diff --git a/devel/addon/bind/install.sh b/devel/addon/bind/install.sh index c75c13516..5d4ced80b 100755 --- a/devel/addon/bind/install.sh +++ b/devel/addon/bind/install.sh @@ -24,11 +24,17 @@ set -o pipefail # Namespace to deploy into NAMESPACE="${NAMESPACE:-bind}" +if [[ "$IS_OPENSHIFT" == "true" ]] ; then + # OpenShift needs bind to be in kube-system due to file ownership restrictions + NAMESPACE="kube-system" +fi SCRIPT_ROOT=$(dirname "${BASH_SOURCE}") source "${SCRIPT_ROOT}/../../lib/lib.sh" SCRIPT_ROOT=$(dirname "${BASH_SOURCE}") +SERVICE_IP_PREFIX="${SERVICE_IP_PREFIX:-10.0.0}" + check_tool kubectl require_image "sameersbn/bind:bazel" "//devel/addon/bind:bundle" @@ -36,4 +42,4 @@ require_image "sameersbn/bind:bazel" "//devel/addon/bind:bundle" kubectl get namespace "${NAMESPACE}" || kubectl create namespace "${NAMESPACE}" # Upgrade or install bind -kubectl apply --namespace "${NAMESPACE}" -f "$SCRIPT_ROOT/manifests/" +sed "s/{SERVICE_IP_PREFIX}/${SERVICE_IP_PREFIX}/g" $SCRIPT_ROOT/manifests/* | kubectl apply --namespace "${NAMESPACE}" -f - diff --git a/devel/addon/bind/manifests/configmap.yaml b/devel/addon/bind/manifests/configmap.yaml index 1ccd1d787..460c1ccad 100644 --- a/devel/addon/bind/manifests/configmap.yaml +++ b/devel/addon/bind/manifests/configmap.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: ConfigMap metadata: @@ -34,8 +35,8 @@ data: 2419200 ; Expire 604800 ) ; Negative Cache TTL @ IN NS localhost. - *.ingress-nginx IN A 10.0.0.15 - ingress-nginx IN A 10.0.0.15 + *.ingress-nginx IN A {SERVICE_IP_PREFIX}.15 + ingress-nginx IN A {SERVICE_IP_PREFIX}.15 db.dns01.example.com: | ; @@ -49,4 +50,4 @@ data: 2419200 ; Expire 604800 ) ; Negative Cache TTL @ IN NS localhost. - dummy IN A 10.0.0.15 + dummy IN A {SERVICE_IP_PREFIX}.15 diff --git a/devel/addon/bind/manifests/deployment.yaml b/devel/addon/bind/manifests/deployment.yaml index af6283021..44d6138c2 100644 --- a/devel/addon/bind/manifests/deployment.yaml +++ b/devel/addon/bind/manifests/deployment.yaml @@ -1,3 +1,4 @@ +--- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/devel/addon/bind/manifests/service.yaml b/devel/addon/bind/manifests/service.yaml index 7d436fba8..063dfb3fb 100644 --- a/devel/addon/bind/manifests/service.yaml +++ b/devel/addon/bind/manifests/service.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Service metadata: @@ -7,7 +8,7 @@ metadata: spec: type: ClusterIP # hardcoded to make this service easily accessible via IP only - clusterIP: 10.0.0.16 + clusterIP: {SERVICE_IP_PREFIX}.16 ports: - port: 53 targetPort: 53 diff --git a/devel/addon/certmanager/install.sh b/devel/addon/certmanager/install.sh index 0d6dab607..62ff41fee 100755 --- a/devel/addon/certmanager/install.sh +++ b/devel/addon/certmanager/install.sh @@ -37,12 +37,11 @@ export APP_VERSION="$(date +"%s")" # Build a copy of the cert-manager release images using the :bazel image tag bazel run --stamp=true --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 "//devel/addon/certmanager:bundle" -# Load all images into the kind cluster -kind load docker-image --name "$KIND_CLUSTER_NAME" "quay.io/jetstack/cert-manager-controller:${APP_VERSION}" & -kind load docker-image --name "$KIND_CLUSTER_NAME" "quay.io/jetstack/cert-manager-acmesolver:${APP_VERSION}" & -kind load docker-image --name "$KIND_CLUSTER_NAME" "quay.io/jetstack/cert-manager-cainjector:${APP_VERSION}" & -kind load docker-image --name "$KIND_CLUSTER_NAME" "quay.io/jetstack/cert-manager-webhook:${APP_VERSION}" & - +# Load all images into the cluster +load_image "quay.io/jetstack/cert-manager-controller:${APP_VERSION}" & +load_image "quay.io/jetstack/cert-manager-acmesolver:${APP_VERSION}" & +load_image "quay.io/jetstack/cert-manager-cainjector:${APP_VERSION}" & +load_image "quay.io/jetstack/cert-manager-webhook:${APP_VERSION}" & wait # Ensure the pebble namespace exists @@ -61,6 +60,6 @@ helm upgrade \ --set webhook.image.tag="${APP_VERSION}" \ --set installCRDs=true \ --set featureGates="${FEATURE_GATES:-}" \ - --set 'extraArgs={--dns01-recursive-nameservers=10.0.0.16:53,--dns01-recursive-nameservers-only=true}' \ + --set "extraArgs={--dns01-recursive-nameservers=${SERVICE_IP_PREFIX}.16:53,--dns01-recursive-nameservers-only=true}" \ "$RELEASE_NAME" \ "$REPO_ROOT/bazel-bin/deploy/charts/cert-manager/cert-manager.tgz" diff --git a/devel/addon/ingressnginx/install.sh b/devel/addon/ingressnginx/install.sh index d58120463..d859f1ef4 100755 --- a/devel/addon/ingressnginx/install.sh +++ b/devel/addon/ingressnginx/install.sh @@ -24,6 +24,10 @@ set -o pipefail # Namespace to deploy into NAMESPACE="${NAMESPACE:-ingress-nginx}" +if [[ "$IS_OPENSHIFT" == "true" ]] ; then + # OpenShift needs bind to be in kube-system due to file ownership restrictions + NAMESPACE="kube-system" +fi # Release name to use with Helm RELEASE_NAME="${RELEASE_NAME:-ingress-nginx}" @@ -37,7 +41,7 @@ check_tool helm require_image "quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.33.0" "//devel/addon/ingressnginx:bundle" require_image "k8s.gcr.io/defaultbackend-amd64:bazel" "//devel/addon/ingressnginx:bundle" -# Ensure the pebble namespace exists +# Ensure the ingress-nginx namespace exists kubectl get namespace "${NAMESPACE}" || kubectl create namespace "${NAMESPACE}" helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx @@ -54,10 +58,12 @@ helm upgrade \ --set controller.image.pullPolicy=Never \ --set defaultBackend.image.tag=bazel \ --set defaultBackend.image.pullPolicy=Never \ - --set controller.service.clusterIP=10.0.0.15 \ + --set "controller.service.clusterIP=${SERVICE_IP_PREFIX}.15"\ --set controller.service.type=ClusterIP \ --set controller.config.no-tls-redirect-locations="" \ --set admissionWebhooks.enabled=false \ --set controller.admissionWebhooks.enabled=false \ + --set controller.image.runAsUser="" \ + --set controller.defaultBackend.runAsUser="" \ "$RELEASE_NAME" \ ingress-nginx/ingress-nginx diff --git a/devel/addon/samplewebhook/chart/templates/deployment.yaml b/devel/addon/samplewebhook/chart/templates/deployment.yaml index ed4946396..1d506aeee 100644 --- a/devel/addon/samplewebhook/chart/templates/deployment.yaml +++ b/devel/addon/samplewebhook/chart/templates/deployment.yaml @@ -27,6 +27,11 @@ spec: args: - --tls-cert-file=/tls/tls.crt - --tls-private-key-file=/tls/tls.key + # These are explicitly set as they are not granted by default in OpenShift + securityContext: + capabilities: + add: + - NET_BIND_SERVICE env: - name: GROUP_NAME value: {{ .Values.groupName | quote }} diff --git a/devel/ci-run-e2e.sh b/devel/ci-run-e2e.sh index 10f95bd7f..8e1237e1e 100755 --- a/devel/ci-run-e2e.sh +++ b/devel/ci-run-e2e.sh @@ -34,10 +34,23 @@ source "${SCRIPT_ROOT}/lib/lib.sh" # Configure PATH to use bazel provided e2e tools setup_tools -trap "export_logs" ERR +echo "Ensuring a cluster exists..." +if [[ "$IS_OPENSHIFT" == "true" ]] ; then + if [[ "$OPENSHIFT_VERSION" =~ 3\..* ]] ; then + "${SCRIPT_ROOT}/cluster/create-openshift3.sh" + else + echo "Unsupported OpenShift version: ${OPENSHIFT_VERSION}" + exit 1 + fi +else + trap "export_logs" ERR + "${SCRIPT_ROOT}/cluster/create-kind.sh" +fi -echo "Ensuring a kind cluster exists..." -"${SCRIPT_ROOT}/cluster/create.sh" +export SERVICE_IP_PREFIX="10.0.0" +if [[ "$IS_OPENSHIFT" == "true" ]] ; then + export SERVICE_IP_PREFIX="172.30.0" +fi echo "Ensuring all e2e test dependencies are installed..." "${SCRIPT_ROOT}/setup-e2e-deps.sh" diff --git a/devel/cluster/BUILD.bazel b/devel/cluster/BUILD.bazel new file mode 100644 index 000000000..7907a0c8a --- /dev/null +++ b/devel/cluster/BUILD.bazel @@ -0,0 +1,17 @@ +package(default_visibility = ["//visibility:public"]) + +OC3 = "//hack/bin:oc3" +KUBECTL = "//hack/bin:kubectl" + +sh_binary( + name = "create-openshift3", + srcs = ["create-openshift3.sh"], + args = [ + "$(location %s)" % OC3, + "$(location %s)" % KUBECTL, + ], + data = [ + OC3, + KUBECTL, + ], +) diff --git a/devel/cluster/config/openshift-coredns.yaml b/devel/cluster/config/openshift-coredns.yaml new file mode 100644 index 000000000..e928a03fb --- /dev/null +++ b/devel/cluster/config/openshift-coredns.yaml @@ -0,0 +1,192 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns + namespace: kube-dns + labels: + app.kubernetes.io/managed-by: cert-manager-e2e-cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/managed-by: cert-manager-e2e-cluster + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + app.kubernetes.io/managed-by: cert-manager-e2e-cluster + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-dns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-dns + labels: + app.kubernetes.io/managed-by: cert-manager-e2e-cluster +data: + Corefile: | + .:53 { + errors + health { + lameduck 5s + } + ready + kubernetes cluster.local in-addr.arpa ip6.arpa { + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + forward . /etc/resolv.conf + cache 30 + loop + reload + loadbalance + } + example.com:53 { + forward . 172.30.0.16 + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns + namespace: kube-dns + labels: + app.kubernetes.io/managed-by: cert-manager-e2e-cluster + k8s-app: kube-dns + kubernetes.io/name: "CoreDNS" +spec: + # replicas: not specified here: + # 1. Default is 1. + # 2. Will be tuned in real time if DNS horizontal auto-scaling is turned on. + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + spec: + serviceAccountName: coredns + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - name: coredns + image: coredns/coredns:1.6.7 + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-dns + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/managed-by: cert-manager-e2e-cluster + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: 172.30.0.2 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP \ No newline at end of file diff --git a/devel/cluster/create-kind.sh b/devel/cluster/create-kind.sh new file mode 100755 index 000000000..f9ae7c2ab --- /dev/null +++ b/devel/cluster/create-kind.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# Copyright 2020 The Jetstack cert-manager contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset +set -o errexit +set -o pipefail + +SCRIPT_ROOT=$(dirname "${BASH_SOURCE}") +source "${SCRIPT_ROOT}/../lib/lib.sh" +SCRIPT_ROOT=$(dirname "${BASH_SOURCE}") + +# Require kind & kubectl available on PATH +check_tool kind +check_tool kubectl + +# Compute the details of the kind image to use +export KIND_IMAGE_SHA="" +export KIND_IMAGE_CONFIG="" +if [[ "$K8S_VERSION" =~ 1\.11 ]]; then + # v1.11.10 @ sha256:e6f3dade95b7cb74081c5b9f3291aaaa6026a90a977e0b990778b6adc9ea6248 + KIND_IMAGE_SHA="sha256:e6f3dade95b7cb74081c5b9f3291aaaa6026a90a977e0b990778b6adc9ea6248" + KIND_IMAGE_CONFIG="v1alpha2" +elif [[ "$K8S_VERSION" =~ 1\.12 ]]; then + # v1.12.10 @ sha256:68a6581f64b54994b824708286fafc37f1227b7b54cbb8865182ce1e036ed1cc + KIND_IMAGE_SHA="sha256:68a6581f64b54994b824708286fafc37f1227b7b54cbb8865182ce1e036ed1cc" + KIND_IMAGE_CONFIG="v1alpha3" +elif [[ "$K8S_VERSION" =~ 1\.13 ]] ; then + # v1.13.12 @sha256:5e8ae1a4e39f3d151d420ef912e18368745a2ede6d20ea87506920cd947a7e3a + KIND_IMAGE_SHA="sha256:5e8ae1a4e39f3d151d420ef912e18368745a2ede6d20ea87506920cd947a7e3a" + KIND_IMAGE_CONFIG="v1beta1" +elif [[ "$K8S_VERSION" =~ 1\.14 ]] ; then + # v1.14.10 @ sha256:81ae5a3237c779efc4dda43cc81c696f88a194abcc4f8fa34f86cf674aa14977 + KIND_IMAGE_SHA="sha256:81ae5a3237c779efc4dda43cc81c696f88a194abcc4f8fa34f86cf674aa14977" + KIND_IMAGE_CONFIG="v1beta1" +elif [[ "$K8S_VERSION" =~ 1\.15 ]] ; then + # v1.15.7 @ sha256:e2df133f80ef633c53c0200114fce2ed5e1f6947477dbc83261a6a921169488d + KIND_IMAGE_SHA="sha256:e2df133f80ef633c53c0200114fce2ed5e1f6947477dbc83261a6a921169488d" + KIND_IMAGE_CONFIG="v1beta2" +elif [[ "$K8S_VERSION" =~ 1\.16 ]] ; then + # v1.16.4 @ sha256:b91a2c2317a000f3a783489dfb755064177dbc3a0b2f4147d50f04825d016f55 + KIND_IMAGE_SHA="sha256:b91a2c2317a000f3a783489dfb755064177dbc3a0b2f4147d50f04825d016f55" + KIND_IMAGE_CONFIG="v1beta2" +elif [[ "$K8S_VERSION" =~ 1\.17 ]] ; then + # v1.17.0 @ sha256:9512edae126da271b66b990b6fff768fbb7cd786c7d39e86bdf55906352fdf62 + KIND_IMAGE_SHA="sha256:9512edae126da271b66b990b6fff768fbb7cd786c7d39e86bdf55906352fdf62" + KIND_IMAGE_CONFIG="v1beta2" +elif [[ "$K8S_VERSION" =~ 1\.18 ]] ; then + # v1.18.0 @ sha256:0e20578828edd939d25eb98496a685c76c98d54084932f76069f886ec315d694 + KIND_IMAGE_SHA="sha256:0e20578828edd939d25eb98496a685c76c98d54084932f76069f886ec315d694" + KIND_IMAGE_CONFIG="v1beta2" +else + echo "Unrecognised Kubernetes version '${K8S_VERSION}'! Aborting..." + exit 1 +fi +export KIND_IMAGE="${KIND_IMAGE_REPO}@${KIND_IMAGE_SHA}" +echo "kind image details:" +echo " repo: ${KIND_IMAGE_REPO}" +echo " sha256: ${KIND_IMAGE_SHA}" +echo " version: ${K8S_VERSION}" +echo " config: ${KIND_IMAGE_CONFIG}" + +if kind get clusters | grep "^$KIND_CLUSTER_NAME\$" &>/dev/null; then + echo "Existing cluster '$KIND_CLUSTER_NAME' found, skipping creating cluster..." + exit 0 +fi + +# Create the kind cluster +kind create cluster \ + --config "${SCRIPT_ROOT}/config/${KIND_IMAGE_CONFIG}.yaml" \ + --image "${KIND_IMAGE}" \ + --name "${KIND_CLUSTER_NAME}" + +# Get the current config +original_coredns_config=$(kubectl get -ogo-template='{{.data.Corefile}}' -n=kube-system configmap/coredns) +additional_coredns_config="$(printf 'example.com:53 {\n forward . 10.0.0.16\n}\n')" +echo "Original CoreDNS config:" +echo "${original_coredns_config}" +# Patch it +fixed_coredns_config=$( + printf '%s\n%s' "${original_coredns_config}" "${additional_coredns_config}" +) +echo "Patched CoreDNS config:" +echo "${fixed_coredns_config}" +kubectl create configmap -oyaml coredns --dry-run --from-literal=Corefile="${fixed_coredns_config}" | kubectl apply --namespace kube-system -f - diff --git a/devel/cluster/create-openshift3.sh b/devel/cluster/create-openshift3.sh new file mode 100755 index 000000000..ea42e9e2a --- /dev/null +++ b/devel/cluster/create-openshift3.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +# Copyright 2020 The Jetstack cert-manager contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset +set -o errexit +set -o pipefail + +SCRIPT_ROOT="$(dirname "${BASH_SOURCE}")" +TMP_DIR="$(mktemp -d)" + +source "${SCRIPT_ROOT}/../lib/lib.sh" +setup_tools + +if docker ps | grep "openshift/origin-node:v${OPENSHIFT_VERSION}" &>/dev/null; then + echo "Existing OpenShift 3 cluster found, skipping creating cluster..." + exit 0 +fi + +# Needed for `oc cluster up` as it places files in the current directory +cd "${TMP_DIR}" + +mkdir -p "${TMP_DIR}/openshift.local.clusterup/kube-apiserver/" + +# Let OpenShift generate all certificates and setup for the controller +echo "Running 'start master'" +docker run -v $(pwd)/openshift.local.clusterup/kube-apiserver/:/var/lib/origin/openshift.local.config/ \ + "openshift/origin-control-plane:v${OPENSHIFT_VERSION}" start master \ + --write-config=/var/lib/origin/openshift.local.config \ + --master=127.0.0.1 \ + --images="openshift/origin-\${component}:v$OPENSHIFT_VERSION" \ + --dns=0.0.0.0:8053 \ + --public-master=https://127.0.0.1:8443 \ + --etcd-dir=/var/lib/etcd + +# Let OpenShift generate all certificates and setup for the node +echo "Running 'adm create-node-config'" +"${OC3}" adm create-node-config \ + --node-dir="${TMP_DIR}/openshift.local.clusterup/node" \ + --certificate-authority="${TMP_DIR}/openshift.local.clusterup/kube-apiserver/ca.crt" \ + --dns-bind-address=0.0.0.0:8053 \ + --hostnames=localhost \ + --hostnames=127.0.0.1 \ + --images="openshift/origin-\${component}:v${OPENSHIFT_VERSION}" \ + --node=localhost \ + --node-client-certificate-authority="${TMP_DIR}/openshift.local.clusterup/kube-apiserver/ca.crt" \ + --signer-cert="${TMP_DIR}/openshift.local.clusterup/kube-apiserver/ca.crt" \ + --signer-key="${TMP_DIR}/openshift.local.clusterup/kube-apiserver/ca.key" \ + --signer-serial="${TMP_DIR}/openshift.local.clusterup/kube-apiserver/ca.serial.txt"\ + --volume-dir=/var/lib/origin/cluster-up/openshift.local.clusterup/openshift.local.volumes + +# Patch the node configuration to disable features that do not work with Docker in Docker +cat << EOF >>"${TMP_DIR}/openshift.local.clusterup/node/node-config.yaml" +kubeletArguments: + cgroups-per-qos: + - "false" + cgroup-driver: + - "systemd" + enforce-node-allocatable: + - "" +EOF + +# Patch for OpenShift using -v ":shared" in Docker +mount --make-shared / +# Set up the cluster itself +echo "Running 'cluster up'" +"${OC3}" cluster up --enable="-automation-service-broker,-centos-imagestreams,-persistent-volumes,-registry,-rhel-imagestreams,-router,-sample-templates,-service-catalog,-template-service-broker,-web-console" + +# Set kubeconfig to be sysadmin +"${OC3}" login -u system:admin + +# Disable restrictions for our test dependancies +"${OC3}" adm policy add-scc-to-group privileged system:authenticated + +# Replace kube-dns with our patched CoreDNS +"${KUBECTL}" apply -n=kube-dns -f "${SCRIPT_ROOT}/config/openshift-coredns.yaml" +"${KUBECTL}" delete -n=kube-dns ds kube-dns +"${KUBECTL}" rollout -n=kube-dns status deploy/coredns diff --git a/devel/cluster/create.sh b/devel/cluster/create.sh index f9ae7c2ab..8b798d22d 100755 --- a/devel/cluster/create.sh +++ b/devel/cluster/create.sh @@ -18,80 +18,12 @@ set -o nounset set -o errexit set -o pipefail -SCRIPT_ROOT=$(dirname "${BASH_SOURCE}") +SCRIPT_ROOT="$(dirname "${BASH_SOURCE}")" source "${SCRIPT_ROOT}/../lib/lib.sh" -SCRIPT_ROOT=$(dirname "${BASH_SOURCE}") -# Require kind & kubectl available on PATH -check_tool kind -check_tool kubectl - -# Compute the details of the kind image to use -export KIND_IMAGE_SHA="" -export KIND_IMAGE_CONFIG="" -if [[ "$K8S_VERSION" =~ 1\.11 ]]; then - # v1.11.10 @ sha256:e6f3dade95b7cb74081c5b9f3291aaaa6026a90a977e0b990778b6adc9ea6248 - KIND_IMAGE_SHA="sha256:e6f3dade95b7cb74081c5b9f3291aaaa6026a90a977e0b990778b6adc9ea6248" - KIND_IMAGE_CONFIG="v1alpha2" -elif [[ "$K8S_VERSION" =~ 1\.12 ]]; then - # v1.12.10 @ sha256:68a6581f64b54994b824708286fafc37f1227b7b54cbb8865182ce1e036ed1cc - KIND_IMAGE_SHA="sha256:68a6581f64b54994b824708286fafc37f1227b7b54cbb8865182ce1e036ed1cc" - KIND_IMAGE_CONFIG="v1alpha3" -elif [[ "$K8S_VERSION" =~ 1\.13 ]] ; then - # v1.13.12 @sha256:5e8ae1a4e39f3d151d420ef912e18368745a2ede6d20ea87506920cd947a7e3a - KIND_IMAGE_SHA="sha256:5e8ae1a4e39f3d151d420ef912e18368745a2ede6d20ea87506920cd947a7e3a" - KIND_IMAGE_CONFIG="v1beta1" -elif [[ "$K8S_VERSION" =~ 1\.14 ]] ; then - # v1.14.10 @ sha256:81ae5a3237c779efc4dda43cc81c696f88a194abcc4f8fa34f86cf674aa14977 - KIND_IMAGE_SHA="sha256:81ae5a3237c779efc4dda43cc81c696f88a194abcc4f8fa34f86cf674aa14977" - KIND_IMAGE_CONFIG="v1beta1" -elif [[ "$K8S_VERSION" =~ 1\.15 ]] ; then - # v1.15.7 @ sha256:e2df133f80ef633c53c0200114fce2ed5e1f6947477dbc83261a6a921169488d - KIND_IMAGE_SHA="sha256:e2df133f80ef633c53c0200114fce2ed5e1f6947477dbc83261a6a921169488d" - KIND_IMAGE_CONFIG="v1beta2" -elif [[ "$K8S_VERSION" =~ 1\.16 ]] ; then - # v1.16.4 @ sha256:b91a2c2317a000f3a783489dfb755064177dbc3a0b2f4147d50f04825d016f55 - KIND_IMAGE_SHA="sha256:b91a2c2317a000f3a783489dfb755064177dbc3a0b2f4147d50f04825d016f55" - KIND_IMAGE_CONFIG="v1beta2" -elif [[ "$K8S_VERSION" =~ 1\.17 ]] ; then - # v1.17.0 @ sha256:9512edae126da271b66b990b6fff768fbb7cd786c7d39e86bdf55906352fdf62 - KIND_IMAGE_SHA="sha256:9512edae126da271b66b990b6fff768fbb7cd786c7d39e86bdf55906352fdf62" - KIND_IMAGE_CONFIG="v1beta2" -elif [[ "$K8S_VERSION" =~ 1\.18 ]] ; then - # v1.18.0 @ sha256:0e20578828edd939d25eb98496a685c76c98d54084932f76069f886ec315d694 - KIND_IMAGE_SHA="sha256:0e20578828edd939d25eb98496a685c76c98d54084932f76069f886ec315d694" - KIND_IMAGE_CONFIG="v1beta2" +# Automatically detect which type of cluster to setup +if [[ "$IS_OPENSHIFT" == "true" ]] ; then + "${SCRIPT_ROOT}/create-openshift3.sh" else - echo "Unrecognised Kubernetes version '${K8S_VERSION}'! Aborting..." - exit 1 -fi -export KIND_IMAGE="${KIND_IMAGE_REPO}@${KIND_IMAGE_SHA}" -echo "kind image details:" -echo " repo: ${KIND_IMAGE_REPO}" -echo " sha256: ${KIND_IMAGE_SHA}" -echo " version: ${K8S_VERSION}" -echo " config: ${KIND_IMAGE_CONFIG}" - -if kind get clusters | grep "^$KIND_CLUSTER_NAME\$" &>/dev/null; then - echo "Existing cluster '$KIND_CLUSTER_NAME' found, skipping creating cluster..." - exit 0 -fi - -# Create the kind cluster -kind create cluster \ - --config "${SCRIPT_ROOT}/config/${KIND_IMAGE_CONFIG}.yaml" \ - --image "${KIND_IMAGE}" \ - --name "${KIND_CLUSTER_NAME}" - -# Get the current config -original_coredns_config=$(kubectl get -ogo-template='{{.data.Corefile}}' -n=kube-system configmap/coredns) -additional_coredns_config="$(printf 'example.com:53 {\n forward . 10.0.0.16\n}\n')" -echo "Original CoreDNS config:" -echo "${original_coredns_config}" -# Patch it -fixed_coredns_config=$( - printf '%s\n%s' "${original_coredns_config}" "${additional_coredns_config}" -) -echo "Patched CoreDNS config:" -echo "${fixed_coredns_config}" -kubectl create configmap -oyaml coredns --dry-run --from-literal=Corefile="${fixed_coredns_config}" | kubectl apply --namespace kube-system -f - + "${SCRIPT_ROOT}/create-kind.sh" +fi \ No newline at end of file diff --git a/devel/cluster/export-logs.sh b/devel/cluster/export-logs.sh index 8d9f7bd8e..be14fd974 100755 --- a/devel/cluster/export-logs.sh +++ b/devel/cluster/export-logs.sh @@ -27,4 +27,10 @@ check_tool kind LOGS_DIR="${ARTIFACTS:-$REPO_ROOT/_artifacts}/cert-manager-e2e-logs" rm -rf $LOGS_DIR && mkdir -p $LOGS_DIR -kind export logs $LOGS_DIR --name "${KIND_CLUSTER_NAME}" +if [[ "$IS_OPENSHIFT" == "true" ]] ; then + for container in $(docker ps -a -q); do + docker logs "$container" >"${LOGS_DIR}/${container}" + done +else + kind export logs $LOGS_DIR --name "${KIND_CLUSTER_NAME}" +fi diff --git a/devel/lib/lib.sh b/devel/lib/lib.sh index 1cc7fb317..7dd67ec98 100644 --- a/devel/lib/lib.sh +++ b/devel/lib/lib.sh @@ -26,15 +26,23 @@ export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" export KIND_IMAGE_REPO="kindest/node" # Default Kubernetes version to use to 1.17 export K8S_VERSION=${K8S_VERSION:-1.17} +# Default OpenShift version to use to 3.11 +export OPENSHIFT_VERSION=${OPENSHIFT_VERSION:-"3.11"} +export SERVICE_IP_PREFIX="${SERVICE_IP_PREFIX:-10.0.0}" +export IS_OPENSHIFT="${IS_OPENSHIFT:-"false"}" +export OPENSHIFT_VERSION="${OPENSHIFT_VERSION:-"3.11"}" +export SERVICE_IP_PREFIX="${SERVICE_IP_PREFIX:-10.0.0}" +export DNS_SERVER="${SERVICE_IP_PREFIX}.16" # setup_tools will build and set up the environment to use bazel-provided # versions of the tools required for development setup_tools() { check_bazel - bazel build //hack/bin:helm //hack/bin:kind //hack/bin:kubectl //devel/bin:ginkgo + bazel build //hack/bin:helm //hack/bin:kind //hack/bin:kubectl //devel/bin:ginkgo //hack/bin:oc3 local bindir="$(bazel info bazel-genfiles)" export HELM="${bindir}/hack/bin/helm" export KIND="${bindir}/hack/bin/kind" + export OC3="${bindir}/hack/bin/oc3" export KUBECTL="${bindir}/hack/bin/kubectl" export GINKGO="${bindir}/devel/bin/ginkgo" # Configure PATH to use bazel provided e2e tools @@ -81,5 +89,19 @@ require_image() { bazel run --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 "${BAZEL_TARGET}" # Load the image into the kind cluster - kind load docker-image --name "$KIND_CLUSTER_NAME" "$IMAGE_NAME" + load_image "$IMAGE_NAME" } + +# load_image will lod an image into the local cluster +# for a kind cluster it will load it into the cluster +# with name $KIND_CLUSTER_NAME +load_image() { + IMAGE_NAME="$1" + if [[ "$IS_OPENSHIFT" == "true" ]] ; then + # No loading into a cluster for OpenShift is needed + # as OpenShift shares the Docker daemon the image was + # built with + return + fi + kind load docker-image --name "$KIND_CLUSTER_NAME" "$IMAGE_NAME" +} \ No newline at end of file diff --git a/devel/run-e2e.sh b/devel/run-e2e.sh index 4c040fc28..f7125bc77 100755 --- a/devel/run-e2e.sh +++ b/devel/run-e2e.sh @@ -20,7 +20,7 @@ set -o pipefail # This script will run the end-to-end test suite against an already configured # kind cluster. -# If a cluster does not already exist, create one with 'cluster/create.sh'. +# If a cluster does not already exist, create one with 'cluster/create-kind.sh'. SCRIPT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )" export REPO_ROOT="${SCRIPT_ROOT}/.." @@ -44,4 +44,5 @@ ginkgo -nodes 10 -flakeAttempts ${FLAKE_ATTEMPTS:-1} \ -- \ --repo-root="${REPO_ROOT}" \ --report-dir="${ARTIFACTS:-$REPO_ROOT/_artifacts}" \ + --acme-dns-server="$DNS_SERVER" \ "$@" diff --git a/devel/setup-e2e-deps.sh b/devel/setup-e2e-deps.sh index dd1c8779b..5e6758010 100755 --- a/devel/setup-e2e-deps.sh +++ b/devel/setup-e2e-deps.sh @@ -31,25 +31,25 @@ source "${SCRIPT_ROOT}/lib/lib.sh" # Configure PATH to use bazel provided e2e tools setup_tools -echo "Installing cert-manager into the kind cluster..." +echo "Installing cert-manager into the cluster..." "${SCRIPT_ROOT}/addon/certmanager/install.sh" check_bazel bazel build --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //devel/addon/... -echo "Installing sample-webhook into the kind cluster..." +echo "Installing sample-webhook into the cluster..." "${SCRIPT_ROOT}/addon/samplewebhook/install.sh" & -echo "Installing bind into the kind cluster..." +echo "Installing bind into the cluster..." "${SCRIPT_ROOT}/addon/bind/install.sh" & -echo "Installing pebble into the kind cluster..." +echo "Installing pebble into the cluster..." "${SCRIPT_ROOT}/addon/pebble/install.sh" & -echo "Installing ingress-nginx into the kind cluster..." +echo "Installing ingress-nginx into the cluster..." "${SCRIPT_ROOT}/addon/ingressnginx/install.sh" & -echo "Loading vault into the kind cluster..." +echo "Loading vault into the cluster..." "${SCRIPT_ROOT}/addon/vault/install.sh" & wait diff --git a/hack/bin/BUILD.bazel b/hack/bin/BUILD.bazel index cd18bc975..c656306b1 100644 --- a/hack/bin/BUILD.bazel +++ b/hack/bin/BUILD.bazel @@ -69,6 +69,17 @@ genrule( visibility = ["//visibility:public"], ) +genrule( + name = "fetch_oc3", + srcs = select({ + ":k8": ["@oc_3_11_linux//:file"], + }), + outs = ["oc3"], + cmd = "cp $(SRCS) $@", + tags = ["manual"], + visibility = ["//visibility:public"], +) + genrule( name = "fetch_buildozer", srcs = select({ diff --git a/hack/bin/deps.bzl b/hack/bin/deps.bzl index c06bd24dd..a542f46f8 100644 --- a/hack/bin/deps.bzl +++ b/hack/bin/deps.bzl @@ -23,6 +23,7 @@ def install(): install_staticcheck() install_helm() install_kubectl() + install_oc3() install_kind() # Install golang.org/x/build as kubernetes/repo-infra requires it for the @@ -152,8 +153,8 @@ def install_helm(): ## the version numbers in these rules. http_archive( name = "helm_darwin", - sha256 = "05c7748da0ea8d5f85576491cd3c615f94063f20986fd82a0f5658ddc286cdb1", - urls = ["https://get.helm.sh/helm-v3.0.2-darwin-amd64.tar.gz"], + sha256 = "92b10652b05a150e76995e08910a662c200a8179cfdb16bd51766d0d5ecc981a", + urls = ["https://get.helm.sh/helm-v3.1.2-darwin-amd64.tar.gz"], build_file_content = """ filegroup( @@ -168,8 +169,8 @@ filegroup( http_archive( name = "helm_linux", - sha256 = "c6b7aa7e4ffc66e8abb4be328f71d48c643cb8f398d95c74d075cfb348710e1d", - urls = ["https://get.helm.sh/helm-v3.0.2-linux-amd64.tar.gz"], + sha256 = "e6be589df85076108c33e12e60cfb85dcd82c5d756a6f6ebc8de0ee505c9fd4c", + urls = ["https://get.helm.sh/helm-v3.1.2-linux-amd64.tar.gz"], build_file_content = """ filegroup( @@ -198,6 +199,24 @@ def install_kubectl(): urls = ["https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl"], ) + +# Define rules for different oc versions +def install_oc3(): + http_archive( + name = "oc_3_11_linux", + sha256 = "4b0f07428ba854174c58d2e38287e5402964c9a9355f6c359d1242efd0990da3", + urls = ["https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz"], + build_file_content = + """ +filegroup( + name = "file", + srcs = [ + "openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit/oc", + ], + visibility = ["//visibility:public"], +) + """, + ) ## Fetch kind images used during e2e tests def install_kind(): # install kind binary diff --git a/test/e2e/framework/config/acme.go b/test/e2e/framework/config/acme.go index d2dfb35ec..623636c9e 100644 --- a/test/e2e/framework/config/acme.go +++ b/test/e2e/framework/config/acme.go @@ -21,11 +21,13 @@ import ( ) type ACMEServer struct { - URL string + URL string + DNSServer string } func (p *ACMEServer) AddFlags(fs *flag.FlagSet) { fs.StringVar(&p.URL, "acme-server-url", "https://pebble.pebble.svc.cluster.local/dir", "URL for the ACME server used during end-to-end tests") + fs.StringVar(&p.DNSServer, "acme-dns-server", "10.0.0.16", "DNS server for ACME DNS01 tests to run against using RFC2136") } func (p *ACMEServer) Validate() []error { diff --git a/test/e2e/suite/conformance/certificates/acme/acme.go b/test/e2e/suite/conformance/certificates/acme/acme.go index 3ff2fb8ed..77fd8c496 100644 --- a/test/e2e/suite/conformance/certificates/acme/acme.go +++ b/test/e2e/suite/conformance/certificates/acme/acme.go @@ -206,7 +206,7 @@ func (a *acmeIssuerProvisioner) createDNS01Issuer(f *framework.Framework) cmmeta ObjectMeta: metav1.ObjectMeta{ GenerateName: "acme-issuer-dns01-", }, - Spec: a.createDNS01IssuerSpec(f.Config.Addons.ACMEServer.URL), + Spec: a.createDNS01IssuerSpec(f.Config.Addons.ACMEServer.URL, f.Config.Addons.ACMEServer.DNSServer), } issuer, err := f.CertManagerClientSet.CertmanagerV1alpha2().Issuers(f.Namespace.Name).Create(context.TODO(), issuer, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred(), "failed to create acme DNS01 Issuer") @@ -226,7 +226,7 @@ func (a *acmeIssuerProvisioner) createDNS01ClusterIssuer(f *framework.Framework) ObjectMeta: metav1.ObjectMeta{ GenerateName: "acme-cluster-issuer-dns01-", }, - Spec: a.createDNS01IssuerSpec(f.Config.Addons.ACMEServer.URL), + Spec: a.createDNS01IssuerSpec(f.Config.Addons.ACMEServer.URL, f.Config.Addons.ACMEServer.DNSServer), } issuer, err := f.CertManagerClientSet.CertmanagerV1alpha2().ClusterIssuers().Create(context.TODO(), issuer, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred(), "failed to create acme DNS01 ClusterIssuer") @@ -238,7 +238,7 @@ func (a *acmeIssuerProvisioner) createDNS01ClusterIssuer(f *framework.Framework) } } -func (a *acmeIssuerProvisioner) createDNS01IssuerSpec(serverURL string) cmapi.IssuerSpec { +func (a *acmeIssuerProvisioner) createDNS01IssuerSpec(serverURL, dnsServer string) cmapi.IssuerSpec { return cmapi.IssuerSpec{ IssuerConfig: cmapi.IssuerConfig{ ACME: &cmacme.ACMEIssuer{ @@ -254,7 +254,7 @@ func (a *acmeIssuerProvisioner) createDNS01IssuerSpec(serverURL string) cmapi.Is { DNS01: &cmacme.ACMEChallengeSolverDNS01{ RFC2136: &cmacme.ACMEIssuerDNS01ProviderRFC2136{ - Nameserver: "10.0.0.16", + Nameserver: dnsServer, }, }, }, diff --git a/test/e2e/suite/issuers/acme/certificate/dns01.go b/test/e2e/suite/issuers/acme/certificate/dns01.go index 661b33f74..4eced1dcd 100644 --- a/test/e2e/suite/issuers/acme/certificate/dns01.go +++ b/test/e2e/suite/issuers/acme/certificate/dns01.go @@ -18,6 +18,8 @@ package certificate import ( "context" + "flag" + "os" "time" . "github.com/onsi/ginkgo" @@ -40,16 +42,24 @@ type dns01Provider interface { } var _ = framework.CertManagerDescribe("ACME Certificate (DNS01)", func() { - rfc := &dnsproviders.RFC2136{} - - testDNSProvider("rfc2136", rfc) + // TODO: add better logic to handle other DNS providers + testRFC2136DNSProvider() }) -func testDNSProvider(name string, p dns01Provider) bool { +func testRFC2136DNSProvider() bool { + name := "rfc2136" return Context("With "+name+" credentials configured", func() { f := framework.NewDefaultFramework("create-acme-certificate-dns01-" + name) h := f.Helper() + // TODO: remove this hack for making config flags load + fs := flag.NewFlagSet("", flag.ContinueOnError) + f.Config.AddFlags(fs) + fs.Parse(os.Args) + + p := &dnsproviders.RFC2136{ + Nameserver: f.Config.Addons.ACMEServer.DNSServer, + } f.RequireAddon(p) issuerName := "test-acme-issuer" diff --git a/test/e2e/suite/issuers/acme/certificaterequest/dns01.go b/test/e2e/suite/issuers/acme/certificaterequest/dns01.go index f73acd203..bc318fde0 100644 --- a/test/e2e/suite/issuers/acme/certificaterequest/dns01.go +++ b/test/e2e/suite/issuers/acme/certificaterequest/dns01.go @@ -44,17 +44,19 @@ const testingACMEEmail = "e2e@cert-manager.io" const testingACMEPrivateKey = "test-acme-private-key" var _ = framework.CertManagerDescribe("ACME CertificateRequest (DNS01)", func() { - // TODO: add additional DNS provider configs here - rfc := &dnsproviders.RFC2136{} - - testDNSProvider("rfc2136", rfc) + // TODO: add better logic to handle other DNS providers + testRFC2136DNSProvider() }) -func testDNSProvider(name string, p dns01Provider) bool { +func testRFC2136DNSProvider() bool { + name := "rfc2136" return Context("With "+name+" credentials configured", func() { f := framework.NewDefaultFramework("create-acme-certificate-request-dns01-" + name) h := f.Helper() + p := &dnsproviders.RFC2136{ + Nameserver: f.Config.Addons.ACMEServer.DNSServer, + } f.RequireAddon(p) issuerName := "test-acme-issuer" diff --git a/test/e2e/suite/issuers/acme/dnsproviders/rfc2136.go b/test/e2e/suite/issuers/acme/dnsproviders/rfc2136.go index ccf7903c6..8d5a21cee 100644 --- a/test/e2e/suite/issuers/acme/dnsproviders/rfc2136.go +++ b/test/e2e/suite/issuers/acme/dnsproviders/rfc2136.go @@ -22,7 +22,8 @@ import ( ) type RFC2136 struct { - details Details + details Details + Nameserver string } func (b *RFC2136) Setup(c *config.Config) error { @@ -35,7 +36,7 @@ func (b *RFC2136) Setup(c *config.Config) error { func (b *RFC2136) Provision() error { b.details.ProviderConfig = cmacme.ACMEChallengeSolverDNS01{ RFC2136: &cmacme.ACMEIssuerDNS01ProviderRFC2136{ - Nameserver: "10.0.0.16", + Nameserver: b.Nameserver, }, } b.details.BaseDomain = "dns01.example.com"