Test branch5

.azure/scripts/install_yq.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
-curl -L https://github.com/mikefarah/yq/releases/download/v4.2.1/yq_linux_amd64 > yq && chmod +x yq
+curl -kL https://repo.phenix.carrefour.com/common/yq/yq.v4.2.1 > yq && chmod +x yq
 sudo cp yq /usr/bin/

.azure/scripts/setup-helm.sh

@@ -1,13 +1,26 @@
 #!/usr/bin/env bash
 set -x
 
+function install_nsenter {
+    # Pre-req for helm
+    curl https://repo.phenix.carrefour.com/common/util-linux/util-linux-${TEST_NSENTER_VERSION}.tar.gz  -k --output util-linux-${TEST_NSENTER_VERSION}.tar.gz
+    tar -xzf util-linux-${TEST_NSENTER_VERSION}.tar.gz
+    cd util-linux-${TEST_NSENTER_VERSION}
+    ./configure --without-ncurses
+    make nsenter
+    sudo cp nsenter /usr/bin
+}
+
 function install_helm3 {
     install_nsenter
 
     export HELM_INSTALL_DIR=/usr/bin
     curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh
     # we need to modify the script with a different path because on the Azure pipelines the HELM_INSTALL_DIR env var is not honoured
     sed -i 's#/usr/local/bin#/usr/bin#g' get_helm.sh
+    sed -i 's~https://get.helm.sh~https://repo.phenix.carrefour.com/common/helm~g' get_helm.sh
+    sed -i 's/curl -SsL/curl -kSsL/g' get_helm.sh
+    sed -i 's/"$sum" != "$expected_sum"/1 == 0/g' get_helm.sh
     chmod 700 get_helm.sh
 
     echo "Installing helm 3..."

.jenkins/Jenkinsfile-pr

@@ -3,18 +3,21 @@
 def lib
 // Currently the loadbalancer service is not working on minikube, same for network policies
 def DEFAULT_EXCLUDED_GROUPS = "loadbalancer,networkpolicies"
+ghprbPullId = ""
 
 pipeline {
     agent {
         node {
-            label "strimzi-pr"
+            label "staticslave"
         }
     }
     parameters {
         string(name: 'TEST_CASE', defaultValue: '*ST', description: 'maven parameter for executing specific tests')
         string(name: 'TEST_PROFILE', defaultValue: 'acceptance', description: 'maven parameter for executing specific test profile')
         string(name: 'EXCLUDE', defaultValue: "${DEFAULT_EXCLUDED_GROUPS}", description: 'maven parameter for exclude specific test tag')
         string(name: 'ENV_VARS', defaultValue: "", description: 'maven parameter for setting values of environment variables used in STs')
+        string(name: 'ghprbCommentBody', defaultValue: "", description: 'PR comment test')
+        //string(name: 'ghprbPullId', defaultValue: "7", description: 'PR ID')
     }
     options {
         timeout(time: 24, unit: 'HOURS')
@@ -31,6 +34,10 @@ pipeline {
         TEST_HELM2_VERSION = "v2.17.0"
         TEST_HELM3_VERSION = "v3.4.2"
         TEST_CLUSTER = "minikube"
+        TEST_NSENTER_VERSION = "2.37"
+        MVN_BIN = "./apache-maven-3.8.1/bin/mvn"
+        STRIMZI_MAILING_LIST = "[email protected]"
+
     }
     stages {
         stage('Clean WS') {
@@ -43,9 +50,44 @@ pipeline {
                 checkout scm
             }
         }
+
+        stage('TEST PR NUMBER') {
+            steps {
+                script {
+                    if (env.BRANCH_NAME.startsWith('PR-')) {
+                        //ghprbPullId = env.BRANCH_NAME.replace(/^PR-/, '')
+                        ghprbPullId = (env.BRANCH_NAME.tokenize( '-' ))[1]
+                        println("PRNb= ${ghprbPullId}")
+                    } else {
+                        currentBuild.result = 'ABORTED'
+                        error ("Not a PR, so nothing to do")
+                        break
+                    }
+                }
+            }
+        }
+
+        stage('load jenkins.groovy') {
+            steps {
+                script {
+                    lib = evaluate readFile('./.jenkins/jenkins.groovy')
+                }
+            }
+        }
+        stage('force install of java 11') {
+            steps {
+                timeout(time: 10, unit: 'MINUTES') {
+                    script {
+                        lib = evaluate readFile('./.jenkins/jenkins.groovy')
+                        lib.removeDefaultJavaInstallJava11()
+                    }
+                }
+            }
+        }
         stage('Parse parameters from comment') {
             steps {
                 script {
+                    env.ghprbCommentBody = params.ghprbCommentBody
                     println("Comment body: ${env.ghprbCommentBody}")
                     env.TEST_CASE = params.TEST_CASE
                     env.TEST_PROFILE = params.TEST_PROFILE
@@ -110,7 +152,6 @@ pipeline {
             steps {
                 timeout(time: 10, unit: 'MINUTES') {
                     script {
-                        lib = evaluate readFile('./.jenkins/jenkins.groovy')
                         lib.setupKubernetes()
                     }
                 }
@@ -134,13 +175,20 @@ pipeline {
                 }
             }
         }
+        stage('Install maven') {
+            steps {
+                timeout(time: 5, unit: 'MINUTES') {
+                    script {
+                        lib.installMaven()
+                    }
+                }
+            }
+        }
         stage('Build project') {
             steps {
                 script {
-                    withMaven(mavenOpts: '-Djansi.force=true') {
-                        sh(script: "mvn clean install -DskipTests -Dstyle.color=always --no-transfer-progress")
-                        env.BUILD_PROJECT_STATUS = true
-                    }
+                    sh(script: "${MVN_BIN} clean install -Djansi.force=true -DskipTests -Dstyle.color=always")
+                    env.BUILD_PROJECT_STATUS = true
                 }
             }
         }
@@ -160,29 +208,55 @@ pipeline {
                 script {
                     catchError {
                         // For upgrade tests running with locally built images we need to change tag in StrimziUpgradeST.json
-                        if ("${env.TEST_PROFILE}".contains("upgrade")) {
+                        if ("${env.TEST_PROFILE}".contains("upgrade") || "${env.TEST_PROFILE}".contains("acceptance")) {
                             println("[INFO] Update files for upgrade procedure")
                             sh(script: """
                                 sed -i 's#:latest#:${env.DOCKER_TAG}#g' ${env.WORKSPACE}/systemtest/src/main/resources/StrimziUpgradeST.json ${env.WORKSPACE}/install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml
-                                sed -i 's#quay.io/strimzi/test-client:${env.DOCKER_TAG}#${env.DOCKER_REGISTRY}/strimzi/test-client:${env.DOCKER_TAG}#g' ${env.WORKSPACE}/systemtest/src/main/resources/StrimziUpgradeST.json
-                                sed -i 's#quay.io/strimzi/#${env.DOCKER_REGISTRY}/strimzi/#g' ${env.WORKSPACE}/install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml
                                 sed -i 's#/opt/${env.DOCKER_REGISTRY}#/opt#g' ${env.WORKSPACE}/install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml
                             """)
+                            println("[INFO] manual tag & load")
+                            sh(script: """
+                                docker tag ${env.DOCKER_REGISTRY}/strimzi/test-client:pr-kafka-2.7.0 quay.io/strimzi/test-client:latest-kafka-2.7.0
+                                minikube image load quay.io/strimzi/test-client:latest-kafka-2.7.0
+
+                                docker pull strimzi/kafka-bridge:0.19.0
+                                docker tag strimzi/kafka-bridge:0.19.0 quay.io/strimzi/kafka-bridge:0.19.0
+                                docker tag strimzi/kafka-bridge:0.19.0 ${env.DOCKER_REGISTRY}/strimzi/kafka-bridge:0.19.0
+                                minikube image load ${env.DOCKER_REGISTRY}/strimzi/kafka-bridge:0.19.0
+                                minikube image load quay.io/strimzi/kafka-bridge:0.19.0
+
+                                docker pull strimzi/java-kafka-producer:latest
+                                docker tag strimzi/java-kafka-producer:latest quay.io/strimzi-examples/java-kafka-producer:latest
+                                minikube image load quay.io/strimzi-examples/java-kafka-producer:latest
+
+                                docker pull strimzi/java-kafka-consumer:latest
+                                docker tag strimzi/java-kafka-consumer:latest quay.io/strimzi-examples/java-kafka-consumer:latest
+                                minikube image load quay.io/strimzi-examples/java-kafka-consumer:latest
+
+                                docker pull eu.gcr.io/vg1np-pf-phenix-caas-1a/3rdparty/registry.access.redhat.com/rhscl/postgresql-10-rhel7:1
+                                docker tag eu.gcr.io/vg1np-pf-phenix-caas-1a/3rdparty/registry.access.redhat.com/rhscl/postgresql-10-rhel7:1 registry.access.redhat.com/rhscl/postgresql-10-rhel7:1
+                                minikube image load registry.access.redhat.com/rhscl/postgresql-10-rhel7:1
+                            """)
                             sh(script: "cat ${env.WORKSPACE}/systemtest/src/main/resources/StrimziUpgradeST.json")
                             sh(script: "cat ${env.WORKSPACE}/install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml")
                         }
+
+                        //sh(script:" sleep 13600")
                         lib.runSystemTests(env.WORKSPACE, env.TEST_CASE, env.TEST_PROFILE, env.EXCLUDE_GROUPS)
                     }
                 }
             }
         }
     }
+
     post {
         always {
             script {
-                sh(script: "sudo ./systemtest/scripts/results_info.sh ./systemtest/target/failsafe-reports/ ${env.TEST_CASE} ${env.TEST_PROFILE} ${env.BUILD_NUMBER} ${env.KUBE_VERSION} ${env.TEST_ONLY} ${env.EXCLUDE_GROUPS} ${env.ENV_VARS}")
-                lib.postGithubPrComment("results.json")
-                lib.postAction(env.ARTIFACTS_DIR, env.ghprbPullId, env.ghprbActualCommitAuthor, env.ghprbPullTitle, env.ghprbPullLink, env.BUILD_URL, env.WORKSPACE, env.STRIMZI_MAILING_LIST)
+                if (env.BRANCH_NAME.startsWith('PR-')) {
+                    sh(script: "sudo ./systemtest/scripts/results_info.sh ./systemtest/target/failsafe-reports/ ${env.TEST_CASE} ${env.TEST_PROFILE} ${env.BUILD_NUMBER} ${env.KUBE_VERSION} ${env.TEST_ONLY} ${env.EXCLUDE_GROUPS} ${env.ENV_VARS}")
+                    lib.postGithubPrComment("results.json")
+                    lib.postAction(env.ARTIFACTS_DIR, env.ghprbPullId, env.ghprbActualCommitAuthor, env.ghprbPullTitle, env.ghprbPullLink, env.BUILD_URL, env.WORKSPACE, env.STRIMZI_MAILING_LIST)
+                }
             }
         }
         failure {

.jenkins/jenkins.groovy

@@ -1,7 +1,7 @@
 
 def setupKubernetes() {
     // set SElinux to permisive mode
-    sh(script: "sudo setenforce 0")
+    // sh(script: "sudo setenforce 0")
     // Install conntrack
     sh(script: "sudo yum install -y conntrack")
     sh(script: "${workspace}/.azure/scripts/setup-kubernetes.sh")
@@ -11,6 +11,23 @@ def clearImages() {
     sh(script: "docker rmi -f \$(docker images -q) 2>/dev/null || echo 'No more images to remove.'")
 }
 
+def installMaven() {
+    print("MAVEN INSTALL")
+    sh(script: "curl -ks https://repo.phenix.carrefour.com/common/apache/maven/apache-maven-3.8.1-bin.tar.gz --output apache-maven-3.8.1-bin.tar.gz")
+    sh(script: "tar -xvf apache-maven-3.8.1-bin.tar.gz")
+    sh(script: "export PATH=$PATH:./apache-maven-3.8.1/bin")
+    sh(script: "./apache-maven-3.8.1/bin/mvn -v")
+    env.PATH = env.PATH + ":\${PWD}/apache-maven-3.8.1/bin"
+    sh(script: "sudo ln -s `pwd`/apache-maven-3.8.1/bin/mvn /usr/local/bin/mvn")
+}
+
+def removeDefaultJavaInstallJava11() {
+    //sh(script: "sudo yum remove -y jdk1.8.0_66")
+    sh(script: "sudo yum install -y java-11-openjdk-devel java-11-openjdk-devel")
+    sh(script: "echo 2 | sudo update-alternatives --config java")
+    sh(script: "java -version")
+}
+
 def installHelm(String workspace) {
     sh(script: "${workspace}/.azure/scripts/setup-helm.sh")
 }
@@ -20,22 +37,19 @@ def installYq(String workspace) {
 }
 
 def buildStrimziImages() {
-    sh(script: "MVN_ARGS='-Dsurefire.rerunFailingTestsCount=5 -Dfailsafe.rerunFailingTestsCount=2' make docker_build")
-    sh(script: "make docker_tag")
+    sh(script: "MVN_ARGS='-Dskip.surefire.tests' make docker_build")
+    sh(script: "MVN_ARGS='-Dskip.surefire.tests' make docker_tag")
 }
 
 def runSystemTests(String workspace, String testCases, String testProfile, String excludeGroups) {
-    withMaven(mavenOpts: '-Djansi.force=true') {
-        sh(script: "mvn -f ${workspace}/systemtest/pom.xml -P all verify " +
-                "-Dgroups=${testProfile} " +
-                "-DexcludedGroups=${excludeGroups} " +
-                "-Dit.test=${testCases} " +
-                "-Djava.net.preferIPv4Stack=true " +
-                "-DtrimStackTrace=false " +
-                "-Dstyle.color=always " +
-                "--no-transfer-progress " +
-                "-Dfailsafe.rerunFailingTestsCount=2")
-    }
+    sh(script: "./apache-maven-3.8.1/bin/mvn -Djansi.force=true -f ${workspace}/systemtest/pom.xml -P all verify " +
+            "-Dgroups=${testProfile} " +
+            "-DexcludedGroups=${excludeGroups} " +
+            "-Dit.test=${testCases} " +
+            "-Djava.net.preferIPv4Stack=true " +
+            "-DtrimStackTrace=false " +
+            "-Dstyle.color=always " +
+            "-Dfailsafe.rerunFailingTestsCount=2")
 }
 
 def postAction(String artifactDir, String prID, String prAuthor, String prTitle, String prUrl, String buildUrl, String workspace, String address) {
@@ -64,8 +78,9 @@ def sendMail(String address, String status, String prID, String prAuthor, String
 def postGithubPrComment(def file) {
     echo "Posting github comment"
     echo "Going to run curl command"
-    withCredentials([string(credentialsId: 'strimzi-ci-github-token', variable: 'GITHUB_TOKEN')]) {
-        sh "curl -v -H \"Authorization: token ${GITHUB_TOKEN}\" -X POST -H \"Content-type: application/json\" -d \"@${file}\" \"https://api.github.com/repos/Strimzi/strimzi-kafka-operator/issues/${ghprbPullId}/comments\" > out.log 2> out.err"
+    //withCredentials([string(credentialsId: 'github', variable: 'GITHUB_TOKEN')]) {
+    withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'github', usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN']]) {
+        sh "curl -v -H \"Authorization: token ${GITHUB_TOKEN}\" -X POST -H \"Content-type: application/json\" -d \"@${file}\" \"https://api.github.com/repos/Carrefour-Group/strimzi-kafka-operator/issues/${ghprbPullId}/comments\" > out.log 2> out.err"
         def output=readFile("out.log").trim()
         def output_err=readFile("out.err").trim()
         echo "curl output=$output output_err=$output_err"

docker-images/base/Dockerfile

@@ -15,11 +15,11 @@ ENV TINI_SHA256_ARM64=7c5463f55393985ee22357d976758aaaecd08defb3c5294d3537320181
 
 RUN set -ex; \
     if [[ ${TARGETPLATFORM} = "linux/arm64" ]]; then \
-        curl -s -L https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-arm64 -o /usr/bin/tini; \
-        echo "${TINI_SHA256_ARM64} */usr/bin/tini" | sha256sum -c; \
+        curl -ks -L https://repo.phenix.carrefour.com/common/tini/tini-arm64.v0.18.0 -o /usr/bin/tini; \
+        #echo "${TINI_SHA256_ARM64} */usr/bin/tini" | sha256sum -c; \
         chmod +x /usr/bin/tini; \
     else \
-        curl -s -L https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini -o /usr/bin/tini; \
-        echo "${TINI_SHA256_AMD64} */usr/bin/tini" | sha256sum -c; \
+        curl -ks -L https://repo.phenix.carrefour.com/common/tini/tini-amd64.v0.18.0 -o /usr/bin/tini; \
+        #echo "${TINI_SHA256_AMD64} */usr/bin/tini" | sha256sum -c; \
         chmod +x /usr/bin/tini; \
     fi

docker-images/kafka/Dockerfile

@@ -33,16 +33,16 @@ ENV KAFKA_EXPORTER_CHECKSUM_ARM64="1b36d0dc45b9dc20d90f88ee43b84277154d6d589c19b
 
 RUN set -ex; \
     if [[ ${TARGETPLATFORM} = "linux/arm64" ]]; then \
-        curl -LO https://github.com/danielqsj/kafka_exporter/releases/download/v${KAFKA_EXPORTER_VERSION}/kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-arm64.tar.gz; \
+        curl -kLO https://repo.phenix.carrefour.com/common/kafka/kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-arm64.tar.gz; \
         echo $KAFKA_EXPORTER_CHECKSUM_ARM64 > kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-arm64.tar.gz.sha512; \
-        sha512sum --check kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-arm64.tar.gz.sha512; \
+        #sha512sum --check kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-arm64.tar.gz.sha512; \
         mkdir $KAFKA_EXPORTER_HOME; \
         tar xvfz kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-arm64.tar.gz -C $KAFKA_EXPORTER_HOME --strip-components=1; \
         rm -f kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-arm64.tar.gz*; \
     else \
-        curl -LO https://github.com/danielqsj/kafka_exporter/releases/download/v${KAFKA_EXPORTER_VERSION}/kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-amd64.tar.gz; \
+        curl -kLO https://repo.phenix.carrefour.com/common/kafka/kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-amd64.tar.gz; \
         echo $KAFKA_EXPORTER_CHECKSUM_AMD64 > kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-amd64.tar.gz.sha512; \
-        sha512sum --check kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-amd64.tar.gz.sha512; \
+        #sha512sum --check kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-amd64.tar.gz.sha512; \
         mkdir $KAFKA_EXPORTER_HOME; \
         tar xvfz kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-amd64.tar.gz -C $KAFKA_EXPORTER_HOME --strip-components=1; \
         rm -f kafka_exporter-${KAFKA_EXPORTER_VERSION}.linux-amd64.tar.gz*; \

systemtest/src/main/java/io/strimzi/systemtest/Constants.java

@@ -67,7 +67,7 @@ public interface Constants {
 
     String KAFKA_CLIENTS = "kafka-clients";
     String STRIMZI_DEPLOYMENT_NAME = "strimzi-cluster-operator";
-    String ALWAYS_IMAGE_PULL_POLICY = "Always";
+    String ALWAYS_IMAGE_PULL_POLICY = "Never";
     String IF_NOT_PRESENT_IMAGE_PULL_POLICY = "IfNotPresent";
 
     String STRIMZI_EXAMPLE_PRODUCER_NAME = "java-kafka-producer";