pipeline { agent { node { label 'jenkins-build-agent-nodejs-16' } } environment { BLUE_OCEAN_URL = "https://jenkins.ozg-sh.de/job/goofy/job/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/" RELEASE_REGEX = /\d+.\d+.\d+/ SNAPSHOT_REGEX = /\d+.\d+.\d+-SNAPSHOT/ BUNDESLAND = "by" SSO_URL = "sso.dev.by.kop-cloud.de" CLUSTER_BASE_URL = "dev.by.kop-cloud.de" FAILED_STAGE = "" IMAGE_TAG = "" VERSION = "" E2E_FAILED = "" HELM_CHART_VERSION = "" SH_SUCCESS_STATUS_CODE = 0 } options { timeout(time: 1, unit: 'HOURS') disableConcurrentBuilds() buildDiscarder(logRotator(numToKeepStr: '5')) } stages { stage('Check Version') { steps { script { FAILED_STAGE = env.STAGE_NAME def rootPom = readMavenPom file: 'pom.xml' VERSION = rootPom.version def serverPom = readMavenPom file: 'goofy-server/pom.xml' def serverVersion = serverPom.parent.version def clientPom = readMavenPom file: 'goofy-client/pom.xml' def clientVersion = clientPom.parent.version if(isReleaseBranch()){ if ( !(VERSION ==~ RELEASE_REGEX) || !(serverVersion ==~ RELEASE_REGEX) || !(clientVersion ==~ RELEASE_REGEX)) { error("Keine Release Version für Branch ${env.BRANCH_NAME}.") } } else { if ( !(VERSION ==~ SNAPSHOT_REGEX) || !(serverVersion ==~ SNAPSHOT_REGEX) || !(clientVersion ==~ SNAPSHOT_REGEX)) { error("Keine Snapshot Version für Branch ${env.BRANCH_NAME}.") } } if( !(VERSION == serverVersion && VERSION == clientVersion)){ error("Versionen sind nicht identisch") } } } } stage('Client') { steps { script { FAILED_STAGE=env.STAGE_NAME sh 'npm --version' dir('goofy-client') { sh 'echo "registry=https://nexus.ozg-sh.de/repository/npm-proxy" >> ~/.npmrc' sh 'echo "//nexus.ozg-sh.de/:_auth=amVua2luczpQaihzX0ZNNFU5ZC8=" >> ~/.npmrc' sh 'npm cache verify' sh 'npm install' if (isReleaseBranch()) { sh 'npm run ci-prodBuild' } else { sh 'npm run ci-build' } sh 'npm run ci-test' try { if (isMasterBranch()) { withSonarQubeEnv('sonarqube-ozg-sh'){ sh 'npm run ci-sonar' } } } catch (Exception e) { unstable("SonarQube failed") } } } } // post { // always{ // junit testResults: 'goofy-client/test-report.xml', skipPublishingChecks: true // } // } } stage('Server') { steps { script { FAILED_STAGE=env.STAGE_NAME IMAGE_TAG = generateImageTag() configFileProvider([configFile(fileId: 'maven-settings', variable: 'MAVEN_SETTINGS')]) { sh 'mvn --version' sh "mvn -s $MAVEN_SETTINGS -pl -goofy-client clean install spring-boot:build-image -Dspring-boot.build-image.imageName=docker.ozg-sh.de/goofy:${IMAGE_TAG} -Dspring-boot.build-image.publish -Dmaven.wagon.http.retryHandler.count=3" try { if (isMasterBranch()) { dir('goofy-server'){ withSonarQubeEnv('sonarqube-ozg-sh'){ sh 'mvn -s $MAVEN_SETTINGS sonar:sonar' } } } } catch (Exception e) { unstable("SonarQube failed") } } } } post { always{ junit testResults: '**/target/surefire-reports/*.xml', skipPublishingChecks: true } } } stage('Deploy Maven Artifacts to Nexus') { when { anyOf { branch 'master' branch 'release' } } steps { script { FAILED_STAGE = env.STAGE_NAME } configFileProvider([configFile(fileId: 'maven-settings', variable: 'MAVEN_SETTINGS')]) { sh 'mvn -s $MAVEN_SETTINGS -pl -goofy-client -DskipTests deploy' } } } stage('Tag and Push Docker Image') { when { anyOf { branch 'master' branch 'release' } } steps { script { FAILED_STAGE = env.STAGE_NAME if (isMasterBranch()) { tagAndPushDockerImage('snapshot-latest') } else if (isReleaseBranch()) { tagAndPushDockerImage('latest') } } } } stage('Test, build and deploy Helm Chart') { steps { script { FAILED_GOOFY_STAGE=env.STAGE_NAME HELM_CHART_VERSION = generateHelmChartVersion() dir('src/main/helm') { sh "helm lint -f test-values.yaml" sh "helm unittest -f '../../test/helm/*.yaml' -v '../../test/unit-values.yaml' ." sh "helm package --version=${HELM_CHART_VERSION} ." deployHelmChart(HELM_CHART_VERSION) } } } } stage('Trigger Dev rollout') { when { branch 'master' } steps { script { FAILED_STAGE = env.STAGE_NAME cloneRolloutGitopsRepo() setNewDevGoofyVersion() pushRolloutGitopsRepo() } } } stage('E2E') { stages { stage('Init k8s') { steps { script { FAILED_STAGE = env.STAGE_NAME E2E_FAILED = "" configFileProvider([configFile(fileId: 'kubeconfig-dev-cluster', variable: 'KUBE_CONFIG')]) { sh 'mkdir ~/.kube' sh 'cp ${KUBE_CONFIG} ~/.kube/config' } sh 'helm version' } } } stage('Install cypress') { steps { script { FAILED_STAGE = env.STAGE_NAME E2E_FAILED = "" dir('goofy-client') { sh "npm run cypress:install" } } } } stage('Rollout E2E Namespaces') { steps { script { FAILED_STAGE = env.STAGE_NAME E2E_FAILED = "" cloneE2eGitopsRepo() env.EA_BEZEICHNER = generateBezeichner("e2e-ea") env.MAIN_BEZEICHNER = generateBezeichner("e2e-main") deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER]) generateEaNamespaceYaml() generateMainNamespaceYaml() pushE2eGitopsRepo() rolloutKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER]) } } } stage('Add E2E User') { steps { script { FAILED_STAGE = env.STAGE_NAME E2E_FAILED = "" cloneProvisioningRepo() initEaKeycloak() initMainKeycloak() } } } stage('Run E2E-Tests') { failFast false parallel { stage('E2E-EA') { steps { script { def stageName = env.STAGE_NAME def bezeichner = env.EA_BEZEICHNER def dbPort = 27018 exposeDatenbank(generateNamespace(bezeichner), dbPort) exposeElasticSearch() def testResult = runTests(bezeichner, 'einheitlicher-ansprechpartner', dbPort) if(!testResult) { E2E_FAILED += "${stageName}, " error("Fehler in Stage ${stageName}") } } } post { always { script { publishE2ETestResult("einheitlicher-ansprechpartner", "Goofy E2E-Tests EA") } } } } stage('E2E-main') { steps { script { def stageName = env.STAGE_NAME def bezeichner = env.MAIN_BEZEICHNER def dbPort = 27019 exposeDatenbank(generateNamespace(bezeichner), dbPort) exposeElasticSearch() def testResult = runTests(bezeichner, 'main-tests', dbPort) if(!testResult) { E2E_FAILED += "${stageName}, " error("Fehler in Stage ${stageName}") } } } post { always { script { publishE2ETestResult("main-tests", "Goofy E2E-Tests main") } } } } } } stage('Delete E2E Namespaces') { steps { script { FAILED_STAGE = env.STAGE_NAME E2E_FAILED = "" deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER]) } } } } post { always { script { if (E2E_FAILED) { FAILED_STAGE = "E2E (${E2E_FAILED.substring(0, E2E_FAILED.length() - 2)})" error("Fehler in E2E-Tests") } } } } } stage('Trigger Test rollout') { when { branch 'release' } steps { script { FAILED_STAGE = env.STAGE_NAME cloneRolloutGitopsRepo() setNewTestGoofyVersion() pushRolloutGitopsRepo() } } } } post { failure { script { if (isMasterBranch() || isReleaseBranch()) { sendFailureMessage() } } } } } Void deployHelmChart(String helmChartVersion) { withCredentials([usernamePassword(credentialsId: 'jenkins-nexus-login', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]){ if (isReleaseBranch()) { result = sh script: '''curl -u $USERNAME:$PASSWORD https://nexus.ozg-sh.de/service/rest/v1/components?repository=ozg-base-apps -F file=@goofy-'''+helmChartVersion+'''.tgz''', returnStdout: true } else { result = sh script: '''curl -u $USERNAME:$PASSWORD https://nexus.ozg-sh.de/service/rest/v1/components?repository=ozg-base-apps-snapshot -F file=@goofy-'''+helmChartVersion+'''.tgz''', returnStdout: true } if (result != '') { error(result) } } } String generateHelmChartVersion() { def chartVersion = "${VERSION}" if (isMasterBranch()) { chartVersion += "-${env.GIT_COMMIT.take(7)}" } else if (env.BRANCH_NAME != 'release') { chartVersion += "-${env.BRANCH_NAME}" } return chartVersion.replaceAll("_", "-") } Void tagAndPushDockerImage(String newTag){ withCredentials([usernamePassword(credentialsId: 'jenkins-nexus-login', usernameVariable: 'USER', passwordVariable: 'PASSWORD')]) { sh 'docker login docker.ozg-sh.de -u ${USER} -p ${PASSWORD}' sh "docker tag docker.ozg-sh.de/goofy:${IMAGE_TAG} docker.ozg-sh.de/goofy:${newTag}" sh "docker push docker.ozg-sh.de/goofy:${newTag}" } } String generateImageTag() { def imageTag = "${env.BRANCH_NAME}-${VERSION}" if (isMasterBranch()) { imageTag += "-${env.GIT_COMMIT.take(7)}" } return imageTag } Void cloneE2eGitopsRepo() { cloneGitopsRepo("e2e") checkoutE2eBranch("e2e") } Void cloneRolloutGitopsRepo() { cloneGitopsRepo("rollout") } Void cloneGitopsRepo(String directory) { withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) { dir(directory) { sh 'git clone https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/gitops.git' } } configureGit("${directory}/gitops") } Void cloneProvisioningRepo() { withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) { dir("e2e") { sh 'git clone https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/provisioning.git' } } } Void pushE2eGitopsRepo() { pushGitopsRepo("e2e") } Void pushRolloutGitopsRepo() { pushGitopsRepo("rollout") } Void pushGitopsRepo(String directory) { withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) { dir("${directory}/gitops") { if (hasUnpushedCommits()) { sh 'git push https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/gitops.git' } } } } Boolean hasUnpushedCommits() { return sh (script: "git cherry -v | grep .", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer } Void configureGit(String directory) { final email = "jenkins@ozg-sh.de" final name = "jenkins" dir(directory) { sh "git config user.email '${email}'" sh "git config user.name '${name}'" } } Void checkoutE2eBranch(String directory) { dir("${directory}/gitops") { sh 'git checkout e2e' } } Void generateEaNamespaceYaml() { generateNamespaceYaml(env.EA_BEZEICHNER, "goofy-client/apps/goofy-e2e/src/fixtures/argocd/by-ea-dev.yaml") } Void generateMainNamespaceYaml() { generateNamespaceYaml(env.MAIN_BEZEICHNER, "goofy-client/apps/goofy-e2e/src/fixtures/argocd/by-main-dev.yaml") } Void generateNamespaceYaml(String bezeichner, String valuesPath) { def envValues = readYaml file: valuesPath envValues.kop.bezeichner = bezeichner envValues.goofy.put("image", ['tag': IMAGE_TAG]) envValues.goofy.put("helm", ['version': HELM_CHART_VERSION]) if (isReleaseBranch()) { envValues.put("argocd", ['source': ['path': 'test/application']]) } writeYaml file: "e2e/gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml", data: envValues, overwrite: true sh "cat e2e/gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml" dir("e2e/gitops") { sh "git add dev/namespace/namespaces/by-${bezeichner}-dev.yaml" sh "git commit -m 'add e2e by-${bezeichner}-dev'" } } Void initEaKeycloak() { addKeycloakGroups(env.EA_BEZEICHNER) addKeycloakUser(env.EA_BEZEICHNER) } Void initMainKeycloak() { addKeycloakGroups(env.MAIN_BEZEICHNER) addKeycloakUser(env.MAIN_BEZEICHNER) } Void addKeycloakGroups(String bezeichner) { def groupFiles = sh (script: 'ls goofy-client/apps/goofy-e2e/src/fixtures/group', returnStdout: true) groupFiles.split("\\n").each { group -> def groupJson = sh (script: "cat goofy-client/apps/goofy-e2e/src/fixtures/group/${group}", returnStdout: true) def ansibleVars = """{"k8s_context":"ozg-dev", \ "kop_env":"dev", \ "keycloak_realm":"${generateNamespace(bezeichner)}", \ "group":${groupJson} }""" dir("e2e/provisioning") { if (isReleaseBranch()) { sh "ansible-playbook playbook/add-keycloak-group.yml --extra-vars '${ansibleVars}'" } else { sh "ansible-playbook playbooks/add-keycloak-group.yml --extra-vars '${ansibleVars}'" } } } } Void addKeycloakUser(String bezeichner) { def userFiles = sh (script: 'ls goofy-client/apps/goofy-e2e/src/fixtures/user', returnStdout: true) userFiles.split("\\n").each { user -> def userJson = sh (script: "cat goofy-client/apps/goofy-e2e/src/fixtures/user/${user}", returnStdout: true) def ansibleVars = """{"k8s_context":"ozg-dev", \ "kop_env":"dev", \ "keycloak_realm":"${generateNamespace(bezeichner)}", \ "user":${userJson} }""" dir("e2e/provisioning") { if (isReleaseBranch()) { sh "ansible-playbook playbook/add-keycloak-user.yml --extra-vars '${ansibleVars}'" } else { sh "ansible-playbook playbooks/add-keycloak-user.yml --extra-vars '${ansibleVars}'" } } } } Void deleteKopStack(kopBezeichner) { for(bezeichner in kopBezeichner) { dir("e2e/gitops/dev/namespace/namespaces") { if (sh (script: "ls | grep 'by-${bezeichner}-dev.yaml'", returnStatus: true) == 1) { return } sh "rm by-${bezeichner}-dev.yaml" sh "git add by-${bezeichner}-dev.yaml" sh "git commit -m 'delete e2e by-${bezeichner}-dev.yaml'" } } pushGitopsRepo("e2e") for(bezeichner in kopBezeichner) { waitForDeletion(bezeichner) } } Void waitForDeletion(String bezeichner) { try { sh "kubectl wait --for=delete applications/by-${bezeichner}-dev-application -n argocd --timeout=300s" } catch (Exception e) { error("Application by-${bezeichner}-dev-application konnte nicht gelöscht werden") } } Void rolloutKopStack(kopBezeichner) { pushGitopsRepo("e2e") for(bezeichner in kopBezeichner) { waitForRollout(bezeichner) } } Void waitForRollout(String bezeichner) { waitForHealthyApplication(bezeichner, 'application') waitForHealthyApplication(bezeichner, 'user-manager') waitForHealthyApplication(bezeichner, 'pluto') waitForHealthyApplication(bezeichner, 'goofy') } Void waitForHealthyApplication(String bezeichner, String application) { try { def countRetry = 0 def maxRetry = 12 while (!isApplicationPresent(bezeichner, application) && countRetry < maxRetry ) { countRetry++ sh "sleep 5" } if (!isApplicationHealthy(bezeichner, application)) { waitForHealthyStatus(bezeichner, application) } } catch (Exception e) { error("Application ${application} unhealthy") } } Boolean isApplicationPresent(String bezeichner, String application) { return sh (script: "kubectl get applications -n argocd | grep 'by-${bezeichner}-dev-${application}'", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer } Boolean isApplicationHealthy(String bezeichner, String application) { return sh (script: "kubectl get application/by-${bezeichner}-dev-${application} -n argocd -o=jsonpath='{.status.health.status}' | grep Healthy", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer } Void waitForHealthyStatus(String bezeichner, String application) { sh "kubectl wait --for=jsonpath='{.status.health.status}'=Healthy applications/by-${bezeichner}-dev-${application} -n argocd --timeout=300s" } Void publishE2ETestResult(String reportFolder, String reportName) { publishHTML ( target: [ allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, reportDir: "goofy-client/apps/goofy-e2e/reports/${reportFolder}", reportFiles: 'report.html', reportName: reportName ] ) } String runTests(String bezeichner, String reportFolder, Integer dbPort) { def configFile = generateCypressConfig(bezeichner, reportFolder, dbPort) try { dir("goofy-client") { sh "npm run cypress:version" sh "npm run cypress:ci-run --CONFIG_FILE=${configFile} --REPORT_FOLDER=${reportFolder}" return true } } catch (Exception e) { sh "ls -l /root/.npm/_logs/*-debug.log" sh "cat /root/.npm/_logs/*-debug.log" return false } } String makeUrlConform(String input) { return input.replaceAll(/[^a-zA-Z0-9]+/, "").toLowerCase() } String generateBezeichner(String stage) { def branchName = makeUrlConform(env.BRANCH_NAME) def stageName = makeUrlConform(stage) return "${cutBranchNameForKeycloakRealm(branchName, stageName)}${stageName}" } String cutBranchNameForKeycloakRealm(String branchName, String stageName) { def maxKeycloakRealmLength = 30 def postPrefixLength = 8 def cutBranchNamePosition = maxKeycloakRealmLength - (branchName.length() + stageName.length() + postPrefixLength) if(cutBranchNamePosition < 0) { branchName = branchName[0..cutBranchNamePosition] } return branchName } String generateCypressConfig(String bezeichner, String testFolder, Integer dbPort) { def namespace = generateNamespace(bezeichner) def configName = "cypress-ci-"+testFolder+".json" dir('goofy-client/apps/goofy-e2e/'){ def config = readJSON file: 'cypress-ci.json' def elasticsearchEnv = getElasticsearchEnv(namespace) config.baseUrl = "https://${bezeichner}.${env.CLUSTER_BASE_URL}" as String config.env.dbUrl = "mongodb://pluto-database-user:XnHhfznNWg65NNd@localhost:${dbPort}/admin?ssl=false&directConnection=true" as String config.env.keycloakUrl = "https://${env.SSO_URL}/" as String config.env.keycloakRealm = namespace as String config.env.keycloakClient = namespace + "-goofy" as String config.env.sabineUuid = getKeycloakUuid(namespace, "sabine") as String config.integrationFolder = "./src/integration/${testFolder}" as String config.videosFolder = "./reports/${testFolder}/videos" as String config.screenshotsFolder = "./reports/${testFolder}/screenshots" as String config.reporterOptions.reportDir = "./reports/${testFolder}/mochawesome-report" as String environment = config.env environment.put("search", elasticsearchEnv) environment.put("userManager", getUserManagerEnv(dbPort)); writeJSON file: configName, json: config sh "cat ${configName}" } return configName } String getUserManagerEnv(dbPort){ return readJSON(text: """{ "dbUrl":"mongodb://user-manager-database-user:5M3N2sVEq5c8@localhost:${dbPort}/admin?ssl=false&directConnection=true", \ "database":"user-manager-database"}"""); } String getKeycloakUuid(realm, userName) { def shScript = """curl -H 'Content-Type: application/json' \ -H 'Authorization: bearer ${getKeycloakAccessToken()}' \ 'https://${env.SSO_URL}/admin/realms/${realm}/users' """ def users = readJSON text: sh(script: shScript, returnStdout: true) for(user in users) { if (user.username == userName) { return user.id } } } String getKeycloakAccessToken() { withCredentials([usernamePassword(credentialsId: 'keycloak-dev-cluster', usernameVariable: 'USER', passwordVariable: 'PASSWORD')]) { def token = readJSON text: sh (script: 'curl -d "client_id=admin-cli" -d "username=$USER" -d "password=$PASSWORD" -d "grant_type=password" https://$SSO_URL/realms/master/protocol/openid-connect/token', returnStdout: true) return token.access_token } } Void sendFailureMessage() { def room = '' def data = """{"msgtype":"m.text", \ "body":"Goofy: Build Failed. Stage: ${FAILED_STAGE} Build-ID: ${env.BUILD_NUMBER} Link: ${BLUE_OCEAN_URL}", \ "format": "org.matrix.custom.html", \ "formatted_body":"Goofy: Build Failed. Stage: ${FAILED_STAGE} Build-ID: <a href='${BLUE_OCEAN_URL}'>${env.BUILD_NUMBER}</a>"}""" if (isMasterBranch()) { room = "!iQPAvQIiRwRpNOszjw:matrix.ozg-sh.de" } else if (isReleaseBranch()) { room = "!oWZpUGTFsxkJIYNfYg:matrix.ozg-sh.de" } sh "curl -XPOST -H 'authorization: Bearer ${getElementAccessToken()}' -d '${data}' https://matrix.ozg-sh.de/_matrix/client/v3/rooms/$room/send/m.room.message" } String getElementAccessToken() { withCredentials([string(credentialsId: 'element-login-json', variable: 'LOGIN_JSON')]) { return readJSON ( text: sh (script: '''curl -XPOST -d \"$LOGIN_JSON\" https://matrix.ozg-sh.de/_matrix/client/v3/login''', returnStdout: true)).access_token } } String getElasticsearchEnv(String namespace) { def elasticsearchSecret = getElasticsearchSecret(namespace) def env = """{ "user":"${decodeString(elasticsearchSecret.username)}", \ "password":"${decodeString(elasticsearchSecret.password)}", \ "index":"${decodeString(elasticsearchSecret.index)}", \ "url":"https://localhost:9200"}""" return readJSON ( text: env) } Void setNewDevGoofyVersion() { setNewGoofyGitopsVersion("rollout", "dev") } Void setNewTestGoofyVersion() { setNewGoofyGitopsVersion("rollout", "test") } Void setNewGoofyGitopsVersion(String directory, String environment) { dir("${directory}/gitops") { def envFile = "${environment}/application/values/goofy-values.yaml" def envVersions = readYaml file: envFile envVersions.goofy.image.tag = IMAGE_TAG envVersions.goofy.helm.version = HELM_CHART_VERSION writeYaml file: envFile, data: envVersions, overwrite: true if (hasGoofyValuesFileChanged(environment)) { sh "git add ${envFile}" sh "git commit -m 'jenkins rollout ${environment} goofy version ${IMAGE_TAG}'" } } } Boolean hasGoofyValuesFileChanged(String environment) { return sh (script: "git status | grep '${environment}/application/values/goofy-values.yaml'", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer } Void exposeElasticSearch() { portStatus = sh (script: "lsof -i -P -n | grep LISTEN | grep :9200", returnStatus: true) as Integer if(portStatus == 1) { sh "kubectl port-forward ozg-search-cluster-es-ozg-search-0 9200:9200 -n elastic-system &" } } Void exposeDatenbank(String namespace, port) { sh "kubectl port-forward pluto-database-0 ${port}:27017 -n ${namespace} &" } String generateNamespace(String bezeichner) { return "${env.BUNDESLAND}-${bezeichner}-dev" } Boolean isReleaseBranch() { return env.BRANCH_NAME == 'release' } Boolean isMasterBranch() { return env.BRANCH_NAME == 'master' } String decodeString(String encoded) { return sh (script: "echo -n ${encoded} | base64 --decode", returnStdout: true) } Map getElasticsearchSecret(String namespace) { return readJSON ( text: sh (script: "kubectl get secret elasticsearch-credentials -n ${namespace} -o jsonpath={.data}", returnStdout: true)) }