From ae63a515d9498588ad97fcd424566500dde3df9f Mon Sep 17 00:00:00 2001 From: OZGCloud <ozgcloud@mgm-tp.com> Date: Fri, 10 Mar 2023 08:36:10 +0100 Subject: [PATCH] jenkinsfile e2e tests --- goofy-client/apps/goofy-e2e/Jenkinsfile | 606 +++++++++++++++++- .../src/fixtures/argocd/by-ea-dev.yaml | 5 +- 2 files changed, 597 insertions(+), 14 deletions(-) diff --git a/goofy-client/apps/goofy-e2e/Jenkinsfile b/goofy-client/apps/goofy-e2e/Jenkinsfile index 586479131e..685b7d3807 100644 --- a/goofy-client/apps/goofy-e2e/Jenkinsfile +++ b/goofy-client/apps/goofy-e2e/Jenkinsfile @@ -1,3 +1,5 @@ +import groovy.json.JsonOutput + pipeline { agent { node { @@ -9,16 +11,40 @@ pipeline { upstream(upstreamProjects: getUpstreamProjects(), threshold: hudson.model.Result.SUCCESS) } + environment { + BLUE_OCEAN_URL = "https://jenkins.ozg-sh.de/job/E2E%20Tests/job/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/" + BUNDESLAND = "by" + SSO_URL = "sso.dev.by.ozg-cloud.de" + CLUSTER_BASE_URL = "dev.by.ozg-cloud.de" + FAILED_STAGE = "" + FAILED_PARALLEL_STAGE = " " + EA_BEZEICHNER = generateBezeichner("e2e-ea") + MAIN_BEZEICHNER = generateBezeichner("e2e-main") + SH_SUCCESS_STATUS_CODE = 0 + } + + options { + timeout(time: 1, unit: 'HOURS') + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '5')) + } + stages { stage("Clone Gitops Repo") { steps { - cloneGitopsRepo() + script { + FAILED_STAGE = env.STAGE_NAME + + cloneGitopsRepo() + } } } stage("Init Default Versions") { steps { script { + FAILED_STAGE = env.STAGE_NAME + initEnvGoofyDefaultVersions() initEnvPlutoDefaultVersions() initEnvUserManagerDefaultVersions() @@ -32,7 +58,9 @@ pipeline { triggeredBy cause: "UserIdCause" } steps { - script { + script { + FAILED_STAGE = env.STAGE_NAME + userVersions = input message: "Edit Default Values", parameters: [ string(name: "GoofyImageTag", defaultValue: env.GOOFY_IMAGE_TAG, trim: true), @@ -48,15 +76,140 @@ pipeline { } } - stage("Print Versions") { + stage("Install Cypress") { + steps { + script { + FAILED_STAGE = env.STAGE_NAME + + sh 'npm --version' + dir('goofy-client') { + sh 'echo "registry=https://nexus.ozg-sh.de/repository/npm-proxy" >> ~/.npmrc' + sh 'echo "//nexus.ozg-sh.de/:_auth=amVua2luczpQaihzX0ZNNFU5ZC8=" >> ~/.npmrc' + + sh 'npm cache verify' + sh 'npm install' + sh "npm run cypress:install" + } + } + } + } + + stage('Init k8s') { + steps { + script { + FAILED_STAGE = env.STAGE_NAME + + configFileProvider([configFile(fileId: 'kubeconfig-ovh-cluster', variable: 'KUBE_CONFIG')]) { + sh 'mkdir ~/.kube' + sh 'cp ${KUBE_CONFIG} ~/.kube/config' + } + + sh 'helm version' + } + } + } + + stage('Rollout E2E Namespaces') { + steps { + script { + FAILED_STAGE = env.STAGE_NAME + + checkoutGitopsE2eBranch() + + deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER]) + + generateEaNamespaceYaml() + generateMainNamespaceYaml() + + pushGitopsRepo() + + waitForKopStackRollout([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER]) + } + } + post { + failure { + script { + deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER]) + } + } + } + } + + stage('Run E2E-Tests') { + failFast false + + parallel { + stage('E2E-EA') { + steps { + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + script { + def bezeichner = env.EA_BEZEICHNER + def dbPort = 27018 + + forwardServices(generateNamespace(bezeichner), dbPort) + + runTests(bezeichner, 'einheitlicher-ansprechpartner', dbPort, env.STAGE_NAME) + } + } + } + post { + failure { + script { + FAILED_PARALLEL_STAGE += "${env.STAGE_NAME} " + } + } + always { + script { + publishE2ETestResult("einheitlicher-ansprechpartner", "Goofy E2E-Tests EA") + } + } + } + } + + stage('E2E-Main') { + steps { + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + script { + def bezeichner = env.MAIN_BEZEICHNER + def dbPort = 27019 + + forwardServices(generateNamespace(bezeichner), dbPort) + + runTests(bezeichner, 'main-tests', dbPort, env.STAGE_NAME) + } + } + } + post { + failure { + script { + FAILED_PARALLEL_STAGE += "${env.STAGE_NAME} " + } + } + always { + script { + publishE2ETestResult("main-tests", "Goofy E2E-Tests main") + } + } + } + } + } + } + + stage('Delete E2E Namespaces') { steps { script { - println "GOOFY_IMAGE_TAG: ${env.GOOFY_IMAGE_TAG}" - println "GOOFY_HELM_CHART_VERSION: ${env.GOOFY_HELM_CHART_VERSION}" - println "PLUTO_IMAGE_TAG: ${env.PLUTO_IMAGE_TAG}" - println "PLUTO_HELM_CHART_VERSION: ${env.PLUTO_HELM_CHART_VERSION}" - println "USER_MANAGER_IMAGE_TAG: ${env.USER_MANAGER_IMAGE_TAG}" - println "USER_MANAGER_HELM_CHART_VERSION: ${env.USER_MANAGER_HELM_CHART_VERSION}" + FAILED_STAGE = env.STAGE_NAME + + deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER]) + } + } + } + } + post { + failure { + script { + if (isMasterBranch() || isReleaseBranch()) { + sendFailureMessage() } } } @@ -68,7 +221,7 @@ String getUpstreamProjects() { return "goofy/${env.BRANCH_NAME},pluto/${env.BRANCH_NAME},user-manager/${env.BRANCH_NAME}" } - return "goofy/${env.BRANCH_NAME}" + return "" } Boolean isReleaseBranch() { @@ -80,8 +233,16 @@ Boolean isMasterBranch() { } def cloneGitopsRepo() { + final email = "jenkins@ozg-sh.de" + final name = "jenkins" + withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) { sh 'git clone https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/gitops.git' + + dir("gitops") { + sh "git config user.email '${email}'" + sh "git config user.name '${name}'" + } } } @@ -108,10 +269,30 @@ String getHelmChartVersion(Map applicationValues) { } Void initEnvGoofyDefaultVersions() { - goofyValues = getApplicationValues("goofy") + if (isMasterBranch() || isReleaseBranch()) { + goofyValues = getApplicationValues("goofy") + + env.GOOFY_IMAGE_TAG = getImageTag(goofyValues) + env.GOOFY_HELM_CHART_VERSION = getHelmChartVersion(goofyValues) + } + else { + env.GOOFY_IMAGE_TAG = getFeatureBranchImageTag() + env.GOOFY_HELM_CHART_VERSION = getFeatureBranchHelmChartVersion() + } +} + +String getFeatureBranchImageTag() { + return "${env.BRANCH_NAME}-${getRootPomVersion()}" +} + +String getFeatureBranchHelmChartVersion() { + return "${getRootPomVersion()}-${env.BRANCH_NAME}".replaceAll("_", "-") +} + +String getRootPomVersion() { + def rootPom = readMavenPom file: 'pom.xml' - env.GOOFY_IMAGE_TAG = getImageTag(goofyValues) - env.GOOFY_HELM_CHART_VERSION = getHelmChartVersion(goofyValues) + return rootPom.version } Void initEnvPlutoDefaultVersions() { @@ -135,4 +316,403 @@ Void initEnvUserVersions(userVersions) { env.PLUTO_HELM_CHART_VERSION = userVersions.PlutoHelmChartVersion env.USER_MANAGER_IMAGE_TAG = userVersions.UserManagerImageTag env.USER_MANAGER_HELM_CHART_VERSION = userVersions.UserManagerHelmChartVersion +} + + +Void pushGitopsRepo() { + withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) { + dir("gitops") { + if (hasUnpushedCommits()) { + sh 'git push https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/gitops.git' + } + } + } +} + +Boolean hasUnpushedCommits() { + return sh (script: "git cherry -v | grep .", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer +} + +Void checkoutGitopsE2eBranch() { + dir("gitops") { + sh 'git checkout e2e' + } +} + +Void generateEaNamespaceYaml() { + generateNamespaceYaml(env.EA_BEZEICHNER, "goofy-client/apps/goofy-e2e/src/fixtures/argocd/by-ea-dev.yaml") +} + +Void generateMainNamespaceYaml() { + generateNamespaceYaml(env.MAIN_BEZEICHNER, "goofy-client/apps/goofy-e2e/src/fixtures/argocd/by-main-dev.yaml") +} + +Void generateNamespaceYaml(String bezeichner, String valuesPath) { + def envValues = readYaml file: valuesPath + + envValues.kop.bezeichner = bezeichner + envValues.goofy.put("image", ['tag': env.GOOFY_IMAGE_TAG]) + envValues.goofy.put("helm", ['version': env.GOOFY_HELM_CHART_VERSION]) + + envValues.pluto.put("image", ['tag': env.PLUTO_IMAGE_TAG]) + envValues.pluto.put("helm", ['version': env.PLUTO_HELM_CHART_VERSION]) + + envValues.user_manager.put("image", ['tag': env.USER_MANAGER_IMAGE_TAG]) + envValues.user_manager.put("helm", ['version': env.USER_MANAGER_HELM_CHART_VERSION]) + + envValues.goofy.sso.put("keycloak_groups", generateKeycloakGroupsForHelmChart()) + envValues.goofy.sso.put("keycloak_users", generateKeycloakUserForHelmChart()) + + writeYaml file: "gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml", data: envValues, overwrite: true + + sh "cat gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml" + + dir("gitops") { + sh "git add dev/namespace/namespaces/by-${bezeichner}-dev.yaml" + sh "git commit -m 'add e2e by-${bezeichner}-dev'" + } +} + +List generateKeycloakUserForHelmChart() { + def userFiles = sh (script: 'ls goofy-client/apps/goofy-e2e/src/fixtures/user', returnStdout: true) + + def helmUsers = [] + + userFiles.split("\\n").each { userFile -> + def userJson = readJSON file: "goofy-client/apps/goofy-e2e/src/fixtures/user/${userFile}" + def user = [ + "name": userJson.name, + "password": userJson.password, + "first_name": userJson.get("firstName", ""), + "last_name": userJson.get("lastName", "") + ] + + if (userJson.containsKey("clientRoles")) { + user.put("client_roles", mapUserClientRoles(userJson.clientRoles)) + } + + if (userJson.containsKey("groups")) { + user.put("groups", userJson.groups) + } + + helmUsers.add(user) + } + + return helmUsers +} + +List mapUserClientRoles(userClientRoles) { + def clientRoles = [] + + for(clientRole in userClientRoles) { + clientRoles.add(['name': "alfa", 'role': clientRole]) + } + + return clientRoles +} + +List generateKeycloakGroupsForHelmChart() { + def groupFiles = sh (script: 'ls goofy-client/apps/goofy-e2e/src/fixtures/group', returnStdout: true) + + def helmGroups = [] + + groupFiles.split("\\n").each { groupFile -> + def groupJson = readJSON file: "goofy-client/apps/goofy-e2e/src/fixtures/group/${groupFile}" + def group = ["name": groupJson.name] + + groupJson.attributes.each { key, values -> + if (!group.containsKey("attributes")) { + group.put("attributes", [["name": key, "value": values]]) + } + else { + group.attributes.add(["name": key, "value": values]) + } + } + + helmGroups.add(group) + } + + return helmGroups +} + +Void deleteKopStack(kopBezeichner) { + for(bezeichner in kopBezeichner) { + if (hasNamespaceFile(bezeichner)) { + removeNamespaceFile(bezeichner) + } + } + + pushGitopsRepo() + + for(bezeichner in kopBezeichner) { + waitForDeletion(bezeichner) + } +} + +Void removeNamespaceFile(String bezeichner) { + dir("gitops/dev/namespace/namespaces") { + sh "rm by-${bezeichner}-dev.yaml" + sh "git add by-${bezeichner}-dev.yaml" + sh "git commit -m 'delete e2e by-${bezeichner}-dev.yaml'" + } +} + +Boolean hasNamespaceFile(String bezeichner) { + return sh (script: "ls gitops/dev/namespace/namespaces | grep 'by-${bezeichner}-dev.yaml'", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer +} + +Void waitForDeletion(String bezeichner) { + try { + sh "kubectl wait --for=delete applications/by-${bezeichner}-dev-application -n argocd --timeout=300s" + } catch (Exception e) { + error("Application by-${bezeichner}-dev-application konnte nicht gelöscht werden") + } +} + +Void waitForKopStackRollout(kopBezeichner) { + for(bezeichner in kopBezeichner) { + waitForRollout(bezeichner) + } +} + +Void waitForRollout(String bezeichner) { + waitForHealthyApplication(bezeichner, 'application') + waitForHealthyApplication(bezeichner, 'user-manager') + waitForHealthyApplication(bezeichner, 'pluto') + waitForHealthyApplication(bezeichner, 'goofy') +} + +Void waitForHealthyApplication(String bezeichner, String application) { + try { + def countRetry = 0 + def maxRetry = 12 + + while (!isApplicationPresent(bezeichner, application) && countRetry < maxRetry ) { + countRetry++ + sh "sleep 5" + } + + if (!isApplicationHealthy(bezeichner, application)) { + waitForHealthyStatus(bezeichner, application) + } + } catch (Exception e) { + error("Application ${application} unhealthy") + } +} + +Boolean isApplicationPresent(String bezeichner, String application) { + return sh (script: "kubectl get applications -n argocd | grep 'by-${bezeichner}-dev-${application}'", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer +} + +Boolean isApplicationHealthy(String bezeichner, String application) { + return sh (script: "kubectl get application/by-${bezeichner}-dev-${application} -n argocd -o=jsonpath='{.status.health.status}' | grep Healthy", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer +} + +Void waitForHealthyStatus(String bezeichner, String application) { + sh "kubectl wait --for=jsonpath='{.status.health.status}'=Healthy applications/by-${bezeichner}-dev-${application} -n argocd --timeout=300s" +} + +Void publishE2ETestResult(String reportFolder, String reportName) { + publishHTML ( + target: [ + allowMissing: false, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: "goofy-client/apps/goofy-e2e/reports/${reportFolder}", + reportFiles: 'report.html', + reportName: reportName + ] + ) +} + +String runTests(String bezeichner, String reportFolder, Integer dbPort, String stageName) { + def configFile = generateCypressConfig(bezeichner, reportFolder, dbPort) + + try { + dir("goofy-client") { + sh "npm run cypress:version" + sh "npm run cypress:ci-run --CONFIG_FILE=${configFile} --REPORT_FOLDER=${reportFolder}" + } + } catch (Exception e) { + printNpmDebugLog() + + error("Fehler in Stage ${stageName}") + } +} + +Void printNpmDebugLog() { + if (hasNpmDebugLog()) { + sh "cat /root/.npm/_logs/*-debug.log" + } + else { + echo "Npm debug log not found" + } +} + +String makeUrlConform(String input) { + return input.replaceAll(/[^a-zA-Z0-9]+/, "").toLowerCase() +} + +String generateBezeichner(String stage) { + def branchName = makeUrlConform(env.BRANCH_NAME) + def stageName = makeUrlConform(stage) + + return "${cutBranchNameForKeycloakRealm(branchName, stageName)}-${stageName}" +} + +String cutBranchNameForKeycloakRealm(String branchName, String stageName) { + final maxKeycloakRealmLength = 30 + + def cutBranchNamePosition = maxKeycloakRealmLength - (stageName.length() + "${env.BUNDESLAND}---dev".length()) + + return branchName.take(cutBranchNamePosition) +} + +String generateCypressConfig(String bezeichner, String testFolder, Integer dbPort) { + def namespace = generateNamespace(bezeichner) + def configName = "cypress-ci-"+testFolder+".json" + + dir('goofy-client/apps/goofy-e2e/'){ + def config = readJSON file: 'cypress-ci.json' + + config.baseUrl = "https://${bezeichner}.${env.CLUSTER_BASE_URL}" as String + config.env.dbUrl = "mongodb://pluto-database-user:XnHhfznNWg65NNd@localhost:${dbPort}/admin?ssl=false&directConnection=true" as String + config.env.keycloakUrl = "https://${env.SSO_URL}/" as String + config.env.keycloakRealm = namespace as String + config.env.keycloakClient = "alfa" as String + config.env.sabineUuid = getKeycloakUuid(namespace, "sabine") as String + config.integrationFolder = "./src/integration/${testFolder}" as String + config.videosFolder = "./reports/${testFolder}/videos" as String + config.screenshotsFolder = "./reports/${testFolder}/screenshots" as String + config.reporterOptions.reportDir = "./reports/${testFolder}/mochawesome-report" as String + + config.env.put("search", getElasticsearchEnv(namespace)) + config.env.put("userManager", getUserManagerEnv(dbPort)) + + writeJSON file: configName, json: config + + sh "cat ${configName}" + } + + return configName +} + +Map getUserManagerEnv(dbPort){ + return [ + "dbUrl": "mongodb://user-manager-database-user:5M3N2sVEq5c8@localhost:${dbPort}/admin?ssl=false&directConnection=true" as String, + "database": "user-manager-database" + ] +} + +String getKeycloakUuid(realm, userName) { + def shScript = """curl -H 'Content-Type: application/json' \ + -H 'Authorization: bearer ${getKeycloakAccessToken()}' \ + 'https://${env.SSO_URL}/admin/realms/${realm}/users' + """ + + def users = readJSON text: sh(script: shScript, returnStdout: true) + + echo "users: ${users}" + + for(user in users) { + if (user.username == userName) { + return user.id + } + } +} + +String getKeycloakAccessToken() { + withCredentials([usernamePassword(credentialsId: 'keycloak-ovh-cluster', usernameVariable: 'USER', passwordVariable: 'PASSWORD')]) { + def token = readJSON text: sh (script: 'curl -d "client_id=admin-cli" -d "username=$USER" -d "password=$PASSWORD" -d "grant_type=password" https://$SSO_URL/realms/master/protocol/openid-connect/token', returnStdout: true) + + return token.access_token + } +} + +Void sendFailureMessage() { + def data = [ + "msgtype": "m.text", + "body": "E2E-Tests: Failed stage: ${getFailedStage()} Build-ID: ${env.BUILD_NUMBER} Link: ${BLUE_OCEAN_URL}" as String, + "format": "org.matrix.custom.html", + "formatted_body": "E2E-Tests: Failed stage: ${getFailedStage()} Build-ID: <a href='${BLUE_OCEAN_URL}'>${env.BUILD_NUMBER}</a>" as String + ] + + sh "curl -XPOST -H 'authorization: Bearer ${getElementAccessToken()}' -d '${JsonOutput.toJson(data)}' https://matrix.ozg-sh.de/_matrix/client/v3/rooms/${getElementRoomId()}/send/m.room.message" +} + +String getFailedStage() { + if (FAILED_PARALLEL_STAGE.trim()) { + return FAILED_PARALLEL_STAGE + } + + return FAILED_STAGE +} + +String getElementRoomId() { + final releaseRoomId = "!oWZpUGTFsxkJIYNfYg:matrix.ozg-sh.de" + final masterRoomId = "!iQPAvQIiRwRpNOszjw:matrix.ozg-sh.de" + + if (isReleaseBranch()) { + return releaseRoomId + } + + return masterRoomId +} + +String getElementAccessToken() { + withCredentials([string(credentialsId: 'element-login-json', variable: 'LOGIN_JSON')]) { + return readJSON ( text: sh (script: '''curl -XPOST -d \"$LOGIN_JSON\" https://matrix.ozg-sh.de/_matrix/client/v3/login''', returnStdout: true)).access_token + } +} + +Map getElasticsearchEnv(String namespace) { + def elasticsearchSecret = getElasticsearchSecret(namespace) + + return [ + "user": decodeString(elasticsearchSecret.username), + "password": decodeString(elasticsearchSecret.password), + "index": decodeString(elasticsearchSecret.index), + "url": "https://localhost:9200" + ] +} + +Void forwardServices(String namespace, Integer dbPort) { + try { + forwardDatenbank(namespace, dbPort) + forwardElasticSearch() + } + catch (Exception e) { + echo "forwardServices Exception: ${e}" + error("Error forwarding service") + } +} + +Void forwardElasticSearch() { + if(!isElasticSearchForwarded()) { + sh "kubectl port-forward ozg-search-cluster-es-ozg-search-0 9200:9200 -n elastic-system &" + } +} + +Boolean isElasticSearchForwarded() { + return sh (script: "lsof -i -P -n | grep LISTEN | grep :9200", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer +} + +Void forwardDatenbank(String namespace, port) { + sh "kubectl port-forward pluto-database-0 ${port}:27017 -n ${namespace} &" +} + +String generateNamespace(String bezeichner) { + return "${env.BUNDESLAND}-${bezeichner}-dev" +} + +String decodeString(String encoded) { + return sh (script: "echo -n ${encoded} | base64 --decode", returnStdout: true) +} + +Map getElasticsearchSecret(String namespace) { + return readJSON ( text: sh (script: "kubectl get secret elasticsearch-credentials -n ${namespace} -o jsonpath={.data}", returnStdout: true)) +} + +Boolean hasNpmDebugLog() { + return sh (script: "ls -l /root/.npm/_logs/*-debug.log", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer } \ No newline at end of file diff --git a/goofy-client/apps/goofy-e2e/src/fixtures/argocd/by-ea-dev.yaml b/goofy-client/apps/goofy-e2e/src/fixtures/argocd/by-ea-dev.yaml index 98bb47af69..ac84437c4d 100644 --- a/goofy-client/apps/goofy-e2e/src/fixtures/argocd/by-ea-dev.yaml +++ b/goofy-client/apps/goofy-e2e/src/fixtures/argocd/by-ea-dev.yaml @@ -8,7 +8,10 @@ goofy: sso: serverUrl: https://sso.dev.by.ozg-cloud.de apiPassword: "Test1234!" - role_einheitlicher_ansprechpartner: true + keycloak_clients: + - client_name: alfa + client_roles: + - name: EINHEITLICHER_ANSPRECHPARTNER ingress: use_staging_cert: true baseUrl: dev.by.ozg-cloud.de -- GitLab