Skip to content
Snippets Groups Projects
Commit 3b6f3e8d authored by OZGCloud's avatar OZGCloud
Browse files

OZG-3294 jenkins argocd e2e tests

parent ca9ec434
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 190 additions and 224 deletions
Jenkinsfile
+ 190
− 224
View file @ 3b6f3e8d
...@@ -125,21 +125,6 @@ pipeline { ...@@ -125,21 +125,6 @@ pipeline {
} }
} }
} }
stage('Init k8s') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
E2E_FAILED = ""
configFileProvider([configFile(fileId: 'kubeconfig-dev-cluster', variable: 'KUBE_CONFIG')]) {
sh 'mkdir ~/.kube'
sh 'cp ${KUBE_CONFIG} ~/.kube/config'
}
sh 'helm version'
}
}
}
stage('Deploy Maven Artifacts to Nexus') { stage('Deploy Maven Artifacts to Nexus') {
when { when {
anyOf { anyOf {
...@@ -207,11 +192,10 @@ pipeline { ...@@ -207,11 +192,10 @@ pipeline {
if(currentBuild.changeSets.size() > 0) { if(currentBuild.changeSets.size() > 0) {
FAILED_STAGE = env.STAGE_NAME FAILED_STAGE = env.STAGE_NAME
checkoutProvisioningRepo() cloneGitopsRepo("rollout")
setNewGoofyProvisioningVersion('dev') setNewGoofyGitopsVersion("rollout", "dev")
pushGitopsRepo("rollout")
pushNewProvisioningVersion('dev')
} }
else { else {
sh 'echo "no code changes found"' sh 'echo "no code changes found"'
...@@ -219,19 +203,75 @@ pipeline { ...@@ -219,19 +203,75 @@ pipeline {
} }
} }
} }
stage('E2E') { stage('E2E') {
stages { stages {
stage('Init k8s') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
E2E_FAILED = ""
configFileProvider([configFile(fileId: 'kubeconfig-dev-cluster', variable: 'KUBE_CONFIG')]) {
sh 'mkdir ~/.kube'
sh 'cp ${KUBE_CONFIG} ~/.kube/config'
}
sh 'helm version'
}
}
}
stage('Install cypress') { stage('Install cypress') {
steps { steps {
script { script {
FAILED_STAGE = env.STAGE_NAME FAILED_STAGE = env.STAGE_NAME
E2E_FAILED = ""
dir('goofy-client') { dir('goofy-client') {
sh "npm run cypress:install" sh "npm run cypress:install"
} }
} }
} }
} }
stage('Rollout E2E Namespaces') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
E2E_FAILED = ""
cloneGitopsRepo("e2e")
checkoutE2eBranch("e2e")
env.EA_BEZEICHNER = generateBezeichner("e2e-ea")
env.MAIN_BEZEICHNER = generateBezeichner("e2e-main")
deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
generateNamespaceYaml(env.EA_BEZEICHNER, true)
generateNamespaceYaml(env.MAIN_BEZEICHNER, false)
pushGitopsRepo("e2e")
rolloutKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
}
}
}
stage('Add E2E User') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
E2E_FAILED = ""
cloneProvisioningRepo("e2e")
addKeycloakGroups(env.EA_BEZEICHNER, "e2e")
addKeycloakUser(env.EA_BEZEICHNER, "e2e")
addKeycloakGroups(env.MAIN_BEZEICHNER, "e2e")
addKeycloakUser(env.MAIN_BEZEICHNER, "e2e")
}
}
}
stage('Run E2E-Tests') { stage('Run E2E-Tests') {
failFast false failFast false
...@@ -240,16 +280,13 @@ pipeline { ...@@ -240,16 +280,13 @@ pipeline {
steps { steps {
script { script {
def stageName = env.STAGE_NAME def stageName = env.STAGE_NAME
def bezeichner = generateBezeichner(stageName) def bezeichner = env.EA_BEZEICHNER
def dbPort = 27018 def dbPort = 27018
startEnvironment(bezeichner, stageName, IMAGE_TAG, true, HELM_CHART_VERSION, dbPort) exposeDatenbank("${env.BUNDESLAND}-${bezeichner}-dev", dbPort)
exposeElasticSearch()
def testResult = runTests(stageName, bezeichner, 'einheitlicher-ansprechpartner', dbPort) def testResult = runTests(bezeichner, 'einheitlicher-ansprechpartner', dbPort)
if (env.BRANCH_NAME != 'master') {
deleteKopStack(bezeichner, stageName)
}
if(!testResult) { if(!testResult) {
E2E_FAILED += "${stageName}, " E2E_FAILED += "${stageName}, "
...@@ -269,16 +306,13 @@ pipeline { ...@@ -269,16 +306,13 @@ pipeline {
steps { steps {
script { script {
def stageName = env.STAGE_NAME def stageName = env.STAGE_NAME
def bezeichner = generateBezeichner(stageName) def bezeichner = env.MAIN_BEZEICHNER
def dbPort = 27019 def dbPort = 27019
startEnvironment(bezeichner, stageName, IMAGE_TAG, false, HELM_CHART_VERSION, dbPort) exposeDatenbank("${env.BUNDESLAND}-${bezeichner}-dev", dbPort)
exposeElasticSearch()
def testResult = runTests(stageName, bezeichner, 'main-tests', dbPort)
if (env.BRANCH_NAME != 'master') { def testResult = runTests(bezeichner, 'main-tests', dbPort)
deleteKopStack(bezeichner, stageName)
}
if(!testResult) { if(!testResult) {
E2E_FAILED += "${stageName}, " E2E_FAILED += "${stageName}, "
...@@ -296,6 +330,16 @@ pipeline { ...@@ -296,6 +330,16 @@ pipeline {
} }
} }
} }
stage('Delete E2E Namespaces') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
E2E_FAILED = ""
deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
}
}
}
} }
post { post {
always { always {
...@@ -308,7 +352,7 @@ pipeline { ...@@ -308,7 +352,7 @@ pipeline {
} }
} }
} }
stage('Trigger Test | Stage rollout') { stage('Trigger Test rollout') {
when { when {
branch 'release' branch 'release'
} }
...@@ -317,12 +361,10 @@ pipeline { ...@@ -317,12 +361,10 @@ pipeline {
script { script {
FAILED_STAGE = env.STAGE_NAME FAILED_STAGE = env.STAGE_NAME
checkoutProvisioningRepo() cloneGitopsRepo("rollout")
setNewGoofyProvisioningVersion('test')
setNewGoofyProvisioningVersion('stage')
pushNewProvisioningVersion('test stage') setNewGoofyGitopsVersion("rollout", "test")
pushGitopsRepo("rollout")
} }
} }
} }
...@@ -385,156 +427,71 @@ String generateImageTag() { ...@@ -385,156 +427,71 @@ String generateImageTag() {
return imageTag return imageTag
} }
Void startEnvironment(String bezeichner, String stage, String imageTag, Boolean isEa, String chartVersion, Integer dbPort) { Void cloneGitopsRepo(String directory) {
setupAnsible(imageTag, stage, isEa, chartVersion) withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) {
dir(directory) {
try { sh 'git clone https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/gitops.git'
deleteKopStack(bezeichner, stage)
} catch (Exception e) {
echo "deleteKopStack Exception"
}
rolloutKopStack(bezeichner, stage)
addKeycloakGroups(bezeichner, stage)
addKeycloakUser(bezeichner, stage)
exposeDatenbank("${env.BUNDESLAND}-${bezeichner}-dev", dbPort)
exposeElasticSearch()
}
Void setupAnsible(String imageTag, String stage, Boolean isEa, String chartVersion) {
checkoutProvisioningRepo(stage)
if (env.BRANCH_NAME == 'release') {
copyTestEnvironmentToDev(stage)
} }
editProvisioningBundesland(stage)
editEnvironemntVersion(stage, imageTag, isEa, chartVersion)
if (isEa) {
setupEaEnvironment(stage)
} }
configGit("${directory}/gitops")
setPlutoDatabasePassword(stage)
} }
Void setAnsibleKubeConfig() { Void cloneProvisioningRepo(String directory) {
configFileProvider([configFile(fileId: 'kubeconfig-dev-cluster', variable: 'KUBE_CONFIG')]) {
sh 'mkdir ~/.kube'
sh 'cp ${KUBE_CONFIG} ~/.kube/config'
}
}
Void checkoutProvisioningRepo(String stage="") {
withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) { withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) {
dir(stage) { dir(directory) {
sh 'git clone https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/provisioning.git' sh 'git clone https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/provisioning.git'
if (env.BRANCH_NAME == 'release') {
dir('provisioning') {
sh 'git checkout release'
}
}
} }
} }
} }
Void copyTestEnvironmentToDev(stage) { Void pushGitopsRepo(String directory) {
dir("${stage}/provisioning") { withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) {
def devEnvFile = "inventories/group_vars/dev/versions" dir("${directory}/gitops") {
def testEnvFile = "inventories/group_vars/test/versions" sh 'git push https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/gitops.git'
def devVersions = readYaml file: devEnvFile
def testVersions = readYaml file: testEnvFile
devVersions.charts = testVersions.charts
devVersions.versions = testVersions.versions
writeYaml file: devEnvFile, data: devVersions, overwrite: true
} }
} }
Void editEnvironemntVersion(String stage, String imageTag, Boolean isEa, String chartVersion) {
dir("${stage}/provisioning") {
def editFile = "inventories/group_vars/dev/versions"
def devVersions = readYaml file: editFile
overrideSpringProfiles = getSpringProfile(isEa)
devVersions.values.goofy.put('env', ['overrideSpringProfiles': overrideSpringProfiles])
devVersions.values.pluto.put('env', ['overrideSpringProfiles': overrideSpringProfiles])
devVersions.values.goofy.put('ingress', ['use_staging_cert': true])
devVersions.versions.goofy.image.tag = imageTag
devVersions.charts.goofy.version = chartVersion
writeYaml file: editFile, data: devVersions, overwrite: true
} }
}
Void editProvisioningBundesland(String stage) {
dir("${stage}/provisioning") {
def editEnvFile = "inventories/group_vars/all/env"
def envVersions = readYaml file: editEnvFile
envVersions.kop_bundesland = env.BUNDESLAND Void configGit(String directory) {
dir(directory) {
writeYaml file: editEnvFile, data: envVersions, overwrite: true sh 'git config user.email "jenkins@ozg-sh.de"'
sh 'git config user.name "jenkins"'
} }
} }
String getSpringProfile(Boolean isEa) { Void checkoutE2eBranch(String directory) {
if (isEa) { dir("${directory}/gitops") {
return "oc,ea,e2e,dev" sh 'git checkout e2e'
} }
return "oc,e2e,dev"
} }
Void setupEaEnvironment(String stage) { Void generateNamespaceYaml(String bezeichner, Boolean isEa) {
dir("${stage}/provisioning") { def envValues = readYaml file: valuesPath(isEa)
def editFile = "inventories/group_vars/all/env"
def groupVars = readYaml file: editFile
groupVars.kop_einheitlicher_ansprechpartner = true envValues.kop.bezeichner = bezeichner
envValues.goofy.image.tag = IMAGE_TAG
envValues.goofy.helm.version = HELM_CHART_VERSION
writeYaml file: editFile, data: groupVars, overwrite: true if (env.BRANCH_NAME == 'release') {
} envValues.put("argocd", ['source': ['path': 'test/application']])
} }
Void setPlutoDatabasePassword(String stage) { writeYaml file: "e2e/gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml", data: envValues, overwrite: true
dir("${stage}/provisioning") {
def editFile = "inventories/group_vars/dev/versions"
def devVars = readYaml file: editFile
devVars.values.pluto.database.password = "XnHhfznNWg65NNd" sh "cat e2e/gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml"
writeYaml file: editFile, data: devVars, overwrite: true sh "git add e2e/gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml"
} sh "git commit -m 'add e2e by-${bezeichner}-dev'"
} }
Void rolloutKopStack(String bezeichner, String stage) { String valuesPath(Boolean isEa) {
dir("${stage}/provisioning") { def path = "goofy-client/apps/goofy-e2e/src/fixtures/argocd"
def ansibleVars = """{"k8s_context":"ozg-dev", \
"kop_env":"dev", \
"kop_bezeichner":${bezeichner}, \
"kop_displayname":${bezeichner}, \
"kop_postfach_api_key":"", \
"install_afm_adapter":false, \
"install_fs_adapter":false, \
"external_db_enabled":false, \
"disable_update":true}"""
if (env.BRANCH_NAME == 'release') { if (isEa) {
sh "ansible-playbook playbook/rollout.yml --extra-vars '${ansibleVars}'" return "${path}/by-ea-dev.yaml"
}
else {
sh "ansible-playbook playbooks/rollout.yml --extra-vars '${ansibleVars}'"
}
} }
return "${path}/by-main-dev.yaml"
} }
Void addKeycloakGroups(String bezeichner, String stage) { Void addKeycloakGroups(String bezeichner, String stage) {
...@@ -583,18 +540,64 @@ Void addKeycloakUser(String bezeichner, String stage) { ...@@ -583,18 +540,64 @@ Void addKeycloakUser(String bezeichner, String stage) {
} }
} }
Void deleteKopStack(String bezeichner, String stage) { Void deleteKopStack(Map kopBezeichner) {
dir("${stage}/provisioning") { for(bezeichner in kopBezeichner) {
def ansibleVars = """{"k8s_context":"ozg-dev", \ dir("e2e/gitops/dev/namespace/namespaces") {
"kop_env":"dev", \ if (sh (script: "ls | grep 'by-${bezeichner}-dev.yaml'", returnStatus: true) == 1) {
"kop_bezeichner":${bezeichner}}""" return
}
if (env.BRANCH_NAME == 'release') { sh "rm by-${bezeichner}-dev.yaml"
sh "ansible-playbook playbook/delete-commune.yml --extra-vars '${ansibleVars}'" sh "git add by-${bezeichner}-dev.yaml"
sh "git commit -m 'delete e2e by-${bezeichner}-dev.yaml'"
} }
else {
sh "ansible-playbook playbooks/delete-commune.yml --extra-vars '${ansibleVars}'"
} }
pushGitopsRepo("e2e")
for(bezeichner in kopBezeichner) {
waitForDeletion(bezeichner)
}
}
Void waitForDeletion(String bezeichner) {
try {
sh "kubectl wait --for=delete applications/by-${bezeichner}-dev-application -n argocd --timeout=300s"
} catch (Exception e) {
error("Application by-${bezeichner}-dev-application konnte nicht gelöscht werden")
}
}
Void rolloutKopStack(Map kopBezeichner) {
pushGitopsRepo("e2e")
for(bezeichner in kopBezeichner) {
waitForHealthyApplication(bezeichner)
}
}
Void waitForRollout(String bezeichner) {
waitForHealthyApplication(bezeichner, 'application')
waitForHealthyApplication(bezeichner, 'user-manager')
waitForHealthyApplication(bezeichner, 'pluto')
waitForHealthyApplication(bezeichner, 'goofy')
}
Void waitForHealthyApplication(String bezeichner, Strin application) {
try {
def countRetry = 0
def maxRetry = 12
while (sh (script: "kubectl get applications -n argocd | grep 'by-${bezeichner}-dev-${application}'", returnStatus: true) == 1 && countRetry < maxRetry ) {
countRetry++
sh "sleep 5"
}
if (sh (script: "kubectl get application/by-${bezeichner}-dev-${application} -n argocd -o=jsonpath='{.status.health.status}' | grep Healthy", returnStatus: true) == 0) {
sh "kubectl wait --for=jsonpath='{.status.health.status}'=Healthy applications/by-${bezeichner}-dev-${application} -n argocd --timeout=300s"
}
} catch (Exception e) {
error("Application ${application} unhealthy")
} }
} }
...@@ -611,8 +614,8 @@ Void publishE2ETestResult(String reportFolder, String reportName) { ...@@ -611,8 +614,8 @@ Void publishE2ETestResult(String reportFolder, String reportName) {
) )
} }
String runTests(String stageName, String bezeichner, String reportFolder, Integer dbPort) { String runTests(String bezeichner, String reportFolder, Integer dbPort) {
def configFile = generateCypressConfig(stageName, bezeichner, reportFolder, dbPort) def configFile = generateCypressConfig(bezeichner, reportFolder, dbPort)
try { try {
dir("goofy-client") { dir("goofy-client") {
...@@ -651,7 +654,7 @@ String cutBranchNameForKeycloakRealm(String branchName, String stageName) { ...@@ -651,7 +654,7 @@ String cutBranchNameForKeycloakRealm(String branchName, String stageName) {
return branchName return branchName
} }
String generateCypressConfig(String stage, String bezeichner, String testFolder, Integer dbPort) { String generateCypressConfig(String bezeichner, String testFolder, Integer dbPort) {
def namespace = "${env.BUNDESLAND}-${bezeichner}-dev" def namespace = "${env.BUNDESLAND}-${bezeichner}-dev"
def configName = "cypress-ci-"+testFolder+".json" def configName = "cypress-ci-"+testFolder+".json"
...@@ -673,7 +676,7 @@ String generateCypressConfig(String stage, String bezeichner, String testFolder, ...@@ -673,7 +676,7 @@ String generateCypressConfig(String stage, String bezeichner, String testFolder,
environment = config.env environment = config.env
environment.put("search", elasticsearchEnv) environment.put("search", elasticsearchEnv)
environment.put("userManager", getUserManagerEnv(namespace, dbPort)); environment.put("userManager", getUserManagerEnv(dbPort));
writeJSON file: configName, json: config writeJSON file: configName, json: config
...@@ -683,21 +686,12 @@ String generateCypressConfig(String stage, String bezeichner, String testFolder, ...@@ -683,21 +686,12 @@ String generateCypressConfig(String stage, String bezeichner, String testFolder,
return configName return configName
} }
String getUserManagerEnv(namespace, dbPort){ String getUserManagerEnv(dbPort){
def secret = getSecrect(namespace, 'pluto-database-admin-user-manager-database-user');
def secretPassword = decodeString(secret.password);
return readJSON(text: """{ return readJSON(text: """{
"dbUrl":"mongodb://user-manager-database-user:${secretPassword}@localhost:${dbPort}/admin?ssl=false&directConnection=true", \ "dbUrl":"mongodb://user-manager-database-user:5M3N2sVEq5c8@localhost:${dbPort}/admin?ssl=false&directConnection=true", \
"database":"user-manager-database"}"""); "database":"user-manager-database"}""");
} }
def getSecrect(namespace, secretName){
script {
return readJSON ( text: sh (script: "kubectl get secret ${secretName} -n ${namespace} -o jsonpath={.data}", returnStdout: true))
}
}
String getKeycloakUuid(realm, userName) { String getKeycloakUuid(realm, userName) {
def shScript = """curl -H 'Content-Type: application/json' \ def shScript = """curl -H 'Content-Type: application/json' \
-H 'Authorization: bearer ${getKeycloakAccessToken()}' \ -H 'Authorization: bearer ${getKeycloakAccessToken()}' \
...@@ -744,57 +738,29 @@ String getElementAccessToken() { ...@@ -744,57 +738,29 @@ String getElementAccessToken() {
} }
} }
def getElasticsearchSecret(namespace) {
script {
def elasticsearch = readJSON ( text: sh (script: "kubectl get secret elasticsearch-credentials -n ${namespace} -o jsonpath={.data}", returnStdout: true))
return elasticsearch
}
}
String getElasticsearchEnv(namespace) { String getElasticsearchEnv(namespace) {
def elasticsearchSecret = getElasticsearchSecret(namespace)
def env = """{ def env = """{
"user":"${decodeString(elasticsearchSecret.username)}", \ "user":"${namespace}", \
"password":"${decodeString(elasticsearchSecret.password)}", \ "password":"vf9W1D8Z3673", \
"index":"${decodeString(elasticsearchSecret.index)}", \ "index":"${namespace}", \
"url":"https://localhost:9200"}""" "url":"https://localhost:9200"}"""
return readJSON ( text: env) return readJSON ( text: env)
} }
String decodeString(encoded) { Void setNewGoofyGitopsVersion(String directory, String environment) {
return sh (script: "echo -n ${encoded} | base64 --decode", returnStdout: true) dir("${directory}/gitops") {
} def envFile = "${environment}/application/values/goofy-values.yaml"
Void setNewGoofyProvisioningVersion(String environment) {
dir("provisioning") {
def envFile = "inventories/group_vars/${environment}/versions"
def envVersions = readYaml file: envFile def envVersions = readYaml file: envFile
envVersions.versions.goofy.image.tag = IMAGE_TAG envVersions.goofy.image.tag = IMAGE_TAG
envVersions.charts.goofy.version = HELM_CHART_VERSION envVersions.goofy.helm.version = HELM_CHART_VERSION
writeYaml file: envFile, data: envVersions, overwrite: true writeYaml file: envFile, data: envVersions, overwrite: true
}
}
Void pushNewProvisioningVersion(String environment) { sh "git add ${environment}/application/values/goofy-values.yaml"
dir('provisioning') {
if (sh (script: "git status | grep 'inventories/group_vars/.*/versions'", returnStatus: true) == 1) {
return
}
withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) {
sh 'git add inventories/group_vars/*/versions'
sh 'git config user.email "jenkins@ozg.de"'
sh 'git config user.name "jenkins"'
sh "git commit -m 'jenkins rollout ${environment} goofy version ${IMAGE_TAG}'" sh "git commit -m 'jenkins rollout ${environment} goofy version ${IMAGE_TAG}'"
sh 'git push https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/provisioning.git'
}
} }
} }
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment

Impressum - Datenschutz - Barrierefreiheit