Skip to content
Snippets Groups Projects
Commit 3b6f3e8d authored by OZGCloud's avatar OZGCloud
Browse files

OZG-3294 jenkins argocd e2e tests

parent ca9ec434
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 190 additions and 224 deletions
Jenkinsfile
+ 190
− 224
View file @ 3b6f3e8d
......@@ -125,21 +125,6 @@ pipeline {
}
}
}
stage('Init k8s') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
E2E_FAILED = ""
configFileProvider([configFile(fileId: 'kubeconfig-dev-cluster', variable: 'KUBE_CONFIG')]) {
sh 'mkdir ~/.kube'
sh 'cp ${KUBE_CONFIG} ~/.kube/config'
}
sh 'helm version'
}
}
}
stage('Deploy Maven Artifacts to Nexus') {
when {
anyOf {
......@@ -207,11 +192,10 @@ pipeline {
if(currentBuild.changeSets.size() > 0) {
FAILED_STAGE = env.STAGE_NAME
checkoutProvisioningRepo()
cloneGitopsRepo("rollout")
setNewGoofyProvisioningVersion('dev')
pushNewProvisioningVersion('dev')
setNewGoofyGitopsVersion("rollout", "dev")
pushGitopsRepo("rollout")
}
else {
sh 'echo "no code changes found"'
......@@ -219,19 +203,75 @@ pipeline {
}
}
}
stage('E2E') {
stages {
stage('Init k8s') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
E2E_FAILED = ""
configFileProvider([configFile(fileId: 'kubeconfig-dev-cluster', variable: 'KUBE_CONFIG')]) {
sh 'mkdir ~/.kube'
sh 'cp ${KUBE_CONFIG} ~/.kube/config'
}
sh 'helm version'
}
}
}
stage('Install cypress') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
E2E_FAILED = ""
dir('goofy-client') {
sh "npm run cypress:install"
}
}
}
}
stage('Rollout E2E Namespaces') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
E2E_FAILED = ""
cloneGitopsRepo("e2e")
checkoutE2eBranch("e2e")
env.EA_BEZEICHNER = generateBezeichner("e2e-ea")
env.MAIN_BEZEICHNER = generateBezeichner("e2e-main")
deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
generateNamespaceYaml(env.EA_BEZEICHNER, true)
generateNamespaceYaml(env.MAIN_BEZEICHNER, false)
pushGitopsRepo("e2e")
rolloutKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
}
}
}
stage('Add E2E User') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
E2E_FAILED = ""
cloneProvisioningRepo("e2e")
addKeycloakGroups(env.EA_BEZEICHNER, "e2e")
addKeycloakUser(env.EA_BEZEICHNER, "e2e")
addKeycloakGroups(env.MAIN_BEZEICHNER, "e2e")
addKeycloakUser(env.MAIN_BEZEICHNER, "e2e")
}
}
}
stage('Run E2E-Tests') {
failFast false
......@@ -240,16 +280,13 @@ pipeline {
steps {
script {
def stageName = env.STAGE_NAME
def bezeichner = generateBezeichner(stageName)
def bezeichner = env.EA_BEZEICHNER
def dbPort = 27018
startEnvironment(bezeichner, stageName, IMAGE_TAG, true, HELM_CHART_VERSION, dbPort)
exposeDatenbank("${env.BUNDESLAND}-${bezeichner}-dev", dbPort)
exposeElasticSearch()
def testResult = runTests(stageName, bezeichner, 'einheitlicher-ansprechpartner', dbPort)
if (env.BRANCH_NAME != 'master') {
deleteKopStack(bezeichner, stageName)
}
def testResult = runTests(bezeichner, 'einheitlicher-ansprechpartner', dbPort)
if(!testResult) {
E2E_FAILED += "${stageName}, "
......@@ -269,16 +306,13 @@ pipeline {
steps {
script {
def stageName = env.STAGE_NAME
def bezeichner = generateBezeichner(stageName)
def bezeichner = env.MAIN_BEZEICHNER
def dbPort = 27019
startEnvironment(bezeichner, stageName, IMAGE_TAG, false, HELM_CHART_VERSION, dbPort)
def testResult = runTests(stageName, bezeichner, 'main-tests', dbPort)
exposeDatenbank("${env.BUNDESLAND}-${bezeichner}-dev", dbPort)
exposeElasticSearch()
if (env.BRANCH_NAME != 'master') {
deleteKopStack(bezeichner, stageName)
}
def testResult = runTests(bezeichner, 'main-tests', dbPort)
if(!testResult) {
E2E_FAILED += "${stageName}, "
......@@ -296,6 +330,16 @@ pipeline {
}
}
}
stage('Delete E2E Namespaces') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
E2E_FAILED = ""
deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
}
}
}
}
post {
always {
......@@ -308,7 +352,7 @@ pipeline {
}
}
}
stage('Trigger Test | Stage rollout') {
stage('Trigger Test rollout') {
when {
branch 'release'
}
......@@ -317,12 +361,10 @@ pipeline {
script {
FAILED_STAGE = env.STAGE_NAME
checkoutProvisioningRepo()
setNewGoofyProvisioningVersion('test')
setNewGoofyProvisioningVersion('stage')
cloneGitopsRepo("rollout")
pushNewProvisioningVersion('test stage')
setNewGoofyGitopsVersion("rollout", "test")
pushGitopsRepo("rollout")
}
}
}
......@@ -385,156 +427,71 @@ String generateImageTag() {
return imageTag
}
Void startEnvironment(String bezeichner, String stage, String imageTag, Boolean isEa, String chartVersion, Integer dbPort) {
setupAnsible(imageTag, stage, isEa, chartVersion)
try {
deleteKopStack(bezeichner, stage)
} catch (Exception e) {
echo "deleteKopStack Exception"
}
rolloutKopStack(bezeichner, stage)
addKeycloakGroups(bezeichner, stage)
addKeycloakUser(bezeichner, stage)
exposeDatenbank("${env.BUNDESLAND}-${bezeichner}-dev", dbPort)
exposeElasticSearch()
}
Void setupAnsible(String imageTag, String stage, Boolean isEa, String chartVersion) {
checkoutProvisioningRepo(stage)
if (env.BRANCH_NAME == 'release') {
copyTestEnvironmentToDev(stage)
Void cloneGitopsRepo(String directory) {
withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) {
dir(directory) {
sh 'git clone https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/gitops.git'
}
editProvisioningBundesland(stage)
editEnvironemntVersion(stage, imageTag, isEa, chartVersion)
if (isEa) {
setupEaEnvironment(stage)
}
setPlutoDatabasePassword(stage)
configGit("${directory}/gitops")
}
Void setAnsibleKubeConfig() {
configFileProvider([configFile(fileId: 'kubeconfig-dev-cluster', variable: 'KUBE_CONFIG')]) {
sh 'mkdir ~/.kube'
sh 'cp ${KUBE_CONFIG} ~/.kube/config'
}
}
Void checkoutProvisioningRepo(String stage="") {
Void cloneProvisioningRepo(String directory) {
withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) {
dir(stage) {
dir(directory) {
sh 'git clone https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/provisioning.git'
if (env.BRANCH_NAME == 'release') {
dir('provisioning') {
sh 'git checkout release'
}
}
}
}
}
Void copyTestEnvironmentToDev(stage) {
dir("${stage}/provisioning") {
def devEnvFile = "inventories/group_vars/dev/versions"
def testEnvFile = "inventories/group_vars/test/versions"
def devVersions = readYaml file: devEnvFile
def testVersions = readYaml file: testEnvFile
devVersions.charts = testVersions.charts
devVersions.versions = testVersions.versions
writeYaml file: devEnvFile, data: devVersions, overwrite: true
Void pushGitopsRepo(String directory) {
withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) {
dir("${directory}/gitops") {
sh 'git push https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/gitops.git'
}
}
Void editEnvironemntVersion(String stage, String imageTag, Boolean isEa, String chartVersion) {
dir("${stage}/provisioning") {
def editFile = "inventories/group_vars/dev/versions"
def devVersions = readYaml file: editFile
overrideSpringProfiles = getSpringProfile(isEa)
devVersions.values.goofy.put('env', ['overrideSpringProfiles': overrideSpringProfiles])
devVersions.values.pluto.put('env', ['overrideSpringProfiles': overrideSpringProfiles])
devVersions.values.goofy.put('ingress', ['use_staging_cert': true])
devVersions.versions.goofy.image.tag = imageTag
devVersions.charts.goofy.version = chartVersion
writeYaml file: editFile, data: devVersions, overwrite: true
}
}
Void editProvisioningBundesland(String stage) {
dir("${stage}/provisioning") {
def editEnvFile = "inventories/group_vars/all/env"
def envVersions = readYaml file: editEnvFile
envVersions.kop_bundesland = env.BUNDESLAND
writeYaml file: editEnvFile, data: envVersions, overwrite: true
Void configGit(String directory) {
dir(directory) {
sh 'git config user.email "jenkins@ozg-sh.de"'
sh 'git config user.name "jenkins"'
}
}
String getSpringProfile(Boolean isEa) {
if (isEa) {
return "oc,ea,e2e,dev"
Void checkoutE2eBranch(String directory) {
dir("${directory}/gitops") {
sh 'git checkout e2e'
}
return "oc,e2e,dev"
}
Void setupEaEnvironment(String stage) {
dir("${stage}/provisioning") {
def editFile = "inventories/group_vars/all/env"
def groupVars = readYaml file: editFile
Void generateNamespaceYaml(String bezeichner, Boolean isEa) {
def envValues = readYaml file: valuesPath(isEa)
groupVars.kop_einheitlicher_ansprechpartner = true
envValues.kop.bezeichner = bezeichner
envValues.goofy.image.tag = IMAGE_TAG
envValues.goofy.helm.version = HELM_CHART_VERSION
writeYaml file: editFile, data: groupVars, overwrite: true
}
if (env.BRANCH_NAME == 'release') {
envValues.put("argocd", ['source': ['path': 'test/application']])
}
Void setPlutoDatabasePassword(String stage) {
dir("${stage}/provisioning") {
def editFile = "inventories/group_vars/dev/versions"
def devVars = readYaml file: editFile
writeYaml file: "e2e/gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml", data: envValues, overwrite: true
devVars.values.pluto.database.password = "XnHhfznNWg65NNd"
sh "cat e2e/gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml"
writeYaml file: editFile, data: devVars, overwrite: true
}
sh "git add e2e/gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml"
sh "git commit -m 'add e2e by-${bezeichner}-dev'"
}
Void rolloutKopStack(String bezeichner, String stage) {
dir("${stage}/provisioning") {
def ansibleVars = """{"k8s_context":"ozg-dev", \
"kop_env":"dev", \
"kop_bezeichner":${bezeichner}, \
"kop_displayname":${bezeichner}, \
"kop_postfach_api_key":"", \
"install_afm_adapter":false, \
"install_fs_adapter":false, \
"external_db_enabled":false, \
"disable_update":true}"""
String valuesPath(Boolean isEa) {
def path = "goofy-client/apps/goofy-e2e/src/fixtures/argocd"
if (env.BRANCH_NAME == 'release') {
sh "ansible-playbook playbook/rollout.yml --extra-vars '${ansibleVars}'"
}
else {
sh "ansible-playbook playbooks/rollout.yml --extra-vars '${ansibleVars}'"
}
if (isEa) {
return "${path}/by-ea-dev.yaml"
}
return "${path}/by-main-dev.yaml"
}
Void addKeycloakGroups(String bezeichner, String stage) {
......@@ -583,18 +540,64 @@ Void addKeycloakUser(String bezeichner, String stage) {
}
}
Void deleteKopStack(String bezeichner, String stage) {
dir("${stage}/provisioning") {
def ansibleVars = """{"k8s_context":"ozg-dev", \
"kop_env":"dev", \
"kop_bezeichner":${bezeichner}}"""
Void deleteKopStack(Map kopBezeichner) {
for(bezeichner in kopBezeichner) {
dir("e2e/gitops/dev/namespace/namespaces") {
if (sh (script: "ls | grep 'by-${bezeichner}-dev.yaml'", returnStatus: true) == 1) {
return
}
if (env.BRANCH_NAME == 'release') {
sh "ansible-playbook playbook/delete-commune.yml --extra-vars '${ansibleVars}'"
sh "rm by-${bezeichner}-dev.yaml"
sh "git add by-${bezeichner}-dev.yaml"
sh "git commit -m 'delete e2e by-${bezeichner}-dev.yaml'"
}
else {
sh "ansible-playbook playbooks/delete-commune.yml --extra-vars '${ansibleVars}'"
}
pushGitopsRepo("e2e")
for(bezeichner in kopBezeichner) {
waitForDeletion(bezeichner)
}
}
Void waitForDeletion(String bezeichner) {
try {
sh "kubectl wait --for=delete applications/by-${bezeichner}-dev-application -n argocd --timeout=300s"
} catch (Exception e) {
error("Application by-${bezeichner}-dev-application konnte nicht gelöscht werden")
}
}
Void rolloutKopStack(Map kopBezeichner) {
pushGitopsRepo("e2e")
for(bezeichner in kopBezeichner) {
waitForHealthyApplication(bezeichner)
}
}
Void waitForRollout(String bezeichner) {
waitForHealthyApplication(bezeichner, 'application')
waitForHealthyApplication(bezeichner, 'user-manager')
waitForHealthyApplication(bezeichner, 'pluto')
waitForHealthyApplication(bezeichner, 'goofy')
}
Void waitForHealthyApplication(String bezeichner, Strin application) {
try {
def countRetry = 0
def maxRetry = 12
while (sh (script: "kubectl get applications -n argocd | grep 'by-${bezeichner}-dev-${application}'", returnStatus: true) == 1 && countRetry < maxRetry ) {
countRetry++
sh "sleep 5"
}
if (sh (script: "kubectl get application/by-${bezeichner}-dev-${application} -n argocd -o=jsonpath='{.status.health.status}' | grep Healthy", returnStatus: true) == 0) {
sh "kubectl wait --for=jsonpath='{.status.health.status}'=Healthy applications/by-${bezeichner}-dev-${application} -n argocd --timeout=300s"
}
} catch (Exception e) {
error("Application ${application} unhealthy")
}
}
......@@ -611,8 +614,8 @@ Void publishE2ETestResult(String reportFolder, String reportName) {
)
}
String runTests(String stageName, String bezeichner, String reportFolder, Integer dbPort) {
def configFile = generateCypressConfig(stageName, bezeichner, reportFolder, dbPort)
String runTests(String bezeichner, String reportFolder, Integer dbPort) {
def configFile = generateCypressConfig(bezeichner, reportFolder, dbPort)
try {
dir("goofy-client") {
......@@ -651,7 +654,7 @@ String cutBranchNameForKeycloakRealm(String branchName, String stageName) {
return branchName
}
String generateCypressConfig(String stage, String bezeichner, String testFolder, Integer dbPort) {
String generateCypressConfig(String bezeichner, String testFolder, Integer dbPort) {
def namespace = "${env.BUNDESLAND}-${bezeichner}-dev"
def configName = "cypress-ci-"+testFolder+".json"
......@@ -673,7 +676,7 @@ String generateCypressConfig(String stage, String bezeichner, String testFolder,
environment = config.env
environment.put("search", elasticsearchEnv)
environment.put("userManager", getUserManagerEnv(namespace, dbPort));
environment.put("userManager", getUserManagerEnv(dbPort));
writeJSON file: configName, json: config
......@@ -683,21 +686,12 @@ String generateCypressConfig(String stage, String bezeichner, String testFolder,
return configName
}
String getUserManagerEnv(namespace, dbPort){
def secret = getSecrect(namespace, 'pluto-database-admin-user-manager-database-user');
def secretPassword = decodeString(secret.password);
String getUserManagerEnv(dbPort){
return readJSON(text: """{
"dbUrl":"mongodb://user-manager-database-user:${secretPassword}@localhost:${dbPort}/admin?ssl=false&directConnection=true", \
"dbUrl":"mongodb://user-manager-database-user:5M3N2sVEq5c8@localhost:${dbPort}/admin?ssl=false&directConnection=true", \
"database":"user-manager-database"}""");
}
def getSecrect(namespace, secretName){
script {
return readJSON ( text: sh (script: "kubectl get secret ${secretName} -n ${namespace} -o jsonpath={.data}", returnStdout: true))
}
}
String getKeycloakUuid(realm, userName) {
def shScript = """curl -H 'Content-Type: application/json' \
-H 'Authorization: bearer ${getKeycloakAccessToken()}' \
......@@ -744,57 +738,29 @@ String getElementAccessToken() {
}
}
def getElasticsearchSecret(namespace) {
script {
def elasticsearch = readJSON ( text: sh (script: "kubectl get secret elasticsearch-credentials -n ${namespace} -o jsonpath={.data}", returnStdout: true))
return elasticsearch
}
}
String getElasticsearchEnv(namespace) {
def elasticsearchSecret = getElasticsearchSecret(namespace)
def env = """{
"user":"${decodeString(elasticsearchSecret.username)}", \
"password":"${decodeString(elasticsearchSecret.password)}", \
"index":"${decodeString(elasticsearchSecret.index)}", \
"user":"${namespace}", \
"password":"vf9W1D8Z3673", \
"index":"${namespace}", \
"url":"https://localhost:9200"}"""
return readJSON ( text: env)
}
String decodeString(encoded) {
return sh (script: "echo -n ${encoded} | base64 --decode", returnStdout: true)
}
Void setNewGoofyProvisioningVersion(String environment) {
dir("provisioning") {
def envFile = "inventories/group_vars/${environment}/versions"
Void setNewGoofyGitopsVersion(String directory, String environment) {
dir("${directory}/gitops") {
def envFile = "${environment}/application/values/goofy-values.yaml"
def envVersions = readYaml file: envFile
envVersions.versions.goofy.image.tag = IMAGE_TAG
envVersions.charts.goofy.version = HELM_CHART_VERSION
envVersions.goofy.image.tag = IMAGE_TAG
envVersions.goofy.helm.version = HELM_CHART_VERSION
writeYaml file: envFile, data: envVersions, overwrite: true
}
}
Void pushNewProvisioningVersion(String environment) {
dir('provisioning') {
if (sh (script: "git status | grep 'inventories/group_vars/.*/versions'", returnStatus: true) == 1) {
return
}
withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) {
sh 'git add inventories/group_vars/*/versions'
sh 'git config user.email "jenkins@ozg.de"'
sh 'git config user.name "jenkins"'
sh "git add ${environment}/application/values/goofy-values.yaml"
sh "git commit -m 'jenkins rollout ${environment} goofy version ${IMAGE_TAG}'"
sh 'git push https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/provisioning.git'
}
}
}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment

Impressum - Datenschutz - Barrierefreiheit