Skip to content
Snippets Groups Projects
Commit ae63a515 authored by OZGCloud's avatar OZGCloud
Browse files

jenkinsfile e2e tests

parent e7107e4e
Branches
Tags
No related merge requests found
import groovy.json.JsonOutput
pipeline {
agent {
node {
......@@ -9,16 +11,40 @@ pipeline {
upstream(upstreamProjects: getUpstreamProjects(), threshold: hudson.model.Result.SUCCESS)
}
environment {
BLUE_OCEAN_URL = "https://jenkins.ozg-sh.de/job/E2E%20Tests/job/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/"
BUNDESLAND = "by"
SSO_URL = "sso.dev.by.ozg-cloud.de"
CLUSTER_BASE_URL = "dev.by.ozg-cloud.de"
FAILED_STAGE = ""
FAILED_PARALLEL_STAGE = " "
EA_BEZEICHNER = generateBezeichner("e2e-ea")
MAIN_BEZEICHNER = generateBezeichner("e2e-main")
SH_SUCCESS_STATUS_CODE = 0
}
options {
timeout(time: 1, unit: 'HOURS')
disableConcurrentBuilds()
buildDiscarder(logRotator(numToKeepStr: '5'))
}
stages {
stage("Clone Gitops Repo") {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
cloneGitopsRepo()
}
}
}
stage("Init Default Versions") {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
initEnvGoofyDefaultVersions()
initEnvPlutoDefaultVersions()
initEnvUserManagerDefaultVersions()
......@@ -33,6 +59,8 @@ pipeline {
}
steps {
script {
FAILED_STAGE = env.STAGE_NAME
userVersions = input message: "Edit Default Values",
parameters: [
string(name: "GoofyImageTag", defaultValue: env.GOOFY_IMAGE_TAG, trim: true),
......@@ -48,15 +76,140 @@ pipeline {
}
}
stage("Print Versions") {
stage("Install Cypress") {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
sh 'npm --version'
dir('goofy-client') {
sh 'echo "registry=https://nexus.ozg-sh.de/repository/npm-proxy" >> ~/.npmrc'
sh 'echo "//nexus.ozg-sh.de/:_auth=amVua2luczpQaihzX0ZNNFU5ZC8=" >> ~/.npmrc'
sh 'npm cache verify'
sh 'npm install'
sh "npm run cypress:install"
}
}
}
}
stage('Init k8s') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
configFileProvider([configFile(fileId: 'kubeconfig-ovh-cluster', variable: 'KUBE_CONFIG')]) {
sh 'mkdir ~/.kube'
sh 'cp ${KUBE_CONFIG} ~/.kube/config'
}
sh 'helm version'
}
}
}
stage('Rollout E2E Namespaces') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
checkoutGitopsE2eBranch()
deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
generateEaNamespaceYaml()
generateMainNamespaceYaml()
pushGitopsRepo()
waitForKopStackRollout([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
}
}
post {
failure {
script {
deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
}
}
}
}
stage('Run E2E-Tests') {
failFast false
parallel {
stage('E2E-EA') {
steps {
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
script {
def bezeichner = env.EA_BEZEICHNER
def dbPort = 27018
forwardServices(generateNamespace(bezeichner), dbPort)
runTests(bezeichner, 'einheitlicher-ansprechpartner', dbPort, env.STAGE_NAME)
}
}
}
post {
failure {
script {
println "GOOFY_IMAGE_TAG: ${env.GOOFY_IMAGE_TAG}"
println "GOOFY_HELM_CHART_VERSION: ${env.GOOFY_HELM_CHART_VERSION}"
println "PLUTO_IMAGE_TAG: ${env.PLUTO_IMAGE_TAG}"
println "PLUTO_HELM_CHART_VERSION: ${env.PLUTO_HELM_CHART_VERSION}"
println "USER_MANAGER_IMAGE_TAG: ${env.USER_MANAGER_IMAGE_TAG}"
println "USER_MANAGER_HELM_CHART_VERSION: ${env.USER_MANAGER_HELM_CHART_VERSION}"
FAILED_PARALLEL_STAGE += "${env.STAGE_NAME} "
}
}
always {
script {
publishE2ETestResult("einheitlicher-ansprechpartner", "Goofy E2E-Tests EA")
}
}
}
}
stage('E2E-Main') {
steps {
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
script {
def bezeichner = env.MAIN_BEZEICHNER
def dbPort = 27019
forwardServices(generateNamespace(bezeichner), dbPort)
runTests(bezeichner, 'main-tests', dbPort, env.STAGE_NAME)
}
}
}
post {
failure {
script {
FAILED_PARALLEL_STAGE += "${env.STAGE_NAME} "
}
}
always {
script {
publishE2ETestResult("main-tests", "Goofy E2E-Tests main")
}
}
}
}
}
}
stage('Delete E2E Namespaces') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
}
}
}
}
post {
failure {
script {
if (isMasterBranch() || isReleaseBranch()) {
sendFailureMessage()
}
}
}
......@@ -68,7 +221,7 @@ String getUpstreamProjects() {
return "goofy/${env.BRANCH_NAME},pluto/${env.BRANCH_NAME},user-manager/${env.BRANCH_NAME}"
}
return "goofy/${env.BRANCH_NAME}"
return ""
}
Boolean isReleaseBranch() {
......@@ -80,8 +233,16 @@ Boolean isMasterBranch() {
}
def cloneGitopsRepo() {
final email = "jenkins@ozg-sh.de"
final name = "jenkins"
withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) {
sh 'git clone https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/gitops.git'
dir("gitops") {
sh "git config user.email '${email}'"
sh "git config user.name '${name}'"
}
}
}
......@@ -108,11 +269,31 @@ String getHelmChartVersion(Map applicationValues) {
}
Void initEnvGoofyDefaultVersions() {
if (isMasterBranch() || isReleaseBranch()) {
goofyValues = getApplicationValues("goofy")
env.GOOFY_IMAGE_TAG = getImageTag(goofyValues)
env.GOOFY_HELM_CHART_VERSION = getHelmChartVersion(goofyValues)
}
else {
env.GOOFY_IMAGE_TAG = getFeatureBranchImageTag()
env.GOOFY_HELM_CHART_VERSION = getFeatureBranchHelmChartVersion()
}
}
String getFeatureBranchImageTag() {
return "${env.BRANCH_NAME}-${getRootPomVersion()}"
}
String getFeatureBranchHelmChartVersion() {
return "${getRootPomVersion()}-${env.BRANCH_NAME}".replaceAll("_", "-")
}
String getRootPomVersion() {
def rootPom = readMavenPom file: 'pom.xml'
return rootPom.version
}
Void initEnvPlutoDefaultVersions() {
plutoValues = getApplicationValues("pluto")
......@@ -136,3 +317,402 @@ Void initEnvUserVersions(userVersions) {
env.USER_MANAGER_IMAGE_TAG = userVersions.UserManagerImageTag
env.USER_MANAGER_HELM_CHART_VERSION = userVersions.UserManagerHelmChartVersion
}
Void pushGitopsRepo() {
withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) {
dir("gitops") {
if (hasUnpushedCommits()) {
sh 'git push https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/gitops.git'
}
}
}
}
Boolean hasUnpushedCommits() {
return sh (script: "git cherry -v | grep .", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer
}
Void checkoutGitopsE2eBranch() {
dir("gitops") {
sh 'git checkout e2e'
}
}
Void generateEaNamespaceYaml() {
generateNamespaceYaml(env.EA_BEZEICHNER, "goofy-client/apps/goofy-e2e/src/fixtures/argocd/by-ea-dev.yaml")
}
Void generateMainNamespaceYaml() {
generateNamespaceYaml(env.MAIN_BEZEICHNER, "goofy-client/apps/goofy-e2e/src/fixtures/argocd/by-main-dev.yaml")
}
Void generateNamespaceYaml(String bezeichner, String valuesPath) {
def envValues = readYaml file: valuesPath
envValues.kop.bezeichner = bezeichner
envValues.goofy.put("image", ['tag': env.GOOFY_IMAGE_TAG])
envValues.goofy.put("helm", ['version': env.GOOFY_HELM_CHART_VERSION])
envValues.pluto.put("image", ['tag': env.PLUTO_IMAGE_TAG])
envValues.pluto.put("helm", ['version': env.PLUTO_HELM_CHART_VERSION])
envValues.user_manager.put("image", ['tag': env.USER_MANAGER_IMAGE_TAG])
envValues.user_manager.put("helm", ['version': env.USER_MANAGER_HELM_CHART_VERSION])
envValues.goofy.sso.put("keycloak_groups", generateKeycloakGroupsForHelmChart())
envValues.goofy.sso.put("keycloak_users", generateKeycloakUserForHelmChart())
writeYaml file: "gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml", data: envValues, overwrite: true
sh "cat gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml"
dir("gitops") {
sh "git add dev/namespace/namespaces/by-${bezeichner}-dev.yaml"
sh "git commit -m 'add e2e by-${bezeichner}-dev'"
}
}
List generateKeycloakUserForHelmChart() {
def userFiles = sh (script: 'ls goofy-client/apps/goofy-e2e/src/fixtures/user', returnStdout: true)
def helmUsers = []
userFiles.split("\\n").each { userFile ->
def userJson = readJSON file: "goofy-client/apps/goofy-e2e/src/fixtures/user/${userFile}"
def user = [
"name": userJson.name,
"password": userJson.password,
"first_name": userJson.get("firstName", ""),
"last_name": userJson.get("lastName", "")
]
if (userJson.containsKey("clientRoles")) {
user.put("client_roles", mapUserClientRoles(userJson.clientRoles))
}
if (userJson.containsKey("groups")) {
user.put("groups", userJson.groups)
}
helmUsers.add(user)
}
return helmUsers
}
List mapUserClientRoles(userClientRoles) {
def clientRoles = []
for(clientRole in userClientRoles) {
clientRoles.add(['name': "alfa", 'role': clientRole])
}
return clientRoles
}
List generateKeycloakGroupsForHelmChart() {
def groupFiles = sh (script: 'ls goofy-client/apps/goofy-e2e/src/fixtures/group', returnStdout: true)
def helmGroups = []
groupFiles.split("\\n").each { groupFile ->
def groupJson = readJSON file: "goofy-client/apps/goofy-e2e/src/fixtures/group/${groupFile}"
def group = ["name": groupJson.name]
groupJson.attributes.each { key, values ->
if (!group.containsKey("attributes")) {
group.put("attributes", [["name": key, "value": values]])
}
else {
group.attributes.add(["name": key, "value": values])
}
}
helmGroups.add(group)
}
return helmGroups
}
Void deleteKopStack(kopBezeichner) {
for(bezeichner in kopBezeichner) {
if (hasNamespaceFile(bezeichner)) {
removeNamespaceFile(bezeichner)
}
}
pushGitopsRepo()
for(bezeichner in kopBezeichner) {
waitForDeletion(bezeichner)
}
}
Void removeNamespaceFile(String bezeichner) {
dir("gitops/dev/namespace/namespaces") {
sh "rm by-${bezeichner}-dev.yaml"
sh "git add by-${bezeichner}-dev.yaml"
sh "git commit -m 'delete e2e by-${bezeichner}-dev.yaml'"
}
}
Boolean hasNamespaceFile(String bezeichner) {
return sh (script: "ls gitops/dev/namespace/namespaces | grep 'by-${bezeichner}-dev.yaml'", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer
}
Void waitForDeletion(String bezeichner) {
try {
sh "kubectl wait --for=delete applications/by-${bezeichner}-dev-application -n argocd --timeout=300s"
} catch (Exception e) {
error("Application by-${bezeichner}-dev-application konnte nicht gelöscht werden")
}
}
Void waitForKopStackRollout(kopBezeichner) {
for(bezeichner in kopBezeichner) {
waitForRollout(bezeichner)
}
}
Void waitForRollout(String bezeichner) {
waitForHealthyApplication(bezeichner, 'application')
waitForHealthyApplication(bezeichner, 'user-manager')
waitForHealthyApplication(bezeichner, 'pluto')
waitForHealthyApplication(bezeichner, 'goofy')
}
Void waitForHealthyApplication(String bezeichner, String application) {
try {
def countRetry = 0
def maxRetry = 12
while (!isApplicationPresent(bezeichner, application) && countRetry < maxRetry ) {
countRetry++
sh "sleep 5"
}
if (!isApplicationHealthy(bezeichner, application)) {
waitForHealthyStatus(bezeichner, application)
}
} catch (Exception e) {
error("Application ${application} unhealthy")
}
}
Boolean isApplicationPresent(String bezeichner, String application) {
return sh (script: "kubectl get applications -n argocd | grep 'by-${bezeichner}-dev-${application}'", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer
}
Boolean isApplicationHealthy(String bezeichner, String application) {
return sh (script: "kubectl get application/by-${bezeichner}-dev-${application} -n argocd -o=jsonpath='{.status.health.status}' | grep Healthy", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer
}
Void waitForHealthyStatus(String bezeichner, String application) {
sh "kubectl wait --for=jsonpath='{.status.health.status}'=Healthy applications/by-${bezeichner}-dev-${application} -n argocd --timeout=300s"
}
Void publishE2ETestResult(String reportFolder, String reportName) {
publishHTML (
target: [
allowMissing: false,
alwaysLinkToLastBuild: false,
keepAll: true,
reportDir: "goofy-client/apps/goofy-e2e/reports/${reportFolder}",
reportFiles: 'report.html',
reportName: reportName
]
)
}
String runTests(String bezeichner, String reportFolder, Integer dbPort, String stageName) {
def configFile = generateCypressConfig(bezeichner, reportFolder, dbPort)
try {
dir("goofy-client") {
sh "npm run cypress:version"
sh "npm run cypress:ci-run --CONFIG_FILE=${configFile} --REPORT_FOLDER=${reportFolder}"
}
} catch (Exception e) {
printNpmDebugLog()
error("Fehler in Stage ${stageName}")
}
}
Void printNpmDebugLog() {
if (hasNpmDebugLog()) {
sh "cat /root/.npm/_logs/*-debug.log"
}
else {
echo "Npm debug log not found"
}
}
String makeUrlConform(String input) {
return input.replaceAll(/[^a-zA-Z0-9]+/, "").toLowerCase()
}
String generateBezeichner(String stage) {
def branchName = makeUrlConform(env.BRANCH_NAME)
def stageName = makeUrlConform(stage)
return "${cutBranchNameForKeycloakRealm(branchName, stageName)}-${stageName}"
}
String cutBranchNameForKeycloakRealm(String branchName, String stageName) {
final maxKeycloakRealmLength = 30
def cutBranchNamePosition = maxKeycloakRealmLength - (stageName.length() + "${env.BUNDESLAND}---dev".length())
return branchName.take(cutBranchNamePosition)
}
String generateCypressConfig(String bezeichner, String testFolder, Integer dbPort) {
def namespace = generateNamespace(bezeichner)
def configName = "cypress-ci-"+testFolder+".json"
dir('goofy-client/apps/goofy-e2e/'){
def config = readJSON file: 'cypress-ci.json'
config.baseUrl = "https://${bezeichner}.${env.CLUSTER_BASE_URL}" as String
config.env.dbUrl = "mongodb://pluto-database-user:XnHhfznNWg65NNd@localhost:${dbPort}/admin?ssl=false&directConnection=true" as String
config.env.keycloakUrl = "https://${env.SSO_URL}/" as String
config.env.keycloakRealm = namespace as String
config.env.keycloakClient = "alfa" as String
config.env.sabineUuid = getKeycloakUuid(namespace, "sabine") as String
config.integrationFolder = "./src/integration/${testFolder}" as String
config.videosFolder = "./reports/${testFolder}/videos" as String
config.screenshotsFolder = "./reports/${testFolder}/screenshots" as String
config.reporterOptions.reportDir = "./reports/${testFolder}/mochawesome-report" as String
config.env.put("search", getElasticsearchEnv(namespace))
config.env.put("userManager", getUserManagerEnv(dbPort))
writeJSON file: configName, json: config
sh "cat ${configName}"
}
return configName
}
Map getUserManagerEnv(dbPort){
return [
"dbUrl": "mongodb://user-manager-database-user:5M3N2sVEq5c8@localhost:${dbPort}/admin?ssl=false&directConnection=true" as String,
"database": "user-manager-database"
]
}
String getKeycloakUuid(realm, userName) {
def shScript = """curl -H 'Content-Type: application/json' \
-H 'Authorization: bearer ${getKeycloakAccessToken()}' \
'https://${env.SSO_URL}/admin/realms/${realm}/users'
"""
def users = readJSON text: sh(script: shScript, returnStdout: true)
echo "users: ${users}"
for(user in users) {
if (user.username == userName) {
return user.id
}
}
}
String getKeycloakAccessToken() {
withCredentials([usernamePassword(credentialsId: 'keycloak-ovh-cluster', usernameVariable: 'USER', passwordVariable: 'PASSWORD')]) {
def token = readJSON text: sh (script: 'curl -d "client_id=admin-cli" -d "username=$USER" -d "password=$PASSWORD" -d "grant_type=password" https://$SSO_URL/realms/master/protocol/openid-connect/token', returnStdout: true)
return token.access_token
}
}
Void sendFailureMessage() {
def data = [
"msgtype": "m.text",
"body": "E2E-Tests: Failed stage: ${getFailedStage()} Build-ID: ${env.BUILD_NUMBER} Link: ${BLUE_OCEAN_URL}" as String,
"format": "org.matrix.custom.html",
"formatted_body": "E2E-Tests: Failed stage: ${getFailedStage()} Build-ID: <a href='${BLUE_OCEAN_URL}'>${env.BUILD_NUMBER}</a>" as String
]
sh "curl -XPOST -H 'authorization: Bearer ${getElementAccessToken()}' -d '${JsonOutput.toJson(data)}' https://matrix.ozg-sh.de/_matrix/client/v3/rooms/${getElementRoomId()}/send/m.room.message"
}
String getFailedStage() {
if (FAILED_PARALLEL_STAGE.trim()) {
return FAILED_PARALLEL_STAGE
}
return FAILED_STAGE
}
String getElementRoomId() {
final releaseRoomId = "!oWZpUGTFsxkJIYNfYg:matrix.ozg-sh.de"
final masterRoomId = "!iQPAvQIiRwRpNOszjw:matrix.ozg-sh.de"
if (isReleaseBranch()) {
return releaseRoomId
}
return masterRoomId
}
String getElementAccessToken() {
withCredentials([string(credentialsId: 'element-login-json', variable: 'LOGIN_JSON')]) {
return readJSON ( text: sh (script: '''curl -XPOST -d \"$LOGIN_JSON\" https://matrix.ozg-sh.de/_matrix/client/v3/login''', returnStdout: true)).access_token
}
}
Map getElasticsearchEnv(String namespace) {
def elasticsearchSecret = getElasticsearchSecret(namespace)
return [
"user": decodeString(elasticsearchSecret.username),
"password": decodeString(elasticsearchSecret.password),
"index": decodeString(elasticsearchSecret.index),
"url": "https://localhost:9200"
]
}
Void forwardServices(String namespace, Integer dbPort) {
try {
forwardDatenbank(namespace, dbPort)
forwardElasticSearch()
}
catch (Exception e) {
echo "forwardServices Exception: ${e}"
error("Error forwarding service")
}
}
Void forwardElasticSearch() {
if(!isElasticSearchForwarded()) {
sh "kubectl port-forward ozg-search-cluster-es-ozg-search-0 9200:9200 -n elastic-system &"
}
}
Boolean isElasticSearchForwarded() {
return sh (script: "lsof -i -P -n | grep LISTEN | grep :9200", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer
}
Void forwardDatenbank(String namespace, port) {
sh "kubectl port-forward pluto-database-0 ${port}:27017 -n ${namespace} &"
}
String generateNamespace(String bezeichner) {
return "${env.BUNDESLAND}-${bezeichner}-dev"
}
String decodeString(String encoded) {
return sh (script: "echo -n ${encoded} | base64 --decode", returnStdout: true)
}
Map getElasticsearchSecret(String namespace) {
return readJSON ( text: sh (script: "kubectl get secret elasticsearch-credentials -n ${namespace} -o jsonpath={.data}", returnStdout: true))
}
Boolean hasNpmDebugLog() {
return sh (script: "ls -l /root/.npm/_logs/*-debug.log", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer
}
\ No newline at end of file
......@@ -8,7 +8,10 @@ goofy:
sso:
serverUrl: https://sso.dev.by.ozg-cloud.de
apiPassword: "Test1234!"
role_einheitlicher_ansprechpartner: true
keycloak_clients:
- client_name: alfa
client_roles:
- name: EINHEITLICHER_ANSPRECHPARTNER
ingress:
use_staging_cert: true
baseUrl: dev.by.ozg-cloud.de
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment