Skip to content
Snippets Groups Projects
Commit ae63a515 authored by OZGCloud's avatar OZGCloud
Browse files

jenkinsfile e2e tests

parent e7107e4e
No related branches found
No related tags found
No related merge requests found
import groovy.json.JsonOutput
pipeline { pipeline {
agent { agent {
node { node {
...@@ -9,16 +11,40 @@ pipeline { ...@@ -9,16 +11,40 @@ pipeline {
upstream(upstreamProjects: getUpstreamProjects(), threshold: hudson.model.Result.SUCCESS) upstream(upstreamProjects: getUpstreamProjects(), threshold: hudson.model.Result.SUCCESS)
} }
environment {
BLUE_OCEAN_URL = "https://jenkins.ozg-sh.de/job/E2E%20Tests/job/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/"
BUNDESLAND = "by"
SSO_URL = "sso.dev.by.ozg-cloud.de"
CLUSTER_BASE_URL = "dev.by.ozg-cloud.de"
FAILED_STAGE = ""
FAILED_PARALLEL_STAGE = " "
EA_BEZEICHNER = generateBezeichner("e2e-ea")
MAIN_BEZEICHNER = generateBezeichner("e2e-main")
SH_SUCCESS_STATUS_CODE = 0
}
options {
timeout(time: 1, unit: 'HOURS')
disableConcurrentBuilds()
buildDiscarder(logRotator(numToKeepStr: '5'))
}
stages { stages {
stage("Clone Gitops Repo") { stage("Clone Gitops Repo") {
steps { steps {
script {
FAILED_STAGE = env.STAGE_NAME
cloneGitopsRepo() cloneGitopsRepo()
} }
} }
}
stage("Init Default Versions") { stage("Init Default Versions") {
steps { steps {
script { script {
FAILED_STAGE = env.STAGE_NAME
initEnvGoofyDefaultVersions() initEnvGoofyDefaultVersions()
initEnvPlutoDefaultVersions() initEnvPlutoDefaultVersions()
initEnvUserManagerDefaultVersions() initEnvUserManagerDefaultVersions()
...@@ -33,6 +59,8 @@ pipeline { ...@@ -33,6 +59,8 @@ pipeline {
} }
steps { steps {
script { script {
FAILED_STAGE = env.STAGE_NAME
userVersions = input message: "Edit Default Values", userVersions = input message: "Edit Default Values",
parameters: [ parameters: [
string(name: "GoofyImageTag", defaultValue: env.GOOFY_IMAGE_TAG, trim: true), string(name: "GoofyImageTag", defaultValue: env.GOOFY_IMAGE_TAG, trim: true),
...@@ -48,15 +76,140 @@ pipeline { ...@@ -48,15 +76,140 @@ pipeline {
} }
} }
stage("Print Versions") { stage("Install Cypress") {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
sh 'npm --version'
dir('goofy-client') {
sh 'echo "registry=https://nexus.ozg-sh.de/repository/npm-proxy" >> ~/.npmrc'
sh 'echo "//nexus.ozg-sh.de/:_auth=amVua2luczpQaihzX0ZNNFU5ZC8=" >> ~/.npmrc'
sh 'npm cache verify'
sh 'npm install'
sh "npm run cypress:install"
}
}
}
}
stage('Init k8s') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
configFileProvider([configFile(fileId: 'kubeconfig-ovh-cluster', variable: 'KUBE_CONFIG')]) {
sh 'mkdir ~/.kube'
sh 'cp ${KUBE_CONFIG} ~/.kube/config'
}
sh 'helm version'
}
}
}
stage('Rollout E2E Namespaces') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
checkoutGitopsE2eBranch()
deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
generateEaNamespaceYaml()
generateMainNamespaceYaml()
pushGitopsRepo()
waitForKopStackRollout([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
}
}
post {
failure {
script {
deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
}
}
}
}
stage('Run E2E-Tests') {
failFast false
parallel {
stage('E2E-EA') {
steps { steps {
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
script {
def bezeichner = env.EA_BEZEICHNER
def dbPort = 27018
forwardServices(generateNamespace(bezeichner), dbPort)
runTests(bezeichner, 'einheitlicher-ansprechpartner', dbPort, env.STAGE_NAME)
}
}
}
post {
failure {
script { script {
println "GOOFY_IMAGE_TAG: ${env.GOOFY_IMAGE_TAG}" FAILED_PARALLEL_STAGE += "${env.STAGE_NAME} "
println "GOOFY_HELM_CHART_VERSION: ${env.GOOFY_HELM_CHART_VERSION}" }
println "PLUTO_IMAGE_TAG: ${env.PLUTO_IMAGE_TAG}" }
println "PLUTO_HELM_CHART_VERSION: ${env.PLUTO_HELM_CHART_VERSION}" always {
println "USER_MANAGER_IMAGE_TAG: ${env.USER_MANAGER_IMAGE_TAG}" script {
println "USER_MANAGER_HELM_CHART_VERSION: ${env.USER_MANAGER_HELM_CHART_VERSION}" publishE2ETestResult("einheitlicher-ansprechpartner", "Goofy E2E-Tests EA")
}
}
}
}
stage('E2E-Main') {
steps {
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
script {
def bezeichner = env.MAIN_BEZEICHNER
def dbPort = 27019
forwardServices(generateNamespace(bezeichner), dbPort)
runTests(bezeichner, 'main-tests', dbPort, env.STAGE_NAME)
}
}
}
post {
failure {
script {
FAILED_PARALLEL_STAGE += "${env.STAGE_NAME} "
}
}
always {
script {
publishE2ETestResult("main-tests", "Goofy E2E-Tests main")
}
}
}
}
}
}
stage('Delete E2E Namespaces') {
steps {
script {
FAILED_STAGE = env.STAGE_NAME
deleteKopStack([env.EA_BEZEICHNER, env.MAIN_BEZEICHNER])
}
}
}
}
post {
failure {
script {
if (isMasterBranch() || isReleaseBranch()) {
sendFailureMessage()
} }
} }
} }
...@@ -68,7 +221,7 @@ String getUpstreamProjects() { ...@@ -68,7 +221,7 @@ String getUpstreamProjects() {
return "goofy/${env.BRANCH_NAME},pluto/${env.BRANCH_NAME},user-manager/${env.BRANCH_NAME}" return "goofy/${env.BRANCH_NAME},pluto/${env.BRANCH_NAME},user-manager/${env.BRANCH_NAME}"
} }
return "goofy/${env.BRANCH_NAME}" return ""
} }
Boolean isReleaseBranch() { Boolean isReleaseBranch() {
...@@ -80,8 +233,16 @@ Boolean isMasterBranch() { ...@@ -80,8 +233,16 @@ Boolean isMasterBranch() {
} }
def cloneGitopsRepo() { def cloneGitopsRepo() {
final email = "jenkins@ozg-sh.de"
final name = "jenkins"
withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) { withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) {
sh 'git clone https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/gitops.git' sh 'git clone https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/gitops.git'
dir("gitops") {
sh "git config user.email '${email}'"
sh "git config user.name '${name}'"
}
} }
} }
...@@ -108,11 +269,31 @@ String getHelmChartVersion(Map applicationValues) { ...@@ -108,11 +269,31 @@ String getHelmChartVersion(Map applicationValues) {
} }
Void initEnvGoofyDefaultVersions() { Void initEnvGoofyDefaultVersions() {
if (isMasterBranch() || isReleaseBranch()) {
goofyValues = getApplicationValues("goofy") goofyValues = getApplicationValues("goofy")
env.GOOFY_IMAGE_TAG = getImageTag(goofyValues) env.GOOFY_IMAGE_TAG = getImageTag(goofyValues)
env.GOOFY_HELM_CHART_VERSION = getHelmChartVersion(goofyValues) env.GOOFY_HELM_CHART_VERSION = getHelmChartVersion(goofyValues)
} }
else {
env.GOOFY_IMAGE_TAG = getFeatureBranchImageTag()
env.GOOFY_HELM_CHART_VERSION = getFeatureBranchHelmChartVersion()
}
}
String getFeatureBranchImageTag() {
return "${env.BRANCH_NAME}-${getRootPomVersion()}"
}
String getFeatureBranchHelmChartVersion() {
return "${getRootPomVersion()}-${env.BRANCH_NAME}".replaceAll("_", "-")
}
String getRootPomVersion() {
def rootPom = readMavenPom file: 'pom.xml'
return rootPom.version
}
Void initEnvPlutoDefaultVersions() { Void initEnvPlutoDefaultVersions() {
plutoValues = getApplicationValues("pluto") plutoValues = getApplicationValues("pluto")
...@@ -136,3 +317,402 @@ Void initEnvUserVersions(userVersions) { ...@@ -136,3 +317,402 @@ Void initEnvUserVersions(userVersions) {
env.USER_MANAGER_IMAGE_TAG = userVersions.UserManagerImageTag env.USER_MANAGER_IMAGE_TAG = userVersions.UserManagerImageTag
env.USER_MANAGER_HELM_CHART_VERSION = userVersions.UserManagerHelmChartVersion env.USER_MANAGER_HELM_CHART_VERSION = userVersions.UserManagerHelmChartVersion
} }
Void pushGitopsRepo() {
withCredentials([usernamePassword(credentialsId: 'jenkins-gitea-access-token', passwordVariable: 'TOKEN', usernameVariable: 'USER')]) {
dir("gitops") {
if (hasUnpushedCommits()) {
sh 'git push https://${USER}:${TOKEN}@git.ozg-sh.de/mgm/gitops.git'
}
}
}
}
Boolean hasUnpushedCommits() {
return sh (script: "git cherry -v | grep .", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer
}
Void checkoutGitopsE2eBranch() {
dir("gitops") {
sh 'git checkout e2e'
}
}
Void generateEaNamespaceYaml() {
generateNamespaceYaml(env.EA_BEZEICHNER, "goofy-client/apps/goofy-e2e/src/fixtures/argocd/by-ea-dev.yaml")
}
Void generateMainNamespaceYaml() {
generateNamespaceYaml(env.MAIN_BEZEICHNER, "goofy-client/apps/goofy-e2e/src/fixtures/argocd/by-main-dev.yaml")
}
Void generateNamespaceYaml(String bezeichner, String valuesPath) {
def envValues = readYaml file: valuesPath
envValues.kop.bezeichner = bezeichner
envValues.goofy.put("image", ['tag': env.GOOFY_IMAGE_TAG])
envValues.goofy.put("helm", ['version': env.GOOFY_HELM_CHART_VERSION])
envValues.pluto.put("image", ['tag': env.PLUTO_IMAGE_TAG])
envValues.pluto.put("helm", ['version': env.PLUTO_HELM_CHART_VERSION])
envValues.user_manager.put("image", ['tag': env.USER_MANAGER_IMAGE_TAG])
envValues.user_manager.put("helm", ['version': env.USER_MANAGER_HELM_CHART_VERSION])
envValues.goofy.sso.put("keycloak_groups", generateKeycloakGroupsForHelmChart())
envValues.goofy.sso.put("keycloak_users", generateKeycloakUserForHelmChart())
writeYaml file: "gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml", data: envValues, overwrite: true
sh "cat gitops/dev/namespace/namespaces/by-${bezeichner}-dev.yaml"
dir("gitops") {
sh "git add dev/namespace/namespaces/by-${bezeichner}-dev.yaml"
sh "git commit -m 'add e2e by-${bezeichner}-dev'"
}
}
List generateKeycloakUserForHelmChart() {
def userFiles = sh (script: 'ls goofy-client/apps/goofy-e2e/src/fixtures/user', returnStdout: true)
def helmUsers = []
userFiles.split("\\n").each { userFile ->
def userJson = readJSON file: "goofy-client/apps/goofy-e2e/src/fixtures/user/${userFile}"
def user = [
"name": userJson.name,
"password": userJson.password,
"first_name": userJson.get("firstName", ""),
"last_name": userJson.get("lastName", "")
]
if (userJson.containsKey("clientRoles")) {
user.put("client_roles", mapUserClientRoles(userJson.clientRoles))
}
if (userJson.containsKey("groups")) {
user.put("groups", userJson.groups)
}
helmUsers.add(user)
}
return helmUsers
}
List mapUserClientRoles(userClientRoles) {
def clientRoles = []
for(clientRole in userClientRoles) {
clientRoles.add(['name': "alfa", 'role': clientRole])
}
return clientRoles
}
List generateKeycloakGroupsForHelmChart() {
def groupFiles = sh (script: 'ls goofy-client/apps/goofy-e2e/src/fixtures/group', returnStdout: true)
def helmGroups = []
groupFiles.split("\\n").each { groupFile ->
def groupJson = readJSON file: "goofy-client/apps/goofy-e2e/src/fixtures/group/${groupFile}"
def group = ["name": groupJson.name]
groupJson.attributes.each { key, values ->
if (!group.containsKey("attributes")) {
group.put("attributes", [["name": key, "value": values]])
}
else {
group.attributes.add(["name": key, "value": values])
}
}
helmGroups.add(group)
}
return helmGroups
}
Void deleteKopStack(kopBezeichner) {
for(bezeichner in kopBezeichner) {
if (hasNamespaceFile(bezeichner)) {
removeNamespaceFile(bezeichner)
}
}
pushGitopsRepo()
for(bezeichner in kopBezeichner) {
waitForDeletion(bezeichner)
}
}
Void removeNamespaceFile(String bezeichner) {
dir("gitops/dev/namespace/namespaces") {
sh "rm by-${bezeichner}-dev.yaml"
sh "git add by-${bezeichner}-dev.yaml"
sh "git commit -m 'delete e2e by-${bezeichner}-dev.yaml'"
}
}
Boolean hasNamespaceFile(String bezeichner) {
return sh (script: "ls gitops/dev/namespace/namespaces | grep 'by-${bezeichner}-dev.yaml'", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer
}
Void waitForDeletion(String bezeichner) {
try {
sh "kubectl wait --for=delete applications/by-${bezeichner}-dev-application -n argocd --timeout=300s"
} catch (Exception e) {
error("Application by-${bezeichner}-dev-application konnte nicht gelöscht werden")
}
}
Void waitForKopStackRollout(kopBezeichner) {
for(bezeichner in kopBezeichner) {
waitForRollout(bezeichner)
}
}
Void waitForRollout(String bezeichner) {
waitForHealthyApplication(bezeichner, 'application')
waitForHealthyApplication(bezeichner, 'user-manager')
waitForHealthyApplication(bezeichner, 'pluto')
waitForHealthyApplication(bezeichner, 'goofy')
}
Void waitForHealthyApplication(String bezeichner, String application) {
try {
def countRetry = 0
def maxRetry = 12
while (!isApplicationPresent(bezeichner, application) && countRetry < maxRetry ) {
countRetry++
sh "sleep 5"
}
if (!isApplicationHealthy(bezeichner, application)) {
waitForHealthyStatus(bezeichner, application)
}
} catch (Exception e) {
error("Application ${application} unhealthy")
}
}
Boolean isApplicationPresent(String bezeichner, String application) {
return sh (script: "kubectl get applications -n argocd | grep 'by-${bezeichner}-dev-${application}'", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer
}
Boolean isApplicationHealthy(String bezeichner, String application) {
return sh (script: "kubectl get application/by-${bezeichner}-dev-${application} -n argocd -o=jsonpath='{.status.health.status}' | grep Healthy", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer
}
Void waitForHealthyStatus(String bezeichner, String application) {
sh "kubectl wait --for=jsonpath='{.status.health.status}'=Healthy applications/by-${bezeichner}-dev-${application} -n argocd --timeout=300s"
}
Void publishE2ETestResult(String reportFolder, String reportName) {
publishHTML (
target: [
allowMissing: false,
alwaysLinkToLastBuild: false,
keepAll: true,
reportDir: "goofy-client/apps/goofy-e2e/reports/${reportFolder}",
reportFiles: 'report.html',
reportName: reportName
]
)
}
String runTests(String bezeichner, String reportFolder, Integer dbPort, String stageName) {
def configFile = generateCypressConfig(bezeichner, reportFolder, dbPort)
try {
dir("goofy-client") {
sh "npm run cypress:version"
sh "npm run cypress:ci-run --CONFIG_FILE=${configFile} --REPORT_FOLDER=${reportFolder}"
}
} catch (Exception e) {
printNpmDebugLog()
error("Fehler in Stage ${stageName}")
}
}
Void printNpmDebugLog() {
if (hasNpmDebugLog()) {
sh "cat /root/.npm/_logs/*-debug.log"
}
else {
echo "Npm debug log not found"
}
}
String makeUrlConform(String input) {
return input.replaceAll(/[^a-zA-Z0-9]+/, "").toLowerCase()
}
String generateBezeichner(String stage) {
def branchName = makeUrlConform(env.BRANCH_NAME)
def stageName = makeUrlConform(stage)
return "${cutBranchNameForKeycloakRealm(branchName, stageName)}-${stageName}"
}
String cutBranchNameForKeycloakRealm(String branchName, String stageName) {
final maxKeycloakRealmLength = 30
def cutBranchNamePosition = maxKeycloakRealmLength - (stageName.length() + "${env.BUNDESLAND}---dev".length())
return branchName.take(cutBranchNamePosition)
}
String generateCypressConfig(String bezeichner, String testFolder, Integer dbPort) {
def namespace = generateNamespace(bezeichner)
def configName = "cypress-ci-"+testFolder+".json"
dir('goofy-client/apps/goofy-e2e/'){
def config = readJSON file: 'cypress-ci.json'
config.baseUrl = "https://${bezeichner}.${env.CLUSTER_BASE_URL}" as String
config.env.dbUrl = "mongodb://pluto-database-user:XnHhfznNWg65NNd@localhost:${dbPort}/admin?ssl=false&directConnection=true" as String
config.env.keycloakUrl = "https://${env.SSO_URL}/" as String
config.env.keycloakRealm = namespace as String
config.env.keycloakClient = "alfa" as String
config.env.sabineUuid = getKeycloakUuid(namespace, "sabine") as String
config.integrationFolder = "./src/integration/${testFolder}" as String
config.videosFolder = "./reports/${testFolder}/videos" as String
config.screenshotsFolder = "./reports/${testFolder}/screenshots" as String
config.reporterOptions.reportDir = "./reports/${testFolder}/mochawesome-report" as String
config.env.put("search", getElasticsearchEnv(namespace))
config.env.put("userManager", getUserManagerEnv(dbPort))
writeJSON file: configName, json: config
sh "cat ${configName}"
}
return configName
}
Map getUserManagerEnv(dbPort){
return [
"dbUrl": "mongodb://user-manager-database-user:5M3N2sVEq5c8@localhost:${dbPort}/admin?ssl=false&directConnection=true" as String,
"database": "user-manager-database"
]
}
String getKeycloakUuid(realm, userName) {
def shScript = """curl -H 'Content-Type: application/json' \
-H 'Authorization: bearer ${getKeycloakAccessToken()}' \
'https://${env.SSO_URL}/admin/realms/${realm}/users'
"""
def users = readJSON text: sh(script: shScript, returnStdout: true)
echo "users: ${users}"
for(user in users) {
if (user.username == userName) {
return user.id
}
}
}
String getKeycloakAccessToken() {
withCredentials([usernamePassword(credentialsId: 'keycloak-ovh-cluster', usernameVariable: 'USER', passwordVariable: 'PASSWORD')]) {
def token = readJSON text: sh (script: 'curl -d "client_id=admin-cli" -d "username=$USER" -d "password=$PASSWORD" -d "grant_type=password" https://$SSO_URL/realms/master/protocol/openid-connect/token', returnStdout: true)
return token.access_token
}
}
Void sendFailureMessage() {
def data = [
"msgtype": "m.text",
"body": "E2E-Tests: Failed stage: ${getFailedStage()} Build-ID: ${env.BUILD_NUMBER} Link: ${BLUE_OCEAN_URL}" as String,
"format": "org.matrix.custom.html",
"formatted_body": "E2E-Tests: Failed stage: ${getFailedStage()} Build-ID: <a href='${BLUE_OCEAN_URL}'>${env.BUILD_NUMBER}</a>" as String
]
sh "curl -XPOST -H 'authorization: Bearer ${getElementAccessToken()}' -d '${JsonOutput.toJson(data)}' https://matrix.ozg-sh.de/_matrix/client/v3/rooms/${getElementRoomId()}/send/m.room.message"
}
String getFailedStage() {
if (FAILED_PARALLEL_STAGE.trim()) {
return FAILED_PARALLEL_STAGE
}
return FAILED_STAGE
}
String getElementRoomId() {
final releaseRoomId = "!oWZpUGTFsxkJIYNfYg:matrix.ozg-sh.de"
final masterRoomId = "!iQPAvQIiRwRpNOszjw:matrix.ozg-sh.de"
if (isReleaseBranch()) {
return releaseRoomId
}
return masterRoomId
}
String getElementAccessToken() {
withCredentials([string(credentialsId: 'element-login-json', variable: 'LOGIN_JSON')]) {
return readJSON ( text: sh (script: '''curl -XPOST -d \"$LOGIN_JSON\" https://matrix.ozg-sh.de/_matrix/client/v3/login''', returnStdout: true)).access_token
}
}
Map getElasticsearchEnv(String namespace) {
def elasticsearchSecret = getElasticsearchSecret(namespace)
return [
"user": decodeString(elasticsearchSecret.username),
"password": decodeString(elasticsearchSecret.password),
"index": decodeString(elasticsearchSecret.index),
"url": "https://localhost:9200"
]
}
Void forwardServices(String namespace, Integer dbPort) {
try {
forwardDatenbank(namespace, dbPort)
forwardElasticSearch()
}
catch (Exception e) {
echo "forwardServices Exception: ${e}"
error("Error forwarding service")
}
}
Void forwardElasticSearch() {
if(!isElasticSearchForwarded()) {
sh "kubectl port-forward ozg-search-cluster-es-ozg-search-0 9200:9200 -n elastic-system &"
}
}
Boolean isElasticSearchForwarded() {
return sh (script: "lsof -i -P -n | grep LISTEN | grep :9200", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer
}
Void forwardDatenbank(String namespace, port) {
sh "kubectl port-forward pluto-database-0 ${port}:27017 -n ${namespace} &"
}
String generateNamespace(String bezeichner) {
return "${env.BUNDESLAND}-${bezeichner}-dev"
}
String decodeString(String encoded) {
return sh (script: "echo -n ${encoded} | base64 --decode", returnStdout: true)
}
Map getElasticsearchSecret(String namespace) {
return readJSON ( text: sh (script: "kubectl get secret elasticsearch-credentials -n ${namespace} -o jsonpath={.data}", returnStdout: true))
}
Boolean hasNpmDebugLog() {
return sh (script: "ls -l /root/.npm/_logs/*-debug.log", returnStatus: true) == env.SH_SUCCESS_STATUS_CODE as Integer
}
\ No newline at end of file
...@@ -8,7 +8,10 @@ goofy: ...@@ -8,7 +8,10 @@ goofy:
sso: sso:
serverUrl: https://sso.dev.by.ozg-cloud.de serverUrl: https://sso.dev.by.ozg-cloud.de
apiPassword: "Test1234!" apiPassword: "Test1234!"
role_einheitlicher_ansprechpartner: true keycloak_clients:
- client_name: alfa
client_roles:
- name: EINHEITLICHER_ANSPRECHPARTNER
ingress: ingress:
use_staging_cert: true use_staging_cert: true
baseUrl: dev.by.ozg-cloud.de baseUrl: dev.by.ozg-cloud.de
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment