Commit a251533c authored by David Huebner's avatar David Huebner

Merge branch 'release/v2.1.0'

parents c40d857f 8b96227e
DIDMOS2_DEMO_FRONTEND='https://gitlab.daasi.int/didmos2-demo/didmos2-demo-frontend.git'
DIDMOS2_DEMO_FRONTEND_VERSION = "develop"
DIDMOS2_DEMO_FRONTEND_VERSION="v2.1.0"
DIDMOS2_DEMO_FRONTEND_TAG="v2.1.0"
DIDMOS2_OPENLDAP='https://gitlab.daasi.int/didmos2/didmos2-openldap.git'
DIDMOS2_OPENLDAP_VERSION = "develop"
DIDMOS2_OPENLDAP_VERSION="v2.3.0"
DIDMOS2_OPENLDAP_TAG="v2.3.0"
DIDMOS2_MONGODB='https://gitlab.daasi.int/didmos2/didmos2-mongodb.git'
DIDMOS2_MONGODB_VERSION = "develop"
DIDMOS2_MONGODB_VERSION="v2.2.0"
DIDMOS2_MONGODB_TAG="v2.2.0"
DIDMOS2_CORE='https://gitlab.daasi.int/didmos2/didmos2-core.git'
DIDMOS2_CORE_VERSION = "develop"
DIDMOS2_LIBPY='https://gitlab.daasi.int/didmos2/didmos2libpy.git'
DIDMOS2_LIBPY_VERSION = "develop"
DIDMOS2_CORE_VERSION="v2.3.0"
DIDMOS2_CORE_TAG="v2.3.0"
DIDMOS2_AUTH='https://gitlab.daasi.int/didmos2/didmos2-auth.git'
DIDMOS2_AUTH_VERSION = "develop"
GIT_SSL_VERIFY=true
FE_HOST='frontend.example.org'
AUTH_HOST='auth.example.org'
BE_HOST='didmos.example.org'
SMTP_SERVER='smtp.example.org'
SMTP_USER='admin'
SMTP_PASSWORD='secret'
DIDMOS2_AUTH_VERSION="v2.3.0"
DIDMOS2_AUTH_TAG="v2.3.0"
BACKEND_HOSTNAME=didmos2-backend.local
FRONTEND_HOSTNAME=didmos2-frontend.local
AUTH_HOSTNAME=didmos2-auth.local
DIDMOS_IMAGE=registry.gitlab.daasi.int/didmos2/didmos2-core:v2.3.0
FRONTEND_IMAGE=registry.gitlab.daasi.int/didmos2-demo/didmos2-demo-frontend:v2.1.0
LDAP_IMAGE=registry.gitlab.daasi.int/didmos2/didmos2-openldap:v2.3.0
MONGO_IMAGE=registry.gitlab.daasi.int/didmos2/didmos2-mongodb:v2.2.0
SATOSA_IMAGE=registry.gitlab.daasi.int/didmos2/didmos2-auth:v2.3.0
\ No newline at end of file
......@@ -9,4 +9,9 @@ didmos2-openldap
docker-compose.override.yml
traefik/certs/*
deploy/.env
deploy/certs/*
\ No newline at end of file
deploy/certs/*
traefik/traefik-both.toml
traefik/traefik-frontend.toml
traefik/traefik-didmos.toml
didmos2env
.vscode/settings.json
......@@ -6,20 +6,17 @@ agent {label 'dockerhost-PRJ'}
AUTH_HOSTNAME='docker-crust-auth.daasi.prj'
BACKEND_HOSTNAME='docker-crust-didmos.daasi.prj'
DOCKER_REGISTRY='registry.gitlab.daasi.int'
//SMTP_PASS_JENKINS_VARIABLE = 'tempTest'
}
stages{
stages{
stage("cleanup environment"){
steps{
script {
//echo "smtp passwd: ${env.SMTP_PASS_JENKINS_VARIABLE}"
try {
sh("docker-compose -f docker-compose.yml down");
}
catch(Exception e) {
}
try {
......@@ -28,27 +25,43 @@ stages{
//nothing to do
}
try {
sh("""docker volume rm didmos2-demo-pipeline_didmos2-openldap-db""")
sh("""docker volume rm didmos2-demo-pipeline_didmos2-demo-openldap-db""")
}
catch(Exception e) {
}
}
}//end steps pull environment
}//end stage pull
}//end steps cleanup environment
}//end stage cleanup environment
stage("pull environment"){
steps{
load "$WORKSPACE/.env-build"
sh("""make get-repos BRANCH=develop """)
script{
echo "BRANCH: ${env.BRANCH}"
echo "RunTests: ${env.RunTests}"
echo "RunSmokeTests: ${env.RunSmokeTests}"
echo "TestMails: ${env.TestMails}"
echo "SMTP_PASS_JENKINS_VARIABLE: ${env.SMTP_PASS_JENKINS_VARIABLE}"
echo "PUSH_EXTERNAL_REGISTRY: ${env.PUSH_EXTERNAL_REGISTRY}"
echo "OVERRIDE_TAG: ${env.OVERRIDE_TAG}"
sh ('git status')
if(env.BRANCH.toString() == "origin/develop") {
sh("""make get-repos""")
}
else {
sh("""make get-repos ENV=release""")
}
}//end script
}//end steps pull environment
}//end stage pull
stage ("build & compose didmos2-demo environment"){
stage ("build & compose didmos2-demo environment"){
steps{
echo "Composing:"
//load "$WORKSPACE/store"
......@@ -56,16 +69,16 @@ stages{
sh ("printenv")
script{
try{
sh("""source ./prepare-image-parameters.sh && docker-compose pull --ignore-pull-failures && docker-compose up -d""")
sh("""source ./prepare-image-parameters.sh && docker-compose -f docker-compose.yml -f docker-compose-build.yml pull --ignore-pull-failures && docker-compose -f docker-compose.yml -f docker-compose-build.yml up -d""")
//start daemonized
}catch(err){
echo "$err"
currentBuild.result = 'FAILURE'
throw new hudson.AbortException('Environment not ready!')
}
}
}
}
}
}//end steps build & compose didmos2-demo environment
}//end stage build & compose didmos2-demo environment
stage ("test environment"){
// so far only a dummy test for all hosts to be up and running
......@@ -75,14 +88,14 @@ stages{
sleep 5
echo 'Make sure all needed VMs are running (is pingable)'
sh 'docker ps'
List<String> neededServers = new ArrayList<String>();
neededServers.add("didmos2-demo-core");
neededServers.add("didmos2-demo-frontend");
neededServers.add("didmos2-demo-auth");
for (String container: neededServers) {
PING_SUCCESS= sh (
script: "docker exec didmos2-demo-auth /bin/bash -c 'ping -c 3 ${container}'",
returnStatus: true
......@@ -91,7 +104,7 @@ stages{
echo("Coult not ping")
currentBuild.result = 'FAILURE'
// full wipe of current build
throw new hudson.AbortException('Composed Environment failed! ')
return
}
......@@ -100,21 +113,40 @@ stages{
}
}
}//end script
}//end steps
}//end stage
}//end steps test environment
}//end stage test environment
stage("Push images") {
steps {
script{
// allways push images to -currently- internal registry
docker.withRegistry('http://${DOCKER_REGISTRY}', 'ad7b56a8-6be3-48e5-afa0-da2810812359'){
sh ("""source ./prepare-image-parameters.sh && docker-compose push""")
sh ("""source ./retag.sh""")
}
if(env.PUSH_EXTERNAL_REGISTRY.toBoolean()) {
if(env.OVERRIDE_TAG.toString() != "") {
sh ("""source ./scripts/jenkins_retag_and_push_to_external.sh $env.OVERRIDE_TAG""")
}
else if (env.BRANCH.toString() == "origin/develop") {
sh ("""source ./scripts/jenkins_retag_and_push_to_external.sh latest""")
}
else {
sh ("""source ./scripts/jenkins_retag_and_push_to_external.sh""")
}
}
else {
docker.withRegistry('http://${DOCKER_REGISTRY}', 'ad7b56a8-6be3-48e5-afa0-da2810812359'){
if(env.OVERRIDE_TAG.toString() != "") {
sh ("""source ./scripts/jenkins_retag_and_push_to_internal.sh $env.OVERRIDE_TAG""")
}
else if (env.BRANCH.toString() == "origin/develop") {
sh ("""source ./scripts/jenkins_retag_and_push_to_internal.sh latest""")
}
else {
sh ("""source ./scripts/jenkins_retag_and_push_to_internal.sh""")
}
}
}
}//end script
}//end steps
}//end stage
}//end steps Push images
}//end stage Push images
stage("Run Selenium Tests") {
when {
anyOf {
......@@ -126,19 +158,17 @@ stages{
script {
def runSmokes = Boolean.valueOf(env.RunSmokeTests);
def testResults = build job: 'SeleniumTestBuild', propagate: false, parameters: [[$class: 'StringParameterValue', name: 'TestClass', value: 'DidmosDemo_Test'],[$class: 'StringParameterValue', name: 'TestMethod', value: ''],[$class: 'StringParameterValue', name: 'TestCustomer', value: 'DidmosDemo-Jenkins'],[$class: 'StringParameterValue', name: 'GITBRANCH', value: "origin/develop"], [$class: 'StringParameterValue', name: 'TestMails', value: "${env.TestMails}"], [ $class: 'BooleanParameterValue', name: 'SmokeTests', value: runSmokes]]
if(testResults.result.equals("FAILURE") || testResults.result.equals("ABORTED")) {
error 'Tests failed!'
}
}
}//end steps
}//end stage
}//end steps Run Selenium Tests"
}//end stage Run Selenium Tests"
}//end stages
post{
always {
sh ('docker-compose down')
......
# !!! ONLY EDIT THIS BLOCK !!!
# !!! DO NOT EDIT BELOW !!!
THIS_FILE := $(lastword $(MAKEFILE_LIST))
include .env-build
export
REPO_FILE=.repos
ifeq ($(ENV), local)
info:
@echo 'local'
else ifeq ($(ENV), release)
info:
@echo 'release'
else
info:
@echo 'default (develop)'
endif
ifneq ("$(wildcard $(REPO_FILE))","")
REPO_OVERRIDE = 1
$(info $(shell tput setaf 1)Using repo url override from file $(REPO_FILE)$(shell tput sgr0))
include .repos
# == Pull all repos and run child make files ==
# If called with ENV=develop, the develop branch will be used
# Otherwise the branches specified in .env-build will be used
ifeq ($(ENV), release)
get-repos:
git clone --branch $(DIDMOS2_DEMO_FRONTEND_VERSION) $(DIDMOS2_DEMO_FRONTEND)
git clone --branch $(DIDMOS2_OPENLDAP_VERSION) $(DIDMOS2_OPENLDAP)
git clone --branch $(DIDMOS2_MONGODB_VERSION) $(DIDMOS2_MONGODB)
git clone --branch $(DIDMOS2_CORE_VERSION) $(DIDMOS2_CORE)
git clone --branch $(DIDMOS2_AUTH_VERSION) $(DIDMOS2_AUTH)
make submodule-init
else
REPO_OVERRIDE = 0
get-repos:
git clone --branch develop $(DIDMOS2_DEMO_FRONTEND)
git clone --branch develop $(DIDMOS2_OPENLDAP)
git clone --branch develop $(DIDMOS2_MONGODB)
git clone --branch develop $(DIDMOS2_CORE)
git clone --branch develop $(DIDMOS2_AUTH)
make submodule-init
endif
submodule-init:
$(MAKE) -C didmos2-demo-frontend
$(MAKE) -C didmos2-core
$(MAKE) -C didmos2-auth
export
# == Pull images ==
ifeq ($(ENV), release)
pull:
@env $(shell cat .env-dev-release) docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" --log-level ERROR pull
@echo "== Pulled the following images according to .env-dev-release: =="
@env $(shell cat .env-dev-release) docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" --log-level ERROR config | grep image: | sed 's/.*: //'
else
pull:
docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" --log-level ERROR pull
@echo "== Pulled the following images according to .env: =="
@docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" --log-level ERROR config | grep image: | sed 's/.*: //'
endif
info:
@echo 'not implemented'
# == Stop development environment ==
down:
@docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" --log-level ERROR down
ifeq ($(BRANCH), master)
get-repos:
@echo 'not implemented'
# == Start development environment ==
ifeq ($(ENV), local)
up:
@echo "== Starting with :local images =="
@docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" -f "docker-compose-dev-local.yml" --log-level ERROR up -d
else ifeq ($(ENV), release)
up:
@echo "== Starting with release images according to .env-dev-release =="
@env $(shell cat .env-dev-release) docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" --log-level ERROR up -d
else
up:
@echo "== Starting with :latest images =="
@docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" --log-level ERROR up -d
endif
else ifeq ($(BRANCH), develop)
get-repos:
git clone $(DIDMOS2_DEMO_FRONTEND)
cd didmos2-demo-frontend;git checkout develop; cd ..
git clone $(DIDMOS2_OPENLDAP)
cd didmos2-openldap;git checkout develop; cd ..
git clone $(DIDMOS2_MONGODB)
cd didmos2-mongodb; git checkout develop; cd ..
git clone $(DIDMOS2_CORE)
cd didmos2-core; git checkout develop; cd ..
git clone $(DIDMOS2_AUTH)
cd didmos2-auth; git checkout develop; cd ..
$(MAKE) -C didmos2-demo-frontend
$(MAKE) -C didmos2-core
$(MAKE) -C didmos2-auth
# == Start development environment with local frontend ==
ifeq ($(ENV), local)
up-frontend:
@echo "== Starting frontend-development with :local images =="
@docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" -f "docker-compose-dev-frontend.yml" -f "docker-compose-dev-local.yml" --log-level ERROR up -d auth mongo ldap didmos traefik
@cd scripts && sh up-frontend.sh $(LIBDEV)
else ifeq ($(ENV), release)
up-frontend:
@echo "== Starting frontend-development with release images according to .env-dev-release =="
@env $(shell cat .env-dev-release) docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" -f "docker-compose-dev-frontend.yml" --log-level ERROR up -d auth mongo ldap didmos traefik
@cd scripts && sh up-frontend.sh $(LIBDEV)
else
get-repos:
@echo 'not implemented'
up-frontend:
@echo "== Starting frontend-development with :latest images =="
@docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" -f "docker-compose-dev-frontend.yml" --log-level ERROR up -d auth mongo ldap didmos traefik
@cd scripts && sh up-frontend.sh $(LIBDEV)
endif
# == Start development environment with local backend ==
ifeq ($(ENV), local)
up-didmos:
@echo "== Starting backend-development with :local images =="
@docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" -f "docker-compose-dev-didmos.yml" -f "docker-compose-dev-local.yml" --log-level ERROR up -d auth mongo ldap frontend traefik
@cd scripts && sh up-backend.sh $(NOSTART)
else ifeq ($(ENV), release)
up-didmos:
@echo "== Starting backend-development with release images according to .env-dev-release =="
@env $(shell cat .env-dev-release) docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" -f "docker-compose-dev-didmos.yml" --log-level ERROR up -d auth mongo ldap frontend traefik
@cd scripts && sh up-backend.sh $(NOSTART)
else
up-didmos:
@echo "== Starting backend-development with :latest images =="
@docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" -f "docker-compose-dev-didmos.yml" --log-level ERROR up -d auth mongo ldap frontend traefik
@cd scripts && sh up-backend.sh $(NOSTART)
endif
# == Start development environment with local frontend & backend ==
ifeq ($(ENV), local)
up-both:
@echo "== Starting front&backend-development with :local images =="
@docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" -f "docker-compose-dev-both.yml" -f "docker-compose-dev-local.yml" --log-level ERROR up -d auth mongo ldap traefik
@cd scripts && sh up-backend.sh $(NOSTART) && sh up-frontend.sh $(LIBDEV)
else ifeq ($(ENV), release)
up-both:
@echo "== Starting front&backend-development with release images according to .env-dev-release =="
@env $(shell cat .env-dev-release) docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" -f "docker-compose-dev-both.yml" --log-level ERROR up -d auth mongo ldap traefik
@cd scripts && sh up-backend.sh $(NOSTART) && sh up-frontend.sh $(LIBDEV)
else
up-both:
@echo "== Starting front&backend-development with :latest images =="
@docker-compose -f "docker-compose.yml" -f "docker-compose-dev.yml" -f "docker-compose-dev-both.yml" --log-level ERROR up -d auth mongo ldap traefik
@cd scripts && sh up-backend.sh $(NOSTART) && sh up-frontend.sh $(LIBDEV)
endif
clean-fs:
# == shortcuts ==
up-both-lib:
make up-both LIBDEV=true
up-frontend-lib:
make up-frontend LIBDEV=true
# == Build local containers ==
build-local:
@echo "== Building :local images of all components based on local files =="
sh ./scripts/manual_build.sh
sh ./scripts/retag_local.sh
# == Remove all locally checked out repositories ==
clean-repos:
rm -rf didmos2-auth didmos2-core didmos2-demo-frontend didmos2-mongodb didmos2-openldap
clean-docker:
docker stop $$(docker ps -aq); docker rm -f $$(docker ps -aq); docker rmi -f $$(docker images -q)
purge-build:
# == Bootstrap local development environment ==
ifeq ($(ENV), release)
bootstrap:
@echo "== Bootstrap with release repositories =="
@cd scripts && sh bootstrap-devel.sh $(INSTALL)
else
bootstrap:
@echo "== Bootstrap with develop repositories =="
@cd scripts && sh bootstrap-devel.sh develop $(INSTALL)
endif
# == Remove all docker containers, images and volumes ==
clean-docker:
#docker stop $$(docker ps -aq); docker rm -f $$(docker ps -aq); docker rmi -f $$(docker images -q)
@echo 'not implemented'
clean-docker-all:
docker ps -a -q | xargs -r docker rm -f
docker network prune
docker volume ls -q -f 'Name=didmos2-demo' | xargs -r docker volume rm
\ No newline at end of file
Introduction
didmos2-demo-compose
============
Didmos 2.0 as Docker-Compose project.
## Requirements for the development environment
## Requirements
- docker
- docker-compose
- Fedora OS with root access to install dependencies
- Docker
- Docker-Compose
- Apache Webserver
- 3 DNS entrys for your host e.g. crust-auth.example.org, crust-didmos.exampe.org, crust.example.org
## Commands
## Usage
Install the recent Docker version directly from the Docker Repo:
```
curl -sSL https://get.docker.com/ | CHANNEL=stable sh
# After the installation process is finished, you may need to enable the service and make sure it is started (e.g. CentOS 7)
systemctl enable docker.service
systemctl start docker.service
```
make bootstrap
```
Install Docker-Compose:
Setup development environment from a clean clone of the didmos2-demo-compose repository. Generally this should only be run once.
```
yum install python-pip
pip install docker-compose
```
This checks out repositories for all components, runs setup for the local development environment (certificates etc.) and installs dependencies for local development (python, node, etc.)
Clone the project with the following command and replace {relese with the current version (e.g. git checkout tags/v2.0.1_RC1)}
By default the develop-branches of all repositories are used.
```
git clone https://gitlab.daasi.de/CRUST/compose.git
cd compose
git checkout {release}
```
Use the flag ```ENV=release``` to setup with release-branches instead. The current release branches are specified in the file .env-build.
Get the additional needed repos by running
Use the flag ```INSTALL=noroot``` to run without running root-commands during bootstrap. This generally works, when another instance of didmos2 was already setup on the system, but might fail due to missing dependencies.
```
make get-repos
```
When succesful, install a Apache webserver (in this case on CentOS 7)
***
```
yum install httpd mod_ssl
```
Copy the vhosts.conf to /etc/httpd/conf.d
make pull
```
```
cp vhosts.conf /etc/httpd/conf.d/vhosts.conf
```
Pull most recent docker images from docker registry (gitlab.daasi.int). TUsually these images are built with the Jenkins pipeline.
Change the values of the vhosts config file according to your subdomains:
By default images tagged with __latest__ are pulled (this is specified in .env).
```
vi /etc/httpd/conf.d/vhosts.conf
```
Use the flag ```ENV=release``` to use release images instead, which are specified in .env-dev-release.
```
#Virtualhost for Satosa (Port 1000)
<VirtualHost *:80>
ServerName crust-auth.example.org
ProxyPreserveHost On
ProxyPass / http://127.0.0.1:1000/
ProxyPassReverse / http://127.0.0.1:1000/
Header set Access-Control-Allow-Origin "*"
Header set Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS, PATCH"
Header set Access-Control-Allow-Headers "x-test-header, Origin, X-Requested-With, Content-Type, Accept, Authorization"
</VirtualHost>
#Virtualhost for Frontend (Port 3000)
<VirtualHost *:80>
ServerName crust.example.org
ProxyPreserveHost On
ProxyPass / http://127.0.0.1:3000/
ProxyPassReverse / http://127.0.0.1:3000/
Header set Access-Control-Allow-Origin "*"
Header set Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS, PATCH"
Header set Access-Control-Allow-Headers "x-test-header, Origin, X-Requested-With, Content-Type, Accept, Authorization"
</VirtualHost>
#Virtualhost for Didmos (Port 2000)
<VirtualHost *:80>
ServerName crust-didmos.example.org
ProxyPreserveHost On
ProxyPass / http://127.0.0.1:2000/
ProxyPassReverse / http://127.0.0.1:2000/
Header set Access-Control-Allow-Origin "*"
Header set Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS, PATCH"
Header set Access-Control-Allow-Headers "x-test-header, Origin, X-Requested-With, Content-Type, Accept, Authorization"
</VirtualHost>
```
Enable HTTPD and run
```
systemctl enable httpd && systemctl start httpd
```
After that install Letsencrypt and get some certificates
```
yum install certbot python2-certbot-apache
```
Get the certificates
```
certbot --apache -d crust.daasi.de -d crust-auth.daasi.de -d crust-didmos.daasi.de
```
If you run into some weird error in python about pyopenssl try this (There are some issues with Letsencrypt and CentOS at this time):
***
```
pip install requests==2.6.0
easy_install --upgrade pip
```
And run certbot cmd again. Enter your mail address and say no to the newsletter.
If the Certbot asks you if you want to redirect all your requests to the webserver to secure connection say yes (Option 2)
make up
```
Now you have to edit the docker-compose file:
Start development environment with all components as docker containers.
```
vi docker-compose.yml
```
By default images tagged with __latest__ are used (this is specified in .env).
Carefully read the documentation of all settings below and adjust to your environment.
Use the flag ```ENV=release``` to use release images instead, which are specified in .env-dev-release.
After that run:
Use the flag ```ENV=local``` to use local images instead (which can be built with ```make build-local```, see below).
```
make docker-build
```
To build the images (this will take a while). When successful run
***
```
docker-compose up -d
```
After that you can reach your new Didmos Installation at https://crust.example.org
Environment Variables in docker-compose.yml
=====================
_Note_: The following documentation assumes that you've set-up the environment with the following three hostnames:
- crust.example.org: Running the frontend
- crust-auth.example.org: Running satosa
- crust-didmos.example.org: Running the backend
## MongoDB