Template project commit

This commit is contained in:
Przemyslaw Nieroda 2022-04-05 23:44:57 +02:00
commit 27035056f9
32 changed files with 3014 additions and 0 deletions

117
.gitignore vendored Normal file
View file

@ -0,0 +1,117 @@
# Created by https://www.toptal.com/developers/gitignore/api/phpstorm
# Edit at https://www.toptal.com/developers/gitignore?templates=phpstorm
### PhpStorm ###
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
# AWS User-specific
.idea/**/aws.xml
# Generated files
.idea/**/contentModel.xml
# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
# Gradle
.idea/**/gradle.xml
.idea/**/libraries
# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using auto-import.
.idea/artifacts
.idea/compiler.xml
.idea/jarRepositories.xml
.idea/modules.xml
.idea/*.iml
.idea/modules
*.iml
*.ipr
# CMake
cmake-build-*/
# Mongo Explorer plugin
.idea/**/mongoSettings.xml
# File-based project format
*.iws
# IntelliJ
out/
.idea/encodings.xml
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Cursive Clojure plugin
.idea/replstate.xml
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Editor-based Rest Client
.idea/httpRequests
# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser
### PhpStorm Patch ###
# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
# *.iml
# modules.xml
# .idea/misc.xml
# *.ipr
# Sonarlint plugin
# https://plugins.jetbrains.com/plugin/7973-sonarlint
.idea/**/sonarlint/
# SonarQube Plugin
# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
.idea/**/sonarIssues.xml
# Markdown Navigator plugin
# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
.idea/**/markdown-navigator.xml
.idea/**/markdown-navigator-enh.xml
.idea/**/markdown-navigator/
# Cache file creation bug
# See https://youtrack.jetbrains.com/issue/JBR-2257
.idea/$CACHE_FILE$
# CodeStream plugin
# https://plugins.jetbrains.com/plugin/12206-codestream
.idea/codestream.xml
# End of https://www.toptal.com/developers/gitignore/api/phpstorm
#Visual Studio Code
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json

9
.gitlab-ci.yml Normal file
View file

@ -0,0 +1,9 @@
include:
- local: 'cicd/initialization.yml'
rules:
- if: '$PROJECT_INIT == "1"'
- local: 'cicd/pipeline.yml'
rules:
- if: '$PROJECT_INIT != "1"'

1
LICENSE Normal file
View file

@ -0,0 +1 @@
TO BE PROVIDED

23
README.md Normal file
View file

@ -0,0 +1,23 @@
# mvp
Short information about the project.
Provide here all valuable links for the project (Wiki, documents, endpoints).
## Quickstart
### Building in local developer environment
How to build and run the software in the local developer environment.
### Building in IH environments
How to build and run the software in the IHLC environment.
## Technical stack
Shortly describe the technical stack of the project.
## Visible endpoints
Describe all exposed endpoints of the project.

35
cicd/.env.dev Normal file
View file

@ -0,0 +1,35 @@
########################################################################################################################
#
# 'docker compose' and 'docker build' variables for IL DEVELOPMENT environment:
# 1. Used to set up 'docker compose' integration variables (ports, tag, image path) for IL DEVELOPMENT environment.
# 2. Used to set up 'docker build' integration variables (mostly EXPOSE port by image, but could be more).
# 3. Generic rule on setup:
# - use dockerfile to customize the image
# - used docker-compose.yml to customize the deployment
# 4. All the values are strongly connected with docker-compose.yml, dockerfiles of the images
# and pipeline configuration. See there before changing.
#
########################################################################################################################
# Generic variables
# Docker project name
COMPOSE_PROJECT_NAME=mvp
# Specification of docker-compose files to be used
COMPOSE_PATH_SEPARATOR=:
COMPOSE_FILE=docker-compose.yml
# Tag used to mark images and name containers and other elements
TAG=dev
# mvp-keycloak
KEYCLOAK_INTERNAL_PORT=8090
KEYCLOAK_PORT_OFFSET=10
# mvp-keycloak-postgresql
KEYCLOAK_POSTGRESQL_INTERNAL_PORT=5432
# mvp-proxy
PROXY_INTERNAL_PORT=8080
PROXY_EXPOSED_PORT=8080

38
cicd/.env.local Normal file
View file

@ -0,0 +1,38 @@
########################################################################################################################
#
# 'docker compose' and 'docker build' variables for local developer environment (own workstation):
# 1. Used to set up 'docker compose' integration variables (ports, tag, image path) for local developer environment.
# 2. Used to set up 'docker build' integration variables (mostly EXPOSE port by image, but could be more).
# 3. Generic rule on setup:
# - use dockerfile to customize the image
# - used docker-compose.yml to customize the deployment
# 4. All the values are strongly connected with docker-compose.yml, dockerfiles of the images
# and pipeline configuration. See there before changing.
#
########################################################################################################################
# Generic variables
# Docker project name
COMPOSE_PROJECT_NAME=mvp
# Specification of docker-compose files to be used
COMPOSE_PATH_SEPARATOR=:
COMPOSE_FILE=docker-compose-build.yml:docker-compose.yml
# Tag used to mark images and name containers and other elements
TAG=local
# mvp-keycloak
KEYCLOAK_INTERNAL_PORT=8090
KEYCLOAK_PORT_OFFSET=10
# mvp-keycloak-postgresql
KEYCLOAK_POSTGRESQL_INTERNAL_PORT=5432
# mvp-proxy
PROXY_INTERNAL_PORT=7070
PROXY_EXPOSED_PORT=7070
# Path to the registry with images (must be empty, required only for local deployments to Docker)
IMAGE_PATH=

35
cicd/.env.staging Normal file
View file

@ -0,0 +1,35 @@
########################################################################################################################
#
# 'docker compose' and 'docker build' variables for IL STAGING environment:
# 1. Used to set up 'docker compose' integration variables (ports, tag, image path) for IL STAGING environment.
# 2. Used to set up 'docker build' integration variables (mostly EXPOSE port by image, but could be more).
# 3. Generic rule on setup:
# - use dockerfile to customize the image
# - used docker-compose.yml to customize the deployment
# 4. All the values are strongly connected with docker-compose.yml, dockerfiles of the images
# and pipeline configuration. See there before changing.
#
########################################################################################################################
# Generic variables
# Docker project name
COMPOSE_PROJECT_NAME=mvp
# Specification of docker-compose files to be used
COMPOSE_PATH_SEPARATOR=:
COMPOSE_FILE=docker-compose.yml
# Tag used to mark images and name containers and other elements
TAG=staging
# mvp-keycloak
KEYCLOAK_INTERNAL_PORT=8090
KEYCLOAK_PORT_OFFSET=10
# mvp-keycloak-postgresql
KEYCLOAK_POSTGRESQL_INTERNAL_PORT=5432
# mvp-proxy
PROXY_INTERNAL_PORT=8080
PROXY_EXPOSED_PORT=8080

0
cicd/.gitkeep Normal file
View file

View file

@ -0,0 +1,25 @@
version: "3.7"
########################################################################################################################
#
# These are additional build step for docker compose:
# 1. To be used only in local developer environment (see Readme).
# 2. Within IL environments docker images are built using GitLab CI/CD pipeline.
#
########################################################################################################################
services:
mvp-keycloak:
build:
context: ./../mvp-keycloak
dockerfile: Dockerfile.${TAG}
mvp-keycloak-postgresql:
build:
context: ./../mvp-keycloak-postgresql
dockerfile: Dockerfile.${TAG}
mvp-proxy:
build:
context: ./../mvp-proxy
dockerfile: Dockerfile.${TAG}

63
cicd/docker-compose.yml Normal file
View file

@ -0,0 +1,63 @@
version: "3.7"
########################################################################################################################
#
# These are deployment steps for docker compose:
# 1. Used for both: local development environment and IL environments.
# 2. This configuration is also used to set up deployment variables for containers.
#
########################################################################################################################
services:
mvp-keycloak:
image: ${IMAGE_PATH}mvp-keycloak:${TAG}
container_name: mvp-keycloak-${TAG}
restart: always
command:
[
"-b 0.0.0.0",
"-Dkeycloak.migration.action=import",
"-Dkeycloak.migration.provider=dir",
"-Dkeycloak.migration.dir=/opt/jboss/keycloak/realm-config",
"-Dkeycloak.migration.strategy=IGNORE_EXISTING",
"-Djboss.socket.binding.port-offset=${KEYCLOAK_PORT_OFFSET}"
]
environment:
DB_VENDOR: POSTGRES
DB_ADDR: mvp-keycloak-postgresql-${TAG}
DB_DATABASE: keycloak_db
DB_SCHEMA: public
DB_USER: keycloakuser
DB_PASSWORD: keycloakuser
KEYCLOAK_USER: adminkeycloak
KEYCLOAK_PASSWORD: adminkeycloak
PROXY_ADDRESS_FORWARDING: "true"
expose:
- "${KEYCLOAK_INTERNAL_PORT}"
depends_on:
- mvp-keycloak-postgresql
mvp-keycloak-postgresql:
image: ${IMAGE_PATH}mvp-keycloak-postgresql:${TAG}
container_name: mvp-keycloak-postgresql-${TAG}
restart: always
volumes:
- keycloak_postgres_data:/var/lib/postgresql/data
environment:
POSTGRES_DB: keycloak_db
POSTGRES_USER: keycloakuser
POSTGRES_PASSWORD: keycloakuser
expose:
- "${KEYCLOAK_POSTGRESQL_INTERNAL_PORT}"
mvp-proxy:
image: ${IMAGE_PATH}mvp-proxy:${TAG}
container_name: mvp-proxy-${TAG}
restart: always
ports:
- "${PROXY_EXPOSED_PORT}:${PROXY_INTERNAL_PORT}"
depends_on:
- mvp-keycloak
volumes:
keycloak_postgres_data:

94
cicd/initialization.yml Normal file
View file

@ -0,0 +1,94 @@
########################################################################################################################
#
# Single pass initialization of the project's repository
#
########################################################################################################################
stages:
- initialize
variables:
TEMPLATE_PROJECT_CODENAME:
value: update_me
description: "Codename of the MVP for in source code, only lowercase [a-z0-9] (e.g. jedi)"
TEMPLATE_DEV_SSH_KEY:
value: update_me
description: "Name of the SSH key for DEV environment (e.g. DEV_DOCKER_01_PEM)"
TEMPLATE_DEV_DOCKER:
value: update_me
description: "Name of the Docker host for DEV environment (e.g. dev-docker-01)"
TEMPLATE_STG_SSH_KEY:
value: update_me
description: "Name of the SSH key for STG environment (e.g. STG_DOCKER_01_PEM)"
TEMPLATE_STG_DOCKER:
value: update_me
description: "Name of the Docker host for STG environment (e.g. stg-docker-01)"
CI_EMAIL:
value: update_me
description: "E-mail address of Gitlab User"
CI_USERNAME:
value: update_me
description: "E-mail address of Gitlab User"
SSH_PRIVATE_KEY:
value: update_me
description: "Private key part of SSH key pair for Gitlab User, ensure newline at the end"
SSH_PUBLIC_KEY:
value: update_me
description: "Public key part of SSH key pair for Gitlab User"
# Deploying and fine tuning project repository
initialize-repository:
stage: initialize
image:
name: docker:git
environment:
name: ops-dev
url: https://dev.ilab.zone/
before_script:
- eval $(ssh-agent -s)
- echo "${SSH_PRIVATE_KEY}" | tr -d '\r' | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo "${SSH_PUBLIC_KEY}" >> ~/.ssh/id_rsa.pub
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
only: # Only possible to trigger run manually from Gitlab Web UI
- web
script:
- sed -i s/DEV_SSH_KEY_VALUE/${TEMPLATE_DEV_SSH_KEY}/g cicd/pipeline.yml
- sed -i s/DEV_DOCKER_VALUE/${TEMPLATE_DEV_DOCKER}/g cicd/pipeline.yml
- sed -i s/STG_SSH_KEY_VALUE/${TEMPLATE_STG_SSH_KEY}/g cicd/pipeline.yml
- sed -i s/STG_DOCKER_VALUE/${TEMPLATE_STG_DOCKER}/g cicd/pipeline.yml
- sed -i s/mvp/${TEMPLATE_PROJECT_CODENAME}/g cicd/pipeline-dev.yml
- sed -i s/mvp/${TEMPLATE_PROJECT_CODENAME}/g cicd/pipeline-stg.yml
- sed -i s/mvp/${TEMPLATE_PROJECT_CODENAME}/g cicd/docker-compose.yml
- sed -i s/mvp/${TEMPLATE_PROJECT_CODENAME}/g cicd/docker-compose-build.yml
- sed -i s/mvp/${TEMPLATE_PROJECT_CODENAME}/g mvp-proxy/nginx/local.conf
- sed -i s/mvp/${TEMPLATE_PROJECT_CODENAME}/g mvp-proxy/nginx/dev.conf
- sed -i s/mvp/${TEMPLATE_PROJECT_CODENAME}/g mvp-proxy/nginx/staging.conf
- sed -i s/mvp/${TEMPLATE_PROJECT_CODENAME}/g cicd/.env.local
- sed -i s/mvp/${TEMPLATE_PROJECT_CODENAME}/g cicd/.env.dev
- sed -i s/mvp/${TEMPLATE_PROJECT_CODENAME}/g cicd/.env.staging
- sed -i s/mvp/${TEMPLATE_PROJECT_CODENAME}/g README.md
- sed -i s/mvp/${TEMPLATE_PROJECT_CODENAME}/g sonar-project.properties
- mv mvp-keycloak ${TEMPLATE_PROJECT_CODENAME}-keycloak
- mv mvp-keycloak-postgresql ${TEMPLATE_PROJECT_CODENAME}-keycloak-postgresql
- mv mvp-proxy ${TEMPLATE_PROJECT_CODENAME}-proxy
- ls -la
- echo "Configure gitlab global user to push changes back"
- git config --global user.email "${CI_EMAIL}"
- git config --global user.name "${CI_USERNAME}"
- git remote rm origin && git remote add origin git@shared-gitlab.ilab.private:$CI_PROJECT_PATH.git
#- echo "Configure new branch"
#- git branch -D ${TEMPLATE_PROJECT_CODENAME} || true # Try to delete cached branched or continue if already deleted
#- git checkout -b ${TEMPLATE_PROJECT_CODENAME}
- echo "Add file changes to new branch"
- git add .
- echo "Commit changes"
- git commit -m "Initialized variables"
- echo "Push to new branch"
#- git push -u origin ${TEMPLATE_PROJECT_CODENAME}
- git push origin HEAD:$CI_COMMIT_REF_NAME # Pushes to the same branch as the trigger
- echo "Done!"
tags:
- docker

36
cicd/pipeline-common.yml Normal file
View file

@ -0,0 +1,36 @@
########################################################################################################################
#
# This configuration file is dedicated to define common to all pipelines elements:
# 1. Set up repeating base steps, which are further used (by extending them) in DEV and STAGING configuration.
# 2. Common steps to all the pipelines.
#
########################################################################################################################
# Generic docker build step (used to build all the images)
.build-image-base:
script:
# Going to building folder
- cd $SERVICE_FOLDER
# Copying environment file
- cp $ENV_FILE .env
# Setting up the environment
- export $(grep -v '^#' .env | xargs)
# Build docker image
- docker build -f $DOCKER_FILE -t $CI_REGISTRY_IMAGE/$SERVICE_FOLDER:$TAG .
# Push docker image
- echo -n $CI_BUILD_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- docker push $CI_REGISTRY_IMAGE/$SERVICE_FOLDER:$TAG
- docker logout $CI_REGISTRY
# Image cleanup
remove-images:
stage: cleanup
script:
# remove all tagged images
- docker rmi -f $(docker images --filter reference="$CI_REGISTRY_IMAGE/*" -q) || true
# remove dangling images but keep newer than 4h (to make frequent builds faster)
- docker image prune -f --filter until="4h" || true
rules:
- if: '$CI_COMMIT_BRANCH == "dev" || $CI_COMMIT_BRANCH == "staging" || $CI_COMMIT_BRANCH == "master"'
tags:
- docker

108
cicd/pipeline-dev.yml Normal file
View file

@ -0,0 +1,108 @@
########################################################################################################################
#
# Development environment pipeline configuration:
# 1. Set up all the phases with respect of development environment specifics.
# 2. Building step use 'docker build', proper dockerfile of the image and variables defined in the environment file.
# 3. Deploying steps utilize 'docker compose', docker-compose.yml file as configuration and variables defined in
# the environment file.
#
########################################################################################################################
########################################################################################################################
#
# Building phase elements
#
########################################################################################################################
# Extended generic building image stage for DEV environment (used to build all DEV images)
.build-image-base-dev:
extends: .build-image-base
stage: build
variables:
ENV_FILE: ../cicd/.env.dev
DOCKER_FILE: Dockerfile.dev
rules:
- if: '$CI_COMMIT_BRANCH == "dev"'
tags:
- docker
# Building image of mvp Keycloak PostgreSQL
build-mvp-keycloak-postgresql-dev:
extends: .build-image-base-dev
variables:
SERVICE_FOLDER: mvp-keycloak-postgresql
needs: []
# Building image of mvp Keycloak
build-mvp-keycloak-dev:
extends: .build-image-base-dev
variables:
SERVICE_FOLDER: mvp-keycloak
needs:
- build-mvp-keycloak-postgresql-dev
# Building image of DragonEgg OpenResty reverse proxy
build-mvp-proxy-dev:
extends: .build-image-base-dev
variables:
SERVICE_FOLDER: mvp-proxy
needs: []
########################################################################################################################
#
# Deployment phase elements
#
########################################################################################################################
# Deploying to project's dev docker
deploy-dev:
stage: deploy
environment:
name: ops-dev
url: https://dev.ilab.zone/
before_script:
- eval $(ssh-agent -s)
- echo "$DEV_SSH_KEY" | base64 -d | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
script:
- ssh -o StrictHostKeyChecking=no $USER@$DEV_DOCKER "mkdir -p /home/$USER/mvp"
- scp -o StrictHostKeyChecking=no ./cicd/docker-compose.yml $USER@$DEV_DOCKER:/home/$USER/mvp
- scp -o StrictHostKeyChecking=no ./cicd/.env.dev $USER@$DEV_DOCKER:/home/$USER/mvp/.env
- ssh -o StrictHostKeyChecking=no $USER@$DEV_DOCKER "echo 'IMAGE_PATH='$CI_REGISTRY_IMAGE/ >> /home/$USER/mvp/.env"
- ssh -o StrictHostKeyChecking=no $USER@$DEV_DOCKER "echo -n $CI_BUILD_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY"
# Set up CI/CD variable RESET_VOLUMES to 1 to remove the volumes of the MVP (to reinitiate MVP from scratch)
- |
if [ "$RESET_VOLUMES" == "1" ]; then
ssh -o StrictHostKeyChecking=no $USER@$DEV_DOCKER "cd /home/$USER/mvp && docker-compose down --remove-orphans --volumes"
else
ssh -o StrictHostKeyChecking=no $USER@$DEV_DOCKER "cd /home/$USER/mvp && docker-compose down --remove-orphans"
fi
- ssh -o StrictHostKeyChecking=no $USER@$DEV_DOCKER "docker rmi -f \$(docker images --filter reference="$CI_REGISTRY_IMAGE/*" -q) || true"
- ssh -o StrictHostKeyChecking=no $USER@$DEV_DOCKER "cd /home/$USER/mvp && docker-compose up -d"
rules:
- if: '$CI_COMMIT_BRANCH == "dev"'
tags:
- docker
needs:
- build-mvp-keycloak-dev
- build-mvp-proxy-dev
########################################################################################################################
#
# Testing phase elements
#
########################################################################################################################
# SAST using SonarQube
sonar-scan-dev:
image: sonarsource/sonar-scanner-cli:4.6
stage: sast
script:
- sonar-scanner -Dsonar.host.url=$SONARQUBE_URL -Dsonar.login=$SONARQUBE_TOKEN
allow_failure: true
rules:
- if: '$CI_COMMIT_BRANCH == "dev"'
tags:
- docker

222
cicd/pipeline-stg.yml Normal file
View file

@ -0,0 +1,222 @@
########################################################################################################################
#
# Staging environment pipeline configuration:
# 1. Set up all the phases with respect of staging environment specifics.
# 2. Building step use 'docker build', proper dockerfile of the image and variables defined in the environment file.
# 3. Deploying steps utilize 'docker compose', docker-compose.yml file as configuration and variables defined in
# the environment file.
#
########################################################################################################################
########################################################################################################################
#
# Building phase elements
#
########################################################################################################################
# Extended generic building image stage for STAGING environment (used to build all STAGING images)
.build-image-base-stg:
extends: .build-image-base
stage: build
variables:
ENV_FILE: ../cicd/.env.staging
DOCKER_FILE: Dockerfile.staging
rules:
- if: '$CI_COMMIT_BRANCH == "staging"'
tags:
- docker
# Building image of mvp Keycloak PostgreSQL
build-mvp-keycloak-postgresql-stg:
extends: .build-image-base-stg
variables:
SERVICE_FOLDER: mvp-keycloak-postgresql
needs: []
# Building image of mvp Keycloak
build-mvp-keycloak-stg:
extends: .build-image-base-stg
variables:
SERVICE_FOLDER: mvp-keycloak
needs:
- build-mvp-keycloak-postgresql-stg
# Building image of the mvp OpenResty reverse proxy
build-mvp-proxy-stg:
extends: .build-image-base-stg
variables:
SERVICE_FOLDER: mvp-proxy
needs: []
########################################################################################################################
#
# Deployment phase elements
#
########################################################################################################################
# Deploying to project's staging docker
deploy-stg:
stage: deploy
environment:
name: ops-stg
url: https://stg.ilab.zone/
before_script:
- eval $(ssh-agent -s)
- echo "$STG_SSH_KEY" | base64 -d | cat - <(echo) | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
script:
- ssh -o StrictHostKeyChecking=no $USER@$STG_DOCKER "mkdir -p /home/$USER/mvp"
- scp -o StrictHostKeyChecking=no ./cicd/docker-compose.yml $USER@$STG_DOCKER:/home/$USER/mvp
- scp -o StrictHostKeyChecking=no ./cicd/.env.staging $USER@$STG_DOCKER:/home/$USER/mvp/.env
- ssh -o StrictHostKeyChecking=no $USER@$STG_DOCKER "echo 'IMAGE_PATH='$CI_REGISTRY_IMAGE/ >> /home/$USER/mvp/.env"
- ssh -o StrictHostKeyChecking=no $USER@$STG_DOCKER "echo -n $CI_BUILD_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY"
# Set up CI/CD variable RESET_VOLUMES to 1 to remove the volumes of the MVP (to reinitiate MVP from scratch)
- |
if [ "$RESET_VOLUMES" == "1" ]; then
ssh -o StrictHostKeyChecking=no $USER@$STG_DOCKER "cd /home/$USER/mvp && docker-compose down --remove-orphans --volumes"
else
ssh -o StrictHostKeyChecking=no $USER@$STG_DOCKER "cd /home/$USER/mvp && docker-compose down --remove-orphans"
fi
- ssh -o StrictHostKeyChecking=no $USER@$STG_DOCKER "docker rmi -f \$(docker images --filter reference="$CI_REGISTRY_IMAGE/*" -q) || true"
- ssh -o StrictHostKeyChecking=no $USER@$STG_DOCKER "cd /home/$USER/mvp && docker-compose up -d"
rules:
- if: '$CI_COMMIT_BRANCH == "staging"'
tags:
- docker
needs:
- build-mvp-keycloak-stg
- build-mvp-proxy-stg
########################################################################################################################
#
# Testing phase elements
#
########################################################################################################################
# SAST using SonarQube
sonar-scan-stg:
image: sonarsource/sonar-scanner-cli:4.6
stage: sast
script:
- sonar-scanner -Dsonar.host.url=$SONARQUBE_URL -Dsonar.login=$SONARQUBE_TOKEN
allow_failure: true
rules:
- if: '$CI_COMMIT_BRANCH == "staging"'
tags:
- docker
# CVA using Trivy
# mvp customization of Trivy - to be used for all images
.cva-mvp-template-stg:
extends: .template_cva_trivy_informative
stage: cva
rules:
- if: '$CI_COMMIT_BRANCH == "staging"'
# CVA - scanning all images for vulnerabilities (informative) using Trivy
cva-mvp-keycloak-postgresql-stg:
extends: .cva-mvp-template-stg
variables:
IMAGE_NAME: mvp-keycloak-postgresql
IMAGE_TAG: staging
needs:
- build-mvp-keycloak-postgresql-stg
cva-mvp-keycloak-stg:
extends: .cva-mvp-template-stg
variables:
IMAGE_NAME: mvp-keycloak
IMAGE_TAG: staging
needs:
- build-mvp-keycloak-stg
cva-mvp-proxy-stg:
extends: .cva-mvp-template-stg
variables:
IMAGE_NAME: mvp-proxy
IMAGE_TAG: staging
needs:
- build-mvp-proxy-stg
# OWASP ZAP DAST - baseline scan
# ZAP is performing baseline scan, fast scan, not doing any advanced attacs (more here: https://www.zaproxy.org/docs/docker/baseline-scan/)
zap-mvp-baseline:
extends: .template_dast_zap_baseline
# Specify the pipeline stage which DAST shall be run in.
# It is recommended to define a dedicated stage for DAST scanning (not named dast, as GitLab already use such stage globally)
stage: dasttest
variables:
# Specify the web application URL the scanning should start with
ZAP_URL: https://dragonegg-stg.ilab.zone
rules:
# Specify which pipeline shall this job be fired in, by indicating the rule on branch.
- if: '$CI_COMMIT_BRANCH == "staging"'
tags:
- docker
needs:
# Make the dependency on the job deploying the application - DAST needs working application
- deploy-stg
# OWASP ZAP DAST - full scan
# ZAP is performing full scan, first passive spidering, than predefined attacs (more here: https://www.zaproxy.org/docs/docker/full-scan/)
# WARNING!!! This task could run very long and may damage the application, as actual attacks are run by ZAP - be careful!
#zap-mvp-full:
# extends: .template_dast_zap_fullscan
# # Specify the pipeline stage which DAST shall be run in.
# # It is recommended to define a dedicated stage for DAST scanning (not named dast, as GitLab already use such stage globally)
# stage: dasttest
# variables:
# # Specify the web application URL the scanning should start with
# ZAP_URL: https://dragonegg-stg.ilab.zone
# rules:
# # Specify which pipeline shall this job be fired in, by indicating the rule on branch.
# - if: '$CI_COMMIT_BRANCH == "staging"'
# tags:
# - docker
# needs:
# # Make the dependency on the job deploying the application - DAST needs working application
# - deploy-stg
# OWASP ZAP DAST - API scan
# ZAP is performing scans against APIs defined by OpenAPI, SOAP, or GraphQL via URL (more here: https://www.zaproxy.org/docs/docker/api-scan/).
#zap-mvp-backend-api-scan:
# extends: .template_dast_zap_apiscan
# # Specify the pipeline stage which DAST shall be run in.
# # It is recommended to define a dedicated stage for DAST scanning (not named dast, as GitLab already use such stage globally)
# stage: dasttest
# variables:
# # One of: openapi, soap, graphql
# ZAP_API_FORMAT: openapi
# # URL pointing to OpenAPI specification of the API
# ZAP_URL: https://dragonegg-dev.ilab.zone/api/api-docs
# rules:
# - if: '$CI_COMMIT_BRANCH == "staging"'
# tags:
# - docker
# needs:
# # Make the dependency on the job deploying the application - DAST needs working application
# - deploy-stg
# Dependencies security scan (Gemnasium)
.gemnasium-mvp-template:
extends: .template-gemnasium-json
# Stage where the test will be run (if the test fails the next stage will not be executed)
stage: test
# Specify which pipeline shall this job be fired in, by indicating the rule on branch.
rules:
- if: '$CI_COMMIT_BRANCH == "staging"'
tags:
- docker
gemnasium-dependency-scanning-frontend:
extends: .gemnasium-mvp-template
variables:
#directory to be scanned against vulnerability database by exposed analyzer
DIRECTORY_TO_SCAN: "mvp-frontend/"
gemnasium-dependency-scanning-backend:
extends: .gemnasium-mvp-template
variables:
#directory to be scanned against vulnerability database by exposed analyzer
DIRECTORY_TO_SCAN: "mvp-backend/"

50
cicd/pipeline.yml Normal file
View file

@ -0,0 +1,50 @@
########################################################################################################################
#
# Main entry to the GitLab Ci/CD pipelines configuration:
# 1. Configures the stages.
# 2. Configures pipeline running conditions.
# 3. Configures variables to be used globally in all configuration files.
#
########################################################################################################################
include:
# Trivy template - to be used for all images
- project: 'ihlc/teams/team-nightwatch/servicestream/configuration-templates'
ref: master
file: 'cicd/template-cva-trivy.yml'
# ZAP
- project: 'ihlc/teams/team-nightwatch/servicestream/configuration-templates'
ref: master
file: 'cicd/template-dast-zap.yml'
# Gemnasium
- project: 'ihlc/teams/team-nightwatch/servicestream/configuration-templates'
ref: master
file: 'cicd/template-sca-gemnasium.yml'
- cicd/pipeline-common.yml
- cicd/pipeline-dev.yml
- cicd/pipeline-stg.yml
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web"'
stages:
- compile
- sast
- build
- deploy
- cva
- apitest
- dasttest
- test
- cleanup
variables:
USER: developer
DEV_SSH_KEY: $DEV_SSH_KEY_VALUE
STG_SSH_KEY: $STG_SSH_KEY_VALUE
DEV_DOCKER: DEV_DOCKER_VALUE
STG_DOCKER: STG_DOCKER_VALUE
SONARQUBE_URL: 'https://sonarqube.ilab.zone/'
SONARQUBE_TOKEN: $SONARQUBE_MASTER_TOKEN

View file

@ -0,0 +1,9 @@
########################################################################################################################
#
# Image configuration for either 'docker build' or 'docker compose' build step:
# 1. Used to set up the image.
# 2. Dedicated to IL DEVELOPMENT environment (environments may vary).
#
########################################################################################################################
FROM postgres:14.0

View file

@ -0,0 +1,9 @@
########################################################################################################################
#
# Image configuration for either 'docker build' or 'docker compose' build step:
# 1. Used to set up the image.
# 2. Dedicated to local developer environment (hosted in own workstation).
#
########################################################################################################################
FROM postgres:14.0

View file

@ -0,0 +1,9 @@
########################################################################################################################
#
# Image configuration for either 'docker build' or 'docker compose' build step:
# 1. Used to set up the image.
# 2. Dedicated to IL STAGING environment (environments may vary).
#
########################################################################################################################
FROM postgres:14.0

View file

@ -0,0 +1,12 @@
########################################################################################################################
#
# Image configuration for either 'docker build' or 'docker compose' build step:
# 1. Used to set up the image.
# 2. Dedicated to IHLC DEVELOPMENT environment (environments may vary).
#
########################################################################################################################
FROM quay.io/keycloak/keycloak:14.0.0
COPY ./configuration/realms/ /opt/jboss/keycloak/realm-config
USER jboss
EXPOSE ${KEYCLOAK_INTERNAL_PORT}

View file

@ -0,0 +1,12 @@
########################################################################################################################
#
# Image configuration for either 'docker build' or 'docker compose' build step:
# 1. Used to set up the image.
# 2. Dedicated to local developer environment (hosted in own workstation).
#
########################################################################################################################
FROM quay.io/keycloak/keycloak:14.0.0
COPY ./configuration/realms/ /opt/jboss/keycloak/realm-config
USER jboss
EXPOSE ${KEYCLOAK_INTERNAL_PORT}

View file

@ -0,0 +1,12 @@
########################################################################################################################
#
# Image configuration for either 'docker build' or 'docker compose' build step:
# 1. Used to set up the image.
# 2. Dedicated to IHLC STAGING environment (environments may vary).
#
########################################################################################################################
FROM quay.io/keycloak/keycloak:14.0.0
COPY ./configuration/realms/ /opt/jboss/keycloak/realm-config
USER jboss
EXPOSE ${KEYCLOAK_INTERNAL_PORT}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,97 @@
{
"realm" : "dragon-egg-green",
"users" : [ {
"id" : "06eb23e8-abbc-48a8-a25d-e40b02319adf",
"createdTimestamp" : 1636751673501,
"username" : "admin",
"enabled" : true,
"totp" : false,
"emailVerified" : true,
"firstName" : "Administrator",
"lastName" : "DragonEgg",
"email" : "admin@dragonegg.com",
"credentials" : [ {
"id" : "caccbbd9-9289-49ea-8e58-e9ae16a0e3ce",
"type" : "password",
"createdDate" : 1636751688980,
"secretData" : "{\"value\":\"5XxCCIB0x9fOFyZsciYhl5xB1WZVrBIBYQGygVhT70Sy3Ec29Aynt58p61eeb62aKOOyq6T0jyGE7eLVPshN4g==\",\"salt\":\"nKWaOjNTxf+UDTf9gW1+0A==\",\"additionalParameters\":{}}",
"credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}"
} ],
"disableableCredentialTypes" : [ ],
"requiredActions" : [ ],
"realmRoles" : [ "dragon-egg-green-administrator", "default-roles-dragon-egg-green" ],
"notBefore" : 0,
"groups" : [ ]
}, {
"id" : "9498963a-2259-49c0-b335-b1da210ea1b7",
"createdTimestamp" : 1637002082543,
"username" : "poweruser",
"enabled" : true,
"totp" : false,
"emailVerified" : true,
"firstName" : "Poweruser",
"lastName" : "DragonEgg",
"email" : "poweruser@dragonegg.com",
"credentials" : [ {
"id" : "af765cdf-a77f-4cef-b3f8-e51b1a7f3322",
"type" : "password",
"createdDate" : 1637012781513,
"secretData" : "{\"value\":\"ieYQlWaHzJKXUKHUfRozksaemZ+WkxZZcnwGeRrPoK1I5kHawbt49gK2K6EqOeIyGZ96aw2Bdl4T2owT4BHNpQ==\",\"salt\":\"kK1jb7nzQHL2u1uKFYSomQ==\",\"additionalParameters\":{}}",
"credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}"
} ],
"disableableCredentialTypes" : [ ],
"requiredActions" : [ ],
"realmRoles" : [ "dragon-egg-green-power-user", "default-roles-dragon-egg-green" ],
"clientConsents" : [ {
"clientId" : "dragon-egg-green",
"grantedClientScopes" : [ "dragon-egg-green", "roles", "email", "profile" ],
"createdDate" : 1637831018182,
"lastUpdatedDate" : 1637831018185
} ],
"notBefore" : 0,
"groups" : [ ]
}, {
"id" : "573980f1-2023-460c-9c12-8e655c1cc4ee",
"createdTimestamp" : 1637934396613,
"username" : "unprivileged",
"enabled" : true,
"totp" : false,
"emailVerified" : true,
"firstName" : "Unprivileged",
"lastName" : "DragonEgg",
"email" : "unprivileged@unprivileged.com",
"credentials" : [ {
"id" : "2e46ca41-3891-4acc-9650-893170df3a4f",
"type" : "password",
"createdDate" : 1637934413610,
"secretData" : "{\"value\":\"a2paWwcF0rTvftY89QFsKbRmml516632AACSntmWGGfHMO0ugR9WknkXK76vQbieKG5gmpiQahhP+e0/0WihtQ==\",\"salt\":\"Y/PL+0+0xmRhfUrpXeT2zg==\",\"additionalParameters\":{}}",
"credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}"
} ],
"disableableCredentialTypes" : [ ],
"requiredActions" : [ ],
"notBefore" : 0,
"groups" : [ ]
}, {
"id" : "c3a5ffd6-715c-46f9-820b-b6c1af6a4c5c",
"createdTimestamp" : 1636985060581,
"username" : "user",
"enabled" : true,
"totp" : false,
"emailVerified" : true,
"firstName" : "User",
"lastName" : "DragonEgg",
"email" : "user@dragonegg.com",
"credentials" : [ {
"id" : "9536076f-6892-4dba-9138-78f762e0fb16",
"type" : "password",
"createdDate" : 1636985072883,
"secretData" : "{\"value\":\"cvjVTg6iMlhzp+mKwCHEqPgz69wfYYxFHTIyzBWS7FQmjFJq9o52PdMvF+RD+WALC+ODyQpHCyamnzsw1TF+dw==\",\"salt\":\"L9GG7/hPuV+uWJrtWLOg7w==\",\"additionalParameters\":{}}",
"credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}"
} ],
"disableableCredentialTypes" : [ ],
"requiredActions" : [ ],
"realmRoles" : [ "default-roles-dragon-egg-green", "dragon-egg-green-user" ],
"notBefore" : 0,
"groups" : [ ]
} ]
}

15
mvp-proxy/Dockerfile.dev Normal file
View file

@ -0,0 +1,15 @@
########################################################################################################################
#
# Image configuration for either 'docker build' or 'docker compose' build step:
# 1. Used to set up the image.
# 2. Dedicated to IL DEVELOPMENT environment (environments may vary).
# 3. See 'nginx/dev.conf' - this is an OpenResty configuration of the reverse proxy of the application.
# 4. See 'nginx/x-forwarded-for.ilab.conf' - this is an OpenResty headers configuration customized to IL environment.
#
########################################################################################################################
FROM openresty/openresty:1.19.9.1-0-alpine
COPY nginx/dev.conf /etc/nginx/conf.d/default.conf
COPY nginx/x-forwarded-for.ilab.conf /etc/nginx/conf.d/
EXPOSE ${PROXY_INTERNAL_PORT}

View file

@ -0,0 +1,16 @@
########################################################################################################################
#
# Image configuration for either 'docker build' or 'docker compose' build step:
# 1. Used to set up the image.
# 2. Dedicated to local developer environment (hosted in own workstation).
# 3. See 'nginx/local.conf' - this is an OpenResty configuration of the reverse proxy of the application.
# 4. See 'nginx/x-forwarded-for.local.conf' - this is an OpenResty headers configuration customized to local developer
# environment.
#
########################################################################################################################
FROM openresty/openresty:1.19.9.1-0-alpine
COPY nginx/local.conf /etc/nginx/conf.d/default.conf
COPY nginx/x-forwarded-for.local.conf /etc/nginx/conf.d/
EXPOSE ${PROXY_INTERNAL_PORT}

View file

@ -0,0 +1,15 @@
########################################################################################################################
#
# Image configuration for either 'docker build' or 'docker compose' build step:
# 1. Used to set up the image.
# 2. Dedicated to IL STAGING environment (environments may vary).
# 3. See 'nginx/staging.conf' - this is an OpenResty configuration of the reverse proxy of the application.
# 4. See 'nginx/x-forwarded-for.ilab.conf' - this is an OpenResty headers configuration customized to IL environment.
#
########################################################################################################################
FROM openresty/openresty:1.19.9.1-0-alpine
COPY nginx/staging.conf /etc/nginx/conf.d/default.conf
COPY nginx/x-forwarded-for.ilab.conf /etc/nginx/conf.d/
EXPOSE ${PROXY_INTERNAL_PORT}

36
mvp-proxy/nginx/dev.conf Normal file
View file

@ -0,0 +1,36 @@
server {
listen 8080;
server_name _;
add_header X-source mvp-reverse-proxy;
root /usr/share/nginx/html;
access_log /dev/stdout;
error_log /dev/stdout;
location = /health {
add_header Content-Type text/plain;
# It's not enough to add X-source at the server level as having add_header in location causes
# higher level add_headers to be ignored
add_header X-source mvp-reverse-proxy;
return 200 'mvp reverse proxy is configured and responding.';
access_log off;
}
proxy_http_version 1.1;
location /auth {
client_max_body_size 10M;
include /etc/nginx/conf.d/x-forwarded-for.ilab.conf;
proxy_pass http://mvp-keycloak-dev:8090;
}
location = / {
add_header Content-Type text/plain;
include /etc/nginx/conf.d/x-forwarded-for.ilab.conf;
# It's not enough to add X-source at the server level as having add_header in location causes
# higher level add_headers to be ignored
add_header X-source mvp-reverse-proxy;
return 200 'mvp root endpoint.';
}
}

View file

@ -0,0 +1,36 @@
server {
listen 7070;
server_name _;
add_header X-source mvp-reverse-proxy;
root /usr/share/nginx/html;
access_log /dev/stdout;
error_log /dev/stdout;
location = /health {
add_header Content-Type text/plain;
# It's not enough to add X-source at the server level as having add_header in location causes
# higher level add_headers to be ignored
add_header X-source mvp-reverse-proxy;
return 200 'mvp reverse proxy is configured and responding.';
access_log off;
}
proxy_http_version 1.1;
location /auth {
client_max_body_size 10M;
include /etc/nginx/conf.d/x-forwarded-for.local.conf;
proxy_pass http://mvp-keycloak-local:8090;
}
location = / {
add_header Content-Type text/plain;
include /etc/nginx/conf.d/x-forwarded-for.local.conf;
# It's not enough to add X-source at the server level as having add_header in location causes
# higher level add_headers to be ignored
add_header X-source mvp-reverse-proxy;
return 200 'mvp root endpoint.';
}
}

View file

@ -0,0 +1,36 @@
server {
listen 8080;
server_name _;
add_header X-source mvp-reverse-proxy;
root /usr/share/nginx/html;
access_log /dev/stdout;
error_log /dev/stdout;
location = /health {
add_header Content-Type text/plain;
# It's not enough to add X-source at the server level as having add_header in location causes
# higher level add_headers to be ignored
add_header X-source mvp-reverse-proxy;
return 200 'mvp reverse proxy is configured and responding.';
access_log off;
}
proxy_http_version 1.1;
location /auth {
client_max_body_size 10M;
include /etc/nginx/conf.d/x-forwarded-for.ilab.conf;
proxy_pass http://mvp-keycloak-staging:8090;
}
location = / {
add_header Content-Type text/plain;
include /etc/nginx/conf.d/x-forwarded-for.ilab.conf;
# It's not enough to add X-source at the server level as having add_header in location causes
# higher level add_headers to be ignored
add_header X-source mvp-reverse-proxy;
return 200 'mvp root endpoint.';
}
}

View file

@ -0,0 +1,19 @@
# To properly handle Cors (using ForwardedHeaderFilter) backend need to have the following headers set properly:
# - X-Forwarded-Proto
# - X-Forwarded-Host
# - X-Forwarded-Port
# When this proxy is run behind AWS ELB X-Forwarded-Proto and X-Forwarded-Port are already set. Additionally
# header Host is set properly In that case. So it is enough to have:
proxy_set_header X-Forwarded-Host $host;
# If this proxy is run not behind AWS ELB but receiving requests directly from clients, all three headers listed
# above would have to be set like this:
#proxy_set_header X-Forwarded-Proto $scheme;
#proxy_set_header X-Forwarded-Host $host;
#proxy_set_header X-Forwarded-Port $server_port;
# Removing unnecessary headers as recommended by
# https://docs.spring.io/spring-framework/docs/5.2.6.RELEASE/spring-framework-reference/web.html#filters-forwarded-headers
proxy_set_header Forwarded "";
proxy_set_header X-Forwarded-Prefix "";
proxy_set_header X-Forwarded-Ssl "";

View file

@ -0,0 +1,19 @@
# To properly handle Cors (using ForwardedHeaderFilter) backend need to have the following headers set properly:
# - X-Forwarded-Proto
# - X-Forwarded-Host
# - X-Forwarded-Port
# When this proxy is run behind AWS ELB X-Forwarded-Proto and X-Forwarded-Port are already set. Additionally
# header Host is set properly In that case. So it is enough to have:
# proxy_set_header X-Forwarded-Host $host;
# If this proxy is run not behind AWS ELB but receiving requests directly from clients, all three headers listed
# above would have to be set like this:
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Port $server_port;
# Removing unnecessary headers as recommended by
# https://docs.spring.io/spring-framework/docs/5.2.6.RELEASE/spring-framework-reference/web.html#filters-forwarded-headers
proxy_set_header Forwarded "";
proxy_set_header X-Forwarded-Prefix "";
proxy_set_header X-Forwarded-Ssl "";

7
sonar-project.properties Normal file
View file

@ -0,0 +1,7 @@
sonar.projectKey=mvp
sonar.projectName=mvp
sonar.projectVersion=1.0.0
sonar.sourceEncoding=UTF-8
sonar.sources=.