Add CI Build (#2)

Co-Authored-By: Marco Massari Calderone <marco@marcomc.com>

.ci/.gitignore

@@ -0,0 +1 @@
+tmp-test-*

.ci/Jenkinsfile

@@ -0,0 +1,65 @@
+pipeline {
+    agent {
+        docker {
+            label 'my127ws'
+            alwaysPull true
+            image 'quay.io/inviqa_images/workspace:latest'
+            args '--entrypoint "" --volume /var/run/docker.sock:/var/run/docker.sock --volume "$HOME/.my127:/root/.my127"'
+        }
+    }
+    environment {
+        COMPOSE_DOCKER_CLI_BUILD = 1
+        DOCKER_BUILDKIT = 1
+        MY127WS_KEY = credentials('base-my127ws-key-20190523')
+    }
+    options {
+        buildDiscarder(logRotator(daysToKeepStr: '30'))
+    }
+    triggers { cron(env.BRANCH_NAME ==~ /^(master|main|\d+\.\d+\.x)$/ ? 'H H(0-6) * * *' : '') }
+    stages {
+        stage('Quality Checks') {
+            steps {
+                sh '.ci/quality'
+                milestone(10)
+            }
+        }
+        stage('Quality Tests') {
+            environment {
+                TEARDOWN_ENVIRONMENT = "no"
+                TEST_MODE = "quality"
+            }
+            steps {
+                sh '.ci/test static'
+                sh '.ci/test dynamic'
+                sh '.ci/test dynamic mutagen'
+                sh '.ci/test static-all-services'
+                milestone(20)
+            }
+        }
+        stage('Acceptance Tests') {
+            environment {
+                REUSE_EXISTING_WORKSPACE = "yes"
+                TEST_MODE = "acceptance"
+            }
+            stages {
+                stage('Static') {
+                    steps { sh '.ci/test static' }
+                }
+                stage('Dynamic') {
+                    steps { sh '.ci/test dynamic' }
+                }
+                stage('Mutagen') {
+                    steps { sh '.ci/test dynamic mutagen' }
+                }
+                stage('Static All Services') {
+                    steps { sh '.ci/test static-all-services' }
+                }
+            }
+        }
+    }
+    post {
+        always {
+            cleanWs()
+        }
+    }
+}

.ci/quality

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e -o pipefail
+
+ANALYSE_PATH="${1-}"
+
+if ! docker image ls -a --format '{{ print .Repository ":" .Tag }}' | grep -q koalaman/shellcheck-alpine:stable; then
+  docker pull koalaman/shellcheck-alpine:stable
+fi
+if ! docker image ls -a --format '{{ print .Repository ":" .Tag }}' | grep -q hadolint/hadolint:latest; then
+  docker pull hadolint/hadolint:latest-alpine
+fi
+
+run_shellcheck()
+{
+  docker run --rm --volume "$(pwd):/app" koalaman/shellcheck-alpine:stable /bin/sh -c "find '/app/$ANALYSE_PATH' -type f ! -path '*/.git/*' ! -path '/app/tmp-test/*' ! -name '*.twig' -and \( -name "*.sh" -or -perm -0111 \) -print -exec shellcheck --exclude=SC1008,SC1091 {} +"
+}
+
+run_hadolint()
+{
+  docker run --rm --volume "$(pwd):/app" hadolint/hadolint:latest-alpine /bin/sh -c "find '/app/$ANALYSE_PATH' -type f -name '*Dockerfile' ! -path '*/.git/*' ! -path '/app/tmp-test/*' ! -path '/app/test/*' ! -name '*.orig' -print -exec hadolint --ignore DL3003 --ignore DL3008 --ignore DL3016 --ignore SC1091 --ignore DL3002 {} +"
+}
+
+run_shellcheck
+run_hadolint

.ci/sample-dynamic/harnesses.json

@@ -0,0 +1,6 @@
+{
+  "inviqa/docker": {
+    "vx.x.x": {
+    }
+  }
+}

.ci/sample-dynamic/workspace.yml

@@ -2,6 +2,8 @@ workspace('ci-docker-sample-dynamic'):
   description: generated local workspace for ci-docker-sample.
   harness: inviqa/docker
 
+harness.repository.source('name'): ./harnesses.json
+
 attributes:
   services:
     chrome:

.ci/sample-static-all-services/harnesses.json

@@ -0,0 +1,6 @@
+{
+  "inviqa/docker": {
+    "vx.x.x": {
+    }
+  }
+}

.ci/sample-static-all-services/workspace.yml

@@ -0,0 +1,44 @@
+workspace('ci-docker-sample-static-all-services'):
+  description: generated local workspace for ci-docker-sample.
+  harness: inviqa/docker
+
+harness.repository.source('name'): ./harnesses.json
+
+attribute('app.build'): static
+
+attributes:
+  services:
+    chrome:
+      enabled: true
+    elasticsearch:
+      enabled: true
+    memcached:
+      enabled: true
+    mongodb:
+      enabled: true
+    mysql:
+      enabled: true
+    postgres:
+      enabled: true
+    rabbitmq:
+      enabled: true
+    redis-session:
+      enabled: true
+    redis:
+      enabled: true
+    solr:
+      enabled: true
+    varnish:
+      enabled: true
+
+  aws:
+    repository: null
+    access_key_id: null
+    secret_access_key: null
+    s3:
+      bucket: null
+
+attribute('mysql.tag'): 8
+attribute('database.platform_version'): 8.0
+
+attribute('varnish.target_service'): elasticsearch

.ci/sample-static/harnesses.json

@@ -0,0 +1,6 @@
+{
+  "inviqa/docker": {
+    "vx.x.x": {
+    }
+  }
+}

.ci/sample-static/workspace.yml

@@ -2,6 +2,8 @@ workspace('ci-docker-sample-static'):
   description: generated local workspace for ci-docker-sample.
   harness: inviqa/docker
 
+harness.repository.source('name'): ./harnesses.json
+
 attribute('app.build'): static
 
 attributes:

.ci/test

@@ -0,0 +1,169 @@
+#!/bin/bash
+
+set -e
+set -x
+
+export path_test=".ci/tmp-test"
+
+export TEST_MODE="${TEST_MODE:-full}"
+export REUSE_EXISTING_WORKSPACE="${REUSE_EXISTING_WORKSPACE:-no}"
+export TEARDOWN_ENVIRONMENT="${TEARDOWN_ENVIRONMENT:-yes}"
+
+function main()
+{
+    local mode="$1"
+    local sync="${2:-}"
+
+    local sync_path=""
+    if [[ -n "$sync" ]]; then
+      sync_path="-${sync}"
+    fi
+    export path_test=".ci/tmp-test-${mode}${sync_path}"
+
+    if [[ "$TEARDOWN_ENVIRONMENT" == "yes" ]]; then
+      trap 'clean' EXIT
+    fi
+
+    if [[ "$mode" == "static" ]]; then
+      export MY127WS_ENV=pipeline
+    fi
+
+    if [[ "$REUSE_EXISTING_WORKSPACE" != "yes" ]]; then
+      # Test default mode of static or dynamic harnesses
+      # For dynamic this means a mountpoint for the application code
+      # or mutagen
+      setup "$mode"
+      if [[ "$sync" == "" ]]; then
+        setup_dynamic_mountpoint
+      elif [[ "$sync" == "mutagen" ]]; then
+        setup_dynamic_mutagen
+      fi
+      prepare_environment
+    fi
+
+    if [[ "$TEST_MODE" == "quality" || "$TEST_MODE" == "full" ]]; then
+      test_harness_quality "$mode" "$sync"
+    fi
+    if [[ "$TEST_MODE" == "acceptance" || "$TEST_MODE" == "full"  ]]; then
+      test_harness_acceptance "$mode" "$sync"
+    fi
+    if [[ "$TEARDOWN_ENVIRONMENT" == "yes" ]]; then
+      teardown
+    fi
+}
+
+function test_harness_quality()
+{
+    .ci/quality "${path_test}/.my127ws"
+}
+
+function test_harness_acceptance()
+(
+    local mode="$1"
+    local sync="$2"
+
+    install_console
+    run_project_quality_tests
+    install_environment
+    check_environment_started "$mode"
+    run_project_acceptance_tests
+    restart_environment
+    check_environment_started "$mode"
+)
+
+function setup()
+(
+    local mode="$1"
+
+    if [ -d "${path_test}" ]; then
+        rm -rf "${path_test}"
+    fi
+
+    cp -ap ".ci/sample-${mode}" "${path_test}"
+    rsync --archive --exclude="**/.git" --exclude="**/.ci" ./ "${path_test}/.my127ws/"
+)
+
+function prepare_environment()
+(
+    cd "${path_test}"
+    ws harness prepare
+)
+
+function install_console()
+(
+    cd "${path_test}"
+    ws enable console
+)
+
+function install_environment()
+(
+    cd "${path_test}"
+    ws install
+)
+
+function check_environment_started()
+(
+    local mode="$1"
+
+    cd "${path_test}"
+
+    project="ci-sample-${mode}"
+    if [[ "$mode" == "static" ]]; then
+      project="$(git log -n 1 --pretty=format:'%H')"
+    fi
+
+    if [ "$(docker-compose -p "$project" ps | grep -v "lighthouse" | grep -c Exit)" -gt 0 ]; then
+      echo 'Some containers failed to start'
+      docker-compose -p "$project" ps
+      return 1
+    fi
+)
+
+function run_project_quality_tests()
+(
+    cd "${path_test}"
+
+    ws helm kubeval --cleanup app
+)
+
+function run_project_acceptance_tests()
+(
+    cd "${path_test}"
+)
+
+function restart_environment()
+(
+    cd "${path_test}"
+    ws disable
+    ws enable
+)
+
+function setup_dynamic_mountpoint()
+(
+    cd "${path_test}"
+    echo "attribute('host.os'): linux" > workspace.override.yml
+)
+
+function setup_dynamic_mutagen()
+(
+    cd "${path_test}"
+    echo "attribute('host.os'): darwin" > workspace.override.yml
+    echo "attribute('mutagen'): yes" >> workspace.override.yml
+)
+
+function teardown()
+(
+    cd "${path_test}"
+    ws destroy || (docker ps -a && return 1)
+    rm -rf .my127ws
+)
+
+function clean()
+{
+    if [ -d "$path_test" ]; then
+        (cd "$path_test" && ws destroy) || (docker ps -a && return 1)
+        rm -rf "$path_test"
+    fi
+}
+
+main "$@"

_twig/docker-compose.yml/service/varnish.yml.twig

@@ -14,7 +14,7 @@
       @('services.varnish.environment_secrets')
       ]), 2, 6) | raw }}
     links:
-      - nginx:nginx
+      - {{ @('varnish.target_service') }}:{{ @('varnish.target_service') }}
     volumes:
       - .my127ws/docker/image/varnish/root/etc/varnish/default.vcl:/etc/varnish/default.vcl:ro
       - type: tmpfs

application/overlay/_twig/Jenkinsfile/test.twig

@@ -5,6 +5,6 @@
                 stage('lighthouse') { steps { sh 'ws lighthouse' } }
 {% endif %}
 {% endif %}
-                stage('helm kubeval qa')  { steps { sh 'ws helm kubeval --cleanup qa' } }
+                stage('helm kubeval app')  { steps { sh 'ws helm kubeval --cleanup app' } }
             }
         }

docker/image/console/Dockerfile.twig

@@ -2,7 +2,12 @@ FROM {{ @('services.console.build.from') }}
 
 COPY .my127ws/docker/image/console/root /
 RUN useradd build \
- && apt-get update && apt-get install -y rsync \
+ && mkdir -p /app \
+ && chown -R build:build /home/build /app \
+ && apt-get update && apt-get install --no-install-recommends -y ca-certificates curl rsync \
+ && apt-get auto-remove -qq -y \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* \
  && ([ -e /sbin/tini ] || curl --fail --silent --show-error --location --output /sbin/tini https://github.com/krallin/tini/releases/download/v0.19.0/tini) \
  && chmod +x /sbin/tini
 

docker/image/lighthouse/Dockerfile.twig

@@ -1,3 +1,4 @@
+# hadolint ignore=DL3007
 FROM {{ @('services.lighthouse.image') }}
 
 COPY root /

harness/attributes/common.yml

@@ -97,7 +97,7 @@ attributes.default:
       enabled: true
     compose:
       file_version: ~
-      host_volume_options: "= ':' ~ @('delegated-volumes') ? 'delegated' : 'cached')"
+      host_volume_options: "= ':' ~ (@('delegated-volumes') ? 'delegated' : 'cached')"
     # If using gitops helm chart repositories, it's recommended to put this configuration
     # in the values.yml in there now instead of project application repositories
     image_pull_config: = @('docker.config')