From 10569470e5126342bda0adec3e7a227668f6af2e Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Thu, 30 Sep 2021 17:03:49 +0100 Subject: [PATCH 1/3] feat(salt): use `pillars_from_directories` & `test/salt/pillar/top.sls` --- ssf/defaults.yaml | 16 +--- ssf/formulas.yaml | 203 ++++++++++++++++------------------------------ 2 files changed, 72 insertions(+), 147 deletions(-) diff --git a/ssf/defaults.yaml b/ssf/defaults.yaml index e1c9efb4..bbc29a59 100644 --- a/ssf/defaults.yaml +++ b/ssf/defaults.yaml @@ -1387,21 +1387,7 @@ ssf: rspamd: *formula_default rstudio: *formula_default rundeck: *formula_default - salt: - <<: *formula_default - context: - <<: *context_default - inspec_suites_kitchen: - <<: *isk_default - 0: - <<: *isk_suite_default - name: 'v3003-py3' - 1: - <<: *isk_suite_default - name: 'v3002-py3' - 2: - <<: *isk_suite_default - name: 'v3001-py3' + salt: *formula_default splunkforwarder: *formula_default sqldeveloper: *formula_default sqlplus: *formula_default diff --git a/ssf/formulas.yaml b/ssf/formulas.yaml index 9cd8977c..9bd6fffb 100644 --- a/ssf/formulas.yaml +++ b/ssf/formulas.yaml @@ -49,12 +49,6 @@ ssf_node_anchors: path: './test/salt' - name: 'states' path: './test/salt' - state_top_salt: &state_top_salt - - '*': - - ._mapdata - - .pkgrepo - - .master - - .minion verifier: &verifier_inspec_tests_default inspec_tests: - default @@ -4338,156 +4332,101 @@ ssf: repo: 'salt-formula' inspec_suites_kitchen: 0: - includes: - # [os , os_ver, salt_ver, py_ver] - - [0 , 0 , 3003.3, 3] - - [freebsd , 0 , 3003.1, 3] - - [windows , 0 , latest, 3] inspec_yml: summary: >- - Verify that Salt `v3003-py3` is setup and configured - provisioner: - pillars: - - '*': - - . - - v3003-py3 - pillars_from_files: - - .sls: 'test/salt/pillar/salt.sls' - - v3003-py3.sls: 'test/salt/pillar/v3003-py3.sls' - state_top: *state_top_salt - 1: - includes: - # [os , os_ver, salt_ver, py_ver] - - [debian , 0 , 3002.7, 3] - - [ubuntu , 0 , 3002.7, 3] - - [centos , 0 , 3002.7, 3] - # - [fedora , 0 , 3002.7, 3] - # - [opensuse/leap, 0 , 3002.7, 3] - # - [opensuse/tmbl, 0 , 3002.7, 3] - - [amazonlinux , 0 , 3002.7, 3] - - [oraclelinux , 0 , 3002.7, 3] - # - [arch-base , 0 , 3002.7, 3] - - [gentoo/stage3, 0 , 3002.7, 3] - - [almalinux , 0 , 3002.7, 3] - - [rockylinux , 0 , 3002.7, 3] - - [openbsd , 0 , 3002.6, 3] - inspec_yml: - summary: >- - Verify that Salt `v3002-py3` is setup and configured - provisioner: - pillars: - - '*': - - . - - v3002-py3 - pillars_from_files: - - .sls: 'test/salt/pillar/salt.sls' - - v3002-py3.sls: 'test/salt/pillar/v3002-py3.sls' - state_top: *state_top_salt - 2: - includes: - # [os , os_ver, salt_ver, py_ver] - - [debian , 0 , 3001.8, 3] - - [ubuntu , 0 , 3001.8, 3] - - [centos , 0 , 3001.8, 3] - # - [fedora , 0 , 3001.8, 3] - # - [opensuse/leap, 0 , 3001.8, 3] - # - [opensuse/tmbl, 0 , 3001.8, 3] - - [amazonlinux , 0 , 3001.8, 3] - - [oraclelinux , 0 , 3001.8, 3] - # - [arch-base , 0 , 3001.8, 3] - # - [gentoo/stage3, 0 , 3001.8, 3] - - [almalinux , 0 , 3001.8, 3] - - [rockylinux , 0 , 3001.8, 3] - - [openbsd , 0 , 3001.1, 3] - inspec_yml: - summary: >- - Verify that Salt `v3001-py3` is setup and configured + Verify that Salt is setup and configured provisioner: - pillars: + state_top: - '*': - - . - - v3001-py3 - pillars_from_files: - - .sls: 'test/salt/pillar/salt.sls' - - v3001-py3.sls: 'test/salt/pillar/v3001-py3.sls' - state_top: *state_top_salt - inspec_suites_matrix: - - v3003-py3 - - v3002-py3 - - v3001-py3 + - ._mapdata + - .pkgrepo + - .master + - .minion + kitchen: + provisioner: + pillars_from_directories: + - test/salt/pillar + top_sls: + - '*': + - salt + - 'G@saltversioninfo:0:3003 and G@pythonversion:0:3': + - v3003-py3 + - 'G@saltversioninfo:0:3002 and G@pythonversion:0:3': + - v3002-py3 + - 'G@saltversioninfo:0:3001 and G@pythonversion:0:3': + - v3001-py3 map_jinja: verification: import: ['salt_settings', 'formulas_settings'] platforms_matrix: # [os , os_ver, salt_ver, py_ver, inspec_suite] + ### `v3004-py3` + # # `arch` unavailable below since only installs `3004.X` + # # This doesn't do anything right now until the `3004.0` pre-salted + # # images are pushed to the formula + - [arch-base , latest, 3004.0, 3, default] + ### `v3003-py3` - - [debian , 11 , 3003.3, 3, v3003-py3] - # - [debian , 10 , 3003.3, 3, v3003-py3] - # - [debian , 9 , 3003.3, 3, v3003-py3] - - [ubuntu , 20.04, 3003.3, 3, v3003-py3] - # - [ubuntu , 18.04, 3003.3, 3, v3003-py3] - - [centos , 8 , 3003.3, 3, v3003-py3] - # - [centos , 7 , 3003.3, 3, v3003-py3] + - [debian , 11 , 3003.3, 3, default] + # - [debian , 10 , 3003.3, 3, default] + # - [debian , 9 , 3003.3, 3, default] + - [ubuntu , 20.04, 3003.3, 3, default] + # - [ubuntu , 18.04, 3003.3, 3, default] + - [centos , 8 , 3003.3, 3, default] + # - [centos , 7 , 3003.3, 3, default] # # `fedora` unavailable below since only installs `3003.X` - - [fedora , 34 , 3003.3, 3, v3003-py3] - - [fedora , 33 , 3003.3, 3, v3003-py3] + - [fedora , 34 , 3003.3, 3, default] + - [fedora , 33 , 3003.3, 3, default] # # `opensuse/*` unavailable below since only installs `3003.X` # # Can't use both `15.3` & `15.2` together within the same # # InSpec suite when there's a shared `_mapdata` verification file -- # # using the currently supported version for now (re: Salt repo) - - [opensuse/leap, 15.3 , 3003.3, 3, v3003-py3] - # - [opensuse/leap, 15.2 , 3003.3, 3, v3003-py3] - # # Was last working properly with `3002.2`, hitting a problem with `3003.3`: - # # `'service' __virtual__ returned False: No service execution module loaded` - # # Actually still works if running `kitchen converge` a second time; - # # thought it might work with the `retry_options` used for FreeBSD - # # but it didn't -- disabling for the time being - # - [opensuse/tmbl, latest, 3003.3, 3, v3003-py3] - - [amazonlinux , 2 , 3003.3, 3, v3003-py3] - - [oraclelinux , 8 , 3003.3, 3, v3003-py3] - # - [oraclelinux , 7 , 3003.3, 3, v3003-py3] - # # `arch` unavailable below since only installs `3003.X` - - [arch-base , latest, 3003.3, 3, v3003-py3] - - [gentoo/stage3, latest, 3003.3, 3, v3003-py3] - - [gentoo/stage3, systemd, 3003.3, 3, v3003-py3] - - [almalinux , 8 , 3003.3, 3, v3003-py3] + - [opensuse/leap, 15.3 , 3003.3, 3, default] + # - [opensuse/leap, 15.2 , 3003.3, 3, default] + - [opensuse/tmbl, latest, 3003.3, 3, default] + - [amazonlinux , 2 , 3003.3, 3, default] + - [oraclelinux , 8 , 3003.3, 3, default] + # - [oraclelinux , 7 , 3003.3, 3, default] + - [gentoo/stage3, latest, 3003.3, 3, default] + - [gentoo/stage3, systemd, 3003.3, 3, default] + - [almalinux , 8 , 3003.3, 3, default] # # TODO: When supported in an official release, move this platform # # to that release (as a minimum, no other Salt versions) - - [rockylinux , 0 , 3003.3, 3, v3003-py3] + - [rockylinux , 0 , 3003.3, 3, default] # # FreeBSD and Windows won't always be in sync with the Linux # # platforms above - - [freebsd , 0 , 3003.1, 3, v3003-py3] - - [windows , 0 , latest, 3, v3003-py3] + - [freebsd , 0 , 3003.1, 3, default] + - [windows , 0 , latest, 3, default] ### `v3002-py3` - # - [debian , 11 , 3002.7, 3, v3002-py3] - - [debian , 10 , 3002.7, 3, v3002-py3] - # - [debian , 9 , 3002.7, 3, v3002-py3] - - [ubuntu , 20.04, 3002.7, 3, v3002-py3] - # - [ubuntu , 18.04, 3002.7, 3, v3002-py3] - - [centos , 8 , 3002.7, 3, v3002-py3] - # - [centos , 7 , 3002.7, 3, v3002-py3] - # - [amazonlinux , 2 , 3002.7, 3, v3002-py3] - - [oraclelinux , 8 , 3002.7, 3, v3002-py3] - # - [oraclelinux , 7 , 3002.7, 3, v3002-py3] - # - [gentoo/stage3, latest, 3002.7, 3, v3002-py3] - # - [gentoo/stage3, systemd, 3002.7, 3, v3002-py3] - - [openbsd , 0 , 3002.6, 3, v3002-py3] + # - [debian , 11 , 3002.7, 3, default] + - [debian , 10 , 3002.7, 3, default] + # - [debian , 9 , 3002.7, 3, default] + # - [ubuntu , 20.04, 3002.7, 3, default] + - [ubuntu , 18.04, 3002.7, 3, default] + # - [centos , 8 , 3002.7, 3, default] + - [centos , 7 , 3002.7, 3, default] + # - [amazonlinux , 2 , 3002.7, 3, default] + # - [oraclelinux , 8 , 3002.7, 3, default] + - [oraclelinux , 7 , 3002.7, 3, default] + # - [gentoo/stage3, latest, 3002.7, 3, default] + # - [gentoo/stage3, systemd, 3002.7, 3, default] + - [openbsd , 0 , 3002.6, 3, default] ### `v3001-py3` - # - [debian , 10 , 3001.8, 3, v3001-py3] - - [debian , 9 , 3001.8, 3, v3001-py3] - # - [ubuntu , 20.04, 3001.8, 3, v3001-py3] - - [ubuntu , 18.04, 3001.8, 3, v3001-py3] - # - [centos , 8 , 3001.8, 3, v3001-py3] - - [centos , 7 , 3001.8, 3, v3001-py3] - # - [amazonlinux , 2 , 3001.8, 3, v3001-py3] - # - [oraclelinux , 8 , 3001.8, 3, v3001-py3] - - [oraclelinux , 7 , 3001.8, 3, v3001-py3] - - [openbsd , 0 , 3001.1, 3, v3001-py3] + # - [debian , 10 , 3001.8, 3, default] + - [debian , 9 , 3001.8, 3, default] + # - [ubuntu , 20.04, 3001.8, 3, default] + # - [ubuntu , 18.04, 3001.8, 3, default] + # - [centos , 8 , 3001.8, 3, default] + # - [centos , 7 , 3001.8, 3, default] + # - [amazonlinux , 2 , 3001.8, 3, default] + # - [oraclelinux , 8 , 3001.8, 3, default] + # - [oraclelinux , 7 , 3001.8, 3, default] + # - [openbsd , 0 , 3001.1, 3, default] platforms_matrix_allow_failure: - - [debian , 11 , 3003.3, 3, v3003-py3] - - [debian , 11 , 3002.7, 3, v3002-py3] + - [debian , 11 , 3003.3, 3, default] + - [debian , 11 , 3002.7, 3, default] testing_freebsd: active: true testing_openbsd: From ea97df5ea6d0944b892e68a58a6a4477cf102213 Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Wed, 17 Nov 2021 22:08:09 +0000 Subject: [PATCH 2/3] feat(kitchen): allow configuration of `platforms.verifier.inputs` --- ssf/defaults.yaml | 5 ++++- ssf/files/default/kitchen.vagrant.yml | 12 ++++++++++++ ssf/files/default/kitchen.windows.yml | 12 ++++++++++++ ssf/files/default/kitchen.yml | 12 ++++++++++++ ssf/formulas.yaml | 10 ++++++++++ 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/ssf/defaults.yaml b/ssf/defaults.yaml index bbc29a59..7e0efb74 100644 --- a/ssf/defaults.yaml +++ b/ssf/defaults.yaml @@ -69,7 +69,7 @@ ssf_node_anchors: # `git describe --abbrev=0 --tags` # yamllint disable rule:line-length rule:quoted-strings title: "ci: use '`'pillars_from_directories'`' & '`'test/salt/pillar/top.sls'`'" - body: '* Automated using https://github.com/myii/ssf-formula/pull/383' + body: '* Automated using https://github.com/myii/ssf-formula/pull/384' # yamllint enable rule:line-length rule:quoted-strings github: owner: 'saltstack-formulas' @@ -147,6 +147,9 @@ ssf_node_anchors: kitchen: driver: run_options: {} + platforms: + verifier: + inputs: {} provisioner: # From a SaltStack Formulas perspective, `pillars_from_directories` # is mutually exclusive to `pillars` & `pillars_from_files` diff --git a/ssf/files/default/kitchen.vagrant.yml b/ssf/files/default/kitchen.vagrant.yml index abe1de73..74ae7bc9 100644 --- a/ssf/files/default/kitchen.vagrant.yml +++ b/ssf/files/default/kitchen.vagrant.yml @@ -68,4 +68,16 @@ platforms: exit 0 # yamllint enable rule:line-length {%- endif %} + {#- Centralise duplication from here and `kitchen*.yml` #} + {%- if kitchen.platforms.verifier.inputs %} + verifier: + inputs: + {%- for input_name, input_val in kitchen.platforms.verifier.inputs.items() %} + {%- if [semrel_formula, input_name] in [['salt', 'saltmajorversion']] %} + {{ input_name }}: {{ input_val.get(salt_ver, salt_ver | int) }} + {%- else %} + {{ input_name }}: {{ input_val }} + {%- endif %} + {%- endfor %} + {%- endif %} {%- endfor %} diff --git a/ssf/files/default/kitchen.windows.yml b/ssf/files/default/kitchen.windows.yml index ba52c9b6..bf489737 100644 --- a/ssf/files/default/kitchen.windows.yml +++ b/ssf/files/default/kitchen.windows.yml @@ -32,4 +32,16 @@ platforms: {%- set salt_ver = platform[2] | d('') %} {%- set py_ver = platform[3] | d('') %} - name: {{ os | replace('/', '-') }}-{{ os_ver | replace('.', '') }}-{{ salt_ver | replace('.', '-') }}-py{{ py_ver }} + {#- Centralise duplication from here and `kitchen*.yml` #} + {%- if kitchen.platforms.verifier.inputs %} + verifier: + inputs: + {%- for input_name, input_val in kitchen.platforms.verifier.inputs.items() %} + {%- if [semrel_formula, input_name] in [['salt', 'saltmajorversion']] %} + {{ input_name }}: {{ input_val.get(salt_ver, salt_ver | int) }} + {%- else %} + {{ input_name }}: {{ input_val }} + {%- endif %} + {%- endfor %} + {%- endif %} {%- endfor %} diff --git a/ssf/files/default/kitchen.yml b/ssf/files/default/kitchen.yml index ffa52376..ac6a4e89 100644 --- a/ssf/files/default/kitchen.yml +++ b/ssf/files/default/kitchen.yml @@ -264,6 +264,18 @@ platforms: transport: max_ssh_sessions: 1 {%- endif %} + {#- Centralise duplication from here and `kitchen*.yml` #} + {%- if kitchen.platforms.verifier.inputs %} + verifier: + inputs: + {%- for input_name, input_val in kitchen.platforms.verifier.inputs.items() %} + {%- if [semrel_formula, input_name] in [['salt', 'saltmajorversion']] %} + {{ input_name }}: {{ input_val.get(salt_ver, salt_ver | int) }} + {%- else %} + {{ input_name }}: {{ input_val }} + {%- endif %} + {%- endfor %} + {%- endif %} {%- endfor %} verifier: diff --git a/ssf/formulas.yaml b/ssf/formulas.yaml index 9bd6fffb..fab22735 100644 --- a/ssf/formulas.yaml +++ b/ssf/formulas.yaml @@ -4343,6 +4343,16 @@ ssf: - .master - .minion kitchen: + platforms: + verifier: + inputs: + # Using this input in a custom fashion; the sub-key is compared + # against the Salt version of the pre-salted image/box and + # translated accordingly (to the formula's current CI state) + saltmajorversion: + latest: 3003 # Only used for Windows, which isn't always up-to-date + master: 3004 + tiamat: 3004 provisioner: pillars_from_directories: - test/salt/pillar From 0c6a1d79cfa9fa97816215d1e0bf84828fadd36d Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Wed, 17 Nov 2021 22:15:49 +0000 Subject: [PATCH 3/3] refactor(salt): centralise and condense `platforms_matrix` for clarity --- ssf/formulas.yaml | 57 +++++++++++------------------------------------ 1 file changed, 13 insertions(+), 44 deletions(-) diff --git a/ssf/formulas.yaml b/ssf/formulas.yaml index fab22735..ea2294ea 100644 --- a/ssf/formulas.yaml +++ b/ssf/formulas.yaml @@ -4370,73 +4370,42 @@ ssf: import: ['salt_settings', 'formulas_settings'] platforms_matrix: # [os , os_ver, salt_ver, py_ver, inspec_suite] - ### `v3004-py3` - # # `arch` unavailable below since only installs `3004.X` - # # This doesn't do anything right now until the `3004.0` pre-salted - # # images are pushed to the formula - - [arch-base , latest, 3004.0, 3, default] - - ### `v3003-py3` - [debian , 11 , 3003.3, 3, default] - # - [debian , 10 , 3003.3, 3, default] - # - [debian , 9 , 3003.3, 3, default] + - [debian , 10 , 3002.7, 3, default] + - [debian , 9 , 3001.8, 3, default] - [ubuntu , 20.04, 3003.3, 3, default] - # - [ubuntu , 18.04, 3003.3, 3, default] + - [ubuntu , 18.04, 3002.7, 3, default] - [centos , 8 , 3003.3, 3, default] - # - [centos , 7 , 3003.3, 3, default] - # # `fedora` unavailable below since only installs `3003.X` + - [centos , 7 , 3002.7, 3, default] + # # `fedora` only installs `3003.X` - [fedora , 34 , 3003.3, 3, default] - [fedora , 33 , 3003.3, 3, default] - # # `opensuse/*` unavailable below since only installs `3003.X` + # # `opensuse/*` only installs `3003.X` # # Can't use both `15.3` & `15.2` together within the same # # InSpec suite when there's a shared `_mapdata` verification file -- # # using the currently supported version for now (re: Salt repo) - [opensuse/leap, 15.3 , 3003.3, 3, default] - # - [opensuse/leap, 15.2 , 3003.3, 3, default] - [opensuse/tmbl, latest, 3003.3, 3, default] - [amazonlinux , 2 , 3003.3, 3, default] - [oraclelinux , 8 , 3003.3, 3, default] - # - [oraclelinux , 7 , 3003.3, 3, default] + - [oraclelinux , 7 , 3002.7, 3, default] + # # `arch` only installs `3004.X` + # # This doesn't do anything right now until the `3004.0` pre-salted + # # images are pushed to the formula + - [arch-base , latest, 3004.0, 3, default] - [gentoo/stage3, latest, 3003.3, 3, default] - [gentoo/stage3, systemd, 3003.3, 3, default] - [almalinux , 8 , 3003.3, 3, default] # # TODO: When supported in an official release, move this platform # # to that release (as a minimum, no other Salt versions) - - [rockylinux , 0 , 3003.3, 3, default] + - [rockylinux , 8 , 3003.3, 3, default] # # FreeBSD and Windows won't always be in sync with the Linux # # platforms above - [freebsd , 0 , 3003.1, 3, default] - - [windows , 0 , latest, 3, default] - - ### `v3002-py3` - # - [debian , 11 , 3002.7, 3, default] - - [debian , 10 , 3002.7, 3, default] - # - [debian , 9 , 3002.7, 3, default] - # - [ubuntu , 20.04, 3002.7, 3, default] - - [ubuntu , 18.04, 3002.7, 3, default] - # - [centos , 8 , 3002.7, 3, default] - - [centos , 7 , 3002.7, 3, default] - # - [amazonlinux , 2 , 3002.7, 3, default] - # - [oraclelinux , 8 , 3002.7, 3, default] - - [oraclelinux , 7 , 3002.7, 3, default] - # - [gentoo/stage3, latest, 3002.7, 3, default] - # - [gentoo/stage3, systemd, 3002.7, 3, default] - [openbsd , 0 , 3002.6, 3, default] - - ### `v3001-py3` - # - [debian , 10 , 3001.8, 3, default] - - [debian , 9 , 3001.8, 3, default] - # - [ubuntu , 20.04, 3001.8, 3, default] - # - [ubuntu , 18.04, 3001.8, 3, default] - # - [centos , 8 , 3001.8, 3, default] - # - [centos , 7 , 3001.8, 3, default] - # - [amazonlinux , 2 , 3001.8, 3, default] - # - [oraclelinux , 8 , 3001.8, 3, default] - # - [oraclelinux , 7 , 3001.8, 3, default] - # - [openbsd , 0 , 3001.1, 3, default] + - [windows , 0 , latest, 3, default] platforms_matrix_allow_failure: - [debian , 11 , 3003.3, 3, default] - - [debian , 11 , 3002.7, 3, default] testing_freebsd: active: true testing_openbsd: