Merge remote-tracking branch 'origin/master' into openstack-caracal

.github/ISSUE_TEMPLATE/generate-purpose-issue-template.md

@@ -38,6 +38,7 @@ assignees: ''
 ## Your Environment
 <!--- Include as many relevant details about the environment you experienced the bug in -->
 * Calico version
+* Calico dataplane (iptables, windows etc.)
 * Orchestrator version (e.g. kubernetes, mesos, rkt):
 * Operating System and version:
 * Link to your project (optional):

.github/workflows/yamllint.yml

@@ -0,0 +1,35 @@
+# Lint YAML files present in the repository.
+#
+# Currently we only lint semaphore yaml files, to
+# avoid excessive confusion from developers by
+# throwing unnecessary warnings or errors on arbitrary
+# yaml files.
+#
+# If you want to lint other YAML files in this
+# repository, *add a second workflow* and make
+# sure that you're specifying file paths in the
+# on:pull_request section and in the `with:file_or_dir`
+# section of the block itself.
+---
+name: Yaml Lint
+on:  # yamllint disable-line rule:truthy
+  pull_request:
+    paths:
+      - '.semaphore/**/*.yml'
+      - '.semaphore/.yamllint.yml'
+
+jobs:
+  lintSemaphoreYaml:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - id: yaml-lint
+        uses: ibiqlik/action-yamllint@v3
+        with:
+          file_or_dir: .semaphore/**/*.yml
+          config_file: .semaphore/.yamllint.yml
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: yamllint-logfile
+          path: ${{ steps.yaml-lint.outputs.logfile }}

.gitignore

@@ -18,6 +18,7 @@ node/windows-packaging/nssm.zip
 node/windows-packaging/nssm.exe
 _output
 builder.coverprofile
+*.log
 
 /* Created by local kind cluster */
 hack/test/kind/kind

.semaphore/.yamllint.yml

@@ -0,0 +1,8 @@
+extends: default
+
+rules:
+  line-length: disable
+  document-start: disable
+  empty-lines: disable
+  indentation:
+    indent-sequences: whatever

.semaphore/release/cut-branch.yml

@@ -0,0 +1,31 @@
+version: v1.0
+name: Create new release branch
+agent:
+  machine:
+    type: f1-standard-2
+    os_image: ubuntu2004
+execution_time_limit:
+  minutes: 30
+
+global_job_config:
+  secrets:
+    # Secret for GitHub access
+    - name: marvin-github-ssh-private-key
+  prologue:
+    commands:
+      - chmod 0600 ~/.keys/*
+      - ssh-add ~/
+      # Unshallow the git repository to get latest tags
+      - retry git fetch --quiet --unshallow
+
+blocks:
+  - name: Cut Release Branch
+    task:
+      jobs:
+        - name: Cut Branch
+          commands:
+            - ./bin/release branch cut
+      prologue:
+        commands:
+          - cd release
+          - make build

.semaphore/release/hashrelease.yml

@@ -40,15 +40,12 @@ blocks:
       jobs:
         - name: Build and publish hashrelease
           commands:
-            - ./bin/release hashrelease build
-            - ./bin/release hashrelease publish
+            - make hashrelease
       prologue:
         commands:
           - export GITHUB_TOKEN=${MARVIN_GITHUB_TOKEN}
           - cd release
           - make build
       env_vars:
-        - name: OPERATOR_BRANCH
-          value: master
         - name: IS_HASHRELEASE
           value: "true"

.semaphore/release/release.yml

@@ -4,10 +4,8 @@ agent:
   machine:
     type: f1-standard-4
     os_image: ubuntu2004
-
 execution_time_limit:
   minutes: 800
-
 blocks:
   - name: "Publish official release"
     dependencies: []
@@ -23,50 +21,50 @@ blocks:
         - name: openstack-signing-publishing
       prologue:
         commands:
-        # Load the github access secrets.  First fix the permissions.
-        - chmod 0600 /home/semaphore/.keys/git_ssh_rsa
-        - ssh-add /home/semaphore/.keys/git_ssh_rsa
-        # For some reason, /mnt is 100 GB and has a qemu-nbd image file.
-        # Let's delete it and use it for our own purposes (building calico
-        # without running out of space)
-        - sudo killall qemu-nbd || true
-        - sudo rm -f /mnt/docker.qcow2
-        - sudo chown $(id -u):$(id -g) /mnt/
-        - mkdir calico
-        - sudo mount --bind /mnt calico
-        # Checkout the code and unshallow it.
-        # (this is going to throw an error because it can't remove
-        # the `calico` directory, which is a mount, but it will
-        # continue anyway)
-        - checkout
-        - retry git fetch --quiet --unshallow
-        # Semaphore mounts a copy-on-write FS as /var/lib/docker in order to provide a pre-loaded cache of
-        # some images. However, the cache is not useful to us and the copy-on-write FS is a big problem given
-        # how much we churn docker containers during the build.  Disable it.
-        - sudo systemctl stop docker
-        - sudo umount /var/lib/docker && sudo killall qemu-nbd || true
-        - sudo systemctl start docker
-        # Log in to container registries needed for release.
-        - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin
-        - echo $QUAY_TOKEN | docker login --username "$QUAY_USER" --password-stdin quay.io
-        # Credentials for accessing gcloud, needed to push images to gcr
-        - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/gcr-credentials.json
-        - gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
-        # Manually log in to GCR until we can test the gcr credentials helper
-        - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://gcr.io
-        - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://eu.gcr.io
-        - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://asia.gcr.io
-        - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://us.gcr.io
+          # Load the github access secrets.  First fix the permissions.
+          - chmod 0600 /home/semaphore/.keys/git_ssh_rsa
+          - ssh-add /home/semaphore/.keys/git_ssh_rsa
+          # For some reason, /mnt is 100 GB and has a qemu-nbd image file.
+          # Let's delete it and use it for our own purposes (building calico
+          # without running out of space)
+          - sudo killall qemu-nbd || true
+          - sudo rm -f /mnt/docker.qcow2
+          - sudo chown $(id -u):$(id -g) /mnt/
+          - mkdir calico
+          - sudo mount --bind /mnt calico
+          # Checkout the code and unshallow it.
+          # (this is going to throw an error because it can't remove
+          # the `calico` directory, which is a mount, but it will
+          # continue anyway)
+          - checkout
+          - retry git fetch --quiet --unshallow
+          # Semaphore mounts a copy-on-write FS as /var/lib/docker in order to provide a pre-loaded cache of
+          # some images. However, the cache is not useful to us and the copy-on-write FS is a big problem given
+          # how much we churn docker containers during the build.  Disable it.
+          - sudo systemctl stop docker
+          - sudo umount /var/lib/docker && sudo killall qemu-nbd || true
+          - sudo systemctl start docker
+          # Log in to container registries needed for release.
+          - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin
+          - echo $QUAY_TOKEN | docker login --username "$QUAY_USER" --password-stdin quay.io
+          # Credentials for accessing gcloud, needed to push images to gcr
+          - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/gcr-credentials.json
+          - gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
+          # Manually log in to GCR until we can test the gcr credentials helper
+          - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://gcr.io
+          - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://eu.gcr.io
+          - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://asia.gcr.io
+          - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://us.gcr.io
       jobs:
-      - name: "Release on Semaphore VM"
-        execution_time_limit:
-          minutes: 360
-        env_vars:
-        - name: VAR_FILE
-          value: /home/semaphore/secrets/release.tfvars
-        commands:
-        - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make GIT_BRANCH=${SEMAPHORE_GIT_BRANCH} release; fi
-        - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make GIT_BRANCH=${SEMAPHORE_GIT_BRANCH} release-publish; fi
+        - name: "Release on Semaphore VM"
+          execution_time_limit:
+            minutes: 360
+          env_vars:
+            - name: VAR_FILE
+              value: /home/semaphore/secrets/release.tfvars
+          commands:
+            - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make GIT_BRANCH=${SEMAPHORE_GIT_BRANCH} release; fi
+            - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make GIT_BRANCH=${SEMAPHORE_GIT_BRANCH} release-publish; fi
   - name: "Build Openstack Packages"
     dependencies: ["Publish official release"]
     skip:
@@ -80,29 +78,25 @@ blocks:
         - name: openstack-signing-publishing
       prologue:
         commands:
-        # Load the github access secrets.  First fix the permissions.
-        - chmod 0600 /home/semaphore/.keys/git_ssh_rsa
-        - ssh-add /home/semaphore/.keys/git_ssh_rsa
-        # Checkout the code and unshallow it.
-        - checkout
-        # Install more tools
-        - sudo apt update
-        - sudo apt install -y moreutils patchelf
+          # Load the github access secrets.  First fix the permissions.
+          - chmod 0600 /home/semaphore/.keys/git_ssh_rsa
+          - ssh-add /home/semaphore/.keys/git_ssh_rsa
+          # Checkout the code (we don't need to unshallow it like we usually do)
+          - checkout
+          # Authenticate to google cloud (to upload RPM binaries to the repo)
+          - gcloud config set project tigera-wp-tcp-redirect
+          - gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS
+          # Install more tools
+          - sudo apt update
+          - sudo apt install -y moreutils patchelf
       jobs:
         - name: "Build Openstack Packages"
           execution_time_limit:
             minutes: 60
-          env_vars:
-          - name: SECRET_KEY
-            value: /home/semaphore/secrets/launchpad-gpg-key-dfox.key
-          - name: GCLOUD_ARGS
-            value: --zone us-east1-c --project tigera-wp-tcp-redirect
-          - name: HOST
-            value: ubuntu@binaries-projectcalico-org
           commands:
             - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make publish-openstack; fi
       epilogue:
         always:
           commands:
-          - test -d release/packaging/output  && mv -v release/packaging/output release/packaging/openstack
-          - artifact push workflow release/packaging/openstack
+            - test -d release/packaging/output  && mv -v release/packaging/output release/packaging/openstack
+            - artifact push workflow release/packaging/openstack

.semaphore/rerun_failed_jobs.yml

@@ -0,0 +1,24 @@
+version: v1.0
+name: Rerun failed jobs
+agent:
+  machine:
+    type: f1-standard-2
+    os_image: ubuntu2204
+
+execution_time_limit:
+  minutes: 10
+
+blocks:
+  - name: Rerun failed jobs
+    dependencies: []
+    task:
+      secrets:
+        - name: semaphore-api
+      jobs:
+        - name: Rerun failed jobs
+          commands:
+            - curl https://storage.googleapis.com/sem-cli-releases/get.sh | bash
+            - export CONNECT_URL=${SEMAPHORE_ORGANIZATION_URL#"https://"}
+            - /usr/local/bin/sem connect $CONNECT_URL $SEMAPHORE_API_TOKEN
+            - export PIPELINE=$(/usr/local/bin/sem get workflows $SEMAPHORE_WORKFLOW_ID -i $SEMAPHORE_PROJECT_ID | tail -n 1 | awk '{print $1}')
+            - /usr/local/bin/sem rebuild pipeline $PIPELINE