Skip to content
Push

Docker image fix #10741

Sign in to view logs

Docker image fix

Docker image fix #10741

Workflow file for this run

.github/workflows/push.yml at 4ce63fc
name: Push
on:
push:
branches:
- master
pull_request_target:
env:
PIP_CACHE: |
~/.cache/pip
~/.local/bin
~/.local/lib/python3.*/site-packages
PYTEST_SENTRY_TRACES_SAMPLE_RATE: 0.001
SENTRY_ENVIRONMENT: test
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
check_membership:
name: Check membership
if: "!contains(github.event.head_commit.message, 'Bump version') || github.event_name != 'push'"
runs-on: ubuntu-22.04
outputs:
skip: ${{ steps.skip.outputs.skip }}
steps:
- uses: tspascoal/get-user-teams-membership@v1
id: membership
if: "github.actor != 'dependabot[bot]'"
with:
username: ${{ github.actor }}
team: 'team'
GITHUB_TOKEN: ${{ secrets.READ_ORG_TOKEN }}
- id: skip
if: "github.event_name == 'pull_request_target' && github.actor != 'dependabot[bot]' && steps.membership.outputs.isTeamMember == 'false'"
run: echo "skip=true" >> $GITHUB_OUTPUT
static_checks:
name: Static checks
if: "!contains(github.event.head_commit.message, 'Bump version') || github.event_name != 'push'"
runs-on: ubuntu-22.04
steps:
- uses: actions-ecosystem/action-get-merged-pull-request@v1
id: get-merged-pull-request
with:
github_token: ${{ github.token }}
# no easy way to "exit 0": https://github.com/actions/runner/issues/662
- uses: tspascoal/get-user-teams-membership@v1
id: membership
if: "github.actor != 'dependabot[bot]'"
with:
username: ${{ github.actor }}
team: 'team'
GITHUB_TOKEN: ${{ secrets.READ_ORG_TOKEN }}
- name: actions/checkout
uses: actions/checkout@v3
if: "github.event_name != 'pull_request_target' && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
with:
token: ${{ secrets.GKWILLIE_TOKEN }}
submodules: false
- name: actions/checkout
uses: actions/checkout@v3
if: "github.event_name == 'pull_request_target' && (github.actor == 'dependabot[bot]' || steps.membership.outputs.isTeamMember) && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
with:
ref: ${{ github.event.pull_request.head.sha }}
token: ${{ secrets.GKWILLIE_TOKEN }}
submodules: false
- name: actions/cache
uses: actions/cache@v3.3.1
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
with:
path: ${{ env.PIP_CACHE }}
key: ubuntu-22.04-pip-static-checks-${{ hashFiles('server/requirements-lint.txt') }}
restore-keys: ubuntu-22.04-pip-static-checks-
- name: pip
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: |
python3 -m pip install --user -r server/requirements-lint.txt --no-warn-script-location
echo "$HOME/.local/bin" >> $GITHUB_PATH
- name: static checks
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
working-directory: server
run: tests/run_static_checks.sh
- name: semgrep security
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
working-directory: server
run: semgrep --config p/r2c-security-audit --severity ERROR --disable-version-check --error
- name: semgrep custom
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
working-directory: server
run: semgrep --config semgrep.yaml --severity ERROR --disable-version-check --error
- name: bandit
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
working-directory: server
run: bandit -c pyproject.toml --severity-level high -r athenian/api
custom_checks:
name: Custom checks
if: "!contains(github.event.head_commit.message, 'Bump version') || github.event_name != 'push'"
runs-on: ubuntu-22.04
services:
postgres:
image: postgres:13.6
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
ports:
- 5432:5432
options: >-
--mount type=tmpfs,destination=/var/lib/postgresql/data
--health-cmd pg_isready
--health-interval 10s
--health-timeout 1s
--health-retries 10
--health-start-period 2s
steps:
- uses: actions-ecosystem/action-get-merged-pull-request@v1
id: get-merged-pull-request
with:
github_token: ${{ github.token }}
- uses: tspascoal/get-user-teams-membership@v1
id: membership
if: "github.actor != 'dependabot[bot]'"
with:
username: ${{ github.actor }}
team: 'team'
GITHUB_TOKEN: ${{ secrets.READ_ORG_TOKEN }}
- name: actions/checkout
uses: actions/checkout@v3
if: "github.event_name != 'pull_request_target' && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
with:
token: ${{ secrets.GKWILLIE_TOKEN }}
submodules: recursive
- name: actions/checkout
uses: actions/checkout@v3
if: "github.event_name == 'pull_request_target' && (github.actor == 'dependabot[bot]' || steps.membership.outputs.isTeamMember) && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
with:
ref: ${{ github.event.pull_request.head.sha }}
token: ${{ secrets.GKWILLIE_TOKEN }}
submodules: recursive
- name: cache pip
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
uses: actions/cache@v3.3.1
with:
path: ${{ env.PIP_CACHE }}
key: ubuntu-22.04-pip-custom-checks-${{ hashFiles('server/requirements.txt', 'server/requirements-lint.txt', 'server/requirements-test.txt') }}
restore-keys: ubuntu-22.04-pip-custom-checks-
- name: chown /usr/local
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: sudo chown $(whoami) /usr/local/lib /usr/local/include
- name: cache libs
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
id: cache-native-libs
uses: actions/cache@v3.3.1
with:
path: |
/usr/local/lib/libsentry.so*
/usr/local/lib/libmimalloc.so*
/usr/local/include/mimalloc.h
/usr/local/include/sentry.h
key: ubuntu-22.04-native-libs-${{ hashFiles('.git/modules/server/athenian/api/sentry_native/refs/heads/master', '.git/modules/server/athenian/api/mimalloc/refs/heads/dev') }}
- name: build-native-libs
if: "steps.cache-native-libs.outputs.cache-hit != 'true' && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: |
set -x
sudo apt-get update
sudo apt-get install -y libcurl4-gnutls-dev
make install-native
- name: pip
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: |
set -x
rm -rf server/athenian/api/sentry_native/*
python3 -m pip install --user --upgrade setuptools pip
sudo apt-get remove -y python3-setuptools
sed -i "s/git+ssh:\/\/git@/git+https:\/\/gkwillie:${{ secrets.GKWILLIE_TOKEN }}@/g" server/requirements.txt
python3 -m pip install --user -r server/requirements.txt --no-warn-script-location
python3 -m pip install --user -r server/requirements-lint.txt --no-warn-script-location
python3 -m pip install --user -r server/requirements-test.txt --no-warn-script-location
git checkout -f server/requirements.txt
python3 -m pip install --user --no-deps -e server/
python3 -m pip list
- name: web models
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
working-directory: server
run: |
set -x
python3 -m athenian.api.models.web
chorny athenian/api/models/web/__init__.py ; isort athenian/api/models/web/__init__.py
git status --porcelain
test -z "$(git status --porcelain)"
- name: migrations
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
working-directory: server
env:
PGPASSWORD: postgres
PGUSER: postgres
run: |
set -e
(! grep -R 'athenian\.api\.models' athenian/api/models/state/versions)
athenian/api/models/state/check_sdb_migrations.sh
test:
name: Test suite
needs: check_membership
if: "(!contains(github.event.head_commit.message, 'Bump version') || github.event_name != 'push') && needs.check_membership.outputs.skip != 'true'"
runs-on: ubuntu-22.04
# yes, we over-commit threads to cores x2
# labels: ubuntu-latest-32-cores
concurrency:
group: auth0-${{ github.actor }}-${{ github.head_ref || github.run_id }}-${{ matrix.db }}-${{ matrix.batch }}-main
cancel-in-progress: true
strategy:
matrix:
db: [sqlite, postgres]
batch: [0, 1, 2, 3]
fail-fast: false
services:
memcached:
image: bitnami/memcached:latest
ports:
- 11211:11211
env:
MEMCACHED_EXTRA_FLAGS: -m 128 -I 64m
postgres:
image: bitnami/postgresql:13
env:
POSTGRESQL_USERNAME: postgres
POSTGRESQL_PASSWORD: postgres
POSTGRESQL_DATABASE: postgres
POSTGRESQL_EXTRA_FLAGS: "-c max_connections=1024 -c shared_buffers=1GB -c fsync=off"
ports:
- 5432:5432
env:
SDB: sqlite:///tests/sdb-master.sqlite
MDB: sqlite:///tests/mdb-master.sqlite
PDB: sqlite:///tests/pdb-master.sqlite
RDB: sqlite:///tests/rdb-master.sqlite
steps:
- uses: actions-ecosystem/action-get-merged-pull-request@v1
id: get-merged-pull-request
with:
github_token: ${{ github.token }}
- uses: tspascoal/get-user-teams-membership@v1
id: membership
if: "github.actor != 'dependabot[bot]'"
with:
username: ${{ github.actor }}
team: 'team'
GITHUB_TOKEN: ${{ secrets.READ_ORG_TOKEN }}
- name: actions/checkout
uses: actions/checkout@v3
if: "github.event_name != 'pull_request_target' && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
with:
token: ${{ secrets.GKWILLIE_TOKEN }}
submodules: recursive
- name: actions/checkout
uses: actions/checkout@v3
if: "github.event_name == 'pull_request_target' && (github.actor == 'dependabot[bot]' || steps.membership.outputs.isTeamMember) && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
with:
ref: ${{ github.event.pull_request.head.sha }}
token: ${{ secrets.GKWILLIE_TOKEN }}
submodules: recursive
- name: Set up Python 3.11
id: python
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
uses: actions/setup-python@v4
with:
python-version: 3.11
- name: cache pip
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
uses: actions/cache@v3.3.1
with:
path: ${{ env.PIP_CACHE }}
key: ubuntu-22.04-${{ steps.python.outputs.python-version }}-pip-main-${{ hashFiles('server/requirements.txt', 'server/requirements-test.txt') }}
restore-keys: ubuntu-22.04-${{ steps.python.outputs.python-version }}-pip-main-
- name: chown /usr/local
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: sudo chown $(whoami) /usr/local/lib /usr/local/include
- name: cache libs
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
id: cache-native-libs
uses: actions/cache@v3.3.1
with:
path: |
/usr/local/lib/libsentry.so*
/usr/local/lib/libmimalloc.so*
/usr/local/include/mimalloc.h
/usr/local/include/sentry.h
key: ubuntu-22.04-native-libs-${{ hashFiles('.git/modules/server/athenian/api/sentry_native/refs/heads/master', '.git/modules/server/athenian/api/mimalloc/refs/heads/dev') }}
- name: build-native-libs
if: "steps.cache-native-libs.outputs.cache-hit != 'true' && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: |
set -x
sudo apt-get update
sudo apt-get install -y libcurl4-gnutls-dev
make install-native
- name: godotenv
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: |
set -x
if [ ! -e $HOME/.local/bin/godotenv ]; then \
mkdir -p $HOME/.local/bin && ls $HOME/.local/bin && \
wget -O - https://github.com/athenianco/godotenv/releases/download/1.3.0-secure/godotenv.gz | gzip -d >$HOME/.local/bin/godotenv && \
chmod +x $HOME/.local/bin/godotenv; \
fi
- name: pip
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: |
set -x
rm -rf server/athenian/api/sentry_native/*
python3 -m pip install --user --upgrade setuptools pip
sudo apt-get remove -y python3-setuptools
sed -i "s/git+ssh:\/\/git@/git+https:\/\/gkwillie:${{ secrets.GKWILLIE_TOKEN }}@/g" server/requirements.txt
python3 -m pip install --user -r server/requirements.txt -r server/requirements-test.txt --no-warn-script-location
python3 -m pip install --user --no-deps -e server/
patch --forward $HOME/.local/lib/python3.*/site-packages/prometheus_client/exposition.py patches/prometheus_client.patch || true
echo "$HOME/.local/bin" >> $GITHUB_PATH
- name: setup postgres
if: "matrix.db == 'postgres' && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
env:
PGPASSWORD: postgres
SDB: postgresql://postgres:postgres@0.0.0.0:5432/state_%s?min_size=2&max_size=3
MDB: postgresql://postgres:postgres@0.0.0.0:5432/metadata_%s?min_size=2&max_size=8
PDB: postgresql://postgres:postgres@0.0.0.0:5432/precomputed_%s?min_size=2&max_size=6
RDB: postgresql://postgres:postgres@0.0.0.0:5432/persistentdata_%s?min_size=1&max_size=4
run: |
set -x
while ! psql -c "show max_connections" -U postgres -h 0.0.0.0 -p 5432; do sleep 1; done
for db in state metadata precomputed persistentdata; do
for i in $(seq 0 3); do
psql -c "create database ${db}_gw$i template 'template0' lc_collate 'C.UTF-8';" -U postgres -h 0.0.0.0 -p 5432
done
psql -c "create database ${db}_master template 'template0' lc_collate 'C.UTF-8';" -U postgres -h 0.0.0.0 -p 5432
done
echo "SDB=$SDB" >> $GITHUB_ENV
echo "MDB=$MDB" >> $GITHUB_ENV
echo "PDB=$PDB" >> $GITHUB_ENV
echo "RDB=$RDB" >> $GITHUB_ENV
echo "OVERRIDE_SDB=$SDB" >> $GITHUB_ENV
echo "OVERRIDE_MDB=$MDB" >> $GITHUB_ENV
echo "OVERRIDE_PDB=$PDB" >> $GITHUB_ENV
echo "OVERRIDE_RDB=$RDB" >> $GITHUB_ENV
- name: setup Google KMS
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: |
echo '${{ secrets.GOOGLE_KMS_SERVICE_ACCOUNT_B64 }}' | base64 -d > google_service.json
echo "GOOGLE_KMS_SERVICE_ACCOUNT_JSON=`pwd`/google_service.json" >> $GITHUB_ENV
- name: setup env
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
working-directory: server
run: |
echo 'AUTH0_AUDIENCE=${{ secrets.AUTH0_AUDIENCE }}' >>.env
echo 'AUTH0_CLIENT_ID=${{ secrets.AUTH0_CLIENT_ID }}' >>.env
echo 'AUTH0_CLIENT_SECRET=${{ secrets.AUTH0_CLIENT_SECRET }}' >>.env
echo 'AUTH0_DOMAIN=${{ secrets.AUTH0_DOMAIN }}' >>.env
echo 'GOOGLE_KMS_KEYNAME=${{ secrets.GOOGLE_KMS_KEYNAME }}' >>.env
echo 'GOOGLE_KMS_KEYRING=${{ secrets.GOOGLE_KMS_KEYRING }}' >>.env
echo 'GOOGLE_KMS_PROJECT=${{ secrets.GOOGLE_PROJECT }}' >>.env
echo 'SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }}' >>.env
echo 'SENTRY_ORG=${{ secrets.SENTRY_ORG }}' >>.env
echo 'SENTRY_PROJECT=${{ secrets.SENTRY_PROJECT }}' >>.env
echo 'SENTRY_KEY=${{ secrets.SENTRY_KEY }}' >>.env
echo 'MANDRILL_API_KEY=${{ secrets.MANDRILL_API_KEY }}' >>.env
echo 'SLACK_API_TOKEN=${{ secrets.SLACK_API_TOKEN }}' >>.env
echo 'SLACK_ACCOUNT_CHANNEL="${{ secrets.SLACK_ACCOUNT_CHANNEL }}"' >>.env
echo 'SLACK_INSTALL_CHANNEL="${{ secrets.SLACK_INSTALL_CHANNEL }}"' >>.env
echo 'SLACK_HEALTH_CHANNEL="${{ secrets.SLACK_ACCOUNT_CHANNEL }}"' >>.env
echo 'SLACK_PERFORMANCE_CHANNEL="${{ secrets.SLACK_ACCOUNT_CHANNEL }}"' >>.env
- name: test suite
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
working-directory: server
run: |
set -x
if [ ${{ matrix.db }} = "sqlite" ]; then
export SLACK_API_TOKEN=xxx
export MANDRILL_API_KEY=
fi
# update "setup postgres" above if you change the number of workers (-n)
godotenv -s pytest --offset ${{ matrix.batch }} --limit 0.25 --seed 777 -n 4 --log-level=info --benchmark-skip --cov-report=xml --cov=athenian.api --durations=20 --timeout 300 --ignore tests/controllers/test_invitation_controller.py --ignore tests/test_auth.py
- name: clear env
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
working-directory: server
run: |
set -x
rm .env
- uses: codecov/codecov-action@v3
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
name: codecov
with:
token: ${{ secrets.CODECOV_TOKEN }}
test-extra:
name: Additional tests
needs: check_membership
if: "(!contains(github.event.head_commit.message, 'Bump version') || github.event_name != 'push') && needs.check_membership.outputs.skip != 'true'"
runs-on: ubuntu-22.04
concurrency:
group: auth0-${{ github.actor }}-${{ matrix.type == 'user' && 'user' || github.head_ref || github.run_id }}-${{ matrix.db }}-${{ matrix.type }}
cancel-in-progress: ${{ matrix.type != 'user' }}
strategy:
matrix:
db: [ sqlite, postgres ]
type: [ heater, user ]
fail-fast: false
services:
memcached:
image: bitnami/memcached:latest
ports:
- 11211:11211
env:
MEMCACHED_EXTRA_FLAGS: -m 128 -I 64m
postgres:
image: postgres:13.6
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
ports:
- 5432:5432
options: >-
--mount type=tmpfs,destination=/var/lib/postgresql/data
--health-cmd pg_isready
--health-interval 10s
--health-timeout 1s
--health-retries 10
--health-start-period 2s
env:
SDB: sqlite:///tests/sdb-master.sqlite
MDB: sqlite:///tests/mdb-master.sqlite
PDB: sqlite:///tests/pdb-master.sqlite
RDB: sqlite:///tests/rdb-master.sqlite
steps:
- uses: actions-ecosystem/action-get-merged-pull-request@v1
id: get-merged-pull-request
with:
github_token: ${{ github.token }}
- uses: tspascoal/get-user-teams-membership@v1
id: membership
if: "github.actor != 'dependabot[bot]'"
with:
username: ${{ github.actor }}
team: 'team'
GITHUB_TOKEN: ${{ secrets.READ_ORG_TOKEN }}
- name: actions/checkout
uses: actions/checkout@v3
if: "github.event_name != 'pull_request_target' && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
with:
token: ${{ secrets.GKWILLIE_TOKEN }}
submodules: recursive
- name: actions/checkout
uses: actions/checkout@v3
if: "github.event_name == 'pull_request_target' && (github.actor == 'dependabot[bot]' || steps.membership.outputs.isTeamMember) && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
with:
ref: ${{ github.event.pull_request.head.sha }}
token: ${{ secrets.GKWILLIE_TOKEN }}
submodules: recursive
- name: Set up Python 3.11
id: python
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
uses: actions/setup-python@v4
with:
python-version: 3.11
- name: cache pip
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
uses: actions/cache@v3.3.1
with:
path: ${{ env.PIP_CACHE }}
key: ubuntu-22.04-${{ steps.python.outputs.python-version }}-pip-main-${{ hashFiles('server/requirements.txt', 'server/requirements-test.txt') }}
restore-keys: ubuntu-22.04-${{ steps.python.outputs.python-version }}-pip-main-
- name: chown /usr/local
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: sudo chown $(whoami) /usr/local/lib /usr/local/include
- name: cache libs
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
id: cache-native-libs
uses: actions/cache@v3.3.1
with:
path: |
/usr/local/lib/libsentry.so*
/usr/local/lib/libmimalloc.so*
/usr/local/include/mimalloc.h
/usr/local/include/sentry.h
key: ubuntu-22.04-native-libs-${{ hashFiles('.git/modules/server/athenian/api/sentry_native/refs/heads/master', '.git/modules/server/athenian/api/mimalloc/refs/heads/dev') }}
- name: build-native-libs
if: "steps.cache-native-libs.outputs.cache-hit != 'true' && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: |
set -x
sudo apt-get update
sudo apt-get install -y libcurl4-gnutls-dev
make install-native
- name: godotenv
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: |
set -x
if [ ! -e $HOME/.local/bin/godotenv ]; then \
mkdir -p $HOME/.local/bin && ls $HOME/.local/bin && \
wget -O - https://github.com/athenianco/godotenv/releases/download/1.3.0-secure/godotenv.gz | gzip -d >$HOME/.local/bin/godotenv && \
chmod +x $HOME/.local/bin/godotenv; \
fi
- name: pip
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: |
set -x
rm -rf server/athenian/api/sentry_native/*
python3 -m pip install --user --upgrade setuptools pip
sudo apt-get remove -y python3-setuptools
sed -i "s/git+ssh:\/\/git@/git+https:\/\/gkwillie:${{ secrets.GKWILLIE_TOKEN }}@/g" server/requirements.txt
python3 -m pip install --user -r server/requirements.txt -r server/requirements-test.txt --no-warn-script-location
python3 -m pip install --user --no-deps -e server/
patch --forward $HOME/.local/lib/python3.*/site-packages/prometheus_client/exposition.py patches/prometheus_client.patch || true
echo "$HOME/.local/bin" >> $GITHUB_PATH
- name: setup postgres
if: "matrix.db == 'postgres' && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
env:
PGPASSWORD: postgres
SDB: postgresql://postgres:postgres@0.0.0.0:5432/state_%s?min_size=2&max_size=3
MDB: postgresql://postgres:postgres@0.0.0.0:5432/metadata_%s?min_size=2&max_size=8
PDB: postgresql://postgres:postgres@0.0.0.0:5432/precomputed_%s?min_size=2&max_size=6
RDB: postgresql://postgres:postgres@0.0.0.0:5432/persistentdata_%s?min_size=2&max_size=5
run: |
set -x
for db in state metadata precomputed persistentdata; do
for i in $(seq 0 3); do
psql -c "create database ${db}_gw$i template 'template0' lc_collate 'C.UTF-8';" -U postgres -h 0.0.0.0 -p 5432
done
psql -c "create database ${db}_master template 'template0' lc_collate 'C.UTF-8';" -U postgres -h 0.0.0.0 -p 5432
done
echo "SDB=$SDB" >> $GITHUB_ENV
echo "MDB=$MDB" >> $GITHUB_ENV
echo "PDB=$PDB" >> $GITHUB_ENV
echo "RDB=$RDB" >> $GITHUB_ENV
echo "OVERRIDE_SDB=$SDB" >> $GITHUB_ENV
echo "OVERRIDE_MDB=$MDB" >> $GITHUB_ENV
echo "OVERRIDE_PDB=$PDB" >> $GITHUB_ENV
echo "OVERRIDE_RDB=$RDB" >> $GITHUB_ENV
- name: setup Google KMS
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: |
echo '${{ secrets.GOOGLE_KMS_SERVICE_ACCOUNT_B64 }}' | base64 -d > google_service.json
echo "GOOGLE_KMS_SERVICE_ACCOUNT_JSON=`pwd`/google_service.json" >> $GITHUB_ENV
- name: setup env
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
working-directory: server
run: |
echo 'AUTH0_AUDIENCE=${{ secrets.AUTH0_AUDIENCE }}' >>.env
echo 'AUTH0_CLIENT_ID=${{ secrets.AUTH0_CLIENT_ID }}' >>.env
echo 'AUTH0_CLIENT_SECRET=${{ secrets.AUTH0_CLIENT_SECRET }}' >>.env
echo 'AUTH0_DOMAIN=${{ secrets.AUTH0_DOMAIN }}' >>.env
echo 'GOOGLE_KMS_KEYNAME=${{ secrets.GOOGLE_KMS_KEYNAME }}' >>.env
echo 'GOOGLE_KMS_KEYRING=${{ secrets.GOOGLE_KMS_KEYRING }}' >>.env
echo 'GOOGLE_KMS_PROJECT=${{ secrets.GOOGLE_PROJECT }}' >>.env
echo 'SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }}' >>.env
echo 'SENTRY_ORG=${{ secrets.SENTRY_ORG }}' >>.env
echo 'SENTRY_PROJECT=${{ secrets.SENTRY_PROJECT }}' >>.env
echo 'SENTRY_KEY=${{ secrets.SENTRY_KEY }}' >>.env
echo 'MANDRILL_API_KEY=${{ secrets.MANDRILL_API_KEY }}' >>.env
echo 'SLACK_API_TOKEN=${{ secrets.SLACK_API_TOKEN }}' >>.env
echo 'SLACK_ACCOUNT_CHANNEL="${{ secrets.SLACK_ACCOUNT_CHANNEL }}"' >>.env
echo 'SLACK_INSTALL_CHANNEL="${{ secrets.SLACK_INSTALL_CHANNEL }}"' >>.env
echo 'SLACK_HEALTH_CHANNEL="${{ secrets.SLACK_ACCOUNT_CHANNEL }}"' >>.env
echo 'SLACK_PERFORMANCE_CHANNEL="${{ secrets.SLACK_ACCOUNT_CHANNEL }}"' >>.env
- name: test user
if: "matrix.type == 'user' && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
working-directory: server
run: |
set -x
if [ ${{ matrix.db }} = "sqlite" ]; then
export SLACK_API_TOKEN=
export MANDRILL_API_KEY=
fi
godotenv -s pytest -n 2 --log-level=info --cov-report=xml --cov=athenian.api --durations=10 --timeout 300 tests/controllers/test_user_controller.py tests/controllers/test_invitation_controller.py tests/test_auth.py
- name: test heater
if: "matrix.type == 'heater' && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
working-directory: server
env:
ATHENIAN_JIRA_INSTALLATION_URL_TEMPLATE: https://whatever-jira/%s
run: |
set -x
export SDB=${SDB/\%s/master}
export MDB=${MDB/\%s/master}
export PDB=${PDB/\%s/master}
export RDB=${RDB/\%s/master}
export OVERRIDE_SDB=$SDB
export OVERRIDE_PDB=$PDB
export OVERRIDE_RDB=$RDB
PYTHONPATH=. PRECOMPUTED=0 godotenv -s python3 tests/gen_mock_db.py
godotenv -s coverage run --source athenian.api -m athenian.api.precompute --metadata-db=$MDB --state-db=$SDB --precomputed-db=$PDB --persistentdata-db=$RDB --memcached=0.0.0.0:11211 accounts 1
# the second run should not crash
godotenv -s python3 -m athenian.api.precompute --metadata-db=$MDB --state-db=$SDB --precomputed-db=$PDB --persistentdata-db=$RDB --memcached=0.0.0.0:11211 accounts 1
coverage xml
- name: clear env
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
working-directory: server
run: |
set -x
rm .env
- uses: codecov/codecov-action@v3
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
name: codecov
with:
token: ${{ secrets.CODECOV_TOKEN }}
docker:
name: Build and test Docker image
if: "!contains(github.event.head_commit.message, 'Bump version') || github.event_name != 'push'"
runs-on: ubuntu-22.04
services:
postgres:
image: postgres:13.6
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
ports:
- 5432:5432
options: >-
--mount type=tmpfs,destination=/var/lib/postgresql/data
--health-cmd pg_isready
--health-interval 10s
--health-timeout 1s
--health-retries 10
--health-start-period 2s
steps:
- uses: actions-ecosystem/action-get-merged-pull-request@v1
id: get-merged-pull-request
with:
github_token: ${{ github.token }}
- uses: tspascoal/get-user-teams-membership@v1
id: membership
if: "github.actor != 'dependabot[bot]'"
with:
username: ${{ github.actor }}
team: 'team'
GITHUB_TOKEN: ${{ secrets.READ_ORG_TOKEN }}
- name: actions/checkout
uses: actions/checkout@v3
if: "github.event_name != 'pull_request_target' && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
with:
token: ${{ secrets.GKWILLIE_TOKEN }}
submodules: recursive
- name: actions/checkout
uses: actions/checkout@v3
if: "github.event_name == 'pull_request_target' && (github.actor == 'dependabot[bot]' || steps.membership.outputs.isTeamMember) && (!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
with:
ref: ${{ github.event.pull_request.head.sha }}
token: ${{ secrets.GKWILLIE_TOKEN }}
submodules: recursive
- uses: satackey/action-docker-layer-caching@v0.0.11
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
continue-on-error: true
- name: setup postgres
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
env:
PGPASSWORD: postgres
run: |
psql -c "create database state template 'template0' lc_collate 'C.UTF-8';" -U postgres -h 0.0.0.0 -p 5432
psql -c "create database metadata template 'template0' lc_collate 'C.UTF-8';" -U postgres -h 0.0.0.0 -p 5432
psql -c "create database precomputed template 'template0' lc_collate 'C.UTF-8';" -U postgres -h 0.0.0.0 -p 5432
- name: docker build
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: |
set -x
python3 -m pip install --no-cache-dir py-cpuinfo resolve-march-native && \
cpuinfo && \
resolve-march-native --vertical && \
docker build --pull --cache-from us-east1-docker.pkg.dev/${{ secrets.GOOGLE_PROJECT }}/images/api:latest --build-arg COMMIT=$GITHUB_SHA --build-arg GKWILLIE_TOKEN=${{ secrets.GKWILLIE_TOKEN }} --build-arg ROOT_ENC_PASSWD='${{ secrets.API_IMAGE_ROOT_PASSWORD }}' -t us-east1-docker.pkg.dev/${{ secrets.GOOGLE_PROJECT }}/images/api .
docker run --rm us-east1-docker.pkg.dev/${{ secrets.GOOGLE_PROJECT }}/images/api --help
docker build -t test --build-arg base=us-east1-docker.pkg.dev/${{ secrets.GOOGLE_PROJECT }}/images/api server/tests
mkdir /tmp/io
sudo chown 1984:1984 /tmp/io # worker user inside container needs write access to directory
run="docker run --network=host --rm -v /tmp/io:/home/worker/io -w /home/worker/io --entrypoint python3 -e ATHENIAN_INVITATION_KEY=777 -e ATHENIAN_INVITATION_URL_PREFIX=https://app.athenian.co/i/ -e ATHENIAN_JIRA_INSTALLATION_URL_TEMPLATE=https://installation.athenian.co/jira/%s/atlassian-connect.json test -m coverage"
$run run -a -m athenian.api.models.state postgresql://postgres:postgres@0.0.0.0:5432/state
$run run -a -m athenian.api.invite_admin postgresql://postgres:postgres@0.0.0.0:5432/state
$run run -a /usr/local/bin/alembic downgrade 34eafe9e7cd9
$run combine
$run xml
docker rmi test
sudo chown $(whoami) -R /tmp/io
sed -i 's/[\./]server[\./]//g' /tmp/io/coverage.xml
sed -i "s\<source></source>\<source>$GITHUB_WORKSPACE/server</source>\g" /tmp/io/coverage.xml
- uses: codecov/codecov-action@v3
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
name: codecov
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: /tmp/io/coverage.xml
- name: docker prune
if: "(!contains(steps.get-merged-pull-request.outputs.labels, 'hotfix') || github.event_name != 'push')"
run: |
(docker image ls -a | grep -v us-east1-docker.pkg.dev/${{ secrets.GOOGLE_PROJECT }}/images/api | tail -n +2 | awk '{print $3}' | xargs docker rmi) || true
(docker image ls -a | grep -v latest | tail -n +2 | awk '{print $3}' | xargs docker rmi) || true
bump_version:
name: Bump the version
needs: [static_checks, custom_checks, test, test-extra, docker]
if: "!contains(github.event.head_commit.message, 'Bump version') && github.ref == 'refs/heads/master' && github.event_name == 'push'"
runs-on: ubuntu-22.04
steps:
- name: actions/checkout
uses: actions/checkout@v3
with:
fetch-depth: 100
persist-credentials: false
token: ${{ secrets.GKWILLIE_TOKEN }}
submodules: true
- name: current_version
run: echo "current_version=$(grep '__version__' server/athenian/api/metadata.py | cut -d\" -f2)" >> $GITHUB_ENV
- name: FragileTech/bump-version
uses: FragileTech/bump-version@main
with:
current_version: "${{ env.current_version }}"
files: server/athenian/api/metadata.py
commit_name: Groundskeeper Willie
commit_email: bot@athenian.co
login: gkwillie
token: ${{ secrets.GKWILLIE_TOKEN }}
deploy:
name: Deploy
if: "contains(github.event.head_commit.message, 'Bump version') && github.ref == 'refs/heads/master' && github.event_name == 'push'"
runs-on: ubuntu-22.04
steps:
- name: actions/checkout
uses: actions/checkout@v3
with:
token: ${{ secrets.GKWILLIE_TOKEN }}
submodules: recursive
- name: install
run: |
curl -sL https://sentry.io/get-cli/ | bash
- uses: satackey/action-docker-layer-caching@v0.0.11
continue-on-error: true
- name: actions/cache google-cloud-sdk
uses: actions/cache@v3.3.1
with:
path: ~/google-cloud-sdk
key: ubuntu-22.04-google-cloud-sdk
- name: actions/cache godotenv
uses: actions/cache@v3.3.1
with:
path: ~/.local/bin/godotenv
key: ubuntu-22.04-godotenv
- name: godotenv
run: |
set -x
if [ ! -e $HOME/.local/bin/godotenv ]; then \
mkdir -p $HOME/.local/bin && \
wget -O - https://github.com/athenianco/godotenv/releases/download/1.3.0-secure/godotenv.gz | gzip -d >$HOME/.local/bin/godotenv && \
chmod +x $HOME/.local/bin/godotenv; \
fi
echo "$HOME/.local/bin" >> $GITHUB_PATH
echo 'SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_ORG=${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT=${{ secrets.SENTRY_PROJECT_NAME }}' >.env
- name: deploy
run: |
set -x
version_file=server/athenian/api/metadata.py
current_version=$(grep __version__ $version_file | cut -d\" -f2)
godotenv -s sentry-cli releases new -p ${{ secrets.SENTRY_PROJECT_NAME }} $current_version
godotenv -s sentry-cli releases set-commits --auto $current_version
export IMAGE=us-east1-docker.pkg.dev/${{ secrets.GOOGLE_PROJECT }}/images/api:$current_version
GOOGLE_CREDENTIALS=${{ secrets.GOOGLE_CREDENTIALS }} \
GOOGLE_PROJECT=${{ secrets.GOOGLE_PROJECT }} \
make gcloud-login
docker build --pull --cache-from us-east1-docker.pkg.dev/${{ secrets.GOOGLE_PROJECT }}/images/api:latest --build-arg COMMIT=$GITHUB_SHA --build-arg GKWILLIE_TOKEN=${{ secrets.GKWILLIE_TOKEN }} --build-arg ROOT_ENC_PASSWD='${{ secrets.API_IMAGE_ROOT_PASSWORD }}' -t $IMAGE .
make docker-push
GOOGLE_CREDENTIALS=${{ secrets.GOOGLE_CREDENTIALS }} \
GOOGLE_PROJECT=${{ secrets.GOOGLE_PROJECT }} \
GOOGLE_PUBSUB_TOPIC=${{ secrets.GOOGLE_PUBSUB_TOPIC }} \
make pubsub-publish
docker tag $IMAGE us-east1-docker.pkg.dev/${{ secrets.GOOGLE_PROJECT }}/images/api
docker run --rm --env-file .env -u root --entrypoint /server/submit_debug_info.sh $IMAGE
IMAGE=us-east1-docker.pkg.dev/${{ secrets.GOOGLE_PROJECT }}/images/api make docker-push
godotenv -s sentry-cli releases finalize $current_version
- name: clear env
run: |
set -x
rm .env
- name: docker prune
run: |
(docker image ls -a | grep -v us-east1-docker.pkg.dev/${{ secrets.GOOGLE_PROJECT }}/images/api | tail -n +2 | awk '{print $3}' | xargs docker rmi) || true
(docker image ls -a | grep -v latest | tail -n +2 | awk '{print $3}' | xargs docker rmi) || true