From 67c44479b56ad496417eeb7901b4b1a470ef4d55 Mon Sep 17 00:00:00 2001 From: Erik Ritter Date: Tue, 24 Nov 2020 11:40:16 -0800 Subject: [PATCH] build: split python CI jobs (#11794) * build: split python CI jobs * Rename superset-python.yml to superset-python-misc.yml --- .github/workflows/superset-python-misc.yml | 72 ++++ .github/workflows/superset-python-mysql.yml | 51 +++ .../workflows/superset-python-postgres.yml | 52 +++ .../workflows/superset-python-presto-hive.yml | 119 +++++++ .github/workflows/superset-python-sqlite.yml | 44 +++ .github/workflows/superset-python.yml | 322 ------------------ 6 files changed, 338 insertions(+), 322 deletions(-) create mode 100644 .github/workflows/superset-python-misc.yml create mode 100644 .github/workflows/superset-python-mysql.yml create mode 100644 .github/workflows/superset-python-postgres.yml create mode 100644 .github/workflows/superset-python-presto-hive.yml create mode 100644 .github/workflows/superset-python-sqlite.yml delete mode 100644 .github/workflows/superset-python.yml diff --git a/.github/workflows/superset-python-misc.yml b/.github/workflows/superset-python-misc.yml new file mode 100644 index 0000000000000..8db08edb82d8a --- /dev/null +++ b/.github/workflows/superset-python-misc.yml @@ -0,0 +1,72 @@ +# Python Misc unit tests +name: Python Misc + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.7] + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + uses: apache-superset/cached-dependencies@b90713b + with: + run: | + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt + - name: pylint + # `-j 0` run Pylint in parallel + run: pylint -j 0 superset + + pre-commit: + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.7] + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + uses: apache-superset/cached-dependencies@b90713b + with: + run: | + apt-get-install + pip-upgrade + pip install -r requirements/integration.txt + - name: pre-commit + run: pre-commit run --all-files + + babel-extract: + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.7] + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + uses: apache-superset/cached-dependencies@b90713b + with: + run: | + apt-get-install + pip-upgrade + pip install -r requirements/base.txt + - name: Test babel extraction + run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct diff --git a/.github/workflows/superset-python-mysql.yml b/.github/workflows/superset-python-mysql.yml new file mode 100644 index 0000000000000..a8b2394373083 --- /dev/null +++ b/.github/workflows/superset-python-mysql.yml @@ -0,0 +1,51 @@ +# Python MySQL unit tests +name: Python MySQL + +on: [push, pull_request] + +jobs: + test-mysql: + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.7] + env: + PYTHONPATH: ${{ github.workspace }} + SUPERSET_CONFIG: tests.superset_test_config + REDIS_PORT: 16379 + SUPERSET__SQLALCHEMY_DATABASE_URI: | + mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true + services: + mysql: + image: mysql:5.7 + env: + MYSQL_ROOT_PASSWORD: root + ports: + - 13306:3306 + redis: + image: redis:5-alpine + options: --entrypoint redis-server + ports: + - 16379:6379 + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + uses: apache-superset/cached-dependencies@b90713b + with: + run: | + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt + setup-mysql + - name: Run celery + run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & + - name: Python unit tests (MySQL) + run: | + ./scripts/python_tests.sh + - name: Upload code coverage + run: | + bash <(curl -s https://codecov.io/bash) -cF python diff --git a/.github/workflows/superset-python-postgres.yml b/.github/workflows/superset-python-postgres.yml new file mode 100644 index 0000000000000..95bd413b8c39a --- /dev/null +++ b/.github/workflows/superset-python-postgres.yml @@ -0,0 +1,52 @@ +# Python Postgres unit tests +name: Python Postgres + +on: [push, pull_request] + +jobs: + test-postgres: + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.7, 3.8] + env: + PYTHONPATH: ${{ github.workspace }} + SUPERSET_CONFIG: tests.superset_test_config + REDIS_PORT: 16379 + SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset + services: + postgres: + image: postgres:10-alpine + env: + POSTGRES_USER: superset + POSTGRES_PASSWORD: superset + ports: + # Use custom ports for services to avoid accidentally connecting to + # GitHub action runner's default installations + - 15432:5432 + redis: + image: redis:5-alpine + ports: + - 16379:6379 + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + uses: apache-superset/cached-dependencies@b90713b + with: + run: | + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt + setup-postgres + - name: Run celery + run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & + - name: Python unit tests (PostgreSQL) + run: | + ./scripts/python_tests.sh + - name: Upload code coverage + run: | + bash <(curl -s https://codecov.io/bash) -cF python diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml new file mode 100644 index 0000000000000..e723c6c7ab5c5 --- /dev/null +++ b/.github/workflows/superset-python-presto-hive.yml @@ -0,0 +1,119 @@ +# Python Presto/Hive unit tests +name: Python Presto/Hive + +on: [push, pull_request] + +jobs: + test-postgres-presto: + runs-on: ubuntu-18.04 + strategy: + matrix: + # run unit tests in multiple version just for fun + python-version: [3.8] + env: + PYTHONPATH: ${{ github.workspace }} + SUPERSET_CONFIG: tests.superset_test_config + REDIS_PORT: 16379 + SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset + SUPERSET__SQLALCHEMY_EXAMPLES_URI: presto://localhost:15433/memory/default + services: + postgres: + image: postgres:10-alpine + env: + POSTGRES_USER: superset + POSTGRES_PASSWORD: superset + ports: + # Use custom ports for services to avoid accidentally connecting to + # GitHub action runner's default installations + - 15432:5432 + presto: + image: prestosql/presto:339 + env: + POSTGRES_USER: superset + POSTGRES_PASSWORD: superset + ports: + # Use custom ports for services to avoid accidentally connecting to + # GitHub action runner's default installations + - 15433:8080 + redis: + image: redis:5-alpine + ports: + - 16379:6379 + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + uses: apache-superset/cached-dependencies@b90713b + with: + run: | + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt + setup-postgres + - name: Run celery + run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & + - name: Python unit tests (PostgreSQL) + run: | + ./scripts/python_tests.sh + - name: Upload code coverage + run: | + bash <(curl -s https://codecov.io/bash) -cF python + + test-postgres-hive: + runs-on: ubuntu-18.04 + strategy: + matrix: + # run unit tests in multiple version just for fun + python-version: [3.7, 3.8] + env: + PYTHONPATH: ${{ github.workspace }} + SUPERSET_CONFIG: tests.superset_test_config + REDIS_PORT: 16379 + SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset + SUPERSET__SQLALCHEMY_EXAMPLES_URI: hive://localhost:10000/default + UPLOAD_FOLDER: /tmp/.superset/uploads/ + services: + postgres: + image: postgres:10-alpine + env: + POSTGRES_USER: superset + POSTGRES_PASSWORD: superset + ports: + # Use custom ports for services to avoid accidentally connecting to + # GitHub action runner's default installations + - 15432:5432 + redis: + image: redis:5-alpine + ports: + - 16379:6379 + steps: + - uses: actions/checkout@v2 + - name: Create csv upload directory + run: sudo mkdir -p /tmp/.superset/uploads + - name: Give write access to the csv upload directory + run: sudo chown -R $USER:$USER /tmp/.superset + - name: Start hadoop and hive + run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + uses: apache-superset/cached-dependencies@b90713b + with: + run: | + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt + setup-postgres + - name: Run celery + run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & + - name: Python unit tests (PostgreSQL) + run: | + ./scripts/python_tests.sh + - name: Upload code coverage + run: | + bash <(curl -s https://codecov.io/bash) -cF python diff --git a/.github/workflows/superset-python-sqlite.yml b/.github/workflows/superset-python-sqlite.yml new file mode 100644 index 0000000000000..63cfed387b038 --- /dev/null +++ b/.github/workflows/superset-python-sqlite.yml @@ -0,0 +1,44 @@ +# Python Sqlite unit tests +name: Python Sqlite + +on: [push, pull_request] + +jobs: + test-sqlite: + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.7] + env: + PYTHONPATH: ${{ github.workspace }} + SUPERSET_CONFIG: tests.superset_test_config + REDIS_PORT: 16379 + SUPERSET__SQLALCHEMY_DATABASE_URI: | + sqlite:///${{ github.workspace }}/.temp/unittest.db + services: + redis: + image: redis:5-alpine + ports: + - 16379:6379 + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + uses: apache-superset/cached-dependencies@b90713b + with: + run: | + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt + mkdir ${{ github.workspace }}/.temp + - name: Run celery + run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & + - name: Python unit tests (SQLite) + run: | + ./scripts/python_tests.sh + - name: Upload code coverage + run: | + bash <(curl -s https://codecov.io/bash) -cF python diff --git a/.github/workflows/superset-python.yml b/.github/workflows/superset-python.yml deleted file mode 100644 index 4221c88165e26..0000000000000 --- a/.github/workflows/superset-python.yml +++ /dev/null @@ -1,322 +0,0 @@ -# Python unit tests -name: Python - -on: [push, pull_request] - -jobs: - lint: - runs-on: ubuntu-18.04 - strategy: - matrix: - python-version: [3.7] - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - uses: apache-superset/cached-dependencies@b90713b - with: - run: | - apt-get-install - pip-upgrade - pip install -r requirements/testing.txt - - name: pylint - # `-j 0` run Pylint in parallel - run: pylint -j 0 superset - - pre-commit: - runs-on: ubuntu-18.04 - strategy: - matrix: - python-version: [3.7] - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - uses: apache-superset/cached-dependencies@b90713b - with: - run: | - apt-get-install - pip-upgrade - pip install -r requirements/integration.txt - - name: pre-commit - run: pre-commit run --all-files - - babel-extract: - runs-on: ubuntu-18.04 - strategy: - matrix: - python-version: [3.7] - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - uses: apache-superset/cached-dependencies@b90713b - with: - run: | - apt-get-install - pip-upgrade - pip install -r requirements/base.txt - - name: Test babel extraction - run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct - - test-postgres-presto: - runs-on: ubuntu-18.04 - strategy: - matrix: - # run unit tests in multiple version just for fun - python-version: [3.8] - env: - PYTHONPATH: ${{ github.workspace }} - SUPERSET_CONFIG: tests.superset_test_config - REDIS_PORT: 16379 - SUPERSET__SQLALCHEMY_DATABASE_URI: - postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset - SUPERSET__SQLALCHEMY_EXAMPLES_URI: - presto://localhost:15433/memory/default - services: - postgres: - image: postgres:10-alpine - env: - POSTGRES_USER: superset - POSTGRES_PASSWORD: superset - ports: - # Use custom ports for services to avoid accidentally connecting to - # GitHub action runner's default installations - - 15432:5432 - presto: - image: prestosql/presto:339 - env: - POSTGRES_USER: superset - POSTGRES_PASSWORD: superset - ports: - # Use custom ports for services to avoid accidentally connecting to - # GitHub action runner's default installations - - 15433:8080 - redis: - image: redis:5-alpine - ports: - - 16379:6379 - steps: - - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - uses: apache-superset/cached-dependencies@b90713b - with: - run: | - apt-get-install - pip-upgrade - pip install -r requirements/testing.txt - setup-postgres - - name: Run celery - run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & - - name: Python unit tests (PostgreSQL) - run: | - ./scripts/python_tests.sh - - name: Upload code coverage - run: | - bash <(curl -s https://codecov.io/bash) -cF python - - test-postgres-hive: - runs-on: ubuntu-18.04 - strategy: - matrix: - # run unit tests in multiple version just for fun - python-version: [3.7, 3.8] - env: - PYTHONPATH: ${{ github.workspace }} - SUPERSET_CONFIG: tests.superset_test_config - REDIS_PORT: 16379 - SUPERSET__SQLALCHEMY_DATABASE_URI: - postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset - SUPERSET__SQLALCHEMY_EXAMPLES_URI: hive://localhost:10000/default - UPLOAD_FOLDER: /tmp/.superset/uploads/ - services: - postgres: - image: postgres:10-alpine - env: - POSTGRES_USER: superset - POSTGRES_PASSWORD: superset - ports: - # Use custom ports for services to avoid accidentally connecting to - # GitHub action runner's default installations - - 15432:5432 - redis: - image: redis:5-alpine - ports: - - 16379:6379 - steps: - - uses: actions/checkout@v2 - - name: Create csv upload directory - run: sudo mkdir -p /tmp/.superset/uploads - - name: Give write access to the csv upload directory - run: sudo chown -R $USER:$USER /tmp/.superset - - name: Start hadoop and hive - run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - uses: apache-superset/cached-dependencies@b90713b - with: - run: | - apt-get-install - pip-upgrade - pip install -r requirements/testing.txt - setup-postgres - - name: Run celery - run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & - - name: Python unit tests (PostgreSQL) - run: | - ./scripts/python_tests.sh - - name: Upload code coverage - run: | - bash <(curl -s https://codecov.io/bash) -cF python - - test-postgres: - runs-on: ubuntu-18.04 - strategy: - matrix: - python-version: [3.7, 3.8] - env: - PYTHONPATH: ${{ github.workspace }} - SUPERSET_CONFIG: tests.superset_test_config - REDIS_PORT: 16379 - SUPERSET__SQLALCHEMY_DATABASE_URI: - postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset - services: - postgres: - image: postgres:10-alpine - env: - POSTGRES_USER: superset - POSTGRES_PASSWORD: superset - ports: - # Use custom ports for services to avoid accidentally connecting to - # GitHub action runner's default installations - - 15432:5432 - redis: - image: redis:5-alpine - ports: - - 16379:6379 - steps: - - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - uses: apache-superset/cached-dependencies@b90713b - with: - run: | - apt-get-install - pip-upgrade - pip install -r requirements/testing.txt - setup-postgres - - name: Run celery - run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & - - name: Python unit tests (PostgreSQL) - run: | - ./scripts/python_tests.sh - - name: Upload code coverage - run: | - bash <(curl -s https://codecov.io/bash) -cF python - - test-mysql: - runs-on: ubuntu-18.04 - strategy: - matrix: - python-version: [3.7] - env: - PYTHONPATH: ${{ github.workspace }} - SUPERSET_CONFIG: tests.superset_test_config - REDIS_PORT: 16379 - SUPERSET__SQLALCHEMY_DATABASE_URI: | - mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true - services: - mysql: - image: mysql:5.7 - env: - MYSQL_ROOT_PASSWORD: root - ports: - - 13306:3306 - redis: - image: redis:5-alpine - options: --entrypoint redis-server - ports: - - 16379:6379 - steps: - - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - uses: apache-superset/cached-dependencies@b90713b - with: - run: | - apt-get-install - pip-upgrade - pip install -r requirements/testing.txt - setup-mysql - - name: Run celery - run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & - - name: Python unit tests (MySQL) - run: | - ./scripts/python_tests.sh - - name: Upload code coverage - run: | - bash <(curl -s https://codecov.io/bash) -cF python - - test-sqlite: - runs-on: ubuntu-18.04 - strategy: - matrix: - python-version: [3.7] - env: - PYTHONPATH: ${{ github.workspace }} - SUPERSET_CONFIG: tests.superset_test_config - REDIS_PORT: 16379 - SUPERSET__SQLALCHEMY_DATABASE_URI: | - sqlite:///${{ github.workspace }}/.temp/unittest.db - services: - redis: - image: redis:5-alpine - ports: - - 16379:6379 - steps: - - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - uses: apache-superset/cached-dependencies@b90713b - with: - run: | - apt-get-install - pip-upgrade - pip install -r requirements/testing.txt - mkdir ${{ github.workspace }}/.temp - - name: Run celery - run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & - - name: Python unit tests (SQLite) - run: | - ./scripts/python_tests.sh - - name: Upload code coverage - run: | - bash <(curl -s https://codecov.io/bash) -cF python