forked from apache/airflow
-
Notifications
You must be signed in to change notification settings - Fork 0
103 lines (100 loc) · 5.95 KB
/
ci-teradata-push-branch-systemtest.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: airflow-teradata-push-branch-systemtest
on:
push:
branches:
- IDE-*
workflow_dispatch:
jobs:
teradata-tests:
name: "Run Teradata Provider Unit and System tests"
runs-on: "ubuntu-22.04"
strategy:
max-parallel: 1
matrix:
python-version: [ "3.11" ]
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout Teradata Airflow Source Code"
uses: actions/checkout@v4
with:
ref: main
persist-credentials: false
- name: "Github Event and Branch Details"
run: echo "The current checked out branch is $GITHUB_REF. Workflow trigged Event- ${{ github.event_name }}. The fully qualified reference (ref) branch that triggered the workflow run- ${{ github.ref }}"
- name: "Install Python"
uses: actions/setup-python@v4
with:
python-version: 3.x
cache: 'pip'
- name: "Python Version"
run: python --version
- name: "Checkout ClearScape Scripts"
run: |
git fetch
git checkout origin/${{vars.WORKFLOW_HELP_BRANCH}} -- .github/workflows/scripts/createTestEnvironments.sh
git checkout origin/${{vars.WORKFLOW_HELP_BRANCH}} -- .github/workflows/scripts/deleteTestEnvironments.sh
- name: "Define CSAE env name"
id: define-environment-name
run: |
echo "env-name=airflow-ci-$(date +%s%N)" >> $GITHUB_OUTPUT
- name: "Create CSAE environment"
id: create-csae-environment
shell: bash
run: |
echo Clearscape Environment name : ${{ steps.define-environment-name.outputs.env-name}}
chmod 777 .github/workflows/scripts/createTestEnvironments.sh
.github/workflows/scripts/createTestEnvironments.sh
env:
CSAE_TOKEN: ${{ secrets.CSAE_AIRFLOW_TOKEN }}
CSAE_ENV_PASSWORD: ${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}
CSAE_ENV_NAME: ${{ steps.define-environment-name.outputs.env-name}}
- name: "Install Airflow Core, Airflow Teradata Provider and Run System Tests"
run: |
echo Clearscape Environment name : ${{ steps.create-csae-environment.outputs.teradata-server-name }}
sudo apt update
sudo apt-get install -y apt-transport-https apt-utils build-essential ca-certificates dirmngr \
freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4 libev-dev libffi-dev libgeos-dev \
libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev libsasl2-modules \
libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client pkgconf sasl2-bin \
software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev python3-venv python3-pip pipx
pip install pytest time-machine pytest-asyncio pytest-md-report cryptography pytest-html tailer pandas jinja2 sqlalchemy hatch
pip install -e ".[devel,teradata,microsoft.azure,amazon]"
export SYSTEM_TESTS_ENV_ID=teradatasystemtest
export AIRFLOW_CONN_TERADATA_DEFAULT='teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}'
export AIRFLOW_CONN_TERADATA_LAKE='teradata://${{ vars.LAKE_ENV_USERNAME }}:${{ secrets.LAKE_ENV_PASSWORD }}@${{ vars.LAKE_SERVER_NAME }}:/${{ vars.LAKE_ENV_USERNAME }}'
export AIRFLOW_CONN_TERADATA_SSL_DEFAULT='teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}?sslmode=allow'
export AIRFLOW_CONN_TERADATA_SP_CALL='teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}?tmode=TERA'
export AIRFLOW_CONN_WASB_DEFAULT='wasb://${{ vars.AZURE_BLOB_ACCOUNTNAME }}:${{ secrets.AZURE_BLOB_ACCOUNT_SECRET_KEY_URI }}@'
export AIRFLOW_CONN_AWS_DEFAULT=aws://${{ vars.AWS_ACCESS_KEY_ID }}:${{ secrets.AWS_SECRET_ACCESS_KEY }}@/?aws_session_token=${{ secrets.AWS_SESSION_TOKEN }}
export AIRFLOW_VAR_AZURE_BLOB_ACCOUNTNAME=${{ vars.AZURE_BLOB_ACCOUNTNAME }}
export AIRFLOW_VAR_AZURE_BLOB_ACCOUNT_SECRET_KEY=${{ secrets.AZURE_BLOB_ACCOUNT_SECRET_KEY }}
export AIRFLOW_VAR_AWS_ACCESS_KEY_ID=${{ secrets.TERADATA_DOWNLOAD_AWS_SECRET_ACCESS_ID }}
export AIRFLOW_VAR_AWS_SECRET_ACCESS_KEY=${{ secrets.TERADATA_DOWNLOAD_AWS_SECRET_ACCESS_KEY }}
pytest --system --junitxml=report_test.xml tests/system/providers/teradata
- name: "Delete CSAE environment"
if: always()
run: |
chmod 777 .github/workflows/scripts/deleteTestEnvironments.sh
.github/workflows/scripts/deleteTestEnvironments.sh
env:
CSAE_TOKEN: ${{ secrets.CSAE_AIRFLOW_TOKEN }}
CSAE_ENV_NAME: ${{ steps.define-environment-name.outputs.env-name}}