From 6f2db67fde098e060a0d1fb0dcaaa9f064a330af Mon Sep 17 00:00:00 2001 From: sreekarjami Date: Mon, 1 Feb 2021 22:15:02 +0000 Subject: [PATCH 1/8] workflow for arm64 tar test --- .github/workflows/staging-test-tar-arm64.yml | 353 ++++++++++++++++++ .../scripts/setup_runners_service.sh | 14 +- 2 files changed, 361 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/staging-test-tar-arm64.yml diff --git a/.github/workflows/staging-test-tar-arm64.yml b/.github/workflows/staging-test-tar-arm64.yml new file mode 100644 index 000000000..ca0ad3931 --- /dev/null +++ b/.github/workflows/staging-test-tar-arm64.yml @@ -0,0 +1,353 @@ +name: Process TAR Artifacts + +on: + # schedule: + # - cron: '0 10 * * *' +# repository_dispatch: +# types: [staging-build-tar] + push: + branches: [sreekarj_arm] + +jobs: + plugin-availability: + name: Check Plugin Availability + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Run check_plugin scripts + run: ./release-tools/scripts/check_plugin.sh; exit `cat /tmp/plugin_status.check` + + + Provision-Runners: + name: Provision-Runners + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_EC2_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_EC2_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: AWS Cli Processing + run: | + #!/bin/bash + # Please add comma at the end of the RUNNERS strings + # Don't add space after commas + RUNNERS+="odfe-tar-ism-nosec-arm64,odfe-tar-alerting-nosec-arm64,odfe-tar-sql-nosec-arm64,odfe-tar-knn-nosec-arm64,odfe-tar-ad-nosec-arm64," + RUNNERS+="odfe-tar-sql-arm64,odfe-tar-ad-arm64,odfe-tar-alerting-arm64" + release-tools/scripts/setup_runners.sh run $RUNNERS ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} ami-055197d43e4ec7482 + + + Test-ISM-NoSec: + needs: [Provision-Runners] + runs-on: ubuntu-18.04 + name: Test-ISM-NoSec + strategy: + fail-fast: false + matrix: + java: [14] + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_ism=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/index-management)" >> $GITHUB_ENV + - name: Checkout ISM + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/index-management + ref: ${{env.p_tag_ism}} + - name: IT + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../index-management; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest + + Test-Alerting-NoSec: + needs: [Provision-Runners] + runs-on: ubuntu-18.04 + name: Test-Alerting-NoSec + strategy: + fail-fast: false + matrix: + java: [14] + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_alerting=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/alerting)" >> $GITHUB_ENV + - name: Checkout Alerting + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/alerting + ref: ${{env.p_tag_alerting}} + - name: IT + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../alerting/alerting; pwd + ../gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest + + Test-SQL-NoSec: + needs: [Provision-Runners] + runs-on: ubuntu-18.04 + name: Test-SQL-NoSec + strategy: + fail-fast: false + matrix: + java: [14] + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_sql=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql)" >> $GITHUB_ENV + - name: Checkout SQL + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/sql + ref: ${{env.p_tag_sql}} + - name: IT + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../sql; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest + + Test-KNN-NoSec: + needs: [Provision-Runners] + runs-on: ubuntu-18.04 + name: Test-KNN-NoSec + strategy: + fail-fast: false + matrix: + java: [14] + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_knn=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/k-NN)" >> $GITHUB_ENV + - name: Checkout k-NN + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/k-NN + ref: ${{env.p_tag_knn}} + - name: IT + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../k-NN; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest + + Test-AD-NoSec: + needs: [Provision-Runners] + runs-on: ubuntu-18.04 + name: Test-AD-NoSec + strategy: + fail-fast: false + matrix: + java: [14] + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_ad=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection)" >> $GITHUB_ENV + - name: Checkout anomaly-detection + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/anomaly-detection + ref: ${{env.p_tag_ad}} + - name: IT + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../anomaly-detection; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="es-integrationtest" + + Test-SQL: + needs: [Provision-Runners] + runs-on: ubuntu-18.04 + name: Test-SQL + strategy: + fail-fast: false + matrix: + java: [14] + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_sql=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql)" >> $GITHUB_ENV + - name: Checkout SQL + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/sql + ref: ${{env.p_tag_sql}} + - name: IT + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../sql; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest -Dhttps=true -Duser=admin -Dpassword=admin + + Test-AD: + needs: [Provision-Runners] + runs-on: ubuntu-18.04 + name: Test-AD + strategy: + fail-fast: false + matrix: + java: [14] + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_ad=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection)" >> $GITHUB_ENV + - name: Checkout anomaly-detection + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/anomaly-detection + ref: ${{env.p_tag_ad}} + - name: IT + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../anomaly-detection; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="es-integrationtest" -Dhttps=true -Duser=admin -Dpassword=admin + + Test-ALERTING: + needs: [Provision-Runners] + runs-on: [ubuntu-18.04] + name: Test-ALERTING + strategy: + fail-fast: false + matrix: + java: [14] + steps: + - uses: actions/checkout@v1 + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_alerting=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/alerting)" >> $GITHUB_ENV + + - uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/alerting + ref: ${{env.p_tag_alerting}} + + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: IT + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../alerting/alerting; pwd + ls -ltr + ../gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest -Dhttps=true -Duser=admin -Dpassword=admin -Dsecurity=true + + CleanUp-Runners: + needs: [Test-ISM-NoSec, Test-ALERTING-NoSec, Test-SQL-NoSec, Test-KNN-NoSec, Test-AD-NoSec, Test-SQL, Test-AD, Test-ALERTING] + if: always() + name: CleanUp-Runners + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_EC2_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_EC2_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: AWS Cli Processing + run: | + #!/bin/bash + # Please add comma at the end of the RUNNERS strings + # Don't add space after commas + RUNNERS+="odfe-tar-ism-nosec-arm64,odfe-tar-alerting-nosec-arm64,odfe-tar-sql-nosec-arm64,odfe-tar-knn-nosec-arm64,odfe-tar-ad-nosec-arm64," + RUNNERS+="odfe-tar-sql-arm64,odfe-tar-ad-arm64,odfe-tar-alerting-arm64" + release-tools/scripts/setup_runners.sh terminate $RUNNERS ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} diff --git a/release-tools/scripts/setup_runners_service.sh b/release-tools/scripts/setup_runners_service.sh index 1ddac498d..078b492b9 100755 --- a/release-tools/scripts/setup_runners_service.sh +++ b/release-tools/scripts/setup_runners_service.sh @@ -7,9 +7,10 @@ # # About: Setup ES/KIBANA for integTests on *NIX based ODFE distros w/wo Security # -# Usage: ./setup_runners_service.sh $SETUP_DISTRO $SETUP_ACTION +# Usage: ./setup_runners_service.sh $SETUP_DISTRO $SETUP_ACTION $ARCHITECTURE # $SETUP_DISTRO: zip | docker | deb | rpm (required) # $SETUP_ACTION: --es | --es-nosec | --kibana | --kibana-nosec (required) +# $ARCHITECTURE: arm64 (optional) # # Requirements: This script assumes java 14 is already installed on the servers # @@ -18,7 +19,7 @@ ############################################################################################### # This script allows users to manually assign parameters -if [ "$#" -ne 2 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ] +if [ "$#" -lt 2 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ] then echo "Please assign 2 parameters when running this script" echo "Example: $0 \$SETUP_DISTRO \$SETUP_ACTION" @@ -28,6 +29,7 @@ fi SETUP_DISTRO=$1 SETUP_ACTION=$2 +ARCHITECTURE="x64"; if [ ! -z "$3" ]; then ARCHITECTURE=$3; fi; echo ARCHITECTURE $ARCHITECTURE SETUP_PACKAGES="python3 git unzip wget jq" echo "install required packages" @@ -72,8 +74,8 @@ sudo chmod -R 777 /dev/shm if [ "$SETUP_DISTRO" = "zip" ] then mkdir -p $ES_ROOT - aws s3 cp s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/$ES_PACKAGE_NAME.tar.gz . --quiet; echo $? - tar -zxf $ES_PACKAGE_NAME.tar.gz -C $ES_ROOT --strip-components 1 + aws s3 cp s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/$ES_PACKAGE_NAME-linux-$ARCHITECTURE.tar.gz . --quiet; echo $? + tar -zxf $ES_PACKAGE_NAME-linux-$ARCHITECTURE.tar.gz -C $ES_ROOT --strip-components 1 fi if [ "$SETUP_DISTRO" = "docker" ] @@ -226,8 +228,8 @@ then if [ "$SETUP_DISTRO" = "zip" ] then mkdir -p $KIBANA_ROOT - aws s3 cp s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/$KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz . --quiet; echo $? - tar -zxf $KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz -C $KIBANA_ROOT --strip-components 1 + aws s3 cp s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/$KIBANA_PACKAGE_NAME-$OD_VERSION-linux-$ARCHITECTURE.tar.gz . --quiet; echo $? + tar -zxf $KIBANA_PACKAGE_NAME-$OD_VERSION-linux-$ARCHITECTURE.tar.gz -C $KIBANA_ROOT --strip-components 1 elif [ "$SETUP_DISTRO" = "docker" ] then echo "FROM opendistroforelasticsearch/opendistroforelasticsearch-kibana:$OD_VERSION" >> Dockerfile.kibana From 22e6126c3727313b123e25f0069cbf6ae8e2779b Mon Sep 17 00:00:00 2001 From: sreekarjami Date: Mon, 1 Feb 2021 22:22:04 +0000 Subject: [PATCH 2/8] workflow for arm64 tar test --- .github/workflows/staging-test-tar-arm64.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/staging-test-tar-arm64.yml b/.github/workflows/staging-test-tar-arm64.yml index ca0ad3931..47cb84a8f 100644 --- a/.github/workflows/staging-test-tar-arm64.yml +++ b/.github/workflows/staging-test-tar-arm64.yml @@ -49,7 +49,7 @@ jobs: Test-ISM-NoSec: needs: [Provision-Runners] - runs-on: ubuntu-18.04 + runs-on: [self-hosted, Linux, ARM64, odfe-tar-ism-nosec-arm64] name: Test-ISM-NoSec strategy: fail-fast: false @@ -84,7 +84,7 @@ jobs: Test-Alerting-NoSec: needs: [Provision-Runners] - runs-on: ubuntu-18.04 + runs-on: [self-hosted, Linux, ARM64, odfe-tar-alerting-nosec-arm64] name: Test-Alerting-NoSec strategy: fail-fast: false @@ -119,7 +119,7 @@ jobs: Test-SQL-NoSec: needs: [Provision-Runners] - runs-on: ubuntu-18.04 + runs-on: [self-hosted, Linux, ARM64, odfe-tar-sql-nosec-arm64] name: Test-SQL-NoSec strategy: fail-fast: false @@ -154,7 +154,7 @@ jobs: Test-KNN-NoSec: needs: [Provision-Runners] - runs-on: ubuntu-18.04 + runs-on: [self-hosted, Linux, ARM64, odfe-tar-knn-nosec-arm64] name: Test-KNN-NoSec strategy: fail-fast: false @@ -189,7 +189,7 @@ jobs: Test-AD-NoSec: needs: [Provision-Runners] - runs-on: ubuntu-18.04 + runs-on: [self-hosted, Linux, ARM64, odfe-tar-ad-nosec-arm64] name: Test-AD-NoSec strategy: fail-fast: false @@ -224,7 +224,7 @@ jobs: Test-SQL: needs: [Provision-Runners] - runs-on: ubuntu-18.04 + runs-on: [self-hosted, Linux, ARM64, odfe-tar-sql-arm64] name: Test-SQL strategy: fail-fast: false @@ -259,7 +259,7 @@ jobs: Test-AD: needs: [Provision-Runners] - runs-on: ubuntu-18.04 + runs-on: [self-hosted, Linux, ARM64, odfe-tar-ad-arm64] name: Test-AD strategy: fail-fast: false @@ -294,7 +294,7 @@ jobs: Test-ALERTING: needs: [Provision-Runners] - runs-on: [ubuntu-18.04] + runs-on: [self-hosted, Linux, ARM64, odfe-tar-alerting-arm64] name: Test-ALERTING strategy: fail-fast: false From 82613ec617a0b66bf9b9333f7554089164651676 Mon Sep 17 00:00:00 2001 From: sreekarjami Date: Mon, 1 Feb 2021 22:28:40 +0000 Subject: [PATCH 3/8] workflow for arm64 tar test --- .github/workflows/staging-build-tar.yml | 7 +- .github/workflows/staging-test-tar-arm64.yml | 74 ++------------------ 2 files changed, 9 insertions(+), 72 deletions(-) diff --git a/.github/workflows/staging-build-tar.yml b/.github/workflows/staging-build-tar.yml index ae1438fcc..299d4c1f2 100644 --- a/.github/workflows/staging-build-tar.yml +++ b/.github/workflows/staging-build-tar.yml @@ -3,8 +3,11 @@ name: Process TAR Artifacts on: # schedule: # - cron: '0 10 * * *' - repository_dispatch: - types: [staging-build-tar] +# repository_dispatch: +# types: [staging-build-tar] + push: + branches: [sreekarj_arm] + jobs: plugin-availability: diff --git a/.github/workflows/staging-test-tar-arm64.yml b/.github/workflows/staging-test-tar-arm64.yml index 47cb84a8f..fdae83e34 100644 --- a/.github/workflows/staging-test-tar-arm64.yml +++ b/.github/workflows/staging-test-tar-arm64.yml @@ -3,10 +3,9 @@ name: Process TAR Artifacts on: # schedule: # - cron: '0 10 * * *' -# repository_dispatch: -# types: [staging-build-tar] - push: - branches: [sreekarj_arm] + repository_dispatch: + types: [staging-build-tar] + jobs: plugin-availability: @@ -47,14 +46,11 @@ jobs: release-tools/scripts/setup_runners.sh run $RUNNERS ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} ami-055197d43e4ec7482 + Test-ISM-NoSec: needs: [Provision-Runners] runs-on: [self-hosted, Linux, ARM64, odfe-tar-ism-nosec-arm64] name: Test-ISM-NoSec - strategy: - fail-fast: false - matrix: - java: [14] steps: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 @@ -62,10 +58,6 @@ jobs: aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - uses: actions/checkout@v1 - name: Required Packages run: ./release-tools/scripts/required_packages.sh @@ -86,10 +78,6 @@ jobs: needs: [Provision-Runners] runs-on: [self-hosted, Linux, ARM64, odfe-tar-alerting-nosec-arm64] name: Test-Alerting-NoSec - strategy: - fail-fast: false - matrix: - java: [14] steps: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 @@ -97,10 +85,6 @@ jobs: aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - uses: actions/checkout@v1 - name: Required Packages run: ./release-tools/scripts/required_packages.sh @@ -121,10 +105,6 @@ jobs: needs: [Provision-Runners] runs-on: [self-hosted, Linux, ARM64, odfe-tar-sql-nosec-arm64] name: Test-SQL-NoSec - strategy: - fail-fast: false - matrix: - java: [14] steps: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 @@ -132,10 +112,6 @@ jobs: aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - uses: actions/checkout@v1 - name: Required Packages run: ./release-tools/scripts/required_packages.sh @@ -156,10 +132,6 @@ jobs: needs: [Provision-Runners] runs-on: [self-hosted, Linux, ARM64, odfe-tar-knn-nosec-arm64] name: Test-KNN-NoSec - strategy: - fail-fast: false - matrix: - java: [14] steps: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 @@ -167,10 +139,6 @@ jobs: aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - uses: actions/checkout@v1 - name: Required Packages run: ./release-tools/scripts/required_packages.sh @@ -191,10 +159,6 @@ jobs: needs: [Provision-Runners] runs-on: [self-hosted, Linux, ARM64, odfe-tar-ad-nosec-arm64] name: Test-AD-NoSec - strategy: - fail-fast: false - matrix: - java: [14] steps: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 @@ -202,10 +166,6 @@ jobs: aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - uses: actions/checkout@v1 - name: Required Packages run: ./release-tools/scripts/required_packages.sh @@ -226,10 +186,6 @@ jobs: needs: [Provision-Runners] runs-on: [self-hosted, Linux, ARM64, odfe-tar-sql-arm64] name: Test-SQL - strategy: - fail-fast: false - matrix: - java: [14] steps: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 @@ -237,10 +193,6 @@ jobs: aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - uses: actions/checkout@v1 - name: Required Packages run: ./release-tools/scripts/required_packages.sh @@ -261,10 +213,6 @@ jobs: needs: [Provision-Runners] runs-on: [self-hosted, Linux, ARM64, odfe-tar-ad-arm64] name: Test-AD - strategy: - fail-fast: false - matrix: - java: [14] steps: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 @@ -272,10 +220,6 @@ jobs: aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - uses: actions/checkout@v1 - name: Required Packages run: ./release-tools/scripts/required_packages.sh @@ -296,10 +240,6 @@ jobs: needs: [Provision-Runners] runs-on: [self-hosted, Linux, ARM64, odfe-tar-alerting-arm64] name: Test-ALERTING - strategy: - fail-fast: false - matrix: - java: [14] steps: - uses: actions/checkout@v1 - name: Set up AWS Cred @@ -317,12 +257,6 @@ jobs: with: repository: opendistro-for-elasticsearch/alerting ref: ${{env.p_tag_alerting}} - - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - name: IT run: | ./release-tools/scripts/setup_runners_service.sh zip --es arm64 From b9d543d72c06170f1e3aae2e6bcab87d20b9c796 Mon Sep 17 00:00:00 2001 From: sreekarjami Date: Mon, 1 Feb 2021 23:19:39 +0000 Subject: [PATCH 4/8] add tar arm64 IT workflow --- .github/workflows/staging-build-tar.yml | 7 ++----- release-tools/scripts/manifest.yml | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/staging-build-tar.yml b/.github/workflows/staging-build-tar.yml index 299d4c1f2..ae1438fcc 100644 --- a/.github/workflows/staging-build-tar.yml +++ b/.github/workflows/staging-build-tar.yml @@ -3,11 +3,8 @@ name: Process TAR Artifacts on: # schedule: # - cron: '0 10 * * *' -# repository_dispatch: -# types: [staging-build-tar] - push: - branches: [sreekarj_arm] - + repository_dispatch: + types: [staging-build-tar] jobs: plugin-availability: diff --git a/release-tools/scripts/manifest.yml b/release-tools/scripts/manifest.yml index f7706c6ef..e59bc1ac0 100644 --- a/release-tools/scripts/manifest.yml +++ b/release-tools/scripts/manifest.yml @@ -151,7 +151,7 @@ plugins: plugin_build: plugin_spec: [noplatform_noarch_zip,noplatform_noarch_deb,noplatform_noarch_rpm] plugin_category: elasticsearch-plugins - release_candidate: false + release_candidate: true plugin_location_staging: [default: "s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-plugins/asynchronous-search/"] plugin_location_prod: [noplatform_noarch_zip: "s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-plugins/opendistro-asynchronous-search/", noplatform_noarch_deb: "s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/debs/opendistro-asynchronous-search/", From 110913f04cc992d47b2fe934e62be0262869dd64 Mon Sep 17 00:00:00 2001 From: sreekarjami Date: Mon, 1 Feb 2021 23:28:59 +0000 Subject: [PATCH 5/8] change workflow name --- .github/workflows/staging-test-tar-arm64.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/staging-test-tar-arm64.yml b/.github/workflows/staging-test-tar-arm64.yml index fdae83e34..b4dc56c04 100644 --- a/.github/workflows/staging-test-tar-arm64.yml +++ b/.github/workflows/staging-test-tar-arm64.yml @@ -1,10 +1,10 @@ -name: Process TAR Artifacts +name: Test ARM64 TAR Artifacts on: # schedule: # - cron: '0 10 * * *' repository_dispatch: - types: [staging-build-tar] + types: [staging-test-arm64-tar] jobs: From c155acca3355097298e06da10c22f66e0454610c Mon Sep 17 00:00:00 2001 From: sreekarjami Date: Mon, 1 Feb 2021 23:43:31 +0000 Subject: [PATCH 6/8] change workflow name --- .github/workflows/staging-build-tar-arm64.yml | 2 +- .github/workflows/staging-test-tar-arm64.yml | 18 +++++++++--------- release-tools/scripts/setup_runners_service.sh | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/staging-build-tar-arm64.yml b/.github/workflows/staging-build-tar-arm64.yml index 5381033ed..1ff2e0b7a 100644 --- a/.github/workflows/staging-build-tar-arm64.yml +++ b/.github/workflows/staging-build-tar-arm64.yml @@ -4,7 +4,7 @@ on: # schedule: # - cron: '0 10 * * *' repository_dispatch: - types: [staging-build-tar] + types: [staging-build-tar-arm64] jobs: plugin-availability: diff --git a/.github/workflows/staging-test-tar-arm64.yml b/.github/workflows/staging-test-tar-arm64.yml index b4dc56c04..1cf3f810f 100644 --- a/.github/workflows/staging-test-tar-arm64.yml +++ b/.github/workflows/staging-test-tar-arm64.yml @@ -4,7 +4,7 @@ on: # schedule: # - cron: '0 10 * * *' repository_dispatch: - types: [staging-test-arm64-tar] + types: [staging-test-tar-arm64] jobs: @@ -68,7 +68,7 @@ jobs: with: repository: opendistro-for-elasticsearch/index-management ref: ${{env.p_tag_ism}} - - name: IT + - name: Integration Test run: | ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../index-management; pwd @@ -95,7 +95,7 @@ jobs: with: repository: opendistro-for-elasticsearch/alerting ref: ${{env.p_tag_alerting}} - - name: IT + - name: Integration Test run: | ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../alerting/alerting; pwd @@ -122,7 +122,7 @@ jobs: with: repository: opendistro-for-elasticsearch/sql ref: ${{env.p_tag_sql}} - - name: IT + - name: Integration Test run: | ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../sql; pwd @@ -149,7 +149,7 @@ jobs: with: repository: opendistro-for-elasticsearch/k-NN ref: ${{env.p_tag_knn}} - - name: IT + - name: Integration Test run: | ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../k-NN; pwd @@ -176,7 +176,7 @@ jobs: with: repository: opendistro-for-elasticsearch/anomaly-detection ref: ${{env.p_tag_ad}} - - name: IT + - name: Integration Test run: | ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../anomaly-detection; pwd @@ -203,7 +203,7 @@ jobs: with: repository: opendistro-for-elasticsearch/sql ref: ${{env.p_tag_sql}} - - name: IT + - name: Integration Test run: | ./release-tools/scripts/setup_runners_service.sh zip --es arm64 export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../sql; pwd @@ -230,7 +230,7 @@ jobs: with: repository: opendistro-for-elasticsearch/anomaly-detection ref: ${{env.p_tag_ad}} - - name: IT + - name: Integration Test run: | ./release-tools/scripts/setup_runners_service.sh zip --es arm64 export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../anomaly-detection; pwd @@ -257,7 +257,7 @@ jobs: with: repository: opendistro-for-elasticsearch/alerting ref: ${{env.p_tag_alerting}} - - name: IT + - name: Integration Test run: | ./release-tools/scripts/setup_runners_service.sh zip --es arm64 export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../alerting/alerting; pwd diff --git a/release-tools/scripts/setup_runners_service.sh b/release-tools/scripts/setup_runners_service.sh index 078b492b9..f7fd6932e 100755 --- a/release-tools/scripts/setup_runners_service.sh +++ b/release-tools/scripts/setup_runners_service.sh @@ -23,7 +23,7 @@ if [ "$#" -lt 2 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ] then echo "Please assign 2 parameters when running this script" echo "Example: $0 \$SETUP_DISTRO \$SETUP_ACTION" - echo "Example: $0 \"zip | docker | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec\"" + echo "Example: $0 \"zip | docker | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec \" \"ARCHITECTURE (optional)\"" exit 1 fi From 656440a2ac99caf2a1e05ce2cb5cb7e83552b38f Mon Sep 17 00:00:00 2001 From: sreekarjami Date: Mon, 1 Feb 2021 23:50:24 +0000 Subject: [PATCH 7/8] change workflow name --- release-tools/scripts/setup_runners_service.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-tools/scripts/setup_runners_service.sh b/release-tools/scripts/setup_runners_service.sh index f7fd6932e..e451cc5a3 100755 --- a/release-tools/scripts/setup_runners_service.sh +++ b/release-tools/scripts/setup_runners_service.sh @@ -21,7 +21,7 @@ # This script allows users to manually assign parameters if [ "$#" -lt 2 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ] then - echo "Please assign 2 parameters when running this script" + echo "Please assign atleast 2 parameters when running this script" echo "Example: $0 \$SETUP_DISTRO \$SETUP_ACTION" echo "Example: $0 \"zip | docker | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec \" \"ARCHITECTURE (optional)\"" exit 1 From 23790cdabd4de35c539cf719e54b866af81f16d2 Mon Sep 17 00:00:00 2001 From: sreekarjami Date: Tue, 2 Feb 2021 00:17:20 +0000 Subject: [PATCH 8/8] change workflow name --- release-tools/scripts/setup_runners_service.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-tools/scripts/setup_runners_service.sh b/release-tools/scripts/setup_runners_service.sh index e451cc5a3..b96921223 100755 --- a/release-tools/scripts/setup_runners_service.sh +++ b/release-tools/scripts/setup_runners_service.sh @@ -22,8 +22,8 @@ if [ "$#" -lt 2 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ] then echo "Please assign atleast 2 parameters when running this script" - echo "Example: $0 \$SETUP_DISTRO \$SETUP_ACTION" - echo "Example: $0 \"zip | docker | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec \" \"ARCHITECTURE (optional)\"" + echo "Example: $0 \$SETUP_DISTRO \$SETUP_ACTION \$ARCHITECTURE (optional)" + echo "Example: $0 \"zip | docker | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec \" \"arm64 (optional)\"" exit 1 fi