diff --git a/.github/workflows/staging-build-tar-arm64.yml b/.github/workflows/staging-build-tar-arm64.yml index 5381033ed..1ff2e0b7a 100644 --- a/.github/workflows/staging-build-tar-arm64.yml +++ b/.github/workflows/staging-build-tar-arm64.yml @@ -4,7 +4,7 @@ on: # schedule: # - cron: '0 10 * * *' repository_dispatch: - types: [staging-build-tar] + types: [staging-build-tar-arm64] jobs: plugin-availability: diff --git a/.github/workflows/staging-test-tar-arm64.yml b/.github/workflows/staging-test-tar-arm64.yml new file mode 100644 index 000000000..1cf3f810f --- /dev/null +++ b/.github/workflows/staging-test-tar-arm64.yml @@ -0,0 +1,287 @@ +name: Test ARM64 TAR Artifacts + +on: + # schedule: + # - cron: '0 10 * * *' + repository_dispatch: + types: [staging-test-tar-arm64] + + +jobs: + plugin-availability: + name: Check Plugin Availability + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Run check_plugin scripts + run: ./release-tools/scripts/check_plugin.sh; exit `cat /tmp/plugin_status.check` + + + Provision-Runners: + name: Provision-Runners + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_EC2_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_EC2_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: AWS Cli Processing + run: | + #!/bin/bash + # Please add comma at the end of the RUNNERS strings + # Don't add space after commas + RUNNERS+="odfe-tar-ism-nosec-arm64,odfe-tar-alerting-nosec-arm64,odfe-tar-sql-nosec-arm64,odfe-tar-knn-nosec-arm64,odfe-tar-ad-nosec-arm64," + RUNNERS+="odfe-tar-sql-arm64,odfe-tar-ad-arm64,odfe-tar-alerting-arm64" + release-tools/scripts/setup_runners.sh run $RUNNERS ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} ami-055197d43e4ec7482 + + + + Test-ISM-NoSec: + needs: [Provision-Runners] + runs-on: [self-hosted, Linux, ARM64, odfe-tar-ism-nosec-arm64] + name: Test-ISM-NoSec + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_ism=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/index-management)" >> $GITHUB_ENV + - name: Checkout ISM + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/index-management + ref: ${{env.p_tag_ism}} + - name: Integration Test + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../index-management; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest + + Test-Alerting-NoSec: + needs: [Provision-Runners] + runs-on: [self-hosted, Linux, ARM64, odfe-tar-alerting-nosec-arm64] + name: Test-Alerting-NoSec + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_alerting=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/alerting)" >> $GITHUB_ENV + - name: Checkout Alerting + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/alerting + ref: ${{env.p_tag_alerting}} + - name: Integration Test + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../alerting/alerting; pwd + ../gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest + + Test-SQL-NoSec: + needs: [Provision-Runners] + runs-on: [self-hosted, Linux, ARM64, odfe-tar-sql-nosec-arm64] + name: Test-SQL-NoSec + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_sql=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql)" >> $GITHUB_ENV + - name: Checkout SQL + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/sql + ref: ${{env.p_tag_sql}} + - name: Integration Test + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../sql; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest + + Test-KNN-NoSec: + needs: [Provision-Runners] + runs-on: [self-hosted, Linux, ARM64, odfe-tar-knn-nosec-arm64] + name: Test-KNN-NoSec + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_knn=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/k-NN)" >> $GITHUB_ENV + - name: Checkout k-NN + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/k-NN + ref: ${{env.p_tag_knn}} + - name: Integration Test + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../k-NN; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest + + Test-AD-NoSec: + needs: [Provision-Runners] + runs-on: [self-hosted, Linux, ARM64, odfe-tar-ad-nosec-arm64] + name: Test-AD-NoSec + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_ad=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection)" >> $GITHUB_ENV + - name: Checkout anomaly-detection + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/anomaly-detection + ref: ${{env.p_tag_ad}} + - name: Integration Test + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../anomaly-detection; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="es-integrationtest" + + Test-SQL: + needs: [Provision-Runners] + runs-on: [self-hosted, Linux, ARM64, odfe-tar-sql-arm64] + name: Test-SQL + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_sql=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql)" >> $GITHUB_ENV + - name: Checkout SQL + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/sql + ref: ${{env.p_tag_sql}} + - name: Integration Test + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../sql; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest -Dhttps=true -Duser=admin -Dpassword=admin + + Test-AD: + needs: [Provision-Runners] + runs-on: [self-hosted, Linux, ARM64, odfe-tar-ad-arm64] + name: Test-AD + steps: + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_ad=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection)" >> $GITHUB_ENV + - name: Checkout anomaly-detection + uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/anomaly-detection + ref: ${{env.p_tag_ad}} + - name: Integration Test + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../anomaly-detection; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="es-integrationtest" -Dhttps=true -Duser=admin -Dpassword=admin + + Test-ALERTING: + needs: [Provision-Runners] + runs-on: [self-hosted, Linux, ARM64, odfe-tar-alerting-arm64] + name: Test-ALERTING + steps: + - uses: actions/checkout@v1 + - name: Set up AWS Cred + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh + - name: Retrieve plugin tags + run: echo "p_tag_alerting=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/alerting)" >> $GITHUB_ENV + + - uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/alerting + ref: ${{env.p_tag_alerting}} + - name: Integration Test + run: | + ./release-tools/scripts/setup_runners_service.sh zip --es arm64 + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../alerting/alerting; pwd + ls -ltr + ../gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest -Dhttps=true -Duser=admin -Dpassword=admin -Dsecurity=true + + CleanUp-Runners: + needs: [Test-ISM-NoSec, Test-ALERTING-NoSec, Test-SQL-NoSec, Test-KNN-NoSec, Test-AD-NoSec, Test-SQL, Test-AD, Test-ALERTING] + if: always() + name: CleanUp-Runners + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_EC2_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_EC2_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: AWS Cli Processing + run: | + #!/bin/bash + # Please add comma at the end of the RUNNERS strings + # Don't add space after commas + RUNNERS+="odfe-tar-ism-nosec-arm64,odfe-tar-alerting-nosec-arm64,odfe-tar-sql-nosec-arm64,odfe-tar-knn-nosec-arm64,odfe-tar-ad-nosec-arm64," + RUNNERS+="odfe-tar-sql-arm64,odfe-tar-ad-arm64,odfe-tar-alerting-arm64" + release-tools/scripts/setup_runners.sh terminate $RUNNERS ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} diff --git a/release-tools/scripts/manifest.yml b/release-tools/scripts/manifest.yml index f7706c6ef..e59bc1ac0 100644 --- a/release-tools/scripts/manifest.yml +++ b/release-tools/scripts/manifest.yml @@ -151,7 +151,7 @@ plugins: plugin_build: plugin_spec: [noplatform_noarch_zip,noplatform_noarch_deb,noplatform_noarch_rpm] plugin_category: elasticsearch-plugins - release_candidate: false + release_candidate: true plugin_location_staging: [default: "s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-plugins/asynchronous-search/"] plugin_location_prod: [noplatform_noarch_zip: "s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-plugins/opendistro-asynchronous-search/", noplatform_noarch_deb: "s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/debs/opendistro-asynchronous-search/", diff --git a/release-tools/scripts/setup_runners_service.sh b/release-tools/scripts/setup_runners_service.sh index 1ddac498d..b96921223 100755 --- a/release-tools/scripts/setup_runners_service.sh +++ b/release-tools/scripts/setup_runners_service.sh @@ -7,9 +7,10 @@ # # About: Setup ES/KIBANA for integTests on *NIX based ODFE distros w/wo Security # -# Usage: ./setup_runners_service.sh $SETUP_DISTRO $SETUP_ACTION +# Usage: ./setup_runners_service.sh $SETUP_DISTRO $SETUP_ACTION $ARCHITECTURE # $SETUP_DISTRO: zip | docker | deb | rpm (required) # $SETUP_ACTION: --es | --es-nosec | --kibana | --kibana-nosec (required) +# $ARCHITECTURE: arm64 (optional) # # Requirements: This script assumes java 14 is already installed on the servers # @@ -18,16 +19,17 @@ ############################################################################################### # This script allows users to manually assign parameters -if [ "$#" -ne 2 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ] +if [ "$#" -lt 2 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ] then - echo "Please assign 2 parameters when running this script" - echo "Example: $0 \$SETUP_DISTRO \$SETUP_ACTION" - echo "Example: $0 \"zip | docker | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec\"" + echo "Please assign atleast 2 parameters when running this script" + echo "Example: $0 \$SETUP_DISTRO \$SETUP_ACTION \$ARCHITECTURE (optional)" + echo "Example: $0 \"zip | docker | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec \" \"arm64 (optional)\"" exit 1 fi SETUP_DISTRO=$1 SETUP_ACTION=$2 +ARCHITECTURE="x64"; if [ ! -z "$3" ]; then ARCHITECTURE=$3; fi; echo ARCHITECTURE $ARCHITECTURE SETUP_PACKAGES="python3 git unzip wget jq" echo "install required packages" @@ -72,8 +74,8 @@ sudo chmod -R 777 /dev/shm if [ "$SETUP_DISTRO" = "zip" ] then mkdir -p $ES_ROOT - aws s3 cp s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/$ES_PACKAGE_NAME.tar.gz . --quiet; echo $? - tar -zxf $ES_PACKAGE_NAME.tar.gz -C $ES_ROOT --strip-components 1 + aws s3 cp s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/$ES_PACKAGE_NAME-linux-$ARCHITECTURE.tar.gz . --quiet; echo $? + tar -zxf $ES_PACKAGE_NAME-linux-$ARCHITECTURE.tar.gz -C $ES_ROOT --strip-components 1 fi if [ "$SETUP_DISTRO" = "docker" ] @@ -226,8 +228,8 @@ then if [ "$SETUP_DISTRO" = "zip" ] then mkdir -p $KIBANA_ROOT - aws s3 cp s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/$KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz . --quiet; echo $? - tar -zxf $KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz -C $KIBANA_ROOT --strip-components 1 + aws s3 cp s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/$KIBANA_PACKAGE_NAME-$OD_VERSION-linux-$ARCHITECTURE.tar.gz . --quiet; echo $? + tar -zxf $KIBANA_PACKAGE_NAME-$OD_VERSION-linux-$ARCHITECTURE.tar.gz -C $KIBANA_ROOT --strip-components 1 elif [ "$SETUP_DISTRO" = "docker" ] then echo "FROM opendistroforelasticsearch/opendistroforelasticsearch-kibana:$OD_VERSION" >> Dockerfile.kibana