diff --git a/.github/workflows/staging-build-tar.yml b/.github/workflows/staging-build-tar.yml index 98d354cc2..b60be8bc4 100644 --- a/.github/workflows/staging-build-tar.yml +++ b/.github/workflows/staging-build-tar.yml @@ -15,12 +15,13 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh - name: Run check_plugin scripts - run: .github/scripts/check_plugin.sh "elasticsearch_zip,kibana_zip"; exit `cat /tmp/plugin_status.check` + run: ./release-tools/scripts/check_plugin.sh "elasticsearch_zip,kibana_zip"; exit `cat /tmp/plugin_status.check` build-es-artifacts: needs: [plugin-availability] @@ -33,9 +34,14 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 + - name: Required Packages + run: | + apt-get update + apt-get install sudo -y + ./release-tools/scripts/required_packages.sh - name: Build tar run: apt install -y jq unzip; set -u; export JAVA_HOME=/openjdk12; export PATH=$JAVA_HOME:$PATH; ./elasticsearch/linux_distributions/opendistro-tar-build.sh @@ -51,9 +57,14 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 + - name: Required Packages + run: | + apt-get update + apt-get install sudo -y + ./release-tools/scripts/required_packages.sh - name: Build Kibana run: apt install -y jq; ./kibana/linux_distributions/opendistro-kibana-build.sh tar @@ -69,16 +80,18 @@ jobs: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Setup Java uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh - name: Retrieve plugin tags - run: echo "p_tag_ism=$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/index-management)" >> $GITHUB_ENV + run: echo "p_tag_ism=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/index-management)" >> $GITHUB_ENV - name: Checkout ISM uses: actions/checkout@v1 with: @@ -86,7 +99,7 @@ jobs: ref: ${{env.p_tag_ism}} - name: IT run: | - .github/scripts/setup_runners_service.sh zip --es-nosec + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../index-management; pwd ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest @@ -102,16 +115,18 @@ jobs: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Setup Java uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh - name: Retrieve plugin tags - run: echo "p_tag_alerting=$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/alerting)" >> $GITHUB_ENV + run: echo "p_tag_alerting=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/alerting)" >> $GITHUB_ENV - name: Checkout Alerting uses: actions/checkout@v1 with: @@ -119,7 +134,7 @@ jobs: ref: ${{env.p_tag_alerting}} - name: IT run: | - .github/scripts/setup_runners_service.sh zip --es-nosec + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../alerting/alerting; pwd ../gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest @@ -135,16 +150,18 @@ jobs: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Setup Java uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh - name: Retrieve plugin tags - run: echo "p_tag_sql=$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql)" >> $GITHUB_ENV + run: echo "p_tag_sql=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql)" >> $GITHUB_ENV - name: Checkout SQL uses: actions/checkout@v1 with: @@ -152,7 +169,7 @@ jobs: ref: ${{env.p_tag_sql}} - name: IT run: | - .github/scripts/setup_runners_service.sh zip --es-nosec + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../sql; pwd ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest @@ -168,16 +185,18 @@ jobs: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Setup Java uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh - name: Retrieve plugin tags - run: echo "p_tag_knn=$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/k-NN)" >> $GITHUB_ENV + run: echo "p_tag_knn=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/k-NN)" >> $GITHUB_ENV - name: Checkout k-NN uses: actions/checkout@v1 with: @@ -185,7 +204,7 @@ jobs: ref: ${{env.p_tag_knn}} - name: IT run: | - .github/scripts/setup_runners_service.sh zip --es-nosec + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../k-NN; pwd ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest @@ -201,16 +220,18 @@ jobs: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Setup Java uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh - name: Retrieve plugin tags - run: echo "p_tag_ad=$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection)" >> $GITHUB_ENV + run: echo "p_tag_ad=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection)" >> $GITHUB_ENV - name: Checkout anomaly-detection uses: actions/checkout@v1 with: @@ -218,7 +239,7 @@ jobs: ref: ${{env.p_tag_ad}} - name: IT run: | - .github/scripts/setup_runners_service.sh zip --es-nosec + ./release-tools/scripts/setup_runners_service.sh zip --es-nosec export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../anomaly-detection; pwd ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="es-integrationtest" @@ -234,16 +255,18 @@ jobs: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Setup Java uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh - name: Retrieve plugin tags - run: echo "p_tag_sql=$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql)" >> $GITHUB_ENV + run: echo "p_tag_sql=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql)" >> $GITHUB_ENV - name: Checkout SQL uses: actions/checkout@v1 with: @@ -251,7 +274,7 @@ jobs: ref: ${{env.p_tag_sql}} - name: IT run: | - .github/scripts/setup_runners_service.sh zip --es + ./release-tools/scripts/setup_runners_service.sh zip --es export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../sql; pwd ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest -Dhttps=true -Duser=admin -Dpassword=admin @@ -267,16 +290,18 @@ jobs: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Setup Java uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - uses: actions/checkout@v1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh - name: Retrieve plugin tags - run: echo "p_tag_ad=$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection)" >> $GITHUB_ENV + run: echo "p_tag_ad=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection)" >> $GITHUB_ENV - name: Checkout anomaly-detection uses: actions/checkout@v1 with: @@ -284,7 +309,7 @@ jobs: ref: ${{env.p_tag_ad}} - name: IT run: | - .github/scripts/setup_runners_service.sh zip --es + ./release-tools/scripts/setup_runners_service.sh zip --es export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../anomaly-detection; pwd ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="es-integrationtest" -Dhttps=true -Duser=admin -Dpassword=admin @@ -301,12 +326,13 @@ jobs: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh - name: Retrieve plugin tags - run: echo "p_tag_alerting=$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/alerting)" >> $GITHUB_ENV + run: echo "p_tag_alerting=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/alerting)" >> $GITHUB_ENV - uses: actions/checkout@v1 with: @@ -320,7 +346,7 @@ jobs: - name: IT run: | - .github/scripts/setup_runners_service.sh zip --es + ./release-tools/scripts/setup_runners_service.sh zip --es export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../alerting/alerting; pwd ls -ltr ../gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest -Dhttps=true -Duser=admin -Dpassword=admin -Dsecurity=true @@ -338,13 +364,15 @@ jobs: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh - name: Retrieve plugin tags run: | - echo "p_tag_ad=$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection-kibana-plugin)" >> $GITHUB_ENV - echo "es_version=$(./bin/version-info --es)" >> $GITHUB_ENV + echo "p_tag_ad=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection-kibana-plugin)" >> $GITHUB_ENV + echo "es_version=$(release-tools/scripts/version-info.sh --es)" >> $GITHUB_ENV - name: Checkout Kibana uses: actions/checkout@v2 @@ -389,7 +417,7 @@ jobs: yarn kbn bootstrap - name: Start ES and Kibana - run: .github/scripts/setup_runners_service.sh zip --kibana-nosec + run: ./release-tools/scripts/setup_runners_service.sh zip --kibana-nosec - name: run IT uses: cypress-io/github-action@v2.5.0 @@ -410,13 +438,15 @@ jobs: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh - name: Retrieve plugin tags run: | - echo "p_tag_sql_kibana=$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql)" >> $GITHUB_ENV - echo "es_version=$(./bin/version-info --es)" >> $GITHUB_ENV + echo "p_tag_sql_kibana=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql)" >> $GITHUB_ENV + echo "es_version=$(release-tools/scripts/version-info.sh --es)" >> $GITHUB_ENV - name: Checkout Kibana uses: actions/checkout@v2 @@ -474,7 +504,7 @@ jobs: - name: Start ES and Kibana run: | - .github/scripts/setup_runners_service.sh zip --kibana-nosec + ./release-tools/scripts/setup_runners_service.sh zip --kibana-nosec echo "load the indices" curl -s https://raw.githubusercontent.com/opendistro-for-elasticsearch/sql/master/integ-test/src/test/resources/accounts.json | curl -s -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/accounts/_bulk?pretty' --data-binary @- > /dev/null 2>&1 curl -s https://raw.githubusercontent.com/opendistro-for-elasticsearch/sql/master/integ-test/src/test/resources/employee_nested.json | curl -s -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/employee_nested/_bulk?pretty' --data-binary @- > /dev/null 2>&1 @@ -498,13 +528,15 @@ jobs: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh - name: Retrieve plugin tags run: | - echo "p_tag_ad=$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection-kibana-plugin)" >> $GITHUB_ENV - echo "es_version=$(./bin/version-info --es)" >> $GITHUB_ENV + echo "p_tag_ad=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection-kibana-plugin)" >> $GITHUB_ENV + echo "es_version=$(release-tools/scripts/version-info.sh --es)" >> $GITHUB_ENV - name: Checkout Kibana uses: actions/checkout@v2 @@ -549,7 +581,7 @@ jobs: yarn kbn bootstrap - name: Start ES and Kibana - run: .github/scripts/setup_runners_service.sh zip --kibana + run: ./release-tools/scripts/setup_runners_service.sh zip --kibana - name: run IT uses: cypress-io/github-action@v2.5.0 @@ -570,14 +602,15 @@ jobs: - name: Set up AWS Cred uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} aws-region: us-east-1 - + - name: Required Packages + run: ./release-tools/scripts/required_packages.sh - name: Retrieve plugin tags run: | echo "p_tag_sec_kibana=$(release-tools/scripts/plugin_tag.sh opendistro-for-elasticsearch/security-kibana-plugin)" >> $GITHUB_ENV - echo "es_version=$(release-tools/scripts/version-info.py --es)" >> $GITHUB_ENV + echo "es_version=$(release-tools/scripts/version-info.sh --es)" >> $GITHUB_ENV - name: Checkout Kibana uses: actions/checkout@v2 diff --git a/elasticsearch/linux_distributions/opendistro-tar-build.sh b/elasticsearch/linux_distributions/opendistro-tar-build.sh index 5faf5cfce..a699e4daf 100755 --- a/elasticsearch/linux_distributions/opendistro-tar-build.sh +++ b/elasticsearch/linux_distributions/opendistro-tar-build.sh @@ -18,26 +18,36 @@ set -e REPO_ROOT=`git rev-parse --show-toplevel` ROOT=`dirname $(realpath $0)`; echo $ROOT; cd $ROOT -ES_VERSION=`$REPO_ROOT/bin/version-info --es`; echo ES_VERSION: $ES_VERSION -OD_VERSION=`$REPO_ROOT/bin/version-info --od`; echo OD_VERSION: $OD_VERSION -IS_CUT=`$REPO_ROOT/bin/version-info --is-cut`; echo IS_CUT: $IS_CUT -S3_BUCKET="artifacts.opendistroforelasticsearch.amazon.com" -ARTIFACTS_URL="https://d3g5vo6xdbdb9a.cloudfront.net" +MANIFEST_FILE=$REPO_ROOT/release-tools/scripts/manifest.yml +ES_VERSION=`$REPO_ROOT/release-tools/scripts/version-info.sh --es`; echo ES_VERSION: $ES_VERSION +OD_VERSION=`$REPO_ROOT/release-tools/scripts/version-info.sh --od`; echo OD_VERSION: $OD_VERSION +S3_RELEASE_BASEURL=`yq eval '.urls.ODFE.releases' $MANIFEST_FILE` +S3_RELEASE_FINAL_BUILD=`yq eval '.urls.ODFE.releases_final_build' $MANIFEST_FILE | sed 's/\///g'` +S3_RELEASE_BUCKET=`echo $S3_RELEASE_BASEURL | awk -F '/' '{print $3}'` PACKAGE_NAME="opendistroforelasticsearch" TARGET_DIR="$ROOT/target" plugin_version=$OD_VERSION -knnlib_version=$OD_VERSION # knnlib version only for tar distros here +# knnlib version only for tar distros here +knnlib_version=`$REPO_ROOT/release-tools/scripts/plugin_parser.sh opendistro-knnlib plugin_version`; echo knnlib_version: $knnlib_version # Please DO NOT change the orders, they have dependencies -PLUGINS=`$REPO_ROOT/bin/plugins-info elasticsearch zip --require-install-true` +PLUGINS=`$REPO_ROOT/release-tools/scripts/plugins-info.sh elasticsearch-plugins plugin_basename` PLUGINS_ARRAY=( $PLUGINS ) -CUT_VERSIONS=`$REPO_ROOT/bin/plugins-info elasticsearch cutversion --require-install-true` -CUT_VERSIONS_ARRAY=( $CUT_VERSIONS ) +PLUGIN_PATH=`yq eval '.urls.ODFE.releases' $MANIFEST_FILE | sed "s/^.*$S3_RELEASE_BUCKET\///g"` basedir="${ROOT}/${PACKAGE_NAME}-${OD_VERSION}/plugins" echo $ROOT +if [ -z "$S3_RELEASE_FINAL_BUILD" ] +then + S3_RELEASE_BUILD=`aws s3api list-objects --bucket $S3_RELEASE_BUCKET --prefix "${PLUGIN_PATH}${OD_VERSION}" --query 'Contents[].[Key]' --output text | awk -F '/' '{print $3}' | uniq | tail -n 1` + echo Latest: $S3_RELEASE_BUILD +else + S3_RELEASE_BUILD=$S3_RELEASE_FINAL_BUILD + echo Final: $S3_RELEASE_BUILD +fi + if [ -z "$PLUGINS" ]; then echo "Provide plugin list to install (separated by space)" exit 1 @@ -54,65 +64,66 @@ tar -xzf elasticsearch-oss-$ES_VERSION-linux-x86_64.tar.gz --strip-components=1 cp -v opendistro-tar-install.sh $PACKAGE_NAME-$OD_VERSION # Install Plugin +rm -rf /tmp/plugins +mkdir -p /tmp/plugins + for index in ${!PLUGINS_ARRAY[@]} do - if [ "$IS_CUT" = "true" ] - then - plugin_version=${CUT_VERSIONS_ARRAY[$index]} - fi + plugin_latest=`aws s3api list-objects --bucket $S3_RELEASE_BUCKET --prefix "${PLUGIN_PATH}${OD_VERSION}/$S3_RELEASE_BUILD/elasticsearch-plugins" --query 'Contents[].[Key]' --output text | grep -v sha512 | grep ${PLUGINS_ARRAY[$index]} | grep zip | sort | tail -n 1` - plugin_path=${PLUGINS_ARRAY[$index]} - plugin_latest=`aws s3api list-objects --bucket $S3_BUCKET --prefix "downloads/elasticsearch-plugins/${plugin_path}-${plugin_version}" --query 'Contents[].[Key]' --output text | sort | tail -n 1` - - if [ "$plugin_path" != "none" ] + if [ ! -z "$plugin_latest" ] then - echo "installing $plugin_latest" - $PACKAGE_NAME-$OD_VERSION/bin/elasticsearch-plugin install --batch "${ARTIFACTS_URL}/${plugin_latest}"; \ + echo "downloading $plugin_latest" + plugin_path=${PLUGINS_ARRAY[$index]} + echo "plugin path: $plugin_path" + aws s3 cp "s3://${S3_RELEASE_BUCKET}/${plugin_latest}" "/tmp/plugins" --quiet; echo $? + rc_candiate=`$REPO_ROOT/release-tools/scripts/plugin_parser.sh $index release_candidate` + if $rc_candiate + then + plugin=`echo $plugin_latest | awk -F '/' '{print $NF}'` + echo "installing $plugin" + $PACKAGE_NAME-$OD_VERSION/bin/elasticsearch-plugin install --batch file:/tmp/plugins/$plugin; \ + fi fi done + # List Plugins echo "List available plugins" ls -lrt $basedir # Move performance-analyzer-rca folder -cp -r $PACKAGE_NAME-$OD_VERSION/plugins/opendistro_performance_analyzer/performance-analyzer-rca $PACKAGE_NAME-$OD_VERSION -chmod -R 755 ${PACKAGE_NAME}-${OD_VERSION}/performance-analyzer-rca +perf_dir=`ls -p $basedir | grep performance` +rca_dir=`ls -p $basedir/$perf_dir | grep rca/` +cp -r $PACKAGE_NAME-$OD_VERSION/plugins/$perf_dir$rca_dir $PACKAGE_NAME-$OD_VERSION +chmod -R 755 ${PACKAGE_NAME}-${OD_VERSION}/$rca_dir + # Move agent script directly into ES_HOME/bin -mv $PACKAGE_NAME-$OD_VERSION/bin/opendistro_performance_analyzer/performance-analyzer-agent-cli $PACKAGE_NAME-$OD_VERSION/bin +perf_analyzer=`ls -p $PACKAGE_NAME-$OD_VERSION/bin/ | grep performance` +mv $PACKAGE_NAME-$OD_VERSION/bin/$perf_analyzer/performance-analyzer-agent-cli $PACKAGE_NAME-$OD_VERSION/bin rm -rf $PACKAGE_NAME-$OD_VERSION/bin/opendistro_performance_analyzer + # Make sure the data folder exists and is writable mkdir -p ${PACKAGE_NAME}-${OD_VERSION}/data chmod 755 ${PACKAGE_NAME}-${OD_VERSION}/data/ # Download Knn lib -# Get knnlib artifact information from plugins.json -plugin_array_noinstall=( `$REPO_ROOT/bin/plugins-info elasticsearch zip --require-install-false` ) -cutversion_array_noinstall=( `$REPO_ROOT/bin/plugins-info elasticsearch cutversion --require-install-false` ) -for index_noinstall in ${!plugin_array_noinstall[@]} -do - if echo ${plugin_array_noinstall[$index_noinstall]} | grep -qi "knnlib" - then - knnlib_path=${plugin_array_noinstall[$index_noinstall]} - - if [ "$IS_CUT" = "true" ] - then - knnlib_version=${cutversion_array_noinstall[$index_noinstall]} - break - fi - fi -done -knnlib_latest=`aws s3api list-objects --bucket $S3_BUCKET --prefix "downloads/${knnlib_path}-${knnlib_version}" --query 'Contents[].[Key]' --output text | sort | tail -n 1` -echo "downloading $knnlib_latest" -aws s3 cp "s3://artifacts.opendistroforelasticsearch.amazon.com/${knnlib_latest}" ./ -unzip opendistro-knnlib*.zip -mkdir -p $PACKAGE_NAME-$OD_VERSION/plugins/opendistro-knn/knn-lib/ -mv -v opendistro-knnlib*/libKNNIndex*.so $PACKAGE_NAME-$OD_VERSION/plugins/opendistro-knn/knn-lib/ +# Get knnlib artifact information from Manifest +knnlib_is_rc=`$REPO_ROOT/release-tools/scripts/plugin_parser.sh opendistro-knnlib release_candidate` +if $knnlib_is_rc +then + echo "" + knnlib_latest=`aws s3api list-objects --bucket $S3_RELEASE_BUCKET --prefix "${PLUGIN_PATH}${OD_VERSION}/$S3_RELEASE_BUILD/opendistro-libs/" --query 'Contents[].[Key]' --output text | grep -v sha512 | grep opendistro-knnlib | grep zip | sort | tail -n 1` + echo "downloading $knnlib_latest" + aws s3 cp "s3://${S3_RELEASE_BUCKET}/$knnlib_latest" ./ + unzip opendistro-knnlib*.zip + mkdir -p $PACKAGE_NAME-$OD_VERSION/plugins/opendistro-knn/knn-lib/ + mv -v opendistro-knnlib*/libKNNIndex*.so $PACKAGE_NAME-$OD_VERSION/plugins/opendistro-knn/knn-lib/ +fi # Tar generation echo "generating tar" tar -czf $TARGET_DIR/$PACKAGE_NAME-$OD_VERSION.tar.gz $PACKAGE_NAME-$OD_VERSION -#tar -tavf $TARGET_DIR/$PACKAGE_NAME-$OD_VERSION.tar.gz cd $TARGET_DIR shasum -a 512 $PACKAGE_NAME-$OD_VERSION.tar.gz > $PACKAGE_NAME-$OD_VERSION.tar.gz.sha512 shasum -a 512 -c $PACKAGE_NAME-$OD_VERSION.tar.gz.sha512 @@ -125,7 +136,7 @@ rm -rf $PACKAGE_NAME-$OD_VERSION ls -ltr $TARGET_DIR tar_artifact=`ls $TARGET_DIR/*.tar.gz` tar_checksum_artifact=`ls $TARGET_DIR/*.tar.gz.sha512` -aws s3 cp $tar_artifact s3://$S3_BUCKET/downloads/tarball/opendistro-elasticsearch/ -aws s3 cp $tar_checksum_artifact s3://$S3_BUCKET/downloads/tarball/opendistro-elasticsearch/ -aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/downloads/*" +echo "Staging destination : s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/" +aws s3 cp $tar_artifact s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/ +aws s3 cp $tar_checksum_artifact s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/ diff --git a/kibana/linux_distributions/opendistro-kibana-build.sh b/kibana/linux_distributions/opendistro-kibana-build.sh index 72f184488..703b63871 100755 --- a/kibana/linux_distributions/opendistro-kibana-build.sh +++ b/kibana/linux_distributions/opendistro-kibana-build.sh @@ -28,26 +28,40 @@ set -e # Initialize directories REPO_ROOT=`git rev-parse --show-toplevel` ROOT=`dirname $(realpath $0)`; echo $ROOT; cd $ROOT -ES_VERSION=`$REPO_ROOT/bin/version-info --es`; echo ES_VERSION: $ES_VERSION -OD_VERSION=`$REPO_ROOT/bin/version-info --od`; echo OD_VERSION: $OD_VERSION -IS_CUT=`$REPO_ROOT/bin/version-info --is-cut`; echo IS_CUT: $IS_CUT +MANIFEST_FILE=$REPO_ROOT/release-tools/scripts/manifest.yml +ES_VERSION=`$REPO_ROOT/release-tools/scripts/version-info.sh --es`; echo ES_VERSION: $ES_VERSION +OD_VERSION=`$REPO_ROOT/release-tools/scripts/version-info.sh --od`; echo OD_VERSION: $OD_VERSION +S3_RELEASE_BASEURL=`yq eval '.urls.ODFE.releases' $MANIFEST_FILE` +S3_RELEASE_FINAL_BUILD=`yq eval '.urls.ODFE.releases_final_build' $MANIFEST_FILE | sed 's/\///g'` +S3_RELEASE_BUCKET=`echo $S3_RELEASE_BASEURL | awk -F '/' '{print $3}'` PACKAGE_TYPE=$1 -S3_BUCKET="artifacts.opendistroforelasticsearch.amazon.com" -ARTIFACTS_URL="https://d3g5vo6xdbdb9a.cloudfront.net" PACKAGE_NAME="opendistroforelasticsearch-kibana" TARGET_DIR="$ROOT/target" plugin_version=$OD_VERSION # Please DO NOT change the orders, they have dependencies -PLUGINS=`$REPO_ROOT/bin/plugins-info kibana zip --require-install-true` +PLUGINS=`$REPO_ROOT/release-tools/scripts/plugins-info.sh kibana-plugins plugin_basename` PLUGINS_ARRAY=($PLUGINS ) -CUT_VERSIONS=`$REPO_ROOT/bin/plugins-info kibana cutversion --require-install-true` -CUT_VERSIONS_ARRAY=( $CUT_VERSIONS ) +PLUGIN_PATH=`yq eval '.urls.ODFE.releases' $MANIFEST_FILE | sed "s/^.*$S3_RELEASE_BUCKET\///g"` basedir="${ROOT}/${PACKAGE_NAME}/plugins" echo $ROOT +if [ -z "$S3_RELEASE_FINAL_BUILD" ] +then + S3_RELEASE_BUILD=`aws s3api list-objects --bucket $S3_RELEASE_BUCKET --prefix "${PLUGIN_PATH}${OD_VERSION}" --query 'Contents[].[Key]' --output text | awk -F '/' '{print $3}' | uniq | tail -n 1` + echo Latest: $S3_RELEASE_BUILD +else + S3_RELEASE_BUILD=$S3_RELEASE_FINAL_BUILD + echo Final: $S3_RELEASE_BUILD +fi + +if [ -z "$PLUGINS" ]; then + echo "Provide plugin list to install (separated by space)" + exit 1 +fi + if [ -z "$PLUGINS" ]; then echo "Provide plugin list to install (separated by space)" exit 1 @@ -67,21 +81,28 @@ mkdir $PACKAGE_NAME echo "Downloading kibana oss" curl -Ls "https://artifacts.elastic.co/downloads/kibana/kibana-oss-$ES_VERSION-linux-x86_64.tar.gz" | tar --strip-components=1 -zxf - --directory $PACKAGE_NAME + # Install required plugins echo "installing open distro plugins" +rm -rf /tmp/plugins +mkdir -p /tmp/plugins for index in ${!PLUGINS_ARRAY[@]} do - if [ "$IS_CUT" = "true" ] - then - plugin_version=${CUT_VERSIONS_ARRAY[$index]} - fi - plugin_path=${PLUGINS_ARRAY[$index]} - plugin_latest=`aws s3api list-objects --bucket $S3_BUCKET --prefix "downloads/kibana-plugins/${plugin_path}-${plugin_version}" --query 'Contents[].[Key]' --output text | sort | tail -n 1` - - if [ "$plugin_path" != "none" ] + plugin_latest=`aws s3api list-objects --bucket $S3_RELEASE_BUCKET --prefix "${PLUGIN_PATH}${OD_VERSION}/$S3_RELEASE_BUILD/kibana-plugins" --query 'Contents[].[Key]' --output text | grep -v sha512 |grep ${PLUGINS_ARRAY[$index]} |grep zip |sort | tail -n 1` + if [ ! -z "$plugin_latest" ] then - echo "installing $plugin_latest" - $PACKAGE_NAME/bin/kibana-plugin --allow-root install "${ARTIFACTS_URL}/${plugin_latest}" + echo "downloading $plugin_latest" + echo `echo $plugin_latest | awk -F '/' '{print $NF}'` >> /tmp/plugins/plugins.list + plugin_path=${PLUGINS_ARRAY[$index]} + echo "plugin path: $plugin_path" + aws s3 cp "s3://${S3_RELEASE_BUCKET}/${plugin_latest}" "/tmp/plugins" --quiet; echo $? + rc_candiate=`$REPO_ROOT/release-tools/scripts/plugin_parser.sh $index release_candidate` + if $rc_candiate + then + plugin=`echo $plugin_latest | awk -F '/' '{print $NF}'` + echo "installing $plugin" + $PACKAGE_NAME/bin/kibana-plugin --allow-root install file:/tmp/plugins/$plugin + fi fi done @@ -132,8 +153,8 @@ if [ $# -eq 0 ] || [ "$PACKAGE_TYPE" = "rpm" ]; then # Upload to S3 ls -ltr $TARGET_DIR rpm_artifact=`ls $TARGET_DIR/*.rpm` - aws s3 cp $rpm_artifact s3://$S3_BUCKET/downloads/rpms/opendistroforelasticsearch-kibana/ - aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/downloads/*" + aws s3 cp $rpm_artifact s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/ +# aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/downloads/*" fi @@ -174,8 +195,7 @@ if [ $# -eq 0 ] || [ "$PACKAGE_TYPE" = "deb" ]; then # Upload to S3 ls -ltr $TARGET_DIR deb_artifact=`ls $TARGET_DIR/*.deb` - aws s3 cp $deb_artifact s3://$S3_BUCKET/downloads/debs/opendistroforelasticsearch-kibana/ - aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/downloads/*" + aws s3 cp $deb_artifact s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/ fi @@ -185,7 +205,6 @@ if [ $# -eq 0 ] || [ "$PACKAGE_TYPE" = "tar" ]; then rm -rf $TARGET_DIR/*tar* echo "generating tar" tar -czf $TARGET_DIR/$PACKAGE_NAME-$OD_VERSION.tar.gz $PACKAGE_NAME - #tar -tzvf $TARGET_DIR/$PACKAGE_NAME-$OD_VERSION.tar.gz cd $TARGET_DIR shasum -a 512 $PACKAGE_NAME-$OD_VERSION.tar.gz > $PACKAGE_NAME-$OD_VERSION.tar.gz.sha512 shasum -a 512 -c $PACKAGE_NAME-$OD_VERSION.tar.gz.sha512 @@ -197,8 +216,8 @@ if [ $# -eq 0 ] || [ "$PACKAGE_TYPE" = "tar" ]; then ls -ltr $TARGET_DIR tar_artifact=`ls $TARGET_DIR/*.tar.gz` tar_checksum_artifact=`ls $TARGET_DIR/*.tar.gz.sha512` - aws s3 cp $tar_artifact s3://$S3_BUCKET/downloads/tarball/opendistroforelasticsearch-kibana/ - aws s3 cp $tar_checksum_artifact s3://$S3_BUCKET/downloads/tarball/opendistroforelasticsearch-kibana/ - aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/downloads/*" + echo "Staging destination : s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/" + aws s3 cp $tar_artifact s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/ + aws s3 cp $tar_checksum_artifact s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/ fi diff --git a/release-tools/scripts/setup_runners_service.sh b/release-tools/scripts/setup_runners_service.sh index ede62caf3..7dd10b4b2 100755 --- a/release-tools/scripts/setup_runners_service.sh +++ b/release-tools/scripts/setup_runners_service.sh @@ -35,8 +35,9 @@ sudo apt install $SETUP_PACKAGES -y || sudo yum install $SETUP_PACKAGES -y REPO_ROOT=`git rev-parse --show-toplevel` ROOT=`dirname $(realpath $0)`; cd $ROOT -OD_VERSION=`$REPO_ROOT/release-tools/scripts/version-info.sh --od` -ES_VERSION=`$REPO_ROOT/release-tools/scripts/version-info.sh --es` +ES_VERSION=`$REPO_ROOT/release-tools/scripts/version-info.sh --es`; echo ES_VERSION: $ES_VERSION +OD_VERSION=`$REPO_ROOT/release-tools/scripts/version-info.sh --od`; echo OD_VERSION: $OD_VERSION +MANIFEST_FILE=$REPO_ROOT/release-tools/scripts/manifest.yml ES_PACKAGE_NAME="opendistroforelasticsearch-${OD_VERSION}" ES_ROOT="${ROOT}/odfe-testing/${ES_PACKAGE_NAME}" @@ -48,7 +49,10 @@ DOCKER_NAME_KIBANA="Test-Docker-Kibana-${OD_VERSION}" DOCKER_NAME_NoSec="Test-Docker-${OD_VERSION}-NoSec" DOCKER_NAME_KIBANA_NoSec="Test-Docker-Kibana-${OD_VERSION}-NoSec" -S3_BUCKET="artifacts.opendistroforelasticsearch.amazon.com" +S3_RELEASE_BASEURL=`yq eval '.urls.ODFE.releases' $MANIFEST_FILE` +S3_RELEASE_FINAL_BUILD=`yq eval '.urls.ODFE.releases_final_build' $MANIFEST_FILE | sed 's/\///g'` +S3_RELEASE_BUCKET=`echo $S3_RELEASE_BASEURL | awk -F '/' '{print $3}'` +PLUGIN_PATH=`yq eval '.urls.ODFE.releases' $MANIFEST_FILE | sed "s/^.*$S3_RELEASE_BUCKET\///g"` ##################################################################################################### @@ -68,7 +72,7 @@ sudo chmod -R 777 /dev/shm if [ "$SETUP_DISTRO" = "zip" ] then mkdir -p $ES_ROOT - aws s3 cp s3://$S3_BUCKET/downloads/tarball/opendistro-elasticsearch/$ES_PACKAGE_NAME.tar.gz . --quiet; echo $? + aws s3 cp s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/$ES_PACKAGE_NAME.tar.gz . --quiet; echo $? tar -zxf $ES_PACKAGE_NAME.tar.gz -C $ES_ROOT --strip-components 1 fi @@ -222,7 +226,7 @@ then if [ "$SETUP_DISTRO" = "zip" ] then mkdir -p $KIBANA_ROOT - aws s3 cp s3://$S3_BUCKET/downloads/tarball/$KIBANA_PACKAGE_NAME/$KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz . --quiet; echo $? + aws s3 cp s3://$S3_RELEASE_BUCKET/${PLUGIN_PATH}${OD_VERSION}/odfe/$KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz . --quiet; echo $? tar -zxf $KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz -C $KIBANA_ROOT --strip-components 1 elif [ "$SETUP_DISTRO" = "docker" ] then