Fixed reg test for cross-region S3 calls (#995)

Author: collado-mike

regtests/docker-compose.yml

@@ -66,6 +66,7 @@ services:
       AWS_CROSS_REGION_TEST_ENABLED: $AWS_CROSS_REGION_TEST_ENABLED
       AWS_CROSS_REGION_BUCKET: $AWS_CROSS_REGION_BUCKET
       AWS_ROLE_FOR_CROSS_REGION_BUCKET: $AWS_ROLE_FOR_CROSS_REGION_BUCKET
+      AWS_REGION_FOR_CROSS_REGION_TEST: $AWS_REGION_FOR_CROSS_REGION_TEST
     volumes:
       - ./output:/tmp/polaris-regtests/
       - ./credentials:/tmp/credentials/

regtests/t_spark_sql/ref/spark_sql_s3_cross_region.sh.ref

@@ -1,4 +1,3 @@
-{"defaults":{"default-base-location":"s3://sfc-role-stage-for-reg-test-do-not-modify-write-only/polaris_test/spark_sql_s3_cross_region_catalog/"},"overrides":{"prefix":"spark_sql_s3_cross_region_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit"]}
 Catalog created
 spark-sql (default)> use polaris;
 spark-sql ()> show namespaces;
@@ -7,23 +6,23 @@ spark-sql ()> create namespace db2;
 spark-sql ()> show namespaces;
 db1
 db2
-spark-sql ()>
+spark-sql ()> 
             > create namespace db1.schema1;
 spark-sql ()> show namespaces;
 db1
 db2
 spark-sql ()> show namespaces in db1;
 db1.schema1
-spark-sql ()>
+spark-sql ()> 
             > create table db1.schema1.tbl1 (col1 int);
 spark-sql ()> show tables in db1;
 spark-sql ()> use db1.schema1;
-spark-sql (db1.schema1)>
+spark-sql (db1.schema1)> 
                        > insert into tbl1 values (123), (234);
 spark-sql (db1.schema1)> select * from tbl1;
 123
 234
-spark-sql (db1.schema1)>
+spark-sql (db1.schema1)> 
                        > drop table tbl1 purge;
 spark-sql (db1.schema1)> show tables;
 spark-sql (db1.schema1)> drop namespace db1.schema1;
@@ -32,4 +31,4 @@ spark-sql (db1.schema1)> show namespaces;
 db2
 spark-sql (db1.schema1)> drop namespace db2;
 spark-sql (db1.schema1)> show namespaces;
-spark-sql (db1.schema1)>
+spark-sql (db1.schema1)> 

regtests/t_spark_sql/src/spark_sql_s3_cross_region.sh

@@ -30,7 +30,7 @@ ROLE_ARN="${AWS_ROLE_FOR_CROSS_REGION_BUCKET}"
 
 curl -i -X POST -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
   http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs \
-  -d '{"name": "spark_sql_s3_cross_region_catalog", "id": 100, "type": "INTERNAL", "readOnly": false, "properties": {"default-base-location": "s3://${BUCKET}/polaris_test/spark_sql_s3_cross_region_catalog/"}, "storageConfigInfo": {"storageType": "S3", "allowedLocations": ["s3://${BUCKET}/polaris_test/"], "roleArn": "${ROLE_ARN}"}}' > /dev/stderr
+  -d "{\"name\": \"spark_sql_s3_cross_region_catalog\", \"id\": 100, \"type\": \"INTERNAL\", \"readOnly\": false, \"properties\": {\"client.region\": \"${AWS_REGION_FOR_CROSS_REGION_TEST}\", \"default-base-location\": \"s3://${BUCKET}/polaris_test/spark_sql_s3_cross_region_catalog/\"}, \"storageConfigInfo\": {\"storageType\": \"S3\", \"allowedLocations\": [\"s3://${BUCKET}/polaris_test/\"], \"roleArn\": \"${ROLE_ARN}\", \"region\": \"${AWS_REGION_FOR_CROSS_REGION_TEST}\"}}" > /dev/stderr
 
 # Add TABLE_WRITE_DATA to the catalog's catalog_admin role since by default it can only manage access and metadata
 curl -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
@@ -42,9 +42,6 @@ curl -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: appl
   http://${POLARIS_HOST:-localhost}:8181/api/management/v1/principal-roles/service_admin/catalog-roles/spark_sql_s3_cross_region_catalog \
   -d '{"name": "catalog_admin"}' > /dev/stderr
 
-curl -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
-  "http://${POLARIS_HOST:-localhost}:8181/api/catalog/v1/config?warehouse=spark_sql_s3_cross_region_catalog"
-echo
 echo "Catalog created"
 cat << EOF | ${SPARK_HOME}/bin/spark-sql -S --conf spark.sql.catalog.polaris.token="${SPARK_BEARER_TOKEN}" --conf spark.sql.catalog.polaris.warehouse=spark_sql_s3_cross_region_catalog
 use polaris;