Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed reg test for cross-region S3 calls #995

Merged
merged 1 commit into from
Feb 14, 2025
Merged

Fixed reg test for cross-region S3 calls #995

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions regtests/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ services:
AWS_CROSS_REGION_TEST_ENABLED: $AWS_CROSS_REGION_TEST_ENABLED
AWS_CROSS_REGION_BUCKET: $AWS_CROSS_REGION_BUCKET
AWS_ROLE_FOR_CROSS_REGION_BUCKET: $AWS_ROLE_FOR_CROSS_REGION_BUCKET
AWS_REGION_FOR_CROSS_REGION_TEST: $AWS_REGION_FOR_CROSS_REGION_TEST
volumes:
- ./output:/tmp/polaris-regtests/
- ./credentials:/tmp/credentials/
11 changes: 5 additions & 6 deletions regtests/t_spark_sql/ref/spark_sql_s3_cross_region.sh.ref
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
{"defaults":{"default-base-location":"s3://sfc-role-stage-for-reg-test-do-not-modify-write-only/polaris_test/spark_sql_s3_cross_region_catalog/"},"overrides":{"prefix":"spark_sql_s3_cross_region_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit"]}
Catalog created
spark-sql (default)> use polaris;
spark-sql ()> show namespaces;
Expand All @@ -7,23 +6,23 @@ spark-sql ()> create namespace db2;
spark-sql ()> show namespaces;
db1
db2
spark-sql ()>
spark-sql ()>
> create namespace db1.schema1;
spark-sql ()> show namespaces;
db1
db2
spark-sql ()> show namespaces in db1;
db1.schema1
spark-sql ()>
spark-sql ()>
> create table db1.schema1.tbl1 (col1 int);
spark-sql ()> show tables in db1;
spark-sql ()> use db1.schema1;
spark-sql (db1.schema1)>
spark-sql (db1.schema1)>
> insert into tbl1 values (123), (234);
spark-sql (db1.schema1)> select * from tbl1;
123
234
spark-sql (db1.schema1)>
spark-sql (db1.schema1)>
> drop table tbl1 purge;
spark-sql (db1.schema1)> show tables;
spark-sql (db1.schema1)> drop namespace db1.schema1;
Expand All @@ -32,4 +31,4 @@ spark-sql (db1.schema1)> show namespaces;
db2
spark-sql (db1.schema1)> drop namespace db2;
spark-sql (db1.schema1)> show namespaces;
spark-sql (db1.schema1)>
spark-sql (db1.schema1)>
5 changes: 1 addition & 4 deletions regtests/t_spark_sql/src/spark_sql_s3_cross_region.sh
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ ROLE_ARN="${AWS_ROLE_FOR_CROSS_REGION_BUCKET}"

curl -i -X POST -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs \
-d '{"name": "spark_sql_s3_cross_region_catalog", "id": 100, "type": "INTERNAL", "readOnly": false, "properties": {"default-base-location": "s3://${BUCKET}/polaris_test/spark_sql_s3_cross_region_catalog/"}, "storageConfigInfo": {"storageType": "S3", "allowedLocations": ["s3://${BUCKET}/polaris_test/"], "roleArn": "${ROLE_ARN}"}}' > /dev/stderr
-d "{\"name\": \"spark_sql_s3_cross_region_catalog\", \"id\": 100, \"type\": \"INTERNAL\", \"readOnly\": false, \"properties\": {\"client.region\": \"${AWS_REGION_FOR_CROSS_REGION_TEST}\", \"default-base-location\": \"s3://${BUCKET}/polaris_test/spark_sql_s3_cross_region_catalog/\"}, \"storageConfigInfo\": {\"storageType\": \"S3\", \"allowedLocations\": [\"s3://${BUCKET}/polaris_test/\"], \"roleArn\": \"${ROLE_ARN}\", \"region\": \"${AWS_REGION_FOR_CROSS_REGION_TEST}\"}}" > /dev/stderr

# Add TABLE_WRITE_DATA to the catalog's catalog_admin role since by default it can only manage access and metadata
curl -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
Expand All @@ -42,9 +42,6 @@ curl -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: appl
http://${POLARIS_HOST:-localhost}:8181/api/management/v1/principal-roles/service_admin/catalog-roles/spark_sql_s3_cross_region_catalog \
-d '{"name": "catalog_admin"}' > /dev/stderr

curl -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
"http://${POLARIS_HOST:-localhost}:8181/api/catalog/v1/config?warehouse=spark_sql_s3_cross_region_catalog"
echo
Comment on lines -45 to -47
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like this is being removed because the output will change depending on AWS_REGION_FOR_CROSS_REGION_TEST, is that right?

Rather than removing it, I wonder if we could do some jq magic to make it env-variable-agnostic.

Anyway, it's not an issue to just remove it imo.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, it would be a useful feature to add, but for right now, i just want to make the tests pass :)

echo "Catalog created"
cat << EOF | ${SPARK_HOME}/bin/spark-sql -S --conf spark.sql.catalog.polaris.token="${SPARK_BEARER_TOKEN}" --conf spark.sql.catalog.polaris.warehouse=spark_sql_s3_cross_region_catalog
use polaris;
Expand Down