Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(python): add capability to read unity catalog (uc://) uris #3113

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+483 −8
Open

feat(python): add capability to read unity catalog (uc://) uris #3113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7d854ee
feat(python): add capability to read unity catalog (uc://) uris
omkar-foss Feb 11, 2025
a904e9a
test(python): add tests with mock APIs for Unity Catalog variants
omkar-foss Feb 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 112 additions & 0 deletions .github/workflows/mockoon_data_files/unitycatalog_databricks.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
{
"uuid": "0696d6d5-62b6-4a0f-8524-917b0c3848d4",
"lastMigration": 33,
"name": "Unity Catalog (Databricks) Test API",
"endpointPrefix": "",
"latency": 0,
"port": 8080,
"hostname": "",
"folders": [],
"routes": [
{
"uuid": "16d88cdc-abcf-4b0d-a4d6-18a3205a9a6c",
"type": "http",
"documentation": "Temporary Table Credentials API",
"method": "post",
"endpoint": "api/2.1/unity-catalog/temporary-table-credentials",
"responses": [
{
"uuid": "50bba5c1-1e2c-4d5b-a611-ab478320f2b0",
"body": "{\n \"aws_temp_credentials\": {\n \"access_key_id\": \"string\",\n \"secret_access_key\": \"string\",\n \"session_token\": \"string\",\n \"access_point\": \"string\"\n },\n \"azure_user_delegation_sas\": {\n \"sas_token\": \"string\"\n },\n \"r2_temp_credentials\": {\n \"access_key_id\": \"string\",\n \"secret_access_key\": \"string\",\n \"session_token\": \"string\"\n },\n \"expiration_time\": 0,\n \"url\": \"string\"\n}",
"latency": 0,
"statusCode": 200,
"label": "Default response",
"headers": [],
"bodyType": "INLINE",
"filePath": "",
"databucketID": "",
"sendFileAsBody": false,
"rules": [],
"rulesOperator": "OR",
"disableTemplating": false,
"fallbackTo404": false,
"default": true,
"crudKey": "id",
"callbacks": []
}
],
"responseMode": null,
"streamingMode": null,
"streamingInterval": 0
},
{
"uuid": "66f51ad5-2ba3-4cb0-b633-c362a9f75836",
"type": "http",
"documentation": "Get Table Details API",
"method": "get",
"endpoint": "api/2.1/unity-catalog/tables/unity.default.testtable",
"responses": [
{
"uuid": "da280b2c-07b6-45ee-8da7-92c50e4e540e",
"body": "{\n \"name\": \"string\",\n \"catalog_name\": \"string\",\n \"schema_name\": \"string\",\n \"table_type\": \"MANAGED\",\n \"data_source_format\": \"DELTA\",\n \"columns\": [\n {\n \"name\": \"string\",\n \"type_text\": \"string\",\n \"type_name\": \"BOOLEAN\",\n \"position\": 0,\n \"type_precision\": 0,\n \"type_scale\": 0,\n \"type_interval_type\": \"string\",\n \"type_json\": \"string\",\n \"comment\": \"string\",\n \"nullable\": true,\n \"partition_index\": 0,\n \"mask\": {\n \"function_name\": \"string\",\n \"using_column_names\": [\n \"string\"\n ]\n }\n }\n ],\n \"storage_location\": \"../crates/test/tests/data/delta-0.8.0-partitioned\",\n \"view_definition\": \"string\",\n \"view_dependencies\": {\n \"dependencies\": [\n {\n \"table\": {\n \"table_full_name\": \"string\"\n },\n \"function\": {\n \"function_full_name\": \"string\"\n }\n }\n ]\n },\n \"sql_path\": \"string\",\n \"owner\": \"string\",\n \"comment\": \"string\",\n \"properties\": {\n \"property1\": \"string\",\n \"property2\": \"string\"\n },\n \"storage_credential_name\": \"string\",\n \"table_constraints\": [\n {\n \"primary_key_constraint\": {\n \"name\": \"string\",\n \"child_columns\": [\n \"string\"\n ]\n },\n \"foreign_key_constraint\": {\n \"name\": \"string\",\n \"child_columns\": [\n \"string\"\n ],\n \"parent_table\": \"string\",\n \"parent_columns\": [\n \"string\"\n ]\n },\n \"named_table_constraint\": {\n \"name\": \"string\"\n }\n }\n ],\n \"row_filter\": {\n \"function_name\": \"string\",\n \"input_column_names\": [\n \"string\"\n ]\n },\n \"enable_predictive_optimization\": \"DISABLE\",\n \"metastore_id\": \"string\",\n \"full_name\": \"string\",\n \"data_access_configuration_id\": \"string\",\n \"created_at\": 0,\n \"created_by\": \"string\",\n \"updated_at\": 0,\n \"updated_by\": \"string\",\n \"deleted_at\": 0,\n \"table_id\": \"string\",\n \"delta_runtime_properties_kvpairs\": {\n \"delta_runtime_properties\": {\n \"property1\": \"string\",\n \"property2\": \"string\"\n }\n },\n \"effective_predictive_optimization_flag\": {\n \"value\": \"DISABLE\",\n \"inherited_from_type\": \"CATALOG\",\n \"inherited_from_name\": \"string\"\n },\n \"access_point\": \"string\",\n \"pipeline_id\": \"string\",\n \"browse_only\": true\n}",
"latency": 0,
"statusCode": 200,
"label": "Default response",
"headers": [],
"bodyType": "INLINE",
"filePath": "",
"databucketID": "",
"sendFileAsBody": false,
"rules": [],
"rulesOperator": "OR",
"disableTemplating": false,
"fallbackTo404": false,
"default": true,
"crudKey": "id",
"callbacks": []
}
],
"responseMode": null,
"streamingMode": null,
"streamingInterval": 0
}
],
"rootChildren": [
{
"type": "route",
"uuid": "16d88cdc-abcf-4b0d-a4d6-18a3205a9a6c"
},
{
"type": "route",
"uuid": "66f51ad5-2ba3-4cb0-b633-c362a9f75836"
}
],
"proxyMode": false,
"proxyHost": "",
"proxyRemovePrefix": false,
"tlsOptions": {
"enabled": false,
"type": "CERT",
"pfxPath": "",
"certPath": "",
"keyPath": "",
"caPath": "",
"passphrase": ""
},
"cors": true,
"headers": [],
"proxyReqHeaders": [
{
"key": "",
"value": ""
}
],
"proxyResHeaders": [
{
"key": "",
"value": ""
}
],
"data": [],
"callbacks": []
}
112 changes: 112 additions & 0 deletions .github/workflows/mockoon_data_files/unitycatalog_oss.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
{
"uuid": "0696d6d5-62b6-4a0f-8524-917b0c3848d4",
"lastMigration": 33,
"name": "Unity Catalog (OSS) Test API",
"endpointPrefix": "",
"latency": 0,
"port": 8080,
"hostname": "",
"folders": [],
"routes": [
{
"uuid": "16d88cdc-abcf-4b0d-a4d6-18a3205a9a6c",
"type": "http",
"documentation": "Temporary Table Credentials API",
"method": "post",
"endpoint": "api/2.1/unity-catalog/temporary-table-credentials",
"responses": [
{
"uuid": "50bba5c1-1e2c-4d5b-a611-ab478320f2b0",
"body": "{\n \"aws_temp_credentials\": {\n \"access_key_id\": \"string\",\n \"secret_access_key\": \"string\",\n \"session_token\": \"string\"\n },\n \"azure_user_delegation_sas\": {\n \"sas_token\": \"string\"\n },\n \"gcp_oauth_token\": {\n \"oauth_token\": \"string\"\n },\n \"expiration_time\": 0\n}",
"latency": 0,
"statusCode": 200,
"label": "Default response",
"headers": [],
"bodyType": "INLINE",
"filePath": "",
"databucketID": "",
"sendFileAsBody": false,
"rules": [],
"rulesOperator": "OR",
"disableTemplating": false,
"fallbackTo404": false,
"default": true,
"crudKey": "id",
"callbacks": []
}
],
"responseMode": null,
"streamingMode": null,
"streamingInterval": 0
},
{
"uuid": "66f51ad5-2ba3-4cb0-b633-c362a9f75836",
"type": "http",
"documentation": "Get Table Details API",
"method": "get",
"endpoint": "api/2.1/unity-catalog/tables/unity.default.testtable",
"responses": [
{
"uuid": "da280b2c-07b6-45ee-8da7-92c50e4e540e",
"body": "{\n \"name\": \"string\",\n \"catalog_name\": \"string\",\n \"schema_name\": \"string\",\n \"table_type\": \"MANAGED\",\n \"data_source_format\": \"DELTA\",\n \"columns\": [\n {\n \"name\": \"string\",\n \"type_text\": \"string\",\n \"type_json\": \"string\",\n \"type_name\": \"BOOLEAN\",\n \"type_precision\": 0,\n \"type_scale\": 0,\n \"type_interval_type\": \"string\",\n \"position\": 0,\n \"comment\": \"string\",\n \"nullable\": true,\n \"partition_index\": 0\n }\n ],\n \"storage_location\": \"string\",\n \"comment\": \"string\",\n \"properties\": {\n \"additionalProp1\": \"string\",\n \"additionalProp2\": \"string\",\n \"additionalProp3\": \"string\"\n },\n \"owner\": \"string\",\n \"created_at\": 0,\n \"created_by\": \"string\",\n \"updated_at\": 0,\n \"updated_by\": \"string\",\n \"table_id\": \"string\"\n}",
"latency": 0,
"statusCode": 200,
"label": "Default response",
"headers": [],
"bodyType": "INLINE",
"filePath": "",
"databucketID": "",
"sendFileAsBody": false,
"rules": [],
"rulesOperator": "OR",
"disableTemplating": false,
"fallbackTo404": false,
"default": true,
"crudKey": "id",
"callbacks": []
}
],
"responseMode": null,
"streamingMode": null,
"streamingInterval": 0
}
],
"rootChildren": [
{
"type": "route",
"uuid": "16d88cdc-abcf-4b0d-a4d6-18a3205a9a6c"
},
{
"type": "route",
"uuid": "66f51ad5-2ba3-4cb0-b633-c362a9f75836"
}
],
"proxyMode": false,
"proxyHost": "",
"proxyRemovePrefix": false,
"tlsOptions": {
"enabled": false,
"type": "CERT",
"pfxPath": "",
"certPath": "",
"keyPath": "",
"caPath": "",
"passphrase": ""
},
"cors": true,
"headers": [],
"proxyReqHeaders": [
{
"key": "",
"value": ""
}
],
"proxyResHeaders": [
{
"key": "",
"value": ""
}
],
"data": [],
"callbacks": []
}
60 changes: 59 additions & 1 deletion .github/workflows/python_build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ jobs:
uv pip uninstall pandas
uv run --no-sync pytest -m "not pandas and not integration and not benchmark"
uv pip install pandas

test-lakefs:
name: Python Build (Python 3.10 LakeFS Integration tests)
runs-on: ubuntu-latest
Expand All @@ -110,6 +110,64 @@ jobs:
- name: Run tests
run: uv run --no-sync pytest -m '(lakefs and integration)' --doctest-modules

test-unitycatalog-databricks:
name: Python Build (Python 3.10 Unity Catalog Integration tests)
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C debuginfo=1"
CARGO_INCREMENTAL: 0

steps:
- uses: actions/checkout@v3

- name: Setup Environment
uses: ./.github/actions/setup-env

- name: Run Mockoon CLI
uses: mockoon/cli-action@v2
with:
version: "latest"
data-file: "./mockoon_data_files/unitycatalog_databricks.json"
port: 8080

- name: Build and install deltalake
run: make develop

- name: Download Data Acceptance Tests (DAT) files
run: make setup-dat

- name: Run tests
run: uv run --no-sync pytest -m '(unitycatalog_databricks and integration)' --doctest-modules

test-unitycatalog-oss:
name: Python Build (Python 3.10 Unity Catalog Integration tests)
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C debuginfo=1"
CARGO_INCREMENTAL: 0

steps:
- uses: actions/checkout@v3

- name: Setup Environment
uses: ./.github/actions/setup-env

- name: Run Mockoon CLI
uses: mockoon/cli-action@v2
with:
version: "latest"
data-file: "./mockoon_data_files/unitycatalog_oss.json"
port: 8080

- name: Build and install deltalake
run: make develop

- name: Download Data Acceptance Tests (DAT) files
run: make setup-dat

- name: Run tests
run: uv run --no-sync pytest -m '(unitycatalog_oss and integration)' --doctest-modules

test-pyspark:
name: PySpark Integration Tests
runs-on: ubuntu-latest
Expand Down
Loading
Loading