Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

New Unity Catalog resource connector #8248

Merged
merged 2 commits into from
Jun 21, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/merge-v5.yml
Original file line number Diff line number Diff line change
@@ -95,7 +95,7 @@ jobs:
uses: docker/build-push-action@v5
with:
push: true
tags: odpi/egeria-platform:${{ env.VERSION }}, odpi/egeria-platform:latest, quay.io/odpi/egeria-platform:${{ env.VERSION }}, quay.io/odpi/egeria-platform:latest
tags: odpi/egeria-platform:${{ env.VERSION }}, odpi/egeria-platform:latest, quay.io/odpi/egeria:${{ env.VERSION }}, quay.io/odpi/egeria:latest
context: ./open-metadata-distribution/omag-server-platform/build/unpacked/egeria-platform-${{ env.VERSION }}-distribution.tar.gz
platforms: linux/amd64,linux/arm64
- name: Build and push(egeria-platform) to quay.io and docker.io (no tag latest)
2 changes: 1 addition & 1 deletion CoreContentPackGUIDMap.json

Large diffs are not rendered by default.

31 changes: 16 additions & 15 deletions MAINTAINERS.md
Original file line number Diff line number Diff line change
@@ -12,21 +12,22 @@ new pull requests into main.
The current Egeria maintainers are:


| Name | Email | GitHub id
| --------------------- | ------------------------------ | --------------
| Mandy Chessell | mandy.e.chessell@gmail.com | mandy-chessell
| Nigel Jones | jonesn@uk.ibm.com | planetf1
| David Radley | david_radley@uk.ibm.com | davidradl
| Maryna Strelchuk | maryna.strelchuk@gmail.com | mstrelchuk
| Bogdan Sava | bogdan.sava@gmail.com | bogdan-sava
| Chris Grote | chris@thegrotes.net | cmgrote
| Raluca Popa | raluca.popa2@ing.com | popa-raluca
| Chris Replogle | Chris.Replogle@sas.com | CDARip2U
| Ljupcho Palashevski | ljupcho.palashevski@ing.com | lpalashevski
| Alexandra Bucur | alexandra-nona.bucur@ing.com | alexandra-bucur
| Cezar Sirbu | sirbunicolaecezar@gmail.com | sarbull
| Liviu Constantin Popa | liviu-constantin.popa@ing.com | lcpopa
| Cristian-Mihai Iliescu | cristianmihaiiliescu@gmail.com | MihaiIliescu
| Name | Email | GitHub id
|------------------------|--------------------------------| --------------
| Mandy Chessell | mandy.e.chessell@gmail.com | mandy-chessell
| Dan Wolfson | dan.wolfson@pdr-associates.com | dwolfson
| Nigel Jones | jonesn@uk.ibm.com | planetf1
| David Radley | david_radley@uk.ibm.com | davidradl
| Maryna Strelchuk | maryna.strelchuk@gmail.com | mstrelchuk
| Bogdan Sava | bogdan.sava@gmail.com | bogdan-sava
| Chris Grote | chris@thegrotes.net | cmgrote
| Raluca Popa | raluca.popa2@ing.com | popa-raluca
| Chris Replogle | Chris.Replogle@sas.com | CDARip2U
| Ljupcho Palashevski | ljupcho.palashevski@ing.com | lpalashevski
| Alexandra Bucur | alexandra-nona.bucur@ing.com | alexandra-bucur
| Cezar Sirbu | sirbunicolaecezar@gmail.com | sarbull
| Liviu Constantin Popa | liviu-constantin.popa@ing.com | lcpopa
| Cristian-Mihai Iliescu | cristianmihaiiliescu@gmail.com | MihaiIliescu

To understand how to become an Egeria maintainer
see the [Egeria Operations Guide](https://egeria-project.org/guides/project-operations/).
Original file line number Diff line number Diff line change
@@ -33,6 +33,14 @@ public enum PostgresPlaceholderProperty
"string",
"myDatabase"),

/**
* The name of the database being catalogued.
*/
DATABASE_DESCRIPTION ("databaseDescription",
"The description of the database being catalogued.",
"string",
null),

/**
* The userId to store in the userId attribute of the connection.
*/
@@ -52,13 +60,23 @@ public enum PostgresPlaceholderProperty
/**
* The name of the server being catalogued.
*/
SERVER_NAME ("serverName", "The name of the database server being catalogued.", "string", "myServer"),
SERVER_NAME ("serverName", "The name of the server being catalogued.", "string", "myServer"),

/**
* The description of the server being catalogued.
*/
SERVER_DESCRIPTION ("serverDescription", "The description of the server being catalogued.", "string", null),

/**
* The name of the schema being catalogued.
*/
SCHEMA_NAME ("schemaName", "The name of the database schema being catalogued.", "string", "MyServer.schema"),

/**
* The description of the schema being catalogued.
*/
SCHEMA_DESCRIPTION ("schemaDescription", "The description of the database schema being catalogued.", "string", null),

/**
* The name of the database table being catalogued.
*/
@@ -158,6 +176,7 @@ public static List<PlaceholderPropertyType> getPostgresServerPlaceholderProperty
placeholderPropertyTypes.add(PostgresPlaceholderProperty.HOST_IDENTIFIER.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.PORT_NUMBER.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SERVER_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SERVER_DESCRIPTION.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_USER_ID.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_PASSWORD.getPlaceholderType());

@@ -178,6 +197,7 @@ public static List<PlaceholderPropertyType> getPostgresDatabasePlaceholderProper
placeholderPropertyTypes.add(PostgresPlaceholderProperty.PORT_NUMBER.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SERVER_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_DESCRIPTION.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_USER_ID.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_PASSWORD.getPlaceholderType());

@@ -199,6 +219,7 @@ public static List<PlaceholderPropertyType> getPostgresSchemaPlaceholderProperty
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SERVER_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SCHEMA_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SCHEMA_DESCRIPTION.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_USER_ID.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_PASSWORD.getPlaceholderType());

Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@

@ucURL=http://localhost:8080

###
# @name list catalogs
GET {{ucURL}}/api/2.1/unity-catalog/catalogs?max_results=1

###
# @name create catalog
POST {{ucURL}}/api/2.1/unity-catalog/catalogs
Content-Type: application/json

{
"name": "test3",
"comment": "Test catalog3",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name get catalog
GET {{ucURL}}/api/2.1/unity-catalog/catalogs/test3
Content-Type: application/json


###
# @name patch catalog
PATCH {{ucURL}}/api/2.1/unity-catalog/catalogs/test4
Content-Type: application/json

{
"comment" : "This is a comment",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name rename catalog
PATCH {{ucURL}}/api/2.1/unity-catalog/catalogs/test3_temp
Content-Type: application/json

{
"new_name" : "test4",
"comment" : "This is a new comment",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name delete catalog
DELETE {{ucURL}}/api/2.1/unity-catalog/catalogs/test3
Content-Type: application/json

Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@

@ucURL=http://localhost:8080

###
# @name list functions
GET {{ucURL}}/api/2.1/unity-catalog/functions?catalog_name=test&schema_name=test4schema1&max_results=10

###
# @name create function
POST {{ucURL}}/api/2.1/unity-catalog/functions
Content-Type: application/json

{
"function_info" :
{
"name": "createWisdom",
"catalog_name" : "test",
"schema_name" : "test4schema1",
"comment": "Test catalog4 function 1",
"parameter_style" : "S",
"routine_body" : "EXTERNAL",
"external_language" : "Java",
"security_type" : "DEFINER"
}
}

###
# @name get function
GET {{ucURL}}/api/2.1/unity-catalog/functions/test.test4schema1.createWisdom
Content-Type: application/json


###
# @name delete function
DELETE {{ucURL}}/api/2.1/unity-catalog/functions/test.test4schema1.createWisdom
Content-Type: application/json

Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@

@ucURL=http://localhost:8080

###
# @name list schemas
GET {{ucURL}}/api/2.1/unity-catalog/schemas?catalog_name=unity&max_results=10

###
# @name create schema
POST {{ucURL}}/api/2.1/unity-catalog/schemas
Content-Type: application/json

{
"name": "test4schema1",
"catalog_name" : "test",
"comment": "Test catalog4 schema 1",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name get schema
GET {{ucURL}}/api/2.1/unity-catalog/schemas/test.test4schema1
Content-Type: application/json


###
# @name patch schema
PATCH {{ucURL}}/api/2.1/unity-catalog/schemas/test.test4schema2
Content-Type: application/json

{
"new_name" : "test4schema1",
"comment" : "This is a comment",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name delete schema
DELETE {{ucURL}}/api/2.1/unity-catalog/schemas/test.test4schema1
Content-Type: application/json

Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@

@ucURL=http://localhost:8080

###
# @name list tables
GET {{ucURL}}/api/2.1/unity-catalog/tables?catalog_name=test&schema_name=test4schema1&max_results=10

###
# @name create table
POST {{ucURL}}/api/2.1/unity-catalog/tables
Content-Type: application/json

{
"table_info" :
{
"name": "bookOfKnowledge",
"catalog_name" : "test",
"schema_name" : "test4schema1",
"comment": "Useful data",
"parameter_style" : "S",
"routine_body" : "EXTERNAL",
"external_language" : "Java",
"security_type" : "DEFINER"
}
}

###
# @name get table
GET {{ucURL}}/api/2.1/unity-catalog/tables/test.test4schema1.createWisdom
Content-Type: application/json


###
# @name delete table
DELETE {{ucURL}}/api/2.1/unity-catalog/tables/test.test4schema1.createWisdom
Content-Type: application/json

Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@

@ucURL=http://localhost:8080

###
# @name list volumes
GET {{ucURL}}/api/2.1/unity-catalog/volumes?catalog_name=unity&schema_name=default&max_results=10

###
# @name create volume
POST {{ucURL}}/api/2.1/unity-catalog/volumes
Content-Type: application/json

{
"name": "test4volume1",
"catalog_name" : "test",
"schema_name": "test4schema1",
"comment": "Test catalog4 volume 1",
"volume_type": "EXTERNAL",
"storage_location": "file:///Users/me/Code/uc/unitycatalog/etc/data/external/unity/default/volumes/json_files/"
}

###
# @name get volume
GET {{ucURL}}/api/2.1/unity-catalog/volumes/test.test4schema1.volume1
Content-Type: application/json


###
# @name patch volume
PATCH {{ucURL}}/api/2.1/unity-catalog/volumes/test.test4schema1.test4volume1
Content-Type: application/json

{
"new_name" : "volume1",
"comment" : "This is an updated comment"
}

###
# @name delete volume
DELETE {{ucURL}}/api/2.1/unity-catalog/volumes/test.test4schema1.test4volume1
Content-Type: application/json

Original file line number Diff line number Diff line change
@@ -25,10 +25,16 @@ dependencies {
compileOnly project(':open-metadata-implementation:access-services:asset-manager:asset-manager-api')
compileOnly project(':open-metadata-implementation:integration-services:catalog-integrator:catalog-integrator-api')
compileOnly project(':open-metadata-implementation:repository-services:repository-services-apis')
compileOnly project(':open-metadata-implementation:adapters:open-connectors:rest-client-connectors:rest-client-connectors-api')
compileOnly project(':open-metadata-implementation:adapters:open-connectors:rest-client-connectors:rest-client-factory')
compileOnly project(':open-metadata-implementation:adapters:open-connectors:rest-client-connectors:spring-rest-client-connector')
testCompileOnly project(':open-metadata-implementation:frameworks:audit-log-framework')
testCompileOnly project(':open-metadata-implementation:repository-services:repository-services-apis')
testImplementation project(':open-metadata-test:open-metadata-ut')
testImplementation 'org.testng:testng'
compileOnly 'org.springframework:spring-core'
compileOnly 'com.fasterxml.jackson.core:jackson-databind'
compileOnly 'com.fasterxml.jackson.core:jackson-annotations'
}

shadowJar {
Original file line number Diff line number Diff line change
@@ -38,7 +38,13 @@ public enum UnityCatalogPlaceholderProperty
/**
* The name of the server being catalogued.
*/
SERVER_NAME ("serverName", "The name of the server being catalogued.", "string", "myOMAGServer"),
SERVER_NAME ("serverName", "The name of the server being catalogued.", "string", "myServer"),

/**
* The description of the server being catalogued.
*/
SERVER_DESCRIPTION ("serverDescription", "The description of the server being catalogued.", "string", null),

;

public final String name;
Original file line number Diff line number Diff line change
@@ -58,6 +58,17 @@ public enum UCAuditCode implements AuditLogMessageSet
"The integration connector continues, looking for another database.",
"This is an information message showing that the integration connector is working, but does not need to do any processing on this database."),



/**
* UNITY-CATALOG-CONNECTOR-0005 - A client-side exception was received from API call {0} to server {1} at {2}. The error message was {3}
*/
CLIENT_SIDE_REST_API_ERROR( "UNITY-CATALOG-CONNECTOR-0005",
AuditLogRecordSeverityLevel.EXCEPTION,
"A client-side exception was received from API call {0} to server {1} at {2}. The error message was {3}",
"The server has issued a call to the open metadata access service REST API in a remote server and has received an exception from the local client libraries.",
"Look for errors in the local server's console to understand and correct the source of the error."),

;

private final String logMessageId;
Original file line number Diff line number Diff line change
@@ -41,6 +41,15 @@ public enum UCErrorCode implements ExceptionMessageSet
"The connector is unable to process the current request.",
"Use the details from the error message to determine the cause of the error and retry the request once it is resolved."),


/**
* UNITY-CATALOG-CONNECTOR-503-001 - A client-side exception {0} was received from API call {1} to URL {2}. The error message was {3}
*/
CLIENT_SIDE_REST_API_ERROR(503, "UNITY-CATALOG-CONNECTOR-503-001",
"A client-side exception of {0} was received from API call {1} to URL {2}. The error message was {3}",
"The connector has issued a call to the open metadata access service REST API in a remote server and has received an exception from the local client libraries.",
"Look for errors in the local server's console to understand and correct the source of the error."),

;

private final int httpErrorCode;
Loading