Skip to content
This repository has been archived by the owner on Jan 9, 2025. It is now read-only.

Commit

Permalink
feat(airbyte): remove local connector and refine definition (#85)
Browse files Browse the repository at this point in the history 
Because

- the Airbyte connector has some schema that are not supported in
Console auto-form.

This commit

- refines definition.
- removes local connectors for security purpose.
  • Loading branch information
@donch1989
donch1989 authored Apr 17, 2024
1 parent 9c0a57d commit 8203316
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 194 deletions.
205 changes: 12 additions & 193 deletions pkg/connector/airbyte/v0/config/definition.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -678,10 +678,9 @@
"type": "string"
},
"region": {
"default": "",
"default": "af-south-1",
"description": "The region of the S3 bucket. See <a href=\"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions\">here</a> for all region codes.",
"enum": [
"",
"af-south-1",
"ap-east-1",
"ap-northeast-1",
Expand Down Expand Up @@ -1896,96 +1895,6 @@
"title": "Convex",
"type": "object"
},
{
"$schema": "http://json-schema.org/draft-07/schema#",
"additionalProperties": true,
"properties": {
"delimiter_type": {
"description": "The character delimiting individual cells in the CSV data.",
"oneOf": [
{
"properties": {
"delimiter": {
"const": "\\u002c",
"type": "string"
}
},
"required": [
"delimiter"
],
"title": "Comma"
},
{
"properties": {
"delimiter": {
"const": "\\u003b",
"type": "string"
}
},
"required": [
"delimiter"
],
"title": "Semicolon"
},
{
"properties": {
"delimiter": {
"const": "\\u007c",
"type": "string"
}
},
"required": [
"delimiter"
],
"title": "Pipe"
},
{
"properties": {
"delimiter": {
"const": "\\u0009",
"type": "string"
}
},
"required": [
"delimiter"
],
"title": "Tab"
},
{
"properties": {
"delimiter": {
"const": "\\u0020",
"type": "string"
}
},
"required": [
"delimiter"
],
"title": "Space"
}
],
"title": "Delimiter",
"type": "object"
},
"destination": {
"const": "airbyte-destination-csv",
"type": "string"
},
"destination_path": {
"description": "Path to the directory where csv files will be written. The destination uses the local mount \"/local\" and any data files will be placed inside that local mount. For more information check out our <a href=\"https://docs.airbyte.com/integrations/destinations/local-csv\">docs</a>",
"examples": [
"/local"
],
"type": "string"
}
},
"required": [
"destination_path",
"destination"
],
"title": "Csv",
"type": "object"
},
{
"$schema": "http://json-schema.org/draft-07/schema#",
"properties": {
Expand Down Expand Up @@ -2260,44 +2169,6 @@
"title": "Databricks",
"type": "object"
},
{
"$schema": "http://json-schema.org/draft-07/schema#",
"additionalProperties": true,
"properties": {
"destination": {
"const": "airbyte-destination-duckdb",
"type": "string"
},
"destination_path": {
"description": "Path to the .duckdb file, or the text 'md:' to connect to MotherDuck. The file will be placed inside that local mount. For more information check out our <a href=\"https://docs.airbyte.io/integrations/destinations/duckdb\">docs</a>",
"examples": [
"/local/destination.duckdb",
"md:",
"motherduck:"
],
"title": "Destination DB",
"type": "string"
},
"motherduck_api_key": {
"instillCredentialField": true,
"description": "API key to use for authentication to a MotherDuck database.",
"title": "MotherDuck API Key",
"type": "string"
},
"schema": {
"description": "Database schema name, default for duckdb is 'main'.",
"example": "main",
"title": "Destination Schema",
"type": "string"
}
},
"required": [
"destination_path",
"destination"
],
"title": "Duckdb",
"type": "object"
},
{
"$schema": "http://json-schema.org/draft-07/schema#",
"additionalProperties": false,
Expand Down Expand Up @@ -2325,10 +2196,9 @@
"type": "string"
},
"dynamodb_region": {
"default": "",
"default": "af-south-1",
"description": "The region of the DynamoDB.",
"enum": [
"",
"af-south-1",
"ap-east-1",
"ap-northeast-1",
Expand Down Expand Up @@ -4296,30 +4166,6 @@
"title": "Langchain",
"type": "object"
},
{
"$schema": "http://json-schema.org/draft-07/schema#",
"additionalProperties": false,
"properties": {
"destination": {
"const": "airbyte-destination-local-json",
"type": "string"
},
"destination_path": {
"description": "Path to the directory where json files will be written. The files will be placed inside that local mount. For more information check out our <a href=\"https://docs.airbyte.io/integrations/destinations/local-json\">docs</a>",
"examples": [
"/json_data"
],
"title": "Destination Path",
"type": "string"
}
},
"required": [
"destination_path",
"destination"
],
"title": "Localjson",
"type": "object"
},
{
"description": "The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration,\nas well as to provide type safety for the configuration passed to the destination.\n\nThe configuration model is composed of four parts:\n* Processing configuration\n* Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\nProcessing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class.",
"groups": [
Expand Down Expand Up @@ -8172,10 +8018,9 @@
"type": "string"
},
"s3_bucket_region": {
"default": "",
"default": "af-south-1",
"description": "The region of the S3 bucket. See <a href=\"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions\">here</a> for all region codes.",
"enum": [
"",
"af-south-1",
"ap-east-1",
"ap-northeast-1",
Expand Down Expand Up @@ -8659,10 +8504,9 @@
"type": "string"
},
"s3_bucket_region": {
"default": "",
"default": "af-south-1",
"description": "The region of the S3 bucket. See <a href=\"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions\">here</a> for all region codes.",
"enum": [
"",
"af-south-1",
"ap-east-1",
"ap-northeast-1",
Expand Down Expand Up @@ -9023,27 +8867,6 @@
"title": "Snowflake",
"type": "object"
},
{
"$schema": "http://json-schema.org/draft-07/schema#",
"additionalProperties": false,
"properties": {
"destination": {
"const": "airbyte-destination-sqlite",
"type": "string"
},
"destination_path": {
"description": "Path to the sqlite.db file. The file will be placed inside that local mount. For more information check out our <a href=\"https://docs.airbyte.io/integrations/destinations/sqlite\">docs</a>",
"example": "/local/sqlite.db",
"type": "string"
}
},
"required": [
"destination_path",
"destination"
],
"title": "Sqlite",
"type": "object"
},
{
"$schema": "http://json-schema.org/draft-07/schema#",
"properties": {
Expand Down Expand Up @@ -10271,48 +10094,44 @@
"uid": "975678a2-5117-48a4-a135-019619dee18e",
"vendor": "Airbyte",
"vendor_attributes": {
"airbyte-destination-astra": "airbyte/destination-astra:0.1.1",
"airbyte-destination-astra": "airbyte/destination-astra:0.1.2",
"airbyte-destination-aws-datalake": "airbyte/destination-aws-datalake:0.1.6",
"airbyte-destination-azure-blob-storage": "airbyte/destination-azure-blob-storage:0.2.1",
"airbyte-destination-bigquery": "airbyte/destination-bigquery:2.4.12",
"airbyte-destination-chroma": "airbyte/destination-chroma:0.0.9",
"airbyte-destination-chroma": "airbyte/destination-chroma:0.0.10",
"airbyte-destination-clickhouse": "airbyte/destination-clickhouse:1.0.0",
"airbyte-destination-convex": "airbyte/destination-convex:0.2.0",
"airbyte-destination-csv": "airbyte/destination-csv:1.0.0",
"airbyte-destination-databricks": "airbyte/destination-databricks:1.1.0",
"airbyte-destination-duckdb": "airbyte/destination-duckdb:0.3.3",
"airbyte-destination-dynamodb": "airbyte/destination-dynamodb:0.1.8",
"airbyte-destination-e2e-test": "airbyte/destination-e2e-test:0.3.2",
"airbyte-destination-e2e-test": "airbyte/destination-e2e-test:0.3.3",
"airbyte-destination-elasticsearch": "airbyte/destination-elasticsearch:0.1.6",
"airbyte-destination-firestore": "airbyte/destination-firestore:0.1.1",
"airbyte-destination-gcs": "airbyte/destination-gcs:0.4.6",
"airbyte-destination-google-sheets": "airbyte/destination-google-sheets:0.2.3",
"airbyte-destination-iceberg": "airbyte/destination-iceberg:0.1.6",
"airbyte-destination-kafka": "airbyte/destination-kafka:0.1.10",
"airbyte-destination-langchain": "airbyte/destination-langchain:0.1.2",
"airbyte-destination-local-json": "airbyte/destination-local-json:0.2.11",
"airbyte-destination-milvus": "airbyte/destination-milvus:0.0.13",
"airbyte-destination-milvus": "airbyte/destination-milvus:0.0.14",
"airbyte-destination-mongodb": "airbyte/destination-mongodb:0.2.0",
"airbyte-destination-mssql": "airbyte/destination-mssql:0.2.0",
"airbyte-destination-mysql": "airbyte/destination-mysql:0.2.0",
"airbyte-destination-oracle": "airbyte/destination-oracle:1.0.0",
"airbyte-destination-pinecone": "airbyte/destination-pinecone:0.0.23",
"airbyte-destination-pinecone": "airbyte/destination-pinecone:0.0.24",
"airbyte-destination-postgres": "airbyte/destination-postgres:2.0.9",
"airbyte-destination-pubsub": "airbyte/destination-pubsub:0.2.0",
"airbyte-destination-qdrant": "airbyte/destination-qdrant:0.0.10",
"airbyte-destination-qdrant": "airbyte/destination-qdrant:0.0.11",
"airbyte-destination-rabbitmq": "airbyte/destination-rabbitmq:0.1.3",
"airbyte-destination-redis": "airbyte/destination-redis:0.1.4",
"airbyte-destination-redshift": "airbyte/destination-redshift:2.4.3",
"airbyte-destination-s3": "airbyte/destination-s3:0.6.0",
"airbyte-destination-s3-glue": "airbyte/destination-s3-glue:0.1.8",
"airbyte-destination-sftp-json": "airbyte/destination-sftp-json:0.1.0",
"airbyte-destination-snowflake": "airbyte/destination-snowflake:3.7.0",
"airbyte-destination-sqlite": "airbyte/destination-sqlite:0.1.0",
"airbyte-destination-starburst-galaxy": "airbyte/destination-starburst-galaxy:0.0.1",
"airbyte-destination-teradata": "airbyte/destination-teradata:0.1.5",
"airbyte-destination-typesense": "airbyte/destination-typesense:0.1.4",
"airbyte-destination-vectara": "airbyte/destination-vectara:0.2.2",
"airbyte-destination-weaviate": "airbyte/destination-weaviate:0.2.16",
"airbyte-destination-vectara": "airbyte/destination-vectara:0.2.3",
"airbyte-destination-weaviate": "airbyte/destination-weaviate:0.2.17",
"airbyte-devmate-cloud": "ghcr.io/devmate-cloud/streamr-airbyte-connectors:0.0.1"
}
}
18 changes: 17 additions & 1 deletion pkg/connector/airbyte/v0/config/seed/generate_definitions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,29 @@
from os.path import dirname


def refine(definition):
newDef = {}
for k, v in definition.items():
if k == "enum" and "" in v:
v = v[1:]
definition["default"] = v[0]
definition[k] = v
if isinstance(v, dict):
newDef[k] = refine(v)
else:
newDef[k] = v
return newDef


url = 'https://connectors.airbyte.com/files/registries/v0/oss_registry.json'
response = urlopen(url)
data_json = json.loads(response.read())
definitions = data_json['destinations']
oneOfs = []
vendor_attribute = {}
for idx, _ in enumerate(definitions):
if (f"{definitions[idx]['dockerRepository'].split('/')[1]}") in ["destination-local-json", "destination-csv", "destination-sqlite", "destination-duckdb"]:
continue
definitions[idx]['uid'] = definitions[idx]['destinationDefinitionId']
definitions[idx][
'id'] = f"airbyte-{definitions[idx]['dockerRepository'].split('/')[1]}"
Expand All @@ -34,7 +50,7 @@
definitions[idx]['spec']['connectionSpecification']

)
definitions[idx]['spec']['resource_specification'] = definitions[idx]['spec']['connectionSpecification']
definitions[idx]['spec']['resource_specification'] = refine(definitions[idx]['spec']['connectionSpecification'])

new_def = {
"available_tasks": [
Expand Down

0 comments on commit 8203316

Please sign in to comment.