Skip to content

Commit

Permalink
feat: publish data to hubspot via target-hubspot (#665)
Browse files Browse the repository at this point in the history
  • Loading branch information
pnadolny13 authored Jul 19, 2023
1 parent f48acaa commit 514aa47
Show file tree
Hide file tree
Showing 8 changed files with 214 additions and 16 deletions.
20 changes: 16 additions & 4 deletions data/environments/cicd.meltano.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,19 +46,31 @@ environments:
- name: tap-snowflake-metrics
config:
tables:
- ${CI_BRANCH}_MELTANO_HUB.FACT_VARIANT_HUB_METRICS
- ${CI_BRANCH}_MELTANO_HUB.FACT_VARIANT_HUB_METRICS
- name: tap-snowflake-audit
config:
tables:
- ${CI_BRANCH}_MELTANO_HUB.HUB_METRICS_AUDIT
- ${CI_BRANCH}_MELTANO_HUB.HUB_METRICS_AUDIT
- name: tap-snowflake-singer-activity
config:
tables:
- ${CI_BRANCH}_SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
- ${CI_BRANCH}_SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
- name: tap-snowflake-meltano-activity
config:
tables:
- ${CI_BRANCH}_SLACK_NOTIFICATIONS.MELTANO_ACTIVITY_NOTIFICATIONS
- ${CI_BRANCH}_SLACK_NOTIFICATIONS.MELTANO_ACTIVITY_NOTIFICATIONS
- name: tap-snowflake-hubspot-companies
config:
tables:
- ${CI_BRANCH}_HUBSPOT.ORG_ACTIVITY
stream_maps:
# TODO: This wont actually work until the nested env vars issue is fixed
${CI_BRANCH}_hubspot-org_activity:
org_first_active_date: str(record.get('org_first_active_date')[:10])
if record.get('org_first_active_date') else None
org_last_active_date: str(record.get('org_last_active_date')[:10]) if
record.get('org_last_active_date') else None

loaders:
- name: target-snowflake
config:
Expand Down
18 changes: 14 additions & 4 deletions data/environments/prod.meltano.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,23 +24,33 @@ environments:
- name: tap-snowflake-metrics-legacy
config:
tables:
- MELTANO_HUB.FACT_HUB_METRICS
- MELTANO_HUB.FACT_HUB_METRICS
- name: tap-snowflake-metrics
config:
tables:
- MELTANO_HUB.FACT_VARIANT_HUB_METRICS
- MELTANO_HUB.FACT_VARIANT_HUB_METRICS
- name: tap-snowflake-audit
config:
tables:
- MELTANO_HUB.HUB_METRICS_AUDIT
- MELTANO_HUB.HUB_METRICS_AUDIT
- name: tap-snowflake-singer-activity
config:
tables:
- SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
- SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
- name: tap-snowflake-meltano-activity
config:
tables:
- SLACK_NOTIFICATIONS.MELTANO_ACTIVITY_NOTIFICATIONS
- name: tap-snowflake-hubspot-companies
config:
tables:
- HUBSPOT.ORG_ACTIVITY
stream_maps:
hubspot-org_activity:
org_first_active_date: str(record.get('org_first_active_date')[:10])
if record.get('org_first_active_date') else None
org_last_active_date: str(record.get('org_last_active_date')[:10]) if
record.get('org_last_active_date') else None
loaders:
- name: target-snowflake
config:
Expand Down
18 changes: 14 additions & 4 deletions data/environments/userdev.meltano.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,29 @@ environments:
- name: tap-snowflake-metrics
config:
tables:
- PNADOLNY_MELTANO_HUB.FACT_VARIANT_HUB_METRICS
- PNADOLNY_MELTANO_HUB.FACT_VARIANT_HUB_METRICS
- name: tap-snowflake-audit
config:
tables:
- PNADOLNY_MELTANO_HUB.HUB_METRICS_AUDIT
- PNADOLNY_MELTANO_HUB.HUB_METRICS_AUDIT
- name: tap-snowflake-singer-activity
config:
tables:
- PNADOLNY_SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
- PNADOLNY_SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
- name: tap-snowflake-meltano-activity
config:
tables:
- PNADOLNY_SLACK_NOTIFICATIONS.MELTANO_ACTIVITY_NOTIFICATIONS
- PNADOLNY_SLACK_NOTIFICATIONS.MELTANO_ACTIVITY_NOTIFICATIONS
- name: tap-snowflake-hubspot-companies
config:
tables:
- PNADOLNY_HUBSPOT.ORG_ACTIVITY
stream_maps:
pnadolny_hubspot-org_activity:
org_first_active_date: str(record.get('org_first_active_date')[:10])
if record.get('org_first_active_date') else None
org_last_active_date: str(record.get('org_last_active_date')[:10]) if
record.get('org_last_active_date') else None
loaders:
- name: target-snowflake
config:
Expand Down
23 changes: 20 additions & 3 deletions data/extract/extractors.meltano.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ plugins:
table_scan_kwargs:
organizations-table:
Select: SPECIFIC_ATTRIBUTES
ProjectionExpression: 'org_name, display_name, tenant_resource_key'
ProjectionExpression: org_name, display_name, tenant_resource_key
tables:
- workload-metadata-table
- projects-table
Expand Down Expand Up @@ -231,11 +231,11 @@ plugins:
- '*hub_metrics_audit.metric_type'
- name: tap-snowflake-singer-activity
inherit_from: tap-snowflake
config:
schema: SLACK_NOTIFICATIONS
metadata:
'*':
replication-method: FULL_TABLE
config:
schema: SLACK_NOTIFICATIONS
select:
- '*singer_activity_notifications.title'
- '*singer_activity_notifications.body'
Expand All @@ -247,6 +247,23 @@ plugins:
select:
- '*meltano_activity_notifications.title'
- '*meltano_activity_notifications.body'
- name: tap-snowflake-hubspot-companies
inherit_from: tap-snowflake
metadata:
'*':
replication-method: FULL_TABLE
select:
- '*org_activity.project_org_name'
- '*org_activity.project_org_domain'
- '*org_activity.project_ids'
- '*org_activity.pipeline_runs_l30'
- '*org_activity.segments'
- '*org_activity.org_first_event_at'
- '*org_activity.org_last_event_at'
- '*org_activity.org_first_active_date'
- '*org_activity.org_last_active_date'
- '*org_activity.is_currently_active'
- '*org_activity.unique_plugins'
- name: tap-smoke-test
variant: meltano
pip_url: git+https://github.com/meltano/tap-smoke-test.git
Expand Down
45 changes: 44 additions & 1 deletion data/load/loaders.meltano.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,47 @@ plugins:
inherit_from: target-apprise
- name: target-jsonl
variant: andyh1203
pip_url: target-jsonl
pip_url: target-jsonl
- name: target-hubspot
variant: meltanolabs
pip_url: git+https://github.com/MeltanoLabs/target-hubspot.git
- name: target-hubspot-companies
inherit_from: target-hubspot
config:
date_format: YEAR_MONTH_DAY
import_operations: UPSERT
column_mapping:
- columnObjectTypeId: 0-2
columnName: project_org_name
propertyName: name
- columnObjectTypeId: 0-2
columnName: project_org_domain
propertyName: domain
columnType: HUBSPOT_ALTERNATE_ID
- columnObjectTypeId: 0-2
columnName: project_ids
propertyName: telemetry__number_of_projects
- columnObjectTypeId: 0-2
columnName: unique_plugins
propertyName: telemetry__unique_plugins
- columnObjectTypeId: 0-2
columnName: segments
propertyName: telemetry__segments
- columnObjectTypeId: 0-2
columnName: pipeline_runs_l30
propertyName: telemetry__pipeline_runs__30_days_
- columnObjectTypeId: 0-2
columnName: is_currently_active
propertyName: telemetry__currently_active
- columnObjectTypeId: 0-2
columnName: org_first_event_at
propertyName: telemetry__first_event_at
- columnObjectTypeId: 0-2
columnName: org_last_event_at
propertyName: telemetry__last_event_at
- columnObjectTypeId: 0-2
columnName: org_first_active_date
propertyName: telemetry__first_active_at
- columnObjectTypeId: 0-2
columnName: org_last_active_date
propertyName: telemetry__last_active_at
1 change: 1 addition & 0 deletions data/orchestrate/orchestrators.meltano.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ jobs:
tasks:
- dbt-snowflake:run_hubspot_publish
- dbt-snowflake:test_hubspot_publish
- tap-snowflake-hubspot-companies target-hubspot-companies

- name: slack_notifications
tasks:
Expand Down
104 changes: 104 additions & 0 deletions data/plugins/loaders/target-hubspot--meltanolabs.lock
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
{
"plugin_type": "loaders",
"name": "target-hubspot",
"namespace": "target_hubspot",
"variant": "meltanolabs",
"label": "HubSpot",
"docs": "https://hub.meltano.com/loaders/target-hubspot--meltanolabs",
"repo": "https://github.com/MeltanoLabs/target-hubspot",
"pip_url": "git+https://github.com/MeltanoLabs/target-hubspot.git",
"executable": "target-hubspot",
"description": "Inbound Marketing software",
"logo_url": "https://hub.meltano.com/assets/logos/loaders/hubspot.png",
"capabilities": [
"about",
"schema-flattening",
"stream-maps"
],
"settings_group_validation": [
[
"access_token",
"column_mapping"
]
],
"settings": [
{
"name": "access_token",
"kind": "password",
"label": "Access Token",
"description": "Your HubSpot private app API access token. See the [docs](https://developers.hubspot.com/docs/api/private-apps) for more details."
},
{
"name": "column_mapping",
"kind": "array",
"label": "Column Mapping",
"description": "An array including an object entry for each column in your import file stream."
},
{
"name": "date_format",
"kind": "options",
"value": "YEAR_MONTH_DAY",
"label": "Date Format",
"description": "The format for dates included in the import file stream.",
"options": [
{
"label": "Month Day Year",
"value": "MONTH_DAY_YEAR"
},
{
"label": "Year Month Day",
"value": "YEAR_MONTH_DAY"
},
{
"label": "Day Month Year",
"value": "DAY_MONTH_YEAR"
}
]
},
{
"name": "flattening_enabled",
"kind": "boolean",
"label": "Flattening Enabled",
"description": "'True' to enable schema flattening and automatically expand nested properties."
},
{
"name": "flattening_max_depth",
"kind": "integer",
"label": "Flattening Max Depth",
"description": "The max depth to flatten schemas."
},
{
"name": "import_operations",
"kind": "options",
"value": "UPDATE",
"label": "Import Operations",
"description": "Used to indicate whether the import should create and update, only create, or only update records for a certain object or activity.",
"options": [
{
"label": "Create",
"value": "CREATE"
},
{
"label": "Update",
"value": "UPDATE"
},
{
"label": "Upsert",
"value": "UPSERT"
}
]
},
{
"name": "stream_map_config",
"kind": "object",
"label": "Stream Map Config",
"description": "User-defined config values to be used within map expressions."
},
{
"name": "stream_maps",
"kind": "object",
"label": "Stream Maps",
"description": "Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html)."
}
]
}
1 change: 1 addition & 0 deletions data/transform/models/publish/hubspot/org_activity.sql
Original file line number Diff line number Diff line change
Expand Up @@ -56,3 +56,4 @@ SELECT
FROM activity
LEFT JOIN plugins
ON plugins.project_org_domain = activity.project_org_domain
WHERE activity.project_org_domain != '-'

0 comments on commit 514aa47

Please sign in to comment.