feat: publish data to hubspot via target-hubspot

data/environments/cicd.meltano.yml

@@ -46,19 +46,31 @@ environments:
       - name: tap-snowflake-metrics
         config:
           tables:
-            - ${CI_BRANCH}_MELTANO_HUB.FACT_VARIANT_HUB_METRICS
+          - ${CI_BRANCH}_MELTANO_HUB.FACT_VARIANT_HUB_METRICS
       - name: tap-snowflake-audit
         config:
           tables:
-            - ${CI_BRANCH}_MELTANO_HUB.HUB_METRICS_AUDIT
+          - ${CI_BRANCH}_MELTANO_HUB.HUB_METRICS_AUDIT
       - name: tap-snowflake-singer-activity
         config:
           tables:
-            - ${CI_BRANCH}_SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
+          - ${CI_BRANCH}_SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
       - name: tap-snowflake-meltano-activity
         config:
           tables:
-            - ${CI_BRANCH}_SLACK_NOTIFICATIONS.MELTANO_ACTIVITY_NOTIFICATIONS
+          - ${CI_BRANCH}_SLACK_NOTIFICATIONS.MELTANO_ACTIVITY_NOTIFICATIONS
+      - name: tap-snowflake-hubspot-companies
+        config:
+          tables:
+          - ${CI_BRANCH}_HUBSPOT.ORG_ACTIVITY
+          stream_maps:
+            # TODO: This wont actually work until the nested env vars issue is fixed
+            ${CI_BRANCH}_hubspot-org_activity:
+              org_first_active_date: str(record.get('org_first_active_date')[:10])
+                if record.get('org_first_active_date') else None
+              org_last_active_date: str(record.get('org_last_active_date')[:10]) if
+                record.get('org_last_active_date') else None
+
       loaders:
       - name: target-snowflake
         config:

data/environments/prod.meltano.yml

@@ -24,23 +24,33 @@ environments:
       - name: tap-snowflake-metrics-legacy
         config:
           tables:
-           - MELTANO_HUB.FACT_HUB_METRICS
+          - MELTANO_HUB.FACT_HUB_METRICS
       - name: tap-snowflake-metrics
         config:
           tables:
-           - MELTANO_HUB.FACT_VARIANT_HUB_METRICS
+          - MELTANO_HUB.FACT_VARIANT_HUB_METRICS
       - name: tap-snowflake-audit
         config:
           tables:
-           - MELTANO_HUB.HUB_METRICS_AUDIT
+          - MELTANO_HUB.HUB_METRICS_AUDIT
       - name: tap-snowflake-singer-activity
         config:
           tables:
-           - SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
+          - SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
       - name: tap-snowflake-meltano-activity
         config:
           tables:
           - SLACK_NOTIFICATIONS.MELTANO_ACTIVITY_NOTIFICATIONS
+      - name: tap-snowflake-hubspot-companies
+        config:
+          tables:
+          - HUBSPOT.ORG_ACTIVITY
+          stream_maps:
+            hubspot-org_activity:
+              org_first_active_date: str(record.get('org_first_active_date')[:10])
+                if record.get('org_first_active_date') else None
+              org_last_active_date: str(record.get('org_last_active_date')[:10]) if
+                record.get('org_last_active_date') else None
       loaders:
       - name: target-snowflake
         config:

data/environments/userdev.meltano.yml

@@ -28,19 +28,29 @@ environments:
       - name: tap-snowflake-metrics
         config:
           tables:
-           -  PNADOLNY_MELTANO_HUB.FACT_VARIANT_HUB_METRICS
+          - PNADOLNY_MELTANO_HUB.FACT_VARIANT_HUB_METRICS
       - name: tap-snowflake-audit
         config:
           tables:
-           - PNADOLNY_MELTANO_HUB.HUB_METRICS_AUDIT
+          - PNADOLNY_MELTANO_HUB.HUB_METRICS_AUDIT
       - name: tap-snowflake-singer-activity
         config:
           tables:
-           - PNADOLNY_SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
+          - PNADOLNY_SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
       - name: tap-snowflake-meltano-activity
         config:
           tables:
-           - PNADOLNY_SLACK_NOTIFICATIONS.MELTANO_ACTIVITY_NOTIFICATIONS
+          - PNADOLNY_SLACK_NOTIFICATIONS.MELTANO_ACTIVITY_NOTIFICATIONS
+      - name: tap-snowflake-hubspot-companies
+        config:
+          tables:
+          - PNADOLNY_HUBSPOT.ORG_ACTIVITY
+          stream_maps:
+            pnadolny_hubspot-org_activity:
+              org_first_active_date: str(record.get('org_first_active_date')[:10])
+                if record.get('org_first_active_date') else None
+              org_last_active_date: str(record.get('org_last_active_date')[:10]) if
+                record.get('org_last_active_date') else None
       loaders:
       - name: target-snowflake
         config:

data/extract/extractors.meltano.yml

@@ -10,7 +10,7 @@ plugins:
       table_scan_kwargs:
         organizations-table:
           Select: SPECIFIC_ATTRIBUTES
-          ProjectionExpression: 'org_name, display_name, tenant_resource_key'
+          ProjectionExpression: org_name, display_name, tenant_resource_key
       tables:
       - workload-metadata-table
       - projects-table
@@ -231,11 +231,11 @@ plugins:
     - '*hub_metrics_audit.metric_type'
   - name: tap-snowflake-singer-activity
     inherit_from: tap-snowflake
+    config:
+      schema: SLACK_NOTIFICATIONS
     metadata:
       '*':
         replication-method: FULL_TABLE
-    config:
-      schema: SLACK_NOTIFICATIONS
     select:
     - '*singer_activity_notifications.title'
     - '*singer_activity_notifications.body'
@@ -247,6 +247,23 @@ plugins:
     select:
     - '*meltano_activity_notifications.title'
     - '*meltano_activity_notifications.body'
+  - name: tap-snowflake-hubspot-companies
+    inherit_from: tap-snowflake
+    metadata:
+      '*':
+        replication-method: FULL_TABLE
+    select:
+    - '*org_activity.project_org_name'
+    - '*org_activity.project_org_domain'
+    - '*org_activity.project_ids'
+    - '*org_activity.pipeline_runs_l30'
+    - '*org_activity.segments'
+    - '*org_activity.org_first_event_at'
+    - '*org_activity.org_last_event_at'
+    - '*org_activity.org_first_active_date'
+    - '*org_activity.org_last_active_date'
+    - '*org_activity.is_currently_active'
+    - '*org_activity.unique_plugins'
   - name: tap-smoke-test
     variant: meltano
     pip_url: git+https://github.com/meltano/tap-smoke-test.git

data/load/loaders.meltano.yml

@@ -43,4 +43,47 @@ plugins:
     inherit_from: target-apprise
   - name: target-jsonl
     variant: andyh1203
-    pip_url: target-jsonl
+    pip_url: target-jsonl
+  - name: target-hubspot
+    variant: meltanolabs
+    pip_url: git+https://github.com/MeltanoLabs/target-hubspot.git
+  - name: target-hubspot-companies
+    inherit_from: target-hubspot
+    config:
+      date_format: YEAR_MONTH_DAY
+      import_operations: UPSERT
+      column_mapping:
+      - columnObjectTypeId: 0-2
+        columnName: project_org_name
+        propertyName: name
+      - columnObjectTypeId: 0-2
+        columnName: project_org_domain
+        propertyName: domain
+        columnType: HUBSPOT_ALTERNATE_ID
+      - columnObjectTypeId: 0-2
+        columnName: project_ids
+        propertyName: telemetry__number_of_projects
+      - columnObjectTypeId: 0-2
+        columnName: unique_plugins
+        propertyName: telemetry__unique_plugins
+      - columnObjectTypeId: 0-2
+        columnName: segments
+        propertyName: telemetry__segments
+      - columnObjectTypeId: 0-2
+        columnName: pipeline_runs_l30
+        propertyName: telemetry__pipeline_runs__30_days_
+      - columnObjectTypeId: 0-2
+        columnName: is_currently_active
+        propertyName: telemetry__currently_active
+      - columnObjectTypeId: 0-2
+        columnName: org_first_event_at
+        propertyName: telemetry__first_event_at
+      - columnObjectTypeId: 0-2
+        columnName: org_last_event_at
+        propertyName: telemetry__last_event_at
+      - columnObjectTypeId: 0-2
+        columnName: org_first_active_date
+        propertyName: telemetry__first_active_at
+      - columnObjectTypeId: 0-2
+        columnName: org_last_active_date
+        propertyName: telemetry__last_active_at

data/orchestrate/orchestrators.meltano.yml

@@ -89,6 +89,7 @@ jobs:
   tasks:
   - dbt-snowflake:run_hubspot_publish
   - dbt-snowflake:test_hubspot_publish
+  - tap-snowflake-hubspot-companies target-hubspot-companies
 
 - name: slack_notifications
   tasks:

data/plugins/loaders/target-hubspot--meltanolabs.lock

@@ -0,0 +1,104 @@
+{
+  "plugin_type": "loaders",
+  "name": "target-hubspot",
+  "namespace": "target_hubspot",
+  "variant": "meltanolabs",
+  "label": "HubSpot",
+  "docs": "https://hub.meltano.com/loaders/target-hubspot--meltanolabs",
+  "repo": "https://github.com/MeltanoLabs/target-hubspot",
+  "pip_url": "git+https://github.com/MeltanoLabs/target-hubspot.git",
+  "executable": "target-hubspot",
+  "description": "Inbound Marketing software",
+  "logo_url": "https://hub.meltano.com/assets/logos/loaders/hubspot.png",
+  "capabilities": [
+    "about",
+    "schema-flattening",
+    "stream-maps"
+  ],
+  "settings_group_validation": [
+    [
+      "access_token",
+      "column_mapping"
+    ]
+  ],
+  "settings": [
+    {
+      "name": "access_token",
+      "kind": "password",
+      "label": "Access Token",
+      "description": "Your HubSpot private app API access token. See the [docs](https://developers.hubspot.com/docs/api/private-apps) for more details."
+    },
+    {
+      "name": "column_mapping",
+      "kind": "array",
+      "label": "Column Mapping",
+      "description": "An array including an object entry for each column in your import file stream."
+    },
+    {
+      "name": "date_format",
+      "kind": "options",
+      "value": "YEAR_MONTH_DAY",
+      "label": "Date Format",
+      "description": "The format for dates included in the import file stream.",
+      "options": [
+        {
+          "label": "Month Day Year",
+          "value": "MONTH_DAY_YEAR"
+        },
+        {
+          "label": "Year Month Day",
+          "value": "YEAR_MONTH_DAY"
+        },
+        {
+          "label": "Day Month Year",
+          "value": "DAY_MONTH_YEAR"
+        }
+      ]
+    },
+    {
+      "name": "flattening_enabled",
+      "kind": "boolean",
+      "label": "Flattening Enabled",
+      "description": "'True' to enable schema flattening and automatically expand nested properties."
+    },
+    {
+      "name": "flattening_max_depth",
+      "kind": "integer",
+      "label": "Flattening Max Depth",
+      "description": "The max depth to flatten schemas."
+    },
+    {
+      "name": "import_operations",
+      "kind": "options",
+      "value": "UPDATE",
+      "label": "Import Operations",
+      "description": "Used to indicate whether the import should create and update, only create, or only update records for a certain object or activity.",
+      "options": [
+        {
+          "label": "Create",
+          "value": "CREATE"
+        },
+        {
+          "label": "Update",
+          "value": "UPDATE"
+        },
+        {
+          "label": "Upsert",
+          "value": "UPSERT"
+        }
+      ]
+    },
+    {
+      "name": "stream_map_config",
+      "kind": "object",
+      "label": "Stream Map Config",
+      "description": "User-defined config values to be used within map expressions."
+    },
+    {
+      "name": "stream_maps",
+      "kind": "object",
+      "label": "Stream Maps",
+      "description": "Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html)."
+    }
+  ]
+}

data/transform/models/publish/hubspot/org_activity.sql

@@ -56,3 +56,4 @@ SELECT
 FROM activity
 LEFT JOIN plugins
     ON plugins.project_org_domain = activity.project_org_domain
+WHERE activity.project_org_domain != '-'