From edcc193d0238dab136ff503cb3c9aec0752d7f74 Mon Sep 17 00:00:00 2001 From: Lucie Milan Date: Thu, 23 Nov 2023 14:57:37 +0100 Subject: [PATCH] fixed hubspot issues --- .../schemas/hubspot/v1/hubspot-v1-issues.txt | 7 -- .../schemas/hubspot/v1/hubspot-v1-tables.yml | 61 ++++----- .../schemas/hubspot/v1/json/campaigns.json | 6 + .../v1/json/hubspot_contacts_by_company.json | 16 --- .../hubspot/v2/hubspot-v2-foreign-keys.yml | 116 +++++++++--------- .../schemas/hubspot/v2/hubspot-v2-issues.txt | 9 -- .../schemas/hubspot/v2/hubspot-v2-tables.yml | 1 + scripts/json/file_exceptions.json | 5 + 8 files changed, 102 insertions(+), 119 deletions(-) delete mode 100644 _data/taps/schemas/hubspot/v1/json/hubspot_contacts_by_company.json delete mode 100644 _data/taps/schemas/hubspot/v2/hubspot-v2-issues.txt diff --git a/_data/taps/schemas/hubspot/v1/hubspot-v1-issues.txt b/_data/taps/schemas/hubspot/v1/hubspot-v1-issues.txt index 3624dd7ce..1fc3ec701 100644 --- a/_data/taps/schemas/hubspot/v1/hubspot-v1-issues.txt +++ b/_data/taps/schemas/hubspot/v1/hubspot-v1-issues.txt @@ -1,9 +1,2 @@ Replication Key properties__lastModifiedDate__value not found in _data/taps/schemas/hubspot/v1/json/contacts.json -Primary Key id not found in _data/taps/schemas/hubspot/v1/json/engagements.json Replication Key lastUpdated not found in _data/taps/schemas/hubspot/v1/json/engagements.json -Primary Key not found in _data/taps/schemas/hubspot/v1/json/deal_pipelines.json -Replication Key not found in _data/taps/schemas/hubspot/v1/json/deal_pipelines.json -Primary Key not found in _data/taps/schemas/hubspot/v1/json/hubspot_contacts_by_company.json -Replication Key not found in _data/taps/schemas/hubspot/v1/json/hubspot_contacts_by_company.json -Primary Key not found in _data/taps/schemas/hubspot/v1/json/workflows.json -Replication Key not found in _data/taps/schemas/hubspot/v1/json/workflows.json diff --git a/_data/taps/schemas/hubspot/v1/hubspot-v1-tables.yml b/_data/taps/schemas/hubspot/v1/hubspot-v1-tables.yml index ba9f282d3..1529a4620 100644 --- a/_data/taps/schemas/hubspot/v1/hubspot-v1-tables.yml +++ b/_data/taps/schemas/hubspot/v1/hubspot-v1-tables.yml @@ -50,7 +50,7 @@ tables: singer-schema: https://github.com/singer-io/tap-hubspot/blob/master/tap_hubspot/schemas/contacts.json api-method: https://developers.hubspot.com/docs/methods/contacts/get_contacts table-details: - replication-method: Incremental + replication-method: Key-based Incremental primary-key: canonical-vid replication-key: properties__lastModifiedDate__value - name: contact_lists @@ -64,7 +64,7 @@ tables: singer-schema: https://github.com/singer-io/tap-hubspot/blob/master/tap_hubspot/schemas/contact_lists.json api-method: https://developers.hubspot.com/docs/methods/lists/get_lists table-details: - replication-method: Incremental + replication-method: Key-based Incremental primary-key: listId replication-key: updatedAt - name: deals @@ -90,7 +90,7 @@ tables: singer-schema: https://github.com/singer-io/tap-hubspot/blob/master/tap_hubspot/schemas/email_events.json api-method: https://developers.hubspot.com/docs/methods/email/get_events table-details: - replication-method: Incremental + replication-method: Key-based Incremental primary-key: id replication-key: id - name: engagements @@ -103,8 +103,8 @@ tables: singer-schema: https://github.com/singer-io/tap-hubspot/blob/master/tap_hubspot/schemas/engagements.json api-method: https://developers.hubspot.com/docs/methods/engagements/get-all-engagements table-details: - replication-method: Incremental - primary-key: id + replication-method: Key-based Incremental + primary-key: engagement_id replication-key: lastUpdated - name: forms description: 'The `forms` table contains info about your HubSpot website forms. @@ -115,7 +115,7 @@ tables: singer-schema: https://github.com/singer-io/tap-hubspot/blob/master/tap_hubspot/schemas/forms.json api-method: https://developers.hubspot.com/docs/methods/forms/v2/get_forms table-details: - replication-method: Incremental + replication-method: Key-based Incremental primary-key: guid replication-key: updatedAt - name: keywords @@ -130,6 +130,7 @@ tables: replication-method: Key-based Incremental primary-key: keyword_guid replication-key: created_at + status: not_found - name: owners description: 'The `owners` table contains info about the owners that exist in your HubSpot portal. Owners are created and updated in HubSpot when new users are added @@ -141,7 +142,7 @@ tables: singer-schema: https://github.com/singer-io/tap-hubspot/blob/master/tap_hubspot/schemas/owners.json api-method: https://developers.hubspot.com/docs/methods/owners/get_owners table-details: - replication-method: Incremental + replication-method: Key-based Incremental primary-keys: - ownerId - portalId @@ -156,36 +157,38 @@ tables: singer-schema: https://github.com/singer-io/tap-hubspot/blob/master/tap_hubspot/schemas/subscription_changes.json api-method: https://developers.hubspot.com/docs/methods/email/get_subscriptions_timeline table-details: - replication-method: Incremental + replication-method: Key-based Incremental primary-keys: - recipient - portalId - timestamp replication-key: timestamp - name: deal_pipelines - description: '' - links: - singer-schema: '' - doc-link: '' - table-details: - replication-method: '' - primary-key: '' - replication-key: '' -- name: hubspot_contacts_by_company - description: '' + description: 'The `{{ table.name }}` table contains info about the `deal stage` + and `pipeline` properties. + + ' links: - singer-schema: '' - doc-link: '' + doc-link: https://developers.hubspot.com/docs/methods/deal-pipelines/overview + singer-schema: https://github.com/singer-io/tap-hubspot/blob/master/tap_hubspot/schemas/deal_pipelines.json + api-method: https://developers.hubspot.com/docs/methods/deal-pipelines/get-all-deal-pipelines table-details: - replication-method: '' - primary-key: '' - replication-key: '' + replication-method: Full Table + primary-key: pipelineId - name: workflows - description: '' + description: 'The `workflows` table contains info about the workflows in your {{ + integration.display_name }} portal. + + + **Note**: The `workflows` stream only supports contact-based workflows. + + ' links: - singer-schema: '' - doc-link: '' + doc-link: https://developers.hubspot.com/docs/methods/workflows/workflows_overview + singer-schema: https://github.com/singer-io/tap-hubspot/blob/master/tap_hubspot/schemas/workflows.json + api-method: https://developers.hubspot.com/docs/methods/workflows/v3/get_workflows table-details: - replication-method: '' - primary-key: '' - replication-key: '' + replication-method: Key-based Incremental + primary-key: id + replication-key: updatedAt +tap-repo-schemas: true diff --git a/_data/taps/schemas/hubspot/v1/json/campaigns.json b/_data/taps/schemas/hubspot/v1/json/campaigns.json index fd945a18f..86edbdecd 100644 --- a/_data/taps/schemas/hubspot/v1/json/campaigns.json +++ b/_data/taps/schemas/hubspot/v1/json/campaigns.json @@ -20,6 +20,12 @@ }, "counters": { "properties": { + "click": { + "type": [ + "null", + "integer" + ] + }, "delievered": { "type": [ "null", diff --git a/_data/taps/schemas/hubspot/v1/json/hubspot_contacts_by_company.json b/_data/taps/schemas/hubspot/v1/json/hubspot_contacts_by_company.json deleted file mode 100644 index 9e2753fec..000000000 --- a/_data/taps/schemas/hubspot/v1/json/hubspot_contacts_by_company.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "additionalProperties": false, - "properties": { - "company-id": { - "type": [ - "integer" - ] - }, - "contact-id": { - "type": [ - "integer" - ] - } - }, - "type": "object" -} \ No newline at end of file diff --git a/_data/taps/schemas/hubspot/v2/hubspot-v2-foreign-keys.yml b/_data/taps/schemas/hubspot/v2/hubspot-v2-foreign-keys.yml index 54dda37be..cd20449ab 100644 --- a/_data/taps/schemas/hubspot/v2/hubspot-v2-foreign-keys.yml +++ b/_data/taps/schemas/hubspot/v2/hubspot-v2-foreign-keys.yml @@ -27,10 +27,10 @@ keys: - company-id - table: deals keys: - - associations.associatedCompanyIds.value + - associations.associatedCompanyIds - table: engagements keys: - - associations.companyIds.value + - associations.companyIds contact_lists_listId: - table: contact_lists keys: @@ -40,15 +40,15 @@ keys: - list-memberships.internal-list-id - table: engagements keys: - - associations.contactIds.value + - associations.contactIds deals_dealId: - table: deals keys: - dealId - - associations.associatedDealIds.value + - associations.associatedDealIds - table: engagements keys: - - associations.dealIds.value + - associations.dealIds deal_pipelines_pipelineId: - table: deal_pipelines keys: @@ -64,14 +64,14 @@ keys: forms_form-id: - table: forms keys: - - form-id + - guid - table: contacts keys: - form-submissions.form-id owners_ownerId: - table: engagements keys: - - ownerId + - engagement.ownerId - table: owners keys: - ownerId @@ -95,7 +95,7 @@ keys: - portalId - table: engagements keys: - - portalId + - engagement.portalId - table: forms keys: - portalId @@ -151,7 +151,7 @@ tables: - table-name: engagements keys: - key: portalId - foreign-key: portalId + foreign-key: engagement.portalId - table-name: owners keys: - key: portalId @@ -178,7 +178,7 @@ tables: foreign-key: portalId - table-name: contacts keys: - - key: form-id + - key: guid foreign-key: form-submissions.form-id - key: portalId foreign-key: portal-id @@ -199,7 +199,7 @@ tables: - table-name: engagements keys: - key: portalId - foreign-key: portalId + foreign-key: engagement.portalId - table-name: owners keys: - key: portalId @@ -229,15 +229,15 @@ tables: - table-name: engagements keys: - key: list-memberships.internal-list-id - foreign-key: associations.contactIds.value + foreign-key: associations.contactIds - key: portal-id - foreign-key: portalId + foreign-key: engagement.portalId - key: form-submissions.portal-id - foreign-key: portalId + foreign-key: engagement.portalId - table-name: forms keys: - key: form-submissions.form-id - foreign-key: form-id + foreign-key: guid - key: portal-id foreign-key: portalId - key: form-submissions.portal-id @@ -293,11 +293,11 @@ tables: - table-name: deals keys: - key: company-id - foreign-key: associations.associatedCompanyIds.value + foreign-key: associations.associatedCompanyIds - table-name: engagements keys: - key: company-id - foreign-key: associations.companyIds.value + foreign-key: associations.companyIds - table-name: companies join: - table-name: contacts_by_company @@ -307,15 +307,15 @@ tables: - table-name: deals keys: - key: companyId - foreign-key: associations.associatedCompanyIds.value + foreign-key: associations.associatedCompanyIds - key: portalId foreign-key: portalId - table-name: engagements keys: - key: companyId - foreign-key: associations.companyIds.value + foreign-key: associations.companyIds - key: portalId - foreign-key: portalId + foreign-key: engagement.portalId - table-name: contact_lists keys: - key: portalId @@ -350,24 +350,24 @@ tables: join: - table-name: companies keys: - - key: associations.associatedCompanyIds.value + - key: associations.associatedCompanyIds foreign-key: companyId - key: portalId foreign-key: portalId - table-name: contacts_by_company keys: - - key: associations.associatedCompanyIds.value + - key: associations.associatedCompanyIds foreign-key: company-id - table-name: engagements keys: - - key: associations.associatedCompanyIds.value - foreign-key: associations.companyIds.value + - key: associations.associatedCompanyIds + foreign-key: associations.companyIds - key: dealId - foreign-key: associations.dealIds.value - - key: associations.associatedDealIds.value - foreign-key: associations.dealIds.value + foreign-key: associations.dealIds + - key: associations.associatedDealIds + foreign-key: associations.dealIds - key: portalId - foreign-key: portalId + foreign-key: engagement.portalId - table-name: contact_lists keys: - key: portalId @@ -402,61 +402,61 @@ tables: join: - table-name: companies keys: - - key: associations.companyIds.value + - key: associations.companyIds foreign-key: companyId - - key: portalId + - key: engagement.portalId foreign-key: portalId - table-name: contacts_by_company keys: - - key: associations.companyIds.value + - key: associations.companyIds foreign-key: company-id - table-name: deals keys: - - key: associations.companyIds.value - foreign-key: associations.associatedCompanyIds.value - - key: associations.dealIds.value + - key: associations.companyIds + foreign-key: associations.associatedCompanyIds + - key: associations.dealIds foreign-key: dealId - - key: associations.dealIds.value - foreign-key: associations.associatedDealIds.value - - key: portalId + - key: associations.dealIds + foreign-key: associations.associatedDealIds + - key: engagement.portalId foreign-key: portalId - table-name: contact_lists keys: - - key: associations.contactIds.value + - key: associations.contactIds foreign-key: listId - - key: portalId + - key: engagement.portalId foreign-key: portalId - table-name: contacts keys: - - key: associations.contactIds.value + - key: associations.contactIds foreign-key: list-memberships.internal-list-id - - key: portalId + - key: engagement.portalId foreign-key: portal-id - - key: portalId + - key: engagement.portalId foreign-key: form-submissions.portal-id - table-name: owners keys: - - key: ownerId + - key: engagement.ownerId foreign-key: ownerId - - key: ownerId + - key: engagement.ownerId foreign-key: remoteList.ownerId - - key: portalId + - key: engagement.portalId foreign-key: portalId - - key: portalId + - key: engagement.portalId foreign-key: remoteList.portalId - table-name: email_events keys: - - key: portalId + - key: engagement.portalId foreign-key: portalId - table-name: forms keys: - - key: portalId + - key: engagement.portalId foreign-key: portalId - table-name: subscription_changes keys: - - key: portalId + - key: engagement.portalId foreign-key: portalId - - key: portalId + - key: engagement.portalId foreign-key: changes.portalId - table-name: contact_lists join: @@ -471,9 +471,9 @@ tables: - table-name: engagements keys: - key: listId - foreign-key: associations.contactIds.value + foreign-key: associations.contactIds - key: portalId - foreign-key: portalId + foreign-key: engagement.portalId - table-name: companies keys: - key: portalId @@ -507,13 +507,13 @@ tables: - table-name: engagements keys: - key: ownerId - foreign-key: ownerId + foreign-key: engagement.ownerId - key: remoteList.ownerId - foreign-key: ownerId + foreign-key: engagement.ownerId - key: portalId - foreign-key: portalId + foreign-key: engagement.portalId - key: remoteList.portalId - foreign-key: portalId + foreign-key: engagement.portalId - table-name: companies keys: - key: portalId @@ -603,9 +603,9 @@ tables: - table-name: engagements keys: - key: portalId - foreign-key: portalId + foreign-key: engagement.portalId - key: changes.portalId - foreign-key: portalId + foreign-key: engagement.portalId - table-name: forms keys: - key: portalId diff --git a/_data/taps/schemas/hubspot/v2/hubspot-v2-issues.txt b/_data/taps/schemas/hubspot/v2/hubspot-v2-issues.txt deleted file mode 100644 index 3c8da6e50..000000000 --- a/_data/taps/schemas/hubspot/v2/hubspot-v2-issues.txt +++ /dev/null @@ -1,9 +0,0 @@ -JSON file versions.json ignored -Key form-id not found in _data/taps/schemas/hubspot/v2/json/forms.json -Key associations.associatedCompanyIds.value not found in _data/taps/schemas/hubspot/v2/json/deals.json -Key associations.associatedDealIds.value not found in _data/taps/schemas/hubspot/v2/json/deals.json -Key associations.companyIds.value not found in _data/taps/schemas/hubspot/v2/json/engagements.json -Key portalId not found in _data/taps/schemas/hubspot/v2/json/engagements.json -Key associations.dealIds.value not found in _data/taps/schemas/hubspot/v2/json/engagements.json -Key associations.contactIds.value not found in _data/taps/schemas/hubspot/v2/json/engagements.json -Key ownerId not found in _data/taps/schemas/hubspot/v2/json/engagements.json diff --git a/_data/taps/schemas/hubspot/v2/hubspot-v2-tables.yml b/_data/taps/schemas/hubspot/v2/hubspot-v2-tables.yml index e143699fc..43329f08f 100644 --- a/_data/taps/schemas/hubspot/v2/hubspot-v2-tables.yml +++ b/_data/taps/schemas/hubspot/v2/hubspot-v2-tables.yml @@ -219,3 +219,4 @@ tables: replication-method: Key-based Incremental primary-key: id replication-key: updatedAt +tap-repo-schemas: true diff --git a/scripts/json/file_exceptions.json b/scripts/json/file_exceptions.json index b0c8e4c6e..29dd00fad 100644 --- a/scripts/json/file_exceptions.json +++ b/scripts/json/file_exceptions.json @@ -35,6 +35,11 @@ } }, "tap-hubspot": { + "1": { + "ignore": [ + "hubspot_contacts_by_company.json" + ] + }, "2": { "keep": [ "custom_objects.json"