Adds Archived Companies and Deals (#10)

tap_hubspot_beta/streams.py

@@ -777,6 +777,65 @@ class CompaniesStream(ObjectSearchV3):
     replication_key_filter = "hs_lastmodifieddate"
     properties_url = "properties/v1/companies/properties"
 
+
+class ArchivedCompaniesStream(hubspotV3Stream):
+    """Archived Companies Stream"""
+
+    name = "companies_archived"
+    replication_key = "archivedAt"
+    path = "crm/v3/objects/companies?archived=true"
+    properties_url = "properties/v1/companies/properties"
+    primary_keys = ["id"]
+
+    base_properties = [
+        th.Property("id", th.StringType),
+        th.Property("archived", th.BooleanType),
+        th.Property("archivedAt", th.DateTimeType),
+        th.Property("createdAt", th.DateTimeType),
+        th.Property("updatedAt", th.DateTimeType)
+    ]
+
+    @property
+    def selected(self) -> bool:
+        """Check if stream is selected.
+
+        Returns:
+            True if the stream is selected.
+        """
+        # It has to be in the catalog or it will cause issues
+        if not self._tap.catalog.get("companies_archived"):
+            return False
+
+        try:
+            # Make this stream auto-select if companies is selected
+            self._tap.catalog["companies_archived"] = self._tap.catalog["companies"]
+            return self.mask.get((), False) or self._tap.catalog["companies"].metadata.get(()).selected
+        except:
+            return self.mask.get((), False)
+
+    def _write_record_message(self, record: dict) -> None:
+        """Write out a RECORD message.
+
+        Args:
+            record: A single stream record.
+        """
+        for record_message in self._generate_record_messages(record):
+            # force this to think it's the companies stream
+            record_message.stream = "companies"
+            singer.write_message(record_message)
+
+    def post_process(self, row, context):
+        row = super().post_process(row, context)
+
+        rep_key = self.get_starting_timestamp(context).replace(tzinfo=pytz.utc)
+        archived_at = parse(row['archivedAt']).replace(tzinfo=pytz.utc)
+
+        if archived_at > rep_key:
+            return row
+
+        return None
+
+
 class TicketsStream(ObjectSearchV3):
     """Companies Stream"""
 
@@ -785,6 +844,7 @@ class TicketsStream(ObjectSearchV3):
     replication_key_filter = "hs_lastmodifieddate"
     properties_url = "properties/v2/tickets/properties"
 
+
 class DealsStream(ObjectSearchV3):
     """Deals Stream"""
 
@@ -797,6 +857,64 @@ def get_child_context(self, record: dict, context) -> dict:
         return {"id": record["id"]}
 
 
+class ArchivedDealsStream(hubspotV3Stream):
+    """Archived Deals Stream"""
+
+    name = "deals_archived"
+    replication_key = "archivedAt"
+    path = "crm/v3/objects/deals?archived=true"
+    properties_url = "properties/v1/deals/properties"
+    primary_keys = ["id"]
+
+    base_properties = [
+        th.Property("id", th.StringType),
+        th.Property("archived", th.BooleanType),
+        th.Property("archivedAt", th.DateTimeType),
+        th.Property("createdAt", th.DateTimeType),
+        th.Property("updatedAt", th.DateTimeType)
+    ]
+
+    @property
+    def selected(self) -> bool:
+        """Check if stream is selected.
+
+        Returns:
+            True if the stream is selected.
+        """
+        # It has to be in the catalog or it will cause issues
+        if not self._tap.catalog.get("deals_archived"):
+            return False
+
+        try:
+            # Make this stream auto-select if deals is selected
+            self._tap.catalog["deals_archived"] = self._tap.catalog["deals"]
+            return self.mask.get((), False) or self._tap.catalog["deals"].metadata.get(()).selected
+        except:
+            return self.mask.get((), False)
+
+    def _write_record_message(self, record: dict) -> None:
+        """Write out a RECORD message.
+
+        Args:
+            record: A single stream record.
+        """
+        for record_message in self._generate_record_messages(record):
+            # force this to think it's the deals stream
+            record_message.stream = "deals"
+            singer.write_message(record_message)
+
+    def post_process(self, row, context):
+        row = super().post_process(row, context)
+
+        rep_key = self.get_starting_timestamp(context).replace(tzinfo=pytz.utc)
+        archived_at = parse(row['archivedAt']).replace(tzinfo=pytz.utc)
+
+        if archived_at > rep_key:
+            return row
+
+        return None
+
+
 class ProductsStream(ObjectSearchV3):
     """Products Stream"""
 
@@ -998,6 +1116,7 @@ class AssociationDealsStream(hubspotV4Stream):
         th.Property("associationTypes", th.CustomType({"type": ["array", "object"]})),
     ).to_dict()
 
+
 class AssociationContactsStream(hubspotV4Stream):
     """Association Base Stream"""
 
@@ -1034,12 +1153,14 @@ class AssociationDealsLineItemsStream(AssociationDealsStream):
     name = "associations_deals_line_items"
     path = "crm/v4/associations/deals/line_items/batch/read"
 
+
 class AssociationContactsTicketsStream(AssociationContactsStream):
     """Association Contacts -> Tickets Stream"""
 
     name = "associations_contacts_tickets"
     path = "crm/v4/associations/contacts/tickets/batch/read"
 
+
 class AssociationContactsStream(hubspotV4Stream):
     """Association Base Stream"""
 
@@ -1062,6 +1183,7 @@ class AssociationContactsCompaniesStream(AssociationContactsStream):
     name = "associations_contacts_companies"
     path = "crm/v4/associations/contacts/companies/batch/read"
 
+
 class MarketingEmailsStream(hubspotV1Stream):
     """Dispositions Stream"""
 
@@ -1165,6 +1287,7 @@ class MarketingEmailsStream(hubspotV1Stream):
         th.Property("vidsIncluded", th.CustomType({"type": ["array", "string"]})),
     ).to_dict()
 
+
 class PostalMailStream(ObjectSearchV3):
     """Owners Stream"""
 
@@ -1188,6 +1311,8 @@ class PostalMailStream(ObjectSearchV3):
         th.Property("archived", th.BooleanType),
         th.Property("associations", th.CustomType({"type": ["object", "array"]})),
     ).to_dict()
+
+
 class CommunicationsStream(ObjectSearchV3):
     """Owners Stream"""
 
@@ -1212,6 +1337,7 @@ class CommunicationsStream(ObjectSearchV3):
         th.Property("associations", th.CustomType({"type": ["object", "array"]})),
     ).to_dict()
 
+
 class QuotesStream(ObjectSearchV3):
     """Products Stream"""
 
@@ -1220,6 +1346,7 @@ class QuotesStream(ObjectSearchV3):
     replication_key_filter = "hs_lastmodifieddate"
     properties_url = "properties/v2/quotes/properties"
 
+
 class AssociationQuotesDealsStream(AssociationDealsStream):
     """Association Quotes -> Deals Stream"""
 

tap_hubspot_beta/tap.py

@@ -43,7 +43,9 @@
     QuotesStream,
     AssociationQuotesDealsStream,
     ListMembershipV3Stream,
-    ListSearchV3Stream
+    ListSearchV3Stream,
+    ArchivedCompaniesStream,
+    ArchivedDealsStream,
 )
 
 STREAM_TYPES = [
@@ -84,7 +86,9 @@
     QuotesStream,
     AssociationQuotesDealsStream,
     ListMembershipV3Stream,
-    ListSearchV3Stream
+    ListSearchV3Stream,
+    ArchivedCompaniesStream,
+    ArchivedDealsStream,
 ]