🐛 Source Chargebee: Ensure no pagination issues during concurrency (#…

…48510) Co-authored-by: Aaron ("AJ") Steers <[email protected]>
airbytehq · Nov 20, 2024 · e5e9b60 · e5e9b60
1 parent 7810e32
commit e5e9b60
Show file tree

Hide file tree

Showing 18 changed files with 345 additions and 285 deletions.
diff --git a/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml b/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml
@@ -32,6 +32,8 @@ acceptance_tests:
             bypass_reason: "Unstable data. Test data is not persistent. Tested with mocker server tests."
           - name: "site_migration_detail"
             bypass_reason: "Cannnot populate with test data."
+          - name: "contact"
+            bypass_reason: "Relies on customer as a parent stream and there is a bug where the parent stream is emitting stream statuses on rate limit which leads to CATs failing. See https://github.com/airbytehq/airbyte-internal-issues/issues/10808 for more information."
           - name: "customer"
             bypass_reason: "Tested with mocker server tests."
           - name: "subscription"
@@ -53,16 +55,7 @@ acceptance_tests:
         timeout_seconds: 2400
         configured_catalog_path: "integration_tests/configured_catalog.json"
         future_state:
-          future_state_path: "integration_tests/future_state.json"
-          missing_streams:
-            - name: contact
-              bypass_reason: "This stream is Full-Refresh only"
-            - name: quote_line_group
-              bypass_reason: "This stream is Full-Refresh only"
-            - name: attached_item
-              bypass_reason: "This stream is Full-Refresh only"
-            - name: subscription_with_scheduled_changes
-              bypass_reason: "This stream is Full-Refresh only"
+          bypass_reason: "This test does not make sense using Concurrent CDK"
   full_refresh:
     tests:
       - config_path: "secrets/config.json"

diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl
@@ -1,6 +1,3 @@
-{"stream": "contact", "data": {"id": "0000002", "first_name": "User2", "last_name": "Sample", "email": "[email protected]", "phone": "+13335556789", "label": "Tag2", "enabled": true, "send_account_email": true, "send_billing_email": true, "object": "contact", "customer_id": "Azz5jBTTJ96UqlvE", "custom_fields": []}, "emitted_at": 1706028645460}
-{"stream": "contact", "data": {"id": "Test 1", "first_name": "Sample Name 1", "last_name": "Sample Lastname 1", "email": "[email protected]", "enabled": true, "send_account_email": false, "send_billing_email": false, "object": "contact", "customer_id": "cbdemo_richard", "custom_fields": []}, "emitted_at": 1706028645946}
-{"stream": "contact", "data": {"id": "Test Contact 2", "first_name": "Sample Name Two", "last_name": "Sample Lastname 2", "email": "[email protected]", "phone": "+13888433888", "enabled": true, "send_account_email": false, "send_billing_email": false, "object": "contact", "customer_id": "Test-Custome-1", "custom_fields": []}, "emitted_at": 1706028645251}
 {"stream": "order", "data": {"id": "1", "document_number": "lol1", "invoice_id": "24", "subscription_id": "6olOsTTHieWUY9", "customer_id": "cbdemo_tyler", "status": "queued", "payment_status": "paid", "order_type": "system_generated", "price_type": "tax_exclusive", "order_date": 1674036524, "shipping_date": 1674036524, "created_by": "Auto generated by system", "tax": 0, "amount_paid": 1000, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 1000, "rounding_adjustement": 0, "paid_on": 1674036524, "exchange_rate": 1.0, "created_at": 1674036525, "updated_at": 1674036525, "is_resent": false, "resource_version": 1674036525755, "deleted": false, "object": "order", "discount": 0, "sub_total": 1000, "order_line_items": [{"id": "o_li169lB6TTHiez02Fb4", "invoice_id": "24", "invoice_line_item_id": "li_6olOsTTHieX6YB", "unit_price": 1000, "amount": 1000, "fulfillment_quantity": 1, "fulfillment_amount": 1000, "tax_amount": 0, "amount_paid": 1000, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 1000, "is_shippable": true, "status": "queued", "object": "order_line_item", "entity_id": "Test-Plan-1-USD-Daily", "discount_amount": 0, "item_level_discount_amount": 0, "description": "Test Plan 1", "entity_type": "plan_item_price"}], "total": 1000, "currency_code": "USD", "base_currency_code": "USD", "is_gifted": false, "billing_address": {"first_name": "Tyler", "last_name": "Durden", "company": "Iselectrics", "validation_status": "not_validated", "object": "billing_address"}, "linked_credit_notes": [], "resent_orders": [], "custom_fields": []}, "emitted_at": 1703026216053}
 {"stream": "order", "data": {"id": "2", "document_number": "lol2", "invoice_id": "25", "subscription_id": "AzZTZgTTHixMHV3", "customer_id": "cbdemo_richard", "status": "queued", "payment_status": "paid", "order_type": "system_generated", "price_type": "tax_exclusive", "order_date": 1674036596, "shipping_date": 1674036596, "created_by": "Auto generated by system", "tax": 0, "amount_paid": 1000, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 1000, "rounding_adjustement": 0, "paid_on": 1674036596, "exchange_rate": 1.0, "created_at": 1674036599, "updated_at": 1674036684, "is_resent": false, "resource_version": 1674036684213, "deleted": false, "object": "order", "discount": 0, "sub_total": 1000, "order_line_items": [{"id": "o_li16CQyCTTHiy9912Tu", "invoice_id": "25", "invoice_line_item_id": "li_AzZTZgTTHixMhV5", "unit_price": 1000, "amount": 1000, "fulfillment_quantity": 1, "fulfillment_amount": 1000, "tax_amount": 0, "amount_paid": 1000, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 1000, "is_shippable": true, "status": "queued", "object": "order_line_item", "entity_id": "Test-Plan-1-USD-Daily", "discount_amount": 0, "item_level_discount_amount": 0, "description": "Test Plan 1", "entity_type": "plan_item_price"}], "total": 1000, "currency_code": "USD", "base_currency_code": "USD", "is_gifted": false, "shipping_address": {"first_name": "Sample Name 1", "last_name": "Sample Lastname 1", "email": "[email protected]", "company": "Semiconductors", "phone": "+1 382 846 3883", "line1": "Ms Ninette Franck", "line2": "4381", "city": "San Francisco", "state_code": "CA", "state": "California", "country": "US", "zip": "94114", "validation_status": "not_validated", "object": "shipping_address"}, "billing_address": {"first_name": "Richard", "last_name": "Hendricks", "company": "Zencorporation", "validation_status": "not_validated", "object": "billing_address"}, "linked_credit_notes": [], "resent_orders": [], "custom_fields": []}, "emitted_at": 1703026216060}
 {"stream": "order", "data": {"id": "3", "document_number": "lol3", "invoice_id": "26", "subscription_id": "AzZTZgTTHmX8Gc1", "customer_id": "cbdemo_simon", "status": "queued", "payment_status": "paid", "order_type": "system_generated", "price_type": "tax_exclusive", "order_date": 1674037448, "shipping_date": 1674037448, "created_by": "Auto generated by system", "tax": 0, "amount_paid": 700, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 700, "rounding_adjustement": 0, "paid_on": 1674037448, "exchange_rate": 1.0, "created_at": 1674037452, "updated_at": 1674037452, "is_resent": false, "resource_version": 1674037452271, "deleted": false, "object": "order", "discount": 300, "sub_total": 700, "order_line_items": [{"id": "o_liAzZZMnTTHmY0s1O7g", "invoice_id": "26", "invoice_line_item_id": "li_AzZTZgTTHmX93c3", "unit_price": 1000, "amount": 1000, "fulfillment_quantity": 1, "fulfillment_amount": 700, "tax_amount": 0, "amount_paid": 700, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 700, "is_shippable": true, "status": "queued", "object": "order_line_item", "entity_id": "Test-Plan-1-USD-Daily", "discount_amount": 300, "item_level_discount_amount": 300, "description": "Test Plan 1", "entity_type": "plan_item_price"}], "line_item_discounts": [{"object": "line_item_discount", "line_item_id": "li_AzZTZgTTHmX93c3", "discount_type": "item_level_coupon", "discount_amount": 300, "coupon_id": "cbdemo_launchoffer", "entity_id": "cbdemo_launchoffer"}], "total": 700, "currency_code": "USD", "base_currency_code": "USD", "is_gifted": false, "billing_address": {"first_name": "Simon", "last_name": "Masrani", "company": "Openlane Ltd", "validation_status": "not_validated", "object": "billing_address"}, "linked_credit_notes": [], "resent_orders": [], "custom_fields": []}, "emitted_at": 1703026216066}

diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json
@@ -2,7 +2,7 @@
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "subscription" }
     }
   },
@@ -16,42 +16,42 @@
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "coupon" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "customer" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "invoice" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "credit_note" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "order" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "item" }
     }
   },
@@ -65,91 +65,91 @@
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "hosted_page" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "item_price" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "payment_source" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "created_at": 2147483647 },
+      "stream_state": { "created_at": "2147483647" },
       "stream_descriptor": { "name": "promotional_credit" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "gift" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "unbilled_charge" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "quote" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "transaction" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "created_at": 2147483647 },
+      "stream_state": { "created_at": "2147483647" },
       "stream_descriptor": { "name": "comment" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "item_family" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "updated_at": 2147483647 },
+      "stream_state": { "updated_at": "2147483647" },
       "stream_descriptor": { "name": "differential_price" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "migrated_at": 2147483647 },
+      "stream_state": { "migrated_at": "2147483647" },
       "stream_descriptor": { "name": "site_migration_detail" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "occurred_at": 2147483647 },
+      "stream_state": { "occurred_at": "2147483647" },
       "stream_descriptor": { "name": "event" }
     }
   }

diff --git a/airbyte-integrations/connectors/source-chargebee/metadata.yaml b/airbyte-integrations/connectors/source-chargebee/metadata.yaml
@@ -10,7 +10,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 686473f1-76d9-4994-9cc7-9b13da46147c
-  dockerImageTag: 0.7.1
+  dockerImageTag: 0.7.2
   dockerRepository: airbyte/source-chargebee
   documentationUrl: https://docs.airbyte.com/integrations/sources/chargebee
   githubIssueLabel: source-chargebee