Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix all_fields_test along with some other clean up #198

Merged
merged 1 commit into from
Apr 18, 2024
Merged

Fix all_fields_test along with some other clean up #198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 19 additions & 9 deletions tests/test_all_fields.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,14 @@
import os
from collections import namedtuple
from datetime import datetime as dt
from dateutil.parser import parse
from pathlib import Path
from random import random
from time import sleep, perf_counter
from datetime import datetime as dt
from dateutil.parser import parse

from collections import namedtuple

from tap_tester import menagerie, runner, connections, LOGGER
from base import BaseTapTest
from utils import \
create_object, delete_object, list_all_object, stripe_obj_to_dict
from tap_tester import menagerie, runner, connections, LOGGER
from utils import create_object, delete_object, list_all_object, stripe_obj_to_dict


# BUG_12478 | https://jira.talendforge.org/browse/TDL-12478
Expand All @@ -24,22 +22,28 @@
'automatic_tax',
'cancellation_details',
'default_tax_rates',
'discounts',
'on_behalf_of',
'payment_settings',
'pending_update',
'trial_settings',
},
'products': {
'features',
'marketing_features',
},
'invoice_items': {
'price',
},
'payouts': {
'application_fee',
'application_fee_amount',
'reconciliation_status',
},
'charges': set(),
'subscription_items': set(),
'subscription_items': {
'discounts',
},
'plans': set(),
'invoice_line_items': {
'margins',
Expand Down Expand Up @@ -271,7 +275,9 @@
'coupons': {
'times_redeemed' # expect 0, get 1
},
'customers': set(),
'customers': {
'discount',
},
'subscriptions': set(),
'products': set(),
'invoice_items': set(),
Expand Down Expand Up @@ -599,6 +605,10 @@ def all_fields_test(self, streams_to_test):

elif actual_field_value and field in FICKLE_FIELDS[stream]:
self.assertIsInstance(actual_field_value, type(expected_field_value))
# if fickle field is a dict at least compare top level keys
if isinstance(actual_field_value, dict):
self.assertTrue(set(actual_field_value.keys()).issubset(
set(expected_field_value.keys())))

elif actual_field_value:
raise AssertionError(f"{base_err_msg} Unexpected field is being fickle.")
Expand Down
35 changes: 21 additions & 14 deletions tests/test_automatic_payout_transactions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
from utils import stripe_obj_to_dict, client, midnight
from datetime import datetime as dt
from datetime import timedelta, time
from tap_tester import runner, connections
from utils import stripe_obj_to_dict, client, midnight

from base import BaseTapTest
from tap_tester import runner, connections


def get_payouts():
"""
Return all the payouts (with pagination), to determine the automatic and non-automatic payouts
Return all the payouts (with pagination), to determine the automatic and non-automatic payouts
"""
# list of all data to return
four_days_ago = int(dt.combine(dt.today()-timedelta(days=4), time.min).timestamp())
data = []
# Api call of 1st page starting from 4 days ago as there is a lag from the Stripe side to reflect
# Api call of 1st page starting from 4 days ago as there is lag from the Stripe side to reflect
# the automatic payout transactions data
stripe_obj = client["payouts"].list(limit=100, created={"gte": four_days_ago})
dict_obj = stripe_obj_to_dict(stripe_obj)
Expand All @@ -20,11 +22,13 @@ def get_payouts():
# add data
data += dict_obj['data']
except KeyError:
raise Exception("No records for 'Payouts' were replicated, please run 'test_all_fields' before re-running.")
raise Exception("No records for 'Payouts' were replicated, please run 'test_all_fields' "
"before re-running.")

# loop over rest of the pages and collect data
while dict_obj.get("has_more"):
stripe_obj = client["payouts"].list(limit=100, created={"gte": four_days_ago}, starting_after=dict_obj.get('data')[-1].get('id'))
stripe_obj = client["payouts"].list(limit=100, created={"gte": four_days_ago},
starting_after=dict_obj.get('data')[-1].get('id'))
dict_obj = stripe_obj_to_dict(stripe_obj)
data += dict_obj['data']

Expand All @@ -33,7 +37,8 @@ def get_payouts():

class AutomaticPayoutTransactionTest(BaseTapTest):
"""
Test case to verify that we only collect payout_transactions for payouts containing "automatic" field as "True"
Test case to verify that we only collect payout_transactions for payouts containing
"automatic" field as "True"
Prerequisite:
Run 'test_all_fields' before running this test case.
"""
Expand All @@ -60,8 +65,8 @@ def setUpClass(cls):
cls.payouts_with_automatic_false.append(record.get("id"))

def test_run(self):
# Decreased the start_date for payout_transactions stream as there is a lag from the Stripe side to reflect
# the automatic payout transactions data
# Decreased the start_date for payout_transactions stream as there is a lag from the Stripe
# side to reflect the automatic payout transactions data
self.start_date = dt.strftime(dt.today() - timedelta(days=4), self.START_DATE_FORMAT)
conn_id = connections.ensure_connection(self, original_properties=False)

Expand Down Expand Up @@ -89,21 +94,23 @@ def test_run(self):
msg="Data isn't set up to be able to test full sync")

# get records
records = [message.get("data") for message in first_sync_records.get(stream).get("messages") if message["action"] == "upsert"]
records = [message.get("data") for message
in first_sync_records.get(stream).get("messages")
if message["action"] == "upsert"]

# collect payout ids for all the payout transaction records
payout_transaction_payout_ids = set()
for record in records:
payout_transaction_payout_ids.add(record.get("payout_id"))

# verify that data exists for payouts with "automatic" field as "True" and "False"
self.assertTrue(self.payouts_with_automatic_true is not None)
self.assertTrue(self.payouts_with_automatic_false is not None)
self.assertGreater(len(self.payouts_with_automatic_true), 0)
self.assertGreater(len(self.payouts_with_automatic_false), 0)

# loop over all the payout ids from the payout transactions to verify
# that we collected "payout transactions" of automatic payouts only
for id in payout_transaction_payout_ids:
# verify that we collect payout transaction record for payout containing "automatic": True
# verify payout transaction record collected for payout containing "automatic": True
self.assertTrue(id in self.payouts_with_automatic_true)
# verify that we do not collect payout transaction record for payout containing "automatic": False
# verify payout transaction rec NOT collected for payout with "automatic": False
self.assertTrue(id not in self.payouts_with_automatic_false)
10 changes: 6 additions & 4 deletions tests/test_create_object.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,14 @@ def test_run(self):
self.conn_id = conn_id

streams_to_create = {
"balance_transactions", # should be created implicity with a create in the payouts or charges streams
"balance_transactions", # should be created implicity with create in payouts or charges
"charges",
"coupons",
"customers",
"invoice_items",
"invoice_line_items", # this is created implicity by invoices, it just creates another invoice TODO get this outa here
# invoice_line_items are created implicity by invoices, this creates another invoice
# TODO update test to remove invoice_line_items from here
"invoice_line_items",
"invoices", # this will create an invoice_item
"payouts",
"plans",
Expand All @@ -41,7 +43,7 @@ def test_run(self):
}

missing_streams_to_create = {
"disputes", # can be created by simulating a dispute transaction with a specific card number
"disputes", # create by simulating a dispute transaction with a specific card number
# no way to create directly, see: https://stripe.com/docs/testing#disputes
"payout_transactions", # BUG_9703 | https://jira.talendforge.org/browse/TDL-9703
# depends on payouts and transactions
Expand Down Expand Up @@ -132,7 +134,7 @@ def test_run(self):
f"new_id: {new_objects[stream]['id']}")
self.assertTrue(new_objects[stream]['id'] in null_date_invoices)
if new_objects[stream]['id'] not in masking_invoices:
LOGGER.warn(f"########## Previous error scenario detected (un-masked failure) ##########")
LOGGER.warn(f"### Previous error scenario detected (un-masked failure) ###")
# TODO END DEBUG

# verify the new object is in the list of created objects
Expand Down