diff --git a/member_card/commands.py b/member_card/commands.py index 15beaf12..b527aa80 100755 --- a/member_card/commands.py +++ b/member_card/commands.py @@ -254,6 +254,14 @@ def minibc(): pass +@minibc.command("sync-subscriptions") +def minibc_sync_subscriptions(): + etl_results = worker.sync_minibc_subscriptions_etl( + message=dict(type="cli-sync-subscriptions"), + ) + logger.info(f"minibc_sync_subscriptions() => {etl_results=}") + + @minibc.command("find-missing-shipping") def minibc_cmd_find_missing_shipping(): minibc_client = Minibc(api_key=app.config["MINIBC_API_KEY"]) diff --git a/member_card/minibc.py b/member_card/minibc.py index b52058b8..5e733870 100644 --- a/member_card/minibc.py +++ b/member_card/minibc.py @@ -162,109 +162,48 @@ def get_profile_by_email(self, email): return self.get(path="profiles/", args=dict(filter=f"email,{email}")) -def insert_order_as_membership(order, skus): - from member_card.models import AnnualMembership - - membership_orders = [] - products = order.get("products", []) - subscription_line_items = [p for p in products if p["sku"] in skus] - ignored_line_items = [p for p in products if p["sku"] not in skus] - logger.debug(f"{ignored_line_items=}") - for subscription_line_item in subscription_line_items: - fulfilled_on = None - if fulfilled_on := order.get("fulfilledOn"): - fulfilled_on = parse(fulfilled_on).replace(tzinfo=timezone.utc) - - customer_email = order["customer"]["email"] - # logger.debug(f"{order=}") - - weird_dates_keys = [ - "created_time", - "last_modified", - "signup_date", - "next_payment_date", - ] - weird_dates = {} - for weird_dates_key in weird_dates_keys: - order[weird_dates_key] = order[weird_dates_key].strip("-") - if order[weird_dates_key] == "0": - weird_dates[weird_dates_key] = None - else: - try: - weird_dates[weird_dates_key] = parse( - order[weird_dates_key] - ).replace(tzinfo=timezone.utc) - except ParserError as err: - logger.warning( - f"Unable to parse {weird_dates_key} for {customer_email}: {err=}" - ) - weird_dates[weird_dates_key] = None - - created_on = weird_dates["signup_date"] - if weird_dates["next_payment_date"] is not None: - created_on = weird_dates["next_payment_date"] - timedelta(days=365) - - logger.debug(f"{weird_dates['next_payment_date']=} => {created_on=}") - membership_kwargs = dict( - order_id=f"minibc_{str(order['id'])}", - order_number=f"minibc_{order['order_id'] or order['id']}_{order['customer']['store_customer_id']}", - channel="minibc", - channel_name="minibc", - billing_address_first_name=order["customer"]["first_name"], - billing_address_last_name=order["customer"]["last_name"], - external_order_reference=order["customer"]["store_customer_id"], - created_on=created_on, - modified_on=weird_dates["last_modified"], - fulfilled_on=fulfilled_on, - customer_email=customer_email, - fulfillment_status=None, - test_mode=False, - line_item_id=subscription_line_item["order_product_id"], - sku=subscription_line_item["sku"], - variant_id=subscription_line_item["name"], - product_id=subscription_line_item["store_product_id"], - product_name=subscription_line_item["name"], - ) - membership = get_or_update( - session=db.session, - model=AnnualMembership, - filters=["order_id"], - kwargs=membership_kwargs, - ) - membership_orders.append(membership) - - membership_user = ensure_user( - email=membership.customer_email, - first_name=membership.billing_address_first_name, - last_name=membership.billing_address_last_name, - ) - membership_user_id = membership_user.id - if not membership.user_id: - logger.debug( - f"No user_id set for {membership=}! Setting to: {membership_user_id=}" - ) - setattr(membership, "user_id", membership_user_id) - return membership_orders - - -def parse_subscriptions(skus, subscriptions): +def parse_subscriptions(subscriptions): logger.info(f"{len(subscriptions)=} retrieved from Minibc...") # Insert oldest orders first (so our internal membership ID generally aligns with order IDs...) subscriptions.reverse() # Loop over all the raw order data and do the ETL bits - memberships = [] + subscription_objs = [] + from member_card.models import Subscription + for subscription in subscriptions: - membership_orders = insert_order_as_membership( - order=subscription, - skus=skus, + product_name = ",".join([p["name"] for p in subscription["products"]]) + shipping_address = " ".join(subscription["shipping_address"].values()) + subscription_kwargs = dict( + subscription_id=subscription["id"], + order_id=subscription["order_id"], + customer_id=subscription["customer"]["id"], + customer_first_name=subscription["customer"]["first_name"], + customer_last_name=subscription["customer"]["last_name"], + customer_email=subscription["customer"]["email"], + product_name=product_name, + status=subscription["status"], + shipping_address=shipping_address, + signup_date=parse_weird_dates(subscription["signup_date"]), + pause_date=parse_weird_dates(subscription["pause_date"]), + cancellation_date=parse_weird_dates(subscription["cancellation_date"]), + next_payment_date=parse_weird_dates(subscription["next_payment_date"]), + created_time=parse_weird_dates(subscription["created_time"]), + last_modified=parse_weird_dates(subscription["last_modified"]), ) - for membership_order in membership_orders: - db.session.add(membership_order) - db.session.commit() - memberships += membership_orders - return memberships + subscription_obj = get_or_update( + session=db.session, + model=Subscription, + filters=["subscription_id"], + kwargs=subscription_kwargs, + ) + subscription_objs.append(subscription_obj) + + for subscription_obj in subscription_objs: + db.session.add(subscription_obj) + db.session.commit() + return subscription_objs def find_missing_shipping(minibc_client: Minibc, skus): @@ -323,21 +262,33 @@ def find_missing_shipping(minibc_client: Minibc, skus): return missing_shipping_subs -def minibc_orders_etl(minibc_client: Minibc, skus, load_all): - from member_card import models +def parse_weird_dates(date_str): + date_str = date_str.strip("-") + if date_str == "0": + return None - # etl_start_time = datetime.now(tz=ZoneInfo("UTC")) + try: + return parse(date_str).replace(tzinfo=timezone.utc) + except ParserError as err: + logger.warning(f"Unable to parse {date_str}: {err=}") + return None - membership_table_name = models.AnnualMembership.__tablename__ + +def minibc_subscriptions_etl(minibc_client: Minibc, skus, load_all=False): + from member_card import models + + subscriptions_table_name = models.Subscription.__tablename__ if load_all: start_page_num = 1 max_pages = 1000 else: - start_page_num = table_metadata.get_last_run_start_page(membership_table_name) + start_page_num = table_metadata.get_last_run_start_page( + subscriptions_table_name + ) max_pages = 20 - memberships = list() + subscription_objs = list() last_page_num = start_page_num end_page_num = start_page_num + max_pages + 1 @@ -360,19 +311,19 @@ def minibc_orders_etl(minibc_client: Minibc, skus, load_all): break last_page_num = page_num - memberships += parse_subscriptions(skus, subscriptions) + subscription_objs += parse_subscriptions(subscriptions) logger.debug(f"after {page_num=} sleeping for 1 second...") sleep(1) if not load_all: logger.debug( - f"Setting start_page_num metadata on {membership_table_name=} to {last_page_num=}" + f"Setting start_page_num metadata on {subscriptions_table_name=} to {last_page_num=}" ) table_metadata.set_last_run_start_page( - membership_table_name, max(1, last_page_num - 1) + subscriptions_table_name, max(1, last_page_num - 1) ) - return memberships + return subscription_objs def load_single_subscription(minibc_client: Minibc, skus, order_id): diff --git a/member_card/worker.py b/member_card/worker.py index 4dcd81c4..3871f576 100644 --- a/member_card/worker.py +++ b/member_card/worker.py @@ -4,6 +4,7 @@ from flask import Blueprint, current_app, request +from member_card import minibc from member_card import bigcommerce, slack from member_card.db import db from member_card.image import ensure_uploaded_card_image @@ -224,6 +225,26 @@ def sync_customers_etl(message): ) +def sync_minibc_subscriptions_etl(message): + log_extra = dict(pubsub_message=message) + logger.debug( + f"sync_minibc_subscriptions_etl(): Processing message: {message}", + extra=log_extra, + ) + + minibc_client = minibc.Minibc(api_key=current_app.config["MINIBC_API_KEY"]) + membership_skus = current_app.config["BIGCOMMERCE_MEMBERSHIP_SKUS"] + + etl_result = minibc.minibc_subscriptions_etl( + minibc_client=minibc_client, + skus=membership_skus, + ) + logger.debug( + f"sync_minibc_subscriptions_etl(): {etl_result=}", + extra=log_extra, + ) + + @worker_bp.route("/pubsub", methods=["POST"]) def pubsub_ingress(): try: @@ -234,6 +255,7 @@ def pubsub_ingress(): MESSAGE_TYPE_HANDLERS = { "email_distribution_request": process_email_distribution_request, "sync_subscriptions_etl": sync_subscriptions_etl, + "sync_minibc_subscriptions_etl": sync_minibc_subscriptions_etl, "sync_customers_etl": sync_customers_etl, "sync_squarespace_order": sync_squarespace_order, "sync_bigcommerce_order": sync_bigcommerce_order,