diff --git a/commerce_coordinator/apps/commercetools/catalog_info/edx_utils.py b/commerce_coordinator/apps/commercetools/catalog_info/edx_utils.py index 85378325..f43c2485 100644 --- a/commerce_coordinator/apps/commercetools/catalog_info/edx_utils.py +++ b/commerce_coordinator/apps/commercetools/catalog_info/edx_utils.py @@ -75,6 +75,11 @@ def get_edx_payment_info(order: CTOrder): return None, None +def get_edx_payment_service_provider(order: CTOrder) -> Union[str, None]: + for pr in order.payment_info.payments: + return pr.obj.payment_method_info.payment_interface + + def get_edx_order_workflow_state_key(order: CTOrder) -> Optional[str]: order_workflow_state = None if order.state and order.state.obj: # it should never be that we have one and not the other. # pragma no cover diff --git a/commerce_coordinator/apps/commercetools/clients.py b/commerce_coordinator/apps/commercetools/clients.py index fad2cd67..7f828f57 100644 --- a/commerce_coordinator/apps/commercetools/clients.py +++ b/commerce_coordinator/apps/commercetools/clients.py @@ -6,10 +6,9 @@ import decimal import logging from types import SimpleNamespace -from typing import Generic, List, Optional, Tuple, TypeVar, Union +from typing import Generic, List, Optional, Tuple, TypedDict, TypeVar, Union import requests -import stripe from commercetools import Client as CTClient from commercetools import CommercetoolsError from commercetools.platform.models import Customer as CTCustomer @@ -23,7 +22,7 @@ OrderAddReturnInfoAction, OrderSetReturnItemCustomTypeAction, OrderSetReturnPaymentStateAction, - OrderTransitionLineItemStateAction + OrderTransitionLineItemStateAction, ) from commercetools.platform.models import Payment as CTPayment from commercetools.platform.models import PaymentAddTransactionAction, PaymentSetTransactionCustomTypeAction @@ -34,7 +33,7 @@ ReturnShipmentState, StateResourceIdentifier, TransactionDraft, - TransactionType + TransactionType, ) from commercetools.platform.models import Type as CTType from commercetools.platform.models import TypeDraft as CTTypeDraft @@ -43,12 +42,17 @@ from django.conf import settings from openedx_filters.exceptions import OpenEdxFilterException -from commerce_coordinator.apps.commercetools.catalog_info.constants import DEFAULT_ORDER_EXPANSION, EdXFieldNames +from commerce_coordinator.apps.commercetools.catalog_info.constants import ( + DEFAULT_ORDER_EXPANSION, + EDX_PAYPAL_PAYMENT_INTERFACE_NAME, + EDX_STRIPE_PAYMENT_INTERFACE_NAME, + EdXFieldNames, +) from commerce_coordinator.apps.commercetools.catalog_info.foundational_types import TwoUCustomTypes from commerce_coordinator.apps.commercetools.utils import ( find_refund_transaction, handle_commercetools_error, - translate_stripe_refund_status_to_transaction_status + translate_refund_status_to_transaction_status, ) from commerce_coordinator.apps.core.constants import ORDER_HISTORY_PER_SYSTEM_REQ_LIMIT @@ -60,7 +64,8 @@ class PaginatedResult(Generic[T]): - """ Planned paginated response wrapper """ + """Planned paginated response wrapper""" + results: List[T] total: int offset: int @@ -84,8 +89,21 @@ def rebuild(self, results: List[T]): return PaginatedResult(results, total=self.total, offset=self.offset) +class Refund(TypedDict): + """ + Refund object definition + """ + + id: str + amount: Union[str, int] + currency: str + created: Union[str, int] + status: str + + class CommercetoolsAPIClient: - """ Commercetools API Client """ + """Commercetools API Client""" + base_client = None def __init__(self): @@ -101,10 +119,10 @@ def __init__(self): self.base_client = CTClient( client_id=config["clientId"], client_secret=config["clientSecret"], - scope=config["scopes"].split(' '), + scope=config["scopes"].split(" "), url=config["apiUrl"], token_url=config["authUrl"], - project_key=config["projectKey"] + project_key=config["projectKey"], ) def ensure_custom_type_exists(self, type_def: CTTypeDraft) -> Optional[CTType]: @@ -153,21 +171,29 @@ def tag_customer_with_lms_user_info(self, customer: CTCustomer, lms_user_id: int # are LOST. if customer.custom and not customer.custom.type.id == type_object.id: - raise ValueError("User already has a custom type, and its not the one were expecting, Refusing to update. " - "(Updating will eradicate the values from the other type, as an object may only have one " - "Custom Type)") + raise ValueError( + "User already has a custom type, and its not the one were expecting, Refusing to update. " + "(Updating will eradicate the values from the other type, as an object may only have one " + "Custom Type)" + ) - ret = self.base_client.customers.update_by_id(customer.id, customer.version, actions=[ - CTCustomerSetCustomTypeAction( - type=CTTypeResourceIdentifier( - key=TwoUCustomTypes.CUSTOMER_TYPE_DRAFT.key, + ret = self.base_client.customers.update_by_id( + customer.id, + customer.version, + actions=[ + CTCustomerSetCustomTypeAction( + type=CTTypeResourceIdentifier( + key=TwoUCustomTypes.CUSTOMER_TYPE_DRAFT.key, + ), + fields=CTFieldContainer( + { + EdXFieldNames.LMS_USER_ID: f"{lms_user_id}", + EdXFieldNames.LMS_USER_NAME: lms_user_name, + } + ), ), - fields=CTFieldContainer({ - EdXFieldNames.LMS_USER_ID: f"{lms_user_id}", - EdXFieldNames.LMS_USER_NAME: lms_user_name - }) - ), - ]) + ], + ) return ret @@ -189,9 +215,9 @@ def get_customer_by_lms_user_id(self, lms_user_id: int) -> Optional[CTCustomer]: start_time = datetime.datetime.now() results = self.base_client.customers.query( - where=f'custom(fields({edx_lms_user_id_key}=:id))', + where=f"custom(fields({edx_lms_user_id_key}=:id))", limit=2, - predicate_var={'id': f"{lms_user_id}"} + predicate_var={"id": f"{lms_user_id}"}, ) duration = (datetime.datetime.now() - start_time).total_seconds() logger.info(f"[Performance Check] - customerId query took {duration} seconds") @@ -199,10 +225,13 @@ def get_customer_by_lms_user_id(self, lms_user_id: int) -> Optional[CTCustomer]: if results.count > 1: # We are unable due to CT Limitations to enforce unique LMS ID values on Customers on the catalog side, so # let's do a backhanded check by trying to pull 2 users and erroring if we find a discrepancy. - logger.info(f"[CommercetoolsAPIClient] - More than one customer found with LMS " - f"user id: {lms_user_id}, raising error") - raise ValueError("More than one user was returned from the catalog with this edX LMS User ID, these must " - "be unique.") + logger.info( + f"[CommercetoolsAPIClient] - More than one customer found with LMS " + f"user id: {lms_user_id}, raising error" + ) + raise ValueError( + "More than one user was returned from the catalog with this edX LMS User ID, these must be unique." + ) if results.count == 0: logger.info(f"[CommercetoolsAPIClient] - No customer found with LMS user id: {lms_user_id}") @@ -211,8 +240,7 @@ def get_customer_by_lms_user_id(self, lms_user_id: int) -> Optional[CTCustomer]: logger.info(f"[CommercetoolsAPIClient] - Customer found with LMS user id: {lms_user_id}") return results.results[0] - def get_order_by_id(self, order_id: str, expand: ExpandList = DEFAULT_ORDER_EXPANSION) \ - -> CTOrder: + def get_order_by_id(self, order_id: str, expand: ExpandList = DEFAULT_ORDER_EXPANSION) -> CTOrder: """ Fetch an order by the Order ID (UUID) @@ -225,8 +253,7 @@ def get_order_by_id(self, order_id: str, expand: ExpandList = DEFAULT_ORDER_EXPA logger.info(f"[CommercetoolsAPIClient] - Attempting to find order with id: {order_id}") return self.base_client.orders.get_by_id(order_id, expand=list(expand)) - def get_order_by_number(self, order_number: str, expand: ExpandList = DEFAULT_ORDER_EXPANSION) \ - -> CTOrder: + def get_order_by_number(self, order_number: str, expand: ExpandList = DEFAULT_ORDER_EXPANSION) -> CTOrder: """ Fetch an order by the Order Number (Human readable order number) @@ -239,11 +266,14 @@ def get_order_by_number(self, order_number: str, expand: ExpandList = DEFAULT_OR logger.info(f"[CommercetoolsAPIClient] - Attempting to find order with number {order_number}") return self.base_client.orders.get_by_order_number(order_number, expand=list(expand)) - def get_orders(self, customer_id: str, offset=0, - limit=ORDER_HISTORY_PER_SYSTEM_REQ_LIMIT, - expand: ExpandList = DEFAULT_ORDER_EXPANSION, - order_state="Complete") -> PaginatedResult[CTOrder]: - + def get_orders( + self, + customer_id: str, + offset=0, + limit=ORDER_HISTORY_PER_SYSTEM_REQ_LIMIT, + expand: ExpandList = DEFAULT_ORDER_EXPANSION, + order_state="Complete", + ) -> PaginatedResult[CTOrder]: """ Call commercetools API overview endpoint for data about historical orders. @@ -259,27 +289,34 @@ def get_orders(self, customer_id: str, offset=0, See sample response in tests.py """ - logger.info(f"[CommercetoolsAPIClient] - Attempting to find all completed orders for " - f"customer with ID {customer_id}") - order_where_clause = f"orderState=\"{order_state}\"" + logger.info( + f"[CommercetoolsAPIClient] - Attempting to find all completed orders for " f"customer with ID {customer_id}" + ) + order_where_clause = f'orderState="{order_state}"' start_time = datetime.datetime.now() values = self.base_client.orders.query( where=["customerId=:cid", order_where_clause], - predicate_var={'cid': customer_id}, + predicate_var={"cid": customer_id}, sort=["completedAt desc", "lastModifiedAt desc"], limit=limit, offset=offset, - expand=list(expand) + expand=list(expand), ) duration = (datetime.datetime.now() - start_time).total_seconds() logger.info(f"[Performance Check] get_orders call took {duration} seconds") return PaginatedResult(values.results, values.total, values.offset) - def get_orders_for_customer(self, edx_lms_user_id: int, offset=0, limit=ORDER_HISTORY_PER_SYSTEM_REQ_LIMIT, - customer_id=None, email=None, - username=None) -> (PaginatedResult[CTOrder], CTCustomer): + def get_orders_for_customer( + self, + edx_lms_user_id: int, + offset=0, + limit=ORDER_HISTORY_PER_SYSTEM_REQ_LIMIT, + customer_id=None, + email=None, + username=None, + ) -> (PaginatedResult[CTOrder], CTCustomer): """ Args: @@ -291,7 +328,7 @@ def get_orders_for_customer(self, edx_lms_user_id: int, offset=0, limit=ORDER_HI customer = self.get_customer_by_lms_user_id(edx_lms_user_id) if customer is None: # pragma: no cover - raise ValueError(f'Unable to locate customer with ID #{edx_lms_user_id}') + raise ValueError(f"Unable to locate customer with ID #{edx_lms_user_id}") customer_id = customer.id else: @@ -301,11 +338,7 @@ def get_orders_for_customer(self, edx_lms_user_id: int, offset=0, limit=ORDER_HI customer = SimpleNamespace( id=customer_id, email=email, - custom=SimpleNamespace( - fields={ - EdXFieldNames.LMS_USER_NAME: username - } - ) + custom=SimpleNamespace(fields={EdXFieldNames.LMS_USER_NAME: username}), ) orders = self.get_orders(customer_id, offset, limit) @@ -328,36 +361,39 @@ def get_payment_by_key(self, payment_key: str) -> CTPayment: logger.info(f"[CommercetoolsAPIClient] - Attempting to find payment with key {payment_key}") return self.base_client.payments.get_by_key(payment_key) + def get_payment_by_transaction_interaction_id(self, interaction_id: str) -> CTPayment: + """ + Fetch a payment by the transaction interaction ID + """ + logger.info(f"[CommercetoolsAPIClient] - Attempting to find payment with interaction ID {interaction_id}") + return self.base_client.payments.query(where=f'transactions(interactionId="{interaction_id}")').results[0] + def get_product_variant_by_course_run(self, cr_id: str) -> Optional[CTProductVariant]: """ Args: cr_id: variant course run key """ start_time = datetime.datetime.now() - results = self.base_client.product_projections.search(False, filter=f"variants.sku:\"{cr_id}\"").results + results = self.base_client.product_projections.search(False, filter=f'variants.sku:"{cr_id}"').results duration = (datetime.datetime.now() - start_time).total_seconds() - logger.info( - f"[Performance Check] get_product_variant_by_course_run took {duration} seconds") + logger.info(f"[Performance Check] get_product_variant_by_course_run took {duration} seconds") if len(results) < 1: # pragma no cover return None # Make 2D List of all variants from all results, and then flatten - all_variants = [listitem for sublist in - list( - map( - lambda selection: [selection.master_variant, *selection.variants], - results - ) - ) - for listitem in sublist] - - matching_variant_list = list( - filter( - lambda v: v.sku == cr_id, - all_variants + all_variants = [ + listitem + for sublist in list( + map( + lambda selection: [selection.master_variant, *selection.variants], + results, + ) ) - ) + for listitem in sublist + ] + + matching_variant_list = list(filter(lambda v: v.sku == cr_id, all_variants)) if len(matching_variant_list) < 1: # pragma no cover return None @@ -376,8 +412,9 @@ def create_return_for_order(self, order_id: str, order_version: int, order_line_ """ try: - return_item_draft_comment = f'Creating return item for order {order_id} with ' \ - f'order line item ID {order_line_item_id}' + return_item_draft_comment = ( + f"Creating return item for order {order_id} with " f"order line item ID {order_line_item_id}" + ) logger.info(f"[CommercetoolsAPIClient] - {return_item_draft_comment}") @@ -388,25 +425,24 @@ def create_return_for_order(self, order_id: str, order_version: int, order_line_ shipment_state=ReturnShipmentState.RETURNED, ) - add_return_info_action = OrderAddReturnInfoAction( - items=[return_item_draft] - ) + add_return_info_action = OrderAddReturnInfoAction(items=[return_item_draft]) returned_order = self.base_client.orders.update_by_id( - id=order_id, - version=order_version, - actions=[add_return_info_action] + id=order_id, version=order_version, actions=[add_return_info_action] ) return returned_order except CommercetoolsError as err: handle_commercetools_error(err, f"Unable to create return for order {order_id}") raise err - def update_return_payment_state_after_successful_refund(self, order_id: str, - order_version: int, - return_line_item_return_id: str, - payment_intent_id: str, - amount_in_cents: decimal) -> Union[CTOrder, None]: + def update_return_payment_state_after_successful_refund( + self, + order_id: str, + order_version: int, + return_line_item_return_id: str, + payment_intent_id: str, + amount_in_cents: decimal, + ) -> Union[CTOrder, None]: """ Update paymentState on the LineItemReturnItem attached to the order. Updated by the Order ID (UUID) @@ -420,45 +456,43 @@ def update_return_payment_state_after_successful_refund(self, order_id: str, Raises Exception: Error if update was unsuccessful. """ try: - logger.info(f"[CommercetoolsAPIClient] - Updating payment state for return " - f"with id {return_line_item_return_id} to '{ReturnPaymentState.REFUNDED}'.") + logger.info( + f"[CommercetoolsAPIClient] - Updating payment state for return " + f"with id {return_line_item_return_id} to '{ReturnPaymentState.REFUNDED}'." + ) return_payment_state_action = OrderSetReturnPaymentStateAction( return_item_id=return_line_item_return_id, - payment_state=ReturnPaymentState.REFUNDED + payment_state=ReturnPaymentState.REFUNDED, ) if not payment_intent_id: - payment_intent_id = '' - logger.info(f'Creating return for order - payment_intent_id: {payment_intent_id}') + payment_intent_id = "" + logger.info(f"Creating return for order - payment_intent_id: {payment_intent_id}") payment = self.get_payment_by_key(payment_intent_id) logger.info(f"Payment found: {payment}") transaction_id = find_refund_transaction(payment, amount_in_cents) update_transaction_id_action = OrderSetReturnItemCustomTypeAction( return_item_id=return_line_item_return_id, type=CTTypeResourceIdentifier( - key='returnItemCustomType', + key="returnItemCustomType", ), - fields=CTFieldContainer({ - 'transactionId': transaction_id - }) + fields=CTFieldContainer({"transactionId": transaction_id}), ) return_transaction_return_item_action = PaymentSetTransactionCustomTypeAction( transaction_id=transaction_id, - type=CTTypeResourceIdentifier(key='transactionCustomType'), - fields=CTFieldContainer({ - 'returnItemId': return_line_item_return_id - }) + type=CTTypeResourceIdentifier(key="transactionCustomType"), + fields=CTFieldContainer({"returnItemId": return_line_item_return_id}), ) logger.info(f"Update return payment state after successful refund - payment_intent_id: {payment_intent_id}") updated_order = self.base_client.orders.update_by_id( id=order_id, version=order_version, - actions=[return_payment_state_action, update_transaction_id_action] + actions=[return_payment_state_action, update_transaction_id_action], ) self.base_client.payments.update_by_id( id=payment.id, version=payment.version, - actions=[return_transaction_return_item_action] + actions=[return_transaction_return_item_action], ) logger.info("Updated transaction with return item id") return updated_order @@ -466,56 +500,77 @@ def update_return_payment_state_after_successful_refund(self, order_id: str, handle_commercetools_error(err, f"Unable to update ReturnPaymentState of order {order_id}") raise OpenEdxFilterException(str(err)) from err + def _preprocess_refund_object(self, refund: Refund, psp: str) -> Refund: + """ + Pre process refund object based on PSP + """ + if psp == EDX_PAYPAL_PAYMENT_INTERFACE_NAME: + # Paypal sends amount in dollars and CT expects it in cents + refund["amount"] = float(refund["amount"]) * 100 + refund["created"] = datetime.datetime.fromisoformat(refund["created"]) + else: + refund["created"] = datetime.datetime.utcfromtimestamp(refund["created"]) + + refund["status"] = translate_refund_status_to_transaction_status(refund["status"]) + refund["currency"] = refund["currency"].upper() + return refund + def create_return_payment_transaction( - self, payment_id: str, - payment_version: int, - stripe_refund: stripe.Refund) -> CTPayment: + self, payment_id: str, payment_version: int, refund: Refund, psp=EDX_STRIPE_PAYMENT_INTERFACE_NAME + ) -> CTPayment: """ Create Commercetools payment transaction for refund Args: payment_id (str): Payment ID (UUID) payment_version (int): Current version of payment - stripe_refund (stripe.Refund): Stripe's refund object + refund (Refund): Refund object Returns (CTPayment): Updated payment object or Raises Exception: Error if creation was unsuccessful. """ try: - logger.info(f"[CommercetoolsAPIClient] - Creating refund transaction for payment with ID {payment_id} " - f"following successful Stripe refund {stripe_refund.id}") + logger.info( + f"[CommercetoolsAPIClient] - Creating refund transaction for payment with ID {payment_id} " + f"following successful refund {refund['id']} in PSP: {psp}" + ) + refund = self._preprocess_refund_object(refund, psp) amount_as_money = CTMoney( - cent_amount=stripe_refund.amount, - currency_code=stripe_refund.currency.upper() + cent_amount=int(refund["amount"]), + currency_code=refund["currency"], ) transaction_draft = TransactionDraft( type=TransactionType.REFUND, amount=amount_as_money, - timestamp=datetime.datetime.utcfromtimestamp(stripe_refund.created), - state=translate_stripe_refund_status_to_transaction_status(stripe_refund.status), - interaction_id=stripe_refund.id + timestamp=refund["created"], + state=refund["status"], + interaction_id=refund["id"], ) - add_transaction_action = PaymentAddTransactionAction( - transaction=transaction_draft - ) + add_transaction_action = PaymentAddTransactionAction(transaction=transaction_draft) returned_payment = self.base_client.payments.update_by_id( - id=payment_id, - version=payment_version, - actions=[add_transaction_action] + id=payment_id, version=payment_version, actions=[add_transaction_action] ) return returned_payment except CommercetoolsError as err: - context = f"Unable to create refund payment transaction for "\ - f"payment {payment_id} and stripe refund {stripe_refund.id}" + context = ( + f"Unable to create refund payment transaction for payment {payment_id}, refund {refund['id']} " + f"with PSP: {psp}" + ) handle_commercetools_error(err, context) raise err - def update_line_item_transition_state_on_fulfillment(self, order_id: str, order_version: int, - line_item_id: str, item_quantity: int, - from_state_id: str, new_state_key: str) -> CTOrder: + def update_line_item_transition_state_on_fulfillment( + self, + order_id: str, + order_version: int, + line_item_id: str, + item_quantity: int, + from_state_id: str, + new_state_key: str, + ) -> CTOrder: """ Update Commercetools order line item state Args: @@ -532,8 +587,10 @@ def update_line_item_transition_state_on_fulfillment(self, order_id: str, order_ from_state_key = self.get_state_by_id(from_state_id).key - logger.info(f"[CommercetoolsAPIClient] - Transitioning line item state for order with ID {order_id}" - f"from {from_state_key} to {new_state_key}") + logger.info( + f"[CommercetoolsAPIClient] - Transitioning line item state for order with ID {order_id} " + f"from {from_state_key} to {new_state_key}" + ) try: if new_state_key != from_state_key: @@ -541,7 +598,7 @@ def update_line_item_transition_state_on_fulfillment(self, order_id: str, order_ line_item_id=line_item_id, quantity=item_quantity, from_state=StateResourceIdentifier(key=from_state_key), - to_state=StateResourceIdentifier(key=new_state_key) + to_state=StateResourceIdentifier(key=new_state_key), ) updated_fulfillment_line_item_order = self.base_client.orders.update_by_id( @@ -552,17 +609,25 @@ def update_line_item_transition_state_on_fulfillment(self, order_id: str, order_ return updated_fulfillment_line_item_order else: - logger.info(f"The line item {line_item_id} already has the correct state {new_state_key}. " - "Not attempting to transition LineItemState") + logger.info( + f"The line item {line_item_id} already has the correct state {new_state_key}. " + "Not attempting to transition LineItemState" + ) return self.get_order_by_id(order_id) except CommercetoolsError as err: # Logs & ignores version conflict errors due to duplicate Commercetools messages handle_commercetools_error(err, f"Unable to update LineItemState of order {order_id}", True) return None - def retire_customer_anonymize_fields(self, customer_id: str, customer_version: int, - retired_first_name: str, retired_last_name: str, - retired_email: str, retired_lms_username: str) -> CTCustomer: + def retire_customer_anonymize_fields( + self, + customer_id: str, + customer_version: int, + retired_first_name: str, + retired_last_name: str, + retired_email: str, + retired_lms_username: str, + ) -> CTCustomer: """ Update Commercetools customer with anonymized fields Args: @@ -577,39 +642,34 @@ def retire_customer_anonymize_fields(self, customer_id: str, customer_version: i """ actions = [] - update_retired_first_name_action = CustomerSetFirstNameAction( - first_name=retired_first_name - ) + update_retired_first_name_action = CustomerSetFirstNameAction(first_name=retired_first_name) - update_retired_last_name_action = CustomerSetLastNameAction( - last_name=retired_last_name - ) + update_retired_last_name_action = CustomerSetLastNameAction(last_name=retired_last_name) - update_retired_email_action = CustomerChangeEmailAction( - email=retired_email - ) + update_retired_email_action = CustomerChangeEmailAction(email=retired_email) update_retired_lms_username_action = CustomerSetCustomFieldAction( - name="edx-lms_user_name", - value=retired_lms_username + name="edx-lms_user_name", value=retired_lms_username ) - actions.extend([ - update_retired_first_name_action, - update_retired_last_name_action, - update_retired_email_action, - update_retired_lms_username_action - ]) + actions.extend( + [ + update_retired_first_name_action, + update_retired_last_name_action, + update_retired_email_action, + update_retired_lms_username_action, + ] + ) try: retired_customer = self.base_client.customers.update_by_id( - id=customer_id, - version=customer_version, - actions=actions + id=customer_id, version=customer_version, actions=actions ) return retired_customer except CommercetoolsError as err: - logger.error(f"[CommercetoolsError] Unable to anonymize customer fields for customer " - f"with ID: {customer_id}, after LMS retirement with " - f"error correlation id {err.correlation_id} and error/s: {err.errors}") + logger.error( + f"[CommercetoolsError] Unable to anonymize customer fields for customer " + f"with ID: {customer_id}, after LMS retirement with " + f"error correlation id {err.correlation_id} and error/s: {err.errors}" + ) raise err diff --git a/commerce_coordinator/apps/commercetools/pipeline.py b/commerce_coordinator/apps/commercetools/pipeline.py index 5a93be28..da5b50c3 100644 --- a/commerce_coordinator/apps/commercetools/pipeline.py +++ b/commerce_coordinator/apps/commercetools/pipeline.py @@ -340,7 +340,7 @@ def run_filter( updated_payment = ct_api_client.create_return_payment_transaction( payment_id=payment_on_order.id, payment_version=payment_on_order.version, - stripe_refund=refund_response + refund=refund_response ) return { diff --git a/commerce_coordinator/apps/commercetools/signals.py b/commerce_coordinator/apps/commercetools/signals.py index 73beedd8..e79945a9 100644 --- a/commerce_coordinator/apps/commercetools/signals.py +++ b/commerce_coordinator/apps/commercetools/signals.py @@ -1,10 +1,12 @@ """ Commercetools signals and receivers. """ + import logging from commerce_coordinator.apps.commercetools.catalog_info.constants import TwoUKeys from commerce_coordinator.apps.commercetools.tasks import ( + refund_from_paypal_task, refund_from_stripe_task, update_line_item_state_on_fulfillment_completion ) @@ -21,7 +23,7 @@ def fulfill_order_completed_send_line_item_state(**kwargs): Update the line item state of the order placed in Commercetools based on LMS enrollment """ - is_fulfilled = kwargs['is_fulfilled'] + is_fulfilled = kwargs["is_fulfilled"] if is_fulfilled: to_state_key = TwoUKeys.SUCCESS_FULFILMENT_STATE @@ -29,12 +31,12 @@ def fulfill_order_completed_send_line_item_state(**kwargs): to_state_key = TwoUKeys.FAILURE_FULFILMENT_STATE result = update_line_item_state_on_fulfillment_completion( - order_id=kwargs['order_id'], - order_version=kwargs['order_version'], - line_item_id=kwargs['line_item_id'], - item_quantity=kwargs['item_quantity'], - from_state_id=kwargs['line_item_state_id'], - to_state_key=to_state_key + order_id=kwargs["order_id"], + order_version=kwargs["order_version"], + line_item_id=kwargs["line_item_id"], + item_quantity=kwargs["item_quantity"], + from_state_id=kwargs["line_item_state_id"], + to_state_key=to_state_key, ) return result @@ -46,7 +48,18 @@ def refund_from_stripe(**kwargs): Create a refund transaction in Commercetools based on a refund created from the Stripe dashboard """ async_result = refund_from_stripe_task.delay( - payment_intent_id=kwargs['payment_intent_id'], - stripe_refund=kwargs['stripe_refund'], + payment_intent_id=kwargs["payment_intent_id"], + stripe_refund=kwargs["stripe_refund"], + ) + return async_result.id + + +@log_receiver(logger) +def refund_from_paypal(**kwargs): + """ + Create a refund transaction in Commercetools based on a refund created from the PayPal dashboard + """ + async_result = refund_from_paypal_task.delay( + paypal_capture_id=kwargs["paypal_capture_id"], refund=kwargs["refund"] ) return async_result.id diff --git a/commerce_coordinator/apps/commercetools/tasks.py b/commerce_coordinator/apps/commercetools/tasks.py index dab53759..d61eb8da 100644 --- a/commerce_coordinator/apps/commercetools/tasks.py +++ b/commerce_coordinator/apps/commercetools/tasks.py @@ -9,6 +9,8 @@ from commercetools import CommercetoolsError from django.conf import settings +from commerce_coordinator.apps.commercetools.catalog_info.constants import EDX_PAYPAL_PAYMENT_INTERFACE_NAME + from .clients import CommercetoolsAPIClient from .utils import has_full_refund_transaction @@ -54,8 +56,6 @@ def refund_from_stripe_task( Celery task for a refund registered in Stripe dashboard and need to create refund payment transaction record via Commercetools API. """ - # Celery serializes stripe_refund to a dict, so we need to explictly convert it back to a Refund object - stripe_refund = stripe.Refund.construct_from(stripe_refund, stripe.api_key) client = CommercetoolsAPIClient() try: payment = client.get_payment_by_key(payment_intent_id) @@ -66,11 +66,37 @@ def refund_from_stripe_task( updated_payment = client.create_return_payment_transaction( payment_id=payment.id, payment_version=payment.version, - stripe_refund=stripe_refund + refund=stripe_refund ) return updated_payment except CommercetoolsError as err: logger.error(f"Unable to create refund transaction for payment [ {payment.id} ] " - f"on Stripe refund {stripe_refund.id} " + f"on Stripe refund {stripe_refund['id']} " + f"with error {err.errors} and correlation id {err.correlation_id}") + return None + + +@shared_task(autoretry_for=(CommercetoolsError,), retry_kwargs={'max_retries': 5, 'countdown': 3}) +def refund_from_paypal_task( + paypal_capture_id, + refund +): + """ + Celery task for a refund registered in PayPal dashboard and need to create + refund payment transaction record via Commercetools API. + """ + client = CommercetoolsAPIClient() + try: + payment = client.get_payment_by_transaction_interaction_id(paypal_capture_id) + updated_payment = client.create_return_payment_transaction( + payment_id=payment.id, + payment_version=payment.version, + refund=refund, + psp=EDX_PAYPAL_PAYMENT_INTERFACE_NAME, + ) + return updated_payment + except CommercetoolsError as err: + logger.error(f"Unable to create refund transaction for payment {payment.key} " + f"on PayPal refund {refund.id} " f"with error {err.errors} and correlation id {err.correlation_id}") return None diff --git a/commerce_coordinator/apps/commercetools/tests/test_clients.py b/commerce_coordinator/apps/commercetools/tests/test_clients.py index ff17e316..a3b6cfde 100644 --- a/commerce_coordinator/apps/commercetools/tests/test_clients.py +++ b/commerce_coordinator/apps/commercetools/tests/test_clients.py @@ -598,14 +598,14 @@ def test_create_refund_transaction_exception(self): self.client_set.client.create_return_payment_transaction( payment_id="mock_payment_id", payment_version=1, - stripe_refund=mock_stripe_refund + refund=mock_stripe_refund ) exception = cm.exception expected_message = ( f"[CommercetoolsError] Unable to create refund payment transaction for " - f"payment mock_payment_id and stripe refund {mock_stripe_refund.id} " + f"payment mock_payment_id, refund {mock_stripe_refund.id} with PSP: stripe_edx " f"- Correlation ID: {exception.correlation_id}, Details: {exception.errors}" ) diff --git a/commerce_coordinator/apps/commercetools/tests/test_tasks.py b/commerce_coordinator/apps/commercetools/tests/test_tasks.py index 3b2e6863..15dcd241 100644 --- a/commerce_coordinator/apps/commercetools/tests/test_tasks.py +++ b/commerce_coordinator/apps/commercetools/tests/test_tasks.py @@ -119,7 +119,7 @@ def test_correct_arguments_passed(self, mock_client): mock_client().create_return_payment_transaction.assert_called_once_with( payment_id=mock_payment.id, payment_version=mock_payment.version, - stripe_refund=mock_stripe_refund + refund=mock_stripe_refund ) def test_full_refund_already_exists(self, mock_client): diff --git a/commerce_coordinator/apps/commercetools/tests/test_utils.py b/commerce_coordinator/apps/commercetools/tests/test_utils.py index 43c7e53b..efe6e417 100644 --- a/commerce_coordinator/apps/commercetools/tests/test_utils.py +++ b/commerce_coordinator/apps/commercetools/tests/test_utils.py @@ -29,7 +29,7 @@ has_refund_transaction, send_order_confirmation_email, send_unsupported_mode_fulfillment_error_email, - translate_stripe_refund_status_to_transaction_status + translate_refund_status_to_transaction_status ) @@ -313,17 +313,17 @@ class TestTranslateStripeRefundStatus(unittest.TestCase): """ def test_translate_stripe_refund_status_succeeded(self): - self.assertEqual(translate_stripe_refund_status_to_transaction_status('succeeded'), TransactionState.SUCCESS) + self.assertEqual(translate_refund_status_to_transaction_status('succeeded'), TransactionState.SUCCESS) def test_translate_stripe_refund_status_pending(self): - self.assertEqual(translate_stripe_refund_status_to_transaction_status('pending'), TransactionState.PENDING) + self.assertEqual(translate_refund_status_to_transaction_status('pending'), TransactionState.PENDING) def test_translate_stripe_refund_status_failed(self): - self.assertEqual(translate_stripe_refund_status_to_transaction_status('failed'), TransactionState.FAILURE) + self.assertEqual(translate_refund_status_to_transaction_status('failed'), TransactionState.FAILURE) def test_translate_stripe_refund_status_other(self): # Test for an unknown status - self.assertEqual(translate_stripe_refund_status_to_transaction_status('unknown_status'), 'unknown_status') + self.assertEqual(translate_refund_status_to_transaction_status('unknown_status'), 'unknown_status') class TestRetirementAnonymizingTestCase(unittest.TestCase): diff --git a/commerce_coordinator/apps/commercetools/utils.py b/commerce_coordinator/apps/commercetools/utils.py index f7e8b415..95557128 100644 --- a/commerce_coordinator/apps/commercetools/utils.py +++ b/commerce_coordinator/apps/commercetools/utils.py @@ -198,16 +198,18 @@ def find_refund_transaction(payment: Payment, amount: decimal): return {} -def translate_stripe_refund_status_to_transaction_status(stripe_refund_status: str): +def translate_refund_status_to_transaction_status(refund_status: str): """ - Utility to translate stripe's refund object's status attribute to a valid CT transaction state + Utility to translate refund object's status attribute to a valid CT transaction state """ translations = { 'succeeded': TransactionState.SUCCESS, + 'completed': TransactionState.SUCCESS, 'pending': TransactionState.PENDING, 'failed': TransactionState.FAILURE, + 'canceled': TransactionState.FAILURE, } - return translations.get(stripe_refund_status.lower(), stripe_refund_status) + return translations.get(refund_status.lower(), TransactionState.SUCCESS) def _create_retired_hash_withsalt(value_to_retire, salt): diff --git a/commerce_coordinator/apps/paypal/pipeline.py b/commerce_coordinator/apps/paypal/pipeline.py index 958e5c05..c4e8037b 100644 --- a/commerce_coordinator/apps/paypal/pipeline.py +++ b/commerce_coordinator/apps/paypal/pipeline.py @@ -26,5 +26,3 @@ def run_filter(self, psp, payment_intent_id, **params): return { 'redirect_url': redirect_url, } - - return None diff --git a/commerce_coordinator/apps/paypal/signals.py b/commerce_coordinator/apps/paypal/signals.py new file mode 100644 index 00000000..6e74096c --- /dev/null +++ b/commerce_coordinator/apps/paypal/signals.py @@ -0,0 +1,7 @@ +""" +Paypal signals and receivers. +""" + +from commerce_coordinator.apps.core.signal_helpers import CoordinatorSignal + +payment_refunded_signal = CoordinatorSignal() diff --git a/commerce_coordinator/apps/paypal/tests/test_views.py b/commerce_coordinator/apps/paypal/tests/test_views.py new file mode 100644 index 00000000..3ad9ea3b --- /dev/null +++ b/commerce_coordinator/apps/paypal/tests/test_views.py @@ -0,0 +1,100 @@ +""" +Paypal views test cases +""" +import base64 +import zlib +from unittest.mock import patch, MagicMock + +from django.conf import settings +from django.urls import reverse +from rest_framework.test import APITestCase + +from commerce_coordinator.apps.paypal.views import PayPalWebhookView + + +class PayPalWebhookViewTests(APITestCase): + """ Tests for PayPalWebhookView """ + + def setUp(self): + super().setUp() + self.url = reverse('paypal:paypal_webhook') + self.headers = { + 'paypal-transmission-id': 'test-transmission-id', + 'paypal-transmission-time': '2023-01-01T00:00:00Z', + 'paypal-transmission-sig': base64.b64encode(b'test-signature').decode('utf-8'), + 'paypal-cert-url': 'https://www.paypal.com/cert.pem', + } + self.body = b'test-body' + self.crc = zlib.crc32(self.body) + self.message = ( + f"{self.headers['paypal-transmission-id']}|{self.headers['paypal-transmission-time']}|" + f"{settings.PAYPAL_WEBHOOK_ID}|{self.crc}" + ) + + @patch('requests.get') + @patch('commerce_coordinator.apps.paypal.views.x509.load_pem_x509_certificate') + def test_post_refund_event(self, mock_load_cert, mock_requests_get): + mock_requests_get.return_value.text = 'test-cert' + mock_load_cert.return_value.public_key.return_value.verify = MagicMock() + + data = { + "event_type": "PAYMENT.CAPTURE.REFUNDED", + "resource": { + "id": "test-refund-id", + "create_time": "2023-01-01T00:00:00Z", + "status": "COMPLETED", + "amount": { + "value": "100.00", + "currency_code": "USD" + }, + "invoice_id": "test-order-number", + "links": [ + {"rel": "up", "href": "https://api.paypal.com/v2/payments/captures/test-capture-id"} + ] + } + } + + response = self.client.post(self.url, data, format='json', headers=self.headers) + self.assertEqual(response.status_code, 200) + + @patch('requests.get') + @patch('commerce_coordinator.apps.paypal.views.x509.load_pem_x509_certificate') + def test_post_invalid_signature(self, mock_load_cert, mock_requests_get): + mock_requests_get.return_value.text = 'test-cert' + mock_load_cert.return_value.public_key.return_value.verify.side_effect = Exception("Invalid signature") + + data = { + "event_type": "PAYMENT.CAPTURE.REFUNDED", + "resource": {} + } + + response = self.client.post(self.url, data, format='json', headers=self.headers) + self.assertEqual(response.status_code, 400) + + @patch('requests.get') + def test_get_certificate_from_url(self, mock_requests_get): + mock_requests_get.return_value.text = 'test-cert' + view = PayPalWebhookView() + cert = view._get_certificate(self.headers['paypal-cert-url']) # pylint: disable=protected-access + self.assertEqual(cert, 'test-cert') + mock_requests_get.assert_called_once_with(self.headers['paypal-cert-url']) + + def test_is_valid_url(self): + view = PayPalWebhookView() + self.assertTrue(view._is_valid_url('https://www.paypal.com/cert.pem')) # pylint: disable=protected-access + self.assertFalse(view._is_valid_url('ftp://www.paypal.com/cert.pem')) # pylint: disable=protected-access + self.assertFalse(view._is_valid_url('https://www.untrusted.com/cert.pem')) # pylint: disable=protected-access + + @patch('requests.get') + @patch('commerce_coordinator.apps.paypal.views.x509.load_pem_x509_certificate') + def test_invalid_event_type(self, mock_load_cert, mock_requests_get): + mock_requests_get.return_value.text = 'test-cert' + mock_load_cert.return_value.public_key.return_value.verify = MagicMock() + + data = { + "event_type": "INVALID.EVENT.TYPE", + "resource": {} + } + + response = self.client.post(self.url, data, format='json', headers=self.headers) + self.assertEqual(response.status_code, 200) diff --git a/commerce_coordinator/apps/paypal/urls.py b/commerce_coordinator/apps/paypal/urls.py new file mode 100644 index 00000000..9d6c37d1 --- /dev/null +++ b/commerce_coordinator/apps/paypal/urls.py @@ -0,0 +1,12 @@ +""" +Paypal app urls +""" + +from django.urls import path + +from commerce_coordinator.apps.paypal.views import PayPalWebhookView + +app_name = 'paypal' +urlpatterns = [ + path('webhook/', PayPalWebhookView.as_view(), name='paypal_webhook'), +] diff --git a/commerce_coordinator/apps/paypal/views.py b/commerce_coordinator/apps/paypal/views.py new file mode 100644 index 00000000..dd35b606 --- /dev/null +++ b/commerce_coordinator/apps/paypal/views.py @@ -0,0 +1,135 @@ +""" +Paypal app views +""" + +import base64 +import logging +import zlib +from urllib.parse import urlparse + +import requests +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import padding +from django.conf import settings +from rest_framework import status +from rest_framework.permissions import AllowAny +from rest_framework.response import Response +from rest_framework.throttling import UserRateThrottle + +from commerce_coordinator.apps.core.views import SingleInvocationAPIView +from commerce_coordinator.apps.paypal.signals import payment_refunded_signal + + +logger = logging.getLogger(__name__) + +WEBHOOK_ID = settings.PAYMENT_PROCESSOR_CONFIG['edx']['paypal']['paypal_webhook_id'] + +class PayPalWebhookView(SingleInvocationAPIView): + """ + PayPal webhook view + """ + ALLOWED_DOMAINS = ['www.paypal.com', 'api.paypal.com', 'api.sandbox.paypal.com', 'www.sandbox.paypal.com'] + http_method_names = ["post"] + authentication_classes = [] + permission_classes = [AllowAny] + # TODO: Limit the view to our paypal webhook servers only and remove throttling + throttle_classes = [UserRateThrottle] + + def _get_certificate(self, url): + """ + Get certificate from the given URL + """ + if not self._is_valid_url(url): + raise ValueError("Invalid or untrusted URL provided") + r = requests.get(url) # pylint: disable=missing-timeout + return r.text + + def _is_valid_url(self, url): + """ + Check if the given URL is valid + """ + try: + parsed_url = urlparse(url) + if parsed_url.scheme not in ['http', 'https']: + return False + if parsed_url.netloc not in self.ALLOWED_DOMAINS: + return False + return True + except Exception: # pylint: disable=broad-exception-caught + return False + + def post(self, request): + """ + Handle POST request + """ + body = request.body + tag = type(self).__name__ + twou_order_number = request.data.get("resource").get("invoice_id", None) + event_type = request.data.get("event_type") + webhook_id = WEBHOOK_ID + + transmission_id = request.headers.get("paypal-transmission-id") + timestamp = request.headers.get("paypal-transmission-time") + crc = zlib.crc32(body) + + message = f"{transmission_id}|{timestamp}|{webhook_id}|{crc}" + signature = base64.b64decode(request.headers.get("paypal-transmission-sig")) + certificate = self._get_certificate(request.headers.get("paypal-cert-url")) + cert = x509.load_pem_x509_certificate( + certificate.encode("utf-8"), default_backend() + ) + public_key = cert.public_key() + + try: + public_key.verify( + signature, message.encode("utf-8"), padding.PKCS1v15(), hashes.SHA256() + ) + except Exception as error: # pylint: disable=broad-exception-caught + logger.exception("Encountered exception %s verifying paypal certificate for ct_order: %s for event %s", + error, + twou_order_number, + event_type) + return Response(status=status.HTTP_400_BAD_REQUEST) + + if event_type == "PAYMENT.CAPTURE.REFUNDED": + refund_id = request.data.get("resource").get("id") + if self._is_running(tag, refund_id): # pragma no cover + self.meta_should_mark_not_running = False + return Response(status=status.HTTP_200_OK) + else: + self.mark_running(tag, refund_id) + refund_urls = request.data.get("resource").get("links", []) + paypal_capture_id = None + for link in refund_urls: + if link.get("rel") == "up" and "captures" in link.get("href"): + paypal_capture_id = link.get("href").split("/")[-1] + break + + logger.info( + "[Paypal webhooks] refund event %s with order_number %s, paypal_capture_id %s received", + event_type, + twou_order_number, + paypal_capture_id, + ) + + refund = { + "id": refund_id, + "created": request.data.get("resource").get("create_time"), + "status": request.data.get("resource").get("status"), + "amount": request.data.get("resource").get("amount").get("value"), + "currency": request.data.get("resource").get("amount").get("currency_code"), + } + + payment_refunded_signal.send_robust( + sender=self.__class__, paypal_capture_id=paypal_capture_id, refund=refund + ) + else: + logger.info( + "[Paypal webhooks] Unhandled Paypal event %s received with payload %s", + event_type, + request.data, + ) + + return Response(status=status.HTTP_200_OK) diff --git a/commerce_coordinator/apps/stripe/pipeline.py b/commerce_coordinator/apps/stripe/pipeline.py index 43a295d9..22368ad5 100644 --- a/commerce_coordinator/apps/stripe/pipeline.py +++ b/commerce_coordinator/apps/stripe/pipeline.py @@ -234,7 +234,6 @@ def run_filter(self, psp, payment_intent_id=None, **params): 'payment_intent': payment_intent, 'redirect_url': receipt_url } - return None class RefundPaymentIntent(PipelineStep): diff --git a/commerce_coordinator/settings/base.py b/commerce_coordinator/settings/base.py index 11d985d2..2efc0676 100644 --- a/commerce_coordinator/settings/base.py +++ b/commerce_coordinator/settings/base.py @@ -331,6 +331,9 @@ def root(*path_fragments): 'commerce_coordinator.apps.stripe.signals.payment_refunded_signal': [ 'commerce_coordinator.apps.commercetools.signals.refund_from_stripe', ], + "commerce_coordinator.apps.paypal.signals.payment_refunded_signal": [ + "commerce_coordinator.apps.commercetools.signals.refund_from_paypal", + ], } # Default timeouts for requests @@ -416,6 +419,7 @@ def root(*path_fragments): } STRIPE_WEBHOOK_ENDPOINT_SECRET = 'SET-ME-PLEASE' +PAYPAL_WEBHOOK_ID = 'SET-ME-PLEASE' # PAYMENT PROCESSING PAYMENT_PROCESSOR_CONFIG = { @@ -431,6 +435,9 @@ def root(*path_fragments): 'source_system_identifier': 'edx/commerce_coordinator?v=1', 'webhook_endpoint_secret': STRIPE_WEBHOOK_ENDPOINT_SECRET, }, + 'paypal': { + 'paypal_webhook_id': PAYPAL_WEBHOOK_ID, + }, }, } # END PAYMENT PROCESSING diff --git a/commerce_coordinator/settings/local.py b/commerce_coordinator/settings/local.py index 6d7803d6..c2b030ab 100644 --- a/commerce_coordinator/settings/local.py +++ b/commerce_coordinator/settings/local.py @@ -144,6 +144,9 @@ 'source_system_identifier': 'edx/commerce_coordinator?v=1', 'webhook_endpoint_secret': 'SET-ME-PLEASE', }, + "paypal": { + "paypal_webhook_id": "SET-ME-PLEASE", + }, }, } diff --git a/commerce_coordinator/settings/test.py b/commerce_coordinator/settings/test.py index 36178042..87f74f2b 100644 --- a/commerce_coordinator/settings/test.py +++ b/commerce_coordinator/settings/test.py @@ -15,6 +15,9 @@ 'source_system_identifier': 'edx/commerce_coordinator?v=1', 'webhook_endpoint_secret': 'SET-ME-PLEASE', }, + "paypal": { + "paypal_webhook_id": "SET-ME-PLEASE", + }, }, } # END PAYMENT PROCESSING diff --git a/commerce_coordinator/urls.py b/commerce_coordinator/urls.py index 56013a56..9e4fbc79 100644 --- a/commerce_coordinator/urls.py +++ b/commerce_coordinator/urls.py @@ -38,6 +38,7 @@ from commerce_coordinator.apps.frontend_app_ecommerce import urls as unified_orders_urls from commerce_coordinator.apps.frontend_app_payment import urls as frontend_app_payment_urls from commerce_coordinator.apps.lms import urls as lms_urls +from commerce_coordinator.apps.paypal import urls as paypal_urls from commerce_coordinator.apps.stripe import urls as stripe_urls from commerce_coordinator.apps.titan import urls as titan_urls from commerce_coordinator.settings.base import FAVICON_URL @@ -63,6 +64,7 @@ re_path(r'^orders/unified/', include(unified_orders_urls), name='frontend_app_ecommerce'), re_path(r'^frontend-app-payment/', include(frontend_app_payment_urls)), re_path(r'^stripe/', include(stripe_urls)), + re_path(r'^paypal/', include(paypal_urls)), # Browser automated hits, this will limit 404s in logging re_path(r'^$', lambda r: JsonResponse(data=[