Merge pull request #180 from edx/michaelroytman/openedx_events_setup

Set up Kafka Open edX event bus.

edx_exams/settings/base.py

@@ -28,11 +28,13 @@ def root(*path_fragments):
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'edx_event_bus_kafka',
     'release_util',
     'drf_yasg',
     'edx_api_doc_tools',
     'lti_consumer.apps.LTIConsumerApp',
     'token_utils',
+    'openedx_events',
 )
 
 THIRD_PARTY_APPS = (

edx_exams/settings/devstack.py

@@ -52,3 +52,15 @@
 })
 
 LMS_ROOT_URL = 'http://edx.devstack.lms:18000'
+
+EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer'
+EVENT_BUS_CONSUMER = 'edx_event_bus_kafka.KafkaEventConsumer'
+EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081'
+EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092'
+EVENT_BUS_TOPIC_PREFIX = 'dev'
+
+# The EVENT_BUS_PRODUCER_CONFIG Django setting introduced in openedx-events 8.6.0 does not work properly.
+# In order to unblock development, EVENT_BUS_PRODUCER_CONFIG should be set to the empty object. Signal handlers
+# should be added as before (i.e. versions <8.6.0).
+# Once openedx-events 8.6.0 is corrected, this Django setting should be set.
+EVENT_BUS_PRODUCER_CONFIG = {}

edx_exams/settings/local.py

@@ -134,3 +134,15 @@
 # Lastly, see if the developer has any local overrides.
 if os.path.isfile(join(dirname(abspath(__file__)), 'private.py')):
     from .private import *  # pylint: disable=import-error
+
+EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer'
+EVENT_BUS_CONSUMER = 'edx_event_bus_kafka.KafkaEventConsumer'
+EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081'
+EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092'
+EVENT_BUS_TOPIC_PREFIX = 'dev'
+
+# The EVENT_BUS_PRODUCER_CONFIG Django setting introduced in openedx-events 8.6.0 does not work properly.
+# In order to unblock development, EVENT_BUS_PRODUCER_CONFIG should be set to the empty object. Signal handlers
+# should be added as before (i.e. versions <8.6.0).
+# Once openedx-events 8.6.0 is corrected, this Django setting should be set.
+EVENT_BUS_PRODUCER_CONFIG = {}

requirements/base.in

@@ -1,6 +1,15 @@
 # Core requirements for using this application
 -c constraints.txt
 
+# The confluent_kafka[avro,schema-registry] dependencies should eventually be moved to some form of optional
+# dependency management because the installation of confluent_kafka poses issues for users of Tutor on M1 Macs, which
+# includes many in the Open edX community. 
+# For now, we will keep this as an explicit dependency until we plan to move edx-exams to the openedx GitHub
+# organization.
+# For more details, please see https://github.com/openedx/event-bus-kafka/blob/main/docs/decisions/
+# 0005-optional-import-of-confluent-kafka.rst.
+# TODO: Move confluent_kafka to optional dependency management: https://2u-internal.atlassian.net/browse/MST-2160
+confluent_kafka[avro,schema-registry]
 Django         # Web application framework
 django-cors-headers
 django-extensions

requirements/base.txt

@@ -12,11 +12,13 @@ attrs==23.1.0
     # via
     #   lti-consumer-xblock
     #   openedx-events
+avro==1.11.2
+    # via confluent-kafka
 bleach==6.0.0
     # via lti-consumer-xblock
-boto3==1.28.42
+boto3==1.28.52
     # via fs-s3fs
-botocore==1.31.42
+botocore==1.31.52
     # via
     #   boto3
     #   s3transfer
@@ -34,13 +36,15 @@ click==8.1.7
     #   edx-django-utils
 code-annotations==1.5.0
     # via edx-toggles
+confluent-kafka[avro,schema-registry]==2.2.0
+    # via -r requirements/base.in
 coreapi==2.3.3
     # via
     #   django-rest-swagger
     #   openapi-codec
 coreschema==0.0.4
     # via coreapi
-cryptography==41.0.3
+cryptography==41.0.4
     # via
     #   pyjwt
     #   social-auth-core
@@ -80,7 +84,7 @@ django==3.2.21
     #   social-auth-app-django
 django-appconf==1.0.5
     # via django-statici18n
-django-config-models==2.5.0
+django-config-models==2.5.1
     # via lti-consumer-xblock
 django-cors-headers==4.2.0
     # via -r requirements/base.in
@@ -90,7 +94,7 @@ django-crum==0.7.9
     #   edx-toggles
 django-extensions==3.2.3
     # via -r requirements/base.in
-django-filter==23.2
+django-filter==23.3
     # via lti-consumer-xblock
 django-model-utils==4.3.1
     # via -r requirements/base.in
@@ -137,11 +141,11 @@ edx-django-utils==5.7.0
     #   edx-event-bus-kafka
     #   edx-rest-api-client
     #   edx-toggles
-edx-drf-extensions==8.9.2
+edx-drf-extensions==8.10.0
     # via -r requirements/base.in
-edx-event-bus-kafka==5.4.0
+edx-event-bus-kafka==5.5.0
     # via -r requirements/base.in
-edx-opaque-keys[django]==2.5.0
+edx-opaque-keys[django]==2.5.1
     # via
     #   -r requirements/base.in
     #   edx-ccx-keys
@@ -155,7 +159,9 @@ edx-toggles==5.1.0
 edx-token-utils==0.2.1
     # via -r requirements/base.in
 fastavro==1.8.3
-    # via openedx-events
+    # via
+    #   confluent-kafka
+    #   openedx-events
 fs==2.4.16
     # via
     #   fs-s3fs
@@ -181,7 +187,7 @@ jmespath==1.0.1
     #   botocore
 jsonfield==3.1.0
     # via lti-consumer-xblock
-lazy==1.5
+lazy==1.6
     # via
     #   lti-consumer-xblock
     #   xblock
@@ -229,7 +235,7 @@ psutil==5.9.5
     # via edx-django-utils
 pycparser==2.21
     # via cffi
-pycryptodomex==3.18.0
+pycryptodomex==3.19.0
     # via
     #   lti-consumer-xblock
     #   pyjwkest
@@ -251,7 +257,6 @@ pynacl==1.5.0
 python-dateutil==2.8.2
     # via
     #   botocore
-    #   edx-drf-extensions
     #   xblock
 python-slugify==8.0.1
     # via code-annotations
@@ -272,6 +277,7 @@ pyyaml==6.0.1
     #   xblock
 requests==2.31.0
     # via
+    #   confluent-kafka
     #   coreapi
     #   edx-drf-extensions
     #   edx-rest-api-client
@@ -295,7 +301,6 @@ six==1.16.0
     #   edx-auth-backends
     #   edx-ccx-keys
     #   edx-django-release-util
-    #   edx-drf-extensions
     #   fs
     #   fs-s3fs
     #   pyjwkest
@@ -317,7 +322,7 @@ stevedore==5.1.0
     #   edx-opaque-keys
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.7.1
+typing-extensions==4.8.0
     # via
     #   asgiref
     #   edx-opaque-keys

requirements/ci.txt

@@ -6,7 +6,7 @@
 #
 distlib==0.3.7
     # via virtualenv
-filelock==3.12.3
+filelock==3.12.4
     # via
     #   tox
     #   virtualenv
@@ -26,7 +26,5 @@ tox==3.28.0
     # via
     #   -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
     #   -r requirements/ci.in
-typing-extensions==4.7.1
-    # via filelock
-virtualenv==20.24.4
+virtualenv==20.24.5
     # via tox