diff --git a/.github/workflows/comment_wrong_branch.yml b/.github/workflows/comment_wrong_branch.yml index 502d5707..0d8bf19a 100644 --- a/.github/workflows/comment_wrong_branch.yml +++ b/.github/workflows/comment_wrong_branch.yml @@ -17,9 +17,33 @@ jobs: uses: actions/github-script@v6 with: script: | + const output = ` + Warning: we only accept PRs to the \`dev\` branch. It looks like you've created a PR to the \`main\` branch. Please edit this PR and select the \`dev\` branch as the target branch instead. If this was intentional please ignore this message. + `; + github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: 'Warning: we only accept PRs to the `dev` branch. It looks like you've created a PR to the `main` branch. Please edit this PR and select the `dev` branch as the target branch instead. If this was intentional please ignore this message.' + body: output + }) + + comment_warning_internal_pr: + if: ${{ github.event.pull_request.head.ref != 'dev' }} + runs-on: ubuntu-latest + steps: + - name: Comment warning about the wrong branch selected for PR + id: comment_docker_image + uses: actions/github-script@v6 + with: + script: | + const output = ` + Warning: we only accept PRs to main from the \`dev\` branch. It looks like you've created a PR from a different branch. Please edit this PR and select the \`dev\` branch as the source branch instead. If this was intentional please ignore this message. + `; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output }) diff --git a/docker/caprover.yaml b/docker/caprover.yaml index 32345213..dd3d228e 100644 --- a/docker/caprover.yaml +++ b/docker/caprover.yaml @@ -39,6 +39,8 @@ caproverOneClickApp: Your service is available at http://$$cap_appname.$$cap_root_domain. You may need to load the data fixtures or create an admin login using the Django cli. You can connect to the running container and run something like `python3 manage.py loaddata fixtures/initial.json` to do this. + We also setup a prometheus exporter for celery at http://$$cap_appname-mm-celery-prom-exporter:9808/metrics. + The prometheus exporter is for celery level, not application level metrics. displayName: "MemberMatters" isOfficial: true description: Open source membership management platform for makerspaces and community groups. @@ -90,7 +92,6 @@ services: # MemberMatters Celery Worker $$cap_appname-mm-celery-worker: image: membermatters/membermatters:$$cap_mm_version - # command: ["celery", "-A", "membermatters.celeryapp", "worker", "-l", "INFO"] restart: always environment: MM_ENV: $$cap_env @@ -113,7 +114,6 @@ services: # MemberMatters Celery Beat $$cap_appname-mm-celery-beat: image: membermatters/membermatters:$$cap_mm_version - # command: ["celery", "-A", "membermatters.celeryapp", "beat", "-l", "INFO"] restart: always environment: MM_ENV: $$cap_env @@ -132,3 +132,15 @@ services: - $$cap_appname-mm-webapp caproverExtra: notExposeAsWebApp: true + + # Celery Prometheus Exporter + $$cap_appname-mm-celery-prom-exporter: + image: danihodovic/celery-exporter + restart: always + environment: + CE_BROKER_URL: "redis://srv-captain--$$cap_appname-mm-redis:6379/0" + depends_on: + - $$cap_appname-mm-redis + - $$cap_appname-mm-celery-worker + caproverExtra: + notExposeAsWebApp: true diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 45c2e815..5a3f015f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -71,6 +71,15 @@ services: - mm-celery-worker - mm-redis + mm-celery-prom-exporter: + image: danihodovic/celery-exporter + restart: always + environment: + CE_BROKER_URL: "redis://mm-redis:6379/0" + depends_on: + - mm-celery-worker + - mm-redis + volumes: mm-postgres-volume: driver: local diff --git a/docs/POST_INSTALL_STEPS.md b/docs/POST_INSTALL_STEPS.md index a1986892..284a0acd 100644 --- a/docs/POST_INSTALL_STEPS.md +++ b/docs/POST_INSTALL_STEPS.md @@ -70,7 +70,6 @@ However, as noted below, currencies will use a hardcoded value set by a configur * "SITE_NAME" - Name of the website. * "SITE_OWNER" - Name of the organisation running this website. * "GOOGLE_ANALYTICS_MEASUREMENT_ID" - Enter your measurement ID to enable Google analytics. Only the new GA4 measurement IDs are supported. It should look something like G-XXXXXXXXXX. - * "API_SECRET_KEY" - Secret key used to authenticate some requests from access control devices. ### Signup * "INDUCTION_ENROL_LINK" - URL to enrol in the Canvas LMS induction course. @@ -189,6 +188,7 @@ as above (NOT recommended for security). * "ENABLE_DISCORD_INTEGRATION" - enable the post to Discord channel feature when an interlock or door swipe is recorded. * "DISCORD_DOOR_WEBHOOK" - URL for the door webhook. * "DISCORD_INTERLOCK_WEBHOOK" - URL for the interlock webhook. + * "DISCORD_MEMBERBUCKS_PURCHASE_WEBHOOK" - URL for the vending/product purchase webhook. ### Home Page and Welcome Email Cards diff --git a/memberportal/access/admin.py b/memberportal/access/admin.py index 3f33ca8b..b502a9af 100644 --- a/memberportal/access/admin.py +++ b/memberportal/access/admin.py @@ -4,7 +4,7 @@ @admin.register(AccessControlledDeviceAPIKey) -class AccessControlledDeviceAPIKey(APIKeyModelAdmin): +class AccessControlledDeviceAPIKeyAdmin(APIKeyModelAdmin): pass diff --git a/memberportal/access/metrics.py b/memberportal/access/metrics.py new file mode 100644 index 00000000..98bd71a2 --- /dev/null +++ b/memberportal/access/metrics.py @@ -0,0 +1,103 @@ +from prometheus_client import Counter, Histogram + +device_connections_total = Counter( + "mm_device_connections_total", + "Number of device connections", + ["type", "id", "name"], +) + +device_disconnections_total = Counter( + "mm_device_disconnections_total", + "Number of device disconnections", + ["type", "id", "name"], +) + +device_authentications_total = Counter( + "mm_device_authentications_total", + "Number of device authentications", + ["type", "id", "name"], +) + +device_checkins_total = Counter( + "mm_device_checkins_total", + "Number of device checkins", + ["type", "id", "name"], +) + +device_access_successes_total = Counter( + "mm_device_access_successes_total", + "Number of successful access swipes logged", + ["type", "id", "name"], +) + +device_access_failures_total = Counter( + "mm_device_access_failures_total", + "Number of failed access swipes logged", + ["type", "id", "name"], +) + +device_force_reboots_total = Counter( + "mm_device_force_reboots_total", + "Number of force reboots", + ["type", "id", "name"], +) + +device_syncs_total = Counter( + "mm_device_syncs_total", + "Number of syncs", + ["type", "id", "name"], +) + +device_force_bumps_total = Counter( + "mm_device_force_bumps_total", + "Number of force bumps", + ["type", "id", "name"], +) + +device_force_locks_total = Counter( + "mm_device_force_locks_total", + "Number of force locks", + ["type", "id", "name"], +) + +device_force_unlocks_total = Counter( + "mm_device_force_unlocks_total", + "Number of force unlocks", + ["type", "id", "name"], +) + +device_interlock_session_activations_total = Counter( + "mm_device_interlock_session_activations_total", + "Number of interlock session activations", + ["type", "id", "name"], +) + +device_interlock_sessions_left_on_total = Counter( + "mm_device_interlock_sessions_left_on_total", + "Number of interlock sessions left on", + ["type", "id", "name"], +) + +device_interlock_sessions_deactivated_total = Counter( + "mm_device_interlock_sessions_deactivated_total", + "Number of interlock session deactivations", + ["type", "id", "name"], +) + +device_interlock_sessions_rejected_total = Counter( + "mm_device_interlock_sessions_rejected_total", + "Number of interlock session rejections", + ["type", "id", "name"], +) + +device_interlock_sessions_cost_cents = Counter( + "mm_device_interlock_sessions_cost_cents", + "Cost of interlock sessions", + ["type", "id", "name"], +) + +device_interlock_session_duration_seconds = Histogram( + "mm_device_interlock_session_duration_seconds", + "Duration of interlock sessions", + ["type", "id", "name"], +) diff --git a/memberportal/access/migrations/0019_auto_20240603_2150.py b/memberportal/access/migrations/0019_auto_20240603_2150.py new file mode 100644 index 00000000..4689c419 --- /dev/null +++ b/memberportal/access/migrations/0019_auto_20240603_2150.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.25 on 2024-06-03 11:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("access", "0018_auto_20240525_0016"), + ] + + operations = [ + migrations.AlterField( + model_name="accesscontrolleddevice", + name="description", + field=models.CharField(max_length=500, verbose_name="Description/Location"), + ), + migrations.AlterField( + model_name="interlocklog", + name="reason", + field=models.CharField(blank=True, max_length=500, null=True), + ), + ] diff --git a/memberportal/access/models.py b/memberportal/access/models.py index 134cac39..b8786bc3 100644 --- a/memberportal/access/models.py +++ b/memberportal/access/models.py @@ -16,6 +16,8 @@ from constance import config import hashlib from django.core.validators import URLValidator +from django_prometheus.models import ExportModelOperationsMixin +import access.metrics as metrics logger = logging.getLogger("access") User = auth.get_user_model() @@ -26,8 +28,7 @@ class AccessControlledDeviceAPIKey(AbstractAPIKey): class Meta: # Add verbose name verbose_name = "API Key For Access Controlled Device" - - pass + app_label = "access" class HasAccessControlledDeviceAPIKey(BaseHasAPIKey): @@ -46,13 +47,15 @@ class HasExternalAccessControlAPIKey(BaseHasAPIKey): model = ExternalAccessControlAPIKey -class AccessControlledDevice(models.Model): +class AccessControlledDevice( + ExportModelOperationsMixin("access-controlled-device"), models.Model +): id = models.AutoField(primary_key=True) authorised = models.BooleanField( "Is this device authorised to access the system?", default=False ) name = models.CharField("Name", max_length=30, unique=True) - description = models.CharField("Description/Location", max_length=100) + description = models.CharField("Description/Location", max_length=500) ip_address = models.GenericIPAddressField( "IP Address of device", null=True, blank=True ) @@ -75,9 +78,19 @@ class AccessControlledDevice(models.Model): "Hidden from members in their access permissions screen", default=False ) + type = "unknown" + + def get_metrics_labels(self): + return { + "type": self.type, + "id": self.id, + "name": self.name, + } + def checkin(self): self.last_seen = timezone.now() self.save(update_fields=["last_seen"]) + metrics.device_checkins_total.labels(**self.get_metrics_labels()).inc() def get_unavailable(self): if self.last_seen: @@ -90,7 +103,7 @@ def __str__(self): return self.name def log_access(self, member_id, success=True): - pass + metrics.device_access_successes_total.labels(**self.get_metrics_labels()).inc() def log_event(self, description=None, data=None): if self.type == "door": @@ -117,41 +130,49 @@ def log_connected(self): self.log_event( description=f"Device connected.", ) + metrics.device_connections_total.labels(**self.get_metrics_labels()).inc() def log_disconnected(self): self.log_event( description=f"Device disconnected.", ) + metrics.device_disconnections_total.labels(**self.get_metrics_labels()).inc() def log_authenticated(self): self.log_event( description=f"Device authenticated.", ) + metrics.device_authentications_total.labels(**self.get_metrics_labels()).inc() def log_force_rebooted(self): self.log_event( description=f"Device manually rebooted.", ) + metrics.device_force_reboots_total.labels(**self.get_metrics_labels()).inc() def log_force_sync(self): self.log_event( description=f"Device manually synced.", ) + metrics.device_syncs_total.labels(**self.get_metrics_labels()).inc() def log_force_bump(self): self.log_event( description=f"Device manually bumped.", ) + metrics.device_force_bumps_total.labels(**self.get_metrics_labels()).inc() def log_force_lock(self): self.log_event( description=f"Device manually locked.", ) + metrics.device_force_locks_total.labels(**self.get_metrics_labels()).inc() def log_force_unlock(self): self.log_event( description=f"Device manually unlocked.", ) + metrics.device_force_unlocks_total.labels(**self.get_metrics_labels()).inc() def sync(self, request=None): logger.info("Sending device sync to channels for {}".format(self.serial_number)) @@ -248,7 +269,9 @@ def get_tags(self): ) -class MemberbucksDevice(AccessControlledDevice): +class MemberbucksDevice( + ExportModelOperationsMixin("memberbucks-device"), AccessControlledDevice +): all_members = True type = "memberbucks" @@ -257,7 +280,7 @@ class Meta: verbose_name_plural = "Memberbucks Devices" -class Doors(AccessControlledDevice): +class Doors(ExportModelOperationsMixin("door"), AccessControlledDevice): type = "door" class Meta: @@ -290,22 +313,40 @@ def bump(self, request=None): return True + metrics.device_force_bumps_total.labels(**self.get_metrics_labels()).inc() return False def log_access(self, member_id, success=True): logger.debug("Logging access for {}".format(self.name)) user_object = User.objects.get(pk=member_id) - door_log = DoorLog.objects.create(user=user_object, door=self, success=success) + door_log = DoorLog.objects.create( + user=user_object, door=self, success=success is True + ) profile = user_object.profile profile.last_seen = timezone.now() profile.save() - if success == True: + if success is True: + metrics.device_access_successes_total.labels( + **self.get_metrics_labels() + ).inc() if self.post_to_discord: post_door_swipe_to_discord(profile.get_full_name(), self.name, success) - elif success == False: + elif success == "locked_out": + metrics.device_access_failures_total.labels( + **self.get_metrics_labels() + ).inc() + post_door_swipe_to_discord(profile.get_full_name(), self.name, "locked_out") + + sms_message = sms.SMS() + sms_message.send_locked_out_swipe_alert(profile.phone) + + elif not success: + metrics.device_access_failures_total.labels( + **self.get_metrics_labels() + ).inc() if self.post_to_discord: post_door_swipe_to_discord( profile.get_full_name(), self.name, "rejected" @@ -313,18 +354,10 @@ def log_access(self, member_id, success=True): sms_message = sms.SMS() sms_message.send_inactive_swipe_alert(profile.phone) - elif success == "locked out": - post_door_swipe_to_discord( - profile.get_full_name(), self.name, "maintenance_lock_out" - ) - - sms_message = sms.SMS() - sms_message.send_locked_out_swipe_alert(profile.phone) - return door_log -class Interlock(AccessControlledDevice): +class Interlock(ExportModelOperationsMixin("interlock"), AccessControlledDevice): type = "interlock" cost_per_session = models.IntegerField( @@ -356,7 +389,7 @@ def session_end_all(self, reason="timeout"): for session in active_sessions: session.session_end(None) - def log_access(self, user, type="activated"): + def log_access(self, user, log_type="activated"): logger.debug("Logging access for {}".format(self.name)) profile = user.profile @@ -365,34 +398,46 @@ def log_access(self, user, type="activated"): if self.post_to_discord: post_interlock_swipe_to_discord( - profile.get_full_name(), self.name, type=type + profile.get_full_name(), self.name, type=log_type ) - if type == "activated": + if log_type == "activated": + metrics.device_interlock_session_activations_total.labels( + **self.get_metrics_labels() + ).inc() return True - elif type == "left_on": + elif log_type == "left_on": + metrics.device_interlock_sessions_left_on_total.labels( + **self.get_metrics_labels() + ).inc() return True - elif type == "deactivated": + elif log_type == "deactivated": + metrics.device_interlock_sessions_deactivated_total.labels( + **self.get_metrics_labels() + ).inc() return True - elif type == "rejected": + elif log_type == "rejected": sms_message = sms.SMS() sms_message.send_inactive_swipe_alert(profile.phone) - elif type == "maintenance_lock_out": + elif log_type == "locked_out": sms_message = sms.SMS() sms_message.send_locked_out_swipe_alert(profile.phone) - elif type == "not_signed_in": + elif log_type == "not_signed_in": pass - self.session_rejected(user, reason=type) + metrics.device_interlock_sessions_rejected_total.labels( + **self.get_metrics_labels() + ).inc() + self.session_rejected(user, reason=log_type) return True -class DoorLog(models.Model): +class DoorLog(ExportModelOperationsMixin("door-log"), models.Model): id = models.AutoField(primary_key=True) user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) door = models.ForeignKey(Doors, on_delete=models.CASCADE) @@ -403,7 +448,7 @@ def __str__(self): return f"{self.user.get_full_name()} ({self.user.profile.screen_name}) swiped at {self.door.name} {'successfully' if self.success else 'unsuccessfully'} on {self.date.date()}" -class InterlockLog(models.Model): +class InterlockLog(ExportModelOperationsMixin("interlock-log"), models.Model): id = models.UUIDField(default=uuid.uuid4, primary_key=True) interlock = models.ForeignKey(Interlock, on_delete=models.CASCADE) user_started = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) @@ -415,7 +460,7 @@ class InterlockLog(models.Model): related_name="user_ended", ) success = models.BooleanField(default=True) - reason = models.CharField(max_length=100, blank=True, null=True) + reason = models.CharField(max_length=500, blank=True, null=True) date_started = models.DateTimeField(default=timezone.now, editable=False) date_updated = models.DateTimeField(default=timezone.now) @@ -426,7 +471,7 @@ class InterlockLog(models.Model): total_cost = models.FloatField(default=None, blank=True, null=True) def __str__(self): - return f"{self.user_started.get_full_name()} ({self.user_started.profile.screen_name}) swiped at {self.interlock.name} {'successfully' if self.success else 'unsuccessfully'} for {round(self.total_time.total_seconds()/60)} mins at {self.date_started.date()}" + return f"{self.user_started.get_full_name()} ({self.user_started.profile.screen_name}) swiped at {self.interlock.name} {'successfully' if self.success else 'unsuccessfully'} for {round(self.total_time.total_seconds() / 60)} mins at {self.date_started.date()}" def calculate_cost(self): total_cost = self.interlock.cost_per_session @@ -459,6 +504,13 @@ def session_end(self, user=None, kwh=None, skip_cost=False): self.date_ended = timezone.now() self.save() + metrics.device_interlock_sessions_cost_cents.labels( + **self.interlock.get_metrics_labels() + ).inc(self.total_cost) + metrics.device_interlock_session_duration_seconds.labels( + **self.interlock.get_metrics_labels() + ).observe(self.total_time.total_seconds()) + if skip_cost or self.total_time.total_seconds() < 10: self.total_cost = 0 self.save() diff --git a/memberportal/api_access/consumers.py b/memberportal/api_access/consumers.py index ed7ce45d..4c3a4afa 100644 --- a/memberportal/api_access/consumers.py +++ b/memberportal/api_access/consumers.py @@ -9,8 +9,8 @@ InterlockLog, MemberbucksDevice, AccessControlledDeviceAPIKey, - AccessControlledDevice, ) +from services.discord import post_purchase_to_discord from memberbucks.models import ( MemberBucks, MemberbucksProductPurchaseLog, @@ -47,6 +47,7 @@ def connect(self): "description": "New device that is yet to be setup.", "serial_number": device_id, "hidden": True, + "report_online_status": False, } # Get or create the device object and check it in @@ -323,7 +324,7 @@ def handle_other_packet(self, content): profile.update_last_seen() if profile.state == "active": if self.device.locked_out: - reason = "maintenance_lock_out" + reason = "locked_out" else: allowed_interlocks = profile.interlocks.all() @@ -336,7 +337,9 @@ def handle_other_packet(self, content): or config.ENABLE_PORTAL_SITE_SIGN_IN is False ): # TODO: check they have enough memberbucks balance - self.device.log_access(profile.user, type="activated") + self.device.log_access( + profile.user, log_type="activated" + ) self.session = self.device.session_start(profile.user) self.send_json( { @@ -588,7 +591,11 @@ def handle_other_packet(self, content): purchase_log.memberbucks_device = self.device purchase_log.save() - description = f"{profile.get_full_name()} ({profile.screen_name}) {product.name} purchased from {self.device.name} ({product.external_id_name}) for {amount}." + description = f"{product.name} purchased from {self.device.name} ({product.external_id_name})." + + post_purchase_to_discord( + f"{profile.get_full_name()} ({profile.screen_name}) just bought something from {self.device.name}." + ) transaction = MemberBucks() transaction.amount = amount @@ -623,6 +630,7 @@ def handle_other_packet(self, content): "success": True, } ) + return True else: diff --git a/memberportal/api_access/metrics.py b/memberportal/api_access/metrics.py new file mode 100644 index 00000000..84f5bc56 --- /dev/null +++ b/memberportal/api_access/metrics.py @@ -0,0 +1,25 @@ +from prometheus_client import Gauge + +devices_total = Gauge( + "mm_devices_total", + "Number of devices", + ["type"], +) + +devices_online_total = Gauge( + "mm_devices_online_total", + "Number of online devices", + ["type"], +) + +devices_offline_total = Gauge( + "mm_devices_offline_total", + "Number of offline devices", + ["type"], +) + +devices_locked_out_total = Gauge( + "mm_devices_locked_out_total", + "Number of locked out devices", + ["type"], +) diff --git a/memberportal/api_access/tasks.py b/memberportal/api_access/tasks.py index 42ef4abf..ca6c56a0 100644 --- a/memberportal/api_access/tasks.py +++ b/memberportal/api_access/tasks.py @@ -1,5 +1,8 @@ from membermatters.celeryapp import app import os +import logging + +logger = logging.getLogger("api_access:tasks") @app.on_after_finalize.connect @@ -14,4 +17,4 @@ def setup_periodic_tasks(sender, **kwargs): @app.task def heartbeat_logger(): - print("Celery is alive!") + logger.debug("Celery is alive!") diff --git a/memberportal/api_access/urls.py b/memberportal/api_access/urls.py index a6e3781b..a48a3aec 100644 --- a/memberportal/api_access/urls.py +++ b/memberportal/api_access/urls.py @@ -42,11 +42,6 @@ views.RebootDoor.as_view(), name="RebootDoor", ), - path( - "api/access/interlocks//sync/", - views.SyncInterlock.as_view(), - name="SyncInterlock", - ), path( "api/access/doors//sync/", views.SyncDoor.as_view(), diff --git a/memberportal/api_access/views.py b/memberportal/api_access/views.py index e0e0293b..862a3417 100644 --- a/memberportal/api_access/views.py +++ b/memberportal/api_access/views.py @@ -1,11 +1,11 @@ -from django.utils import timezone from access.models import ( Doors, Interlock, + MemberbucksDevice, HasExternalAccessControlAPIKey, - AccessControlledDeviceAPIKey, ) from profile.models import User +import api_access.metrics as metrics from rest_framework import status, permissions from rest_framework.response import Response @@ -24,13 +24,40 @@ def get(self, request): statusObject = { "doors": [], "interlocks": [], + "memberbucksDevices": [], } error_if_offline = request.GET.get("errorIfOffline", False) a_device_is_offline = False + total_count, offline_count, online_count, locked_out_count = 0, 0, 0, 0 + + def reset_count(): + nonlocal total_count, offline_count, online_count, locked_out_count + total_count, offline_count, online_count, locked_out_count = 0, 0, 0, 0 + + def update_count(device_offline=False, device_locked_out=False): + nonlocal total_count, offline_count, online_count, locked_out_count + total_count += 1 + if device_offline: + offline_count += 1 + else: + online_count += 1 + if device_locked_out: + locked_out_count += 1 + + def report_count(device_type: str): + nonlocal total_count, offline_count, online_count, locked_out_count + metrics.devices_total.labels(type=device_type).set(total_count) + metrics.devices_online_total.labels(type=device_type).set(online_count) + metrics.devices_offline_total.labels(type=device_type).set(offline_count) + metrics.devices_locked_out_total.labels(type=device_type).set( + locked_out_count + ) for door in Doors.objects.all(): offline = door.get_unavailable() + update_count(offline, door.locked_out) + statusObject["doors"].append( { "id": door.id, @@ -43,8 +70,14 @@ def get(self, request): if offline and door.report_online_status: a_device_is_offline = True + # report door metrics + report_count("door") + reset_count() + for interlock in Interlock.objects.all(): offline = interlock.get_unavailable() + update_count(offline, interlock.locked_out) + statusObject["interlocks"].append( { "id": interlock.id, @@ -57,6 +90,30 @@ def get(self, request): if offline and interlock.report_online_status: a_device_is_offline = True + # report interlock metrics + report_count("interlock") + reset_count() + + for memberbucksDevice in MemberbucksDevice.objects.all(): + offline = memberbucksDevice.get_unavailable() + update_count(offline, memberbucksDevice.locked_out) + + statusObject["memberbucksDevices"].append( + { + "id": memberbucksDevice.id, + "name": memberbucksDevice.name, + "lastSeen": memberbucksDevice.last_seen, + "lockedOut": memberbucksDevice.locked_out, + "offline": offline, + } + ) + if offline and memberbucksDevice.report_online_status: + a_device_is_offline = True + + # report spacebucksDevices metrics + report_count("spacebucksDevice") + reset_count() + if error_if_offline and a_device_is_offline: return Response(statusObject, status=status.HTTP_503_SERVICE_UNAVAILABLE) @@ -144,20 +201,6 @@ def put(self, request, interlock_id, user_id): return Response() -class SyncInterlock(APIView): - """ - post: This method will force sync the specified interlock. - """ - - permission_classes = (permissions.IsAdminUser,) - - def post(self, request, interlock_id): - interlock = Interlock.objects.get(pk=interlock_id) - interlock.log_force_sync() - - return Response({"success": interlock.sync()}) - - class RebootInterlock(APIView): """ post: This method will reboot the specified interlock. diff --git a/memberportal/api_admin_tools/models.py b/memberportal/api_admin_tools/models.py index a24b4c07..01e5d5d3 100644 --- a/memberportal/api_admin_tools/models.py +++ b/memberportal/api_admin_tools/models.py @@ -1,8 +1,9 @@ from django.db import models +from django_prometheus.models import ExportModelOperationsMixin # This is a Stripe Product -class MemberTier(models.Model): +class MemberTier(ExportModelOperationsMixin("kiosk"), models.Model): """A membership tier that a member can be billed for.""" id = models.AutoField(primary_key=True) @@ -31,7 +32,7 @@ def get_object(self): # This is a Stripe Price -class PaymentPlan(models.Model): +class PaymentPlan(ExportModelOperationsMixin("payment-plan"), models.Model): """A Membership Plan that specifies how a member is billed for a member tier.""" BILLING_PERIODS = [("Months", "month"), ("Weeks", "week"), ("Days", "days")] diff --git a/memberportal/api_general/models.py b/memberportal/api_general/models.py index 9cdc4243..315b482e 100644 --- a/memberportal/api_general/models.py +++ b/memberportal/api_general/models.py @@ -4,11 +4,12 @@ import pytz from django.conf import settings from uuid import uuid4 +from django_prometheus.models import ExportModelOperationsMixin utc = pytz.UTC -class Kiosk(models.Model): +class Kiosk(ExportModelOperationsMixin("kiosk"), models.Model): id = models.AutoField(primary_key=True) name = models.CharField("Name", max_length=30, unique=True) kiosk_id = models.CharField("Kiosk Id", max_length=70, unique=True) @@ -34,7 +35,7 @@ def __str__(self): return self.name -class SiteSession(models.Model): +class SiteSession(ExportModelOperationsMixin("site-session"), models.Model): id = models.AutoField(primary_key=True) user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) signin_date = models.DateTimeField(default=timezone.now) @@ -49,7 +50,9 @@ def __str__(self): return f"{self.user.profile.get_full_name()} - in: {self.signin_date} out: {self.signout_date}" -class EmailVerificationToken(models.Model): +class EmailVerificationToken( + ExportModelOperationsMixin("email-verification-token"), models.Model +): id = models.AutoField(primary_key=True) user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) creation_date = models.DateTimeField(default=timezone.now) diff --git a/memberportal/api_general/tasks.py b/memberportal/api_general/tasks.py new file mode 100644 index 00000000..562d0c65 --- /dev/null +++ b/memberportal/api_general/tasks.py @@ -0,0 +1,24 @@ +from celery import signals +import logging + +logger = logging.getLogger("api_general:tasks") + + +@signals.task_failure.connect +@signals.task_revoked.connect +def on_task_failure(**kwargs): + """Abort transaction on task errors.""" + # celery exceptions will not be published to `sys.excepthook`. therefore we have to create another handler here. + from traceback import format_tb + + logger.error( + "[task:%s:%s]" + % ( + kwargs.get("task_id"), + kwargs["sender"].request.correlation_id, + ) + + "\n" + + "".join(format_tb(kwargs.get("traceback", []))) + + "\n" + + str(kwargs.get("exception", "")) + ) diff --git a/memberportal/api_general/urls.py b/memberportal/api_general/urls.py index c96b2df1..32f69459 100644 --- a/memberportal/api_general/urls.py +++ b/memberportal/api_general/urls.py @@ -42,10 +42,5 @@ views.UserSiteSession.as_view(), name="api_user_site_session", ), - path( - "api/statistics/", - views.Statistics.as_view(), - name="api_statistics", - ), path("api/kiosks//", views.Kiosks.as_view(), name="api_kiosks"), ] diff --git a/memberportal/api_general/views.py b/memberportal/api_general/views.py index 4207be86..39cbccbc 100644 --- a/memberportal/api_general/views.py +++ b/memberportal/api_general/views.py @@ -627,23 +627,6 @@ def get(self, request): return Response(status=status.HTTP_401_UNAUTHORIZED) -class Statistics(APIView): - """ - get: gets site statistics. - """ - - def get(self, request): - members = SiteSession.objects.filter(signout_date=None).order_by("-signin_date") - member_list = [] - - for member in members: - member_list.append(member.user.profile.get_full_name()) - - statistics = {"onSite": {"members": member_list, "count": members.count()}} - - return Response(statistics) - - class Register(APIView): """ post: registers a new member. diff --git a/memberportal/api_meeting/models.py b/memberportal/api_meeting/models.py index e50fc67c..b57452d2 100644 --- a/memberportal/api_meeting/models.py +++ b/memberportal/api_meeting/models.py @@ -1,8 +1,9 @@ from django.db import models from django.conf import settings +from django_prometheus.models import ExportModelOperationsMixin -class Meeting(models.Model): +class Meeting(ExportModelOperationsMixin("meeting"), models.Model): """ Model to store meeting objects. """ @@ -26,7 +27,7 @@ def __str__(self): return f"{self.date} - {self.type} meeting" -class ProxyVote(models.Model): +class ProxyVote(ExportModelOperationsMixin("proxy-vote"), models.Model): """ Model to store proxy votes that may be attached to a Meeting object. """ diff --git a/memberportal/api_member_bucks/views.py b/memberportal/api_member_bucks/views.py index e34ac65d..896dc225 100644 --- a/memberportal/api_member_bucks/views.py +++ b/memberportal/api_member_bucks/views.py @@ -6,6 +6,9 @@ from constance import config from django.db.utils import OperationalError from sentry_sdk import capture_exception +import logging + +logger = logging.getLogger("api_member_bucks") class StripeAPIView(APIView): @@ -90,7 +93,8 @@ def post(self, request, amount=None): payment_intent_id = err.payment_intent["id"] payment_intent = stripe.PaymentIntent.retrieve(payment_intent_id) - print(payment_intent) + logger.error("Error charging card!") + logger.error(payment_intent) return Response("Error charging card", status=status.HTTP_400_BAD_REQUEST) @@ -104,7 +108,8 @@ def post(self, request, amount=None): return Response() else: - print(payment_intent.status) + logger.error("Error charging card! (unexpected payment intent status)") + logger.error(payment_intent.status) return Response("Error charging card", status=status.HTTP_400_BAD_REQUEST) diff --git a/memberportal/api_member_tools/models.py b/memberportal/api_member_tools/models.py deleted file mode 100644 index e69de29b..00000000 diff --git a/memberportal/api_member_tools/serializers.py b/memberportal/api_member_tools/serializers.py deleted file mode 100644 index b3af5a30..00000000 --- a/memberportal/api_member_tools/serializers.py +++ /dev/null @@ -1,3 +0,0 @@ -from rest_framework_simplejwt.serializers import TokenObtainPairSerializer -from rest_framework import serializers -from access.models import DoorLog, InterlockLog diff --git a/memberportal/api_member_tools/views.py b/memberportal/api_member_tools/views.py index a571fe93..4dfb962a 100644 --- a/memberportal/api_member_tools/views.py +++ b/memberportal/api_member_tools/views.py @@ -1,4 +1,5 @@ -from profile.models import User, Profile +from access.models import DoorLog, InterlockLog +from profile.models import Profile from api_meeting.models import Meeting from constance import config from services.emails import send_email_to_admin @@ -9,7 +10,6 @@ from rest_framework import status, permissions from rest_framework.response import Response from rest_framework.views import APIView -from .serializers import * class SwipesList(APIView): diff --git a/memberportal/api_access/models.py b/memberportal/api_metrics/__init__.py similarity index 100% rename from memberportal/api_access/models.py rename to memberportal/api_metrics/__init__.py diff --git a/memberportal/api_metrics/admin.py b/memberportal/api_metrics/admin.py new file mode 100644 index 00000000..139a330c --- /dev/null +++ b/memberportal/api_metrics/admin.py @@ -0,0 +1,7 @@ +from django.contrib import admin +from api_metrics.models import * + + +@admin.register(Metric) +class Metric(admin.ModelAdmin): + pass diff --git a/memberportal/api_metrics/metrics.py b/memberportal/api_metrics/metrics.py new file mode 100644 index 00000000..e266811f --- /dev/null +++ b/memberportal/api_metrics/metrics.py @@ -0,0 +1,143 @@ +import logging +from dateutil.relativedelta import relativedelta +from django.db.models import Count, Sum +from django.utils import timezone +from prometheus_client import Gauge + +from api_metrics.models import Metric +from profile.models import Profile +from memberbucks.models import MemberBucks + +logger = logging.getLogger("celery:api_metrics") + +member_count_total = Gauge( + "mm_member_count_total", + "Number of members in the system", + ["state"], +) + +member_count_6_months_total = Gauge( + "member_count_6_months_total", + "Number of members in the system >6 months old", + ["state"], +) + +member_count_12_months_total = Gauge( + "member_count_12_months_total", + "Number of members in the system >12 months old", + ["state"], +) + +subscription_count_total = Gauge( + "mm_subscription_count_total", + "Number of subscriptions in the system", + ["state"], +) + +memberbucks_balance_total = Gauge( + "mm_memberbucks_balance_total", + "Total balance of memberbucks in the system", +) + +memberbucks_transactions_total = Gauge( + "mm_memberbucks_transactions_total", + "Total balance of memberbucks transactions in the system", + ["type"], +) + + +def calculate_member_count(): + # get the count of all the different member profile states + logger.debug("Calculating member count total") + profile_states = [] + + for state in ( + Profile.objects.values("state").annotate(count=Count("pk")).order_by("count") + ): + profile_states.append({"state": state["state"], "total": state["count"]}) + + Metric.objects.create( + name=Metric.MetricName.MEMBER_COUNT_TOTAL, data=profile_states + ).full_clean() + + +def calculate_member_count_6_months(): + logger.debug("Calculating member 6 months count total") + profile_states = [] + + for state in ( + Profile.objects.filter(created__lt=timezone.now() + relativedelta(months=-6)) + .values("state") + .annotate(count=Count("pk")) + .order_by("count") + ): + profile_states.append({"state": state["state"], "total": state["count"]}) + + Metric.objects.create( + name=Metric.MetricName.MEMBER_COUNT_6_MONTHS, data=profile_states + ).full_clean() + + +def calculate_member_count_12_months(): + logger.debug("Calculating member 12 months count total") + profile_states = [] + + for state in ( + Profile.objects.filter(created__lt=timezone.now() + relativedelta(months=-12)) + .values("state") + .annotate(count=Count("pk")) + .order_by("count") + ): + profile_states.append({"state": state["state"], "total": state["count"]}) + + Metric.objects.create( + name=Metric.MetricName.MEMBER_COUNT_12_MONTHS, data=profile_states + ).full_clean() + + +def calculate_subscription_count(): + # get the count of all the different subscription states + logger.debug("Calculating subscription count total") + subscription_states_data = [] + for state in ( + Profile.objects.values("subscription_status") + .annotate(count=Count("pk")) + .order_by("count") + ): + subscription_states_data.append( + {"state": state["subscription_status"], "total": state["count"]} + ) + Metric.objects.create( + name=Metric.MetricName.SUBSCRIPTION_COUNT_TOTAL, + data=subscription_states_data, + ).full_clean() + + +def calculate_memberbucks_balance(): + logger.debug("Calculating memberbucks balance total") + total_balance = Profile.objects.aggregate(Sum("memberbucks_balance")) + Metric.objects.create( + name=Metric.MetricName.MEMBERBUCKS_BALANCE_TOTAL, + data={"value": total_balance["memberbucks_balance__sum"]}, + ).full_clean() + + +def calculate_memberbucks_transactions(): + # get the sum of all the different subscription states + logger.debug("Calculating subscription count total") + transaction_data = [] + for transaction_type in ( + MemberBucks.objects.values("transaction_type") + .annotate(amount=Sum("amount")) + .order_by("-amount") + ): + transaction_data.append( + { + "type": transaction_type["transaction_type"], + "total": transaction_type["amount"], + } + ) + Metric.objects.create( + name=Metric.MetricName.MEMBERBUCKS_TRANSACTIONS_TOTAL, + data=transaction_data, + ).full_clean() diff --git a/memberportal/api_metrics/migrations/0001_initial.py b/memberportal/api_metrics/migrations/0001_initial.py new file mode 100644 index 00000000..dda86d50 --- /dev/null +++ b/memberportal/api_metrics/migrations/0001_initial.py @@ -0,0 +1,42 @@ +# Generated by Django 3.2.25 on 2024-05-31 03:39 + +from django.db import migrations, models +import django.utils.timezone +import django_prometheus.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Metric", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ( + "creation_date", + models.DateTimeField(default=django.utils.timezone.now), + ), + ("data", models.JSONField(verbose_name="Data")), + ( + "name", + models.CharField( + choices=[ + ("member_count_total", "Member Count Total"), + ("subscription_count_total", "Subscription Count Total"), + ], + default=None, + max_length=250, + verbose_name="Metric Name", + ), + ), + ], + bases=( + django_prometheus.models.ExportModelOperationsMixin("metric"), + models.Model, + ), + ), + ] diff --git a/memberportal/api_metrics/migrations/0002_alter_metric_name.py b/memberportal/api_metrics/migrations/0002_alter_metric_name.py new file mode 100644 index 00000000..53e7ca5f --- /dev/null +++ b/memberportal/api_metrics/migrations/0002_alter_metric_name.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.25 on 2024-06-01 16:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api_metrics", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="metric", + name="name", + field=models.CharField( + choices=[ + ("member_count_total", "Member Count Total"), + ("member_count_6_months", "Member Count 6 Months"), + ("member_count_12_months", "Member Count 12 Months"), + ("subscription_count_total", "Subscription Count Total"), + ], + default=None, + max_length=250, + verbose_name="Metric Name", + ), + ), + ] diff --git a/memberportal/api_metrics/migrations/0003_alter_metric_name.py b/memberportal/api_metrics/migrations/0003_alter_metric_name.py new file mode 100644 index 00000000..76f8cc5f --- /dev/null +++ b/memberportal/api_metrics/migrations/0003_alter_metric_name.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.25 on 2024-06-03 11:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api_metrics", "0002_alter_metric_name"), + ] + + operations = [ + migrations.AlterField( + model_name="metric", + name="name", + field=models.CharField( + choices=[ + ("member_count_total", "Member Count Total"), + ("member_count_6_months_total", "Member Count 6 Months"), + ("member_count_12_months_total", "Member Count 12 Months"), + ("subscription_count_total", "Subscription Count Total"), + ("memberbucks_balance_total", "Memberbucks Balance Total"), + ( + "memberbucks_transactions_total", + "Memberbucks Transactions Total", + ), + ], + default=None, + max_length=250, + verbose_name="Metric Name", + ), + ), + ] diff --git a/memberportal/api_member_bucks/models.py b/memberportal/api_metrics/migrations/__init__.py similarity index 100% rename from memberportal/api_member_bucks/models.py rename to memberportal/api_metrics/migrations/__init__.py diff --git a/memberportal/api_metrics/models.py b/memberportal/api_metrics/models.py new file mode 100644 index 00000000..fe660828 --- /dev/null +++ b/memberportal/api_metrics/models.py @@ -0,0 +1,43 @@ +from django.db import models +from django.utils import timezone +import pytz +from django_prometheus.models import ExportModelOperationsMixin + +utc = pytz.UTC + + +class Metric(ExportModelOperationsMixin("metric"), models.Model): + """Stores a single instance of a metric value.""" + + class MetricName(models.TextChoices): + MEMBER_COUNT_TOTAL = "member_count_total", "Member Count Total" + MEMBER_COUNT_6_MONTHS = "member_count_6_months_total", "Member Count 6 Months" + MEMBER_COUNT_12_MONTHS = ( + "member_count_12_months_total", + "Member Count 12 Months", + ) + SUBSCRIPTION_COUNT_TOTAL = ( + "subscription_count_total", + "Subscription Count Total", + ) + MEMBERBUCKS_BALANCE_TOTAL = ( + "memberbucks_balance_total", + "Memberbucks Balance Total", + ) + MEMBERBUCKS_TRANSACTIONS_TOTAL = ( + "memberbucks_transactions_total", + "Memberbucks Transactions Total", + ) + + id = models.AutoField(primary_key=True) + creation_date = models.DateTimeField(default=timezone.now) + data = models.JSONField("Data") + name = models.CharField( + "Metric Name", + max_length=250, + choices=MetricName.choices, + default=None, + ) + + def __str__(self): + return f"{self.name} - {self.creation_date}" diff --git a/memberportal/api_metrics/tasks.py b/memberportal/api_metrics/tasks.py new file mode 100644 index 00000000..1f2acb0d --- /dev/null +++ b/memberportal/api_metrics/tasks.py @@ -0,0 +1,36 @@ +from membermatters.celeryapp import app +from api_metrics.metrics import * + +import requests +from constance import config +import logging + +logger = logging.getLogger("celery:api_metrics") + + +@app.on_after_finalize.connect +def setup_periodic_tasks(sender, **kwargs): + sender.add_periodic_task( + config.METRICS_INTERVAL, + calculate_metrics.s(), + expires=60, + name="celery_calculate_metrics", + ) + + +@app.task +def calculate_metrics(): + logger.info("Calculating metrics!") + + calculate_member_count() + calculate_member_count_6_months() + calculate_member_count_12_months() + calculate_subscription_count() + calculate_memberbucks_balance() + calculate_memberbucks_transactions() + + try: + requests.post(config.SITE_URL + "/api/update-prom-metrics/") + + except Exception as e: + logger.error(f"Failed to update Prometheus metrics: {e}") diff --git a/memberportal/api_metrics/urls.py b/memberportal/api_metrics/urls.py new file mode 100644 index 00000000..74f8f638 --- /dev/null +++ b/memberportal/api_metrics/urls.py @@ -0,0 +1,16 @@ +from django.urls import path +from rest_framework_simplejwt import views as jwt_views +from . import views + +urlpatterns = [ + path( + "api/statistics/", + views.Statistics.as_view(), + name="api_statistics", + ), + path( + "api/update-prom-metrics/", + views.UpdatePromMetrics.as_view(), + name="api_update_prom_metrics", + ), +] diff --git a/memberportal/api_metrics/views.py b/memberportal/api_metrics/views.py new file mode 100644 index 00000000..57e81468 --- /dev/null +++ b/memberportal/api_metrics/views.py @@ -0,0 +1,80 @@ +import api_metrics.metrics +from api_metrics.models import Metric +from api_general.models import SiteSession + +from rest_framework.response import Response +from rest_framework.views import APIView +from rest_framework import permissions +import logging + +logger = logging.getLogger("metrics") + + +class Statistics(APIView): + """ + get: gets site statistics. + """ + + def get(self, request): + members = SiteSession.objects.filter(signout_date=None).order_by("-signin_date") + member_list = [] + + for member in members: + member_list.append(member.user.profile.get_full_name()) + + statistics = {"onSite": {"members": member_list, "count": members.count()}} + + return Response(statistics) + + +class UpdatePromMetrics(APIView): + """ + post: triggers Django to update the Prometheus site metrics from the database. + """ + + permission_classes = (permissions.AllowAny,) + + def post(self, request): + metrics = [] + + # get the latest metric for each type + for name in Metric.MetricName.values: + metric = Metric.objects.filter(name=name).order_by("-creation_date").first() + if metric: + metrics.append(metric) + + for metric in metrics: + if metric.name in [ + Metric.MetricName.MEMBER_COUNT_TOTAL, + Metric.MetricName.MEMBER_COUNT_6_MONTHS, + Metric.MetricName.MEMBER_COUNT_12_MONTHS, + Metric.MetricName.SUBSCRIPTION_COUNT_TOTAL, + Metric.MetricName.MEMBERBUCKS_BALANCE_TOTAL, + Metric.MetricName.MEMBERBUCKS_TRANSACTIONS_TOTAL, + ]: + prom_metric = getattr(api_metrics.metrics, metric.name) + + if not prom_metric: + logger.error(f"Prometheus metric {metric.name} not found.") + continue + + for state in metric.data: + if metric.name in [ + Metric.MetricName.MEMBERBUCKS_TRANSACTIONS_TOTAL, + ]: + logger.debug( + f"Setting {metric.name} {state['type']} to {state['total']}" + ) + prom_metric.labels(type=state["type"]).set(state["total"]) + + elif metric.name in [Metric.MetricName.MEMBERBUCKS_BALANCE_TOTAL]: + logger.debug(f"Setting {metric.name} to {metric.data[state]}") + prom_metric.set(metric.data[state]) + + else: + logger.debug( + f"Setting {metric.name} {state['state']} to {state['total']}" + ) + prom_metric.labels(state=state["state"]).set(state["total"]) + + return Response() diff --git a/memberportal/api_spacedirectory/models.py b/memberportal/api_spacedirectory/models.py index 78801649..2de310d8 100644 --- a/memberportal/api_spacedirectory/models.py +++ b/memberportal/api_spacedirectory/models.py @@ -1,9 +1,9 @@ from django.core.exceptions import ValidationError from django.db import models -from django.conf import settings +from django_prometheus.models import ExportModelOperationsMixin -class SpaceAPI(models.Model): +class SpaceAPI(ExportModelOperationsMixin("space-api"), models.Model): # The Hackspace is always closed by default to prevent people from showing up by accident. space_is_open = models.BooleanField(default=False) space_message = models.CharField(max_length=255, blank=True, null=True) @@ -23,7 +23,7 @@ def __str__(self): # Because Sensors are tied to a space, and we only have one space in the system for now # We have the luxury of not requiring foreign keys here! -class SpaceAPISensor(models.Model): +class SpaceAPISensor(ExportModelOperationsMixin("space-api-sensor"), models.Model): SENSOR_TYPE_CHOICES = [ ("temperature", "Temperature"), ("barometer", "Barometer"), @@ -50,7 +50,9 @@ def __str__(self): # Some sensors have properties, let's track those in a separate model -class SpaceAPISensorProperties(models.Model): +class SpaceAPISensorProperties( + ExportModelOperationsMixin("space-api-sensor-properties"), models.Model +): sensor_id = models.ForeignKey( SpaceAPISensor, related_name="properties", on_delete=models.CASCADE ) diff --git a/memberportal/api_spacedirectory/views.py b/memberportal/api_spacedirectory/views.py index 97d4e14a..9bf1ac2a 100644 --- a/memberportal/api_spacedirectory/views.py +++ b/memberportal/api_spacedirectory/views.py @@ -7,6 +7,9 @@ from profile.models import Profile from api_general.models import SiteSession import json +import logging + +logger = logging.getLogger("api_spacedirectory") class SpaceDirectoryStatus(APIView): @@ -145,7 +148,7 @@ def post(self, request): current_sensor = ( SpaceAPISensor.objects.all().filter(name=sensor["name"]).first() ) - print(f"Found sensor: {current_sensor}") + logger.debug(f"Found sensor: {current_sensor}") if "type" in sensor: current_sensor.sensor_type = sensor["type"] if "value" in sensor: @@ -165,7 +168,7 @@ def post(self, request): current_prop = SpaceAPISensorProperties.objects.filter( name=prop["name"], sensor_id=current_sensor ).get() - print(f"Found property: {current_prop}") + logger.debug(f"Found property: {current_prop}") if "name" in prop: current_prop.name = prop["name"] if "value" in prop: diff --git a/memberportal/memberbucks/migrations/0008_alter_memberbucks_description.py b/memberportal/memberbucks/migrations/0008_alter_memberbucks_description.py new file mode 100644 index 00000000..dc2c5254 --- /dev/null +++ b/memberportal/memberbucks/migrations/0008_alter_memberbucks_description.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.25 on 2024-06-03 11:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("memberbucks", "0007_memberbucksproduct_memberbucksproductpurchaselog"), + ] + + operations = [ + migrations.AlterField( + model_name="memberbucks", + name="description", + field=models.CharField( + max_length=500, verbose_name="Description of Transaction" + ), + ), + ] diff --git a/memberportal/memberbucks/models.py b/memberportal/memberbucks/models.py index 22ec2b9c..32986470 100644 --- a/memberportal/memberbucks/models.py +++ b/memberportal/memberbucks/models.py @@ -3,9 +3,10 @@ from django.conf import settings from django.db.models import Sum from django.utils import timezone +from django_prometheus.models import ExportModelOperationsMixin -class MemberBucks(models.Model): +class MemberBucks(ExportModelOperationsMixin("memberbucks"), models.Model): class Meta: verbose_name = "Memberbucks" verbose_name_plural = "Memberbucks" @@ -31,12 +32,12 @@ class Meta: transaction_type = models.CharField( "Transaction Type", max_length=10, choices=TRANSACTION_TYPES ) - description = models.CharField("Description of Transaction", max_length=100) + description = models.CharField("Description of Transaction", max_length=500) date = models.DateTimeField(auto_now_add=True, blank=True) logging_info = models.TextField("Detailed logging info from stripe.", blank=True) def __str__(self): - return f"{self.user.get_full_name()} {'debited' if self.amount > 0 else 'credited'} ${abs(self.amount)} for {self.description} on {self.date.date()}" + return f"{self.user.get_full_name()} {'debited' if self.amount < 0 else 'credited'} ${abs(self.amount)} for {self.description} on {self.date.date()}" def save(self, *args, **kwargs): super(MemberBucks, self).save(*args, **kwargs) @@ -55,7 +56,9 @@ def get_transaction_display(self): } -class MemberbucksProduct(models.Model): +class MemberbucksProduct( + ExportModelOperationsMixin("memberbucks-products"), models.Model +): """A new Memberbucks product should be created for each vending machine if you want to use the stock tracking or reporting features, or their external_id is different.""" @@ -88,7 +91,9 @@ def __str__(self): ) # (used for profit estimates) -class MemberbucksProductPurchaseLog(models.Model): +class MemberbucksProductPurchaseLog( + ExportModelOperationsMixin("memberbucks-product-purchase-log"), models.Model +): id = models.AutoField(primary_key=True) user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) product = models.ForeignKey(MemberbucksProduct, on_delete=models.CASCADE) diff --git a/memberportal/membermatters/celeryapp.py b/memberportal/membermatters/celeryapp.py index db7a6573..9c981aa9 100644 --- a/memberportal/membermatters/celeryapp.py +++ b/memberportal/membermatters/celeryapp.py @@ -1,6 +1,8 @@ import os - from celery import Celery +import logging + +logger = logging.getLogger("celery:celeryapp") # Set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "membermatters.settings") @@ -19,4 +21,4 @@ @app.task(bind=True) def debug_task(self): - print(f"Request: {self.request!r}") + logger.debug(f"Request: {self.request!r}") diff --git a/memberportal/membermatters/constance_config.py b/memberportal/membermatters/constance_config.py index 0a15d77e..ec22c9ac 100644 --- a/memberportal/membermatters/constance_config.py +++ b/memberportal/membermatters/constance_config.py @@ -120,6 +120,11 @@ False, "Enable integration with stripe for membership payments.", ), + # Vikunja config + "VIKUNJA_TEAMS": ( + '[{"name": "Members", "oidcID": "members", "description": "The default team for all members.", "isPublic": false}]', + "A JSON array of Vikunja teams to add users to when they login via SSO. Returned as an OIDC claim with the 'vikunja_teams' scope. Check Vikunja docs for syntax.", + ), # Trello config "ENABLE_TRELLO_INTEGRATION": ( False, @@ -203,6 +208,10 @@ "https://discordapp.com/api/webhooks/", "Discord URL to send webhook notifications to.", ), + "DISCORD_MEMBERBUCKS_PURCHASE_WEBHOOK": ( + "https://discordapp.com/api/webhooks/", + "Discord URL to send webhook notifications to for vending/memberbucks purchases.", + ), "ENABLE_DISCOURSE_SSO_PROTOCOL": ( False, "Enable support for the discourse SSO protocol.", @@ -215,10 +224,6 @@ "", "Enter your measurement ID to enable Google analytics. Only the new GA4 measurement IDs are supported. It should look something like G-XXXXXXXXXX.", ), - "API_SECRET_KEY": ( - "PLEASE_CHANGE_ME", - "The API key used by the internal access system for device authentication.", - ), "SENTRY_DSN_FRONTEND": ( "https://577dc95136cd402bb273d00f46c2a017@sentry.serv02.binarydigital.com.au/5/", "Enter a Sentry DSN to enable sentry logging of frontend errors.", @@ -325,6 +330,10 @@ '{"inactive_swipe": "Hi! Your swipe was just declined due to inactive membership. Please contact us if you need assistance.", "deactivated_access": "Hi! Your site access was just turned off. Please check your email and contact us if you need assistance.", "activated_access": "Hi! Your site access was just turned on. Please make sure you stay up to date with our policies and rules by visiting our website.", "locked_out_swipe": "Hi! Your swipe was just declined due to a temporary maintenance lockout. Please contact us if you need assistance."}', "The SMS messages to send when a user attempts to swipe with an inactive card.", ), + "METRICS_INTERVAL": ( + 3600, + "The interval in seconds to calculate and store application level metrics data like member count and door swipes.", + ), } CONSTANCE_CONFIG_FIELDSETS = OrderedDict( @@ -336,8 +345,8 @@ "SITE_OWNER", "SITE_LOCALE_CURRENCY", "GOOGLE_ANALYTICS_MEASUREMENT_ID", - "API_SECRET_KEY", "SITE_BANNER", + "METRICS_INTERVAL", ), ), ( @@ -414,6 +423,7 @@ "MEMBERBUCKS_CURRENCY", ), ), + ("Vikunja Integration", ("VIKUNJA_TEAMS",)), ( "Trello Integration", ( @@ -488,6 +498,7 @@ ( "DISCORD_DOOR_WEBHOOK", "DISCORD_INTERLOCK_WEBHOOK", + "DISCORD_MEMBERBUCKS_PURCHASE_WEBHOOK", ), ), ] diff --git a/memberportal/membermatters/mdns.py b/memberportal/membermatters/mdns.py deleted file mode 100644 index bd82ef6f..00000000 --- a/memberportal/membermatters/mdns.py +++ /dev/null @@ -1,38 +0,0 @@ -import socket -import time -from zeroconf import IPVersion, ServiceInfo, Zeroconf, InterfaceChoice - - -def run(): - try: - zeroconf = Zeroconf(ip_version=IPVersion.All, interfaces=InterfaceChoice.All) - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect(("1.1.1.1", 80)) - ip = s.getsockname()[0] - - info = ServiceInfo( - "_http._tcp.local.", - "Member Matters Server._http._tcp.local.", - addresses=[socket.inet_aton(ip)], - properties={"ip": ip}, - port=80, - server="membermatters.local.", - ) - - zeroconf.register_service(info) - except: - pass - - try: - while True: - time.sleep(0.1) - except KeyboardInterrupt: - pass - finally: - print("Unregistering...") - zeroconf.unregister_service(info) - zeroconf.close() - - -if __name__ == "__main__": - run() diff --git a/memberportal/membermatters/oidc_provider_settings.py b/memberportal/membermatters/oidc_provider_settings.py index a7fd5da1..5e625ce8 100644 --- a/memberportal/membermatters/oidc_provider_settings.py +++ b/memberportal/membermatters/oidc_provider_settings.py @@ -1,3 +1,12 @@ +import json +import logging +from django.utils.translation import ugettext_lazy as _ +from oidc_provider.lib.claims import ScopeClaims +from constance import config + +logger = logging.getLogger("oidc_provider") + + def userinfo(claims, user): # Populate claims dict. claims["name"] = user.get_full_name() or "NO_NAME" @@ -14,10 +23,6 @@ def userinfo(claims, user): return claims -from django.utils.translation import ugettext_lazy as _ -from oidc_provider.lib.claims import ScopeClaims - - class CustomScopeClaims(ScopeClaims): info_membershipinfo = ( _("Membership Info"), @@ -26,6 +31,11 @@ class CustomScopeClaims(ScopeClaims): ), ) + info_vikunja_teams = ( + _("Vikunja Teams"), + _("Vikunja teams all members should be added to automatically."), + ) + def scope_membershipinfo(self): groups = [] state = self.user.profile.state @@ -54,3 +64,22 @@ def scope_membershipinfo(self): "firstSubscribedDate": firstSubscribed, "groups": groups, } + + def scope_vikunja_teams(self): + if config.VIKUNJA_TEAMS: + try: + teams = json.loads(config.VIKUNJA_TEAMS) + if self.user.profile.state == "active": + return { + "vikunja_groups": teams, + } + else: + return { + "vikunja_groups": [], + } + except json.JSONDecodeError: + logger.error( + "VIKUNJA_TEAMS is not a valid JSON object and the Vikunja teams claim wasn't added." + ) + + return {} diff --git a/memberportal/membermatters/settings.py b/memberportal/membermatters/settings.py index d919facb..524793a0 100644 --- a/memberportal/membermatters/settings.py +++ b/memberportal/membermatters/settings.py @@ -15,7 +15,6 @@ import json from datetime import timedelta from multiprocessing import Process -from . import mdns import logging from .constance_config import CONSTANCE_CONFIG_FIELDSETS, CONSTANCE_CONFIG @@ -68,6 +67,7 @@ "api_meeting", "api_admin_tools", "api_billing", + "api_metrics", "corsheaders", "rest_framework", "rest_framework_api_key", @@ -267,6 +267,46 @@ "level": os.environ.get("MM_LOG_LEVEL_SMS", "INFO"), "propagate": False, }, + "api_general:tasks": { + "handlers": ["console", "file"], + "level": os.environ.get("MM_LOG_LEVEL_GENERAL_TASKS", "INFO"), + "propagate": False, + }, + "api_access:tasks": { + "handlers": ["console", "file"], + "level": os.environ.get("MM_LOG_LEVEL_ACCESS_TASKS", "INFO"), + "propagate": False, + }, + "celery:api_metrics": { + "handlers": ["console", "file"], + "level": os.environ.get("MM_LOG_LEVEL_CELERY_METRICS", "INFO"), + "propagate": False, + }, + "api_member_bucks": { + "handlers": ["console", "file"], + "level": os.environ.get("MM_LOG_LEVEL_MEMBER_BUCKS", "INFO"), + "propagate": False, + }, + "api_spacedirectory": { + "handlers": ["console", "file"], + "level": os.environ.get("MM_LOG_LEVEL_SPACEDIRECTORY", "INFO"), + "propagate": False, + }, + "metrics": { + "handlers": ["console", "file"], + "level": os.environ.get("MM_LOG_LEVEL_METRICS", "INFO"), + "propagate": False, + }, + "celery:celeryapp": { + "handlers": ["console", "file"], + "level": os.environ.get("MM_LOG_LEVEL_CELERY_APP", "INFO"), + "propagate": False, + }, + "oidc_provider": { + "handlers": ["console", "file"], + "level": os.environ.get("MM_LOG_LEVEL_OIDC_PROVIDER", "INFO"), + "propagate": False, + }, "daphne": {"handlers": ["console", "file"], "level": "WARNING"}, }, } @@ -335,5 +375,10 @@ USE_X_FORWARDED_HOST = True SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") +# Enable celery events for danihodovic/celery-exporter +CELERY_WORKER_SEND_TASK_EVENTS = True +CELERY_TASK_SEND_SENT_EVENT = True +CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True + # Needed for testing OIDC on local development environment with ngrok (oauth requires HTTPS) # SITE_URL = "https://1bd0-122-148-148-138.ngrok-free.app" diff --git a/memberportal/membermatters/urls.py b/memberportal/membermatters/urls.py index 3d933e30..342654d7 100644 --- a/memberportal/membermatters/urls.py +++ b/memberportal/membermatters/urls.py @@ -40,6 +40,7 @@ def safe_constance_get(fld: str): urlpatterns = [ path("api/openid/", include("oidc_provider.urls", namespace="oidc_provider")), + path("", include("api_metrics.urls")), path("", include("api_spacedirectory.urls")), path("", include("api_general.urls")), path("", include("api_access.urls")), diff --git a/memberportal/profile/models.py b/memberportal/profile/models.py index 78e6f046..fee4edb1 100644 --- a/memberportal/profile/models.py +++ b/memberportal/profile/models.py @@ -2,7 +2,6 @@ from django.utils import timezone from datetime import timedelta, datetime import pytz -import os from django.utils.timezone import make_aware from django.contrib.auth.models import ( BaseUserManager, @@ -19,6 +18,8 @@ import logging from services.emails import send_single_email, send_email_to_admin from services import sms +from django_prometheus.models import ExportModelOperationsMixin + logger = logging.getLogger("profile") @@ -41,7 +42,7 @@ ) -class Log(models.Model): +class Log(ExportModelOperationsMixin("log"), models.Model): id = models.AutoField(primary_key=True) logtype = models.CharField( "Type of action/event", choices=LOG_TYPES, max_length=30, default="generic" @@ -74,14 +75,14 @@ class Log(models.Model): ) -class UserEventLog(Log): +class UserEventLog(ExportModelOperationsMixin("user-event-log"), Log): user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) def __str__(self): return f"{self.user.get_full_name()} - {self.description}" -class EventLog(Log): +class EventLog(ExportModelOperationsMixin("event-log"), Log): def __str__(self): if self.door: return f"{self.door.name} {self.logtype} log - {self.description}" @@ -156,7 +157,7 @@ def create_superuser(self, email, password): return user -class User(AbstractBaseUser, PermissionsMixin): +class User(ExportModelOperationsMixin("user"), AbstractBaseUser, PermissionsMixin): email = models.EmailField( verbose_name="email address", max_length=255, @@ -295,7 +296,7 @@ def reset_password(self): return True -class Profile(models.Model): +class Profile(ExportModelOperationsMixin("profile"), models.Model): STATES = ( ("noob", "Needs Induction"), ("active", "Active"), diff --git a/memberportal/requirements.txt b/memberportal/requirements.txt index f97c7c7d..1b7ee403 100644 --- a/memberportal/requirements.txt +++ b/memberportal/requirements.txt @@ -1,31 +1,33 @@ django~=3.2.13 -requests~=2.27.1 -stripe~=9.7.0 -humanize~=4.1.0 +requests~=2.32.3 +stripe~=9.8.0 +humanize~=4.9.0 django-constance~=2.9.0 -django-picklefield~=3.0.1 -django-cors-headers~=3.12.0 -black==24.1.0 -pre-commit==2.19.0 -djangorestframework~=3.13.1 -djangorestframework-simplejwt>=4.7.2 -django-csp~=3.7 -sentry-sdk==1.14.0 -mysqlclient +django-picklefield~=3.2 +django-cors-headers~=4.3.1 +black==24.4.2 +pre-commit==3.7.1 +djangorestframework~=3.15.1 +djangorestframework-simplejwt>=5.3.1 +django-csp~=3.8 +sentry-sdk==2.3.1 +mysqlclient~=2.2.4 git+https://github.com/mailchimp/mailchimp-marketing-python.git channels~=3.0.4 -zeroconf~=0.38.6 +zeroconf~=0.132.2 postmarker~=1.0 channels_redis~=3.4.0 -ics~=0.7 +ics~=0.7.2 python_http_client~=3.3.7 -pwned-passwords-django~=1.6.0 -celery~=5.2.7 -django-celery-results~=2.3.1 -django-celery-beat~=2.2.1 +pwned-passwords-django~=2.1 +celery~=5.4.0 +django-celery-results~=2.5.1 +django-celery-beat~=2.6.0 redis~=4.3.1 -twilio~=7.9.2 +twilio~=9.1.0 django-prometheus==2.3.1 -psycopg2-binary~=2.9.6 -django-oidc-provider~=0.8.0 -djangorestframework-api-key==2.* +psycopg2-binary~=2.9.9 +django-oidc-provider~=0.8.2 +djangorestframework-api-key~=3.0.0 +prometheus-client~=0.20.0 +python-dateutil~=2.9.0 \ No newline at end of file diff --git a/memberportal/services/discord.py b/memberportal/services/discord.py index e786338b..ce878dc6 100644 --- a/memberportal/services/discord.py +++ b/memberportal/services/discord.py @@ -7,7 +7,7 @@ def post_door_swipe_to_discord(name, door, status): - if config.ENABLE_DISCORD_INTEGRATION: + if config.ENABLE_DISCORD_INTEGRATION and config.DISCORD_DOOR_WEBHOOK: logger.debug("Posting door swipe to Discord!") url = config.DISCORD_DOOR_WEBHOOK @@ -33,7 +33,7 @@ def post_door_swipe_to_discord(name, door, status): } ) - elif status == "maintenance_lock_out": + elif status == "locked_out": json_message["embeds"].append( { "description": ":x: {} tried to access the {} but it is currently under a " @@ -61,7 +61,7 @@ def post_door_swipe_to_discord(name, door, status): def post_interlock_swipe_to_discord(name, interlock, type, time=None): - if config.ENABLE_DISCORD_INTEGRATION: + if config.ENABLE_DISCORD_INTEGRATION and config.DISCORD_INTERLOCK_WEBHOOK: logger.debug("Posting interlock swipe to Discord!") url = config.DISCORD_INTERLOCK_WEBHOOK @@ -106,7 +106,7 @@ def post_interlock_swipe_to_discord(name, interlock, type, time=None): } ) - elif type == "maintenance_lock_out": + elif type == "locked_out": json_message["embeds"].append( { "description": "{} tried to access the {} but it is currently under a " @@ -134,7 +134,7 @@ def post_interlock_swipe_to_discord(name, interlock, type, time=None): def post_kiosk_swipe_to_discord(name, sign_in): - if config.ENABLE_DISCORD_INTEGRATION: + if config.ENABLE_DISCORD_INTEGRATION and config.DISCORD_DOOR_WEBHOOK: logger.debug("Posting kiosk swipe to Discord!") url = config.DISCORD_DOOR_WEBHOOK @@ -153,3 +153,28 @@ def post_kiosk_swipe_to_discord(name, sign_in): return True return True + + +def post_purchase_to_discord(description): + if ( + config.ENABLE_DISCORD_INTEGRATION + and config.DISCORD_MEMBERBUCKS_PURCHASE_WEBHOOK + ): + logger.debug("Posting memberbucks purchase to Discord!") + url = config.DISCORD_MEMBERBUCKS_PURCHASE_WEBHOOK + + json_message = {"description": "", "embeds": []} + + json_message["embeds"].append( + { + "description": f":coin: {description}", + "color": 5025616, + } + ) + + try: + requests.post(url, json=json_message, timeout=settings.REQUEST_TIMEOUT) + except requests.exceptions.ReadTimeout: + return True + + return True diff --git a/src-frontend/src/components/AdminTools/DeviceDialog.vue b/src-frontend/src/components/AdminTools/DeviceDialog.vue index 7d8a8d46..0137bd50 100644 --- a/src-frontend/src/components/AdminTools/DeviceDialog.vue +++ b/src-frontend/src/components/AdminTools/DeviceDialog.vue @@ -146,7 +146,9 @@
String(item.id) === this.deviceId ); } else if (this.deviceType === 'interlocks') { + this.disabled.sync = true; this.deviceIndex = this.interlocks.findIndex( (item) => String(item.id) === this.deviceId ); } else if (this.deviceType === 'memberbucks-devices') { + this.disabled = { + unlock: true, + lock: true, + reboot: true, + sync: true, + }; this.deviceIndex = this.memberbucksDevices.findIndex( (item) => String(item.id) === this.deviceId ); @@ -568,7 +585,6 @@ export default { }, onNextClick() { - let newDevice; this.deviceIndex = this.deviceIndex + 1; this.initForm(); },