From ba0b050ba760d867fd839c08a1605cd2c1788ca6 Mon Sep 17 00:00:00 2001 From: Matthew Pugh Date: Fri, 2 Aug 2024 15:41:07 +0100 Subject: [PATCH] Some minor changes to get the pipeline working on the interim solution --- Dockerfile | 1 + external_dataset/__init__.py | 0 liiatools_pipeline/sensors/sufficiency_sensor.py | 8 ++++++-- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 external_dataset/__init__.py diff --git a/Dockerfile b/Dockerfile index 7f8cf5be..b06696ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,7 @@ ENV VIRTUAL_ENV=/code/.venv \ COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} COPY liiatools /code/liiatools COPY liiatools_pipeline /code/liiatools_pipeline +COPY external_dataset /code/external_dataset RUN apt-get update && apt-get install -y --no-install-recommends diff --git a/external_dataset/__init__.py b/external_dataset/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/liiatools_pipeline/sensors/sufficiency_sensor.py b/liiatools_pipeline/sensors/sufficiency_sensor.py index c592f87b..af154ccc 100644 --- a/liiatools_pipeline/sensors/sufficiency_sensor.py +++ b/liiatools_pipeline/sensors/sufficiency_sensor.py @@ -1,9 +1,13 @@ -from dagster import RunRequest, RunsFilter, DagsterRunStatus, sensor +from dagster import RunRequest, RunsFilter, DagsterRunStatus, sensor, DefaultSensorStatus from liiatools_pipeline.jobs.ssda903_la import ssda903_clean from liiatools_pipeline.jobs.ssda903_org import ssda903_sufficiency -@sensor(job=ssda903_sufficiency) +@sensor( + job=ssda903_sufficiency, + description="Completion of the 903 cleanfile job before running the sufficiency process for 903 Files", + default_status=DefaultSensorStatus.RUNNING, +) def sufficiency_sensor(context): run_records = context.instance.get_run_records( filters=RunsFilter(