From 58793a566070132ae0762cae8c055f917db534eb Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Mon, 2 Dec 2024 16:03:01 -0500 Subject: [PATCH 01/14] Utils function was statically initializing log https://github.com/ORNL/flowcept/issues/172 --- src/flowcept/commons/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/flowcept/commons/utils.py b/src/flowcept/commons/utils.py index 7864823c..cdb3008a 100644 --- a/src/flowcept/commons/utils.py +++ b/src/flowcept/commons/utils.py @@ -40,15 +40,15 @@ def get_utc_minutes_ago(minutes_ago=1): return rounded.timestamp() -def perf_log(func_name, t0: float, logger=FlowceptLogger()): +def perf_log(func_name, t0: float, logger=None): """Configure the performance log.""" if PERF_LOG: + _logger = logger or FlowceptLogger() t1 = time() - logger.debug(f"[PERFEVAL][{func_name}]={t1 - t0}") + _logger.debug(f"[PERFEVAL][{func_name}]={t1 - t0}") return t1 return None - def get_status_from_str(status_str: str) -> Status: """Get the status.""" # TODO: complete this utility function From 554e75d8a82a94029defab433fefab7b2e09a0d1 Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Mon, 2 Dec 2024 16:03:48 -0500 Subject: [PATCH 02/14] Reformat code --- .../commons/flowcept_dataclasses/workflow_object.py | 7 +++---- src/flowcept/commons/utils.py | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/flowcept/commons/flowcept_dataclasses/workflow_object.py b/src/flowcept/commons/flowcept_dataclasses/workflow_object.py index 6e093a71..872e02fe 100644 --- a/src/flowcept/commons/flowcept_dataclasses/workflow_object.py +++ b/src/flowcept/commons/flowcept_dataclasses/workflow_object.py @@ -4,9 +4,8 @@ import msgpack from omegaconf import OmegaConf -import flowcept -from flowcept import __version__ - +from flowcept.version import __version__ +from flowcept.commons.utils import get_utc_now from flowcept.configs import ( settings, FLOWCEPT_USER, @@ -71,7 +70,7 @@ def to_dict(self): def enrich(self, adapter_key=None): """Enrich it.""" - self.utc_timestamp = flowcept.commons.utils.get_utc_now() + self.utc_timestamp = get_utc_now() self.flowcept_settings = OmegaConf.to_container(settings) if adapter_key is not None: diff --git a/src/flowcept/commons/utils.py b/src/flowcept/commons/utils.py index cdb3008a..a7e8e427 100644 --- a/src/flowcept/commons/utils.py +++ b/src/flowcept/commons/utils.py @@ -49,6 +49,7 @@ def perf_log(func_name, t0: float, logger=None): return t1 return None + def get_status_from_str(status_str: str) -> Status: """Get the status.""" # TODO: complete this utility function From e2dd0f53723468bb7cd54c968d8a35219c7bcd29 Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Tue, 3 Dec 2024 12:06:00 -0500 Subject: [PATCH 03/14] Adding log clean up --- .github/workflows/run-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 3f0e7423..84761865 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -83,8 +83,8 @@ jobs: - name: Shut down docker compose run: make services-stop - - name: Start docker compose with kafka - run: docker compose -f deployment/compose-kafka.yml up -d + - name: Clean up log files + run: find /home/runner/runners/ -type f -name "*.log" -exec sh -c '>"{}"' \; - name: Wait for one minute run: sleep 60 From e2cf3ddf937a11732ba17230f4fdc441f27a5cb0 Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Tue, 3 Dec 2024 14:00:38 -0500 Subject: [PATCH 04/14] Adding log clean up --- .github/workflows/run-tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 84761865..7060d0c2 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -84,7 +84,10 @@ jobs: run: make services-stop - name: Clean up log files - run: find /home/runner/runners/ -type f -name "*.log" -exec sh -c '>"{}"' \; + run: find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; + + - name: Start docker compose with kafka + run: docker compose -f deployment/compose-kafka.yml up -d - name: Wait for one minute run: sleep 60 From 2ed40dfad62181da1e6ad592d0a952a1e99902ec Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Tue, 3 Dec 2024 15:07:45 -0500 Subject: [PATCH 05/14] Adding services shut down after all tests --- .github/workflows/run-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 7060d0c2..5a73f0b9 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -104,3 +104,6 @@ jobs: export MQ_TYPE=kafka export MQ_PORT=9092 make tests + + - name: Stop docker compose with kafka + run: docker compose -f deployment/compose-kafka.yml down \ No newline at end of file From a119e6833554f71593ac6be4b7fa062ceed29c81 Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Tue, 3 Dec 2024 16:37:13 -0500 Subject: [PATCH 06/14] Adding more clean up in the GH workflow --- .github/workflows/run-tests-kafka.yml | 9 --------- .github/workflows/run-tests.yml | 5 ++++- src/flowcept/commons/utils.py | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/run-tests-kafka.yml b/.github/workflows/run-tests-kafka.yml index a7aedc49..ef340586 100644 --- a/.github/workflows/run-tests-kafka.yml +++ b/.github/workflows/run-tests-kafka.yml @@ -47,12 +47,3 @@ jobs: - name: Test notebooks run: pytest --ignore=notebooks/zambeze.ipynb --nbmake "notebooks/" --nbmake-timeout=600 --ignore=notebooks/dask_from_CLI.ipynb - -# - name: Test notebooks -# run: | -# pip install -e .[all] # Installing stuff again may not be needed -# export MQ_TYPE=kafka -# export MQ_PORT=9092 -# python -c 'from flowcept.configs import MQ_TYPE, MQ_PORT; print(f"MQ_TYPE={MQ_TYPE}"); print(f"MQ_PORT={MQ_PORT}")' -# python -c 'from flowcept import Flowcept; assert Flowcept.services_alive()' -# pytest --ignore=notebooks/zambeze.ipynb --nbmake "notebooks/" --nbmake-timeout=600 --ignore=notebooks/dask_from_CLI.ipynb diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 5a73f0b9..f62aad7e 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -106,4 +106,7 @@ jobs: make tests - name: Stop docker compose with kafka - run: docker compose -f deployment/compose-kafka.yml down \ No newline at end of file + run: docker compose -f deployment/compose-kafka.yml down + + - name: Clean up log files again + run: find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; diff --git a/src/flowcept/commons/utils.py b/src/flowcept/commons/utils.py index a7e8e427..43744e68 100644 --- a/src/flowcept/commons/utils.py +++ b/src/flowcept/commons/utils.py @@ -43,8 +43,8 @@ def get_utc_minutes_ago(minutes_ago=1): def perf_log(func_name, t0: float, logger=None): """Configure the performance log.""" if PERF_LOG: - _logger = logger or FlowceptLogger() t1 = time() + _logger = logger or FlowceptLogger() _logger.debug(f"[PERFEVAL][{func_name}]={t1 - t0}") return t1 return None From 0e256ec9a13c0115afde833ce8ad2568a75d59bb Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Tue, 3 Dec 2024 17:24:14 -0500 Subject: [PATCH 07/14] Removing docker volume --- .github/workflows/run-tests.yml | 6 ++++++ deployment/compose-kafka.yml | 11 +++++++++-- deployment/compose.yml | 5 +++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f62aad7e..b2afb40f 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -86,6 +86,9 @@ jobs: - name: Clean up log files run: find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; + - name: Clean up Docker volume + run: docker volume rm flowcept_mongo_data + - name: Start docker compose with kafka run: docker compose -f deployment/compose-kafka.yml up -d @@ -110,3 +113,6 @@ jobs: - name: Clean up log files again run: find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; + + - name: Clean up Docker volume + run: docker volume rm flowcept_mongo_data \ No newline at end of file diff --git a/deployment/compose-kafka.yml b/deployment/compose-kafka.yml index 6bfa4783..4c2600c9 100644 --- a/deployment/compose-kafka.yml +++ b/deployment/compose-kafka.yml @@ -9,10 +9,10 @@ services: flowcept_mongo: container_name: flowcept_mongo image: mongo:latest - # volumes: - # - /Users/rsr/Downloads/mongo_data/db:/data/db ports: - 27017:27017 + volumes: + - mongo_data:/data/db zookeeper: image: confluentinc/cp-zookeeper:6.1.1 @@ -54,3 +54,10 @@ services: echo -e 'Successfully created the following topics:' kafka-topics --bootstrap-server kafka:29092 --list " + +networks: + flowcept: + driver: bridge + +volumes: + mongo_data: \ No newline at end of file diff --git a/deployment/compose.yml b/deployment/compose.yml index 712de8f7..5e5490ee 100644 --- a/deployment/compose.yml +++ b/deployment/compose.yml @@ -11,11 +11,16 @@ services: image: mongo:latest ports: - 27017:27017 + volumes: + - mongo_data:/data/db networks: flowcept: driver: bridge +volumes: + mongo_data: + # This is just for the cases where one does not want to use the same Redis instance for caching and messaging, but # it's not required to have separate instances. # local_interceptor_cache: From b16e5e570c2719bd8900395eda25b254c01889cf Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Tue, 3 Dec 2024 21:26:32 -0500 Subject: [PATCH 08/14] Listing large files in GH actions --- .github/workflows/run-tests.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index b2afb40f..679ed503 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -86,9 +86,6 @@ jobs: - name: Clean up log files run: find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; - - name: Clean up Docker volume - run: docker volume rm flowcept_mongo_data - - name: Start docker compose with kafka run: docker compose -f deployment/compose-kafka.yml up -d @@ -114,5 +111,5 @@ jobs: - name: Clean up log files again run: find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; - - name: Clean up Docker volume - run: docker volume rm flowcept_mongo_data \ No newline at end of file + - name: List large files + run: find . -type f -exec du -h {} + | sort -h From c4120c8f2e2961c39ebf543f1c13e91302ca5470 Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Tue, 3 Dec 2024 22:16:29 -0500 Subject: [PATCH 09/14] Clean up in run-tests.yml --- .github/workflows/run-tests.yml | 12 ++++++++---- Makefile | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 679ed503..575934f2 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -83,8 +83,10 @@ jobs: - name: Shut down docker compose run: make services-stop - - name: Clean up log files - run: find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; + - name: Clean up + run: | + make clean + find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; - name: Start docker compose with kafka run: docker compose -f deployment/compose-kafka.yml up -d @@ -108,8 +110,10 @@ jobs: - name: Stop docker compose with kafka run: docker compose -f deployment/compose-kafka.yml down - - name: Clean up log files again - run: find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; + - name: Clean up + run: | + make clean + find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; - name: List large files run: find . -type f -exec du -h {} + | sort -h diff --git a/Makefile b/Makefile index e4671cc9..6b1323de 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,9 @@ clean: find . -type f -name "*.log" -exec rm -f {} \; find . -type f -name "*.pth" -exec rm -f {} \; find . -type f -name "mlflow.db" -exec rm -f {} \; - find . -type d -name "mlruns" -exec rm -rf {} \; + find . -type d -name "mlruns" -exec rm -rf {} \; 2>/dev/null find . -type d -name "__pycache__" -exec rm -rf {} \; 2>/dev/null - sphinx-build -M clean docs docs/_build + # sphinx-build -M clean docs docs/_build This needs to be fixed. # Build the HTML documentation using Sphinx .PHONY: docs From c126e314579402a6592f34d529dc961721427401 Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Tue, 3 Dec 2024 22:47:46 -0500 Subject: [PATCH 10/14] Fix in makefile cleang --- Makefile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 6b1323de..c20f5d30 100644 --- a/Makefile +++ b/Makefile @@ -32,11 +32,12 @@ clean: rm -rf tensorboard_events rm -f docs_dump_tasks_* rm -f dump_test.json - find . -type f -name "*.log" -exec rm -f {} \; - find . -type f -name "*.pth" -exec rm -f {} \; - find . -type f -name "mlflow.db" -exec rm -f {} \; - find . -type d -name "mlruns" -exec rm -rf {} \; 2>/dev/null - find . -type d -name "__pycache__" -exec rm -rf {} \; 2>/dev/null + find . -type f -name "*.log" -exec rm -f {} \; || true + find . -type f -name "*.pth" -exec rm -f {} \; || true + find . -type f -name "mlflow.db" -exec rm -f {} \; || true + find . -type d -name "mlruns" -exec rm -rf {} \; 2>/dev/null || true + find . -type d -name "mlruns" -exec rm -rf {} \; 2>/dev/null || true + find . -type d -name "__pycache__" -exec rm -rf {} \; 2>/dev/null || true # sphinx-build -M clean docs docs/_build This needs to be fixed. # Build the HTML documentation using Sphinx From 2374b48529646d4e69453aa170e6b7e70b5ff51d Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Wed, 4 Dec 2024 10:16:04 -0500 Subject: [PATCH 11/14] Adding tensorevents to make clean --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index c20f5d30..963fbcfd 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,7 @@ clean: find . -type d -name "mlruns" -exec rm -rf {} \; 2>/dev/null || true find . -type d -name "mlruns" -exec rm -rf {} \; 2>/dev/null || true find . -type d -name "__pycache__" -exec rm -rf {} \; 2>/dev/null || true + find . -type d -name "*tfevents*" -exec rm -rf {} \; 2>/dev/null || true # sphinx-build -M clean docs docs/_build This needs to be fixed. # Build the HTML documentation using Sphinx From edd57fabf33f02f411b8a8cb2ad204dd9a64d4f7 Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Wed, 4 Dec 2024 10:16:25 -0500 Subject: [PATCH 12/14] Updating Readme with docker info and recent features --- README.md | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 75049d8e..dff167b6 100644 --- a/README.md +++ b/README.md @@ -6,15 +6,40 @@ # FlowCept -FlowCept is a runtime data integration system that empowers any data processing system to capture and query workflow provenance data using data observability, requiring minimal or no changes in the target system code. It seamlessly integrates data from multiple workflows, enabling users to comprehend complex, heterogeneous, and large-scale data from various sources in federated environments. - -FlowCept is intended to address scenarios where multiple workflows in a science campaign or in an enterprise run and generate important data to be analyzed in an integrated manner. Since these workflows may use different data manipulation tools (e.g., provenance or lineage capture tools, database systems, performance profiling tools) or can be executed within different parallel computing systems (e.g., Dask, Spark, Workflow Management Systems), its key differentiator is the capability to seamless and automatically integrate data from various workflows using data observability. It builds an integrated data view at runtime enabling end-to-end exploratory data analysis and monitoring. It follows [W3C PROV](https://www.w3.org/TR/prov-overview/) recommendations for its data schema. It does not require changes in user codes or systems (i.e., instrumentation). All users need to do is to create adapters for their systems or tools, if one is not available yet. In addition to observability, we provide instrumentation options for convenience. For example, by adding a `@flowcept_task` decorator on functions, FlowCept will observe their executions when they run. Also, we provide special features for PyTorch modules. Adding `@torch_task` to them will enable extra model inspection to be captured and integrated in the database at runtime. +FlowCept is a runtime data integration system that enables any data processing system to capture and query workflow provenance with minimal or no code changes. It integrates data across workflows, providing insights into complex, large-scale, and heterogeneous data in federated environments. It has additional features if there are Machine Learning (ML) workflows involved. + +FlowCept is designed for scenarios where multiple workflows generate critical data requiring integrated analysis. These workflows may use diverse tools (e.g., provenance capture, databases, performance profiling, ML frameworks) or run on different data processing systems. FlowCept’s key capability is to seamlessly integrate data using observability, creating a unified data view at runtime for end-to-end analysis and monitoring. + +Other capabilities include: + +- Automatic multi-workflow provenance data capture; +- Data observability, enabling minimal intrusion to user workflows; +- Explicit user workflow instrumentation, if this is preferred over data observability; +- ML data capture in various levels of details: workflow, model fitting or evaluation task, epoch iteration, layer forwarding; +- ML model management; +- Adapter-based system architecture, making it easy to plug and play with different data processing systems and backend database (e.g., MongoDB) or MQ services (e.g., Redis, Kafka); +- Low-overhead focused system architecture, to avoid adding performance overhead particularly to workloads that run on HPC machines; +- Telemetry data capture (e.g., CPU, GPU, Memory consumption) linked to the application dataflow; +- Highly customizable to multiple use cases, enabling easy toggle between settings (e.g., with/without provenance capture; with/without telemetry and which telemetry type to capture; which adapters or backend services to run with); +- [W3C PROV](https://www.w3.org/TR/prov-overview/) adherence; + +Notes: -Currently, FlowCept provides adapters for: [Dask](https://www.dask.org/), [MLFlow](https://mlflow.org/), [TensorBoard](https://www.tensorflow.org/tensorboard), and [Zambeze](https://github.com/ORNL/zambeze). +- Currently implemented data observability adapters: + - MLFlow + - Dask + - TensorBoard +- Python scripts can be easily instrumented via `@decorators` using `@flowcept_task` (for generic Python method) or `@torch_task` (for methods that encapsulate PyTorch model manipulation, such as training or evaluation). +- Currently supported MQ systems: + - Kafka + - Redis +- Currently supported database systems: + - MongoDB + - Lightning Memory-Mapped Database (lightweight file-only database system) -See the [Jupyter Notebooks](notebooks) and [Examples](examples) for utilization examples. +Explore [Jupyter Notebooks](notebooks) and [Examples](examples) for usage. -See the [Contributing](CONTRIBUTING.md) file for guidelines to contribute with new adapters. Note that we may use the term 'plugin' in the codebase as a synonym to adapter. Future releases should standardize the terminology to use adapter. +Refer to [Contributing](CONTRIBUTING.md) for adding new adapters. Note: The term "plugin" in the codebase is synonymous with "adapter," and future updates will standardize terminology. ## Install and Setup: @@ -47,6 +72,37 @@ For convenience, the default needed services can be started using a [docker-comp 4. See the [Jupyter Notebooks](notebooks) and [Examples directory](examples) for utilization examples. +## Installing and Running with Docker + +To use containers instead of installing FlowCept's dependencies on your host system, we provide a [Dockerfile](deployment/Dockerfile) alongside a [docker-compose.yml](deployment/compose.yml) for dependent services (e.g., Redis, MongoDB). + +#### Notes: +- As seen in the steps below, there are [Makefile](Makefile) commands to build and run the image. Please use them instead of running the Docker commands to build and run the image. +- The Dockerfile builds from a local `miniconda` image, which will be built first using the [build-image.sh](deployment/build-image.sh) script. +- All dependencies for all adapters are installed, increasing build time. Edit the Dockerfile to customize dependencies based on our [pyproject.toml](pyproject.toml) to reduce build time if needed. + +#### Steps: + +1. Build the Docker image: + ```bash + make build + ``` + +2. Start dependent services: + ```bash + make services + ``` + +3. Run the image interactively: + ```bash + make run + ``` + +4. Optionally, run Unit tests in the container: + ```bash + make tests-in-container + ``` + ### Simple Example with Decorators Instrumentation In addition to existing adapters to Dask, MLFlow, and others (it's extensible for any system that generates data), FlowCept also offers instrumentation via @decorators. From dcb17bcde845adbb7e3bbfdf7487006c0e6a9fb0 Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Wed, 4 Dec 2024 10:16:54 -0500 Subject: [PATCH 13/14] Adding clean up and services shutdown in all GH workflows --- .../workflows/create-release-n-publish.yml | 31 +++++++++++++++++-- .github/workflows/run-checks.yml | 7 +++++ .github/workflows/run-tests-in-container.yml | 10 ++++++ .github/workflows/run-tests-kafka.yml | 10 ++++++ .github/workflows/run-tests-py11.yml | 17 +++++++++- .github/workflows/run-tests.yml | 4 ++- 6 files changed, 74 insertions(+), 5 deletions(-) diff --git a/.github/workflows/create-release-n-publish.yml b/.github/workflows/create-release-n-publish.yml index 7b7c3629..fe729cda 100644 --- a/.github/workflows/create-release-n-publish.yml +++ b/.github/workflows/create-release-n-publish.yml @@ -12,6 +12,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -21,11 +23,13 @@ jobs: - name: Get branch names id: branch-name uses: tj-actions/branch-names@v6 + - name: Update version.py run: | export PYTHONPATH=$PYTHONPATH:flowcept export BRANCH_NAME="${{ steps.branch-name.outputs.current_branch }}" python .github/workflows/version_bumper.py + - name: Commit new version run: | git config --global user.name 'Flowcept CI Bot' @@ -70,6 +74,7 @@ jobs: pip install build --user + - name: Build a binary wheel and a source tarball run: >- python -m @@ -78,6 +83,7 @@ jobs: --wheel --outdir dist/ . + - name: Publish distribution to Test PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: @@ -91,29 +97,48 @@ jobs: with: password: ${{ secrets.PYPI_API_TOKEN }} verbose: true + - name: Wait pypi do its thing run: sleep 120 + - name: Test pip install run: pip install flowcept + - name: Print installed version run: pip list | grep flowcept + - name: Test pip install one adapter run: pip install flowcept[dask] + - name: Test pip install multiple adapters run: pip install flowcept[mlflow,tensorboard] + - name: Install our dependencies run: pip install flowcept[all] + - name: Install ml_dev dependencies run: pip install flowcept[ml_dev] + - name: Pip list - run: pip list - - name: Run Docker Compose - run: docker compose -f deployment/compose.yml up -d + run: pip list + + - name: Start up services + run: make services + - name: Test with pytest run: pytest --ignore=tests/decorator_tests/ml_tests/llm_tests + - name: Test notebooks run: | # export FLOWCEPT_SETTINGS_PATH=~/.flowcept/settings.yaml python src/flowcept/flowcept_webserver/app.py & sleep 3 pytest --nbmake "notebooks/" --nbmake-timeout=600 --ignore=notebooks/dask_from_CLI.ipynb + + - name: Stop services + run: make services-stop + + - name: Clean up + run: | + make clean + find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; || true diff --git a/.github/workflows/run-checks.yml b/.github/workflows/run-checks.yml index cd1671cb..9f7d2410 100644 --- a/.github/workflows/run-checks.yml +++ b/.github/workflows/run-checks.yml @@ -11,6 +11,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -29,3 +31,8 @@ jobs: - name: Run HTML builder for Sphinx documentation run: make docs + + - name: Clean up + run: | + make clean + find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; || true diff --git a/.github/workflows/run-tests-in-container.yml b/.github/workflows/run-tests-in-container.yml index e6218d84..c243beef 100644 --- a/.github/workflows/run-tests-in-container.yml +++ b/.github/workflows/run-tests-in-container.yml @@ -10,6 +10,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Show OS Info run: '[[ "$OSTYPE" == "linux-gnu"* ]] && { echo "OS Type: Linux"; (command -v lsb_release &> /dev/null && lsb_release -a) || cat /etc/os-release; uname -r; } || [[ "$OSTYPE" == "darwin"* ]] && { echo "OS Type: macOS"; sw_vers; uname -r; } || echo "Unsupported OS type: $OSTYPE"' @@ -22,3 +24,11 @@ jobs: - name: Run tests in container run: make tests-in-container + + - name: Stop services + run: make services-stop + + - name: Clean up + run: | + make clean + find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; || true diff --git a/.github/workflows/run-tests-kafka.yml b/.github/workflows/run-tests-kafka.yml index ef340586..f6cd2757 100644 --- a/.github/workflows/run-tests-kafka.yml +++ b/.github/workflows/run-tests-kafka.yml @@ -13,6 +13,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -47,3 +49,11 @@ jobs: - name: Test notebooks run: pytest --ignore=notebooks/zambeze.ipynb --nbmake "notebooks/" --nbmake-timeout=600 --ignore=notebooks/dask_from_CLI.ipynb + + - name: Stop services + run: docker compose -f deployment/compose-kafka.yml down + + - name: Clean up + run: | + make clean + find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; || true diff --git a/.github/workflows/run-tests-py11.yml b/.github/workflows/run-tests-py11.yml index 29bbefc5..d192d89d 100644 --- a/.github/workflows/run-tests-py11.yml +++ b/.github/workflows/run-tests-py11.yml @@ -10,6 +10,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Set up Python 3.11 uses: actions/setup-python@v5 @@ -68,9 +70,14 @@ jobs: - name: Test notebooks with pytest and redis run: make tests-notebooks - - name: Shut down docker compose + - name: Stop services run: make services-stop + - name: Clean up + run: | + make clean + find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; || true + - name: Start docker compose with kafka run: docker compose -f deployment/compose-kafka.yml up -d @@ -90,3 +97,11 @@ jobs: export MQ_PORT=9092 # Ignoring heavy tests. They are executed with Kafka in another GH Action. pytest --ignore=tests/decorator_tests/ml_tests --ignore=tests/adapters/test_tensorboard.py + + - name: Stop services + run: docker compose -f deployment/compose-kafka.yml down + + - name: Clean up + run: | + make clean + find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; || true diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 575934f2..3597c0bf 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -13,6 +13,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -113,7 +115,7 @@ jobs: - name: Clean up run: | make clean - find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; + find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; || true - name: List large files run: find . -type f -exec du -h {} + | sort -h From b9a70ae631a09c5a1f4e63ca19478bd9d81e940d Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Wed, 4 Dec 2024 12:08:39 -0500 Subject: [PATCH 14/14] Pruning images in all GH actions --- .github/workflows/create-release-n-publish.yml | 2 ++ .github/workflows/run-tests-in-container.yml | 1 + .github/workflows/run-tests-kafka.yml | 1 + .github/workflows/run-tests-py11.yml | 1 + .github/workflows/run-tests.yml | 1 + Makefile | 12 ++++++------ 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/create-release-n-publish.yml b/.github/workflows/create-release-n-publish.yml index fe729cda..f986ca2f 100644 --- a/.github/workflows/create-release-n-publish.yml +++ b/.github/workflows/create-release-n-publish.yml @@ -142,3 +142,5 @@ jobs: run: | make clean find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; || true + docker image prune -a -f + diff --git a/.github/workflows/run-tests-in-container.yml b/.github/workflows/run-tests-in-container.yml index c243beef..df4bb36f 100644 --- a/.github/workflows/run-tests-in-container.yml +++ b/.github/workflows/run-tests-in-container.yml @@ -32,3 +32,4 @@ jobs: run: | make clean find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; || true + docker image prune -a -f diff --git a/.github/workflows/run-tests-kafka.yml b/.github/workflows/run-tests-kafka.yml index f6cd2757..019349a9 100644 --- a/.github/workflows/run-tests-kafka.yml +++ b/.github/workflows/run-tests-kafka.yml @@ -57,3 +57,4 @@ jobs: run: | make clean find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; || true + docker image prune -a -f diff --git a/.github/workflows/run-tests-py11.yml b/.github/workflows/run-tests-py11.yml index d192d89d..40b5a717 100644 --- a/.github/workflows/run-tests-py11.yml +++ b/.github/workflows/run-tests-py11.yml @@ -105,3 +105,4 @@ jobs: run: | make clean find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; || true + docker image prune -a -f diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 3597c0bf..1985824b 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -116,6 +116,7 @@ jobs: run: | make clean find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; || true + docker image prune -a -f - name: List large files run: find . -type f -exec du -h {} + | sort -h diff --git a/Makefile b/Makefile index 963fbcfd..19874b98 100644 --- a/Makefile +++ b/Makefile @@ -26,12 +26,12 @@ reformat: # Remove cache directories and Sphinx build output clean: - rm -rf .ruff_cache - rm -rf .pytest_cache - rm -rf mnist_data - rm -rf tensorboard_events - rm -f docs_dump_tasks_* - rm -f dump_test.json + rm -rf .ruff_cache || true + rm -rf .pytest_cache || true + rm -rf mnist_data || true + rm -rf tensorboard_events || true + rm -f docs_dump_tasks_* || true + rm -f dump_test.json || true find . -type f -name "*.log" -exec rm -f {} \; || true find . -type f -name "*.pth" -exec rm -f {} \; || true find . -type f -name "mlflow.db" -exec rm -f {} \; || true