From d8e8324d6da1088400ad20da2d7818ed2ce83c94 Mon Sep 17 00:00:00 2001 From: Tom Wier Date: Thu, 21 Nov 2024 12:52:43 +0300 Subject: [PATCH 1/3] fix(#182): add dbt-local-path --- dbt/dbt-run.py | 5 +++++ docker-compose.local.yml | 43 ++++++++++++++++++++++++++++++++++++++++ env.template | 3 +++ 3 files changed, 51 insertions(+) create mode 100644 docker-compose.local.yml diff --git a/dbt/dbt-run.py b/dbt/dbt-run.py index 7906e94..f5e9f2e 100644 --- a/dbt/dbt-run.py +++ b/dbt/dbt-run.py @@ -61,6 +61,11 @@ def get_package(): "revision": init_package.fragment }]}) + if os.getenv("DBT_LOCAL_PATH"): + package_json = json.dumps({"packages": [{ + "local": '/dbt/local/' + }]}) + with open("/dbt/packages.yml", "w") as f: f.write(package_json) diff --git a/docker-compose.local.yml b/docker-compose.local.yml new file mode 100644 index 0000000..fa62659 --- /dev/null +++ b/docker-compose.local.yml @@ -0,0 +1,43 @@ +name: ${COMPOSE_PROJECT_NAME:-cht-sync} +services: + couch2pg: + build: ./couch2pg/ + extra_hosts: + - "host.docker.internal:host-gateway" + logging: + driver: "json-file" + options: + max-size: "512m" + environment: + - COUCHDB_USER=${COUCHDB_USER} + - COUCHDB_PASSWORD=${COUCHDB_PASSWORD} + - COUCHDB_HOST=${COUCHDB_HOST} + - COUCHDB_DBS=${COUCHDB_DBS} + - COUCHDB_PORT=${COUCHDB_PORT} + - COUCHDB_SECURE=${COUCHDB_SECURE:-true} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_HOST=${POSTGRES_HOST} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_PORT=5432 + - POSTGRES_SCHEMA=${POSTGRES_SCHEMA} + - POSTGRES_TABLE=${POSTGRES_TABLE} + restart: always + + dbt: + build: ./dbt/ + working_dir: /dbt/ + volumes: + - "${DBT_LOCAL_PATH:-dbt}:/dbt/local/" + environment: + - POSTGRES_HOST=${POSTGRES_HOST} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_TABLE=${POSTGRES_TABLE} + - POSTGRES_SCHEMA=${POSTGRES_SCHEMA} + - ROOT_POSTGRES_SCHEMA=${POSTGRES_SCHEMA} + - CHT_PIPELINE_BRANCH_URL=${CHT_PIPELINE_BRANCH_URL} + - DATAEMON_INTERVAL=${DATAEMON_INTERVAL} + - DBT_PACKAGE_TARBALL_URL=${DBT_PACKAGE_TARBALL_URL} + - DBT_LOCAL_PATH=${DBT_LOCAL_PATH} diff --git a/env.template b/env.template index eb25151..96206c6 100644 --- a/env.template +++ b/env.template @@ -11,6 +11,9 @@ POSTGRES_PORT=5432 CHT_PIPELINE_BRANCH_URL="https://github.com/medic/cht-pipeline.git#main" DATAEMON_INTERVAL=5 +# if running locally, path to pipeline +DBT_LOCAL_PATH="/path/to/cht-pipeline/" + # couchdb COUCHDB_USER=medic COUCHDB_PASSWORD=password From c7bf41f12c3a72370cb3caf2d440e7dedb4c99a6 Mon Sep 17 00:00:00 2001 From: Tom Wier Date: Thu, 21 Nov 2024 13:01:41 +0300 Subject: [PATCH 2/3] fix(#182): changing path name --- dbt/dbt-run.py | 2 +- docker-compose.local.yml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dbt/dbt-run.py b/dbt/dbt-run.py index f5e9f2e..8f30dd1 100644 --- a/dbt/dbt-run.py +++ b/dbt/dbt-run.py @@ -63,7 +63,7 @@ def get_package(): if os.getenv("DBT_LOCAL_PATH"): package_json = json.dumps({"packages": [{ - "local": '/dbt/local/' + "local": '/dbt/package/' }]}) with open("/dbt/packages.yml", "w") as f: diff --git a/docker-compose.local.yml b/docker-compose.local.yml index fa62659..cae6b09 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -28,7 +28,7 @@ services: build: ./dbt/ working_dir: /dbt/ volumes: - - "${DBT_LOCAL_PATH:-dbt}:/dbt/local/" + - "${DBT_LOCAL_PATH}:/dbt/package/" environment: - POSTGRES_HOST=${POSTGRES_HOST} - POSTGRES_USER=${POSTGRES_USER} @@ -37,7 +37,5 @@ services: - POSTGRES_TABLE=${POSTGRES_TABLE} - POSTGRES_SCHEMA=${POSTGRES_SCHEMA} - ROOT_POSTGRES_SCHEMA=${POSTGRES_SCHEMA} - - CHT_PIPELINE_BRANCH_URL=${CHT_PIPELINE_BRANCH_URL} - DATAEMON_INTERVAL=${DATAEMON_INTERVAL} - - DBT_PACKAGE_TARBALL_URL=${DBT_PACKAGE_TARBALL_URL} - DBT_LOCAL_PATH=${DBT_LOCAL_PATH} From 6e546621ffd6c7dc6f1ae18d249aa2cc5be0895c Mon Sep 17 00:00:00 2001 From: Tom Wier Date: Thu, 21 Nov 2024 13:18:30 +0300 Subject: [PATCH 3/3] chore(#182): changing tests to use local path --- package.json | 6 +++--- tests/.e2e-env | 2 +- tests/dbt/Dockerfile | 15 --------------- tests/dbt/docker-compose.yml | 5 ----- tests/dbt/nginx.conf | 21 --------------------- 5 files changed, 4 insertions(+), 45 deletions(-) delete mode 100644 tests/dbt/Dockerfile delete mode 100644 tests/dbt/docker-compose.yml delete mode 100644 tests/dbt/nginx.conf diff --git a/package.json b/package.json index d3ecf66..8558d9b 100644 --- a/package.json +++ b/package.json @@ -5,10 +5,10 @@ "main": "", "scripts": { "postinstall": "cd couch2pg && npm ci", - "test:e2e": "npm run test:e2e-data && npm run test:e2e-containers && mocha tests/**/*.spec.js --timeout 50000; npm run test:e2e-stop-containers ", + "test:e2e": "npm run test:e2e-data && npm run test:e2e-containers && mocha tests/**/*.spec.js --timeout 50000 && npm run test:e2e-stop-containers ", "lint": "eslint --color --cache .", - "test:e2e-stop-containers": "docker compose --env-file ./tests/.e2e-env -f docker-compose.yml -f docker-compose.couchdb.yml -f docker-compose.postgres.yml down -v", - "test:e2e-containers": "docker compose --env-file ./tests/.e2e-env -f docker-compose.yml -f docker-compose.couchdb.yml -f docker-compose.postgres.yml -f tests/dbt/docker-compose.yml up -d --build --force-recreate", + "test:e2e-stop-containers": "docker compose --env-file ./tests/.e2e-env -f docker-compose.local.yml -f docker-compose.couchdb.yml -f docker-compose.postgres.yml down -v", + "test:e2e-containers": "docker compose --env-file ./tests/.e2e-env -f docker-compose.local.yml -f docker-compose.couchdb.yml -f docker-compose.postgres.yml up -d --build --force-recreate", "test:e2e-data": "cd tests/data && rm -rf ./json_docs && cht csv-to-docs", "test": "cd couch2pg && npm run test" }, diff --git a/tests/.e2e-env b/tests/.e2e-env index 9d57327..51f0892 100644 --- a/tests/.e2e-env +++ b/tests/.e2e-env @@ -7,7 +7,7 @@ DBT_POSTGRES_USER="postgres" DBT_POSTGRES_PASSWORD="postgres" DBT_POSTGRES_SCHEMA="dbt" DBT_POSTGRES_HOST="postgres" -DBT_PACKAGE_TARBALL_URL="http://dbt-package/dbt/package.tar.gz" +DBT_LOCAL_PATH="./tests/dbt/package" DATAEMON_INTERVAL=0 COUCHDB_USER="medic" COUCHDB_PASSWORD="password" diff --git a/tests/dbt/Dockerfile b/tests/dbt/Dockerfile deleted file mode 100644 index 5d90812..0000000 --- a/tests/dbt/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM nginx:1.25.1-alpine as base_nginx - -RUN apk add --update --no-cache \ - curl \ - socat \ - sed \ - bash \ - tar - -WORKDIR /dbt - -COPY nginx.conf /etc/nginx/nginx.conf -COPY package ./package -RUN tar -czvf ./package.tar.gz ./package - diff --git a/tests/dbt/docker-compose.yml b/tests/dbt/docker-compose.yml deleted file mode 100644 index 32f0378..0000000 --- a/tests/dbt/docker-compose.yml +++ /dev/null @@ -1,5 +0,0 @@ -services: - dbt-package: - build: ./tests/dbt/ - ports: - - "10080:80" diff --git a/tests/dbt/nginx.conf b/tests/dbt/nginx.conf deleted file mode 100644 index 2e743aa..0000000 --- a/tests/dbt/nginx.conf +++ /dev/null @@ -1,21 +0,0 @@ -user nginx; -worker_processes auto; - -events { - worker_connections 10240; -} - -http { - sendfile on; - keepalive_timeout 3600; - - server { - listen 80; - listen [::]:80; - - root /; - location / { - try_files $uri $uri/ =404; - } - } -}