diff --git a/dbt/dbt-run.py b/dbt/dbt-run.py index 7906e94..8f30dd1 100644 --- a/dbt/dbt-run.py +++ b/dbt/dbt-run.py @@ -61,6 +61,11 @@ def get_package(): "revision": init_package.fragment }]}) + if os.getenv("DBT_LOCAL_PATH"): + package_json = json.dumps({"packages": [{ + "local": '/dbt/package/' + }]}) + with open("/dbt/packages.yml", "w") as f: f.write(package_json) diff --git a/docker-compose.local.yml b/docker-compose.local.yml new file mode 100644 index 0000000..cae6b09 --- /dev/null +++ b/docker-compose.local.yml @@ -0,0 +1,41 @@ +name: ${COMPOSE_PROJECT_NAME:-cht-sync} +services: + couch2pg: + build: ./couch2pg/ + extra_hosts: + - "host.docker.internal:host-gateway" + logging: + driver: "json-file" + options: + max-size: "512m" + environment: + - COUCHDB_USER=${COUCHDB_USER} + - COUCHDB_PASSWORD=${COUCHDB_PASSWORD} + - COUCHDB_HOST=${COUCHDB_HOST} + - COUCHDB_DBS=${COUCHDB_DBS} + - COUCHDB_PORT=${COUCHDB_PORT} + - COUCHDB_SECURE=${COUCHDB_SECURE:-true} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_HOST=${POSTGRES_HOST} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_PORT=5432 + - POSTGRES_SCHEMA=${POSTGRES_SCHEMA} + - POSTGRES_TABLE=${POSTGRES_TABLE} + restart: always + + dbt: + build: ./dbt/ + working_dir: /dbt/ + volumes: + - "${DBT_LOCAL_PATH}:/dbt/package/" + environment: + - POSTGRES_HOST=${POSTGRES_HOST} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_TABLE=${POSTGRES_TABLE} + - POSTGRES_SCHEMA=${POSTGRES_SCHEMA} + - ROOT_POSTGRES_SCHEMA=${POSTGRES_SCHEMA} + - DATAEMON_INTERVAL=${DATAEMON_INTERVAL} + - DBT_LOCAL_PATH=${DBT_LOCAL_PATH} diff --git a/env.template b/env.template index eb25151..96206c6 100644 --- a/env.template +++ b/env.template @@ -11,6 +11,9 @@ POSTGRES_PORT=5432 CHT_PIPELINE_BRANCH_URL="https://github.com/medic/cht-pipeline.git#main" DATAEMON_INTERVAL=5 +# if running locally, path to pipeline +DBT_LOCAL_PATH="/path/to/cht-pipeline/" + # couchdb COUCHDB_USER=medic COUCHDB_PASSWORD=password diff --git a/package.json b/package.json index d3ecf66..8558d9b 100644 --- a/package.json +++ b/package.json @@ -5,10 +5,10 @@ "main": "", "scripts": { "postinstall": "cd couch2pg && npm ci", - "test:e2e": "npm run test:e2e-data && npm run test:e2e-containers && mocha tests/**/*.spec.js --timeout 50000; npm run test:e2e-stop-containers ", + "test:e2e": "npm run test:e2e-data && npm run test:e2e-containers && mocha tests/**/*.spec.js --timeout 50000 && npm run test:e2e-stop-containers ", "lint": "eslint --color --cache .", - "test:e2e-stop-containers": "docker compose --env-file ./tests/.e2e-env -f docker-compose.yml -f docker-compose.couchdb.yml -f docker-compose.postgres.yml down -v", - "test:e2e-containers": "docker compose --env-file ./tests/.e2e-env -f docker-compose.yml -f docker-compose.couchdb.yml -f docker-compose.postgres.yml -f tests/dbt/docker-compose.yml up -d --build --force-recreate", + "test:e2e-stop-containers": "docker compose --env-file ./tests/.e2e-env -f docker-compose.local.yml -f docker-compose.couchdb.yml -f docker-compose.postgres.yml down -v", + "test:e2e-containers": "docker compose --env-file ./tests/.e2e-env -f docker-compose.local.yml -f docker-compose.couchdb.yml -f docker-compose.postgres.yml up -d --build --force-recreate", "test:e2e-data": "cd tests/data && rm -rf ./json_docs && cht csv-to-docs", "test": "cd couch2pg && npm run test" }, diff --git a/tests/.e2e-env b/tests/.e2e-env index 9d57327..51f0892 100644 --- a/tests/.e2e-env +++ b/tests/.e2e-env @@ -7,7 +7,7 @@ DBT_POSTGRES_USER="postgres" DBT_POSTGRES_PASSWORD="postgres" DBT_POSTGRES_SCHEMA="dbt" DBT_POSTGRES_HOST="postgres" -DBT_PACKAGE_TARBALL_URL="http://dbt-package/dbt/package.tar.gz" +DBT_LOCAL_PATH="./tests/dbt/package" DATAEMON_INTERVAL=0 COUCHDB_USER="medic" COUCHDB_PASSWORD="password" diff --git a/tests/dbt/Dockerfile b/tests/dbt/Dockerfile deleted file mode 100644 index 5d90812..0000000 --- a/tests/dbt/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM nginx:1.25.1-alpine as base_nginx - -RUN apk add --update --no-cache \ - curl \ - socat \ - sed \ - bash \ - tar - -WORKDIR /dbt - -COPY nginx.conf /etc/nginx/nginx.conf -COPY package ./package -RUN tar -czvf ./package.tar.gz ./package - diff --git a/tests/dbt/docker-compose.yml b/tests/dbt/docker-compose.yml deleted file mode 100644 index 32f0378..0000000 --- a/tests/dbt/docker-compose.yml +++ /dev/null @@ -1,5 +0,0 @@ -services: - dbt-package: - build: ./tests/dbt/ - ports: - - "10080:80" diff --git a/tests/dbt/nginx.conf b/tests/dbt/nginx.conf deleted file mode 100644 index 2e743aa..0000000 --- a/tests/dbt/nginx.conf +++ /dev/null @@ -1,21 +0,0 @@ -user nginx; -worker_processes auto; - -events { - worker_connections 10240; -} - -http { - sendfile on; - keepalive_timeout 3600; - - server { - listen 80; - listen [::]:80; - - root /; - location / { - try_files $uri $uri/ =404; - } - } -}