From c32828c23ae36ff08ea1b4f588a6b1b977a2d566 Mon Sep 17 00:00:00 2001
From: Sadiq Khoja <sadiqkhoja@gmail.com>
Date: Mon, 4 Nov 2024 15:29:53 -0500
Subject: [PATCH 1/7] Added a docker compose file for development

* Added Makefile for quick start and stop
* Modified start-enketo.sh for dev with hardcode secrets, single redis config
* Modified start-postgres.sh to listen postgres14-upgrade even if data directory is absent
* Added instruction in README.md
---
 Makefile                           | 11 +++++++
 README.md                          |  2 ++
 docker-compose.dev.yml             | 51 ++++++++++++++++++++++++++++++
 files/enketo/start-enketo.sh       |  9 ++++++
 files/postgres14/start-postgres.sh |  2 +-
 5 files changed, 74 insertions(+), 1 deletion(-)
 create mode 100644 Makefile
 create mode 100644 docker-compose.dev.yml

diff --git a/Makefile b/Makefile
new file mode 100644
index 00000000..e8b2bb26
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,11 @@
+.PHONY: dev
+dev:
+	docker compose --profile central -f docker-compose.yml -f docker-compose.dev.yml up -d
+
+.PHONY: upgrade-successful
+upgrade-successful:
+	docker compose exec postgres14 touch /var/lib/odk/postgresql/14/.postgres14-upgrade-successful
+
+.PHONY: stop
+stop:
+	docker compose stop
diff --git a/README.md b/README.md
index 2cfe34dd..cb94de91 100644
--- a/README.md
+++ b/README.md
@@ -28,6 +28,8 @@ In addition to the Backend and the Frontend, Central deploys services:
 * Central relies on [pyxform-http](https://github.com/getodk/pyxform-http) for converting Forms from XLSForm. It generally shouldn't be needed in development but can be run locally.
 * Central relies on [Enketo](https://github.com/enketo/enketo-express) for Web Form functionality. Enketo can be run locally and configured to work with Frontend and Backend in development by following [these instructions](https://github.com/getodk/central-frontend/blob/master/docs/enketo.md).
 
+If you want to work on Central codebase and don't want to setup dependent services like Postgresql, Enketo, etc manually then you can run `make dev`, which will start those services as Docker containers. If you are setting up development environment for the first time, please run `make upgrade-successful` to skip the Postgresql upgrade.
+
 Operations
 ----------
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
new file mode 100644
index 00000000..0f8c88e1
--- /dev/null
+++ b/docker-compose.dev.yml
@@ -0,0 +1,51 @@
+# Assumption: This file will be merged by `docker compose`
+#
+# What it does:
+# Sets profiles for each service depending on whether that 
+# service is required for development of that particular application
+# For Central be/fe development, we need postgres14, pyxform, enketo, redis_main
+# For Enketo development, we need postgres14, service, nginx, redis_main
+# 'none' profile is set for services that are not need for either
+#  
+# depends_on of some services are reset using !reset custom yml tag
+# nb: can't replace depends_on so we have removed all the values, ok for dev
+services:
+  postgres14:
+    profiles:
+      - central
+    network_mode: host
+  postgres:
+    profiles:
+      - none
+  mail:
+    profiles:
+      - none
+  service:
+    profiles:
+      - none
+  nginx:
+    profiles:
+      - none
+  pyxform:
+    profiles:
+      - central
+    # changing port here - can't use `ports` mapping with host networking
+    command: ["gunicorn", "--bind", "0.0.0.0:5001", "--workers", "5", "--timeout", "600", "--max-requests", "1", "--max-requests-jitter", "3", "main:app()"]
+    network_mode: host
+  secrets:
+    profiles:
+      - none
+  enketo:
+    profiles:
+      - central    
+    depends_on: !reset []
+    environment:
+      - ENV=DEV
+    network_mode: host
+  enketo_redis_main:
+    profiles:
+      - central
+    network_mode: host
+  enketo_redis_cache:
+    profiles:
+      - none
diff --git a/files/enketo/start-enketo.sh b/files/enketo/start-enketo.sh
index b9a43ac0..08829969 100755
--- a/files/enketo/start-enketo.sh
+++ b/files/enketo/start-enketo.sh
@@ -3,6 +3,15 @@
 CONFIG_PATH=${ENKETO_SRC_DIR}/config/config.json
 echo "generating enketo configuration..."
 
+if [ "$ENV" = "DEV" ]; then
+    sed -i -e 's/enketo_redis_main/localhost/g' \
+       -e 's/enketo_redis_cache/localhost/g' \
+       -e 's/6380/6379/g' \
+       -e 's/${SECRET}/s0m3v3rys3cr3tk3y/g' \
+       -e 's/${LESS_SECRET}/this $3cr3t key is crackable/g' \
+       -e 's/${API_KEY}/enketorules/g' "$CONFIG_PATH.template"
+fi
+
 BASE_URL=$( [ "${HTTPS_PORT}" = 443 ] && echo https://"${DOMAIN}" || echo https://"${DOMAIN}":"${HTTPS_PORT}" ) \
 SECRET=$(cat /etc/secrets/enketo-secret) \
 LESS_SECRET=$(cat /etc/secrets/enketo-less-secret) \
diff --git a/files/postgres14/start-postgres.sh b/files/postgres14/start-postgres.sh
index 972c6147..978f6c4c 100755
--- a/files/postgres14/start-postgres.sh
+++ b/files/postgres14/start-postgres.sh
@@ -1,7 +1,7 @@
 #!/bin/bash -eu
 set -o pipefail
 
-flag_upgradeCompletedOk="$PGDATA/../.postgres14-upgrade-successful"
+flag_upgradeCompletedOk=$(readlink -f -m "$PGDATA/../.postgres14-upgrade-successful")
 
 logPrefix="$(basename "$0")"
 log() {

From 644e16aeb58d93881ffc9b4468eba460d1187624 Mon Sep 17 00:00:00 2001
From: Sadiq Khoja <sadiqkhoja@gmail.com>
Date: Wed, 27 Nov 2024 16:48:26 -0500
Subject: [PATCH 2/7] update the comment in docker-compose.dev.yml

---
 docker-compose.dev.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 0f8c88e1..6317e2bb 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -2,10 +2,7 @@
 #
 # What it does:
 # Sets profiles for each service depending on whether that 
-# service is required for development of that particular application
-# For Central be/fe development, we need postgres14, pyxform, enketo, redis_main
-# For Enketo development, we need postgres14, service, nginx, redis_main
-# 'none' profile is set for services that are not need for either
+# service is required for Central development or not.
 #  
 # depends_on of some services are reset using !reset custom yml tag
 # nb: can't replace depends_on so we have removed all the values, ok for dev

From e10956ce7484f0c4da59cb25ed7ff60905a91cda Mon Sep 17 00:00:00 2001
From: Sadiq Khoja <sadiqkhoja@gmail.com>
Date: Wed, 27 Nov 2024 16:50:44 -0500
Subject: [PATCH 3/7] remove the extra step of running 'make
 upgrade-successful'

---
 Makefile  | 3 ---
 README.md | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index e8b2bb26..d0e56b2e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,9 +1,6 @@
 .PHONY: dev
 dev:
 	docker compose --profile central -f docker-compose.yml -f docker-compose.dev.yml up -d
-
-.PHONY: upgrade-successful
-upgrade-successful:
 	docker compose exec postgres14 touch /var/lib/odk/postgresql/14/.postgres14-upgrade-successful
 
 .PHONY: stop
diff --git a/README.md b/README.md
index cb94de91..f2afb037 100644
--- a/README.md
+++ b/README.md
@@ -28,7 +28,7 @@ In addition to the Backend and the Frontend, Central deploys services:
 * Central relies on [pyxform-http](https://github.com/getodk/pyxform-http) for converting Forms from XLSForm. It generally shouldn't be needed in development but can be run locally.
 * Central relies on [Enketo](https://github.com/enketo/enketo-express) for Web Form functionality. Enketo can be run locally and configured to work with Frontend and Backend in development by following [these instructions](https://github.com/getodk/central-frontend/blob/master/docs/enketo.md).
 
-If you want to work on Central codebase and don't want to setup dependent services like Postgresql, Enketo, etc manually then you can run `make dev`, which will start those services as Docker containers. If you are setting up development environment for the first time, please run `make upgrade-successful` to skip the Postgresql upgrade.
+If you want to work on Central codebase and don't want to setup dependent services like Postgresql, Enketo, etc manually then you can run `make dev`, which will start those services as Docker containers.
 
 Operations
 ----------

From 6338341ea67f8373f3afd23832d5d1a0ad1504a1 Mon Sep 17 00:00:00 2001
From: Sadiq Khoja <sadiqkhoja@gmail.com>
Date: Fri, 29 Nov 2024 15:21:28 -0500
Subject: [PATCH 4/7] move enketo dev secrets to docker-compose.dev.yml

---
 docker-compose.dev.yml       | 16 +++++++++++++---
 files/enketo/start-enketo.sh |  5 +----
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 6317e2bb..6785e559 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -31,11 +31,19 @@ services:
     network_mode: host
   secrets:
     profiles:
-      - none
+      - central
+    volumes:
+      - dev_secrets:/etc/secrets
+    working_dir: /etc/secrets
+    command: /bin/sh -c 'echo "s0m3v3rys3cr3tk3y" > enketo-secret && echo "this $3cr3t key is crackable" > enketo-less-secret && echo "enketorules" > enketo-api-key'
   enketo:
     profiles:
-      - central    
-    depends_on: !reset []
+      - central
+    volumes:
+      - dev_secrets:/etc/secrets
+    depends_on: !override
+      - secrets
+      - enketo_redis_main
     environment:
       - ENV=DEV
     network_mode: host
@@ -46,3 +54,5 @@ services:
   enketo_redis_cache:
     profiles:
       - none
+volumes:
+  dev_secrets:
\ No newline at end of file
diff --git a/files/enketo/start-enketo.sh b/files/enketo/start-enketo.sh
index 08829969..9c107a39 100755
--- a/files/enketo/start-enketo.sh
+++ b/files/enketo/start-enketo.sh
@@ -6,10 +6,7 @@ echo "generating enketo configuration..."
 if [ "$ENV" = "DEV" ]; then
     sed -i -e 's/enketo_redis_main/localhost/g' \
        -e 's/enketo_redis_cache/localhost/g' \
-       -e 's/6380/6379/g' \
-       -e 's/${SECRET}/s0m3v3rys3cr3tk3y/g' \
-       -e 's/${LESS_SECRET}/this $3cr3t key is crackable/g' \
-       -e 's/${API_KEY}/enketorules/g' "$CONFIG_PATH.template"
+       -e 's/6380/6379/g' "$CONFIG_PATH.template"
 fi
 
 BASE_URL=$( [ "${HTTPS_PORT}" = 443 ] && echo https://"${DOMAIN}" || echo https://"${DOMAIN}":"${HTTPS_PORT}" ) \

From 97f2bb962a4cfb9c9bcfc1777a5973271b7de66b Mon Sep 17 00:00:00 2001
From: Sadiq Khoja <sadiqkhoja@gmail.com>
Date: Mon, 2 Dec 2024 15:21:35 -0500
Subject: [PATCH 5/7] PR Feedback:

- Start two instances of redis
- remove readlink
- use postgres service to create data dir and .postgres14-upgrade-successful
---
 Makefile                           | 1 -
 docker-compose.dev.yml             | 7 +++++--
 files/enketo/start-enketo.sh       | 3 +--
 files/postgres14/start-postgres.sh | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index d0e56b2e..c020af70 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,6 @@
 .PHONY: dev
 dev:
 	docker compose --profile central -f docker-compose.yml -f docker-compose.dev.yml up -d
-	docker compose exec postgres14 touch /var/lib/odk/postgresql/14/.postgres14-upgrade-successful
 
 .PHONY: stop
 stop:
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 6785e559..7f455bf9 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -13,7 +13,9 @@ services:
     network_mode: host
   postgres:
     profiles:
-      - none
+      - central
+    image: debian:bullseye
+    command: /bin/sh -c 'mkdir -p /var/lib/postgresql/14/data && touch /var/lib/postgresql/14/.postgres14-upgrade-successful'
   mail:
     profiles:
       - none
@@ -53,6 +55,7 @@ services:
     network_mode: host
   enketo_redis_cache:
     profiles:
-      - none
+      - central
+    network_mode: host
 volumes:
   dev_secrets:
\ No newline at end of file
diff --git a/files/enketo/start-enketo.sh b/files/enketo/start-enketo.sh
index 9c107a39..6b6d7055 100755
--- a/files/enketo/start-enketo.sh
+++ b/files/enketo/start-enketo.sh
@@ -5,8 +5,7 @@ echo "generating enketo configuration..."
 
 if [ "$ENV" = "DEV" ]; then
     sed -i -e 's/enketo_redis_main/localhost/g' \
-       -e 's/enketo_redis_cache/localhost/g' \
-       -e 's/6380/6379/g' "$CONFIG_PATH.template"
+       -e 's/enketo_redis_cache/localhost/g' "$CONFIG_PATH.template"
 fi
 
 BASE_URL=$( [ "${HTTPS_PORT}" = 443 ] && echo https://"${DOMAIN}" || echo https://"${DOMAIN}":"${HTTPS_PORT}" ) \
diff --git a/files/postgres14/start-postgres.sh b/files/postgres14/start-postgres.sh
index 978f6c4c..972c6147 100755
--- a/files/postgres14/start-postgres.sh
+++ b/files/postgres14/start-postgres.sh
@@ -1,7 +1,7 @@
 #!/bin/bash -eu
 set -o pipefail
 
-flag_upgradeCompletedOk=$(readlink -f -m "$PGDATA/../.postgres14-upgrade-successful")
+flag_upgradeCompletedOk="$PGDATA/../.postgres14-upgrade-successful"
 
 logPrefix="$(basename "$0")"
 log() {

From 96967808881bf8692ea5340534ecf0295dcda472 Mon Sep 17 00:00:00 2001
From: Sadiq Khoja <sadiqkhoja@gmail.com>
Date: Fri, 6 Dec 2024 16:11:21 -0500
Subject: [PATCH 6/7] changed Makefile to start and stop with both docker
 compose files

---
 Makefile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index c020af70..dd192f47 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,9 @@
+docker-compose-dev := docker compose --profile central -f docker-compose.yml -f docker-compose.dev.yml
+
 .PHONY: dev
 dev:
-	docker compose --profile central -f docker-compose.yml -f docker-compose.dev.yml up -d
+	$(docker-compose-dev) up -d
 
 .PHONY: stop
 stop:
-	docker compose stop
+	$(docker-compose-dev) stop

From aa4df38308174455caaea77f0ffc42103c81e43e Mon Sep 17 00:00:00 2001
From: Sadiq Khoja <sadiqkhoja@gmail.com>
Date: Thu, 12 Dec 2024 23:50:55 -0500
Subject: [PATCH 7/7] Change the network mode from host to bridge

---
 README.md                    |  6 +++++-
 docker-compose.dev.yml       | 17 +++++++++--------
 files/enketo/start-enketo.sh |  5 -----
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/README.md b/README.md
index f2afb037..b45f2d90 100644
--- a/README.md
+++ b/README.md
@@ -28,7 +28,11 @@ In addition to the Backend and the Frontend, Central deploys services:
 * Central relies on [pyxform-http](https://github.com/getodk/pyxform-http) for converting Forms from XLSForm. It generally shouldn't be needed in development but can be run locally.
 * Central relies on [Enketo](https://github.com/enketo/enketo-express) for Web Form functionality. Enketo can be run locally and configured to work with Frontend and Backend in development by following [these instructions](https://github.com/getodk/central-frontend/blob/master/docs/enketo.md).
 
-If you want to work on Central codebase and don't want to setup dependent services like Postgresql, Enketo, etc manually then you can run `make dev`, which will start those services as Docker containers.
+If you want to work on Central codebase and don't want to setup dependent services like Postgresql, Enketo, etc manually then you can run `make dev`, which will start those services as Docker containers. With this setup you would need a local domain name, `central-dev` is a good choice. Following are the places where you need to add this:
+
+* Set `DOMAIN=central-dev` in .env file. 
+* Add an entry in your `/etc/hosts` file for `127.0.0.1 central-dev`.
+* Create `local.json` in central-backend directory and set value of default.env.domain to `http://central-dev:8989`
 
 Operations
 ----------
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 7f455bf9..94c6f42e 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -10,7 +10,8 @@ services:
   postgres14:
     profiles:
       - central
-    network_mode: host
+    ports:
+      - 5432:5432
   postgres:
     profiles:
       - central
@@ -28,9 +29,8 @@ services:
   pyxform:
     profiles:
       - central
-    # changing port here - can't use `ports` mapping with host networking
-    command: ["gunicorn", "--bind", "0.0.0.0:5001", "--workers", "5", "--timeout", "600", "--max-requests", "1", "--max-requests-jitter", "3", "main:app()"]
-    network_mode: host
+    ports:
+      - 5001:80
   secrets:
     profiles:
       - central
@@ -48,14 +48,15 @@ services:
       - enketo_redis_main
     environment:
       - ENV=DEV
-    network_mode: host
+    extra_hosts:
+    - "${DOMAIN}:host-gateway"
+    ports:
+      - 8005:8005
   enketo_redis_main:
     profiles:
       - central
-    network_mode: host
   enketo_redis_cache:
     profiles:
       - central
-    network_mode: host
 volumes:
-  dev_secrets:
\ No newline at end of file
+  dev_secrets:
diff --git a/files/enketo/start-enketo.sh b/files/enketo/start-enketo.sh
index 6b6d7055..b9a43ac0 100755
--- a/files/enketo/start-enketo.sh
+++ b/files/enketo/start-enketo.sh
@@ -3,11 +3,6 @@
 CONFIG_PATH=${ENKETO_SRC_DIR}/config/config.json
 echo "generating enketo configuration..."
 
-if [ "$ENV" = "DEV" ]; then
-    sed -i -e 's/enketo_redis_main/localhost/g' \
-       -e 's/enketo_redis_cache/localhost/g' "$CONFIG_PATH.template"
-fi
-
 BASE_URL=$( [ "${HTTPS_PORT}" = 443 ] && echo https://"${DOMAIN}" || echo https://"${DOMAIN}":"${HTTPS_PORT}" ) \
 SECRET=$(cat /etc/secrets/enketo-secret) \
 LESS_SECRET=$(cat /etc/secrets/enketo-less-secret) \