From 815f20bdffa642a3cef60a6212a264a36fed7c9b Mon Sep 17 00:00:00 2001 From: lruzicki <56487722+lruzicki@users.noreply.github.com> Date: Mon, 29 May 2023 11:21:02 +0200 Subject: [PATCH 01/15] OTC-1080: PK updated when serial was needed (#14) Co-authored-by: lruzicki --- database scripts/00_dump.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/database scripts/00_dump.sql b/database scripts/00_dump.sql index 0362cdc..06c0254 100644 --- a/database scripts/00_dump.sql +++ b/database scripts/00_dump.sql @@ -3844,7 +3844,7 @@ ALTER SEQUENCE "public"."tblPayer_PayerID_seq" OWNED BY "public"."tblPayer"."Pay -- CREATE TABLE "public"."tblPayment" ( - "PaymentID" bigint NOT NULL, + "PaymentID" bigserial NOT NULL, "PaymentUUID" "uuid" NOT NULL, "ExpectedAmount" numeric(18,2), "ReceivedAmount" numeric(18,2), @@ -3883,7 +3883,7 @@ CREATE TABLE "public"."tblPayment" ( -- CREATE TABLE "public"."tblPaymentDetails" ( - "PaymentDetailsID" bigint NOT NULL, + "PaymentDetailsID" bigserial NOT NULL, "PaymentID" bigint NOT NULL, "ProductCode" character varying(8), "InsuranceNumber" character varying(12), @@ -3987,7 +3987,7 @@ CREATE TABLE "public"."tblPolicy" ( -- CREATE TABLE "public"."tblPolicyRenewalDetails" ( - "RenewalDetailID" integer NOT NULL, + "RenewalDetailID" SERIAL NOT NULL, "RenewalID" integer NOT NULL, "InsureeID" integer NOT NULL, "ValidityFrom" timestamp with time zone NOT NULL, @@ -4005,7 +4005,7 @@ CREATE TABLE "public"."tblPolicyRenewalDetails" ( -- CREATE TABLE "public"."tblPolicyRenewals" ( - "RenewalID" integer NOT NULL, + "RenewalID" SERIAL NOT NULL, "RenewalPromptDate" "date" NOT NULL, "RenewalDate" "date" NOT NULL, "NewOfficerID" integer, From 4941c2480480a744a56f83b18b566088ceb493c1 Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Thu, 22 Jun 2023 16:15:33 +0200 Subject: [PATCH 02/15] Added jsonb schema extension installation --- Dockerfile | 6 ++++++ install_postgres_json_schema_extension.sh | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 install_postgres_json_schema_extension.sh diff --git a/Dockerfile b/Dockerfile index 38b357d..4a4feed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,3 +3,9 @@ FROM postgres:13-alpine # Script to detect whether the database has finished initializing COPY ["true_isready.sh", "/usr/local/bin/"] COPY ["database scripts/00_dump.sql", "database scripts/0[2345]_*.sql", "database scripts/demo_db.sql", "/docker-entrypoint-initdb.d/"] + +# Install https://github.com/gavinwahl/postgres-json-schema/ +# extension that allows validation of jsonb fields against jsonschema +COPY ["install_postgres_json_schema_extension.sh", "install_postgres_json_schema_extension.sh"] +RUN chmod u+x install_postgres_json_schema_extension.sh +RUN ./install_postgres_json_schema_extension.sh diff --git a/install_postgres_json_schema_extension.sh b/install_postgres_json_schema_extension.sh new file mode 100644 index 0000000..7db5317 --- /dev/null +++ b/install_postgres_json_schema_extension.sh @@ -0,0 +1,20 @@ +#!/bin/bash +apk add --update apt +apk add --update make +apk add --update git + +# Clone the repository +git clone https://github.com/gavinwahl/postgres-json-schema/ + +apt install libpq-dev -y +# Move into the directory +cd postgres-json-schema + +# Build the extension +make & make install + + +apt install postgresql-client -y + +/bin/mkdir -p '/usr/local/share/postgresql/extension' + From 6bd35f5b2f9d077b50c9c116f28102c710ef3111 Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Fri, 23 Jun 2023 10:31:38 +0200 Subject: [PATCH 03/15] Change apt to apk --- install_postgres_json_schema_extension.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/install_postgres_json_schema_extension.sh b/install_postgres_json_schema_extension.sh index 7db5317..46fba01 100644 --- a/install_postgres_json_schema_extension.sh +++ b/install_postgres_json_schema_extension.sh @@ -6,7 +6,7 @@ apk add --update git # Clone the repository git clone https://github.com/gavinwahl/postgres-json-schema/ -apt install libpq-dev -y +apk install libpq-dev # Move into the directory cd postgres-json-schema @@ -14,7 +14,11 @@ cd postgres-json-schema make & make install -apt install postgresql-client -y +apk install postgresql-client /bin/mkdir -p '/usr/local/share/postgresql/extension' +chmod 666 /usr/local/share/postgresql/extension/postgres-json-schema.control +# Add extension to PostgreSQL +# Assuming you're operating in the psql shell +psql -U $POSTGRES_USER -d $POSTGRES_DB -c "CREATE EXTENSION \"postgres-json-schema\";" From 76a71386d4be9438f532ac56b98f9061402ca168 Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Mon, 26 Jun 2023 12:24:06 +0200 Subject: [PATCH 04/15] Fixed apk add command --- install_postgres_json_schema_extension.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/install_postgres_json_schema_extension.sh b/install_postgres_json_schema_extension.sh index 46fba01..e8d6220 100644 --- a/install_postgres_json_schema_extension.sh +++ b/install_postgres_json_schema_extension.sh @@ -1,12 +1,11 @@ #!/bin/bash -apk add --update apt apk add --update make apk add --update git # Clone the repository git clone https://github.com/gavinwahl/postgres-json-schema/ -apk install libpq-dev +apk add libpq-dev # Move into the directory cd postgres-json-schema @@ -14,7 +13,7 @@ cd postgres-json-schema make & make install -apk install postgresql-client +apk add postgresql-client /bin/mkdir -p '/usr/local/share/postgresql/extension' From c0d1e639439637efb7d3072358c093e3b04a9a6a Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Mon, 17 Jul 2023 16:35:48 +0200 Subject: [PATCH 05/15] CM-235: Added CD for develop image of database --- .github/workflows/docker-dev-cd.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/docker-dev-cd.yml diff --git a/.github/workflows/docker-dev-cd.yml b/.github/workflows/docker-dev-cd.yml new file mode 100644 index 0000000..758258d --- /dev/null +++ b/.github/workflows/docker-dev-cd.yml @@ -0,0 +1,23 @@ +name: publish develop image +on: + push: + branches: + - develop + +jobs: + publish-docker-image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push develop Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: openimis/openimis-pgsql:develop From 696bb24bf824ed4d12010f96012c6de3e6c5bf5a Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Tue, 18 Jul 2023 11:03:00 +0200 Subject: [PATCH 06/15] CM-235: Fixed json_schema extension --- Dockerfile | 2 +- database scripts/json_schema_extension.sql | 1 + install_postgres_json_schema_extension.sh | 3 --- 3 files changed, 2 insertions(+), 4 deletions(-) create mode 100644 database scripts/json_schema_extension.sql diff --git a/Dockerfile b/Dockerfile index 4a4feed..6e2eafe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM postgres:13-alpine # Script to detect whether the database has finished initializing COPY ["true_isready.sh", "/usr/local/bin/"] -COPY ["database scripts/00_dump.sql", "database scripts/0[2345]_*.sql", "database scripts/demo_db.sql", "/docker-entrypoint-initdb.d/"] +COPY ["database scripts/00_dump.sql", "database scripts/0[2345]_*.sql", "database scripts/demo_db.sql", "database scripts/json_schema_extension.sql", "/docker-entrypoint-initdb.d/"] # Install https://github.com/gavinwahl/postgres-json-schema/ # extension that allows validation of jsonb fields against jsonschema diff --git a/database scripts/json_schema_extension.sql b/database scripts/json_schema_extension.sql new file mode 100644 index 0000000..54335c5 --- /dev/null +++ b/database scripts/json_schema_extension.sql @@ -0,0 +1 @@ +CREATE EXTENSION "postgres-json-schema"; \ No newline at end of file diff --git a/install_postgres_json_schema_extension.sh b/install_postgres_json_schema_extension.sh index e8d6220..38889d0 100644 --- a/install_postgres_json_schema_extension.sh +++ b/install_postgres_json_schema_extension.sh @@ -18,6 +18,3 @@ apk add postgresql-client /bin/mkdir -p '/usr/local/share/postgresql/extension' chmod 666 /usr/local/share/postgresql/extension/postgres-json-schema.control -# Add extension to PostgreSQL -# Assuming you're operating in the psql shell -psql -U $POSTGRES_USER -d $POSTGRES_DB -c "CREATE EXTENSION \"postgres-json-schema\";" From d404f15d0f5f5c0465b35631d7bc312424da7c25 Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Tue, 18 Jul 2023 11:05:56 +0200 Subject: [PATCH 07/15] CM-235: Added CI check for docker image build --- .github/workflows/docker-build-check.yaml | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/docker-build-check.yaml diff --git a/.github/workflows/docker-build-check.yaml b/.github/workflows/docker-build-check.yaml new file mode 100644 index 0000000..f70793c --- /dev/null +++ b/.github/workflows/docker-build-check.yaml @@ -0,0 +1,29 @@ +name: Docker Build Check + +on: + pull_request: + branches: + - main + - develop + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Build Docker image + run: docker build -t : . + + - name: Check Docker image build status + run: | + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "Docker image build failed." + exit 1 + else + echo "Docker image build successful." + exit 0 + fi \ No newline at end of file From f32ccb5914ac564412a825c09378016211ef03d5 Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Tue, 18 Jul 2023 11:25:59 +0200 Subject: [PATCH 08/15] CM-235: Fixed build config --- .github/workflows/docker-build-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-check.yaml b/.github/workflows/docker-build-check.yaml index f70793c..3a6c296 100644 --- a/.github/workflows/docker-build-check.yaml +++ b/.github/workflows/docker-build-check.yaml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v2 - name: Build Docker image - run: docker build -t : . + run: docker build -t . - name: Check Docker image build status run: | From 8b7966bfe7678e456ba2446a6ae8bd722e5a71eb Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Tue, 18 Jul 2023 11:26:49 +0200 Subject: [PATCH 09/15] CM-235: Fixed build config --- .github/workflows/docker-build-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-check.yaml b/.github/workflows/docker-build-check.yaml index 3a6c296..8b05a27 100644 --- a/.github/workflows/docker-build-check.yaml +++ b/.github/workflows/docker-build-check.yaml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v2 - name: Build Docker image - run: docker build -t . + run: docker build . - name: Check Docker image build status run: | From 06aa315717979008dfa7058fcefbe23e9c7dce19 Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Tue, 18 Jul 2023 11:39:36 +0200 Subject: [PATCH 10/15] CM-235: Changed build and push method of CD to one used by manual trigger --- .github/workflows/docker-dev-cd.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-dev-cd.yml b/.github/workflows/docker-dev-cd.yml index 758258d..c1a7348 100644 --- a/.github/workflows/docker-dev-cd.yml +++ b/.github/workflows/docker-dev-cd.yml @@ -15,9 +15,7 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push develop Docker image - uses: docker/build-push-action@v2 - with: - context: . - push: true - tags: openimis/openimis-pgsql:develop + - name: Build the pgsql Docker image + run: | + docker build . --tag ghcr.io/openimis/openimis-pgsql:develop + docker push ghcr.io/openimis/openimis-pgsql:develop \ No newline at end of file From 2f365a3fb8140700bfee805f397f679428ebd44f Mon Sep 17 00:00:00 2001 From: Patrick Delcroix Date: Thu, 31 Aug 2023 10:03:27 +0200 Subject: [PATCH 11/15] Add admin user to empty (#19) --- database scripts/00_dump.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/database scripts/00_dump.sql b/database scripts/00_dump.sql index 06c0254..259de47 100644 --- a/database scripts/00_dump.sql +++ b/database scripts/00_dump.sql @@ -4755,6 +4755,7 @@ CREATE TABLE "public"."tblUsers" ( "password" "bytea" ); +INSERT INTO "tblUsers" ("UserID", "LanguageID", "LastName", "OtherNames", "Phone", "LoginName", "RoleID", "HFID", "ValidityFrom", "ValidityTo", "LegacyID", "AuditUserID", "password", "DummyPwd", "EmailId", "StoredPassword", "PrivateKey", "PasswordValidity", "IsAssociated", "UserUUID") VALUES (1, N'en', N'Admin', N'Admin', N'', N'Admin', 1023, NULL, CAST(N'2018-03-19T08:54:48.923' AS timestamptz), NULL, NULL, 2, 'x001699E55A06FA79F4CA0D06EF15096C02000000DF691E2CE66AA7ABDF65B3E6210C1C04CAAE1A3B1FEE5E266B5FAF4F7D4E95109C92E3205F0145CC'::bytea, NULL, N'test@testing.com', N'59E66831C680C19E8736751D5480A7C3291BD8775DF47C19C4D0361FBC1C3438', N'C1C224B03CD9BC7B6A86D77F5DACE40191766C485CD55DC48CAF9AC873335D6F', NULL, NULL, N'281953c0-1c1a-4cf8-b93c-8e8b33c617c2'); -- ALTER TABLE "public"."tblUsers" OWNER TO "postgres"; From 766cd5b980548b071049ea6138328908df704c79 Mon Sep 17 00:00:00 2001 From: Kamil Malinowski Date: Mon, 4 Sep 2023 10:00:57 +0200 Subject: [PATCH 12/15] Empty db fix (#20) * Fixed admin user for empty db * Moved en to empty db * Removed admin from demo script --- database scripts/00_dump.sql | 4 +++- database scripts/demo_db.sql | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/database scripts/00_dump.sql b/database scripts/00_dump.sql index 259de47..ee5d184 100644 --- a/database scripts/00_dump.sql +++ b/database scripts/00_dump.sql @@ -3439,6 +3439,8 @@ CREATE TABLE "public"."tblLanguages" ( "CountryCode" character varying(10) NULL ); +INSERT INTO "public"."tblLanguages" ("LanguageCode", "LanguageName", "SortOrder") VALUES (N'en', N'English', NULL); + -- ALTER TABLE "public"."tblLanguages" OWNER TO "postgres"; @@ -4755,7 +4757,7 @@ CREATE TABLE "public"."tblUsers" ( "password" "bytea" ); -INSERT INTO "tblUsers" ("UserID", "LanguageID", "LastName", "OtherNames", "Phone", "LoginName", "RoleID", "HFID", "ValidityFrom", "ValidityTo", "LegacyID", "AuditUserID", "password", "DummyPwd", "EmailId", "StoredPassword", "PrivateKey", "PasswordValidity", "IsAssociated", "UserUUID") VALUES (1, N'en', N'Admin', N'Admin', N'', N'Admin', 1023, NULL, CAST(N'2018-03-19T08:54:48.923' AS timestamptz), NULL, NULL, 2, 'x001699E55A06FA79F4CA0D06EF15096C02000000DF691E2CE66AA7ABDF65B3E6210C1C04CAAE1A3B1FEE5E266B5FAF4F7D4E95109C92E3205F0145CC'::bytea, NULL, N'test@testing.com', N'59E66831C680C19E8736751D5480A7C3291BD8775DF47C19C4D0361FBC1C3438', N'C1C224B03CD9BC7B6A86D77F5DACE40191766C485CD55DC48CAF9AC873335D6F', NULL, NULL, N'281953c0-1c1a-4cf8-b93c-8e8b33c617c2'); +INSERT INTO "public"."tblUsers" ("UserID", "LanguageID", "LastName", "OtherNames", "Phone", "LoginName", "RoleID", "HFID", "ValidityFrom", "ValidityTo", "LegacyID", "AuditUserID", "password", "DummyPwd", "EmailId", "StoredPassword", "PrivateKey", "PasswordValidity", "IsAssociated", "UserUUID") VALUES (1, N'en', N'Admin', N'Admin', N'', N'Admin', 1023, NULL, CAST(N'2018-03-19T08:54:48.923' AS timestamptz), NULL, NULL, 2, 'x001699E55A06FA79F4CA0D06EF15096C02000000DF691E2CE66AA7ABDF65B3E6210C1C04CAAE1A3B1FEE5E266B5FAF4F7D4E95109C92E3205F0145CC'::bytea, NULL, N'test@testing.com', N'59E66831C680C19E8736751D5480A7C3291BD8775DF47C19C4D0361FBC1C3438', N'C1C224B03CD9BC7B6A86D77F5DACE40191766C485CD55DC48CAF9AC873335D6F', NULL, NULL, N'281953c0-1c1a-4cf8-b93c-8e8b33c617c2'); -- ALTER TABLE "public"."tblUsers" OWNER TO "postgres"; diff --git a/database scripts/demo_db.sql b/database scripts/demo_db.sql index 2d89339..7f26b77 100644 --- a/database scripts/demo_db.sql +++ b/database scripts/demo_db.sql @@ -1,8 +1,7 @@ set search_path to public; --- tblLanguages -INSERT INTO "tblLanguages" ("LanguageCode", "LanguageName", "SortOrder") VALUES (N'en', N'English', NULL); +-- tblLanguages (en added in empty db) INSERT INTO "tblLanguages" ("LanguageCode", "LanguageName", "SortOrder") VALUES (N'fr', N'Français', NULL); -- tblGender @@ -1842,7 +1841,6 @@ INSERT INTO "tblExtracts" ("ExtractID", "ExtractDirection", "ExtractType", "Extr INSERT INTO "tblExtracts" ("ExtractID", "ExtractDirection", "ExtractType", "ExtractSequence", "ExtractDate", "ExtractFileName", "ExtractFolder", "LocationId", "HFID", "AppVersionBackend", "ValidityFrom", "ValidityTo", "LegacyID", "AuditUserID", "RowID", "ExtractUUID") VALUES (7, 0, 1, 3, CAST(N'2019-01-28T08:36:39.410' AS timestamptz), N'ImisData.db3', N'', 17, 0, CAST(1.2 AS Decimal(3, 1)), CAST(N'2019-01-28T08:36:39.410' AS timestamptz), NULL, NULL, 1, 0, N'7ed9f259-c742-41e7-af22-b5d2bc2d4cf5'::uuid); -- tblUsers -INSERT INTO "tblUsers" ("UserID", "LanguageID", "LastName", "OtherNames", "Phone", "LoginName", "RoleID", "HFID", "ValidityFrom", "ValidityTo", "LegacyID", "AuditUserID", "password", "DummyPwd", "EmailId", "StoredPassword", "PrivateKey", "PasswordValidity", "IsAssociated", "UserUUID") VALUES (1, N'en', N'Admin', N'Admin', N'', N'Admin', 1023, NULL, CAST(N'2018-03-19T08:54:48.923' AS timestamptz), NULL, NULL, 2, 'x001699E55A06FA79F4CA0D06EF15096C02000000DF691E2CE66AA7ABDF65B3E6210C1C04CAAE1A3B1FEE5E266B5FAF4F7D4E95109C92E3205F0145CC'::bytea, NULL, N'test@testing.com', N'59E66831C680C19E8736751D5480A7C3291BD8775DF47C19C4D0361FBC1C3438', N'C1C224B03CD9BC7B6A86D77F5DACE40191766C485CD55DC48CAF9AC873335D6F', NULL, NULL, N'281953c0-1c1a-4cf8-b93c-8e8b33c617c2'); INSERT INTO "tblUsers" ("UserID", "LanguageID", "LastName", "OtherNames", "Phone", "LoginName", "RoleID", "HFID", "ValidityFrom", "ValidityTo", "LegacyID", "AuditUserID", "password", "DummyPwd", "EmailId", "StoredPassword", "PrivateKey", "PasswordValidity", "IsAssociated", "UserUUID") VALUES (2, N'en', N'Srivastava', N'Siddharth', N'', N'siddharth', 1023, 0, CAST(N'2019-05-10T05:13:34.897' AS timestamptz), CAST(N'2019-05-10T05:13:34.897' AS timestamptz), NULL, 1, 'x001699E55A06FA79F4CA0D06EF15096C020000002C9B22886522612FD8687D520E820178EC956FE690415250829FC35042E4C253EDB00622B818C321578B877F24B6BED5'::bytea, NULL, N'siddharth.srivastava@swisstph.ch', N'568BD443C04F9B2BB7EB8089213B868E2865623F140BC25A77239169B24C7C1B', N'7BF4FB3F97CD903A9AF16BA419A1A3947FFA293371E2BCB1B868EE6E4BAF3AEC', NULL, NULL, N'326b5d5c-5630-4b10-bda3-b7334034ad61'); INSERT INTO "tblUsers" ("UserID", "LanguageID", "LastName", "OtherNames", "Phone", "LoginName", "RoleID", "HFID", "ValidityFrom", "ValidityTo", "LegacyID", "AuditUserID", "password", "DummyPwd", "EmailId", "StoredPassword", "PrivateKey", "PasswordValidity", "IsAssociated", "UserUUID") VALUES (3, N'en', N'Admin', N'Admin', N'', N'Admin', 1023, 0, CAST(N'2017-12-16T14:26:37.710' AS timestamptz), CAST(N'2018-03-19T08:54:48.907' AS timestamptz), 1, 0, 'x001699E55A06FA79F4CA0D06EF15096C01000000AABB2D46EA963BEEBEC2B6A7C0195BEAD56D875BF16D75F92DD047EAE987E4CC'::bytea, NULL, NULL, NULL, NULL, NULL, NULL, N'5bcc1389-a47b-464c-a32a-7a60d9fd1cad'); INSERT INTO "tblUsers" ("UserID", "LanguageID", "LastName", "OtherNames", "Phone", "LoginName", "RoleID", "HFID", "ValidityFrom", "ValidityTo", "LegacyID", "AuditUserID", "password", "DummyPwd", "EmailId", "StoredPassword", "PrivateKey", "PasswordValidity", "IsAssociated", "UserUUID") VALUES (4, N'en', N'sri', N'sid', N'', N'sidsri', 1, 0, CAST(N'2019-05-10T05:13:38.223' AS timestamptz), CAST(N'2019-05-10T05:13:38.223' AS timestamptz), NULL, 1, 'x001699E55A06FA79F4CA0D06EF15096C02000000D7925623C394F680CF9A1BD06ED155960019C8567100E8A869147E30E6353296FACCD73063C0D08B'::bytea, NULL, N'siddharth.srivastava@swisstph.ch', N'6BA12D6E86F3B6FFBEBECD7E25D156B04B474F707B25B09A822F688B99426F5B', N'D9193C4A617CEE0AAEA870AAEC59FBE9B90A59A16319142CC55FF6FE2A0D6C57', NULL, NULL, N'e3d5ed51-fdf6-4755-a6a2-9d8012abac51'); From 63766daa66439e91db924634054a8d8c6aa5aa11 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 18 Sep 2023 11:42:25 +0200 Subject: [PATCH 13/15] OP-1538: update dockerfile (#21) * OP-1538: update dockerfile * OP-1538: change as to upper case --------- Co-authored-by: Jan --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6e2eafe..6e72e74 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,14 @@ -FROM postgres:13-alpine +FROM postgres:13-alpine AS base # Script to detect whether the database has finished initializing COPY ["true_isready.sh", "/usr/local/bin/"] -COPY ["database scripts/00_dump.sql", "database scripts/0[2345]_*.sql", "database scripts/demo_db.sql", "database scripts/json_schema_extension.sql", "/docker-entrypoint-initdb.d/"] +COPY ["database scripts/00_dump.sql", "database scripts/0[2345]_*.sql", "database scripts/json_schema_extension.sql", "/docker-entrypoint-initdb.d/"] # Install https://github.com/gavinwahl/postgres-json-schema/ # extension that allows validation of jsonb fields against jsonschema COPY ["install_postgres_json_schema_extension.sh", "install_postgres_json_schema_extension.sh"] RUN chmod u+x install_postgres_json_schema_extension.sh RUN ./install_postgres_json_schema_extension.sh + +FROM base AS demo +COPY ["database scripts/demo_db.sql", "/docker-entrypoint-initdb.d/"] From c4411ec482e8d833c9a98ca849c5e48a0958c86a Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Thu, 23 Nov 2023 09:28:50 +0100 Subject: [PATCH 14/15] Split demo and base images + cleanup migtool (#22) --- .github/workflows/docker-dev-cd.yml | 8 +- .github/workflows/docker.yml | 6 +- migtool/migtool.py | 111 ++++++++++++++++++---------- 3 files changed, 82 insertions(+), 43 deletions(-) diff --git a/.github/workflows/docker-dev-cd.yml b/.github/workflows/docker-dev-cd.yml index c1a7348..f4e5fc2 100644 --- a/.github/workflows/docker-dev-cd.yml +++ b/.github/workflows/docker-dev-cd.yml @@ -15,7 +15,11 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build the pgsql Docker image + - name: Build the pgsql EMPTY Docker image + run: | + docker build . --tag ghcr.io/openimis/openimis-pgsql:develop-base --target base + docker push ghcr.io/openimis/openimis-pgsql:develop-base + - name: Build the pgsql DEMO Docker image run: | docker build . --tag ghcr.io/openimis/openimis-pgsql:develop - docker push ghcr.io/openimis/openimis-pgsql:develop \ No newline at end of file + docker push ghcr.io/openimis/openimis-pgsql:develop diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index af4a318..52ca2ee 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,11 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build the pgsql Docker image + - name: Build the pgsql EMPTY Docker image + run: | + docker build . --target base --tag ghcr.io/openimis/openimis-pgsql:${{ github.event.inputs.tag }}-base + docker push ghcr.io/openimis/openimis-pgsql:${{ github.event.inputs.tag }}-base + - name: Build the pgsql DEMO Docker image run: | docker build . --tag ghcr.io/openimis/openimis-pgsql:${{ github.event.inputs.tag }} docker push ghcr.io/openimis/openimis-pgsql:${{ github.event.inputs.tag }} diff --git a/migtool/migtool.py b/migtool/migtool.py index 85db04b..0110505 100644 --- a/migtool/migtool.py +++ b/migtool/migtool.py @@ -1,9 +1,24 @@ +import re + import pyodbc # adapter for SQL Server import psycopg2 # adapter for PostgreSQL import configparser # used to read settings from file import datetime # used to properly format dates and datetimes import time # used to calculate time taken +# This script was created with global variables without initializing them here. Ideally, we should store a settings +# object rather than all of them separately but this works. +settings = None +EXCLUDED_COLUMNS = ["RowID"] +delete_data = False +historical = False +demo_fix = False +migration_modules = [] +old_connection = None +new_connection = None +today = datetime.date.today() +now = datetime.datetime.now() + # loads connection configuration and migration settings from a file. # In future the settings file could be specified with a parameter. @@ -29,10 +44,6 @@ def get_settings_from_file(): print(" Histrical data will be migrated to the new database.") else: print(" No historical data will be migrated.") - global today - global now - today = datetime.date.today() - now = datetime.datetime.now() except KeyError as e: print("\x1b[0;31;48m" + "Error while trying to load settings. " +\ "Please make sure the settings.ini file exists in your working directory." + "\x1b[0m") @@ -50,8 +61,9 @@ def connect(): global old_connection old_connection = pyodbc.connect(old_connection_string) except pyodbc.InterfaceError as exc: - print("\x1b[0;31;48m" + - "ERROR: Could not connect to the SQL Server database. Make sure the server is running and check your settings." + + print("\x1b[0;31;48m" + "ERROR: Could not connect to the SQL Server database. " + "Make sure the server is running and check your settings." "\x1b[0m") print(exc) exit(1) @@ -61,13 +73,15 @@ def connect(): new_db = settings["NewDB"] new_connection_string = f'host={new_db["host"]} port={new_db["port"]} dbname={new_db["name"]} ' \ f'user={new_db["user"]} password={new_db["pwd"]}' - new_connection_string = f'postgres://{new_db["user"]}@{new_db["host"]}:{new_db["port"]}/{new_db["name"]}' + # new_connection_string = \ + # f'postgres://{new_db["user"]}:{new_db["pwd"]}@{new_db["host"]}:{new_db["port"]}/{new_db["name"]}' try: global new_connection new_connection = psycopg2.connect(new_connection_string) except psycopg2.OperationalError as exc: - print("\x1b[0;31;48m" + - "ERROR: Could not connect to the PostgreSQL database. Make sure the server is running and check your settings." + + print("\x1b[0;31;48m" + "ERROR: Could not connect to the PostgreSQL database. " + "Make sure the server is running and check your settings." "\x1b[0m") print(exc) exit(1) @@ -108,43 +122,35 @@ def get_db_tables(): print("Finding tables in both databases.\n") old_cursor.execute("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE';") new_cursor.execute("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public';") - old_tables = list() - for x in old_cursor: - # Remove special characters at the start and end of each item when adding it to the list. - # This way the entries in the old and new list match - old_tables.append(str(x)[2:-4]) - new_tables = list() - for x in new_cursor: - # Remove special characters at the start and end of each item when adding it to the list. - # This way the entries in the old and new list match - new_tables.append(str(x)[2:-3]) + old_tables = [x[0] for x in old_cursor] + new_tables = [x[0] for x in new_cursor] return old_tables, new_tables # This function puts the data from a SELECT statement into string and formats it correctly so that postgres can work # with it. def generate_insertion_string(row): - row_str = "(" + row_list = [] for x in row: # Strings must be enclosed in apostrophes, also escape singe quotes in a string by doubling them if isinstance(x, str): - row_str = row_str + "'" + str(x).replace("'", "''") + "', " + row_list.append("'" + str(x).replace("'", "''") + "'") # Dates and datetimes must be enclosed in apostrophes elif isinstance(x, datetime.datetime) or isinstance(x, datetime.date): - row_str = row_str + "'" + str(x) + "', " - # If x is NoneType then str(x) get transtlated to "None", but sql wants "null" + row_list.append("'" + str(x) + "'") + # If x is NoneType then str(x) get translated to "None", but sql wants "null" elif x is None: - row_str = row_str + "null, " + row_list.append("null") # If x is bytes we need to make them nice (start with \x and append the data converted to hex): elif isinstance(x, bytes): - row_str = row_str + "'\\x" + str(x.hex()) + "', " + row_list.append("'\\x" + str(x.hex()) + "'") else: - row_str = row_str + str(x) + ", " - row_str = row_str[:-2] + ")" + row_list.append(str(x)) + row_str = f"({', '.join(row_list)})" return row_str -# When not migrating historical data, this function figures out what colums "ValidityTo" is so we can later check for +# When not migrating historical data, this function figures out what columns "ValidityTo" is so we can later check for # each row if it is still valid or already historical def get_validity_index(rows): vi = -1 @@ -164,6 +170,8 @@ def get_validity_index(rows): def get_validity(vi, row): + global today + global now if historical or ((not historical) and vi == -1): return True elif (not historical) and vi != -1: @@ -185,6 +193,17 @@ def get_validity(vi, row): return True +def extract_sequence_name(column_default): + if not column_default: + return None + pattern = r"nextval\('([^']*)" + match = re.search(pattern, column_default) + if match: + return match.group(1) + else: + return None + + def migrate(): # This list collects all db tables that exist only in one of the databases but not the other. lonely_tables = list() @@ -206,7 +225,8 @@ def migrate(): "\"FeedbackUUID\", \"AuditUserID\") VALUES ('2000 01 01 00:00:00.000000', 0, 0, 0);") # Set up all the columns we're going to migrate. - new_cursor.execute("SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '" + table + "';") + new_cursor.execute("SELECT COLUMN_NAME, COLUMN_DEFAULT " + "FROM information_schema.COLUMNS WHERE TABLE_NAME = '" + table + "';") rows = new_cursor.fetchall() # While we have the data ready: find out where dates are stored for historical data stuff. validity_index # stores in which column the date (ValidityTo) is stored @@ -214,18 +234,24 @@ def migrate(): if not historical: validity_index = get_validity_index(rows) # Finally, set up the columns to migrate - old_cols = "" - new_cols = "(" + sequence_columns = {} + old_cols_list = [] + new_cols_list = [] for row in rows: - old_cols = old_cols + str(row)[2:-3] + ", " - new_cols = new_cols + "\"" + str(row)[2:-3] + "\", " - old_cols = old_cols[:-2] - new_cols = new_cols[:-2] + ")" + if row[0] not in EXCLUDED_COLUMNS: + col_default = extract_sequence_name(row[1]) + if col_default: + sequence_columns[row[0]] = col_default + old_cols_list.append(row[0]) + new_cols_list.append(f'"{row[0]}"') + old_cols = ", ".join(old_cols_list) + new_cols = "(" + ", ".join(new_cols_list) + ")" # Get the data from the old db with these column specifications print(" Fetching data from old database.") old_cursor.execute("SELECT COUNT(*) FROM " + table + ";") - print(" Found " + str(old_cursor.fetchone())[1:-3] + " entries.") + print(f" Found {old_cursor.fetchone()[0]} entries.") + print(f" == old_cols: {old_cols} from {table} ==") old_cursor.execute("SELECT " + old_cols + " FROM " + table + ";") # Set up the values for the insert statement and execute @@ -247,14 +273,19 @@ def migrate(): # Not rolling back leads to an InFailedSqlTransaction exception. new_connection.rollback() pass - + except Exception as e: + print("Failed: INSERT INTO \"" + table + "\" " + new_cols + " VALUES " + row_str + ";") + raise + if sequence_columns: + print(" Data transferred, updating sequences.") + for column, sequence in sequence_columns.items(): + new_cursor.execute(f"select setval('{sequence}', max(\"{column}\")) from \"{table}\";") print(" Table " + table + " has been migrated.\n") # Table doesn't exist else: - print("\x1b[0;31;48m" + "WARNING: Table " + table + \ - " only exists in one of the databases (but not the other)! Is this correct?" + "\x1b[0m\n") - print("") + print(f"\x1b[0;31;48mWARNING: Table {table} only exists in one of the databases " + f"new: {table in new_tables}, old:{table in old_tables})! Is this correct?\x1b[0m\n") lonely_tables.append(table) # Print all tables that have not been migrated due to missing schemas: From 0e418c2dbfd59b547372db7b7abaa23683492e04 Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Wed, 20 Dec 2023 18:32:27 +0100 Subject: [PATCH 15/15] A few fixes to migtool uuid & common list of cols The MS webapp used to create uppercase UUIDs. Most setups won't experience it because they used the hybrid version but older setups might (like Niger). We tried to detect the data type of DB columns but it's a mess. Some are of `uniqueidentifier` type but others are plain char(32) or char(36). So we resorted to adding a check in the string formatting itself. Additionally, the tool was only based on the new DB column list, so in tblLanguages, a country_code appeared in Postgres, causing a missing column error on the source. We now take the common denominator between the old and new columns. --- migtool/migtool.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/migtool/migtool.py b/migtool/migtool.py index 0110505..e00d4c8 100644 --- a/migtool/migtool.py +++ b/migtool/migtool.py @@ -1,4 +1,5 @@ import re +import uuid import pyodbc # adapter for SQL Server import psycopg2 # adapter for PostgreSQL @@ -50,6 +51,14 @@ def get_settings_from_file(): exit(1) +def is_uuid(value): + try: + uuid.UUID(value, version=4) + return True + except ValueError: + return False + + # tries to connect to both databases def connect(): print("Setting up connection to the databases:") @@ -134,6 +143,9 @@ def generate_insertion_string(row): for x in row: # Strings must be enclosed in apostrophes, also escape singe quotes in a string by doubling them if isinstance(x, str): + # The .NET webapp used to create uppercase UUIDs, so we try to detect it and lowercase it + if 32 <= len(x) <= 36 and is_uuid(x): + x = x.lower() row_list.append("'" + str(x).replace("'", "''") + "'") # Dates and datetimes must be enclosed in apostrophes elif isinstance(x, datetime.datetime) or isinstance(x, datetime.date): @@ -225,6 +237,9 @@ def migrate(): "\"FeedbackUUID\", \"AuditUserID\") VALUES ('2000 01 01 00:00:00.000000', 0, 0, 0);") # Set up all the columns we're going to migrate. + cursor = old_cursor.execute("SELECT TOP 1 * FROM " + table + ";") + old_columns_with_types = {column[0].lower(): column[1] for column in cursor.description} + new_cursor.execute("SELECT COLUMN_NAME, COLUMN_DEFAULT " "FROM information_schema.COLUMNS WHERE TABLE_NAME = '" + table + "';") rows = new_cursor.fetchall() @@ -238,7 +253,7 @@ def migrate(): old_cols_list = [] new_cols_list = [] for row in rows: - if row[0] not in EXCLUDED_COLUMNS: + if row[0] not in EXCLUDED_COLUMNS and row[0].lower() in old_columns_with_types: col_default = extract_sequence_name(row[1]) if col_default: sequence_columns[row[0]] = col_default