From e6a39e7aee691773b34f6d10a2a546682d19095e Mon Sep 17 00:00:00 2001 From: Miikka Kallio Date: Wed, 13 Mar 2024 06:47:06 +0200 Subject: [PATCH 1/5] Changed .bat files to read database connection from .env-file. -Changed .bat files to read database connection from .env-file -Changed conf.py to read the .env file from specified location. --- jkrimporter/conf.py | 7 +++- scripts/drop_schemas.bat | 35 ++++++++++++++--- scripts/import_and_create_kohteet.bat | 56 +++++++++++++++++++-------- scripts/import_dvv.bat | 46 +++++++++++++++++----- scripts/import_posti.bat | 35 ++++++++++++++--- scripts/import_taajama.bat | 41 +++++++++++++++----- scripts/update_dvv.bat | 43 +++++++++++++++----- scripts/update_huoneistomaara.bat | 36 +++++++++++++---- 8 files changed, 234 insertions(+), 65 deletions(-) diff --git a/jkrimporter/conf.py b/jkrimporter/conf.py index 8d651065..680fb4d3 100644 --- a/jkrimporter/conf.py +++ b/jkrimporter/conf.py @@ -2,11 +2,16 @@ from dotenv import dotenv_values +# Path to the .env file in the user's %APPDATA%/jkr directory +dotenv_path = os.path.join(os.getenv('APPDATA'), 'jkr', '.env') + +# Read the environment variables from the .env file env = { - **dotenv_values(".env"), + **dotenv_values(dotenv_path), **os.environ, } +# Define database configuration using environment variables dbconf = { "host": env.get("JKR_DB_HOST", None), "port": env.get("JKR_DB_PORT", None), diff --git a/scripts/drop_schemas.bat b/scripts/drop_schemas.bat index 781bc672..1ed96a64 100644 --- a/scripts/drop_schemas.bat +++ b/scripts/drop_schemas.bat @@ -5,11 +5,34 @@ CHCP 65001 REM Kerrotaan Postgresille myös että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST=localhost -SET PORT=5435 -SET DB_NAME=jkr -SET USER=jkr_admin -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) ECHO Muunnetaan jkr-muotoon... -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -f "drop_schemas.sql" \ No newline at end of file +psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "drop_schemas.sql" \ No newline at end of file diff --git a/scripts/import_and_create_kohteet.bat b/scripts/import_and_create_kohteet.bat index d76803cb..9b55d152 100644 --- a/scripts/import_and_create_kohteet.bat +++ b/scripts/import_and_create_kohteet.bat @@ -1,7 +1,35 @@ @echo off - setlocal +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) + REM Tarkistetaan, että dvv-aineisto on syötetty IF "%~1"=="" ( echo Anna dvv-aineiston tiedostopolku. @@ -35,38 +63,34 @@ CHCP 65001 REM Kerrotaan Postgresille myös että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST= -SET PORT= -SET DB_NAME= -SET USER= REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa -REM Määritä polku QGISin psql asennukseen. Esim "C:\\Program Files\\QGIS 3.30.2\\bin". -SET PSQL_PATH="" +REM Määritä polku QGISin psql asennukseen. Esim "C:\\Program Files\\QGIS 3.28.10\\bin". +SET PSQL_PATH="C:\\Program Files\\QGIS 3.28.11\\bin" -REM Määritä polku QGISin ogr2ogr asennukseen. Esim "C:\\Program Files\\QGIS 3.30.2\\bin". -SET OGR2OGR_PATH="" +REM Määritä polku QGISin ogr2ogr asennukseen. Esim "C:\\Program Files\\QGIS 3.28.10\\bin". +SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.11\\bin" REM Tarkistetaan halutaanko importoida posti data. IF "%~3"=="posti" ( - %PSQL_PATH%\\psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -f "./scripts/jkr_posti.sql" + %PSQL_PATH%\\psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "./scripts/jkr_posti.sql" ) ECHO Rakennukset - %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus %DVV% "R1 rakennus" + %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus %DVV% "R1 rakennus" ECHO Osoitteet - %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite %DVV% "R3 osoite" + %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite %DVV% "R3 osoite" ECHO Omistajat - %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja %DVV% "R4 omistaja" + %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja %DVV% "R4 omistaja" ECHO Asukkaat - %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin %DVV% "R9 huon asukk" + %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin %DVV% "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -%PSQL_PATH%\\psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -v formatted_date="%formatted_date%" -f "./scripts/import_dvv.sql" +%PSQL_PATH%\\psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -v formatted_date="%formatted_date%" -f "./scripts/import_dvv.sql" -endlocal +endlocal \ No newline at end of file diff --git a/scripts/import_dvv.bat b/scripts/import_dvv.bat index cb6b18e3..f820a4da 100755 --- a/scripts/import_dvv.bat +++ b/scripts/import_dvv.bat @@ -25,23 +25,49 @@ CHCP 65001 REM Kerrotaan Postgresille myös että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST=localhost -SET PORT=5435 -SET DB_NAME=jkr -SET USER=jkr_admin -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) + +REM Määritä polku QGISin ogr2ogr asennukseen. Esim "C:\\Program Files\\QGIS 3.30.2\\bin". +SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.11\\bin" ECHO Rakennukset -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "../data/dvv/DVV_rakennukset.xlsx" "R1 rakennus" +%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "../data/dvv/DVV_rakennukset.xlsx" "R1 rakennus" ECHO Osoitteet -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "../data/dvv/DVV_rakennukset.xlsx" "R3 osoite" +%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "../data/dvv/DVV_rakennukset.xlsx" "R3 osoite" ECHO Omistajat -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "../data/dvv/DVV_rakennukset.xlsx" "R4 omistaja" +%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "../data/dvv/DVV_rakennukset.xlsx" "R4 omistaja" ECHO Asukkaat -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "../data/dvv/DVV_rakennukset.xlsx" "R9 huon asukk" +%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "../data/dvv/DVV_rakennukset.xlsx" "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -v formatted_date="%formatted_date%" -f "import_dvv.sql" +psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -v formatted_date="%formatted_date%" -f "import_dvv.sql" diff --git a/scripts/import_posti.bat b/scripts/import_posti.bat index a00afb13..5c52a14c 100644 --- a/scripts/import_posti.bat +++ b/scripts/import_posti.bat @@ -5,15 +5,38 @@ CHCP 65001 REM Kerrotaan Postgresille myös että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST=localhost -SET PORT=5435 -SET DB_NAME=jkr -SET USER=jkr_admin -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Check if required variables are set +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) ECHO Kunnat ja postinumerot REM # Kunnat ja postinumerot on tuotava tietokantaan ennen dvv-aineiston tuontia -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -f "import_posti.sql" +psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "import_posti.sql" REM Kadut REM Katuja ei tarvitse tuoda; tarvittavat kadut löytyvät dvv-osoiteaineistosta diff --git a/scripts/import_taajama.bat b/scripts/import_taajama.bat index 96d8cfa0..25b9f1b0 100644 --- a/scripts/import_taajama.bat +++ b/scripts/import_taajama.bat @@ -1,6 +1,35 @@ @echo off setlocal +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) + REM Tarkistetaan parametrit IF "%~3"=="" ( ECHO Anna parametrit järjestyksessä @@ -16,12 +45,6 @@ CHCP 65001 REM Kerrotaan Postgresille myäs että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST= -SET PORT= -SET DB_NAME= -SET USER= -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa - SET SHP_FILE=%~1 SET DATE_FROM=%~2 SET POPULATION=%~3 @@ -30,9 +53,9 @@ for %%A in ("%SHP_FILE%") do ( SET "SHP_TABLE=%%~nA" ) -REM Määritä polku QGISin ogr2ogr-asennukseen. Esim. "C:\\Program Files\\QGIS 3.30.2\\bin". -SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.9\\bin" +REM Määritä polku QGISin ogr2ogr-asennukseen. Esim. "C:\\Program Files\\QGIS 3.28.10\\bin". +SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.11\\bin" -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -update -append PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr" -nln taajama -nlt MULTIPOLYGON -dialect SQLITE -sql "SELECT ""Geometry"" as geom, ""Urakkaraja"" as nimi, %POPULATION% as vaesto_lkm, ""fid"" as taajama_id, '%DATE_FROM%' as alkupvm FROM ""%SHP_TABLE%""" "%SHP_FILE%" +%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -update -append PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr" -nln taajama -nlt MULTIPOLYGON -dialect SQLITE -sql "SELECT ""Geometry"" as geom, ""Urakkaraja"" as nimi, %POPULATION% as vaesto_lkm, ""fid"" as taajama_id, '%DATE_FROM%' as alkupvm FROM ""%SHP_TABLE%""" "%SHP_FILE%" ECHO Valmis! diff --git a/scripts/update_dvv.bat b/scripts/update_dvv.bat index 7f44a21b..6d3fb32c 100644 --- a/scripts/update_dvv.bat +++ b/scripts/update_dvv.bat @@ -25,23 +25,46 @@ CHCP 65001 REM Kerrotaan Postgresille myös että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST=localhost -SET PORT=5435 -SET DB_NAME=jkr -SET USER=jkr_admin -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) ECHO Rakennukset -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "../data/dvv/DVV_rakennukset_uusi.xlsx" "R1 rakennus" +ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "../data/dvv/DVV_rakennukset_uusi.xlsx" "R1 rakennus" ECHO Osoitteet -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "../data/dvv/DVV_rakennukset_uusi.xlsx" "R3 osoite" +ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "../data/dvv/DVV_rakennukset_uusi.xlsx" "R3 osoite" ECHO Omistajat -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "../data/dvv/DVV_rakennukset_uusi.xlsx" "R4 omistaja" +ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "../data/dvv/DVV_rakennukset_uusi.xlsx" "R4 omistaja" ECHO Asukkaat -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "../data/dvv/DVV_rakennukset_uusi.xlsx" "R9 huon asukk" +ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "../data/dvv/DVV_rakennukset_uusi.xlsx" "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -v formatted_date="%formatted_date%" -f "import_dvv.sql" +psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -v formatted_date="%formatted_date%" -f "import_dvv.sql" diff --git a/scripts/update_huoneistomaara.bat b/scripts/update_huoneistomaara.bat index 9718d3aa..dbeac263 100644 --- a/scripts/update_huoneistomaara.bat +++ b/scripts/update_huoneistomaara.bat @@ -11,12 +11,34 @@ CHCP 65001 REM Kerrotaan Postgresille myäs että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET EXCEL_FILE=%~1 -SET HOST= -SET PORT= -SET DB_NAME= -SET USER= -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) REM Määritä polku QGISin psql-asennukseen. Esim. "C:\\Program Files\\QGIS 3.30.2\\bin". SET PSQL_PATH="" @@ -28,4 +50,4 @@ ECHO Luetaan huoneistomäärät %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln huoneistomaara %EXCEL_FILE% "Huoneistolkm" ECHO Päivitetään huoneistomäärät rakennus-tauluun -%PSQL_PATH%\\psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -f "update_huoneistomaara.sql" +%PSQL_PATH%\\psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "update_huoneistomaara.sql" From d75c1a7addaee058604a7d8849adf60f42b0aba5 Mon Sep 17 00:00:00 2001 From: Miikka Kallio Date: Thu, 14 Mar 2024 09:27:13 +0200 Subject: [PATCH 2/5] Fixes for tests --- jkrimporter/conf.py | 10 ++++++ scripts/import_posti.bat | 2 +- tests/scripts/init_database.bat | 55 ++++++++++++++++++++++-------- tests/scripts/update_database.bat | 50 ++++++++++++++++++++------- tests/test_akppoistosyy.py | 2 +- tests/test_data_import.py | 13 +++++-- tests/test_kompostori.py | 18 +++++----- tests/test_kompostori_end_dates.py | 8 ++--- tests/test_lahti_siirtotiedosto.py | 4 +-- tests/test_paatostulos.py | 2 +- tests/test_tapahtumalaji.py | 2 +- tests/test_viranomaispaatokset.py | 7 ++-- 12 files changed, 121 insertions(+), 52 deletions(-) diff --git a/jkrimporter/conf.py b/jkrimporter/conf.py index 680fb4d3..4d8f71e9 100644 --- a/jkrimporter/conf.py +++ b/jkrimporter/conf.py @@ -18,6 +18,16 @@ "username": env.get("JKR_USER", None), "password": env.get("JKR_PASSWORD", None), "dbname": env.get("JKR_DB", None), + # "test_port": env.get("JKR_TEST_DB_PORT", None), + # "test_password": env.get("JKR_TEST_PASSWORD", None), + # "test_dbname": env.get("JKR_TEST_DB", None), +} +dbtestconf = { + "host": env.get("JKR_DB_HOST", None), + "port": env.get("JKR_TEST_DB_PORT", None), + "username": env.get("JKR_USER", None), + "password": env.get("JKR_TEST_PASSWORD", None), + "dbname": env.get("JKR_TEST_DB", None), } __all__ = ["dbconf"] diff --git a/scripts/import_posti.bat b/scripts/import_posti.bat index 5c52a14c..19cb737a 100644 --- a/scripts/import_posti.bat +++ b/scripts/import_posti.bat @@ -16,7 +16,7 @@ for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( set "%%a=%%b" ) -REM Check if required variables are set +REM Tarkistetaan onko tarvittavat muuttujat asetettu. if "%JKR_DB_HOST%"=="" ( echo Error: HOST variable not set in .env file exit /b 1 diff --git a/tests/scripts/init_database.bat b/tests/scripts/init_database.bat index be01f05c..15fd6379 100644 --- a/tests/scripts/init_database.bat +++ b/tests/scripts/init_database.bat @@ -4,38 +4,63 @@ REM Luodaan kanta alusta dockerilla docker stop jkr_test_database docker rm jkr_test_database docker volume rm jkr-core_postgis-data-test -docker compose -f ..\\docker-compose.yml up -d db_test -docker compose -f ..\\docker-compose.yml run --rm flyway_test migrate +docker compose --env-file "%APPDATA%\jkr\.env" -f ..\\docker-compose.yml up -d db_test +docker compose --env-file "%APPDATA%\jkr\.env" -f ..\\docker-compose.yml run --rm flyway_test migrate REM Vaihdetaan terminaalin code page UTF-8:ksi CHCP 65001 REM Kerrotaan Postgresille my├Âs terminaalin encoding UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST=localhost -SET PORT=5436 -SET DB_NAME=jkr_test -SET USER=jkr_admin -SET PGPASSWORD=qwerty -REM Salasana v├ñlitet├ñ├ñn ymp├ñrist├Âmuuttujassa ainoastaan testikannalle, joka ei sis├ñll├ñ todellista dataa +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_TEST_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_TEST_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) + + +SET PGPASSWORD=%JKR_PASSWORD% ECHO Kunnat ja postinumerot REM Kunnat ja postinumerot on tuotava tietokantaan ennen dvv-aineiston tuontia -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -f "./scripts/import_posti_test.sql" +psql -h %JKR_DB_HOST% -p %JKR_TEST_DB_PORT% -d %JKR_TEST_DB% -U %JKR_USER% -f "./scripts/import_posti_test.sql" -SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.9\\bin" +SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.11\\bin" ECHO Rakennukset -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "./data/test_data_import/DVV_original.xlsx" "R1 rakennus" +%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "./data/test_data_import/DVV_original.xlsx" "R1 rakennus" ECHO Osoitteet -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "./data/test_data_import/DVV_original.xlsx" "R3 osoite" +%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "./data/test_data_import/DVV_original.xlsx" "R3 osoite" ECHO Omistajat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "./data/test_data_import/DVV_original.xlsx" "R4 omistaja" +%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "./data/test_data_import/DVV_original.xlsx" "R4 omistaja" ECHO Asukkaat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "./data/test_data_import/DVV_original.xlsx" "R9 huon asukk" +%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "./data/test_data_import/DVV_original.xlsx" "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -v formatted_date=20220128 -f "../scripts/import_dvv.sql" +psql -h %JKR_DB_HOST% -p %JKR_TEST_DB_PORT% -d %JKR_TEST_DB% -U %JKR_USER% -v formatted_date=20220128 -f "../scripts/import_dvv.sql" diff --git a/tests/scripts/update_database.bat b/tests/scripts/update_database.bat index f98a498e..23af2d6c 100644 --- a/tests/scripts/update_database.bat +++ b/tests/scripts/update_database.bat @@ -5,26 +5,50 @@ CHCP 65001 REM Kerrotaan Postgresille my├Âs terminaalin encoding UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST=localhost -SET PORT=5436 -SET DB_NAME=jkr_test -SET USER=jkr_admin -SET PGPASSWORD=qwerty -REM Salasana v├ñlitet├ñ├ñn ymp├ñrist├Âmuuttujassa ainoastaan testikannalle, joka ei sis├ñll├ñ todellista dataa - -SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.9\\bin" +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_TEST_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_TEST_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) + +SET PGPASSWORD=%JKR_PASSWORD% + +SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.11\\bin" ECHO Rakennukset -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "./data/test_data_import/DVV_update.xlsx" "R1 rakennus" +%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "./data/test_data_import/DVV_update.xlsx" "R1 rakennus" ECHO Osoitteet -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "./data/test_data_import/DVV_update.xlsx" "R3 osoite" +%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "./data/test_data_import/DVV_update.xlsx" "R3 osoite" ECHO Omistajat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "./data/test_data_import/DVV_update.xlsx" "R4 omistaja" +%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "./data/test_data_import/DVV_update.xlsx" "R4 omistaja" ECHO Asukkaat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "./data/test_data_import/DVV_update.xlsx" "R9 huon asukk" +%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "./data/test_data_import/DVV_update.xlsx" "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -v formatted_date=20230131 -f "../scripts/import_dvv.sql" +psql -h %JKR_DB_HOST% -p %JKR_TEST_DB_PORT% -d %JKR_TEST_DB% -U %JKR_USER% -v formatted_date=20230131 -f "../scripts/import_dvv.sql" diff --git a/tests/test_akppoistosyy.py b/tests/test_akppoistosyy.py index 275ae0a5..8d1a6647 100644 --- a/tests/test_akppoistosyy.py +++ b/tests/test_akppoistosyy.py @@ -11,7 +11,7 @@ def engine(): engine = create_engine( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbconf + **conf.dbtestconf ), future=True, json_serializer=json_dumps, diff --git a/tests/test_data_import.py b/tests/test_data_import.py index d6630721..06899da5 100644 --- a/tests/test_data_import.py +++ b/tests/test_data_import.py @@ -1,5 +1,5 @@ -from datetime import datetime import subprocess +from datetime import datetime import pytest from sqlalchemy import and_, create_engine, distinct, func, or_, select @@ -9,13 +9,20 @@ from jkrimporter.providers.db.codes import init_code_objects from jkrimporter.providers.db.database import json_dumps from jkrimporter.providers.db.dbprovider import import_dvv_kohteet -from jkrimporter.providers.db.models import Kohde, KohteenOsapuolet, Osapuoli, Osapuolenrooli +from jkrimporter.providers.db.models import ( + Kohde, + KohteenOsapuolet, + Osapuolenrooli, + Osapuoli, +) @pytest.fixture(scope="module", autouse=True) def engine(): engine = create_engine( - "postgresql://{username}:{password}@{host}:{port}/{dbname}".format(**conf.dbconf), + "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( + **conf.dbtestconf + ), future=True, json_serializer=json_dumps ) diff --git a/tests/test_kompostori.py b/tests/test_kompostori.py index f5368231..9dc398cd 100644 --- a/tests/test_kompostori.py +++ b/tests/test_kompostori.py @@ -1,33 +1,36 @@ -import pytest +import json import os -from openpyxl.reader.excel import load_workbook +import pytest +from openpyxl.reader.excel import load_workbook from sqlalchemy import create_engine, func from sqlalchemy.orm import Session from jkrimporter import conf from jkrimporter.cli.jkr import import_ilmoitukset -from jkrimporter.providers.db.database import json_dumps +from jkrimporter.providers.db.database import JSONEncoderWithDateSupport, json_dumps from jkrimporter.providers.db.models import Kompostori, KompostorinKohteet from jkrimporter.providers.lahti.ilmoitustiedosto import Ilmoitustiedosto +# Redefine json_dumps with test-specific behavior +def json_dumps(value): + return json.dumps(value, cls=JSONEncoderWithDateSupport) + @pytest.fixture(scope="module", autouse=True) def engine(): engine = create_engine( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbconf + **conf.dbtestconf ), future=True, - json_serializer=json_dumps, + json_serializer=json_dumps, # Use the test-specific json_dumps function here ) return engine - def test_readable(datadir): assert Ilmoitustiedosto.readable_by_me(datadir + "/ilmoitukset.xlsx") - def test_kompostori(engine, datadir): import_ilmoitukset(datadir + "/ilmoitukset.xlsx") session = Session(engine) @@ -45,7 +48,6 @@ def test_kompostori(engine, datadir): sheet = workbook[workbook.sheetnames[0]] assert sheet.max_row == 6 - def test_kompostori_osakkaan_lisays(engine, datadir): import_ilmoitukset(datadir + "/ilmoitukset_lisaa_komposti_osakas.xlsx") session = Session(engine) diff --git a/tests/test_kompostori_end_dates.py b/tests/test_kompostori_end_dates.py index a373485a..78a49902 100644 --- a/tests/test_kompostori_end_dates.py +++ b/tests/test_kompostori_end_dates.py @@ -1,8 +1,8 @@ -import pytest -import os import datetime -from openpyxl.reader.excel import load_workbook +import os +import pytest +from openpyxl.reader.excel import load_workbook from sqlalchemy import create_engine, func from sqlalchemy.orm import Session @@ -17,7 +17,7 @@ def engine(): engine = create_engine( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbconf + **conf.dbtestconf ), future=True, json_serializer=json_dumps, diff --git a/tests/test_lahti_siirtotiedosto.py b/tests/test_lahti_siirtotiedosto.py index 870df45c..3768530b 100644 --- a/tests/test_lahti_siirtotiedosto.py +++ b/tests/test_lahti_siirtotiedosto.py @@ -1,8 +1,8 @@ import csv import os from pathlib import Path -import pytest +import pytest from sqlalchemy import create_engine, func, or_, select, text from sqlalchemy.orm import Session @@ -28,7 +28,7 @@ def engine(): engine = create_engine( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbconf + **conf.dbtestconf ), future=True, json_serializer=json_dumps diff --git a/tests/test_paatostulos.py b/tests/test_paatostulos.py index 34d5107a..3626490f 100644 --- a/tests/test_paatostulos.py +++ b/tests/test_paatostulos.py @@ -11,7 +11,7 @@ def engine(): engine = create_engine( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbconf + **conf.dbtestconf ), future=True, json_serializer=json_dumps, diff --git a/tests/test_tapahtumalaji.py b/tests/test_tapahtumalaji.py index c0f83271..6de4d00a 100644 --- a/tests/test_tapahtumalaji.py +++ b/tests/test_tapahtumalaji.py @@ -11,7 +11,7 @@ def engine(): engine = create_engine( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbconf + **conf.dbtestconf ), future=True, json_serializer=json_dumps, diff --git a/tests/test_viranomaispaatokset.py b/tests/test_viranomaispaatokset.py index c6a3873c..6f4883cd 100644 --- a/tests/test_viranomaispaatokset.py +++ b/tests/test_viranomaispaatokset.py @@ -1,7 +1,7 @@ -from datetime import date import os -import pytest +from datetime import date +import pytest from openpyxl.reader.excel import load_workbook from sqlalchemy import create_engine, func from sqlalchemy.orm import Session @@ -23,8 +23,9 @@ @pytest.fixture(scope="module", autouse=True) def engine(): engine = create_engine( + # "postgresql://{username}:{test_password}@{host}:{test_port}/{test_dbname}".format( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbconf + **conf.dbtestconf ), future=True, json_serializer=json_dumps, From 3661cae16ef8cee6027b38cfef643253b882c735 Mon Sep 17 00:00:00 2001 From: Miikka Kallio Date: Thu, 14 Mar 2024 11:06:36 +0200 Subject: [PATCH 3/5] Add QGIS_BIN_PATH to .env, used for psql and ogr2ogr Add QGIS_BIN_PATH to .env, used for psql and ogr2ogr. Updated .env.template --- .env.template | 2 ++ jkrimporter/conf.py | 3 --- scripts/drop_schemas.bat | 7 ++++++- scripts/import_and_create_kohteet.bat | 25 +++++++++++-------------- scripts/import_dvv.bat | 17 +++++++++-------- scripts/import_posti.bat | 6 +++++- scripts/import_taajama.bat | 9 +++++---- scripts/update_dvv.bat | 14 +++++++++----- scripts/update_huoneistomaara.bat | 13 ++++++------- tests/scripts/init_database.bat | 17 ++++++++++------- tests/scripts/update_database.bat | 17 ++++++++++------- 11 files changed, 73 insertions(+), 57 deletions(-) diff --git a/.env.template b/.env.template index 86f4becb..37111ee5 100644 --- a/.env.template +++ b/.env.template @@ -10,3 +10,5 @@ JKR_PASSWORD=qwerty JKR_TEST_DB=jkr_test JKR_TEST_DB_PORT=5436 JKR_TEST_PASSWORD=qwerty + +QGIS_BIN_PATH=C:\\Program Files\\QGIS 3.28.11\\bin diff --git a/jkrimporter/conf.py b/jkrimporter/conf.py index 4d8f71e9..a5e72d29 100644 --- a/jkrimporter/conf.py +++ b/jkrimporter/conf.py @@ -18,9 +18,6 @@ "username": env.get("JKR_USER", None), "password": env.get("JKR_PASSWORD", None), "dbname": env.get("JKR_DB", None), - # "test_port": env.get("JKR_TEST_DB_PORT", None), - # "test_password": env.get("JKR_TEST_PASSWORD", None), - # "test_dbname": env.get("JKR_TEST_DB", None), } dbtestconf = { "host": env.get("JKR_DB_HOST", None), diff --git a/scripts/drop_schemas.bat b/scripts/drop_schemas.bat index 1ed96a64..0e266470 100644 --- a/scripts/drop_schemas.bat +++ b/scripts/drop_schemas.bat @@ -33,6 +33,11 @@ if "%JKR_USER%"=="" ( echo Error: USER variable not set in .env file exit /b 1 ) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b 1 +) + ECHO Muunnetaan jkr-muotoon... -psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "drop_schemas.sql" \ No newline at end of file +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "drop_schemas.sql" \ No newline at end of file diff --git a/scripts/import_and_create_kohteet.bat b/scripts/import_and_create_kohteet.bat index 9b55d152..0ba2b3f3 100644 --- a/scripts/import_and_create_kohteet.bat +++ b/scripts/import_and_create_kohteet.bat @@ -29,6 +29,11 @@ if "%JKR_USER%"=="" ( echo Error: USER variable not set in .env file exit /b 1 ) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) + REM Tarkistetaan, että dvv-aineisto on syötetty IF "%~1"=="" ( @@ -63,34 +68,26 @@ CHCP 65001 REM Kerrotaan Postgresille myös että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa - -REM Määritä polku QGISin psql asennukseen. Esim "C:\\Program Files\\QGIS 3.28.10\\bin". -SET PSQL_PATH="C:\\Program Files\\QGIS 3.28.11\\bin" - -REM Määritä polku QGISin ogr2ogr asennukseen. Esim "C:\\Program Files\\QGIS 3.28.10\\bin". -SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.11\\bin" - REM Tarkistetaan halutaanko importoida posti data. IF "%~3"=="posti" ( - %PSQL_PATH%\\psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "./scripts/jkr_posti.sql" + "%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "./scripts/jkr_posti.sql" ) ECHO Rakennukset - %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus %DVV% "R1 rakennus" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus %DVV% "R1 rakennus" ECHO Osoitteet - %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite %DVV% "R3 osoite" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite %DVV% "R3 osoite" ECHO Omistajat - %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja %DVV% "R4 omistaja" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja %DVV% "R4 omistaja" ECHO Asukkaat - %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin %DVV% "R9 huon asukk" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin %DVV% "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -%PSQL_PATH%\\psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -v formatted_date="%formatted_date%" -f "./scripts/import_dvv.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -v formatted_date="%formatted_date%" -f "./scripts/import_dvv.sql" endlocal \ No newline at end of file diff --git a/scripts/import_dvv.bat b/scripts/import_dvv.bat index f820a4da..fe4e0531 100755 --- a/scripts/import_dvv.bat +++ b/scripts/import_dvv.bat @@ -53,21 +53,22 @@ if "%JKR_USER%"=="" ( echo Error: USER variable not set in .env file exit /b 1 ) - -REM Määritä polku QGISin ogr2ogr asennukseen. Esim "C:\\Program Files\\QGIS 3.30.2\\bin". -SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.11\\bin" +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) ECHO Rakennukset -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "../data/dvv/DVV_rakennukset.xlsx" "R1 rakennus" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "../tests/data/test_data_import/DVV_original.xlsx" "R1 rakennus" ECHO Osoitteet -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "../data/dvv/DVV_rakennukset.xlsx" "R3 osoite" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "../tests/data/test_data_import/DVV_original.xlsx" "R3 osoite" ECHO Omistajat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "../data/dvv/DVV_rakennukset.xlsx" "R4 omistaja" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "../tests/data/test_data_import/DVV_original.xlsx" "R4 omistaja" ECHO Asukkaat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "../data/dvv/DVV_rakennukset.xlsx" "R9 huon asukk" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "../tests/data/test_data_import/DVV_original.xlsx" "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -v formatted_date="%formatted_date%" -f "import_dvv.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -v formatted_date="%formatted_date%" -f "import_dvv.sql" diff --git a/scripts/import_posti.bat b/scripts/import_posti.bat index 19cb737a..04856106 100644 --- a/scripts/import_posti.bat +++ b/scripts/import_posti.bat @@ -33,10 +33,14 @@ if "%JKR_USER%"=="" ( echo Error: USER variable not set in .env file exit /b 1 ) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b 1 +) ECHO Kunnat ja postinumerot REM # Kunnat ja postinumerot on tuotava tietokantaan ennen dvv-aineiston tuontia -psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "import_posti.sql" +"%QGIS_BIN_PATH%\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "import_posti.sql" REM Kadut REM Katuja ei tarvitse tuoda; tarvittavat kadut löytyvät dvv-osoiteaineistosta diff --git a/scripts/import_taajama.bat b/scripts/import_taajama.bat index 25b9f1b0..374aba10 100644 --- a/scripts/import_taajama.bat +++ b/scripts/import_taajama.bat @@ -29,6 +29,10 @@ if "%JKR_USER%"=="" ( echo Error: USER variable not set in .env file exit /b 1 ) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) REM Tarkistetaan parametrit IF "%~3"=="" ( @@ -53,9 +57,6 @@ for %%A in ("%SHP_FILE%") do ( SET "SHP_TABLE=%%~nA" ) -REM Määritä polku QGISin ogr2ogr-asennukseen. Esim. "C:\\Program Files\\QGIS 3.28.10\\bin". -SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.11\\bin" - -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -update -append PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr" -nln taajama -nlt MULTIPOLYGON -dialect SQLITE -sql "SELECT ""Geometry"" as geom, ""Urakkaraja"" as nimi, %POPULATION% as vaesto_lkm, ""fid"" as taajama_id, '%DATE_FROM%' as alkupvm FROM ""%SHP_TABLE%""" "%SHP_FILE%" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -update -append PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr" -nln taajama -nlt MULTIPOLYGON -dialect SQLITE -sql "SELECT ""Geometry"" as geom, ""Urakkaraja"" as nimi, %POPULATION% as vaesto_lkm, ""fid"" as taajama_id, '%DATE_FROM%' as alkupvm FROM ""%SHP_TABLE%""" "%SHP_FILE%" ECHO Valmis! diff --git a/scripts/update_dvv.bat b/scripts/update_dvv.bat index 6d3fb32c..543fd953 100644 --- a/scripts/update_dvv.bat +++ b/scripts/update_dvv.bat @@ -53,18 +53,22 @@ if "%JKR_USER%"=="" ( echo Error: USER variable not set in .env file exit /b 1 ) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) ECHO Rakennukset -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "../data/dvv/DVV_rakennukset_uusi.xlsx" "R1 rakennus" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "../data/dvv/DVV_rakennukset_uusi.xlsx" "R1 rakennus" ECHO Osoitteet -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "../data/dvv/DVV_rakennukset_uusi.xlsx" "R3 osoite" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "../data/dvv/DVV_rakennukset_uusi.xlsx" "R3 osoite" ECHO Omistajat -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "../data/dvv/DVV_rakennukset_uusi.xlsx" "R4 omistaja" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "../data/dvv/DVV_rakennukset_uusi.xlsx" "R4 omistaja" ECHO Asukkaat -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "../data/dvv/DVV_rakennukset_uusi.xlsx" "R9 huon asukk" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "../data/dvv/DVV_rakennukset_uusi.xlsx" "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -v formatted_date="%formatted_date%" -f "import_dvv.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -v formatted_date="%formatted_date%" -f "import_dvv.sql" diff --git a/scripts/update_huoneistomaara.bat b/scripts/update_huoneistomaara.bat index dbeac263..697311b7 100644 --- a/scripts/update_huoneistomaara.bat +++ b/scripts/update_huoneistomaara.bat @@ -39,15 +39,14 @@ if "%JKR_USER%"=="" ( echo Error: USER variable not set in .env file exit /b 1 ) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) -REM Määritä polku QGISin psql-asennukseen. Esim. "C:\\Program Files\\QGIS 3.30.2\\bin". -SET PSQL_PATH="" - -REM Määritä polku QGISin ogr2ogr-asennukseen. Esim. "C:\\Program Files\\QGIS 3.30.2\\bin". -SET OGR2OGR_PATH="" ECHO Luetaan huoneistomäärät -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln huoneistomaara %EXCEL_FILE% "Huoneistolkm" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln huoneistomaara %EXCEL_FILE% "Huoneistolkm" ECHO Päivitetään huoneistomäärät rakennus-tauluun -%PSQL_PATH%\\psql -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "update_huoneistomaara.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "update_huoneistomaara.sql" diff --git a/tests/scripts/init_database.bat b/tests/scripts/init_database.bat index 15fd6379..8eb26642 100644 --- a/tests/scripts/init_database.bat +++ b/tests/scripts/init_database.bat @@ -40,27 +40,30 @@ if "%JKR_USER%"=="" ( echo Error: USER variable not set in .env file exit /b 1 ) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) SET PGPASSWORD=%JKR_PASSWORD% ECHO Kunnat ja postinumerot REM Kunnat ja postinumerot on tuotava tietokantaan ennen dvv-aineiston tuontia -psql -h %JKR_DB_HOST% -p %JKR_TEST_DB_PORT% -d %JKR_TEST_DB% -U %JKR_USER% -f "./scripts/import_posti_test.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_TEST_DB_PORT% -d %JKR_TEST_DB% -U %JKR_USER% -f "./scripts/import_posti_test.sql" -SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.11\\bin" ECHO Rakennukset -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "./data/test_data_import/DVV_original.xlsx" "R1 rakennus" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "./data/test_data_import/DVV_original.xlsx" "R1 rakennus" ECHO Osoitteet -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "./data/test_data_import/DVV_original.xlsx" "R3 osoite" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "./data/test_data_import/DVV_original.xlsx" "R3 osoite" ECHO Omistajat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "./data/test_data_import/DVV_original.xlsx" "R4 omistaja" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "./data/test_data_import/DVV_original.xlsx" "R4 omistaja" ECHO Asukkaat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "./data/test_data_import/DVV_original.xlsx" "R9 huon asukk" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "./data/test_data_import/DVV_original.xlsx" "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -psql -h %JKR_DB_HOST% -p %JKR_TEST_DB_PORT% -d %JKR_TEST_DB% -U %JKR_USER% -v formatted_date=20220128 -f "../scripts/import_dvv.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_TEST_DB_PORT% -d %JKR_TEST_DB% -U %JKR_USER% -v formatted_date=20220128 -f "../scripts/import_dvv.sql" diff --git a/tests/scripts/update_database.bat b/tests/scripts/update_database.bat index 23af2d6c..dde25778 100644 --- a/tests/scripts/update_database.bat +++ b/tests/scripts/update_database.bat @@ -33,22 +33,25 @@ if "%JKR_USER%"=="" ( echo Error: USER variable not set in .env file exit /b 1 ) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) -SET PGPASSWORD=%JKR_PASSWORD% -SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.11\\bin" +SET PGPASSWORD=%JKR_PASSWORD% ECHO Rakennukset -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "./data/test_data_import/DVV_update.xlsx" "R1 rakennus" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "./data/test_data_import/DVV_update.xlsx" "R1 rakennus" ECHO Osoitteet -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "./data/test_data_import/DVV_update.xlsx" "R3 osoite" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "./data/test_data_import/DVV_update.xlsx" "R3 osoite" ECHO Omistajat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "./data/test_data_import/DVV_update.xlsx" "R4 omistaja" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "./data/test_data_import/DVV_update.xlsx" "R4 omistaja" ECHO Asukkaat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "./data/test_data_import/DVV_update.xlsx" "R9 huon asukk" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "./data/test_data_import/DVV_update.xlsx" "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -psql -h %JKR_DB_HOST% -p %JKR_TEST_DB_PORT% -d %JKR_TEST_DB% -U %JKR_USER% -v formatted_date=20230131 -f "../scripts/import_dvv.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_TEST_DB_PORT% -d %JKR_TEST_DB% -U %JKR_USER% -v formatted_date=20230131 -f "../scripts/import_dvv.sql" From 3718ccc319b841d9ad8b57746b7dc9d8ac1c65b4 Mon Sep 17 00:00:00 2001 From: Ismo Lahtinen Date: Thu, 14 Mar 2024 12:14:33 +0200 Subject: [PATCH 4/5] Add different connection parameters for test environment --- jkrimporter/conf.py | 14 +++++++------- tests/conftest.py | 2 ++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/jkrimporter/conf.py b/jkrimporter/conf.py index a5e72d29..1051799f 100644 --- a/jkrimporter/conf.py +++ b/jkrimporter/conf.py @@ -2,6 +2,9 @@ from dotenv import dotenv_values +from tests.conftest import TEST_ENV + + # Path to the .env file in the user's %APPDATA%/jkr directory dotenv_path = os.path.join(os.getenv('APPDATA'), 'jkr', '.env') @@ -19,13 +22,10 @@ "password": env.get("JKR_PASSWORD", None), "dbname": env.get("JKR_DB", None), } -dbtestconf = { - "host": env.get("JKR_DB_HOST", None), - "port": env.get("JKR_TEST_DB_PORT", None), - "username": env.get("JKR_USER", None), - "password": env.get("JKR_TEST_PASSWORD", None), - "dbname": env.get("JKR_TEST_DB", None), -} +if TEST_ENV: + dbconf["port"] = env.get("JKR_TEST_DB_PORT", None) + dbconf["password"] = env.get("JKR_TEST_PASSWORD", None) + dbconf["dbname"] = env.get("JKR_TEST_DB", None) __all__ = ["dbconf"] diff --git a/tests/conftest.py b/tests/conftest.py index 15cd424f..89036135 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,8 @@ from pytest import fixture +TEST_ENV = True + @fixture def datadir(tmpdir, request): From 000d5671e7a06563c9a0c894f31ad3d7451ab10d Mon Sep 17 00:00:00 2001 From: Ismo Lahtinen Date: Thu, 14 Mar 2024 12:34:08 +0200 Subject: [PATCH 5/5] Use dbconf in tests instead of dbtestconf --- tests/test_akppoistosyy.py | 2 +- tests/test_data_import.py | 2 +- tests/test_kompostori.py | 2 +- tests/test_kompostori_end_dates.py | 2 +- tests/test_lahti_siirtotiedosto.py | 2 +- tests/test_paatostulos.py | 2 +- tests/test_tapahtumalaji.py | 2 +- tests/test_viranomaispaatokset.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_akppoistosyy.py b/tests/test_akppoistosyy.py index 8d1a6647..275ae0a5 100644 --- a/tests/test_akppoistosyy.py +++ b/tests/test_akppoistosyy.py @@ -11,7 +11,7 @@ def engine(): engine = create_engine( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbtestconf + **conf.dbconf ), future=True, json_serializer=json_dumps, diff --git a/tests/test_data_import.py b/tests/test_data_import.py index 06899da5..1b892907 100644 --- a/tests/test_data_import.py +++ b/tests/test_data_import.py @@ -21,7 +21,7 @@ def engine(): engine = create_engine( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbtestconf + **conf.dbconf ), future=True, json_serializer=json_dumps diff --git a/tests/test_kompostori.py b/tests/test_kompostori.py index 9dc398cd..1028d45a 100644 --- a/tests/test_kompostori.py +++ b/tests/test_kompostori.py @@ -21,7 +21,7 @@ def json_dumps(value): def engine(): engine = create_engine( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbtestconf + **conf.dbconf ), future=True, json_serializer=json_dumps, # Use the test-specific json_dumps function here diff --git a/tests/test_kompostori_end_dates.py b/tests/test_kompostori_end_dates.py index 78a49902..810e3b1c 100644 --- a/tests/test_kompostori_end_dates.py +++ b/tests/test_kompostori_end_dates.py @@ -17,7 +17,7 @@ def engine(): engine = create_engine( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbtestconf + **conf.dbconf ), future=True, json_serializer=json_dumps, diff --git a/tests/test_lahti_siirtotiedosto.py b/tests/test_lahti_siirtotiedosto.py index 3768530b..391f27d8 100644 --- a/tests/test_lahti_siirtotiedosto.py +++ b/tests/test_lahti_siirtotiedosto.py @@ -28,7 +28,7 @@ def engine(): engine = create_engine( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbtestconf + **conf.dbconf ), future=True, json_serializer=json_dumps diff --git a/tests/test_paatostulos.py b/tests/test_paatostulos.py index 3626490f..34d5107a 100644 --- a/tests/test_paatostulos.py +++ b/tests/test_paatostulos.py @@ -11,7 +11,7 @@ def engine(): engine = create_engine( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbtestconf + **conf.dbconf ), future=True, json_serializer=json_dumps, diff --git a/tests/test_tapahtumalaji.py b/tests/test_tapahtumalaji.py index 6de4d00a..c0f83271 100644 --- a/tests/test_tapahtumalaji.py +++ b/tests/test_tapahtumalaji.py @@ -11,7 +11,7 @@ def engine(): engine = create_engine( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbtestconf + **conf.dbconf ), future=True, json_serializer=json_dumps, diff --git a/tests/test_viranomaispaatokset.py b/tests/test_viranomaispaatokset.py index 6f4883cd..6bff473e 100644 --- a/tests/test_viranomaispaatokset.py +++ b/tests/test_viranomaispaatokset.py @@ -25,7 +25,7 @@ def engine(): engine = create_engine( # "postgresql://{username}:{test_password}@{host}:{test_port}/{test_dbname}".format( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( - **conf.dbtestconf + **conf.dbconf ), future=True, json_serializer=json_dumps,