diff --git a/.env.template b/.env.template index 86f4becb..37111ee5 100644 --- a/.env.template +++ b/.env.template @@ -10,3 +10,5 @@ JKR_PASSWORD=qwerty JKR_TEST_DB=jkr_test JKR_TEST_DB_PORT=5436 JKR_TEST_PASSWORD=qwerty + +QGIS_BIN_PATH=C:\\Program Files\\QGIS 3.28.11\\bin diff --git a/jkrimporter/conf.py b/jkrimporter/conf.py index 8d651065..1051799f 100644 --- a/jkrimporter/conf.py +++ b/jkrimporter/conf.py @@ -2,11 +2,19 @@ from dotenv import dotenv_values +from tests.conftest import TEST_ENV + + +# Path to the .env file in the user's %APPDATA%/jkr directory +dotenv_path = os.path.join(os.getenv('APPDATA'), 'jkr', '.env') + +# Read the environment variables from the .env file env = { - **dotenv_values(".env"), + **dotenv_values(dotenv_path), **os.environ, } +# Define database configuration using environment variables dbconf = { "host": env.get("JKR_DB_HOST", None), "port": env.get("JKR_DB_PORT", None), @@ -14,6 +22,10 @@ "password": env.get("JKR_PASSWORD", None), "dbname": env.get("JKR_DB", None), } +if TEST_ENV: + dbconf["port"] = env.get("JKR_TEST_DB_PORT", None) + dbconf["password"] = env.get("JKR_TEST_PASSWORD", None) + dbconf["dbname"] = env.get("JKR_TEST_DB", None) __all__ = ["dbconf"] diff --git a/scripts/drop_schemas.bat b/scripts/drop_schemas.bat index 781bc672..0e266470 100644 --- a/scripts/drop_schemas.bat +++ b/scripts/drop_schemas.bat @@ -5,11 +5,39 @@ CHCP 65001 REM Kerrotaan Postgresille myös että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST=localhost -SET PORT=5435 -SET DB_NAME=jkr -SET USER=jkr_admin -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b 1 +) + ECHO Muunnetaan jkr-muotoon... -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -f "drop_schemas.sql" \ No newline at end of file +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "drop_schemas.sql" \ No newline at end of file diff --git a/scripts/import_and_create_kohteet.bat b/scripts/import_and_create_kohteet.bat index d76803cb..0ba2b3f3 100644 --- a/scripts/import_and_create_kohteet.bat +++ b/scripts/import_and_create_kohteet.bat @@ -1,7 +1,40 @@ @echo off - setlocal +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) + + REM Tarkistetaan, että dvv-aineisto on syötetty IF "%~1"=="" ( echo Anna dvv-aineiston tiedostopolku. @@ -35,38 +68,26 @@ CHCP 65001 REM Kerrotaan Postgresille myös että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST= -SET PORT= -SET DB_NAME= -SET USER= -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa - -REM Määritä polku QGISin psql asennukseen. Esim "C:\\Program Files\\QGIS 3.30.2\\bin". -SET PSQL_PATH="" - -REM Määritä polku QGISin ogr2ogr asennukseen. Esim "C:\\Program Files\\QGIS 3.30.2\\bin". -SET OGR2OGR_PATH="" - REM Tarkistetaan halutaanko importoida posti data. IF "%~3"=="posti" ( - %PSQL_PATH%\\psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -f "./scripts/jkr_posti.sql" + "%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "./scripts/jkr_posti.sql" ) ECHO Rakennukset - %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus %DVV% "R1 rakennus" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus %DVV% "R1 rakennus" ECHO Osoitteet - %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite %DVV% "R3 osoite" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite %DVV% "R3 osoite" ECHO Omistajat - %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja %DVV% "R4 omistaja" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja %DVV% "R4 omistaja" ECHO Asukkaat - %OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin %DVV% "R9 huon asukk" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin %DVV% "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -%PSQL_PATH%\\psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -v formatted_date="%formatted_date%" -f "./scripts/import_dvv.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -v formatted_date="%formatted_date%" -f "./scripts/import_dvv.sql" -endlocal +endlocal \ No newline at end of file diff --git a/scripts/import_dvv.bat b/scripts/import_dvv.bat index cb6b18e3..fe4e0531 100755 --- a/scripts/import_dvv.bat +++ b/scripts/import_dvv.bat @@ -25,23 +25,50 @@ CHCP 65001 REM Kerrotaan Postgresille myös että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST=localhost -SET PORT=5435 -SET DB_NAME=jkr -SET USER=jkr_admin -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) ECHO Rakennukset -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "../data/dvv/DVV_rakennukset.xlsx" "R1 rakennus" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "../tests/data/test_data_import/DVV_original.xlsx" "R1 rakennus" ECHO Osoitteet -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "../data/dvv/DVV_rakennukset.xlsx" "R3 osoite" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "../tests/data/test_data_import/DVV_original.xlsx" "R3 osoite" ECHO Omistajat -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "../data/dvv/DVV_rakennukset.xlsx" "R4 omistaja" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "../tests/data/test_data_import/DVV_original.xlsx" "R4 omistaja" ECHO Asukkaat -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "../data/dvv/DVV_rakennukset.xlsx" "R9 huon asukk" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "../tests/data/test_data_import/DVV_original.xlsx" "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -v formatted_date="%formatted_date%" -f "import_dvv.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -v formatted_date="%formatted_date%" -f "import_dvv.sql" diff --git a/scripts/import_posti.bat b/scripts/import_posti.bat index a00afb13..04856106 100644 --- a/scripts/import_posti.bat +++ b/scripts/import_posti.bat @@ -5,15 +5,42 @@ CHCP 65001 REM Kerrotaan Postgresille myös että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST=localhost -SET PORT=5435 -SET DB_NAME=jkr -SET USER=jkr_admin -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b 1 +) ECHO Kunnat ja postinumerot REM # Kunnat ja postinumerot on tuotava tietokantaan ennen dvv-aineiston tuontia -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -f "import_posti.sql" +"%QGIS_BIN_PATH%\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "import_posti.sql" REM Kadut REM Katuja ei tarvitse tuoda; tarvittavat kadut löytyvät dvv-osoiteaineistosta diff --git a/scripts/import_taajama.bat b/scripts/import_taajama.bat index 96d8cfa0..374aba10 100644 --- a/scripts/import_taajama.bat +++ b/scripts/import_taajama.bat @@ -1,6 +1,39 @@ @echo off setlocal +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) + REM Tarkistetaan parametrit IF "%~3"=="" ( ECHO Anna parametrit järjestyksessä @@ -16,12 +49,6 @@ CHCP 65001 REM Kerrotaan Postgresille myäs että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST= -SET PORT= -SET DB_NAME= -SET USER= -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa - SET SHP_FILE=%~1 SET DATE_FROM=%~2 SET POPULATION=%~3 @@ -30,9 +57,6 @@ for %%A in ("%SHP_FILE%") do ( SET "SHP_TABLE=%%~nA" ) -REM Määritä polku QGISin ogr2ogr-asennukseen. Esim. "C:\\Program Files\\QGIS 3.30.2\\bin". -SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.9\\bin" - -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -update -append PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr" -nln taajama -nlt MULTIPOLYGON -dialect SQLITE -sql "SELECT ""Geometry"" as geom, ""Urakkaraja"" as nimi, %POPULATION% as vaesto_lkm, ""fid"" as taajama_id, '%DATE_FROM%' as alkupvm FROM ""%SHP_TABLE%""" "%SHP_FILE%" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -update -append PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr" -nln taajama -nlt MULTIPOLYGON -dialect SQLITE -sql "SELECT ""Geometry"" as geom, ""Urakkaraja"" as nimi, %POPULATION% as vaesto_lkm, ""fid"" as taajama_id, '%DATE_FROM%' as alkupvm FROM ""%SHP_TABLE%""" "%SHP_FILE%" ECHO Valmis! diff --git a/scripts/update_dvv.bat b/scripts/update_dvv.bat index 7f44a21b..543fd953 100644 --- a/scripts/update_dvv.bat +++ b/scripts/update_dvv.bat @@ -25,23 +25,50 @@ CHCP 65001 REM Kerrotaan Postgresille myös että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST=localhost -SET PORT=5435 -SET DB_NAME=jkr -SET USER=jkr_admin -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) ECHO Rakennukset -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "../data/dvv/DVV_rakennukset_uusi.xlsx" "R1 rakennus" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "../data/dvv/DVV_rakennukset_uusi.xlsx" "R1 rakennus" ECHO Osoitteet -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "../data/dvv/DVV_rakennukset_uusi.xlsx" "R3 osoite" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "../data/dvv/DVV_rakennukset_uusi.xlsx" "R3 osoite" ECHO Omistajat -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "../data/dvv/DVV_rakennukset_uusi.xlsx" "R4 omistaja" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "../data/dvv/DVV_rakennukset_uusi.xlsx" "R4 omistaja" ECHO Asukkaat -ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "../data/dvv/DVV_rakennukset_uusi.xlsx" "R9 huon asukk" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_DB_PORT% dbname=%JKR_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "../data/dvv/DVV_rakennukset_uusi.xlsx" "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -v formatted_date="%formatted_date%" -f "import_dvv.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -v formatted_date="%formatted_date%" -f "import_dvv.sql" diff --git a/scripts/update_huoneistomaara.bat b/scripts/update_huoneistomaara.bat index 9718d3aa..697311b7 100644 --- a/scripts/update_huoneistomaara.bat +++ b/scripts/update_huoneistomaara.bat @@ -11,21 +11,42 @@ CHCP 65001 REM Kerrotaan Postgresille myäs että terminaalin encoding on UTF-8 SET PGCLIENTENCODING=UTF8 -SET EXCEL_FILE=%~1 -SET HOST= -SET PORT= -SET DB_NAME= -SET USER= -REM Määritä salasana %APPDATA%\postgresql\pgpass.conf tiedostossa +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) -REM Määritä polku QGISin psql-asennukseen. Esim. "C:\\Program Files\\QGIS 3.30.2\\bin". -SET PSQL_PATH="" +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) -REM Määritä polku QGISin ogr2ogr-asennukseen. Esim. "C:\\Program Files\\QGIS 3.30.2\\bin". -SET OGR2OGR_PATH="" ECHO Luetaan huoneistomäärät -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln huoneistomaara %EXCEL_FILE% "Huoneistolkm" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln huoneistomaara %EXCEL_FILE% "Huoneistolkm" ECHO Päivitetään huoneistomäärät rakennus-tauluun -%PSQL_PATH%\\psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -f "update_huoneistomaara.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_DB_PORT% -d %JKR_DB% -U %JKR_USER% -f "update_huoneistomaara.sql" diff --git a/tests/conftest.py b/tests/conftest.py index 15cd424f..89036135 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,8 @@ from pytest import fixture +TEST_ENV = True + @fixture def datadir(tmpdir, request): diff --git a/tests/scripts/init_database.bat b/tests/scripts/init_database.bat index be01f05c..8eb26642 100644 --- a/tests/scripts/init_database.bat +++ b/tests/scripts/init_database.bat @@ -4,38 +4,66 @@ REM Luodaan kanta alusta dockerilla docker stop jkr_test_database docker rm jkr_test_database docker volume rm jkr-core_postgis-data-test -docker compose -f ..\\docker-compose.yml up -d db_test -docker compose -f ..\\docker-compose.yml run --rm flyway_test migrate +docker compose --env-file "%APPDATA%\jkr\.env" -f ..\\docker-compose.yml up -d db_test +docker compose --env-file "%APPDATA%\jkr\.env" -f ..\\docker-compose.yml run --rm flyway_test migrate REM Vaihdetaan terminaalin code page UTF-8:ksi CHCP 65001 REM Kerrotaan Postgresille my├Âs terminaalin encoding UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST=localhost -SET PORT=5436 -SET DB_NAME=jkr_test -SET USER=jkr_admin -SET PGPASSWORD=qwerty -REM Salasana v├ñlitet├ñ├ñn ymp├ñrist├Âmuuttujassa ainoastaan testikannalle, joka ei sis├ñll├ñ todellista dataa +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_TEST_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_TEST_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) + + +SET PGPASSWORD=%JKR_PASSWORD% ECHO Kunnat ja postinumerot REM Kunnat ja postinumerot on tuotava tietokantaan ennen dvv-aineiston tuontia -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -f "./scripts/import_posti_test.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_TEST_DB_PORT% -d %JKR_TEST_DB% -U %JKR_USER% -f "./scripts/import_posti_test.sql" -SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.9\\bin" ECHO Rakennukset -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "./data/test_data_import/DVV_original.xlsx" "R1 rakennus" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "./data/test_data_import/DVV_original.xlsx" "R1 rakennus" ECHO Osoitteet -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "./data/test_data_import/DVV_original.xlsx" "R3 osoite" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "./data/test_data_import/DVV_original.xlsx" "R3 osoite" ECHO Omistajat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "./data/test_data_import/DVV_original.xlsx" "R4 omistaja" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "./data/test_data_import/DVV_original.xlsx" "R4 omistaja" ECHO Asukkaat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "./data/test_data_import/DVV_original.xlsx" "R9 huon asukk" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "./data/test_data_import/DVV_original.xlsx" "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -v formatted_date=20220128 -f "../scripts/import_dvv.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_TEST_DB_PORT% -d %JKR_TEST_DB% -U %JKR_USER% -v formatted_date=20220128 -f "../scripts/import_dvv.sql" diff --git a/tests/scripts/update_database.bat b/tests/scripts/update_database.bat index f98a498e..dde25778 100644 --- a/tests/scripts/update_database.bat +++ b/tests/scripts/update_database.bat @@ -5,26 +5,53 @@ CHCP 65001 REM Kerrotaan Postgresille my├Âs terminaalin encoding UTF-8 SET PGCLIENTENCODING=UTF8 -SET HOST=localhost -SET PORT=5436 -SET DB_NAME=jkr_test -SET USER=jkr_admin -SET PGPASSWORD=qwerty -REM Salasana v├ñlitet├ñ├ñn ymp├ñrist├Âmuuttujassa ainoastaan testikannalle, joka ei sis├ñll├ñ todellista dataa - -SET OGR2OGR_PATH="C:\\Program Files\\QGIS 3.28.9\\bin" +REM Tarkistetaan .env tiedosto. +if not exist "%APPDATA%\jkr\.env" ( + echo Error: .env file not found at %APPDATA%\jkr\.env + exit /b 1 +) + +REM Ladataan muuttujat .env tiedostosta. +for /f "usebackq tokens=1,* delims==" %%a in ("%APPDATA%\jkr\.env") do ( + set "%%a=%%b" +) + +REM Tarkistetaan onko tarvittavat muuttujat asetettu. +if "%JKR_DB_HOST%"=="" ( + echo Error: HOST variable not set in .env file + exit /b 1 +) +if "%JKR_TEST_DB_PORT%"=="" ( + echo Error: PORT variable not set in .env file + exit /b 1 +) +if "%JKR_TEST_DB%"=="" ( + echo Error: DB_NAME variable not set in .env file + exit /b 1 +) +if "%JKR_USER%"=="" ( + echo Error: USER variable not set in .env file + exit /b 1 +) +if "%QGIS_BIN_PATH%"=="" ( + echo Error: QGIS_BIN_PATH variable not set in .env file + exit /b1 +) + + +SET PGPASSWORD=%JKR_PASSWORD% ECHO Rakennukset -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "./data/test_data_import/DVV_update.xlsx" "R1 rakennus" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln rakennus "./data/test_data_import/DVV_update.xlsx" "R1 rakennus" ECHO Osoitteet -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "./data/test_data_import/DVV_update.xlsx" "R3 osoite" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln osoite "./data/test_data_import/DVV_update.xlsx" "R3 osoite" ECHO Omistajat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "./data/test_data_import/DVV_update.xlsx" "R4 omistaja" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln omistaja "./data/test_data_import/DVV_update.xlsx" "R4 omistaja" ECHO Asukkaat -%OGR2OGR_PATH%\\ogr2ogr -f PostgreSQL -overwrite -progress PG:"host=%HOST% port=%PORT% dbname=%DB_NAME% user=%USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "./data/test_data_import/DVV_update.xlsx" "R9 huon asukk" +"%QGIS_BIN_PATH%\\ogr2ogr" -f PostgreSQL -overwrite -progress PG:"host=%JKR_DB_HOST% port=%JKR_TEST_DB_PORT% dbname=%JKR_TEST_DB% user=%JKR_USER% ACTIVE_SCHEMA=jkr_dvv" -nln vanhin "./data/test_data_import/DVV_update.xlsx" "R9 huon asukk" ECHO Muunnetaan jkr-muotoon... -psql -h %HOST% -p %PORT% -d %DB_NAME% -U %USER% -v formatted_date=20230131 -f "../scripts/import_dvv.sql" +"%QGIS_BIN_PATH%\\psql" -h %JKR_DB_HOST% -p %JKR_TEST_DB_PORT% -d %JKR_TEST_DB% -U %JKR_USER% -v formatted_date=20230131 -f "../scripts/import_dvv.sql" diff --git a/tests/test_data_import.py b/tests/test_data_import.py index d6630721..1b892907 100644 --- a/tests/test_data_import.py +++ b/tests/test_data_import.py @@ -1,5 +1,5 @@ -from datetime import datetime import subprocess +from datetime import datetime import pytest from sqlalchemy import and_, create_engine, distinct, func, or_, select @@ -9,13 +9,20 @@ from jkrimporter.providers.db.codes import init_code_objects from jkrimporter.providers.db.database import json_dumps from jkrimporter.providers.db.dbprovider import import_dvv_kohteet -from jkrimporter.providers.db.models import Kohde, KohteenOsapuolet, Osapuoli, Osapuolenrooli +from jkrimporter.providers.db.models import ( + Kohde, + KohteenOsapuolet, + Osapuolenrooli, + Osapuoli, +) @pytest.fixture(scope="module", autouse=True) def engine(): engine = create_engine( - "postgresql://{username}:{password}@{host}:{port}/{dbname}".format(**conf.dbconf), + "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( + **conf.dbconf + ), future=True, json_serializer=json_dumps ) diff --git a/tests/test_kompostori.py b/tests/test_kompostori.py index f5368231..1028d45a 100644 --- a/tests/test_kompostori.py +++ b/tests/test_kompostori.py @@ -1,17 +1,22 @@ -import pytest +import json import os -from openpyxl.reader.excel import load_workbook +import pytest +from openpyxl.reader.excel import load_workbook from sqlalchemy import create_engine, func from sqlalchemy.orm import Session from jkrimporter import conf from jkrimporter.cli.jkr import import_ilmoitukset -from jkrimporter.providers.db.database import json_dumps +from jkrimporter.providers.db.database import JSONEncoderWithDateSupport, json_dumps from jkrimporter.providers.db.models import Kompostori, KompostorinKohteet from jkrimporter.providers.lahti.ilmoitustiedosto import Ilmoitustiedosto +# Redefine json_dumps with test-specific behavior +def json_dumps(value): + return json.dumps(value, cls=JSONEncoderWithDateSupport) + @pytest.fixture(scope="module", autouse=True) def engine(): engine = create_engine( @@ -19,15 +24,13 @@ def engine(): **conf.dbconf ), future=True, - json_serializer=json_dumps, + json_serializer=json_dumps, # Use the test-specific json_dumps function here ) return engine - def test_readable(datadir): assert Ilmoitustiedosto.readable_by_me(datadir + "/ilmoitukset.xlsx") - def test_kompostori(engine, datadir): import_ilmoitukset(datadir + "/ilmoitukset.xlsx") session = Session(engine) @@ -45,7 +48,6 @@ def test_kompostori(engine, datadir): sheet = workbook[workbook.sheetnames[0]] assert sheet.max_row == 6 - def test_kompostori_osakkaan_lisays(engine, datadir): import_ilmoitukset(datadir + "/ilmoitukset_lisaa_komposti_osakas.xlsx") session = Session(engine) diff --git a/tests/test_kompostori_end_dates.py b/tests/test_kompostori_end_dates.py index a373485a..810e3b1c 100644 --- a/tests/test_kompostori_end_dates.py +++ b/tests/test_kompostori_end_dates.py @@ -1,8 +1,8 @@ -import pytest -import os import datetime -from openpyxl.reader.excel import load_workbook +import os +import pytest +from openpyxl.reader.excel import load_workbook from sqlalchemy import create_engine, func from sqlalchemy.orm import Session diff --git a/tests/test_lahti_siirtotiedosto.py b/tests/test_lahti_siirtotiedosto.py index 870df45c..391f27d8 100644 --- a/tests/test_lahti_siirtotiedosto.py +++ b/tests/test_lahti_siirtotiedosto.py @@ -1,8 +1,8 @@ import csv import os from pathlib import Path -import pytest +import pytest from sqlalchemy import create_engine, func, or_, select, text from sqlalchemy.orm import Session diff --git a/tests/test_viranomaispaatokset.py b/tests/test_viranomaispaatokset.py index c6a3873c..6bff473e 100644 --- a/tests/test_viranomaispaatokset.py +++ b/tests/test_viranomaispaatokset.py @@ -1,7 +1,7 @@ -from datetime import date import os -import pytest +from datetime import date +import pytest from openpyxl.reader.excel import load_workbook from sqlalchemy import create_engine, func from sqlalchemy.orm import Session @@ -23,6 +23,7 @@ @pytest.fixture(scope="module", autouse=True) def engine(): engine = create_engine( + # "postgresql://{username}:{test_password}@{host}:{test_port}/{test_dbname}".format( "postgresql://{username}:{password}@{host}:{port}/{dbname}".format( **conf.dbconf ),