Skip to content

Commit

Permalink
Merge pull request #592 from MTES-MCT/staging
Browse files Browse the repository at this point in the history 
MEP v8.1.2
  • Loading branch information
@alexisig
alexisig authored Oct 2, 2024
2 parents f5ec5eb + 876fdf0 commit 4921457
Show file tree
Hide file tree
Showing 18 changed files with 616 additions and 454 deletions.
27 changes: 18 additions & 9 deletions airflow/dags/ingest_gpu.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,21 +20,23 @@
def ingest_gpu():
bucket_name = "airflow-staging"
wfs_du_filename = "wfs_du.gpkg"
path_on_bucket = f"{bucket_name}/gpu/{wfs_du_filename}"
wfs_folder = "/pub/export-wfs/latest/gpkg/"
wfs_filepath = f"{wfs_folder}{wfs_du_filename}"
localpath = f"/tmp/{wfs_du_filename}"

@task.python
def download() -> str:
path_on_bucket = f"{bucket_name}/gpu/{wfs_du_filename}"
with Container.gpu_sftp() as sftp:
sftp.get(f"/pub/export-wfs/latest/gpkg/{wfs_du_filename}", f"/tmp/{wfs_du_filename}")
sftp.get(wfs_filepath, localpath)

Container().s3().put_file(f"/tmp/{wfs_du_filename}", path_on_bucket)
Container().s3().put_file(localpath, path_on_bucket)

return path_on_bucket

@task.python
def ingest(path_on_bucket: str) -> str:
wfs_du_temp = f"/tmp/{wfs_du_filename}"
Container().s3().get_file(path_on_bucket, wfs_du_temp)
def ingest():
Container().s3().get_file(path_on_bucket, localpath)
sql = """
SELECT
MD5Checksum(ST_AsText(geom)) AS checksum,
Expand Down Expand Up @@ -81,7 +83,7 @@ def ingest(path_on_bucket: str) -> str:
"MULTIPOLYGON",
"-nlt",
"PROMOTE_TO_MULTI",
wfs_du_temp,
localpath,
"-sql",
f'"{multiline_string_to_single_line(sql)}"',
"--config",
Expand All @@ -93,8 +95,15 @@ def ingest(path_on_bucket: str) -> str:
bash_command=" ".join(cmd),
).execute(context={})

path_on_bucket = download()
ingest(path_on_bucket)
@task.bash
def dbt_build() -> str:
return 'cd "${AIRFLOW_HOME}/include/sql/sparte" && dbt build -s zonage_urbanisme.sql+'

@task.bash
def cleanup() -> str:
return f"rm -f {localpath}"

download() >> ingest() >> dbt_build() >> cleanup()


ingest_gpu()
133 changes: 133 additions & 0 deletions project/migrations/0090_alter_historicalrequest_first_name_and_more.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
# Generated by Django 4.2.13 on 2024-09-30 09:42

from django.db import migrations, models
import utils.validators


class Migration(migrations.Migration):
dependencies = [
("project", "0089_rnupackagerequest"),
]

operations = [
migrations.AlterField(
model_name="historicalrequest",
name="first_name",
field=models.CharField(
max_length=150, null=True, validators=[utils.validators.is_alpha_validator], verbose_name="Prénom"
),
),
migrations.AlterField(
model_name="historicalrequest",
name="function",
field=models.CharField(
max_length=250, null=True, validators=[utils.validators.is_alpha_validator], verbose_name="Fonction"
),
),
migrations.AlterField(
model_name="historicalrequest",
name="last_name",
field=models.CharField(
max_length=150, null=True, validators=[utils.validators.is_alpha_validator], verbose_name="Nom"
),
),
migrations.AlterField(
model_name="historicalrequest",
name="organism",
field=models.CharField(
choices=[
("AGENCE", "Agence d'urbanisme"),
("ASSOCI", "Association"),
("BUREAU", "Bureau d'études"),
("COMMUN", "Commune"),
("DDT", "DDT"),
("DDTM", "DDTM"),
("DEAL", "DEAL"),
("DREAL", "DREAL"),
("DRIEAT", "DRIEAT"),
("EPCI", "EPCI"),
("PARTIC", "Particulier"),
("SCOT", "SCOT"),
("AUTRE", "Autre"),
],
default="COMMUN",
max_length=30,
validators=[utils.validators.is_alpha_validator],
verbose_name="Organisme",
),
),
migrations.AlterField(
model_name="historicalrequest",
name="requested_document",
field=models.CharField(
choices=[
("rapport-conso", "Rapport de consommation"),
("rapport-complet", "Rapport complet"),
("rapport-local", "Rapport local"),
],
max_length=30,
validators=[utils.validators.is_alpha_validator],
verbose_name="Document demandé",
),
),
migrations.AlterField(
model_name="request",
name="first_name",
field=models.CharField(
max_length=150, null=True, validators=[utils.validators.is_alpha_validator], verbose_name="Prénom"
),
),
migrations.AlterField(
model_name="request",
name="function",
field=models.CharField(
max_length=250, null=True, validators=[utils.validators.is_alpha_validator], verbose_name="Fonction"
),
),
migrations.AlterField(
model_name="request",
name="last_name",
field=models.CharField(
max_length=150, null=True, validators=[utils.validators.is_alpha_validator], verbose_name="Nom"
),
),
migrations.AlterField(
model_name="request",
name="organism",
field=models.CharField(
choices=[
("AGENCE", "Agence d'urbanisme"),
("ASSOCI", "Association"),
("BUREAU", "Bureau d'études"),
("COMMUN", "Commune"),
("DDT", "DDT"),
("DDTM", "DDTM"),
("DEAL", "DEAL"),
("DREAL", "DREAL"),
("DRIEAT", "DRIEAT"),
("EPCI", "EPCI"),
("PARTIC", "Particulier"),
("SCOT", "SCOT"),
("AUTRE", "Autre"),
],
default="COMMUN",
max_length=30,
validators=[utils.validators.is_alpha_validator],
verbose_name="Organisme",
),
),
migrations.AlterField(
model_name="request",
name="requested_document",
field=models.CharField(
choices=[
("rapport-conso", "Rapport de consommation"),
("rapport-complet", "Rapport complet"),
("rapport-local", "Rapport local"),
],
max_length=30,
validators=[utils.validators.is_alpha_validator],
verbose_name="Document demandé",
),
),
]
32 changes: 32 additions & 0 deletions project/migrations/0091_auto_20240930_1152.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Generated by Django 4.2.13 on 2024-09-30 09:52

from django.db import migrations

from utils.validators import is_alpha_valid


def remove_invalid_user_inputs(apps, schema_editor):
Request = apps.get_model("project", "Request")

for request in Request.objects.all():
if not is_alpha_valid(request.first_name):
request.first_name = "empty"
if not is_alpha_valid(request.last_name):
request.last_name = "empty"
if not is_alpha_valid(request.function):
request.function = "empty"
if not is_alpha_valid(request.organism):
request.organism = "COMMUN"
if not is_alpha_valid(request.requested_document):
request.requested_document = "rapport-complet"
request.save()


class Migration(migrations.Migration):
dependencies = [
("project", "0090_alter_historicalrequest_first_name_and_more"),
]

operations = [
migrations.RunPython(remove_invalid_user_inputs),
]
83 changes: 83 additions & 0 deletions project/migrations/0092_remove_historicalproject_couverture_usage_and_more.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# Generated by Django 4.2.13 on 2024-10-01 09:14

from django.db import migrations, models
import utils.validators


class Migration(migrations.Migration):
dependencies = [
("project", "0091_auto_20240930_1152"),
]

operations = [
migrations.RemoveField(
model_name="historicalproject",
name="couverture_usage",
),
migrations.RemoveField(
model_name="historicalproject",
name="import_date",
),
migrations.RemoveField(
model_name="historicalproject",
name="import_error",
),
migrations.RemoveField(
model_name="historicalproject",
name="import_status",
),
migrations.RemoveField(
model_name="project",
name="couverture_usage",
),
migrations.RemoveField(
model_name="project",
name="import_date",
),
migrations.RemoveField(
model_name="project",
name="import_error",
),
migrations.RemoveField(
model_name="project",
name="import_status",
),
migrations.AlterField(
model_name="historicalproject",
name="name",
field=models.CharField(
max_length=100, validators=[utils.validators.is_alpha_validator], verbose_name="Nom"
),
),
migrations.AlterField(
model_name="historicalproject",
name="territory_name",
field=models.CharField(
blank=True,
help_text="C'est le nom qui est utilisé pour désigner votre territoire, notamment dans le rapport word.",
max_length=250,
null=True,
validators=[utils.validators.is_alpha_validator],
verbose_name="Territoire",
),
),
migrations.AlterField(
model_name="project",
name="name",
field=models.CharField(
max_length=100, validators=[utils.validators.is_alpha_validator], verbose_name="Nom"
),
),
migrations.AlterField(
model_name="project",
name="territory_name",
field=models.CharField(
blank=True,
help_text="C'est le nom qui est utilisé pour désigner votre territoire, notamment dans le rapport word.",
max_length=250,
null=True,
validators=[utils.validators.is_alpha_validator],
verbose_name="Territoire",
),
),
]
3 changes: 2 additions & 1 deletion project/models/create.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,8 @@ def create_from_public_key(
user=user if user and user.is_authenticated else None,
)
project._change_reason = ProjectChangeReason.CREATED_FROM_PUBLIC_KEY
project.set_success(save=True)

project.save()

trigger_async_tasks(project, public_key)

Expand Down
Loading

0 comments on commit 4921457

Please sign in to comment.