+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
24 from io
import BytesIO
+
25 from json
import dumps, loads
+
26 from typing
import List, Optional
+
27 from zipfile
import ZipFile
+
+
+
+
+
+
+
+
+
36 import shapely.wkb
as wkblib
+
+
38 from fastapi
import File, HTTPException, UploadFile
+
39 from geoalchemy2.shape
import from_shape, to_shape
+
40 from geojson
import dump
+
41 from loguru
import logger
as log
+
42 from osm_fieldwork.basemapper
import create_basemap_file
+
43 from osm_fieldwork.data_models
import data_models_path
+
44 from osm_fieldwork.filter_data
import FilterData
+
45 from osm_fieldwork.json2osm
import json2osm
+
46 from osm_fieldwork.OdkCentral
import OdkAppUser
+
47 from osm_fieldwork.xlsforms
import xlsforms_path
+
48 from osm_rawdata.postgres
import PostgresClient
+
49 from shapely
import wkt
+
50 from shapely.geometry
import (
+
+
+
+
+
+
+
+
58 from shapely.ops
import transform
+
59 from sqlalchemy
import and_, column, func, inspect, select, table, text
+
60 from sqlalchemy.dialects.postgresql
import insert
+
61 from sqlalchemy.orm
import Session
+
62 from sqlalchemy.sql
import text
+
+
64 from ..central
import central_crud
+
65 from ..config
import settings
+
66 from ..db
import database, db_models
+
67 from ..db.postgis_utils
import geometry_to_geojson, timestamp
+
68 from ..tasks
import tasks_crud
+
69 from ..users
import user_crud
+
70 from .
import project_schemas
+
+
72 QR_CODES_DIR =
"QR_codes/"
+
73 TASK_GEOJSON_DIR =
"geojson/"
+
74 TILESDIR =
"/opt/tiles"
+
+
+
+
+
+
+
+
82 db_objects: bool =
False,
+
83 hashtags: List[str] =
None,
+
+
+
+
+
88 filters.append(db_models.DbProject.author_id == user_id)
+
+
+
91 filters.append(db_models.DbProject.hashtags.op(
"&&")(hashtags))
+
+
+
94 filters.append(db_models.DbProject.project_name_prefix.ilike(f
"%{search}%"))
+
+
+
+
98 db.query(db_models.DbProject)
+
99 .filter(and_(*filters))
+
100 .order_by(db_models.DbProject.id.desc())
+
+
+
+
+
105 project_count = db.query(db_models.DbProject).filter(and_(*filters)).count()
+
+
+
+
109 db.query(db_models.DbProject)
+
110 .order_by(db_models.DbProject.id.desc())
+
+
+
+
+
115 project_count = db.query(db_models.DbProject).count()
+
+
117 return project_count, db_projects
+
118 return project_count, convert_to_app_projects(db_projects)
+
+
+
121 def get_project_summaries(
+
+
+
+
+
126 hashtags: str =
None,
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
144 project_count, db_projects = get_projects(
+
145 db, user_id, skip, limit,
True, hashtags, search
+
+
147 return project_count, convert_to_project_summaries(db_projects)
+
+
+
150 def get_project(db: Session, project_id: int):
+
+
152 db.query(db_models.DbProject)
+
153 .filter(db_models.DbProject.id == project_id)
+
+
+
+
+
+
159 def get_project_by_id(db: Session, project_id: int):
+
+
161 db.query(db_models.DbProject)
+
162 .filter(db_models.DbProject.id == project_id)
+
163 .order_by(db_models.DbProject.id)
+
+
+
166 return convert_to_app_project(db_project)
+
+
+
169 def get_project_info_by_id(db: Session, project_id: int):
+
+
171 db.query(db_models.DbProjectInfo)
+
172 .filter(db_models.DbProjectInfo.project_id == project_id)
+
173 .order_by(db_models.DbProjectInfo.project_id)
+
+
+
176 return convert_to_app_project_info(db_project_info)
+
+
+
179 def delete_project_by_id(db: Session, project_id: int):
+
+
+
182 db.query(db_models.DbProject)
+
183 .filter(db_models.DbProject.id == project_id)
+
184 .order_by(db_models.DbProject.id)
+
+
+
+
188 db.delete(db_project)
+
+
190 except Exception
as e:
+
+
192 raise HTTPException(e)
from e
+
193 return f
"Project {project_id} deleted"
+
+
+
196 def partial_update_project_info(
+
197 db: Session, project_metadata: project_schemas.ProjectUpdate, project_id
+
+
+
200 db_project = get_project_by_id(db, project_id)
+
+
+
+
+
205 status_code=428, detail=f
"Project with id {project_id} does not exist"
+
+
+
+
209 db_project_info = get_project_info_by_id(db, project_id)
+
+
+
212 if project_metadata.name:
+
213 db_project.project_name_prefix = project_metadata.name
+
214 db_project_info.name = project_metadata.name
+
215 if project_metadata.description:
+
216 db_project_info.description = project_metadata.description
+
217 if project_metadata.short_description:
+
218 db_project_info.short_description = project_metadata.short_description
+
+
+
221 db.refresh(db_project)
+
+
223 return convert_to_app_project(db_project)
+
+
+
226 def update_project_info(
+
227 db: Session, project_metadata: project_schemas.ProjectUpload, project_id
+
+
229 user = project_metadata.author
+
230 project_info = project_metadata.project_info
+
+
+
+
234 raise HTTPException(
"No user passed in")
+
+
236 raise HTTPException(
"No project info passed in")
+
+
+
239 db_user = user_crud.get_user(db, user.id)
+
+
+
242 status_code=400, detail=f
"User {user.username} does not exist"
+
+
+
+
246 db_project = get_project_by_id(db, project_id)
+
+
+
249 status_code=428, detail=f
"Project with id {project_id} does not exist"
+
+
+
+
253 project_info = project_metadata.project_info
+
+
+
256 db_project.author = db_user
+
257 db_project.project_name_prefix = project_info.name
+
+
+
260 db_project_info = get_project_info_by_id(db, project_id)
+
+
+
263 db_project_info.name = project_info.name
+
264 db_project_info.short_description = project_info.short_description
+
265 db_project_info.description = project_info.description
+
+
+
268 db.refresh(db_project)
+
+
270 return convert_to_app_project(db_project)
+
+
+
273 def create_project_with_project_info(
+
274 db: Session, project_metadata: project_schemas.ProjectUpload, odk_project_id: int
+
+
276 project_user = project_metadata.author
+
277 project_info = project_metadata.project_info
+
278 xform_title = project_metadata.xform_title
+
279 odk_credentials = project_metadata.odk_central
+
280 hashtags = project_metadata.hashtags
+
281 organisation_id = project_metadata.organisation_id
+
282 task_split_type = project_metadata.task_split_type
+
283 task_split_dimension = project_metadata.task_split_dimension
+
284 task_num_buildings = project_metadata.task_num_buildings
+
+
+
+
288 raise HTTPException(
"User details are missing")
+
+
290 raise HTTPException(
"Project info is missing")
+
291 if not odk_project_id:
+
292 raise HTTPException(
"ODK Central project id is missing")
+
+
+
295 "Creating project in FMTM database with vars: "
+
296 f
"project_user: {project_user} | "
+
297 f
"project_info: {project_info} | "
+
298 f
"xform_title: {xform_title} | "
+
299 f
"hashtags: {hashtags}| "
+
300 f
"organisation_id: {organisation_id}"
+
+
+
+
+
305 url = odk_credentials.odk_central_url
+
306 user = odk_credentials.odk_central_user
+
307 pw = odk_credentials.odk_central_password
+
+
+
310 log.debug(
"ODKCentral connection variables not set in function")
+
311 log.debug(
"Attempting extraction from environment variables")
+
312 url = settings.ODK_CENTRAL_URL
+
313 user = settings.ODK_CENTRAL_USER
+
314 pw = settings.ODK_CENTRAL_PASSWD
+
+
+
+
+
319 db_user = user_crud.get_user(db, project_user.id)
+
+
+
322 status_code=400, detail=f
"User {project_user.username} does not exist"
+
+
+
+
+
+
328 lambda hashtag: hashtag
if hashtag.startswith(
"#")
else f
"#{hashtag}",
+
+
+
+
+
+
+
+
336 db_project = db_models.DbProject(
+
+
338 odkid=odk_project_id,
+
339 project_name_prefix=project_info.name,
+
340 xform_title=xform_title,
+
+
342 odk_central_user=user,
+
343 odk_central_password=pw,
+
+
345 organisation_id=organisation_id,
+
346 task_split_type=task_split_type,
+
347 task_split_dimension=task_split_dimension,
+
348 task_num_buildings=task_num_buildings,
+
+
+
+
+
+
+
355 db_project_info = db_models.DbProjectInfo(
+
+
357 name=project_info.name,
+
358 short_description=project_info.short_description,
+
359 description=project_info.description,
+
+
361 db.add(db_project_info)
+
+
+
364 db.refresh(db_project)
+
+
366 return convert_to_app_project(db_project)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
384 ins = insert(forms).values(title=name, xls=xlsform, category=category)
+
385 sql = ins.on_conflict_do_update(
+
386 constraint=
"xlsforms_title_key",
+
387 set_=dict(title=name, xls=xlsform, category=category),
+
+
+
+
+
392 except Exception
as e:
+
393 raise HTTPException(status=400, detail={
"message": str(e)})
from e
+
+
+
396 def update_multi_polygon_project_boundary(
+
+
+
+
+
401 """This function receives the project_id and boundary as a parameter
+
402 and creates a task for each polygon in the database.
+
403 This function also creates a project outline from the multiple polygons received.
+
+
+
406 if isinstance(boundary, str):
+
407 boundary = json.loads(boundary)
+
+
+
410 db_project = get_project_by_id(db, project_id)
+
+
412 log.error(f
"Project {project_id} doesn't exist!")
+
+
+
+
416 if boundary[
"type"] ==
"Feature":
+
417 polygons = [boundary]
+
+
419 polygons = boundary[
"features"]
+
420 log.debug(f
"Processing {len(polygons)} task geometries")
+
421 for polygon
in polygons:
+
+
423 if polygon[
"geometry"][
"type"] ==
"MultiPolygon":
+
424 log.debug(
"Converting MultiPolygon to Polygon")
+
425 polygon[
"geometry"][
"type"] =
"Polygon"
+
426 polygon[
"geometry"][
"coordinates"] = polygon[
"geometry"][
"coordinates"][
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
441 db_task = db_models.DbTask(
+
442 project_id=project_id,
+
443 outline=wkblib.dumps(shape(polygon[
"geometry"]), hex=
True),
+
444 project_task_index=1,
+
+
+
+
+
+
450 db_task.project_task_name = str(db_task.id)
+
+
452 "Created database task | "
+
453 f
"Project ID {project_id} | "
+
454 f
"Task ID {db_task.project_task_name}"
+
+
+
+
+
+
460 f
"""SELECT ST_AsText(ST_ConvexHull(ST_Collect(outline)))
+
+
462 WHERE project_id={project_id};"""
+
+
+
465 log.debug(
"Generating project outline from tasks")
+
466 result = db.execute(query)
+
467 data = result.fetchone()
+
+
469 update_project_location_info(db_project, data[0])
+
+
+
472 db.refresh(db_project)
+
473 log.debug(
"COMPLETE: creating project boundary, based on task boundaries")
+
+
+
476 except Exception
as e:
+
+
478 raise HTTPException(e)
from e
+
+
+
481 def preview_tasks(boundary: str, dimension: int):
+
482 """Preview tasks by returning a list of task objects."""
+
483 """Use a lambda function to remove the "z" dimension from each coordinate in the feature's geometry """
+
+
485 def remove_z_dimension(coord):
+
486 return coord.pop()
if len(coord) == 3
else None
+
+
488 """ Check if the boundary is a Feature or a FeatureCollection """
+
489 if boundary[
"type"] ==
"Feature":
+
490 features = [boundary]
+
491 elif boundary[
"type"] ==
"FeatureCollection":
+
492 features = boundary[
"features"]
+
493 elif boundary[
"type"] ==
"Polygon":
+
+
+
+
+
498 "geometry": boundary,
+
+
+
+
+
503 status_code=400, detail=f
"Invalid GeoJSON type: {boundary['type']}"
+
+
+
506 """ Apply the lambda function to each coordinate in its geometry ro remove the z-dimension - if it exists"""
+
+
508 for feature
in features:
+
509 list(map(remove_z_dimension, feature[
"geometry"][
"coordinates"][0]))
+
510 if feature[
"geometry"][
"type"] ==
"MultiPolygon":
+
511 multi_polygons.append(Polygon(feature[
"geometry"][
"coordinates"][0][0]))
+
+
513 """Update the boundary polyon on the database."""
+
+
515 boundary = multi_polygons[0]
+
516 for geom
in multi_polygons[1:]:
+
517 boundary = boundary.union(geom)
+
+
519 boundary = shape(features[0][
"geometry"])
+
+
521 minx, miny, maxx, maxy = boundary.bounds
+
+
+
524 value = dimension / 111139
+
+
526 nx = int((maxx - minx) / value)
+
527 ny = int((maxy - miny) / value)
+
+
+
530 xdiff = abs(maxx - minx)
+
531 ydiff = abs(maxy - miny)
+
+
533 gx, gy = np.linspace(minx, maxx, ny), np.linspace(miny, miny + xdiff, ny)
+
+
535 gx, gy = np.linspace(minx, minx + ydiff, nx), np.linspace(miny, maxy, nx)
+
+
+
+
539 for i
in range(len(gx) - 1):
+
540 for j
in range(len(gy) - 1):
+
+
+
+
+
545 [gx[i + 1], gy[j + 1]],
+
+
+
+
+
+
551 if boundary.intersection(poly):
+
552 feature = geojson.Feature(
+
553 geometry=boundary.intersection(poly), properties={
"id": str(id)}
+
+
+
+
557 geom = shape(feature[
"geometry"])
+
+
559 if geom.geom_type ==
"MultiPolygon":
+
+
561 polygons = geom.geoms
+
+
563 for x
in range(len(polygons)):
+
+
+
+
+
568 "geometry": mapping(polygons[x]),
+
+
570 grid.append(feature1)
+
+
+
+
574 collection = geojson.FeatureCollection(grid)
+
+
+
+
578 if len(collection[
"features"]) == 0:
+
579 boundary = mapping(boundary)
+
+
581 "type":
"FeatureCollection",
+
582 "features": [{
"type":
"Feature",
"geometry": boundary,
"properties": {}}],
+
+
+
+
+
+
+
589 def get_osm_extracts(boundary: str):
+
590 """Request an extract from raw-data-api and extract the file contents.
+
+
592 - The query is posted to raw-data-api and job initiated for fetching the extract.
+
593 - The status of the job is polled every few seconds, until 'SUCCESS' is returned.
+
594 - The resulting zip file is downloaded, extracted, and data returned.
+
+
+
+
+
+
+
601 "join_or": {
"building": [],
"highway": [],
"waterway": []}
+
+
+
+
+
+
+
608 json_boundary = json.loads(boundary)
+
609 if json_boundary.get(
"features",
None)
is not None:
+
610 query[
"geometry"] = json_boundary
+
+
+
613 query[
"geometry"] = json_boundary
+
+
+
616 query[
"fileName"] =
"extract"
+
+
618 query[
"outputType"] =
"geojson"
+
619 extract_filename = f
'{query["fileName"]}.{query["outputType"]}'
+
620 log.debug(f
"Setting data extract file name to: {extract_filename}")
+
+
622 log.debug(f
"Query for raw data api: {query}")
+
623 base_url = settings.UNDERPASS_API_URL
+
624 query_url = f
"{base_url}/snapshot/"
+
625 headers = {
"accept":
"application/json",
"Content-Type":
"application/json"}
+
+
+
628 result = requests.post(query_url, data=json.dumps(query), headers=headers)
+
+
630 if result.status_code == 200:
+
631 task_id = result.json()[
"task_id"]
+
+
+
+
+
636 task_url = f
"{base_url}/tasks/status/{task_id}"
+
+
638 result = requests.get(task_url, headers=headers)
+
639 if result.json()[
"status"] ==
"PENDING":
+
+
+
642 elif result.json()[
"status"] ==
"SUCCESS":
+
+
+
+
+
+
648 zip_url = result.json()[
"result"][
"download_url"]
+
649 result = requests.get(zip_url, headers=headers)
+
+
651 fp = BytesIO(result.content)
+
652 zfp = zipfile.ZipFile(fp,
"r")
+
653 zfp.extract(extract_filename,
"/tmp/")
+
654 data = json.loads(zfp.read(extract_filename))
+
+
656 for feature
in data[
"features"]:
+
657 properties = feature[
"properties"]
+
658 tags = properties.pop(
"tags", {})
+
659 properties.update(tags)
+
+
+
+
+
664 def split_into_tasks(
+
665 db: Session, outline: str, no_of_buildings: int, has_data_extracts: bool
+
+
667 """Splits a project into tasks.
+
+
+
670 db (Session): A database session.
+
671 boundary (str): A GeoJSON string representing the boundary of the project to split into tasks.
+
672 no_of_buildings (int): The number of buildings to include in each task.
+
+
+
675 Any: A GeoJSON object containing the tasks for the specified project.
+
+
677 project_id = uuid.uuid4()
+
+
+
+
+
682 if outline[
"type"] ==
"FeatureCollection":
+
683 log.debug(
"Project boundary GeoJSON = FeatureCollection")
+
684 boundary_data.extend(feature[
"geometry"]
for feature
in outline[
"features"])
+
+
686 split_polygon_into_tasks(
+
687 db, project_id, data, no_of_buildings, has_data_extracts
+
+
689 for data
in boundary_data
+
+
+
692 for inner_list
in result:
+
+
694 all_results.extend(iter(inner_list))
+
+
696 elif outline[
"type"] ==
"GeometryCollection":
+
697 log.debug(
"Project boundary GeoJSON = GeometryCollection")
+
698 geometries = outline[
"geometries"]
+
699 boundary_data.extend(iter(geometries))
+
+
701 split_polygon_into_tasks(
+
702 db, project_id, data, no_of_buildings, has_data_extracts
+
+
704 for data
in boundary_data
+
+
706 for inner_list
in result:
+
+
708 all_results.extend(iter(inner_list))
+
+
710 elif outline[
"type"] ==
"Feature":
+
711 log.debug(
"Project boundary GeoJSON = Feature")
+
712 boundary_data = outline[
"geometry"]
+
713 result = split_polygon_into_tasks(
+
714 db, project_id, boundary_data, no_of_buildings, has_data_extracts
+
+
716 all_results.extend(iter(result))
+
+
718 elif outline[
"type"] ==
"Polygon":
+
719 log.debug(
"Project boundary GeoJSON = Polygon")
+
720 boundary_data = outline
+
721 result = split_polygon_into_tasks(
+
722 db, project_id, boundary_data, no_of_buildings, has_data_extracts
+
+
724 all_results.extend(result)
+
+
+
+
728 "Project boundary not one of: Polygon, Feature, GeometryCollection,"
+
729 " FeatureCollection. Task splitting failed."
+
+
+
732 "type":
"FeatureCollection",
+
733 "features": all_results,
+
+
+
+
737 def split_polygon_into_tasks(
+
+
739 project_id: uuid.UUID,
+
+
741 no_of_buildings: int,
+
742 has_data_extracts: bool,
+
+
744 outline = shape(boundary_data)
+
745 db_task = db_models.DbProjectAOI(
+
746 project_id=project_id,
+
+
+
+
+
+
+
+
+
+
756 if not has_data_extracts:
+
757 data = get_osm_extracts(json.dumps(boundary_data))
+
+
+
760 for feature
in data[
"features"]:
+
761 feature_shape = shape(feature[
"geometry"])
+
762 wkb_element = from_shape(feature_shape, srid=4326)
+
763 if feature[
"properties"].get(
"building") ==
"yes":
+
764 db_feature = db_models.DbBuildings(
+
765 project_id=project_id, geom=wkb_element, tags=feature[
"properties"]
+
+
+
768 elif "highway" in feature[
"properties"]:
+
769 db_feature = db_models.DbOsmLines(
+
770 project_id=project_id, geom=wkb_element, tags=feature[
"properties"]
+
+
+
+
+
+
+
+
+
779 DELETE FROM ways_poly
+
780 WHERE NOT ST_Within(ST_Centroid(ways_poly.geom), (SELECT geom FROM project_aoi WHERE project_id = '{project_id}'));
+
+
+
783 result = db.execute(query)
+
+
+
+
787 with open(
"app/db/split_algorithm.sql",
"r")
as sql_file:
+
788 query = sql_file.read()
+
789 log.debug(f
"STARTED project {project_id} task splitting")
+
790 result = db.execute(text(query), params={
"num_buildings": no_of_buildings})
+
791 result = result.fetchall()
+
792 db.query(db_models.DbBuildings).delete()
+
793 db.query(db_models.DbOsmLines).delete()
+
794 db.query(db_models.DbProjectAOI).delete()
+
+
796 log.debug(f
"COMPLETE project {project_id} task splitting")
+
+
798 features = result[0][0][
"features"]
+
+
+
801 f
"Project {project_id}: no tasks returned from splitting algorithm. "
+
802 f
"Params: 'num_buildings': {no_of_buildings}"
+
+
+
+
806 log.debug(f
"Project {project_id} split into {len(features)} tasks")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
882 def update_project_boundary(
+
883 db: Session, project_id: int, boundary: str, dimension: int
+
+
+
886 db_project = get_project_by_id(db, project_id)
+
+
888 log.error(f
"Project {project_id} doesn't exist!")
+
+
+
891 """Use a lambda function to remove the "z" dimension from each coordinate in the feature's geometry """
+
+
893 def remove_z_dimension(coord):
+
894 return coord.pop()
if len(coord) == 3
else None
+
+
896 """ Check if the boundary is a Feature or a FeatureCollection """
+
897 if boundary[
"type"] ==
"Feature":
+
898 features = [boundary]
+
899 elif boundary[
"type"] ==
"FeatureCollection":
+
900 features = boundary[
"features"]
+
901 elif boundary[
"type"] ==
"Polygon":
+
+
+
+
+
906 "geometry": boundary,
+
+
+
+
+
911 db.delete(db_project)
+
+
+
+
+
916 status_code=400, detail=f
"Invalid GeoJSON type: {boundary['type']}"
+
+
+
919 """ Apply the lambda function to each coordinate in its geometry """
+
+
921 for feature
in features:
+
922 list(map(remove_z_dimension, feature[
"geometry"][
"coordinates"][0]))
+
923 if feature[
"geometry"][
"type"] ==
"MultiPolygon":
+
924 multi_polygons.append(Polygon(feature[
"geometry"][
"coordinates"][0][0]))
+
+
926 """Update the boundary polyon on the database."""
+
+
928 outline = multi_polygons[0]
+
929 for geom
in multi_polygons[1:]:
+
930 outline = outline.union(geom)
+
+
932 outline = shape(features[0][
"geometry"])
+
+
934 update_project_location_info(db_project, outline.wkt)
+
+
+
937 db.refresh(db_project)
+
938 log.debug(
"Added project boundary!")
+
+
940 result = create_task_grid(db, project_id=project_id, delta=dimension)
+
+
+
943 db.query(db_models.DbFeatures).filter(
+
944 db_models.DbFeatures.project_id == project_id
+
+
+
+
948 db.query(db_models.DbTask).filter(
+
949 db_models.DbTask.project_id == project_id
+
+
+
+
953 for poly
in tasks[
"features"]:
+
+
955 task_name = str(poly[
"properties"][
"id"])
+
956 db_task = db_models.DbTask(
+
957 project_id=project_id,
+
958 project_task_name=task_name,
+
959 outline=wkblib.dumps(shape(poly[
"geometry"]), hex=
True),
+
+
+
+
963 project_task_index=1,
+
+
+
+
+
+
+
+
+
+
+
974 def update_project_with_zip(
+
+
+
977 project_name_prefix: str,
+
978 task_type_prefix: str,
+
979 uploaded_zip: UploadFile,
+
+
+
+
+
984 if uploaded_zip.content_type
not in [
+
+
986 "application/zip-compressed",
+
987 "application/x-zip-compressed",
+
+
+
+
991 detail=f
"File must be a zip. Uploaded file was {uploaded_zip.content_type}",
+
+
+
994 with ZipFile(io.BytesIO(uploaded_zip.file.read()),
"r")
as zip:
+
+
996 bad_file = zip.testzip()
+
+
+
999 status_code=400, detail=f
"Zip contained a bad file: {bad_file}"
+
+
+
+
1003 listed_files = zip.namelist()
+
+
1005 if QR_CODES_DIR
not in listed_files:
+
1006 raise HTTPException(
+
+
1008 detail=f
"Zip must contain directory named {QR_CODES_DIR}",
+
+
+
1011 if TASK_GEOJSON_DIR
not in listed_files:
+
1012 raise HTTPException(
+
+
1014 detail=f
"Zip must contain directory named {TASK_GEOJSON_DIR}",
+
+
+
1017 outline_filename = f
"{project_name_prefix}.geojson"
+
1018 if outline_filename
not in listed_files:
+
1019 raise HTTPException(
+
+
1021 detail=f
'Zip must contain file named "{outline_filename}" that contains a FeatureCollection outlining the project',
+
+
+
1024 task_outlines_filename = f
"{project_name_prefix}_polygons.geojson"
+
1025 if task_outlines_filename
not in listed_files:
+
1026 raise HTTPException(
+
+
1028 detail=f
'Zip must contain file named "{task_outlines_filename}" that contains a FeatureCollection where each Feature outlines a task',
+
+
+
+
1032 db_project = get_project_by_id(db, project_id)
+
+
1034 raise HTTPException(
+
1035 status_code=428, detail=f
"Project with id {project_id} does not exist"
+
+
+
+
1039 db_project.project_name_prefix = project_name_prefix
+
1040 db_project.task_type_prefix = task_type_prefix
+
+
+
1043 outline_shape = get_outline_from_geojson_file_in_zip(
+
1044 zip, outline_filename, f
"Could not generate Shape from {outline_filename}"
+
+
1046 update_project_location_info(db_project, outline_shape.wkt)
+
+
+
1049 project_tasks_feature_collection = get_json_from_zip(
+
+
1051 task_outlines_filename,
+
1052 f
"Could not generate FeatureCollection from {task_outlines_filename}",
+
+
+
+
+
+
1058 db_project.total_tasks = len(project_tasks_feature_collection[
"features"])
+
1059 for feature
in project_tasks_feature_collection[
"features"]:
+
1060 task_name = feature[
"properties"][
"task"]
+
+
+
+
1064 f
"{project_name_prefix}_{task_type_prefix}__{task_name}.png"
+
+
1066 db_qr = get_dbqrcode_from_file(
+
+
1068 QR_CODES_DIR + qr_filename,
+
1069 f
"QRCode for task {task_name} does not exist. File should be in {qr_filename}",
+
+
+
+
+
1074 task_outline_shape = get_shape_from_json_str(
+
+
1076 f
"Could not create task outline for {task_name} using {feature}",
+
+
+
+
1080 task_geojson_filename = (
+
1081 f
"{project_name_prefix}_{task_type_prefix}__{task_name}.geojson"
+
+
1083 task_geojson = get_json_from_zip(
+
+
1085 TASK_GEOJSON_DIR + task_geojson_filename,
+
1086 f
"Geojson for task {task_name} does not exist",
+
+
+
+
+
+
1092 task = db_models.DbTask(
+
1093 project_id=project_id,
+
1094 project_task_index=feature[
"properties"][
"fid"],
+
1095 project_task_name=task_name,
+
+
1097 qr_code_id=db_qr.id,
+
1098 outline=task_outline_shape.wkt,
+
+
1100 initial_feature_count=len(task_geojson[
"features"]),
+
+
+
+
+
1105 task_count = task_count + 1
+
1106 db_project.last_updated = timestamp()
+
+
+
+
1110 db.refresh(db_project)
+
+
+
+
+
1115 except HTTPException
as e:
+
+
+
+
1119 except Exception
as e:
+
1120 raise HTTPException(
+
+
1122 detail=f
"{task_count} tasks were created before the following error was thrown: {e}, on feature: {feature}",
+
+
+
+
+
+
+
+
+
+
+
+
+
1135 """Read the list of XLSForms from the disk."""
+
+
1137 package_name =
"osm_fieldwork"
+
1138 for xls
in os.listdir(directory):
+
1139 if xls.endswith(
".xls")
or xls.endswith(
".xlsx"):
+
1140 file_name = xls.split(
".")[0]
+
1141 yaml_file_name = f
"data_models/{file_name}.yaml"
+
1142 if pkg_resources.resource_exists(package_name, yaml_file_name):
+
1143 xlsforms.append(xls)
+
+
+
+
1147 inspect(db_models.DbXForm)
+
+
1149 "xlsforms", column(
"title"), column(
"xls"), column(
"xml"), column(
"id")
+
+
+
+
+
1154 for xlsform
in xlsforms:
+
1155 infile = f
"{directory}/{xlsform}"
+
1156 if os.path.getsize(infile) <= 0:
+
1157 log.warning(f
"{infile} is empty!")
+
+
1159 xls = open(infile,
"rb")
+
1160 name = xlsform.split(
".")[0]
+
+
+
+
1164 ins = insert(forms).values(title=name, xls=data)
+
1165 sql = ins.on_conflict_do_update(
+
1166 constraint=
"xlsforms_title_key", set_=dict(title=name, xls=data)
+
+
+
+
+
+
+
+
1174 def get_odk_id_for_project(db: Session, project_id: int):
+
1175 """Get the odk project id for the fmtm project id."""
+
+
+
+
+
+
1181 where = f
"id={project_id}"
+
1182 sql = select(project).where(text(where))
+
+
1184 result = db.execute(sql)
+
+
+
1187 if result.rowcount != 1:
+
1188 log.warning(str(sql))
+
+
1190 project_info = result.first()
+
1191 return project_info.odkid
+
+
+
1194 def upload_custom_data_extracts(
+
+
+
+
1198 category: str =
"buildings",
+
+
1200 """Uploads custom data extracts to the database.
+
+
+
1203 db (Session): The database session object.
+
1204 project_id (int): The ID of the project.
+
1205 contents (str): The custom data extracts contents.
+
+
+
1208 bool: True if the upload is successful.
+
+
1210 project = get_project(db, project_id)
+
1211 log.debug(f
"Uploading custom data extract for project: {project}")
+
+
+
1214 raise HTTPException(status_code=404, detail=
"Project not found")
+
+
1216 project_geojson = db.query(func.ST_AsGeoJSON(project.outline)).scalar()
+
1217 log.debug(f
"Generated project geojson: {project_geojson}")
+
1218 json.loads(project_geojson)
+
+
1220 features_data = json.loads(contents)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
1234 for feature
in features_data[
"features"]:
+
1235 feature_shape = shape(feature[
"geometry"])
+
1236 if isinstance(feature_shape, MultiPolygon):
+
1237 wkb_element = from_shape(
+
1238 Polygon(feature[
"geometry"][
"coordinates"][0][0]), srid=4326
+
+
1240 elif isinstance(feature_shape, MultiLineString):
+
1241 wkb_element = from_shape(
+
1242 LineString(feature[
"geometry"][
"coordinates"][0]), srid=4326
+
+
+
1245 wkb_element = from_shape(feature_shape, srid=4326)
+
+
+
1248 feature[
"properties"][
"title"] =
""
+
1249 properties = flatten_dict(feature[
"properties"])
+
+
1251 db_feature = db_models.DbFeatures(
+
1252 project_id=project_id,
+
1253 geometry=wkb_element,
+
1254 properties=properties,
+
1255 category_title=category,
+
+
+
+
+
+
+
+
1263 def flatten_dict(d, parent_key="", sep="_"):
+
1264 """Recursively flattens a nested dictionary into a single-level dictionary.
+
+
+
1267 d (dict): The input dictionary.
+
1268 parent_key (str): The parent key (used for recursion).
+
1269 sep (str): The separator character to use in flattened keys.
+
+
+
1272 dict: The flattened dictionary.
+
+
+
1275 for k, v
in d.items():
+
1276 new_key = f
"{parent_key}{sep}{k}" if parent_key
else k
+
1277 if isinstance(v, dict):
+
1278 items.update(flatten_dict(v, new_key, sep=sep))
+
+
+
+
+
+
1284 def generate_task_files(
+
+
+
+
+
+
1290 odk_credentials: project_schemas.ODKCentral,
+
+
1292 project_log = log.bind(task=
"create_project", project_id=project_id)
+
+
1294 project_log.info(f
"Generating files for task {task_id}")
+
1295 project = get_project(db, project_id)
+
1296 odk_id = project.odkid
+
1297 project_name = project.project_name_prefix
+
1298 category = project.xform_title
+
1299 name = f
"{project_name}_{category}_{task_id}"
+
+
+
1302 project_log.info(f
"Creating odkcentral app user for task {task_id}")
+
1303 appuser = central_crud.create_appuser(odk_id, name, odk_credentials)
+
+
+
+
1307 project_log.error(
"Couldn't create appuser for project")
+
+
+
+
1311 project_log.info(f
"Creating qr code for task {task_id}")
+
1312 create_qr = create_qrcode(
+
+
+
1315 appuser.json()[
"token"],
+
+
1317 odk_credentials.odk_central_url,
+
+
+
1320 task = tasks_crud.get_task(db, task_id)
+
1321 task.qr_code_id = create_qr[
"qr_code_id"]
+
+
+
+
+
1326 xform = f
"/tmp/{name}.xml"
+
1327 extracts = f
"/tmp/{name}.geojson"
+
+
+
1330 xform_id = f
"{name}".split(
"_")[2]
+
+
+
+
+
+
+
1337 SET task_id={task_id}
+
+
+
+
1341 WHERE project_id={project_id}
+
1342 AND ST_IsValid(geometry)
+
1343 AND ST_IsValid('{task.outline}'::Geometry)
+
1344 AND ST_Contains('{task.outline}'::Geometry, ST_Centroid(geometry))
+
+
+
+
1348 result = db.execute(query)
+
+
+
+
1352 f
"""SELECT jsonb_build_object(
+
1353 'type', 'FeatureCollection',
+
1354 'features', jsonb_agg(feature)
+
+
+
1357 SELECT jsonb_build_object(
+
+
+
1360 'geometry', ST_AsGeoJSON(geometry)::jsonb,
+
1361 'properties', properties
+
+
+
1364 WHERE project_id={project_id} and task_id={task_id}
+
+
+
+
1368 result = db.execute(query)
+
+
1370 features = result.fetchone()[0]
+
+
1372 upload_media =
False if features[
"features"]
is None else True
+
+
+
1375 with open(extracts,
"w")
as jsonfile:
+
1376 jsonfile.truncate(0)
+
1377 dump(features, jsonfile)
+
+
+
1380 f
"Generating xform for task: {task_id} "
+
1381 f
"using xform: {xform} | form_type: {form_type}"
+
+
1383 outfile = central_crud.generate_updated_xform(xlsform, xform, form_type)
+
+
+
1386 project_log.info(f
"Uploading media in {task_id}")
+
1387 result = central_crud.create_odk_xform(
+
1388 odk_id, task_id, outfile, odk_credentials,
False, upload_media
+
+
+
+
1392 project_log.info(f
"Updating role for app user in task {task_id}")
+
+
+
+
+
1397 url = odk_credentials.odk_central_url
+
1398 user = odk_credentials.odk_central_user
+
1399 pw = odk_credentials.odk_central_password
+
+
+
1402 log.debug(
"ODKCentral connection variables not set in function")
+
1403 log.debug(
"Attempting extraction from environment variables")
+
1404 url = settings.ODK_CENTRAL_URL
+
1405 user = settings.ODK_CENTRAL_USER
+
1406 pw = settings.ODK_CENTRAL_PASSWD
+
+
1408 odk_app = OdkAppUser(url, user, pw)
+
+
+
1411 projectId=odk_id, xform=xform_id, actorId=appuser.json()[
"id"]
+
+
1413 except Exception
as e:
+
+
+
1416 project.extract_completed_count += 1
+
+
+
+
+
+
+
1423 def generate_task_files_wrapper(project_id, task, xlsform, form_type, odk_credentials):
+
1424 for db
in database.get_db():
+
1425 generate_task_files(db, project_id, task, xlsform, form_type, odk_credentials)
+
+
+
1428 def generate_appuser_files(
+
+
+
1431 extract_polygon: bool,
+
1432 custom_xls_form: str,
+
1433 extracts_contents: str,
+
+
+
1436 background_task_id: Optional[uuid.UUID] =
None,
+
+
1438 """Generate the files for each appuser.
+
1439 QR code, new XForm, and the OSM data extract.
+
+
+
1442 - db: the database session
+
1443 - project_id: Project ID
+
1444 - extract_polygon: boolean to determine if we should extract the polygon
+
1445 - custom_xls_form: the xls file to upload if we have a custom form
+
1446 - category: the category of the project
+
1447 - form_type: weather the form is xls, xlsx or xml
+
1448 - background_task_id: the task_id of the background task running this function.
+
+
+
1451 project_log = log.bind(task=
"create_project", project_id=project_id)
+
+
1453 project_log.info(f
"Starting generate_appuser_files for project {project_id}")
+
+
+
+
+
1458 column(
"project_name_prefix"),
+
1459 column(
"xform_title"),
+
+
1461 column(
"odk_central_url"),
+
1462 column(
"odk_central_user"),
+
1463 column(
"odk_central_password"),
+
+
+
+
1467 where = f
"id={project_id}"
+
+
1469 project.c.project_name_prefix,
+
1470 project.c.xform_title,
+
+
1472 project.c.odk_central_url,
+
1473 project.c.odk_central_user,
+
1474 project.c.odk_central_password,
+
1475 geoalchemy2.functions.ST_AsGeoJSON(project.c.outline).label(
"outline"),
+
1476 ).where(text(where))
+
1477 result = db.execute(sql)
+
+
+
1480 if result.rowcount != 1:
+
1481 log.warning(str(sql))
+
1482 if result.rowcount < 1:
+
1483 raise HTTPException(status_code=400, detail=
"Project not found")
+
+
1485 raise HTTPException(status_code=400, detail=
"Multiple projects found")
+
+
1487 one = result.first()
+
+
+
+
+
1492 "odk_central_url": one.odk_central_url,
+
1493 "odk_central_user": one.odk_central_user,
+
1494 "odk_central_password": one.odk_central_password,
+
+
+
1497 odk_credentials = project_schemas.ODKCentral(**odk_credentials)
+
+
1499 xform_title = one.xform_title
if one.xform_title
else None
+
+
1501 category = xform_title
+
+
1503 xlsform = f
"/tmp/{category}.{form_type}"
+
1504 contents = custom_xls_form
+
1505 with open(xlsform,
"wb")
as f:
+
+
+
1508 xlsform = f
"{xlsforms_path}/{xform_title}.xls"
+
+
+
1511 if extracts_contents
is not None:
+
1512 project_log.info(
"Uploading data extracts")
+
1513 upload_custom_data_extracts(db, project_id, extracts_contents)
+
+
+
+
1517 db.query(db_models.DbProject)
+
1518 .filter(db_models.DbProject.id == project_id)
+
+
+
1521 config_file_contents = project.form_config_file
+
+
1523 project_log.info(
"Extracting Data from OSM")
+
+
1525 config_path =
"/tmp/config.yaml"
+
1526 if config_file_contents:
+
1527 with open(config_path,
"w", encoding=
"utf-8")
as config_file_handle:
+
1528 config_file_handle.write(config_file_contents.decode(
"utf-8"))
+
+
1530 config_path = f
"{data_models_path}/{category}.yaml"
+
+
+
1533 pg = PostgresClient(
"underpass", config_path)
+
1534 outline = json.loads(one.outline)
+
1535 boundary = {
"type":
"Feature",
"properties": {},
"geometry": outline}
+
1536 data_extract = pg.execQuery(boundary)
+
1537 filter = FilterData(xlsform)
+
+
1539 updated_data_extract = {
"type":
"FeatureCollection",
"features": []}
+
1540 filtered_data_extract = (
+
1541 filter.cleanData(data_extract)
+
+
1543 else updated_data_extract
+
+
+
+
1547 feature_mappings = []
+
+
1549 for feature
in filtered_data_extract[
"features"]:
+
+
1551 feature[
"properties"][
"title"] =
""
+
+
1553 feature_shape = shape(feature[
"geometry"])
+
+
+
1556 if extract_polygon
and (
+
1557 not shape(outline).contains(shape(feature_shape.centroid))
+
+
+
+
1561 wkb_element = from_shape(feature_shape, srid=4326)
+
+
1563 "project_id": project_id,
+
1564 "category_title": category,
+
1565 "geometry": wkb_element,
+
1566 "properties": feature[
"properties"],
+
+
1568 updated_data_extract[
"features"].append(feature)
+
1569 feature_mappings.append(feature_mapping)
+
+
1571 db.bulk_insert_mappings(db_models.DbFeatures, feature_mappings)
+
+
+
+
1575 tasks_list = tasks_crud.get_task_lists(db, project_id)
+
+
+
+
+
+
+
+
+
+
1585 for task
in tasks_list:
+
+
1587 generate_task_files(
+
+
+
+
+
+
+
+
1595 except Exception
as e:
+
+
+
+
+
1600 if background_task_id:
+
1601 update_background_task_status_in_database(
+
1602 db, background_task_id, 4
+
+
+
1605 except Exception
as e:
+
+
+
+
1609 if background_task_id:
+
1610 update_background_task_status_in_database(
+
1611 db, background_task_id, 2, str(e)
+
+
+
+
+
+
+
+
+
+
+
+
+
1624 odk_central_url: str =
None,
+
+
+
1627 log.debug(f
"Generating base64 encoded QR settings for token: {token}")
+
1628 qrcode_data = central_crud.create_qrcode(
+
1629 odk_id, token, project_name, odk_central_url
+
+
+
1632 log.debug(
"Generating QR code from base64 settings")
+
1633 qrcode = segno.make(qrcode_data, micro=
False)
+
+
1635 log.debug(
"Saving to buffer and decoding")
+
1636 buffer = io.BytesIO()
+
1637 qrcode.save(buffer, kind=
"png", scale=5)
+
1638 qrcode_binary = buffer.getvalue()
+
+
1640 log.debug(f
"Writing QR code to database for token {token}")
+
1641 qrdb = db_models.DbQrCode(image=qrcode_binary)
+
+
+
1644 codes = table(
"qr_code", column(
"id"))
+
1645 sql = select(sqlalchemy.func.count(codes.c.id))
+
1646 result = db.execute(sql)
+
1647 rows = result.fetchone()[0]
+
1648 return {
"data": qrcode,
"id": rows + 1,
"qr_code_id": qrdb.id}
+
+
+
1651 def get_project_geometry(db: Session, project_id: int):
+
1652 """Retrieves the geometry of a project.
+
+
+
1655 db (Session): The database session.
+
1656 project_id (int): The ID of the project.
+
+
+
1659 str: A geojson of the project outline.
+
+
1661 projects = table(
"projects", column(
"outline"), column(
"id"))
+
1662 where = f
"projects.id={project_id}"
+
1663 sql = select(geoalchemy2.functions.ST_AsGeoJSON(projects.c.outline)).where(
+
+
+
1666 result = db.execute(sql)
+
+
1668 if result.rowcount != 1:
+
1669 log.warning(str(sql))
+
+
1671 row = eval(result.first()[0])
+
1672 return json.dumps(row)
+
+
+
1675 def get_task_geometry(db: Session, project_id: int):
+
1676 """Retrieves the geometry of tasks associated with a project.
+
+
+
1679 db (Session): The database session.
+
1680 project_id (int): The ID of the project.
+
+
+
1683 str: A geojson of the task boundaries
+
+
1685 db_tasks = tasks_crud.get_tasks(db, project_id,
None)
+
+
1687 for task
in db_tasks:
+
1688 geom = to_shape(task.outline)
+
+
1690 geometry = geom.__geo_interface__
+
+
+
+
1694 feature = {
"type":
"Feature",
"geometry": geometry,
"properties": properties}
+
1695 features.append(feature)
+
+
1697 feature_collection = {
"type":
"FeatureCollection",
"features": features}
+
1698 return json.dumps(feature_collection)
+
+
+
1701 async
def get_project_features_geojson(db: Session, project_id: int):
+
+
1703 db.query(db_models.DbFeatures)
+
1704 .filter(db_models.DbFeatures.project_id == project_id)
+
+
+
+
1708 """Get a geojson of all features for a task."""
+
+
1710 f
"""SELECT jsonb_build_object(
+
1711 'type', 'FeatureCollection',
+
1712 'features', jsonb_agg(feature)
+
+
+
1715 SELECT jsonb_build_object(
+
+
+
1718 'geometry', ST_AsGeoJSON(geometry)::jsonb,
+
1719 'properties', properties
+
+
+
1722 WHERE project_id={project_id}
+
+
+
+
+
1727 result = db.execute(query)
+
1728 features = result.fetchone()[0]
+
+
+
1731 task_feature_mapping = {feat.id: feat.task_id
for feat
in db_features}
+
+
1733 for feature
in features[
"features"]:
+
1734 if (feat_id := feature[
"id"])
in task_feature_mapping:
+
1735 feature[
"properties"][
"task_id"] = task_feature_mapping[feat_id]
+
+
+
+
+
1740 def create_task_grid(db: Session, project_id: int, delta: int):
+
+
+
1743 projects = table(
"projects", column(
"outline"), column(
"id"))
+
1744 where = f
"projects.id={project_id}"
+
1745 sql = select(geoalchemy2.functions.ST_AsGeoJSON(projects.c.outline)).where(
+
+
+
1748 result = db.execute(sql)
+
+
1750 if result.rowcount != 1:
+
1751 log.warning(str(sql))
+
+
1753 data = result.fetchall()
+
1754 boundary = shape(loads(data[0][0]))
+
1755 minx, miny, maxx, maxy = boundary.bounds
+
+
+
1758 value = delta / 111139
+
+
1760 nx = int((maxx - minx) / value)
+
1761 ny = int((maxy - miny) / value)
+
+
+
+
+
+
1767 gx, gy = np.linspace(minx, maxx, ny), np.linspace(miny, miny + xdiff, ny)
+
+
1769 gx, gy = np.linspace(minx, minx + ydiff, nx), np.linspace(miny, maxy, nx)
+
+
+
+
+
1774 for i
in range(len(gx) - 1):
+
1775 for j
in range(len(gy) - 1):
+
+
+
+
+
1780 [gx[i + 1], gy[j + 1]],
+
+
+
+
+
+
1786 if boundary.intersection(poly):
+
1787 feature = geojson.Feature(
+
1788 geometry=boundary.intersection(poly), properties={
"id": str(id)}
+
+
+
1791 geom = shape(feature[
"geometry"])
+
+
1793 if geom.geom_type ==
"MultiPolygon":
+
+
1795 polygons = geom.geoms
+
+
1797 for x
in range(len(polygons)):
+
+
+
+
+
1802 "properties": {
"id": str(id)},
+
1803 "geometry": mapping(polygons[x]),
+
+
1805 grid.append(feature1)
+
+
+
1808 grid.append(feature)
+
+
1810 collection = geojson.FeatureCollection(grid)
+
+
+
1813 out = dumps(collection)
+
+
+
+
1817 result = json.loads(out)
+
1818 if len(result[
"features"]) == 0:
+
1819 geom = loads(data[0][0])
+
+
1821 "type":
"FeatureCollection",
+
+
+
+
+
1826 "properties": {
"id": project_id},
+
+
+
+
1830 out = json.dumps(out)
+
+
+
1833 except Exception
as e:
+
+
+
+
1837 def get_json_from_zip(zip, filename: str, error_detail: str):
+
+
1839 with zip.open(filename)
as file:
+
+
1841 return json.loads(data)
+
1842 except Exception
as e:
+
1843 raise HTTPException(status_code=400, detail=f
"{error_detail} ----- Error: {e}")
+
+
+
1846 def get_outline_from_geojson_file_in_zip(
+
1847 zip, filename: str, error_detail: str, feature_index: int = 0
+
+
+
1850 with zip.open(filename)
as file:
+
+
1852 json_dump = json.loads(data)
+
1853 check_crs(json_dump)
+
1854 feature_collection = geojson.FeatureCollection(json_dump)
+
1855 feature = feature_collection[
"features"][feature_index]
+
1856 geom = feature[
"geometry"]
+
1857 shape_from_geom = shape(geom)
+
1858 return shape_from_geom
+
1859 except Exception
as e:
+
+
1861 raise HTTPException(
+
+
1863 detail=f
"{error_detail} ----- Error: {e} ----",
+
+
+
+
1867 def get_shape_from_json_str(feature: str, error_detail: str):
+
+
1869 geom = feature[
"geometry"]
+
+
1871 except Exception
as e:
+
+
1873 raise HTTPException(
+
+
1875 detail=f
"{error_detail} ----- Error: {e} ---- Json: {feature}",
+
+
+
+
1879 def get_dbqrcode_from_file(zip, qr_filename: str, error_detail: str):
+
+
1881 with zip.open(qr_filename)
as qr_file:
+
1882 binary_qrcode = qr_file.read()
+
+
1884 return db_models.DbQrCode(
+
1885 filename=qr_filename,
+
1886 image=binary_qrcode,
+
+
+
1889 raise HTTPException(
+
1890 status_code=400, detail=f
"{qr_filename} is an empty file"
+
+
1892 except Exception
as e:
+
+
1894 raise HTTPException(
+
1895 status_code=400, detail=f
"{error_detail} ----- Error: {e}"
+
+
+
+
+
+
+
+
+
+
+
1906 def convert_to_app_project(db_project: db_models.DbProject):
+
+
1908 log.debug(
"Converting db project to app project")
+
1909 app_project: project_schemas.Project = db_project
+
+
1911 if db_project.outline:
+
1912 log.debug(
"Converting project outline to geojson")
+
1913 app_project.outline_geojson = geometry_to_geojson(
+
1914 db_project.outline, {
"id": db_project.id}, db_project.id
+
+
+
1917 app_project.project_tasks = tasks_crud.convert_to_app_tasks(db_project.tasks)
+
+
+
+
1921 log.debug(
"convert_to_app_project called, but no project provided")
+
+
+
+
1925 def convert_to_app_project_info(db_project_info: db_models.DbProjectInfo):
+
+
1927 app_project_info: project_schemas.ProjectInfo = db_project_info
+
1928 return app_project_info
+
+
+
+
+
1933 def convert_to_app_projects(db_projects: List[db_models.DbProject]):
+
1934 if db_projects
and len(db_projects) > 0:
+
+
1936 for project
in db_projects:
+
+
1938 app_projects.append(convert_to_app_project(project))
+
1939 app_projects_without_nones = [i
for i
in app_projects
if i
is not None]
+
1940 return app_projects_without_nones
+
+
+
+
+
1945 def convert_to_project_summary(db_project: db_models.DbProject):
+
+
1947 summary: project_schemas.ProjectSummary = db_project
+
+
1949 if db_project.project_info:
+
1950 summary.location_str = db_project.location_str
+
1951 summary.title = db_project.project_info.name
+
1952 summary.description = db_project.project_info.short_description
+
+
1954 summary.num_contributors = (
+
1955 db_project.tasks_mapped + db_project.tasks_validated
+
+
1957 summary.organisation_logo = (
+
1958 db_project.organisation.logo
if db_project.organisation
else None
+
+
+
+
+
+
+
+
1966 def convert_to_project_summaries(db_projects: List[db_models.DbProject]):
+
1967 if db_projects
and len(db_projects) > 0:
+
1968 project_summaries = []
+
1969 for project
in db_projects:
+
+
1971 project_summaries.append(convert_to_project_summary(project))
+
1972 app_projects_without_nones = [i
for i
in project_summaries
if i
is not None]
+
1973 return app_projects_without_nones
+
+
+
+
+
1978 def convert_to_project_feature(db_project_feature: db_models.DbFeatures):
+
1979 if db_project_feature:
+
1980 app_project_feature: project_schemas.Feature = db_project_feature
+
+
1982 if db_project_feature.geometry:
+
1983 app_project_feature.geometry = geometry_to_geojson(
+
1984 db_project_feature.geometry,
+
1985 db_project_feature.properties,
+
1986 db_project_feature.id,
+
+
+
1989 return app_project_feature
+
+
+
+
+
1994 def convert_to_project_features(db_project_features: List[db_models.DbFeatures]):
+
1995 if db_project_features
and len(db_project_features) > 0:
+
1996 app_project_features = []
+
1997 for project_feature
in db_project_features:
+
+
1999 app_project_features.append(convert_to_project_feature(project_feature))
+
2000 return app_project_features
+
+
+
+
+
2005 def get_project_features(db: Session, project_id: int, task_id: int =
None):
+
+
+
2008 db.query(db_models.DbFeatures)
+
2009 .filter(db_models.DbFeatures.project_id == project_id)
+
2010 .filter(db_models.DbFeatures.task_id == task_id)
+
+
+
+
+
2015 db.query(db_models.DbFeatures)
+
2016 .filter(db_models.DbFeatures.project_id == project_id)
+
+
+
2019 return convert_to_project_features(features)
+
+
+
2022 async
def get_extract_completion_count(project_id: int, db: Session):
+
+
2024 db.query(db_models.DbProject)
+
2025 .filter(db_models.DbProject.id == project_id)
+
+
+
2028 return project.extract_completed_count
+
+
+
2031 async
def get_background_task_status(task_id: uuid.UUID, db: Session):
+
2032 """Get the status of a background task."""
+
+
2034 db.query(db_models.BackgroundTasks)
+
2035 .filter(db_models.BackgroundTasks.id == str(task_id))
+
+
+
+
2039 log.warning(f
"No background task with found with UUID: {task_id}")
+
2040 raise HTTPException(status_code=404, detail=
"Task not found")
+
2041 return task.status, task.message
+
+
+
2044 async
def insert_background_task_into_database(
+
2045 db: Session, name: str =
None, project_id=
None
+
+
2047 """Inserts a new task into the database
+
+
2049 db: database session
+
2050 name: name of the task.
+
2051 project_id: associated project id
+
+
+
2054 task_id(uuid.uuid4): The background task uuid for tracking.
+
+
2056 task_id = uuid.uuid4()
+
+
2058 task = db_models.BackgroundTasks(
+
2059 id=str(task_id), name=name, status=1, project_id=project_id
+
+
+
+
+
+
+
+
+
+
2069 def update_background_task_status_in_database(
+
2070 db: Session, task_id: uuid.UUID, status: int, message: str =
None
+
+
2072 """Updates the status of a task in the database
+
+
2074 db: database session
+
2075 task_id: uuid of the task
+
2076 status: status of the task.
+
+
2078 db.query(db_models.BackgroundTasks).filter(
+
2079 db_models.BackgroundTasks.id == str(task_id)
+
+
+
2082 db_models.BackgroundTasks.status: status,
+
2083 db_models.BackgroundTasks.message: message,
+
+
+
+
+
+
+
+
2091 def add_features_into_database(
+
+
+
2094 background_task_id: uuid.UUID,
+
+
+
2097 """Inserts a new task into the database
+
+
2099 db: database session
+
2100 project_id: id of the project
+
2101 features: features to be added.
+
+
+
+
+
2106 project_id = uuid.uuid4()
+
2107 if feature_type ==
"buildings":
+
2108 for feature
in features[
"features"]:
+
+
2110 feature_geometry = feature[
"geometry"]
+
2111 feature_shape = shape(feature_geometry)
+
+
2113 wkb_element = from_shape(feature_shape, srid=4326)
+
2114 building_obj = db_models.DbBuildings(
+
2115 project_id=project_id,
+
+
2117 tags=feature[
"properties"],
+
+
2119 db.add(building_obj)
+
+
+
+
+
+
+
+
2127 update_background_task_status_in_database(
+
2128 db, background_task_id, 4
+
+
+
2131 elif feature_type ==
"lines":
+
2132 for feature
in features[
"features"]:
+
+
2134 feature_geometry = feature[
"geometry"]
+
2135 feature_shape = shape(feature_geometry)
+
2136 feature[
"properties"][
"highway"] =
"yes"
+
+
2138 wkb_element = from_shape(feature_shape, srid=4326)
+
2139 db_feature = db_models.DbOsmLines(
+
2140 project_id=project_id,
+
+
2142 tags=feature[
"properties"],
+
+
+
+
+
+
+
+
+
+
+
2153 update_background_task_status_in_database(
+
2154 db, background_task_id, 4
+
+
+
+
2158 except Exception
as e:
+
+
+
+
2162 update_background_task_status_in_database(
+
2163 db, background_task_id, 2, str(e)
+
+
+
+
2167 async
def update_project_form(
+
2168 db: Session, project_id: int, form_type: str, form: UploadFile = File(
None)
+
+
2170 project = get_project(db, project_id)
+
2171 category = project.xform_title
+
2172 project_title = project.project_name_prefix
+
2173 odk_id = project.odkid
+
+
+
2176 odk_credentials = project_schemas.ODKCentral(
+
2177 odk_central_url=project.odk_central_url,
+
2178 odk_central_user=project.odk_central_user,
+
2179 odk_central_password=project.odk_central_password,
+
+
+
+
2183 xlsform = f
"/tmp/custom_form.{form_type}"
+
2184 contents = await form.read()
+
2185 with open(xlsform,
"wb")
as f:
+
+
+
2188 xlsform = f
"{xlsforms_path}/{category}.xls"
+
+
2190 db.query(db_models.DbFeatures).filter(
+
2191 db_models.DbFeatures.project_id == project_id
+
+
+
+
+
2196 pg = PostgresClient(
"underpass")
+
+
2198 f
"/tmp/{project_title}_{category}.geojson"
+
+
+
2201 extract_polygon =
True if project.data_extract_type ==
"polygon" else False
+
+
2203 project = table(
"projects", column(
"outline"))
+
+
+
+
2207 geoalchemy2.functions.ST_AsGeoJSON(project.c.outline).label(
"outline"),
+
2208 ).where(text(f
"id={project_id}"))
+
2209 result = db.execute(sql)
+
2210 project_outline = result.first()
+
+
2212 final_outline = json.loads(project_outline.outline)
+
+
2214 outline_geojson = pg.getFeatures(
+
2215 boundary=final_outline,
+
+
2217 polygon=extract_polygon,
+
2218 xlsfile=f
"{category}.xls",
+
+
+
+
2222 updated_outline_geojson = {
"type":
"FeatureCollection",
"features": []}
+
+
+
2225 feature_mappings = []
+
+
2227 for feature
in outline_geojson[
"features"]:
+
+
2229 feature[
"properties"][
"title"] =
""
+
+
2231 feature_shape = shape(feature[
"geometry"])
+
+
+
+
+
+
2237 wkb_element = from_shape(feature_shape, srid=4326)
+
+
2239 "project_id": project_id,
+
2240 "category_title": category,
+
2241 "geometry": wkb_element,
+
2242 "properties": feature[
"properties"],
+
+
2244 updated_outline_geojson[
"features"].append(feature)
+
2245 feature_mappings.append(feature_mapping)
+
+
+
2248 db_feature = db_models.DbFeatures(
+
2249 project_id=project_id,
+
2250 category_title=category,
+
2251 geometry=wkb_element,
+
2252 properties=feature[
"properties"],
+
+
+
+
+
2257 tasks_list = tasks_crud.get_task_lists(db, project_id)
+
+
2259 for task
in tasks_list:
+
2260 task_obj = tasks_crud.get_task(db, task)
+
+
+
+
+
+
+
+
+
+
+
+
2272 WHERE project_id={project_id} and ST_Intersects(geometry, '{task_obj.outline}'::Geometry)
+
+
+
+
+
2277 result = db.execute(query)
+
+
+
+
2281 f
"""SELECT jsonb_build_object(
+
2282 'type', 'FeatureCollection',
+
2283 'features', jsonb_agg(feature)
+
+
+
2286 SELECT jsonb_build_object(
+
+
+
2289 'geometry', ST_AsGeoJSON(geometry)::jsonb,
+
2290 'properties', properties
+
+
+
2293 WHERE project_id={project_id} and task_id={task}
+
+
+
+
2297 result = db.execute(query)
+
2298 features = result.fetchone()[0]
+
+
2300 xform = f
"/tmp/{project_title}_{category}_{task}.xml"
+
2301 extracts = f
"/tmp/{project_title}_{category}_{task}.geojson"
+
+
+
2304 with open(extracts,
"w")
as jsonfile:
+
2305 jsonfile.truncate(0)
+
2306 dump(features, jsonfile)
+
+
2308 outfile = central_crud.generate_updated_xform(xlsform, xform, form_type)
+
+
+
2311 result = central_crud.create_odk_xform(
+
2312 odk_id, task, xform, odk_credentials,
True,
True,
False
+
+
+
+
+
+
2318 async
def update_odk_credentials(
+
2319 project_instance: project_schemas.ProjectUpload,
+
2320 odk_central_cred: project_schemas.ODKCentral,
+
+
+
+
2324 project_instance.odkid = odkid
+
2325 project_instance.odk_central_url = odk_central_cred.odk_central_url
+
2326 project_instance.odk_central_user = odk_central_cred.odk_central_user
+
2327 project_instance.odk_central_password = odk_central_cred.odk_central_password
+
+
+
2330 db.refresh(project_instance)
+
+
+
2333 async
def get_extracted_data_from_db(db: Session, project_id: int, outfile: str):
+
2334 """Get the geojson of those features for this project."""
+
+
2336 f
"""SELECT jsonb_build_object(
+
2337 'type', 'FeatureCollection',
+
2338 'features', jsonb_agg(feature)
+
+
+
2341 SELECT jsonb_build_object(
+
+
+
2344 'geometry', ST_AsGeoJSON(geometry)::jsonb,
+
2345 'properties', properties
+
+
+
2348 WHERE project_id={project_id}
+
+
+
+
2352 result = db.execute(query)
+
2353 features = result.fetchone()[0]
+
+
+
2356 with open(outfile,
"w")
as jsonfile:
+
2357 jsonfile.truncate(0)
+
2358 dump(features, jsonfile)
+
+
+
2361 def get_project_tiles(
+
+
+
2364 background_task_id: uuid.UUID,
+
+
2366 output_format: str =
"mbtiles",
+
+
+
2369 """Get the tiles for a project.
+
+
+
2372 project_id (int): ID of project to create tiles for.
+
2373 background_task_id (uuid.UUID): UUID of background task to track.
+
2374 source (str): Tile source ("esri", "bing", "topo", "google", "oam").
+
2375 output_format (str, optional): Default "mbtiles".
+
2376 Other options: "pmtiles", "sqlite3".
+
2377 tms (str, optional): Default None. Custom TMS provider URL.
+
+
+
2380 tiles_path_id = uuid.uuid4()
+
2381 tiles_dir = f
"{TILESDIR}/{tiles_path_id}"
+
2382 outfile = f
"{tiles_dir}/{project_id}_{source}tiles.{output_format}"
+
+
2384 tile_path_instance = db_models.DbTilesPath(
+
2385 project_id=project_id,
+
2386 background_task_id=str(background_task_id),
+
+
+
+
+
+
+
2393 db.add(tile_path_instance)
+
+
+
+
2397 log.debug(f
"Getting bbox for project: {project_id}")
+
+
2399 f
"""SELECT ST_XMin(ST_Envelope(outline)) AS min_lon,
+
2400 ST_YMin(ST_Envelope(outline)) AS min_lat,
+
2401 ST_XMax(ST_Envelope(outline)) AS max_lon,
+
2402 ST_YMax(ST_Envelope(outline)) AS max_lat
+
+
2404 WHERE id = {project_id};"""
+
+
+
2407 result = db.execute(query)
+
2408 project_bbox = result.fetchone()
+
2409 log.debug(f
"Extracted project bbox: {project_bbox}")
+
+
+
2412 min_lon, min_lat, max_lon, max_lat = project_bbox
+
+
2414 log.error(f
"Failed to get bbox from project: {project_id}")
+
+
+
2417 "Creating basemap with params: "
+
2418 f
"boundary={min_lon},{min_lat},{max_lon},{max_lat} | "
+
2419 f
"outfile={outfile} | "
+
+
2421 f
"outdir={tiles_dir} | "
+
2422 f
"source={source} | "
+
+
+
+
2426 create_basemap_file(
+
2427 boundary=f
"{min_lon},{min_lat},{max_lon},{max_lat}",
+
+
+
+
+
+
+
+
2435 log.info(f
"Basemap created for project ID {project_id}: {outfile}")
+
+
2437 tile_path_instance.status = 4
+
+
+
+
2441 update_background_task_status_in_database(
+
2442 db, background_task_id, 4
+
+
+
2445 log.info(f
"Tiles generation process completed for project id {project_id}")
+
+
2447 except Exception
as e:
+
2448 log.error(f
"Tiles generation process failed for project id {project_id}")
+
+
+
2451 tile_path_instance.status = 2
+
+
+
+
2455 update_background_task_status_in_database(
+
2456 db, background_task_id, 2, str(e)
+
+
+
+
2460 async
def get_mbtiles_list(db: Session, project_id: int):
+
+
+
+
2464 db_models.DbTilesPath.id,
+
2465 db_models.DbTilesPath.project_id,
+
2466 db_models.DbTilesPath.status,
+
2467 db_models.DbTilesPath.tile_source,
+
+
2469 .filter(db_models.DbTilesPath.project_id == str(project_id))
+
+
+
+
2473 processed_tiles_list = [
+
+
+
2476 "project_id": x.project_id,
+
2477 "status": x.status.name,
+
2478 "tile_source": x.tile_source,
+
+
+
+
+
2483 return processed_tiles_list
+
+
2485 except Exception
as e:
+
2486 raise HTTPException(status_code=400, detail=str(e))
+
+
+
2489 async
def convert_geojson_to_osm(geojson_file: str):
+
2490 """Convert a GeoJSON file to OSM format."""
+
2491 return json2osm(geojson_file)
+
+
+
2494 def generate_appuser_files_for_janakpur(
+
+
+
+
2498 building_extracts_contents: str,
+
2499 road_extracts_contents: str,
+
+
+
2502 background_task_id: uuid.UUID,
+
+
2504 project_log = log.bind(task=
"create_project", project_id=project_id)
+
+
2506 project_log.info(f
"Starting generate_appuser_files for project {project_id}")
+
+
+
+
+
2511 column(
"project_name_prefix"),
+
2512 column(
"xform_title"),
+
+
+
2515 column(
"odk_central_url"),
+
2516 column(
"odk_central_user"),
+
2517 column(
"odk_central_password"),
+
+
+
+
2521 where = f
"id={project_id}"
+
+
2523 project.c.project_name_prefix,
+
2524 project.c.xform_title,
+
+
+
2527 project.c.odk_central_url,
+
2528 project.c.odk_central_user,
+
2529 project.c.odk_central_password,
+
2530 geoalchemy2.functions.ST_AsGeoJSON(project.c.outline).label(
"outline"),
+
2531 ).where(text(where))
+
2532 result = db.execute(sql)
+
+
+
2535 if result.rowcount != 1:
+
2536 log.warning(str(sql))
+
2537 if result.rowcount < 1:
+
2538 raise HTTPException(status_code=400, detail=
"Project not found")
+
+
2540 raise HTTPException(status_code=400, detail=
"Multiple projects found")
+
+
2542 one = result.first()
+
+
+
2545 prefix = one.project_name_prefix
+
+
+
+
2549 "odk_central_url": one.odk_central_url,
+
2550 "odk_central_user": one.odk_central_user,
+
2551 "odk_central_password": one.odk_central_password,
+
+
+
2554 odk_credentials = project_schemas.ODKCentral(**odk_credentials)
+
+
2556 xform_title = one.xform_title
if one.xform_title
else None
+
+
+
2559 xlsform = f
"/tmp/custom_form.{form_type}"
+
+
2561 with open(xlsform,
"wb")
as f:
+
+
+
2564 xlsform = f
"{xlsforms_path}/{xform_title}.xls"
+
+
2566 category = xform_title
+
+
+
+
2570 if building_extracts_contents
is not None:
+
2571 project_log.info(
"Uploading data extracts")
+
2572 upload_custom_data_extracts(db, project_id, building_extracts_contents)
+
+
2574 if road_extracts_contents
is not None:
+
2575 project_log.info(
"Uploading roads data")
+
2576 upload_custom_data_extracts(
+
2577 db, project_id, road_extracts_contents,
"highways"
+
+
+
+
+
2582 tasks_list = tasks_crud.get_task_lists(db, project_id)
+
+
2584 project_name = prefix
+
+
2586 project_obj = get_project(db, project_id)
+
+
2588 for task_id
in tasks_list:
+
+
2590 name = f
"{project_name}_{category}_{task_id}"
+
+
2592 appuser = central_crud.create_appuser(odk_id, name, odk_credentials)
+
+
+
+
2596 project_log.error(
"Couldn't create appuser for project")
+
+
+
+
2600 project_log.info(f
"Creating qr code for task_id {task_id}")
+
2601 create_qr = create_qrcode(
+
+
+
2604 appuser.json()[
"token"],
+
+
2606 odk_credentials.odk_central_url,
+
+
+
2609 task = tasks_crud.get_task(db, task_id)
+
2610 task.qr_code_id = create_qr[
"qr_code_id"]
+
+
+
+
+
2615 xform = f
"/tmp/{name}.xml"
+
+
2617 buildings_extracts = (
+
2618 f
"/tmp/buildings_{name}.geojson"
+
+
+
2621 f
"/tmp/roads_{name}.geojson"
+
+
+
+
2625 xform_id = f
"{name}".split(
"_")[2]
+
+
+
+
+
+
+
2632 SET task_id={task_id}
+
+
+
+
2636 WHERE project_id={project_id}
+
2637 AND ST_IsValid(geometry)
+
2638 AND ST_IsValid('{task.outline}'::Geometry)
+
2639 AND ST_Contains('{task.outline}'::Geometry, ST_Centroid(geometry))
+
+
+
2642 result = db.execute(query)
+
+
+
2645 buildings_query = text(
+
2646 f
"""SELECT jsonb_build_object(
+
2647 'type', 'FeatureCollection',
+
2648 'features', jsonb_agg(feature)
+
+
+
2651 SELECT jsonb_build_object(
+
+
+
2654 'geometry', ST_AsGeoJSON(geometry)::jsonb,
+
2655 'properties', properties
+
+
+
2658 WHERE project_id={project_id} and task_id={task_id} and category_title='buildings'
+
+
+
2661 result = db.execute(buildings_query)
+
2662 features = result.fetchone()[0]
+
+
2664 highway_query = text(
+
2665 f
"""SELECT jsonb_build_object(
+
2666 'type', 'FeatureCollection',
+
2667 'features', jsonb_agg(feature)
+
+
+
2670 SELECT jsonb_build_object(
+
+
+
2673 'geometry', ST_AsGeoJSON(geometry)::jsonb,
+
2674 'properties', properties
+
+
+
2677 WHERE project_id={project_id} and category_title='highways'
+
+
+
2680 highway_result = db.execute(highway_query)
+
2681 highway_features = highway_result.fetchone()[0]
+
+
+
+
+
+
2687 with open(buildings_extracts,
"w")
as jsonfile:
+
2688 jsonfile.truncate(0)
+
2689 dump(features, jsonfile)
+
+
+
2692 with open(roads_extracts,
"w")
as jsonfile:
+
2693 jsonfile.truncate(0)
+
2694 dump(highway_features, jsonfile)
+
+
2696 project_log.info(f
"Generating xform for task {task_id}")
+
2697 outfile = central_crud.generate_updated_xform_for_janakpur(
+
2698 xlsform, xform, form_type
+
+
+
+
2702 project_log.info(f
"Uploading media in {task_id}")
+
2703 result = central_crud.create_odk_xform_for_janakpur(
+
2704 odk_id, task, outfile, odk_credentials,
False, upload_media
+
+
+
2707 project_log.info(f
"Updating role for app user in task {task_id}")
+
+
+
+
+
2712 url = odk_credentials.odk_central_url
+
2713 user = odk_credentials.odk_central_user
+
2714 pw = odk_credentials.odk_central_password
+
+
+
2717 log.debug(
"ODKCentral connection variables not set in function")
+
2718 log.debug(
"Attempting extraction from environment variables")
+
2719 url = settings.ODK_CENTRAL_URL
+
2720 user = settings.ODK_CENTRAL_USER
+
2721 pw = settings.ODK_CENTRAL_PASSWD
+
+
2723 odk_app = OdkAppUser(url, user, pw)
+
+
+
2726 projectId=odk_id, xform=xform_id, actorId=appuser.json()[
"id"]
+
+
+
2729 except Exception
as e:
+
+
+
2732 project_obj.extract_completed_count += 1
+
+
2734 db.refresh(project_obj)
+
+
+
2737 update_background_task_status_in_database(
+
2738 db, background_task_id, 4
+
+
+
+
2742 def get_address_from_lat_lon(latitude, longitude):
+
2743 """Get address using Nominatim, using lat,lon."""
+
2744 base_url =
"https://nominatim.openstreetmap.org/reverse"
+
+
+
+
+
+
+
+
2752 headers = {
"Accept-Language":
"en"}
+
+
2754 response = requests.get(base_url, params=params, headers=headers)
+
2755 data = response.json()
+
2756 address = data[
"address"][
"country"]
+
+
2758 if response.status_code == 200:
+
2759 if "city" in data[
"address"]:
+
2760 city = data[
"address"][
"city"]
+
2761 address = f
"{city}" +
"," + address
+
+
+
2764 return "Address not found."
+
+
+
2767 def update_project_location_info(
+
2768 db_project: sqlalchemy.orm.declarative_base, project_boundary: str
+
+
2770 """Update project boundary, centroid, address.
+
+
+
2773 db_project(sqlalchemy.orm.declarative_base): The project database record.
+
2774 project_boundary(str): WKT string geometry.
+
+
2776 db_project.outline = project_boundary
+
2777 centroid = (wkt.loads(project_boundary)).centroid.wkt
+
2778 db_project.centroid = centroid
+
2779 geometry = wkt.loads(centroid)
+
2780 longitude, latitude = geometry.x, geometry.y
+
2781 address = get_address_from_lat_lon(latitude, longitude)
+
2782 db_project.location_str = address
if address
is not None else ""
+
+
+
2785 def convert_geojson_to_epsg4326(input_geojson):
+
2786 source_crs = pyproj.CRS(
+
2787 input_geojson.get(
"crs", {}).get(
"properties", {}).get(
"name",
"EPSG:4326")
+
+
2789 transformer = pyproj.Transformer.from_crs(source_crs,
"EPSG:4326", always_xy=
True)
+
+
+
2792 transformed_features = []
+
2793 for feature
in input_geojson.get(
"features", []):
+
2794 geom = shape(feature.get(
"geometry", {}))
+
2795 transformed_geom = transform(transformer.transform, geom)
+
2796 transformed_feature = {
+
+
2798 "geometry": transformed_geom.__geo_interface__,
+
2799 "properties": feature.get(
"properties", {}),
+
+
2801 transformed_features.append(transformed_feature)
+
+
+
2804 output_geojson = {
"type":
"FeatureCollection",
"features": transformed_features}
+
+
2806 return output_geojson
+
+
+
2809 def check_crs(input_geojson: dict):
+
2810 log.debug(
"validating coordinate reference system")
+
+
2812 def is_valid_crs(crs_name):
+
+
2814 "urn:ogc:def:crs:OGC:1.3:CRS84",
+
2815 "urn:ogc:def:crs:EPSG::4326",
+
+
+
2818 return crs_name
in valid_crs_list
+
+
2820 def is_valid_coordinate(coord):
+
2821 return -180 <= coord[0] <= 180
and -90 <= coord[1] <= 90
+
+
2823 error_message =
"ERROR: Unsupported coordinate system, it is recommended to use a GeoJSON file in WGS84(EPSG 4326) standard."
+
2824 if "crs" in input_geojson:
+
2825 crs = input_geojson[
"crs"][
"properties"][
"name"]
+
2826 if not is_valid_crs(crs):
+
2827 log.error(error_message)
+
2828 raise HTTPException(status_code=400, detail=error_message)
+
+
+
2831 if input_geojson[
"type"] ==
"FeatureCollection":
+
2832 coordinates = input_geojson[
"features"][0][
"geometry"][
"coordinates"]
+
2833 elif input_geojson[
"type"] ==
"Feature":
+
2834 coordinates = input_geojson[
"geometry"][
"coordinates"]
+
+
2836 input_geojson[
"features"][0][
"geometry"][
"type"]
+
2837 if input_geojson[
"type"] ==
"FeatureCollection"
+
2838 else input_geojson[
"geometry"][
"type"]
+
+
+
2841 if geometry_type ==
"MultiPolygon":
+
2842 first_coordinate = coordinates[0][0][
+
+
+
+
2846 first_coordinate = coordinates[0][0]
+
+
2848 if not is_valid_coordinate(first_coordinate):
+
2849 log.error(error_message)
+
2850 raise HTTPException(status_code=400, detail=error_message)
+
+
+
2853 def get_tasks_count(db: Session, project_id: int):
+
+
2855 db.query(db_models.DbProject)
+
2856 .filter(db_models.DbProject.id == project_id)
+
+
+
2859 task_count = len(db_task.tasks)
+
+
+
+
2863 def get_pagintaion(page: int, count: int, results_per_page: int, total: int):
+
2864 total_pages = (count + results_per_page - 1) // results_per_page
+
2865 hasNext = (page * results_per_page) < count
+
+
+
2868 pagination = project_schemas.PaginationInfo(
+
+
+
2871 nextNum=page + 1
if hasNext
else None,
+
+
+
2874 prevNum=page - 1
if hasPrev
else None,
+
2875 perPage=results_per_page,
+
+
+
+
+
First off, We are really glad you're reading this, because we need volunteer developers to help improve the Field Mapping Tasking Manager (FMTM)! 
Thank you¶