From 14c3cd910b602306a83bf32a493a46adbbc0c350 Mon Sep 17 00:00:00 2001 From: Sam <78538841+spwoodcock@users.noreply.github.com> Date: Mon, 2 Oct 2023 23:12:35 +0100 Subject: [PATCH] Various: task splitting error handling, compose cleanup, default svcfmtm user (#872) * refactor: rename frontend container ui-main --> ui (fmtm) * fix: remove route protection during dev, populate odk vars * docs: lengthen default odk pass example * fix: improve task splitting logs and error handling * fix: handle empty task split on frontend * fix: add default svc user for fmtm * build: remove refs to redundant APP_NAME for frontend * build: add bind mount to migration container * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docker-compose.deploy.yml | 4 +- docker-compose.noodk.yml | 4 +- docker-compose.yml | 8 +-- src/backend/app/projects/project_crud.py | 51 +++++++++++++++---- .../createproject/FormSelection.tsx | 4 +- src/frontend/src/store/slices/LoginSlice.ts | 4 +- 6 files changed, 55 insertions(+), 20 deletions(-) diff --git a/docker-compose.deploy.yml b/docker-compose.deploy.yml index b516807b13..3e07fb70ec 100644 --- a/docker-compose.deploy.yml +++ b/docker-compose.deploy.yml @@ -118,7 +118,7 @@ services: command: ["alembic", "upgrade", "head"] restart: "on-failure:3" - ui-main: + ui: image: "ghcr.io/hotosm/fmtm/frontend:${FRONTEND_MAIN_VERSION}-${GIT_BRANCH}" build: context: src/frontend @@ -127,7 +127,7 @@ services: APP_VERSION: ${FRONTEND_MAIN_VERSION} API_URL: ${URL_SCHEME}://${API_URL} FRONTEND_MAIN_URL: ${URL_SCHEME}://${FRONTEND_MAIN_URL} - container_name: fmtm_main + container_name: fmtm depends_on: - api - traefik diff --git a/docker-compose.noodk.yml b/docker-compose.noodk.yml index 5c70dc1afb..bbbca5a2b8 100644 --- a/docker-compose.noodk.yml +++ b/docker-compose.noodk.yml @@ -80,7 +80,7 @@ services: command: ["alembic", "upgrade", "head"] restart: "on-failure:3" - ui-main: + ui: image: "ghcr.io/hotosm/fmtm/frontend:debug" build: context: src/frontend @@ -89,7 +89,7 @@ services: APP_VERSION: ${FRONTEND_MAIN_VERSION} API_URL: ${URL_SCHEME}://${API_URL} FRONTEND_MAIN_URL: ${URL_SCHEME}://${FRONTEND_MAIN_URL} - container_name: fmtm_main + container_name: fmtm depends_on: - api volumes: diff --git a/docker-compose.yml b/docker-compose.yml index fe62b685de..ca5359ef26 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,16 +87,15 @@ services: command: ["alembic", "upgrade", "head"] restart: "on-failure:3" - ui-main: + ui: image: "ghcr.io/hotosm/fmtm/frontend:debug" build: context: src/frontend dockerfile: debug.dockerfile args: - APP_NAME: main API_URL: ${URL_SCHEME}://${API_URL} FRONTEND_MAIN_URL: ${URL_SCHEME}://${FRONTEND_MAIN_URL} - container_name: fmtm_main + container_name: fmtm depends_on: - api volumes: @@ -105,6 +104,9 @@ services: environment: - API_URL=${URL_SCHEME}://${API_URL} - FRONTEND_MAIN_URL=${URL_SCHEME}://${FRONTEND_MAIN_URL} + - ODK_CENTRAL_URL=${ODK_CENTRAL_URL} + - ODK_CENTRAL_USER=${ODK_CENTRAL_USER} + - ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD} ports: - "8080:8080" networks: diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 35efac7c30..aa28ce322e 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -646,44 +646,63 @@ async def split_into_tasks( all_results = [] boundary_data = [] result = [] + if outline["type"] == "FeatureCollection": + log.debug("Project boundary GeoJSON = FeatureCollection") boundary_data.extend(feature["geometry"] for feature in outline["features"]) result.extend( - process_polygon(db, project_id, data, no_of_buildings, has_data_extracts) + split_polygon_into_tasks( + db, project_id, data, no_of_buildings, has_data_extracts + ) for data in boundary_data ) + for inner_list in result: - all_results.extend(iter(inner_list)) + if inner_list: + all_results.extend(iter(inner_list)) elif outline["type"] == "GeometryCollection": + log.debug("Project boundary GeoJSON = GeometryCollection") geometries = outline["geometries"] boundary_data.extend(iter(geometries)) result.extend( - process_polygon(db, project_id, data, no_of_buildings, has_data_extracts) + split_polygon_into_tasks( + db, project_id, data, no_of_buildings, has_data_extracts + ) for data in boundary_data ) for inner_list in result: - all_results.extend(iter(inner_list)) + if inner_list: + all_results.extend(iter(inner_list)) elif outline["type"] == "Feature": + log.debug("Project boundary GeoJSON = Feature") boundary_data = outline["geometry"] - result = process_polygon( + result = split_polygon_into_tasks( db, project_id, boundary_data, no_of_buildings, has_data_extracts ) all_results.extend(iter(result)) - else: + + elif outline["type"] == "Polygon": + log.debug("Project boundary GeoJSON = Polygon") boundary_data = outline - result = process_polygon( + result = split_polygon_into_tasks( db, project_id, boundary_data, no_of_buildings, has_data_extracts ) all_results.extend(result) + + else: + log.error( + "Project boundary not one of: Polygon, Feature, GeometryCollection," + " FeatureCollection. Task splitting failed." + ) return { "type": "FeatureCollection", "features": all_results, } -def process_polygon( +def split_polygon_into_tasks( db: Session, project_id: uuid.UUID, boundary_data: str, @@ -727,16 +746,30 @@ def process_polygon( ) result = db.execute(query) db.commit() + + # TODO replace with fmtm_splitter algo with open("app/db/split_algorithm.sql", "r") as sql_file: query = sql_file.read() + log.debug(f"STARTED project {project_id} task splitting") result = db.execute(text(query), params={"num_buildings": no_of_buildings}) result = result.fetchall() db.query(db_models.DbBuildings).delete() db.query(db_models.DbOsmLines).delete() db.query(db_models.DbProjectAOI).delete() db.commit() + log.debug(f"COMPLETE project {project_id} task splitting") - return result[0][0]["features"] + features = result[0][0]["features"] + if not features: + log.warning( + f"Project {project_id}: no tasks returned from splitting algorithm. " + f"Params: 'num_buildings': {no_of_buildings}" + ) + return [] + + features = json.loads(features) + log.debug(f"Project {project_id} split into {len(features)} tasks") + return features # def update_project_boundary( diff --git a/src/frontend/src/components/createproject/FormSelection.tsx b/src/frontend/src/components/createproject/FormSelection.tsx index 41c0f97edb..ecdafd131d 100755 --- a/src/frontend/src/components/createproject/FormSelection.tsx +++ b/src/frontend/src/components/createproject/FormSelection.tsx @@ -92,8 +92,8 @@ const FormSelection: React.FC = ({ description: projectDetails.description, }, author: { - username: userDetails.username, - id: userDetails.id, + username: userDetails?.username || 'svcfmtm', + id: userDetails?.id || 20386219, }, odk_central: { odk_central_url: projectDetails.odk_central_url, diff --git a/src/frontend/src/store/slices/LoginSlice.ts b/src/frontend/src/store/slices/LoginSlice.ts index 85a39ccd1c..6afdd45ee1 100755 --- a/src/frontend/src/store/slices/LoginSlice.ts +++ b/src/frontend/src/store/slices/LoginSlice.ts @@ -3,8 +3,8 @@ import storage from 'redux-persist/lib/storage'; const LoginSlice = CoreModules.createSlice({ name: 'login', initialState: { - loginToken: null, - authDetails: null, + loginToken: { id: 20386219, username: 'svcfmtm' }, + authDetails: { id: 20386219, username: 'svcfmtm' }, }, reducers: { SetLoginToken(state, action) {