Skip to content

Commit

Permalink
Various: task splitting error handling, compose cleanup, default svcf…
Browse files Browse the repository at this point in the history
…mtm user (#872)

* refactor: rename frontend container ui-main --> ui (fmtm)

* fix: remove route protection during dev, populate odk vars

* docs: lengthen default odk pass example

* fix: improve task splitting logs and error handling

* fix: handle empty task split on frontend

* fix: add default svc user for fmtm

* build: remove refs to redundant APP_NAME for frontend

* build: add bind mount to migration container

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
spwoodcock and pre-commit-ci[bot] committed Oct 3, 2023
1 parent 9a2d886 commit 14c3cd9
Show file tree
Hide file tree
Showing 6 changed files with 55 additions and 20 deletions.
4 changes: 2 additions & 2 deletions docker-compose.deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ services:
command: ["alembic", "upgrade", "head"]
restart: "on-failure:3"

ui-main:
ui:
image: "ghcr.io/hotosm/fmtm/frontend:${FRONTEND_MAIN_VERSION}-${GIT_BRANCH}"
build:
context: src/frontend
Expand All @@ -127,7 +127,7 @@ services:
APP_VERSION: ${FRONTEND_MAIN_VERSION}
API_URL: ${URL_SCHEME}://${API_URL}
FRONTEND_MAIN_URL: ${URL_SCHEME}://${FRONTEND_MAIN_URL}
container_name: fmtm_main
container_name: fmtm
depends_on:
- api
- traefik
Expand Down
4 changes: 2 additions & 2 deletions docker-compose.noodk.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ services:
command: ["alembic", "upgrade", "head"]
restart: "on-failure:3"

ui-main:
ui:
image: "ghcr.io/hotosm/fmtm/frontend:debug"
build:
context: src/frontend
Expand All @@ -89,7 +89,7 @@ services:
APP_VERSION: ${FRONTEND_MAIN_VERSION}
API_URL: ${URL_SCHEME}://${API_URL}
FRONTEND_MAIN_URL: ${URL_SCHEME}://${FRONTEND_MAIN_URL}
container_name: fmtm_main
container_name: fmtm
depends_on:
- api
volumes:
Expand Down
8 changes: 5 additions & 3 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,16 +87,15 @@ services:
command: ["alembic", "upgrade", "head"]
restart: "on-failure:3"

ui-main:
ui:
image: "ghcr.io/hotosm/fmtm/frontend:debug"
build:
context: src/frontend
dockerfile: debug.dockerfile
args:
APP_NAME: main
API_URL: ${URL_SCHEME}://${API_URL}
FRONTEND_MAIN_URL: ${URL_SCHEME}://${FRONTEND_MAIN_URL}
container_name: fmtm_main
container_name: fmtm
depends_on:
- api
volumes:
Expand All @@ -105,6 +104,9 @@ services:
environment:
- API_URL=${URL_SCHEME}://${API_URL}
- FRONTEND_MAIN_URL=${URL_SCHEME}://${FRONTEND_MAIN_URL}
- ODK_CENTRAL_URL=${ODK_CENTRAL_URL}
- ODK_CENTRAL_USER=${ODK_CENTRAL_USER}
- ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD}
ports:
- "8080:8080"
networks:
Expand Down
51 changes: 42 additions & 9 deletions src/backend/app/projects/project_crud.py
Original file line number Diff line number Diff line change
Expand Up @@ -646,44 +646,63 @@ async def split_into_tasks(
all_results = []
boundary_data = []
result = []

if outline["type"] == "FeatureCollection":
log.debug("Project boundary GeoJSON = FeatureCollection")
boundary_data.extend(feature["geometry"] for feature in outline["features"])
result.extend(
process_polygon(db, project_id, data, no_of_buildings, has_data_extracts)
split_polygon_into_tasks(
db, project_id, data, no_of_buildings, has_data_extracts
)
for data in boundary_data
)

for inner_list in result:
all_results.extend(iter(inner_list))
if inner_list:
all_results.extend(iter(inner_list))

elif outline["type"] == "GeometryCollection":
log.debug("Project boundary GeoJSON = GeometryCollection")
geometries = outline["geometries"]
boundary_data.extend(iter(geometries))
result.extend(
process_polygon(db, project_id, data, no_of_buildings, has_data_extracts)
split_polygon_into_tasks(
db, project_id, data, no_of_buildings, has_data_extracts
)
for data in boundary_data
)
for inner_list in result:
all_results.extend(iter(inner_list))
if inner_list:
all_results.extend(iter(inner_list))

elif outline["type"] == "Feature":
log.debug("Project boundary GeoJSON = Feature")
boundary_data = outline["geometry"]
result = process_polygon(
result = split_polygon_into_tasks(
db, project_id, boundary_data, no_of_buildings, has_data_extracts
)
all_results.extend(iter(result))
else:

elif outline["type"] == "Polygon":
log.debug("Project boundary GeoJSON = Polygon")
boundary_data = outline
result = process_polygon(
result = split_polygon_into_tasks(
db, project_id, boundary_data, no_of_buildings, has_data_extracts
)
all_results.extend(result)

else:
log.error(
"Project boundary not one of: Polygon, Feature, GeometryCollection,"
" FeatureCollection. Task splitting failed."
)
return {
"type": "FeatureCollection",
"features": all_results,
}


def process_polygon(
def split_polygon_into_tasks(
db: Session,
project_id: uuid.UUID,
boundary_data: str,
Expand Down Expand Up @@ -727,16 +746,30 @@ def process_polygon(
)
result = db.execute(query)
db.commit()

# TODO replace with fmtm_splitter algo
with open("app/db/split_algorithm.sql", "r") as sql_file:
query = sql_file.read()
log.debug(f"STARTED project {project_id} task splitting")
result = db.execute(text(query), params={"num_buildings": no_of_buildings})
result = result.fetchall()
db.query(db_models.DbBuildings).delete()
db.query(db_models.DbOsmLines).delete()
db.query(db_models.DbProjectAOI).delete()
db.commit()
log.debug(f"COMPLETE project {project_id} task splitting")

return result[0][0]["features"]
features = result[0][0]["features"]
if not features:
log.warning(
f"Project {project_id}: no tasks returned from splitting algorithm. "
f"Params: 'num_buildings': {no_of_buildings}"
)
return []

features = json.loads(features)
log.debug(f"Project {project_id} split into {len(features)} tasks")
return features


# def update_project_boundary(
Expand Down
4 changes: 2 additions & 2 deletions src/frontend/src/components/createproject/FormSelection.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,8 @@ const FormSelection: React.FC<any> = ({
description: projectDetails.description,
},
author: {
username: userDetails.username,
id: userDetails.id,
username: userDetails?.username || 'svcfmtm',
id: userDetails?.id || 20386219,
},
odk_central: {
odk_central_url: projectDetails.odk_central_url,
Expand Down
4 changes: 2 additions & 2 deletions src/frontend/src/store/slices/LoginSlice.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ import storage from 'redux-persist/lib/storage';
const LoginSlice = CoreModules.createSlice({
name: 'login',
initialState: {
loginToken: null,
authDetails: null,
loginToken: { id: 20386219, username: 'svcfmtm' },
authDetails: { id: 20386219, username: 'svcfmtm' },
},
reducers: {
SetLoginToken(state, action) {
Expand Down

0 comments on commit 14c3cd9

Please sign in to comment.