From ac05dea952e23b1fb534b4404e590063fb4b8636 Mon Sep 17 00:00:00 2001 From: Sam <78538841+spwoodcock@users.noreply.github.com> Date: Tue, 23 Apr 2024 00:11:31 +0000 Subject: [PATCH] feat: frontend buttons to load Entities in ODK Collect by intent (#1449) * feat: add endpoint for easy entities upload from csv file * feat: add logic to get project entities, get and update entity mapping status * feat: add project/entities and project/entity-mapping-status endpoints * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * feat: endpoints for getting and updating entity status * feat: add endpoint for minimal entity_uuid:osm_id mapping * fix(backend): handle cases when select_one_from_file is either geojson or csv * feat(frontend): add popup for task feature (entity) selection with link to odk by intent * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(backend): handle edge case when task area contains no geometries (fmtm-splitter #28) * build: update osm-fiedwork --> v0.8.2 * refactor: wrap usage of OdkEntity in helper func with error handling * refactor: directly pass project_info to model during proj create * refactor: simplify project creation, divide into separate odk function * refactor: rename state osm:entity mapping var for clarity * refactor: change label on odk intent button --> map feature in odk * fix(backend): functions to update existing xform after entity update * refactor(frontend): allow default odk creds label to be clicked to toggle * fix(backend): do not pretty print final odk xml (minify) * build: upgrade osm-fieldwork --> 0.9.0 for entities workflow * refactor: update references to osm_fieldwork entity registration form * fix: update xform manipulation to factor in entities fields --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- src/backend/app/central/central_crud.py | 143 ++++---- src/backend/app/central/central_deps.py | 43 +++ src/backend/app/central/central_schemas.py | 7 + src/backend/app/db/postgis_utils.py | 2 +- src/backend/app/helpers/helper_routes.py | 8 +- src/backend/app/projects/project_crud.py | 313 +++++++----------- src/backend/app/projects/project_routes.py | 37 ++- src/backend/pdm.lock | 8 +- src/backend/pyproject.toml | 2 +- src/frontend/src/api/Project.js | 14 + .../FeatureSelectionPopup.tsx | 108 ++++++ .../ProjectDetailsV2/TaskSelectionPopup.tsx | 2 +- .../src/components/common/Checkbox.tsx | 8 +- .../createnewproject/ProjectDetailsForm.tsx | 1 + src/frontend/src/store/slices/ProjectSlice.ts | 8 + src/frontend/src/store/slices/TaskSlice.ts | 6 + src/frontend/src/store/types/IProject.ts | 8 + src/frontend/src/store/types/ITask.ts | 1 + src/frontend/src/views/ProjectDetailsV2.tsx | 97 +++--- 19 files changed, 498 insertions(+), 318 deletions(-) create mode 100644 src/backend/app/central/central_deps.py create mode 100644 src/frontend/src/components/ProjectDetailsV2/FeatureSelectionPopup.tsx diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index bd59b28bd2..52fc3432e9 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -30,12 +30,12 @@ from loguru import logger as log from osm_fieldwork.CSVDump import CSVDump from osm_fieldwork.OdkCentral import OdkAppUser, OdkForm, OdkProject -from osm_fieldwork.OdkCentralAsync import OdkEntity from pyxform.builder import create_survey_element_from_dict from pyxform.xls2json import parse_file_to_json from sqlalchemy import text from sqlalchemy.orm import Session +from app.central import central_deps from app.config import settings from app.db.postgis_utils import ( geojson_to_javarosa_geom, @@ -322,7 +322,6 @@ async def update_project_xform( odk_id: int, xform_data: BytesIO, form_file_ext: str, - project_name: str, category: str, odk_credentials: project_schemas.ODKCentralDecrypted, ) -> None: @@ -333,15 +332,9 @@ async def update_project_xform( odk_id (int): ODK Central form ID. xform_data (BytesIO): XForm data. form_file_ext (str): Extension of the form file. - project_name (str): Name (title) of the project. category (str): Category of the XForm. odk_credentials (project_schemas.ODKCentralDecrypted): ODK Central creds. """ - # TODO in the future we may possibly support multiple forms per project. - # TODO to faciliate this we need to add the _{category} suffix and track. - # TODO this in the new xforms.category field/table. - form_name = project_name - xform_data = await read_and_test_xform( xform_data, form_file_ext, @@ -349,27 +342,21 @@ async def update_project_xform( ) updated_xform_data = await update_survey_xform( xform_data, - form_name, category, task_ids, ) - try: - xform = get_odk_form(odk_credentials) - except Exception as e: - log.error(e) - raise HTTPException( - status_code=500, detail={"message": "Connection failed to odk central"} - ) from e + xform_obj = get_odk_form(odk_credentials) - # NOTE calling createForm with the form_name specified should update - xform.createForm( + # NOTE calling createForm for an existing form will update it + form_name = category + xform_obj.createForm( odk_id, updated_xform_data, form_name, ) # The draft form must be published after upload - xform.publishForm(odk_id, form_name) + xform_obj.publishForm(odk_id, form_name) def download_submissions( @@ -425,6 +412,7 @@ async def read_and_test_xform( xform_bytesio = BytesIO( generated_xform.to_xml( validate=False, + pretty_print=False, ).encode("utf-8") ) except Exception as e: @@ -505,25 +493,28 @@ async def update_entity_registration_xform( for instance_elem in root.findall(".//xforms:instance[@src]", namespaces): src_value = instance_elem.get("src", "") if src_value.endswith(".csv"): - instance_elem.set("src", f"jr://file/{category}.csv") + # NOTE geojson files require jr://file/{category}.geojson + # NOTE csv files require jr://file-csv/{category}.csv + instance_elem.set("src", f"jr://file-csv/{category}.csv") return BytesIO(ElementTree.tostring(root)) async def update_survey_xform( form_data: BytesIO, - form_name: str, category: str, - task_ids: list, + task_ids: list[int], ) -> BytesIO: """Update fields in the XForm to work with FMTM. - Updates the 'id' and 'name' fields for the form. - Updates the csv filename to match the dataset name. + The 'id' field is set to random UUID (xFormId) + The 'name' field is set to the category name. + The upload media must match the (entity) dataset name (with .csv). + The task_id options are populated as choices in the form. + The form_category value is also injected to display in the instructions. Args: form_data (str): The input form data. - form_name (str): Name of the XForm to set. category (str): The form category, used to name the dataset (entity list) and the .csv file containing the geometries. task_ids (list): List of task IDs to insert as choices in form. @@ -540,38 +531,77 @@ async def update_survey_xform( "entities": "http://www.opendatakit.org/xforms/entities", } - # Parse the XML + # Parse the XML from BytesIO obj root = ElementTree.fromstring(form_data.getvalue()) - # Update id attribute to equal the form name to be generated xform_data = root.findall(".//xforms:data[@id]", namespaces) for dt in xform_data: - dt.set("id", form_name) + # This sets the xFormId in ODK Central (the form reference via API) + dt.set("id", category) # Update the form title (displayed in ODK Collect) existing_title = root.find(".//h:title", namespaces) if existing_title is not None: - existing_title.text = form_name + existing_title.text = category # Update the attachment name to {category}.csv, to link to the entity list xform_instance_src = root.findall(".//xforms:instance[@src]", namespaces) for inst in xform_instance_src: src_value = inst.get("src", "") - if src_value.endswith(".csv"): - inst.set("src", f"jr://file/{category}.csv") + if src_value.endswith(".geojson") or src_value.endswith(".csv"): + # NOTE geojson files require jr://file/{category}.geojson + # NOTE csv files require jr://file-csv/{category}.csv + inst.set("src", f"jr://file-csv/{category}.csv") - # must be defined inside key + # NOTE add the task ID choices to the XML + # must be defined inside root element model_element = root.find(".//xforms:model", namespaces) + # The existing dummy value for task_id must be removed + existing_instance = model_element.find( + ".//xforms:instance[@id='task_id']", namespaces + ) + if existing_instance is not None: + model_element.remove(existing_instance) + # Create a new instance element instance_task_ids = Element("instance", id="task_id") - # Create sub-elements for each task ID,