Add initial support for entity creation (#624)

pyxform/aliases.py

@@ -105,6 +105,7 @@
     "required message": "bind::jr:requiredMsg",
     "body": "control",
     "parameters": "parameters",
+    constants.ENTITIES_SAVETO: "bind::entities:saveto",
 }
 # Key is the pyxform internal name, Value is the name used in error/warning messages.
 TRANSLATABLE_SURVEY_COLUMNS = {

pyxform/builder.py

@@ -7,6 +7,7 @@
 import re
 
 from pyxform import file_utils, utils
+from pyxform.entities.entity_declaration import EntityDeclaration
 from pyxform.errors import PyXFormError
 from pyxform.external_instance import ExternalInstance
 from pyxform.question import (
@@ -94,6 +95,7 @@ def create_survey_element_from_dict(self, d):
         """
         if "add_none_option" in d:
             self._add_none_option = d["add_none_option"]
+
         if d["type"] in self.SECTION_CLASSES:
             section = self._create_section_from_dict(d)
 
@@ -116,6 +118,8 @@ def create_survey_element_from_dict(self, d):
             return full_survey.children
         elif d["type"] in ["xml-external", "csv-external"]:
             return ExternalInstance(**d)
+        elif d["type"] == "entity":
+            return EntityDeclaration(**d)
         else:
             self._save_trigger_as_setvalue_and_remove_calculate(d)
 

pyxform/constants.py

@@ -12,6 +12,7 @@
 TYPE = "type"
 TITLE = "title"
 NAME = "name"
+ENTITIES_SAVETO = "save_to"
 ID_STRING = "id_string"
 SMS_KEYWORD = "sms_keyword"
 SMS_FIELD = "sms_field"
@@ -68,6 +69,7 @@
 SURVEY = "survey"
 SETTINGS = "settings"
 EXTERNAL_CHOICES = "external_choices"
+ENTITIES = "entities"
 
 OSM = "osm"
 OSM_TYPE = "binary"
@@ -81,6 +83,7 @@
     SETTINGS,
     EXTERNAL_CHOICES,
     OSM,
+    ENTITIES,
 ]
 XLS_EXTENSIONS = [".xls"]
 XLSX_EXTENSIONS = [".xlsx", ".xlsm"]
@@ -99,6 +102,11 @@
 # The ODK XForms version that generated forms comply to
 CURRENT_XFORMS_VERSION = "1.0.0"
 
+# The ODK entities spec version that generated forms comply to
+CURRENT_ENTITIES_VERSION = "2022.1.0"
+ENTITY_RELATED = "entity_related"
+ENTITIES_RESERVED_PREFIX = "__"
+
 DEPRECATED_DEVICE_ID_METADATA_FIELDS = ["subscriberid", "simserial"]
 
 AUDIO_QUALITY_VOICE_ONLY = "voice-only"
@@ -115,3 +123,9 @@
 EXTERNAL_CHOICES_ITEMSET_REF_VALUE_GEOJSON = "id"
 
 ROW_FORMAT_STRING: str = "[row : %s]"
+XML_IDENTIFIER_ERROR_MESSAGE = "must begin with a letter, colon, or underscore. Other characters can include numbers, dashes, and periods."
+_MSG_SUPPRESS_SPELLING = (
+    " If you do not mean to include a sheet, to suppress this message, "
+    "prefix the sheet name with an underscore. For example 'setting' "
+    "becomes '_setting'."
+)

pyxform/entities/entities_parsing.py

@@ -0,0 +1,95 @@
+from typing import Dict, List
+
+from pyxform import constants
+from pyxform.errors import PyXFormError
+from pyxform.xlsparseutils import find_sheet_misspellings, is_valid_xml_tag
+
+
+def get_entity_declaration(workbook_dict: Dict, warnings: List) -> Dict:
+    entities_sheet = workbook_dict.get(constants.ENTITIES, [])
+
+    if len(entities_sheet) == 0:
+        similar = find_sheet_misspellings(
+            key=constants.ENTITIES, keys=workbook_dict.keys()
+        )
+        if similar is not None:
+            warnings.append(similar + constants._MSG_SUPPRESS_SPELLING)
+        return {}
+    elif len(entities_sheet) > 1:
+        raise PyXFormError(
+            "This version of pyxform only supports declaring a single entity per form. Please make sure your entities sheet only declares one entity."
+        )
+
+    entity = entities_sheet[0]
+    dataset = entity["dataset"]
+
+    if dataset.startswith(constants.ENTITIES_RESERVED_PREFIX):
+        raise PyXFormError(
+            f"Invalid dataset name: '{dataset}' starts with reserved prefix {constants.ENTITIES_RESERVED_PREFIX}."
+        )
+
+    if "." in dataset:
+        raise PyXFormError(
+            f"Invalid dataset name: '{dataset}'. Dataset names may not include periods."
+        )
+
+    if not is_valid_xml_tag(dataset):
+        if isinstance(dataset, bytes):
+            dataset = dataset.encode("utf-8")
+
+        raise PyXFormError(
+            f"Invalid dataset name: '{dataset}'. Dataset names must begin with a letter, colon, or underscore. Other characters can include numbers or dashes."
+        )
+
+    if not ("label" in entity):
+        raise PyXFormError("The entities sheet is missing the required label column.")
+
+    creation_condition = entity["create_if"] if "create_if" in entity else "1"
+
+    return {
+        "name": "entity",
+        "type": "entity",
+        "parameters": {
+            "dataset": dataset,
+            "create": creation_condition,
+            "label": entity["label"],
+        },
+    }
+
+
+def validate_entity_saveto(row: Dict, row_number: int, entity_declaration: Dict):
+    save_to = row.get("bind", {}).get("entities:saveto", "")
+    if not save_to:
+        return
+
+    if len(entity_declaration) == 0:
+        raise PyXFormError(
+            "To save entity properties using the save_to column, you must add an entities sheet and declare an entity."
+        )
+
+    if constants.GROUP in row.get(constants.TYPE) or constants.REPEAT in row.get(
+        constants.TYPE
+    ):
+        raise PyXFormError(
+            f"{constants.ROW_FORMAT_STRING % row_number} Groups and repeats can't be saved as entity properties."
+        )
+
+    error_start = f"{constants.ROW_FORMAT_STRING % row_number} Invalid save_to name:"
+
+    if save_to == "name" or save_to == "label":
+        raise PyXFormError(
+            f"{error_start} the entity property name '{save_to}' is reserved."
+        )
+
+    if save_to.startswith(constants.ENTITIES_RESERVED_PREFIX):
+        raise PyXFormError(
+            f"{error_start} the entity property name '{save_to}' starts with reserved prefix {constants.ENTITIES_RESERVED_PREFIX}."
+        )
+
+    if not is_valid_xml_tag(save_to):
+        if isinstance(save_to, bytes):
+            save_to = save_to.encode("utf-8")
+
+        raise PyXFormError(
+            f"{error_start} '{save_to}'. Entity property names {constants.XML_IDENTIFIER_ERROR_MESSAGE}"
+        )

pyxform/entities/entity_declaration.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+
+from pyxform.survey_element import SurveyElement
+from pyxform.utils import node
+
+
+class EntityDeclaration(SurveyElement):
+    def xml_instance(self, **kwargs):
+        attributes = {}
+        attributes["dataset"] = self.get("parameters", {}).get("dataset", "")
+        attributes["create"] = "1"
+        attributes["id"] = ""
+
+        label_node = node("label")
+        return node("entity", label_node, **attributes)
+
+    def xml_bindings(self):
+        survey = self.get_root()
+
+        create_expr = survey.insert_xpaths(
+            self.get("parameters", {}).get("create", "true()"), context=self
+        )
+        create_bind = {
+            "calculate": create_expr,
+            "type": "string",
+            "readonly": "true()",
+        }
+        create_node = node("bind", nodeset=self.get_xpath() + "/@create", **create_bind)
+
+        id_bind = {"type": "string", "readonly": "true()"}
+        id_node = node("bind", nodeset=self.get_xpath() + "/@id", **id_bind)
+
+        id_setvalue_attrs = {
+            "event": "odk-instance-first-load",
+            "type": "string",
+            "readonly": "true()",
+            "value": "uuid()",
+        }
+        id_setvalue = node("setvalue", ref=self.get_xpath() + "/@id", **id_setvalue_attrs)
+
+        label_expr = survey.insert_xpaths(
+            self.get("parameters", {}).get("label", ""), context=self
+        )
+        label_bind = {
+            "calculate": label_expr,
+            "type": "string",
+            "readonly": "true()",
+        }
+        label_node = node("bind", nodeset=self.get_xpath() + "/label", **label_bind)
+        return [create_node, id_node, id_setvalue, label_node]

pyxform/survey.py

@@ -173,6 +173,7 @@ class Survey(Section):
             "style": str,
             "attribute": dict,
             "namespaces": str,
+            constants.ENTITY_RELATED: str,
         }
     )  # yapf: disable
 
@@ -204,7 +205,9 @@ def _validate_uniqueness_of_section_names(self):
 
     def get_nsmap(self):
         """Add additional namespaces"""
-        namespaces = getattr(self, constants.NAMESPACES, None)
+        namespaces = getattr(self, constants.NAMESPACES, "")
+        if getattr(self, constants.ENTITY_RELATED, "false") == "true":
+            namespaces += " entities=http://www.opendatakit.org/xforms/entities"
 
         if namespaces and isinstance(namespaces, str):
             nslist = [
@@ -550,13 +553,15 @@ def xml_model(self):
         self._add_empty_translations()
 
         model_kwargs = {"odk:xforms-version": constants.CURRENT_XFORMS_VERSION}
+        if getattr(self, constants.ENTITY_RELATED, "false") == "true":
+            model_kwargs["entities:entities-version"] = constants.CURRENT_ENTITIES_VERSION
 
         model_children = []
         if self._translations:
             model_children.append(self.itext())
         model_children += [node("instance", self.xml_instance())]
         model_children += list(self._generate_instances())
-        model_children += self.xml_bindings()
+        model_children += self.xml_descendent_bindings()
         model_children += self.xml_actions()
 
         if self.submission_url or self.public_key or self.auto_send or self.auto_delete:

pyxform/survey_element.py

@@ -14,10 +14,10 @@
     BRACKETED_TAG_REGEX,
     INVALID_XFORM_TAG_REGEXP,
     default_is_dynamic,
-    is_valid_xml_tag,
     node,
 )
 from pyxform.xls2json import print_pyobj_to_json
+from pyxform.xlsparseutils import is_valid_xml_tag
 
 if TYPE_CHECKING:
     from typing import List
@@ -157,13 +157,9 @@ def add_children(self, children):
     def validate(self):
         if not is_valid_xml_tag(self.name):
             invalid_char = re.search(INVALID_XFORM_TAG_REGEXP, self.name)
-            msg = (
-                "The name '{}' is an invalid XML tag, it contains an "
-                "invalid character '{}'. Names must begin with a letter, "
-                "colon, or underscore, subsequent characters can include "
-                "numbers, dashes, and periods".format(self.name, invalid_char.group(0))
+            raise PyXFormError(
+                f"The name '{self.name}' contains an invalid character '{invalid_char.group(0)}'. Names {constants.XML_IDENTIFIER_ERROR_MESSAGE}"
             )
-            raise PyXFormError(msg)
 
     # TODO: Make sure renaming this doesn't cause any problems
     def iter_descendants(self):
@@ -438,9 +434,9 @@ def xml_label_and_hint(self) -> "List[DetachableElement]":
 
         return result
 
-    def xml_binding(self):
+    def xml_bindings(self):
         """
-        Return the binding for this survey element.
+        Return the binding(s) for this survey element.
         """
         survey = self.get_root()
         bind_dict = self.bind.copy()
@@ -472,18 +468,18 @@ def xml_binding(self):
                 if k == "jr:noAppErrorString" and type(v) is dict:
                     v = "jr:itext('%s')" % self._translation_path("jr:noAppErrorString")
                 bind_dict[k] = survey.insert_xpaths(v, context=self)
-            return node("bind", nodeset=self.get_xpath(), **bind_dict)
+            return [node("bind", nodeset=self.get_xpath(), **bind_dict)]
         return None
 
-    def xml_bindings(self):
+    def xml_descendent_bindings(self):
         """
         Return a list of bindings for this node and all its descendants.
         """
         result = []
         for e in self.iter_descendants():
-            xml_binding = e.xml_binding()
-            if xml_binding is not None:
-                result.append(xml_binding)
+            xml_bindings = e.xml_bindings()
+            if xml_bindings is not None:
+                result.extend(xml_bindings)
 
             # dynamic defaults for repeats go in the body. All other dynamic defaults (setvalue actions) go in the model
             if (

pyxform/utils.py

@@ -20,11 +20,6 @@
 
 SEP = "_"
 
-# http://www.w3.org/TR/REC-xml/
-TAG_START_CHAR = r"[a-zA-Z:_]"
-TAG_CHAR = r"[a-zA-Z:_0-9\-.]"
-XFORM_TAG_REGEXP = "%(start)s%(char)s*" % {"start": TAG_START_CHAR, "char": TAG_CHAR}
-
 INVALID_XFORM_TAG_REGEXP = r"[^a-zA-Z:_][^a-zA-Z:_0-9\-.]*"
 
 LAST_SAVED_INSTANCE_NAME = "__last-saved"
@@ -67,13 +62,6 @@ def writexml(self, writer, indent="", addindent="", newl=""):
         writer.write(data)
 
 
-def is_valid_xml_tag(tag):
-    """
-    Use a regex to see if there are any invalid characters (i.e. spaces).
-    """
-    return re.search(r"^" + XFORM_TAG_REGEXP + r"$", tag)
-
-
 def node(*args, **kwargs):
     """
     args[0] -- a XML tag