diff --git a/pyxform/aliases.py b/pyxform/aliases.py index f33523f5..2d24fd53 100644 --- a/pyxform/aliases.py +++ b/pyxform/aliases.py @@ -105,6 +105,7 @@ "required message": "bind::jr:requiredMsg", "body": "control", "parameters": "parameters", + constants.ENTITIES_SAVETO: "bind::entities:saveto", } # Key is the pyxform internal name, Value is the name used in error/warning messages. TRANSLATABLE_SURVEY_COLUMNS = { diff --git a/pyxform/builder.py b/pyxform/builder.py index 38ca086d..95e52936 100644 --- a/pyxform/builder.py +++ b/pyxform/builder.py @@ -7,6 +7,7 @@ import re from pyxform import file_utils, utils +from pyxform.entities.entity_declaration import EntityDeclaration from pyxform.errors import PyXFormError from pyxform.external_instance import ExternalInstance from pyxform.question import ( @@ -94,6 +95,7 @@ def create_survey_element_from_dict(self, d): """ if "add_none_option" in d: self._add_none_option = d["add_none_option"] + if d["type"] in self.SECTION_CLASSES: section = self._create_section_from_dict(d) @@ -116,6 +118,8 @@ def create_survey_element_from_dict(self, d): return full_survey.children elif d["type"] in ["xml-external", "csv-external"]: return ExternalInstance(**d) + elif d["type"] == "entity": + return EntityDeclaration(**d) else: self._save_trigger_as_setvalue_and_remove_calculate(d) diff --git a/pyxform/constants.py b/pyxform/constants.py index a9261fce..7576d569 100644 --- a/pyxform/constants.py +++ b/pyxform/constants.py @@ -12,6 +12,7 @@ TYPE = "type" TITLE = "title" NAME = "name" +ENTITIES_SAVETO = "save_to" ID_STRING = "id_string" SMS_KEYWORD = "sms_keyword" SMS_FIELD = "sms_field" @@ -68,6 +69,7 @@ SURVEY = "survey" SETTINGS = "settings" EXTERNAL_CHOICES = "external_choices" +ENTITIES = "entities" OSM = "osm" OSM_TYPE = "binary" @@ -81,6 +83,7 @@ SETTINGS, EXTERNAL_CHOICES, OSM, + ENTITIES, ] XLS_EXTENSIONS = [".xls"] XLSX_EXTENSIONS = [".xlsx", ".xlsm"] @@ -99,6 +102,11 @@ # The ODK XForms version that generated forms comply to CURRENT_XFORMS_VERSION = "1.0.0" +# The ODK entities spec version that generated forms comply to +CURRENT_ENTITIES_VERSION = "2022.1.0" +ENTITY_RELATED = "entity_related" +ENTITIES_RESERVED_PREFIX = "__" + DEPRECATED_DEVICE_ID_METADATA_FIELDS = ["subscriberid", "simserial"] AUDIO_QUALITY_VOICE_ONLY = "voice-only" @@ -115,3 +123,9 @@ EXTERNAL_CHOICES_ITEMSET_REF_VALUE_GEOJSON = "id" ROW_FORMAT_STRING: str = "[row : %s]" +XML_IDENTIFIER_ERROR_MESSAGE = "must begin with a letter, colon, or underscore. Other characters can include numbers, dashes, and periods." +_MSG_SUPPRESS_SPELLING = ( + " If you do not mean to include a sheet, to suppress this message, " + "prefix the sheet name with an underscore. For example 'setting' " + "becomes '_setting'." +) diff --git a/pyxform/entities/entities_parsing.py b/pyxform/entities/entities_parsing.py new file mode 100644 index 00000000..e985ee44 --- /dev/null +++ b/pyxform/entities/entities_parsing.py @@ -0,0 +1,95 @@ +from typing import Dict, List + +from pyxform import constants +from pyxform.errors import PyXFormError +from pyxform.xlsparseutils import find_sheet_misspellings, is_valid_xml_tag + + +def get_entity_declaration(workbook_dict: Dict, warnings: List) -> Dict: + entities_sheet = workbook_dict.get(constants.ENTITIES, []) + + if len(entities_sheet) == 0: + similar = find_sheet_misspellings( + key=constants.ENTITIES, keys=workbook_dict.keys() + ) + if similar is not None: + warnings.append(similar + constants._MSG_SUPPRESS_SPELLING) + return {} + elif len(entities_sheet) > 1: + raise PyXFormError( + "This version of pyxform only supports declaring a single entity per form. Please make sure your entities sheet only declares one entity." + ) + + entity = entities_sheet[0] + dataset = entity["dataset"] + + if dataset.startswith(constants.ENTITIES_RESERVED_PREFIX): + raise PyXFormError( + f"Invalid dataset name: '{dataset}' starts with reserved prefix {constants.ENTITIES_RESERVED_PREFIX}." + ) + + if "." in dataset: + raise PyXFormError( + f"Invalid dataset name: '{dataset}'. Dataset names may not include periods." + ) + + if not is_valid_xml_tag(dataset): + if isinstance(dataset, bytes): + dataset = dataset.encode("utf-8") + + raise PyXFormError( + f"Invalid dataset name: '{dataset}'. Dataset names must begin with a letter, colon, or underscore. Other characters can include numbers or dashes." + ) + + if not ("label" in entity): + raise PyXFormError("The entities sheet is missing the required label column.") + + creation_condition = entity["create_if"] if "create_if" in entity else "1" + + return { + "name": "entity", + "type": "entity", + "parameters": { + "dataset": dataset, + "create": creation_condition, + "label": entity["label"], + }, + } + + +def validate_entity_saveto(row: Dict, row_number: int, entity_declaration: Dict): + save_to = row.get("bind", {}).get("entities:saveto", "") + if not save_to: + return + + if len(entity_declaration) == 0: + raise PyXFormError( + "To save entity properties using the save_to column, you must add an entities sheet and declare an entity." + ) + + if constants.GROUP in row.get(constants.TYPE) or constants.REPEAT in row.get( + constants.TYPE + ): + raise PyXFormError( + f"{constants.ROW_FORMAT_STRING % row_number} Groups and repeats can't be saved as entity properties." + ) + + error_start = f"{constants.ROW_FORMAT_STRING % row_number} Invalid save_to name:" + + if save_to == "name" or save_to == "label": + raise PyXFormError( + f"{error_start} the entity property name '{save_to}' is reserved." + ) + + if save_to.startswith(constants.ENTITIES_RESERVED_PREFIX): + raise PyXFormError( + f"{error_start} the entity property name '{save_to}' starts with reserved prefix {constants.ENTITIES_RESERVED_PREFIX}." + ) + + if not is_valid_xml_tag(save_to): + if isinstance(save_to, bytes): + save_to = save_to.encode("utf-8") + + raise PyXFormError( + f"{error_start} '{save_to}'. Entity property names {constants.XML_IDENTIFIER_ERROR_MESSAGE}" + ) diff --git a/pyxform/entities/entity_declaration.py b/pyxform/entities/entity_declaration.py new file mode 100644 index 00000000..bc55ea51 --- /dev/null +++ b/pyxform/entities/entity_declaration.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +from pyxform.survey_element import SurveyElement +from pyxform.utils import node + + +class EntityDeclaration(SurveyElement): + def xml_instance(self, **kwargs): + attributes = {} + attributes["dataset"] = self.get("parameters", {}).get("dataset", "") + attributes["create"] = "1" + attributes["id"] = "" + + label_node = node("label") + return node("entity", label_node, **attributes) + + def xml_bindings(self): + survey = self.get_root() + + create_expr = survey.insert_xpaths( + self.get("parameters", {}).get("create", "true()"), context=self + ) + create_bind = { + "calculate": create_expr, + "type": "string", + "readonly": "true()", + } + create_node = node("bind", nodeset=self.get_xpath() + "/@create", **create_bind) + + id_bind = {"type": "string", "readonly": "true()"} + id_node = node("bind", nodeset=self.get_xpath() + "/@id", **id_bind) + + id_setvalue_attrs = { + "event": "odk-instance-first-load", + "type": "string", + "readonly": "true()", + "value": "uuid()", + } + id_setvalue = node("setvalue", ref=self.get_xpath() + "/@id", **id_setvalue_attrs) + + label_expr = survey.insert_xpaths( + self.get("parameters", {}).get("label", ""), context=self + ) + label_bind = { + "calculate": label_expr, + "type": "string", + "readonly": "true()", + } + label_node = node("bind", nodeset=self.get_xpath() + "/label", **label_bind) + return [create_node, id_node, id_setvalue, label_node] diff --git a/pyxform/survey.py b/pyxform/survey.py index 90a91d59..bbbf26a6 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -173,6 +173,7 @@ class Survey(Section): "style": str, "attribute": dict, "namespaces": str, + constants.ENTITY_RELATED: str, } ) # yapf: disable @@ -204,7 +205,9 @@ def _validate_uniqueness_of_section_names(self): def get_nsmap(self): """Add additional namespaces""" - namespaces = getattr(self, constants.NAMESPACES, None) + namespaces = getattr(self, constants.NAMESPACES, "") + if getattr(self, constants.ENTITY_RELATED, "false") == "true": + namespaces += " entities=http://www.opendatakit.org/xforms/entities" if namespaces and isinstance(namespaces, str): nslist = [ @@ -550,13 +553,15 @@ def xml_model(self): self._add_empty_translations() model_kwargs = {"odk:xforms-version": constants.CURRENT_XFORMS_VERSION} + if getattr(self, constants.ENTITY_RELATED, "false") == "true": + model_kwargs["entities:entities-version"] = constants.CURRENT_ENTITIES_VERSION model_children = [] if self._translations: model_children.append(self.itext()) model_children += [node("instance", self.xml_instance())] model_children += list(self._generate_instances()) - model_children += self.xml_bindings() + model_children += self.xml_descendent_bindings() model_children += self.xml_actions() if self.submission_url or self.public_key or self.auto_send or self.auto_delete: diff --git a/pyxform/survey_element.py b/pyxform/survey_element.py index 99091422..ade87aba 100644 --- a/pyxform/survey_element.py +++ b/pyxform/survey_element.py @@ -14,10 +14,10 @@ BRACKETED_TAG_REGEX, INVALID_XFORM_TAG_REGEXP, default_is_dynamic, - is_valid_xml_tag, node, ) from pyxform.xls2json import print_pyobj_to_json +from pyxform.xlsparseutils import is_valid_xml_tag if TYPE_CHECKING: from typing import List @@ -157,13 +157,9 @@ def add_children(self, children): def validate(self): if not is_valid_xml_tag(self.name): invalid_char = re.search(INVALID_XFORM_TAG_REGEXP, self.name) - msg = ( - "The name '{}' is an invalid XML tag, it contains an " - "invalid character '{}'. Names must begin with a letter, " - "colon, or underscore, subsequent characters can include " - "numbers, dashes, and periods".format(self.name, invalid_char.group(0)) + raise PyXFormError( + f"The name '{self.name}' contains an invalid character '{invalid_char.group(0)}'. Names {constants.XML_IDENTIFIER_ERROR_MESSAGE}" ) - raise PyXFormError(msg) # TODO: Make sure renaming this doesn't cause any problems def iter_descendants(self): @@ -438,9 +434,9 @@ def xml_label_and_hint(self) -> "List[DetachableElement]": return result - def xml_binding(self): + def xml_bindings(self): """ - Return the binding for this survey element. + Return the binding(s) for this survey element. """ survey = self.get_root() bind_dict = self.bind.copy() @@ -472,18 +468,18 @@ def xml_binding(self): if k == "jr:noAppErrorString" and type(v) is dict: v = "jr:itext('%s')" % self._translation_path("jr:noAppErrorString") bind_dict[k] = survey.insert_xpaths(v, context=self) - return node("bind", nodeset=self.get_xpath(), **bind_dict) + return [node("bind", nodeset=self.get_xpath(), **bind_dict)] return None - def xml_bindings(self): + def xml_descendent_bindings(self): """ Return a list of bindings for this node and all its descendants. """ result = [] for e in self.iter_descendants(): - xml_binding = e.xml_binding() - if xml_binding is not None: - result.append(xml_binding) + xml_bindings = e.xml_bindings() + if xml_bindings is not None: + result.extend(xml_bindings) # dynamic defaults for repeats go in the body. All other dynamic defaults (setvalue actions) go in the model if ( diff --git a/pyxform/utils.py b/pyxform/utils.py index 73a53051..de5c7fde 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -20,11 +20,6 @@ SEP = "_" -# http://www.w3.org/TR/REC-xml/ -TAG_START_CHAR = r"[a-zA-Z:_]" -TAG_CHAR = r"[a-zA-Z:_0-9\-.]" -XFORM_TAG_REGEXP = "%(start)s%(char)s*" % {"start": TAG_START_CHAR, "char": TAG_CHAR} - INVALID_XFORM_TAG_REGEXP = r"[^a-zA-Z:_][^a-zA-Z:_0-9\-.]*" LAST_SAVED_INSTANCE_NAME = "__last-saved" @@ -67,13 +62,6 @@ def writexml(self, writer, indent="", addindent="", newl=""): writer.write(data) -def is_valid_xml_tag(tag): - """ - Use a regex to see if there are any invalid characters (i.e. spaces). - """ - return re.search(r"^" + XFORM_TAG_REGEXP + r"$", tag) - - def node(*args, **kwargs): """ args[0] -- a XML tag diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index 282eb15a..3e524dff 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -8,24 +8,29 @@ import re import sys from collections import Counter -from typing import Any, Dict, KeysView, List, Optional +from typing import Any, Dict, List from pyxform import aliases, constants -from pyxform.constants import EXTERNAL_INSTANCE_EXTENSIONS, ROW_FORMAT_STRING +from pyxform.constants import ( + _MSG_SUPPRESS_SPELLING, + EXTERNAL_INSTANCE_EXTENSIONS, + ROW_FORMAT_STRING, + XML_IDENTIFIER_ERROR_MESSAGE, +) +from pyxform.entities.entities_parsing import ( + get_entity_declaration, + validate_entity_saveto, +) from pyxform.errors import PyXFormError -from pyxform.utils import default_is_dynamic, is_valid_xml_tag, levenshtein_distance +from pyxform.utils import default_is_dynamic from pyxform.validators.pyxform import parameters_generic, select_from_file_params from pyxform.validators.pyxform.missing_translations_check import ( missing_translations_check, ) from pyxform.xls2json_backends import csv_to_dict, xls_to_dict, xlsx_to_dict +from pyxform.xlsparseutils import find_sheet_misspellings, is_valid_xml_tag SMART_QUOTES = {"\u2018": "'", "\u2019": "'", "\u201c": '"', "\u201d": '"'} -_MSG_SUPPRESS_SPELLING = ( - " If you do not mean to include a sheet, to suppress this message, " - "prefix the sheet name with an underscore. For example 'setting' " - "becomes '_setting'." -) def print_pyobj_to_json(pyobj, path=None): @@ -317,34 +322,6 @@ def process_image_default(default_value): return default_value -def find_sheet_misspellings(key: str, keys: "KeysView") -> "Optional[str]": - """ - Find possible sheet name misspellings to warn the user about. - - It's possible that this will warn about sheet names for sheets that have - auxilliary metadata that is not meant for processing by pyxform. For - example the "osm" sheet name may be similar to many other initialisms. - - :param key: The sheet name to look for. - :param keys: The workbook sheet names. - """ - candidates = tuple( - _k # thanks to black - for _k in keys - if 2 >= levenshtein_distance(_k.lower(), key) - and _k not in constants.SUPPORTED_SHEET_NAMES - and not _k.startswith("_") - ) - if 0 < len(candidates): - msg = ( - "When looking for a sheet named '{k}', the following sheets with " - "similar names were found: {c}." - ).format(k=key, c=str(", ".join(("'{}'".format(c) for c in candidates)))) - return msg - else: - return None - - def workbook_to_json( workbook_dict, form_name=None, @@ -560,6 +537,9 @@ def workbook_to_json( ) ) # noqa + # ########## Entities sheet ########### + entity_declaration = get_entity_declaration(workbook_dict, warnings) + # ########## Survey sheet ########### survey_sheet = workbook_dict[constants.SURVEY] # Process the headers: @@ -901,13 +881,12 @@ def workbook_to_json( if not is_valid_xml_tag(question_name): if isinstance(question_name, bytes): question_name = question_name.encode("utf-8") - error_message = ROW_FORMAT_STRING % row_number - error_message += " Invalid question name [" + question_name + "] " - error_message += "Names must begin with a letter, colon," + " or underscore." - error_message += ( - "Subsequent characters can include numbers," + " dashes, and periods." + + raise PyXFormError( + f"{ROW_FORMAT_STRING % row_number} Invalid question name '{question_name}'. Names {XML_IDENTIFIER_ERROR_MESSAGE}" ) - raise PyXFormError(error_message) + + validate_entity_saveto(row, row_number, entity_declaration) # Try to parse question as begin control statement # (i.e. begin loop/repeat/group): @@ -1357,7 +1336,6 @@ def workbook_to_json( parent_children_array.append(new_dict) continue - # TODO: Consider adding some question_type validation here. # Put the row in the json dict as is: @@ -1401,6 +1379,10 @@ def workbook_to_json( } ) + if len(entity_declaration) > 0: + json_dict[constants.ENTITY_RELATED] = "true" + meta_children.append(entity_declaration) + if len(meta_children) > 0: meta_element = { "name": "meta", @@ -1411,7 +1393,7 @@ def workbook_to_json( survey_children_array = stack[0]["parent_children"] survey_children_array.append(meta_element) - # print_pyobj_to_json(json_dict) + print_pyobj_to_json(json_dict) return json_dict diff --git a/pyxform/xlsparseutils.py b/pyxform/xlsparseutils.py new file mode 100644 index 00000000..9a10a770 --- /dev/null +++ b/pyxform/xlsparseutils.py @@ -0,0 +1,45 @@ +import re +from pyxform import constants +from typing import KeysView, Optional + +from pyxform.utils import levenshtein_distance + +# http://www.w3.org/TR/REC-xml/ +TAG_START_CHAR = r"[a-zA-Z:_]" +TAG_CHAR = r"[a-zA-Z:_0-9\-.]" +XFORM_TAG_REGEXP = "%(start)s%(char)s*" % {"start": TAG_START_CHAR, "char": TAG_CHAR} + + +def find_sheet_misspellings(key: str, keys: "KeysView") -> "Optional[str]": + """ + Find possible sheet name misspellings to warn the user about. + + It's possible that this will warn about sheet names for sheets that have + auxilliary metadata that is not meant for processing by pyxform. For + example the "osm" sheet name may be similar to many other initialisms. + + :param key: The sheet name to look for. + :param keys: The workbook sheet names. + """ + candidates = tuple( + _k # thanks to black + for _k in keys + if 2 >= levenshtein_distance(_k.lower(), key) + and _k not in constants.SUPPORTED_SHEET_NAMES + and not _k.startswith("_") + ) + if 0 < len(candidates): + msg = ( + "When looking for a sheet named '{k}', the following sheets with " + "similar names were found: {c}." + ).format(k=key, c=str(", ".join(("'{}'".format(c) for c in candidates)))) + return msg + else: + return None + + +def is_valid_xml_tag(tag): + """ + Use a regex to see if there are any invalid characters (i.e. spaces). + """ + return re.search(r"^" + XFORM_TAG_REGEXP + r"$", tag) diff --git a/tests/j2x_question_tests.py b/tests/j2x_question_tests.py index 30b4e9d6..bc242ade 100644 --- a/tests/j2x_question_tests.py +++ b/tests/j2x_question_tests.py @@ -16,6 +16,8 @@ def ctw(control): ctw stands for control_test_wrap, but ctw is shorter and easier. using begin_str and end_str to take out the wrap that xml gives us """ + if isinstance(control, list) and len(control) == 1: + control = control[0] return control.toxml() @@ -55,7 +57,7 @@ def test_question_type_string(self): self.assertEqual(ctw(q.xml_control()), expected_string_control_xml) if TESTING_BINDINGS: - self.assertEqual(ctw(q.xml_binding()), expected_string_binding_xml) + self.assertEqual(ctw(q.xml_bindings()), expected_string_binding_xml) def test_select_one_question_multilingual(self): """ @@ -86,7 +88,7 @@ def test_select_one_question_multilingual(self): self.assertEqual(ctw(q.xml_control()), expected_select_one_control_xml) if TESTING_BINDINGS: - self.assertEqual(ctw(q.xml_binding()), expected_select_one_binding_xml) + self.assertEqual(ctw(q.xml_bindings()), expected_select_one_binding_xml) def test_simple_integer_question_type_multilingual(self): """ @@ -114,7 +116,7 @@ def test_simple_integer_question_type_multilingual(self): self.assertEqual(ctw(q.xml_control()), expected_integer_control_xml) if TESTING_BINDINGS: - self.assertEqual(ctw(q.xml_binding()), expected_integer_binding_xml) + self.assertEqual(ctw(q.xml_bindings()), expected_integer_binding_xml) def test_simple_date_question_type_multilingual(self): """ @@ -140,7 +142,7 @@ def test_simple_date_question_type_multilingual(self): self.assertEqual(ctw(q.xml_control()), expected_date_control_xml) if TESTING_BINDINGS: - self.assertEqual(ctw(q.xml_binding()), expected_date_binding_xml) + self.assertEqual(ctw(q.xml_bindings()), expected_date_binding_xml) def test_simple_phone_number_question_type_multilingual(self): """ @@ -165,7 +167,7 @@ def test_simple_phone_number_question_type_multilingual(self): self.assertEqual(ctw(q.xml_control()), expected_phone_number_control_xml) if TESTING_BINDINGS: - self.assertEqual(ctw(q.xml_binding()), expected_phone_number_binding_xml) + self.assertEqual(ctw(q.xml_bindings()), expected_phone_number_binding_xml) def test_simple_select_all_question_multilingual(self): """ @@ -195,7 +197,7 @@ def test_simple_select_all_question_multilingual(self): self.assertEqual(ctw(q.xml_control()), expected_select_all_control_xml) if TESTING_BINDINGS: - self.assertEqual(ctw(q.xml_binding()), expected_select_all_binding_xml) + self.assertEqual(ctw(q.xml_bindings()), expected_select_all_binding_xml) def test_simple_decimal_question_multilingual(self): """ @@ -221,4 +223,4 @@ def test_simple_decimal_question_multilingual(self): self.assertEqual(ctw(q.xml_control()), expected_decimal_control_xml) if TESTING_BINDINGS: - self.assertEqual(ctw(q.xml_binding()), expected_decimal_binding_xml) + self.assertEqual(ctw(q.xml_bindings()), expected_decimal_binding_xml) diff --git a/tests/test_entities.py b/tests/test_entities.py new file mode 100644 index 00000000..4b62365a --- /dev/null +++ b/tests/test_entities.py @@ -0,0 +1,387 @@ +# -*- coding: utf-8 -*- +from tests.pyxform_test_case import PyxformTestCase + + +class EntitiesTest(PyxformTestCase): + def test_dataset_in_entities_sheet__adds_meta_entity_block(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | label | | + | | trees | a | | + """, + xml__xpath_match=["/h:html/h:head/x:model/x:instance/x:data/x:meta/x:entity"], + ) + + def test_multiple_dataset_rows_in_entities_sheet__errors(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | | | + | | trees | | | + | | shovels | | | + """, + errored=True, + error__contains=[ + "This version of pyxform only supports declaring a single entity per form. Please make sure your entities sheet only declares one entity." + ], + ) + + def test_dataset_in_entities_sheet__adds_dataset_attribute_to_entity(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | label | | + | | trees | a | | + """, + xml__xpath_match=[ + '/h:html/h:head/x:model/x:instance/x:data/x:meta/x:entity[@dataset = "trees"]' + ], + ) + + def test_dataset_with_reserved_prefix__errors(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | label | | + | | __sweet | a | | + """, + errored=True, + error__contains=[ + "Invalid dataset name: '__sweet' starts with reserved prefix __." + ], + ) + + def test_dataset_with_invalid_xml_name__errors(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | label | | + | | $sweet | a | | + """, + errored=True, + error__contains=[ + "Invalid dataset name: '$sweet'. Dataset names must begin with a letter, colon, or underscore. Other characters can include numbers or dashes." + ], + ) + + def test_dataset_with_period_in_name__errors(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | label | | + | | s.w.eet | a | | + """, + errored=True, + error__contains=[ + "Invalid dataset name: 's.w.eet'. Dataset names may not include periods." + ], + ) + + def test_worksheet_name_close_to_entities__produces_warning(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entitoes | | | | + | | dataset | label | | + | | trees | a | | + """, + warnings__contains=[ + "When looking for a sheet named 'entities', the following sheets with similar names were found: 'entitoes'." + ], + ) + + def test_dataset_in_entities_sheet__defaults_to_always_creating(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | label | | + | | trees | a | | + """, + xml__xpath_match=[ + '/h:html/h:head/x:model/x:instance/x:data/x:meta/x:entity[@create = "1"]' + ], + ) + + def test_create_if_in_entities_sheet__puts_expression_on_bind(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | create_if | label | + | | trees | string-length(a) > 3 | a | + """, + xml__xpath_match=[ + '/h:html/h:head/x:model/x:bind[@nodeset = "/data/meta/entity/@create" and @calculate = "string-length(a) > 3"]', + '/h:html/h:head/x:model/x:instance/x:data/x:meta/x:entity[@create = "1"]', + ], + ) + + def test_dataset_in_entities_sheet__adds_id_attribute_and_model_nodes_to_entity(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | label | | + | | trees | a | | + """, + xml__xpath_match=[ + '/h:html/h:head/x:model/x:instance/x:data/x:meta/x:entity[@id = ""]', + '/h:html/h:head/x:model/x:bind[@nodeset = "/data/meta/entity/@id" and @type = "string" and @readonly = "true()"]', + '/h:html/h:head/x:model/x:setvalue[@event = "odk-instance-first-load" and @type = "string" and @ref = "/data/meta/entity/@id" and @value = "uuid()"]', + ], + ) + + def test_label_in_entities_sheet__adds_label_and_bind_to_entity(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | label | | + | | trees | a | | + """, + xml__xpath_match=[ + "/h:html/h:head/x:model/x:instance/x:data/x:meta/x:entity/x:label", + '/h:html/h:head/x:model/x:bind[@nodeset = "/data/meta/entity/label" and @type = "string" and @readonly = "true()" and @calculate = "a"]', + ], + ) + + def test_label_and_create_if_in_entities_sheet__expand_node_selectors_to_xpath(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | label | create_if | + | | trees | ${a} | string-length(${a}) > 3 | + """, + xml__xpath_match=[ + '/h:html/h:head/x:model/x:bind[@nodeset = "/data/meta/entity/@create" and @calculate = "string-length( /data/a ) > 3"]', + '/h:html/h:head/x:model/x:bind[@nodeset = "/data/meta/entity/label" and @type = "string" and @readonly = "true()" and @calculate = " /data/a "]', + ], + ) + + def test_entity_label__required(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | | | + | | trees | | | + """, + errored=True, + error__contains=["The entities sheet is missing the required label column."], + ) + + def test_dataset_in_entities_sheet__adds_entities_namespace(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | label | | + | | trees | a | | + """, + xml__contains=['xmlns:entities="http://www.opendatakit.org/xforms/entities"'], + ) + + def test_entities_namespace__omitted_if_no_entities_sheet(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + """, + xml__excludes=['xmlns:entities="http://www.opendatakit.org/xforms/entities"'], + ) + + def test_dataset_in_entities_sheet__adds_entities_version(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + | entities | | | | + | | dataset | label | | + | | trees | a | | + """, + xml__xpath_match=[ + '/h:html/h:head/x:model[@entities:entities-version = "2022.1.0"]' + ], + ) + + def test_entities_version__omitted_if_no_entities_sheet(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | + | | type | name | label | + | | text | a | A | + """, + xml__excludes=['entities:entities-version = "2022.1.0"'], + ) + + def test_saveto_column__added_to_xml(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | | + | | type | name | label | save_to | + | | text | a | A | foo | + | entities | | | | | + | | dataset | label | | | + | | trees | a | | | + """, + xml__xpath_match=[ + '/h:html/h:head/x:model/x:bind[@nodeset = "/data/a" and @entities:saveto = "foo"]' + ], + ) + + def test_saveto_without_entities_sheet__errors(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | | + | | type | name | label | save_to | + | | text | a | A | foo | + """, + errored=True, + error__contains=[ + "To save entity properties using the save_to column, you must add an entities sheet and declare an entity." + ], + ) + + def test_name_in_saveto_column__errors(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | | + | | type | name | label | save_to | + | | text | a | A | name | + | entities | | | | | + | | dataset | label | | | + | | trees | a | | | + """, + errored=True, + error__contains=[ + "[row : 2] Invalid save_to name: the entity property name 'name' is reserved." + ], + ) + + def test_label_in_saveto_column__errors(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | | + | | type | name | label | save_to | + | | text | a | A | label | + | entities | | | | | + | | dataset | label | | | + | | trees | a | | | + """, + errored=True, + error__contains=[ + "[row : 2] Invalid save_to name: the entity property name 'label' is reserved." + ], + ) + + def test_system_prefix_in_saveto_column__errors(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | | + | | type | name | label | save_to | + | | text | a | A | __a | + | entities | | | | | + | | dataset | label | | | + | | trees | a | | | + """, + errored=True, + error__contains=[ + "[row : 2] Invalid save_to name: the entity property name '__a' starts with reserved prefix __." + ], + ) + + def test_invalid_xml_identifier_in_saveto_column__errors(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | | + | | type | name | label | save_to | + | | text | a | A | $a | + | entities | | | | | + | | dataset | label | | | + | | trees | a | | | + """, + errored=True, + error__contains=[ + "[row : 2] Invalid save_to name: '$a'. Entity property names must begin with a letter, colon, or underscore. Other characters can include numbers, dashes, and periods." + ], + ) + + def test_saveto_on_group__errors(self): + self.assertPyxformXform( + name="data", + md=""" + | survey | | | | | + | | type | name | label | save_to | + | | begin_group | a | A | a | + | | end_group | | | | + | entities | | | | | + | | dataset | label | | | + | | trees | a | | | + """, + errored=True, + error__contains=[ + "[row : 2] Groups and repeats can't be saved as entity properties." + ], + ) diff --git a/tests/test_sheet_columns.py b/tests/test_sheet_columns.py index 2fff4d48..106d5c04 100644 --- a/tests/test_sheet_columns.py +++ b/tests/test_sheet_columns.py @@ -158,12 +158,7 @@ def test_missing_list_name(self): def test_clear_filename_error_message(self): """Test clear filename error message""" - error_message = ( - "The name 'bad@filename' is an invalid XML tag, it " - "contains an invalid character '@'. Names must begin" - " with a letter, colon, or underscore, subsequent " - "characters can include numbers, dashes, and periods" - ) + error_message = "The name 'bad@filename' contains an invalid character '@'. Names must begin with a letter, colon, or underscore. Other characters can include numbers, dashes, and periods." self.assertPyxformXform( name="bad@filename", ss_structure=self._simple_choice_ss(