diff --git a/pyxform/builder.py b/pyxform/builder.py index 0ade9150f..86675f142 100644 --- a/pyxform/builder.py +++ b/pyxform/builder.py @@ -2,10 +2,9 @@ Survey builder functionality. """ -import copy import os from collections import defaultdict -from typing import TYPE_CHECKING, Any, Union +from typing import Any from pyxform import constants as const from pyxform import file_utils, utils @@ -15,6 +14,7 @@ from pyxform.question import ( InputQuestion, MultipleChoiceQuestion, + Option, OsmUploadQuestion, Question, RangeQuestion, @@ -24,15 +24,9 @@ from pyxform.question_type_dictionary import QUESTION_TYPE_DICT from pyxform.section import GroupedSection, RepeatingSection from pyxform.survey import Survey +from pyxform.survey_element import SurveyElement from pyxform.xls2json import SurveyReader -if TYPE_CHECKING: - from pyxform.survey_element import SurveyElement - -OR_OTHER_CHOICE = { - const.NAME: "other", - const.LABEL: "Other", -} QUESTION_CLASSES = { "": Question, "action": Question, @@ -52,29 +46,6 @@ } -def copy_json_dict(json_dict): - """ - Returns a deep copy of the input json_dict - """ - json_dict_copy = None - items = None - - if isinstance(json_dict, list): - json_dict_copy = [None] * len(json_dict) - items = enumerate(json_dict) - elif isinstance(json_dict, dict): - json_dict_copy = {} - items = json_dict.items() - - for key, value in items: - if isinstance(value, dict | list): - json_dict_copy[key] = copy_json_dict(value) - else: - json_dict_copy[key] = value - - return json_dict_copy - - class SurveyElementBuilder: def __init__(self, **kwargs): # I don't know why we would need an explicit none option for @@ -87,8 +58,6 @@ def __init__(self, **kwargs): self.setvalues_by_triggering_ref = defaultdict(list) # dictionary of setgeopoint target and value tuple indexed by triggering element self.setgeopoint_by_triggering_ref = defaultdict(list) - # For tracking survey-level choices while recursing through the survey. - self._choices: dict[str, Any] = {} def set_sections(self, sections): """ @@ -101,30 +70,28 @@ def set_sections(self, sections): self._sections = sections def create_survey_element_from_dict( - self, d: dict[str, Any] - ) -> Union["SurveyElement", list["SurveyElement"]]: + self, d: dict[str, Any], choices: dict[str, tuple[Option, ...]] | None = None + ) -> SurveyElement | list[SurveyElement]: """ Convert from a nested python dictionary/array structure (a json dict I call it because it corresponds directly with a json object) to a survey object + + :param d: data to use for constructing SurveyElements. """ if "add_none_option" in d: self._add_none_option = d["add_none_option"] if d[const.TYPE] in SECTION_CLASSES: - if d[const.TYPE] == const.SURVEY: - self._choices = copy.deepcopy(d.get(const.CHOICES, {})) - - section = self._create_section_from_dict(d) + section = self._create_section_from_dict(d=d, choices=choices) if d[const.TYPE] == const.SURVEY: section.setvalues_by_triggering_ref = self.setvalues_by_triggering_ref section.setgeopoint_by_triggering_ref = self.setgeopoint_by_triggering_ref - section.choices = self._choices return section elif d[const.TYPE] == const.LOOP: - return self._create_loop_from_dict(d) + return self._create_loop_from_dict(d=d, choices=choices) elif d[const.TYPE] == "include": section_name = d[const.NAME] if section_name not in self._sections: @@ -134,16 +101,19 @@ def create_survey_element_from_dict( self._sections.keys(), ) d = self._sections[section_name] - full_survey = self.create_survey_element_from_dict(d) + full_survey = self.create_survey_element_from_dict(d=d, choices=choices) return full_survey.children - elif d[const.TYPE] in ["xml-external", "csv-external"]: + elif d[const.TYPE] in {"xml-external", "csv-external"}: return ExternalInstance(**d) elif d[const.TYPE] == "entity": return EntityDeclaration(**d) else: self._save_trigger(d=d) return self._create_question_from_dict( - d, copy_json_dict(QUESTION_TYPE_DICT), self._add_none_option + d=d, + question_type_dictionary=QUESTION_TYPE_DICT, + add_none_option=self._add_none_option, + choices=choices, ) def _save_trigger(self, d: dict) -> None: @@ -163,70 +133,35 @@ def _create_question_from_dict( d: dict[str, Any], question_type_dictionary: dict[str, Any], add_none_option: bool = False, - ) -> Question | list[Question]: + choices: dict[str, tuple[Option, ...]] | None = None, + ) -> Question | tuple[Question, ...]: question_type_str = d[const.TYPE] - d_copy = d.copy() # TODO: Keep add none option? if add_none_option and question_type_str.startswith(const.SELECT_ALL_THAT_APPLY): - SurveyElementBuilder._add_none_option_to_select_all_that_apply(d_copy) - - # Handle or_other on select type questions - or_other_len = len(const.SELECT_OR_OTHER_SUFFIX) - if question_type_str.endswith(const.SELECT_OR_OTHER_SUFFIX): - question_type_str = question_type_str[: len(question_type_str) - or_other_len] - d_copy[const.TYPE] = question_type_str - SurveyElementBuilder._add_other_option_to_multiple_choice_question(d_copy) - return [ - SurveyElementBuilder._create_question_from_dict( - d_copy, question_type_dictionary, add_none_option - ), - SurveyElementBuilder._create_specify_other_question_from_dict(d_copy), - ] + SurveyElementBuilder._add_none_option_to_select_all_that_apply(d) question_class = SurveyElementBuilder._get_question_class( question_type_str, question_type_dictionary ) - # todo: clean up this spaghetti code - d_copy["question_type_dictionary"] = question_type_dictionary if question_class: - return question_class(**d_copy) - - return [] - - @staticmethod - def _add_other_option_to_multiple_choice_question(d: dict[str, Any]) -> None: - # ideally, we'd just be pulling from children - choice_list = d.get(const.CHOICES, d.get(const.CHILDREN, [])) - if len(choice_list) <= 0: - raise PyXFormError("There should be choices for this question.") - if not any(c[const.NAME] == OR_OTHER_CHOICE[const.NAME] for c in choice_list): - choice_list.append(SurveyElementBuilder._get_or_other_choice(choice_list)) + if const.CHOICES in d and choices: + return question_class( + question_type_dictionary=question_type_dictionary, + choices=choices.get(d[const.ITEMSET], d[const.CHOICES]), + **{k: v for k, v in d.items() if k != const.CHOICES}, + ) + else: + return question_class( + question_type_dictionary=question_type_dictionary, **d + ) - @staticmethod - def _get_or_other_choice( - choice_list: list[dict[str, Any]], - ) -> dict[str, str | dict]: - """ - If the choices have any translations, return an OR_OTHER choice for each lang. - """ - if any(isinstance(c.get(const.LABEL), dict) for c in choice_list): - langs = { - lang - for c in choice_list - for lang in c[const.LABEL] - if isinstance(c.get(const.LABEL), dict) - } - return { - const.NAME: OR_OTHER_CHOICE[const.NAME], - const.LABEL: {lang: OR_OTHER_CHOICE[const.LABEL] for lang in langs}, - } - return OR_OTHER_CHOICE + return () @staticmethod def _add_none_option_to_select_all_that_apply(d_copy): - choice_list = d_copy.get(const.CHOICES, d_copy.get(const.CHILDREN, [])) + choice_list = d_copy.get(const.CHOICES, d_copy.get(const.CHILDREN, ())) if len(choice_list) <= 0: raise PyXFormError("There should be choices for this question.") none_choice = {const.NAME: "none", const.LABEL: "None"} @@ -247,79 +182,65 @@ def _get_question_class(question_type_str, question_type_dictionary): and find what class it maps to going through type_dictionary -> QUESTION_CLASSES """ - question_type = question_type_dictionary.get(question_type_str, {}) - control_dict = question_type.get(const.CONTROL, {}) - control_tag = control_dict.get("tag", "") - if control_tag == "upload" and control_dict.get("mediatype") == "osm/*": - control_tag = "osm" + control_tag = "" + question_type = question_type_dictionary.get(question_type_str) + if question_type: + control_dict = question_type.get(const.CONTROL) + if control_dict: + control_tag = control_dict.get("tag") + if control_tag == "upload" and control_dict.get("mediatype") == "osm/*": + control_tag = "osm" return QUESTION_CLASSES[control_tag] - @staticmethod - def _create_specify_other_question_from_dict(d: dict[str, Any]) -> InputQuestion: - kwargs = { - const.TYPE: "text", - const.NAME: f"{d[const.NAME]}_other", - const.LABEL: "Specify other.", - const.BIND: {"relevant": f"selected(../{d[const.NAME]}, 'other')"}, - } - return InputQuestion(**kwargs) - - def _create_section_from_dict(self, d): - d_copy = d.copy() - children = d_copy.pop(const.CHILDREN, []) - section_class = SECTION_CLASSES[d_copy[const.TYPE]] + def _create_section_from_dict( + self, d: dict[str, Any], choices: dict[str, tuple[Option, ...]] | None = None + ) -> Survey | GroupedSection | RepeatingSection: + children = d.get(const.CHILDREN) + section_class = SECTION_CLASSES[d[const.TYPE]] if d[const.TYPE] == const.SURVEY and const.TITLE not in d: - d_copy[const.TITLE] = d[const.NAME] - result = section_class(**d_copy) - for child in children: - # Deep copying the child is a hacky solution to the or_other bug. - # I don't know why it works. - # And I hope it doesn't break something else. - # I think the good solution would be to rewrite this class. - survey_element = self.create_survey_element_from_dict(copy.deepcopy(child)) - if child[const.TYPE].endswith(const.SELECT_OR_OTHER_SUFFIX): - select_question = survey_element[0] - itemset_choices = self._choices.get(select_question[const.ITEMSET], None) - if ( - itemset_choices is not None - and isinstance(itemset_choices, list) - and not any( - c[const.NAME] == OR_OTHER_CHOICE[const.NAME] - for c in itemset_choices + d[const.TITLE] = d[const.NAME] + result = section_class(**d) + if children: + for child in children: + if isinstance(result, Survey): + survey_element = self.create_survey_element_from_dict( + d=child, choices=result.choices + ) + else: + survey_element = self.create_survey_element_from_dict( + d=child, choices=choices ) - ): - itemset_choices.append(self._get_or_other_choice(itemset_choices)) - # This is required for builder_tests.BuilderTests.test_loop to pass. - self._add_other_option_to_multiple_choice_question(d=child) - if survey_element: result.add_children(survey_element) return result - def _create_loop_from_dict(self, d): + def _create_loop_from_dict( + self, d: dict[str, Any], choices: dict[str, tuple[Option, ...]] | None = None + ): """ Takes a json_dict of "loop" type Returns a GroupedSection """ - d_copy = d.copy() - children = d_copy.pop(const.CHILDREN, []) - columns = d_copy.pop(const.COLUMNS, []) - result = GroupedSection(**d_copy) + children = d.get(const.CHILDREN) + result = GroupedSection(**d) # columns is a left over from when this was # create_table_from_dict, I will need to clean this up - for column_dict in columns: + for column_dict in d.get(const.COLUMNS, ()): # If this is a none option for a select all that apply # question then we should skip adding it to the result if column_dict[const.NAME] == "none": continue - column = GroupedSection(**column_dict) - for child in children: - question_dict = self._name_and_label_substitutions(child, column_dict) - question = self.create_survey_element_from_dict(question_dict) - column.add_child(question) + column = GroupedSection(type=const.GROUP, **column_dict) + if children is not None: + for child in children: + question_dict = self._name_and_label_substitutions(child, column_dict) + question = self.create_survey_element_from_dict( + d=question_dict, choices=choices + ) + column.add_child(question) result.add_child(column) if result.name != "": return result @@ -331,7 +252,7 @@ def _create_loop_from_dict(self, d): def _name_and_label_substitutions(question_template, column_headers): # if the label in column_headers has multiple languages setup a # dictionary by language to do substitutions. - info_by_lang = {} + info_by_lang = None if isinstance(column_headers[const.LABEL], dict): info_by_lang = { lang: { @@ -348,7 +269,7 @@ def _name_and_label_substitutions(question_template, column_headers): elif isinstance(result[key], dict): result[key] = result[key].copy() for key2 in result[key].keys(): - if isinstance(column_headers[const.LABEL], dict): + if info_by_lang and isinstance(column_headers[const.LABEL], dict): result[key][key2] %= info_by_lang.get(key2, column_headers) else: result[key][key2] %= column_headers @@ -356,7 +277,8 @@ def _name_and_label_substitutions(question_template, column_headers): def create_survey_element_from_json(self, str_or_path): d = utils.get_pyobj_from_json(str_or_path) - return self.create_survey_element_from_dict(d) + # Loading JSON creates a new dictionary structure so no need to re-copy. + return self.create_survey_element_from_dict(d=d) def create_survey_element_from_dict(d, sections=None): diff --git a/pyxform/constants.py b/pyxform/constants.py index bd569af5a..c33323e96 100644 --- a/pyxform/constants.py +++ b/pyxform/constants.py @@ -87,17 +87,17 @@ NAMESPACES = "namespaces" # The following are the possible sheet names: -SUPPORTED_SHEET_NAMES = [ +SUPPORTED_SHEET_NAMES = { SURVEY, CHOICES, SETTINGS, EXTERNAL_CHOICES, OSM, ENTITIES, -] -XLS_EXTENSIONS = [".xls"] -XLSX_EXTENSIONS = [".xlsx", ".xlsm"] -SUPPORTED_FILE_EXTENSIONS = XLS_EXTENSIONS + XLSX_EXTENSIONS +} +XLS_EXTENSIONS = {".xls"} +XLSX_EXTENSIONS = {".xlsx", ".xlsm"} +SUPPORTED_FILE_EXTENSIONS = {*XLS_EXTENSIONS, *XLSX_EXTENSIONS} LOCATION_PRIORITY = "location-priority" LOCATION_MIN_INTERVAL = "location-min-interval" @@ -107,7 +107,7 @@ TRACK_CHANGES_REASONS = "track-changes-reasons" # supported bind keywords for which external instances will be created for pulldata function -EXTERNAL_INSTANCES = ["calculate", "constraint", "readonly", "required", "relevant"] +EXTERNAL_INSTANCES = {"calculate", "constraint", "readonly", "required", "relevant"} # The ODK XForms version that generated forms comply to CURRENT_XFORMS_VERSION = "1.0.0" @@ -130,14 +130,14 @@ class EntityColumns(StrEnum): OFFLINE = "offline" -DEPRECATED_DEVICE_ID_METADATA_FIELDS = ["subscriberid", "simserial"] +DEPRECATED_DEVICE_ID_METADATA_FIELDS = {"subscriberid", "simserial"} AUDIO_QUALITY_VOICE_ONLY = "voice-only" AUDIO_QUALITY_LOW = "low" AUDIO_QUALITY_NORMAL = "normal" AUDIO_QUALITY_EXTERNAL = "external" -EXTERNAL_INSTANCE_EXTENSIONS = [".xml", ".csv", ".geojson"] +EXTERNAL_INSTANCE_EXTENSIONS = {".xml", ".csv", ".geojson"} EXTERNAL_CHOICES_ITEMSET_REF_LABEL = "label" EXTERNAL_CHOICES_ITEMSET_REF_VALUE = "name" @@ -153,13 +153,13 @@ class EntityColumns(StrEnum): "becomes '_setting'." ) -CONVERTIBLE_BIND_ATTRIBUTES = ( +CONVERTIBLE_BIND_ATTRIBUTES = { "readonly", "required", "relevant", "constraint", "calculate", -) +} NSMAP = { "xmlns": "http://www.w3.org/2002/xforms", "xmlns:h": "http://www.w3.org/1999/xhtml", @@ -169,3 +169,5 @@ class EntityColumns(StrEnum): "xmlns:orx": "http://openrosa.org/xforms", "xmlns:odk": "http://www.opendatakit.org/xforms", } +SUPPORTED_MEDIA_TYPES = {"image", "big-image", "audio", "video"} +OR_OTHER_CHOICE = {NAME: "other", LABEL: "Other"} diff --git a/pyxform/entities/entity_declaration.py b/pyxform/entities/entity_declaration.py index b34950a0b..ed0460032 100644 --- a/pyxform/entities/entity_declaration.py +++ b/pyxform/entities/entity_declaration.py @@ -1,8 +1,19 @@ +from typing import TYPE_CHECKING + from pyxform import constants as const -from pyxform.survey_element import SurveyElement +from pyxform.survey_element import SURVEY_ELEMENT_FIELDS, SurveyElement from pyxform.utils import node +if TYPE_CHECKING: + from pyxform.survey import Survey + + EC = const.EntityColumns +ENTITY_EXTRA_FIELDS = ( + const.TYPE, + const.PARAMETERS, +) +ENTITY_FIELDS = (*SURVEY_ELEMENT_FIELDS, *ENTITY_EXTRA_FIELDS) class EntityDeclaration(SurveyElement): @@ -23,6 +34,17 @@ class EntityDeclaration(SurveyElement): 0 1 1 error, need id to update """ + __slots__ = ENTITY_EXTRA_FIELDS + + @staticmethod + def get_slot_names() -> tuple[str, ...]: + return ENTITY_FIELDS + + def __init__(self, name: str, type: str, parameters: dict, **kwargs): + self.parameters: dict = parameters + self.type: str = type + super().__init__(name=name, **kwargs) + def xml_instance(self, **kwargs): parameters = self.get(const.PARAMETERS, {}) @@ -50,11 +72,10 @@ def xml_instance(self, **kwargs): else: return node(const.ENTITY, **attributes) - def xml_bindings(self): + def xml_bindings(self, survey: "Survey"): """ See the class comment for an explanation of the logic for generating bindings. """ - survey = self.get_root() parameters = self.get(const.PARAMETERS, {}) entity_id_expression = parameters.get(EC.ENTITY_ID, None) create_condition = parameters.get(EC.CREATE_IF, None) @@ -125,5 +146,5 @@ def _get_bind_node(self, survey, expression, destination): return node(const.BIND, nodeset=self.get_xpath() + destination, **bind_attrs) - def xml_control(self): + def xml_control(self, survey: "Survey"): raise NotImplementedError() diff --git a/pyxform/external_instance.py b/pyxform/external_instance.py index 50ea4a701..50301ddb5 100644 --- a/pyxform/external_instance.py +++ b/pyxform/external_instance.py @@ -2,11 +2,31 @@ ExternalInstance class module """ -from pyxform.survey_element import SurveyElement +from typing import TYPE_CHECKING + +from pyxform import constants +from pyxform.survey_element import SURVEY_ELEMENT_FIELDS, SurveyElement + +if TYPE_CHECKING: + from pyxform.survey import Survey + + +EXTERNAL_INSTANCE_EXTRA_FIELDS = (constants.TYPE,) +EXTERNAL_INSTANCE_FIELDS = (*SURVEY_ELEMENT_FIELDS, *EXTERNAL_INSTANCE_EXTRA_FIELDS) class ExternalInstance(SurveyElement): - def xml_control(self): + __slots__ = EXTERNAL_INSTANCE_EXTRA_FIELDS + + @staticmethod + def get_slot_names() -> tuple[str, ...]: + return EXTERNAL_INSTANCE_FIELDS + + def __init__(self, name: str, type: str, **kwargs): + self.type: str = type + super().__init__(name=name, **kwargs) + + def xml_control(self, survey: "Survey"): """ No-op since there is no associated form control to place under . diff --git a/pyxform/instance.py b/pyxform/instance.py index 17b77f9f7..ae47fedbf 100644 --- a/pyxform/instance.py +++ b/pyxform/instance.py @@ -22,7 +22,7 @@ def __init__(self, survey_object, **kwargs): # get xpaths # - prep for xpaths. self._survey.xml() - self._xpaths = self._survey._xpath.values() + self._xpaths = [x.get_xpath() for x in self._survey._xpath.values()] # see "answers(self):" below for explanation of this dict self._answers = {} diff --git a/pyxform/parsing/expression.py b/pyxform/parsing/expression.py index 29df4fa1d..335864e61 100644 --- a/pyxform/parsing/expression.py +++ b/pyxform/parsing/expression.py @@ -1,10 +1,9 @@ import re from collections.abc import Iterable from functools import lru_cache -from typing import NamedTuple -def get_expression_lexer() -> re.Scanner: +def get_expression_lexer(name_only: bool = False) -> re.Scanner: """ Get a expression lexer (scanner) for parsing. """ @@ -62,7 +61,9 @@ def get_expression_lexer() -> re.Scanner: } def get_tokenizer(name): - def tokenizer(scan, value): + def tokenizer(scan, value) -> ExpLexerToken | str: + if name_only: + return name return ExpLexerToken(name, value, scan.match.start(), scan.match.end()) return tokenizer @@ -74,17 +75,21 @@ def tokenizer(scan, value): # Scanner takes a few 100ms to compile so use this shared instance. -class ExpLexerToken(NamedTuple): - name: str - value: str - start: int - end: int +class ExpLexerToken: + __slots__ = ("name", "value", "start", "end") + + def __init__(self, name: str, value: str, start: int, end: int) -> None: + self.name: str = name + self.value: str = value + self.start: int = start + self.end: int = end _EXPRESSION_LEXER = get_expression_lexer() +_TOKEN_NAME_LEXER = get_expression_lexer(name_only=True) -@lru_cache(maxsize=1024) +@lru_cache(maxsize=128) def parse_expression(text: str) -> tuple[list[ExpLexerToken], str]: """ Parse an expression. @@ -102,8 +107,10 @@ def is_single_token_expression(expression: str, token_types: Iterable[str]) -> b """ Does the expression contain single token of one of the provided token types? """ - tokens, _ = parse_expression(expression.strip()) - if 1 == len(tokens) and tokens[0].name in token_types: + if not expression: + return False + tokens, _ = _TOKEN_NAME_LEXER.scan(expression.strip()) + if 1 == len(tokens) and tokens[0] in token_types: return True else: return False @@ -113,6 +120,8 @@ def is_pyxform_reference(value: str) -> bool: """ Does the input string contain only a valid Pyxform reference? e.g. ${my_question} """ + if not value or len(value) <= 3: # Needs 3 characters for "${}", plus a name inside. + return False return is_single_token_expression(expression=value, token_types=("PYXFORM_REF",)) @@ -120,4 +129,6 @@ def is_xml_tag(value: str) -> bool: """ Does the input string contain only a valid XML tag / element name? """ + if not value: + return False return is_single_token_expression(expression=value, token_types=("NAME",)) diff --git a/pyxform/parsing/instance_expression.py b/pyxform/parsing/instance_expression.py index 4b3f82ed2..7ab5fbb2d 100644 --- a/pyxform/parsing/instance_expression.py +++ b/pyxform/parsing/instance_expression.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING -from pyxform.parsing.expression import ExpLexerToken, parse_expression +from pyxform.parsing.expression import parse_expression from pyxform.utils import BRACKETED_TAG_REGEX, node if TYPE_CHECKING: @@ -8,18 +8,6 @@ from pyxform.survey_element import SurveyElement -def instance_func_start(token: ExpLexerToken) -> bool: - """ - Determine if the token is the start of an instance expression. - - :param token: The token to examine. - :return: If True, the token is the start of an instance expression. - """ - if token is None: - return False - return token.name == "FUNC_CALL" and token.value == "instance(" - - def find_boundaries(xml_text: str) -> list[tuple[int, int]]: """ Find token boundaries of any instance() expression. @@ -43,14 +31,18 @@ def find_boundaries(xml_text: str) -> list[tuple[int, int]]: for t in tokens: emit = False # If an instance expression had started, note the string position boundary. - if instance_func_start(token=t) and not instance_enter: + if not instance_enter and t.name == "FUNC_CALL" and t.value == "instance(": instance_enter = True emit = True boundaries.append(t.start) # Tokens that are part of an instance expression. elif instance_enter: # Tokens that are part of the instance call. - if instance_func_start(token=last_token) and t.name == "SYSTEM_LITERAL": + if ( + t.name == "SYSTEM_LITERAL" + and last_token.name == "FUNC_CALL" + and last_token.value == "instance(" + ): emit = True elif last_token.name == "SYSTEM_LITERAL" and t.name == "CLOSE_PAREN": emit = True diff --git a/pyxform/question.py b/pyxform/question.py index 6c8f6a914..211626d85 100644 --- a/pyxform/question.py +++ b/pyxform/question.py @@ -3,7 +3,11 @@ """ import os.path +from collections.abc import Iterable +from itertools import chain +from typing import TYPE_CHECKING +from pyxform import constants from pyxform.constants import ( EXTERNAL_CHOICES_ITEMSET_REF_LABEL, EXTERNAL_CHOICES_ITEMSET_REF_LABEL_GEOJSON, @@ -13,24 +17,128 @@ ) from pyxform.errors import PyXFormError from pyxform.question_type_dictionary import QUESTION_TYPE_DICT -from pyxform.survey_element import SurveyElement +from pyxform.survey_element import SURVEY_ELEMENT_FIELDS, SurveyElement from pyxform.utils import ( PYXFORM_REFERENCE_REGEX, DetachableElement, + combine_lists, default_is_dynamic, node, ) +if TYPE_CHECKING: + from pyxform.survey import Survey + + +QUESTION_EXTRA_FIELDS = ( + "_itemset_dyn_label", + "_itemset_has_media", + "_itemset_multi_language", + "_qtd_defaults", + "_qtd_kwargs", + "action", + "default", + "guidance_hint", + "instance", + "query", + "sms_field", + "trigger", + constants.BIND, + constants.CHOICE_FILTER, + constants.COMPACT_TAG, # used for compact (sms) representation + constants.CONTROL, + constants.HINT, + constants.MEDIA, + constants.PARAMETERS, + constants.TYPE, +) +QUESTION_FIELDS = (*SURVEY_ELEMENT_FIELDS, *QUESTION_EXTRA_FIELDS) + +SELECT_QUESTION_EXTRA_FIELDS = ( + constants.CHILDREN, + constants.ITEMSET, + constants.LIST_NAME_U, +) +SELECT_QUESTION_FIELDS = (*QUESTION_FIELDS, *SELECT_QUESTION_EXTRA_FIELDS) + +OSM_QUESTION_EXTRA_FIELDS = (constants.CHILDREN,) +OSM_QUESTION_FIELDS = (*QUESTION_FIELDS, *SELECT_QUESTION_EXTRA_FIELDS) + +OPTION_EXTRA_FIELDS = ( + "_choice_itext_id", + constants.MEDIA, + "sms_option", +) +OPTION_FIELDS = (*SURVEY_ELEMENT_FIELDS, *OPTION_EXTRA_FIELDS) + +TAG_EXTRA_FIELDS = (constants.CHILDREN,) +TAG_FIELDS = (*SURVEY_ELEMENT_FIELDS, *TAG_EXTRA_FIELDS) + class Question(SurveyElement): - FIELDS = SurveyElement.FIELDS.copy() - FIELDS.update( - { - "_itemset_multi_language": bool, - "_itemset_has_media": bool, - "_itemset_dyn_label": bool, - } - ) + __slots__ = QUESTION_EXTRA_FIELDS + + @staticmethod + def get_slot_names() -> tuple[str, ...]: + return QUESTION_FIELDS + + def __init__(self, fields: tuple[str, ...] | None = None, **kwargs): + # Internals + self._qtd_defaults: dict | None = None + self._qtd_kwargs: dict | None = None + + # Structure + self.action: dict[str, str] | None = None + self.bind: dict | None = None + self.control: dict | None = None + self.instance: dict | None = None + self.media: dict | None = None + self.type: str | None = None + + # Common / template settings + self.choice_filter: str | None = None + self.default: str | None = None + self.guidance_hint: str | dict | None = None + self.hint: str | dict | None = None + # constraint_message, required_message are placed in bind dict. + self.parameters: dict | None = None + self.query: str | None = None + self.trigger: str | None = None + + # SMS / compact settings + self.compact_tag: str | None = None + self.sms_field: str | None = None + + qtd = kwargs.pop("question_type_dictionary", QUESTION_TYPE_DICT) + type_arg = kwargs.get("type") + default_type = qtd.get(type_arg) + if default_type is None: + raise PyXFormError(f"Unknown question type '{type_arg}'.") + + # Keeping original qtd_kwargs is only needed if output of QTD data is not + # acceptable in to_json_dict() i.e. to exclude default bind/control values. + self._qtd_defaults = qtd.get(type_arg) + qtd_kwargs = None + for k, v in self._qtd_defaults.items(): + if isinstance(v, dict): + template = v.copy() + if k in kwargs: + template.update(kwargs[k]) + if qtd_kwargs is None: + qtd_kwargs = {} + qtd_kwargs[k] = kwargs[k] + kwargs[k] = template + elif k not in kwargs: + kwargs[k] = v + + if qtd_kwargs: + self._qtd_kwargs = qtd_kwargs + + if fields is None: + fields = QUESTION_EXTRA_FIELDS + else: + fields = chain(QUESTION_EXTRA_FIELDS, fields) + super().__init__(fields=fields, **kwargs) def validate(self): SurveyElement.validate(self) @@ -40,21 +148,29 @@ def validate(self): if self.type not in QUESTION_TYPE_DICT: raise PyXFormError(f"Unknown question type '{self.type}'.") - def xml_instance(self, **kwargs): - survey = self.get_root() - attributes = {} - attributes.update(self.get("instance", {})) - for key, value in attributes.items(): - attributes[key] = survey.insert_xpaths(value, self) + def xml_instance(self, survey: "Survey", **kwargs): + attributes = self.get("instance") + if attributes is None: + attributes = {} + else: + for key, value in attributes.items(): + attributes[key] = survey.insert_xpaths(value, self) if self.get("default") and not default_is_dynamic(self.default, self.type): return node(self.name, str(self.get("default")), **attributes) return node(self.name, **attributes) - def xml_control(self): - survey = self.get_root() + def xml_control(self, survey: "Survey"): if self.type == "calculate" or ( - ("calculate" in self.bind or self.trigger) and not (self.label or self.hint) + ( + ( + hasattr(self, "bind") + and self.bind is not None + and "calculate" in self.bind + ) + or self.trigger + ) + and not (self.label or self.hint) ): nested_setvalues = survey.get_trigger_values_for_question_name( self.name, "setvalue" @@ -69,7 +185,7 @@ def xml_control(self): raise PyXFormError(msg) return None - xml_node = self.build_xml() + xml_node = self.build_xml(survey=survey) if xml_node: # Get nested setvalue and setgeopoint items @@ -90,6 +206,19 @@ def xml_control(self): return xml_node + def xml_action(self): + """ + Return the action for this survey element. + """ + if self.action: + return node( + self.action["name"], + ref=self.get_xpath(), + **{k: v for k, v in self.action.items() if k != "name"}, + ) + + return None + def nest_set_nodes(self, survey, xml_node, tag, nested_items): for item in nested_items: node_attrs = { @@ -101,9 +230,22 @@ def nest_set_nodes(self, survey, xml_node, tag, nested_items): set_node = node(tag, **node_attrs) xml_node.appendChild(set_node) - def build_xml(self) -> DetachableElement | None: + def build_xml(self, survey: "Survey") -> DetachableElement | None: return None + def to_json_dict(self, delete_keys: Iterable[str] | None = None) -> dict: + to_delete = (k for k in self.get_slot_names() if k.startswith("_")) + if self._qtd_defaults: + to_delete = chain(to_delete, self._qtd_defaults) + if delete_keys is not None: + to_delete = chain(to_delete, delete_keys) + result = super().to_json_dict(delete_keys=to_delete) + if self._qtd_kwargs: + for k, v in self._qtd_kwargs.items(): + if v: + result[k] = v + return result + class InputQuestion(Question): """ @@ -111,10 +253,9 @@ class InputQuestion(Question): dates, geopoints, barcodes ... """ - def build_xml(self): + def build_xml(self, survey: "Survey"): control_dict = self.control - label_and_hint = self.xml_label_and_hint() - survey = self.get_root() + label_and_hint = self.xml_label_and_hint(survey=survey) # Resolve field references in attributes for key, value in control_dict.items(): control_dict[key] = survey.insert_xpaths(value, self) @@ -122,54 +263,73 @@ def build_xml(self): result = node(**control_dict) if label_and_hint: - for element in self.xml_label_and_hint(): - result.appendChild(element) + for element in self.xml_label_and_hint(survey=survey): + if element: + result.appendChild(element) # Input types are used for selects with external choices sheets. if self["query"]: - choice_filter = self.get("choice_filter") - query = "instance('" + self["query"] + "')/root/item" - choice_filter = survey.insert_xpaths(choice_filter, self, True) - if choice_filter: - query += "[" + choice_filter + "]" + choice_filter = self.get(constants.CHOICE_FILTER) + if choice_filter is not None: + pred = survey.insert_xpaths(choice_filter, self, True) + query = f"""instance('{self["query"]}')/root/item[{pred}]""" + else: + query = f"""instance('{self["query"]}')/root/item""" result.setAttribute("query", query) return result class TriggerQuestion(Question): - def build_xml(self): + def build_xml(self, survey: "Survey"): control_dict = self.control - survey = self.get_root() # Resolve field references in attributes for key, value in control_dict.items(): control_dict[key] = survey.insert_xpaths(value, self) control_dict["ref"] = self.get_xpath() - return node("trigger", *self.xml_label_and_hint(), **control_dict) + return node("trigger", *self.xml_label_and_hint(survey=survey), **control_dict) class UploadQuestion(Question): def _get_media_type(self): return self.control["mediatype"] - def build_xml(self): + def build_xml(self, survey: "Survey"): control_dict = self.control - survey = self.get_root() # Resolve field references in attributes for key, value in control_dict.items(): control_dict[key] = survey.insert_xpaths(value, self) control_dict["ref"] = self.get_xpath() control_dict["mediatype"] = self._get_media_type() - return node("upload", *self.xml_label_and_hint(), **control_dict) + return node("upload", *self.xml_label_and_hint(survey=survey), **control_dict) class Option(SurveyElement): + __slots__ = OPTION_EXTRA_FIELDS + + @staticmethod + def get_slot_names() -> tuple[str, ...]: + return OPTION_FIELDS + + def __init__( + self, + name: str, + label: str | dict | None = None, + media: dict | None = None, + sms_option: str | None = None, + **kwargs, + ): + self._choice_itext_id: str | None = None + self.media: dict | None = media + self.sms_option: str | None = sms_option + + super().__init__(name=name, label=label, **kwargs) + def xml_value(self): return node("value", self.name) - def xml(self): + def xml(self, survey: "Survey"): item = node("item") - self.xml_label() - item.appendChild(self.xml_label()) + item.appendChild(self.xml_label(survey=survey)) item.appendChild(self.xml_value()) return item @@ -177,74 +337,89 @@ def xml(self): def validate(self): pass - def xml_control(self): + def xml_control(self, survey: "Survey"): raise NotImplementedError() def _translation_path(self, display_element): - choice_itext_id = self.get("_choice_itext_id") - if choice_itext_id is not None: - return choice_itext_id + if self._choice_itext_id is not None: + return self._choice_itext_id return super()._translation_path(display_element=display_element) + def to_json_dict(self, delete_keys: Iterable[str] | None = None) -> dict: + to_delete = (k for k in self.get_slot_names() if k.startswith("_")) + if delete_keys is not None: + to_delete = chain(to_delete, delete_keys) + return super().to_json_dict(delete_keys=to_delete) + class MultipleChoiceQuestion(Question): - def __init__(self, **kwargs): - kwargs_copy = kwargs.copy() + __slots__ = SELECT_QUESTION_EXTRA_FIELDS + + @staticmethod + def get_slot_names() -> tuple[str, ...]: + return SELECT_QUESTION_FIELDS + + def __init__( + self, itemset: str | None = None, list_name: str | None = None, **kwargs + ): + # Internals + self._itemset_dyn_label: bool = False + self._itemset_has_media: bool = False + self._itemset_multi_language: bool = False + + # Structure + self.children: tuple[Option, ...] | None = None + self.itemset: str | None = itemset + self.list_name: str | None = list_name + # Notice that choices can be specified under choices or children. # I'm going to try to stick to just choices. # Aliases in the json format will make it more difficult # to use going forward. - choices = list(kwargs_copy.pop("choices", [])) + list( - kwargs_copy.pop("children", []) + choices = combine_lists( + a=kwargs.pop(constants.CHOICES, None), b=kwargs.pop(constants.CHILDREN, None) ) - Question.__init__(self, **kwargs_copy) - for choice in choices: - self.add_choice(**choice) - - def add_choice(self, **kwargs): - option = Option(**kwargs) - self.add_child(option) + if choices: + self.children = tuple( + c if isinstance(c, Option) else Option(**c) for c in choices + ) + super().__init__(**kwargs) def validate(self): Question.validate(self) - descendants = self.iter_descendants() - next(descendants) # iter_descendants includes self; we need to pop it - - for choice in descendants: - choice.validate() + if self.children: + for child in self.children: + child.validate() - def build_xml(self): - if self.bind["type"] not in ["string", "odk:rank"]: + def build_xml(self, survey: "Survey"): + if self.bind["type"] not in {"string", "odk:rank"}: raise PyXFormError("""Invalid value for `self.bind["type"]`.""") - survey = self.get_root() - control_dict = self.control.copy() + # Resolve field references in attributes - for key, value in control_dict.items(): - control_dict[key] = survey.insert_xpaths(value, self) + control_dict = { + key: survey.insert_xpaths(value, self) for key, value in self.control.items() + } control_dict["ref"] = self.get_xpath() result = node(**control_dict) - for element in self.xml_label_and_hint(): - result.appendChild(element) + for element in self.xml_label_and_hint(survey=survey): + if element: + result.appendChild(element) # itemset are only supposed to be strings, # check to prevent the rare dicts that show up if self["itemset"] and isinstance(self["itemset"], str): - choice_filter = self.get("choice_filter") - itemset, file_extension = os.path.splitext(self["itemset"]) - itemset_value_ref = self.parameters.get( - "value", - EXTERNAL_CHOICES_ITEMSET_REF_VALUE_GEOJSON - if file_extension == ".geojson" - else EXTERNAL_CHOICES_ITEMSET_REF_VALUE, - ) - itemset_label_ref = self.parameters.get( - "label", - EXTERNAL_CHOICES_ITEMSET_REF_LABEL_GEOJSON - if file_extension == ".geojson" - else EXTERNAL_CHOICES_ITEMSET_REF_LABEL, - ) + + if file_extension == ".geojson": + itemset_value_ref = EXTERNAL_CHOICES_ITEMSET_REF_VALUE_GEOJSON + itemset_label_ref = EXTERNAL_CHOICES_ITEMSET_REF_LABEL_GEOJSON + else: + itemset_value_ref = EXTERNAL_CHOICES_ITEMSET_REF_VALUE + itemset_label_ref = EXTERNAL_CHOICES_ITEMSET_REF_LABEL + if hasattr(self, "parameters") and self.parameters is not None: + itemset_value_ref = self.parameters.get("value", itemset_value_ref) + itemset_label_ref = self.parameters.get("label", itemset_label_ref) multi_language = self.get("_itemset_multi_language", False) has_media = self.get("_itemset_has_media", False) @@ -261,9 +436,11 @@ def build_xml(self): itemset = self["itemset"] itemset_label_ref = "jr:itext(itextId)" - choice_filter = survey.insert_xpaths( - choice_filter, self, True, is_previous_question - ) + choice_filter = self.get(constants.CHOICE_FILTER) + if choice_filter is not None: + choice_filter = survey.insert_xpaths( + choice_filter, self, True, is_previous_question + ) if is_previous_question: path = ( survey.insert_xpaths(self["itemset"], self, reference_parent=True) @@ -283,10 +460,10 @@ def build_xml(self): name = path[-1] choice_filter = f"./{name} != ''" else: - nodeset = "instance('" + itemset + "')/root/item" + nodeset = f"instance('{itemset}')/root/item" if choice_filter: - nodeset += "[" + choice_filter + "]" + nodeset += f"[{choice_filter}]" if self["parameters"]: params = self["parameters"] @@ -311,83 +488,83 @@ def build_xml(self): node("label", ref=itemset_label_ref), ] result.appendChild(node("itemset", *itemset_children, nodeset=nodeset)) - else: + elif self.children: for child in self.children: - result.appendChild(child.xml()) + result.appendChild(child.xml(survey=survey)) return result -class SelectOneQuestion(MultipleChoiceQuestion): - def __init__(self, **kwargs): - super().__init__(**kwargs) - self._dict[self.TYPE] = "select one" - - class Tag(SurveyElement): - def __init__(self, **kwargs): - kwargs_copy = kwargs.copy() - choices = kwargs_copy.pop("choices", []) + kwargs_copy.pop("children", []) + __slots__ = TAG_EXTRA_FIELDS - super().__init__(**kwargs_copy) + @staticmethod + def get_slot_names() -> tuple[str, ...]: + return TAG_FIELDS - if choices: - self.children = [] + def __init__(self, name: str, label: str | dict | None = None, **kwargs): + self.children: tuple[Option, ...] | None = None - for choice in choices: - option = Option(**choice) - self.add_child(option) + choices = combine_lists( + a=kwargs.pop(constants.CHOICES, None), b=kwargs.pop(constants.CHILDREN, None) + ) + if choices: + self.children = tuple( + c if isinstance(c, Option) else Option(**c) for c in choices + ) + super().__init__(name=name, label=label, **kwargs) - def xml(self): + def xml(self, survey: "Survey"): result = node("tag", key=self.name) - self.xml_label() - result.appendChild(self.xml_label()) - for choice in self.children: - result.appendChild(choice.xml()) + result.appendChild(self.xml_label(survey=survey)) + if self.children: + for choice in self.children: + result.appendChild(choice.xml(survey=survey)) return result def validate(self): pass - def xml_control(self): + def xml_control(self, survey: "Survey"): raise NotImplementedError() class OsmUploadQuestion(UploadQuestion): - def __init__(self, **kwargs): - kwargs_copy = kwargs.copy() - tags = kwargs_copy.pop("tags", []) + kwargs_copy.pop("children", []) + __slots__ = OSM_QUESTION_EXTRA_FIELDS - super().__init__(**kwargs_copy) + @staticmethod + def get_slot_names() -> tuple[str, ...]: + return OSM_QUESTION_FIELDS - if tags: - self.children = [] + def __init__(self, **kwargs): + self.children: tuple[Option, ...] | None = None - for tag in tags: - self.add_tag(**tag) + choices = combine_lists( + a=kwargs.pop("tags", None), b=kwargs.pop(constants.CHILDREN, None) + ) + if choices: + self.children = tuple(Tag(**c) for c in choices) - def add_tag(self, **kwargs): - tag = Tag(**kwargs) - self.add_child(tag) + super().__init__(**kwargs) - def build_xml(self): + def build_xml(self, survey: "Survey"): control_dict = self.control control_dict["ref"] = self.get_xpath() control_dict["mediatype"] = self._get_media_type() - result = node("upload", *self.xml_label_and_hint(), **control_dict) + result = node("upload", *self.xml_label_and_hint(survey=survey), **control_dict) - for osm_tag in self.children: - result.appendChild(osm_tag.xml()) + if self.children: + for osm_tag in self.children: + result.appendChild(osm_tag.xml(survey=survey)) return result class RangeQuestion(Question): - def build_xml(self): + def build_xml(self, survey: "Survey"): control_dict = self.control - label_and_hint = self.xml_label_and_hint() - survey = self.get_root() + label_and_hint = self.xml_label_and_hint(survey=survey) # Resolve field references in attributes for key, value in control_dict.items(): control_dict[key] = survey.insert_xpaths(value, self) @@ -396,7 +573,7 @@ def build_xml(self): control_dict.update(params) result = node(**control_dict) if label_and_hint: - for element in self.xml_label_and_hint(): + for element in self.xml_label_and_hint(survey=survey): result.appendChild(element) return result diff --git a/pyxform/question_type_dictionary.py b/pyxform/question_type_dictionary.py index b22e0b7d6..0d5e0857d 100644 --- a/pyxform/question_type_dictionary.py +++ b/pyxform/question_type_dictionary.py @@ -2,6 +2,8 @@ XForm survey question type mapping dictionary module. """ +from types import MappingProxyType + from pyxform.xls2json import QuestionTypesReader, print_pyobj_to_json @@ -16,7 +18,7 @@ def generate_new_dict(): print_pyobj_to_json(json_dict, "new_question_type_dict.json") -QUESTION_TYPE_DICT = { +_QUESTION_TYPE_DICT = { "q picture": { "control": {"tag": "upload", "mediatype": "image/*"}, "bind": {"type": "binary"}, @@ -387,3 +389,6 @@ def generate_new_dict(): "bind": {"type": "geopoint"}, }, } + +# Read-only view of the types. +QUESTION_TYPE_DICT = MappingProxyType(_QUESTION_TYPE_DICT) diff --git a/pyxform/section.py b/pyxform/section.py index a52ea9e9f..2111980ca 100644 --- a/pyxform/section.py +++ b/pyxform/section.py @@ -2,13 +2,76 @@ Section survey element module. """ +from collections.abc import Generator, Iterable +from itertools import chain +from typing import TYPE_CHECKING + +from pyxform import constants from pyxform.errors import PyXFormError from pyxform.external_instance import ExternalInstance -from pyxform.survey_element import SurveyElement -from pyxform.utils import node +from pyxform.survey_element import SURVEY_ELEMENT_FIELDS, SurveyElement +from pyxform.utils import DetachableElement, node + +if TYPE_CHECKING: + from pyxform.question import Question + from pyxform.survey import Survey + + +SECTION_EXTRA_FIELDS = ( + constants.BIND, + constants.CHILDREN, + constants.CONTROL, + constants.HINT, + constants.MEDIA, + constants.TYPE, + "instance", + "flat", + "sms_field", +) +SECTION_FIELDS = (*SURVEY_ELEMENT_FIELDS, *SECTION_EXTRA_FIELDS) class Section(SurveyElement): + __slots__ = SECTION_EXTRA_FIELDS + + @staticmethod + def get_slot_names() -> tuple[str, ...]: + return SECTION_FIELDS + + def __init__( + self, + name: str, + type: str, + label: str | dict | None = None, + hint: str | dict | None = None, + bind: dict | None = None, + control: dict | None = None, + instance: dict | None = None, + media: dict | None = None, + flat: bool | None = None, + sms_field: str | None = None, + fields: tuple[str, ...] | None = None, + **kwargs, + ): + # Structure + self.bind: dict | None = bind + self.children: list[Section | Question] | None = None + self.control: dict | None = control + # instance is for custom instance attrs from survey e.g. instance::abc:xyz + self.instance: dict | None = instance + # TODO: is media valid for groups? No tests for it, but it behaves like Questions. + self.media: dict | None = media + self.type: str | None = type + + # Group settings are generally put in bind/control dicts. + self.hint: str | dict | None = hint + self.flat: bool | None = flat + self.sms_field: str | None = sms_field + + # Recursively creating child objects currently handled by the builder module. + kwargs.pop(constants.CHILDREN, None) + super().__init__(name=name, label=label, fields=fields, **kwargs) + def validate(self): super().validate() for element in self.children: @@ -28,7 +91,7 @@ def _validate_uniqueness_of_element_names(self): ) element_slugs.add(elem_lower) - def xml_instance(self, **kwargs): + def xml_instance(self, survey: "Survey", **kwargs): """ Creates an xml representation of the section """ @@ -36,8 +99,8 @@ def xml_instance(self, **kwargs): attributes = {} attributes.update(kwargs) - attributes.update(self.get("instance", {})) - survey = self.get_root() + if self.instance: + attributes.update(self.instance) # Resolve field references in attributes for key, value in attributes.items(): attributes[key] = survey.insert_xpaths(value, self) @@ -45,54 +108,56 @@ def xml_instance(self, **kwargs): for child in self.children: repeating_template = None - if child.get("flat"): - for grandchild in child.xml_instance_array(): + if hasattr(child, "flat") and child.get("flat"): + for grandchild in child.xml_instance_array(survey=survey): result.appendChild(grandchild) elif isinstance(child, ExternalInstance): continue else: if isinstance(child, RepeatingSection) and not append_template: append_template = not append_template - repeating_template = child.generate_repeating_template() - result.appendChild(child.xml_instance(append_template=append_template)) + repeating_template = child.generate_repeating_template(survey=survey) + result.appendChild( + child.xml_instance(survey=survey, append_template=append_template) + ) if append_template and repeating_template: append_template = not append_template result.insertBefore(repeating_template, result._get_lastChild()) return result - def generate_repeating_template(self, **kwargs): + def generate_repeating_template(self, survey: "Survey", **kwargs): attributes = {"jr:template": ""} result = node(self.name, **attributes) for child in self.children: if isinstance(child, RepeatingSection): - result.appendChild(child.template_instance()) + result.appendChild(child.template_instance(survey=survey)) else: - result.appendChild(child.xml_instance()) + result.appendChild(child.xml_instance(survey=survey)) return result - def xml_instance_array(self): + def xml_instance_array(self, survey: "Survey"): """ This method is used for generating flat instances. """ for child in self.children: - if child.get("flat"): - yield from child.xml_instance_array() + if hasattr(child, "flat") and child.get("flat"): + yield from child.xml_instance_array(survey=survey) else: - yield child.xml_instance() + yield child.xml_instance(survey=survey) - def xml_control(self): + def xml_control(self, survey: "Survey"): """ Ideally, we'll have groups up and rolling soon, but for now let's just yield controls from all the children of this section """ for e in self.children: - control = e.xml_control() + control = e.xml_control(survey=survey) if control is not None: yield control class RepeatingSection(Section): - def xml_control(self): + def xml_control(self, survey: "Survey"): """ @@ -106,44 +171,49 @@ def xml_control(self): """ - control_dict = self.control.copy() - survey = self.get_root() # Resolve field references in attributes - for key, value in control_dict.items(): - control_dict[key] = survey.insert_xpaths(value, self) - repeat_node = node("repeat", nodeset=self.get_xpath(), **control_dict) - - for n in Section.xml_control(self): + if self.control: + control_dict = { + key: survey.insert_xpaths(value, self) + for key, value in self.control.items() + } + repeat_node = node("repeat", nodeset=self.get_xpath(), **control_dict) + else: + repeat_node = node("repeat", nodeset=self.get_xpath()) + + for n in Section.xml_control(self, survey=survey): repeat_node.appendChild(n) - setvalue_nodes = self._get_setvalue_nodes_for_dynamic_defaults() - - for setvalue_node in setvalue_nodes: + for setvalue_node in self._dynamic_defaults_helper(current=self, survey=survey): repeat_node.appendChild(setvalue_node) - label = self.xml_label() + label = self.xml_label(survey=survey) if label: - return node("group", self.xml_label(), repeat_node, ref=self.get_xpath()) - return node("group", repeat_node, ref=self.get_xpath(), **self.control) + return node("group", label, repeat_node, ref=self.get_xpath()) + if self.control: + return node("group", repeat_node, ref=self.get_xpath(), **self.control) + else: + return node("group", repeat_node, ref=self.get_xpath()) # Get setvalue nodes for all descendants of this repeat that have dynamic defaults and aren't nested in other repeats. - def _get_setvalue_nodes_for_dynamic_defaults(self): - setvalue_nodes = [] - self._dynamic_defaults_helper(self, setvalue_nodes) - return setvalue_nodes - - def _dynamic_defaults_helper(self, current, nodes): + def _dynamic_defaults_helper( + self, current: "Section", survey: "Survey" + ) -> Generator[DetachableElement, None, None]: + if not isinstance(current, Section): + return for e in current.children: if e.type != "repeat": # let nested repeats handle their own defaults - dynamic_default = e.get_setvalue_node_for_dynamic_default(in_repeat=True) + dynamic_default = e.get_setvalue_node_for_dynamic_default( + in_repeat=True, survey=survey + ) if dynamic_default: - nodes.append(dynamic_default) - self._dynamic_defaults_helper(e, nodes) + yield dynamic_default + yield from self._dynamic_defaults_helper(current=e, survey=survey) # I'm anal about matching function signatures when overriding a function, # but there's no reason for kwargs to be an argument - def template_instance(self, **kwargs): - return super().generate_repeating_template(**kwargs) + def template_instance(self, survey: "Survey", **kwargs): + return super().generate_repeating_template(survey=survey, **kwargs) class GroupedSection(Section): @@ -159,42 +229,39 @@ class GroupedSection(Section): # kwargs["children"].insert(0, kwargs["children"][0]) # super(GroupedSection, self).__init__(kwargs) - def xml_control(self): - control_dict = self.control - - if control_dict.get("bodyless"): + def xml_control(self, survey: "Survey"): + if self.control and self.control.get("bodyless"): return None children = [] - attributes = {} - attributes.update(self.control) - - survey = self.get_root() # Resolve field references in attributes - for key, value in attributes.items(): - attributes[key] = survey.insert_xpaths(value, self) + if self.control: + attributes = { + key: survey.insert_xpaths(value, self) + for key, value in self.control.items() + } + if "appearance" in self.control: + attributes["appearance"] = self.control["appearance"] + else: + attributes = {} if not self.get("flat"): attributes["ref"] = self.get_xpath() - if "label" in self and len(self["label"]) > 0: - children.append(self.xml_label()) - for n in Section.xml_control(self): + if "label" in self and self.label is not None and len(self["label"]) > 0: + children.append(self.xml_label(survey=survey)) + for n in Section.xml_control(self, survey=survey): children.append(n) - if "appearance" in control_dict: - attributes["appearance"] = control_dict["appearance"] - - if "intent" in control_dict: - survey = self.get_root() - attributes["intent"] = survey.insert_xpaths(control_dict["intent"], self) - return node("group", *children, **attributes) - def to_json_dict(self): + def to_json_dict(self, delete_keys: Iterable[str] | None = None) -> dict: + to_delete = (constants.BIND,) + if delete_keys is not None: + to_delete = chain(to_delete, delete_keys) + result = super().to_json_dict(delete_keys=to_delete) # This is quite hacky, might want to think about a smart way # to approach this problem. - result = super().to_json_dict() result["type"] = "group" return result diff --git a/pyxform/survey.py b/pyxform/survey.py index 2b6e21eda..a45ca14cc 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -7,31 +7,33 @@ import tempfile import xml.etree.ElementTree as ETree from collections import defaultdict -from collections.abc import Generator, Iterator +from collections.abc import Generator, Iterable from datetime import datetime from functools import lru_cache +from itertools import chain from pathlib import Path from pyxform import aliases, constants from pyxform.constants import EXTERNAL_INSTANCE_EXTENSIONS, NSMAP +from pyxform.entities.entity_declaration import EntityDeclaration from pyxform.errors import PyXFormError, ValidationError from pyxform.external_instance import ExternalInstance from pyxform.instance import SurveyInstance from pyxform.parsing import instance_expression -from pyxform.question import Option, Question -from pyxform.section import Section -from pyxform.survey_element import SurveyElement +from pyxform.question import MultipleChoiceQuestion, Option, Question, Tag +from pyxform.section import SECTION_EXTRA_FIELDS, Section +from pyxform.survey_element import SURVEY_ELEMENT_FIELDS, SurveyElement from pyxform.utils import ( BRACKETED_TAG_REGEX, LAST_SAVED_INSTANCE_NAME, LAST_SAVED_REGEX, DetachableElement, PatchedText, - get_languages_with_bad_tags, has_dynamic_label, node, ) from pyxform.validators import enketo_validate, odk_validate +from pyxform.validators.pyxform.iana_subtags.validation import get_languages_with_bad_tags RE_BRACKET = re.compile(r"\[([^]]+)\]") RE_FUNCTION_ARGS = re.compile(r"\b[^()]+\((.*)\)$") @@ -74,7 +76,7 @@ def register_nsmap(): register_nsmap() -@lru_cache(maxsize=65536) # 2^16 +@lru_cache(maxsize=128) def is_parent_a_repeat(survey, xpath): """ Returns the XPATH of the first repeat of the given xpath in the survey, @@ -84,13 +86,14 @@ def is_parent_a_repeat(survey, xpath): if not parent_xpath: return False - if survey.any_repeat(parent_xpath): - return parent_xpath + for item in survey.iter_descendants(condition=lambda i: isinstance(i, Section)): + if item.type == constants.REPEAT and item.get_xpath() == parent_xpath: + return parent_xpath return is_parent_a_repeat(survey, parent_xpath) -@lru_cache(maxsize=65536) # 2^16 +@lru_cache(maxsize=128) def share_same_repeat_parent(survey, xpath, context_xpath, reference_parent=False): """ Returns a tuple of the number of steps from the context xpath to the shared @@ -98,7 +101,7 @@ def share_same_repeat_parent(survey, xpath, context_xpath, reference_parent=Fals parent. For example, - xpath = /data/repeat_a/group_a/name + xpath = /data/repeat_a/group_a/name context_xpath = /data/repeat_a/group_b/age returns (2, '/group_a/name')' @@ -168,7 +171,7 @@ def _get_steps_and_target_xpath(context_parent, xpath_parent, include_parent=Fal return (None, None) -@lru_cache(maxsize=65536) # 2^16 +@lru_cache(maxsize=128) def is_label_dynamic(label: str) -> bool: return ( label is not None @@ -181,44 +184,117 @@ def recursive_dict(): return defaultdict(recursive_dict) +SURVEY_EXTRA_FIELDS = ( + "_created", + "_search_lists", + "_translations", + "_xpath", + "add_none_option", + "clean_text_values", + "attribute", + "auto_delete", + "auto_send", + "choices", + "default_language", + "file_name", + "id_string", + "instance_name", + "instance_xmlns", + "namespaces", + "omit_instanceID", + "public_key", + "setgeopoint_by_triggering_ref", + "setvalues_by_triggering_ref", + "sms_allow_media", + "sms_date_format", + "sms_datetime_format", + "sms_keyword", + "sms_response", + "sms_separator", + "style", + "submission_url", + "title", + "version", + constants.ALLOW_CHOICE_DUPLICATES, + constants.COMPACT_DELIMITER, + constants.COMPACT_PREFIX, + constants.ENTITY_FEATURES, +) +SURVEY_FIELDS = (*SURVEY_ELEMENT_FIELDS, *SECTION_EXTRA_FIELDS, *SURVEY_EXTRA_FIELDS) + + class Survey(Section): """ Survey class - represents the full XForm XML. """ - FIELDS = Section.FIELDS.copy() - FIELDS.update( - { - "_xpath": dict, - "_created": datetime.now, # This can't be dumped to json - "setvalues_by_triggering_ref": dict, - "setgeopoint_by_triggering_ref": dict, - "title": str, - "id_string": str, - "sms_keyword": str, - "sms_separator": str, - "sms_allow_media": bool, - "sms_date_format": str, - "sms_datetime_format": str, - "sms_response": str, - constants.COMPACT_PREFIX: str, - constants.COMPACT_DELIMITER: str, - "file_name": str, - "default_language": str, - "_translations": recursive_dict, - "submission_url": str, - "auto_send": str, - "auto_delete": str, - "public_key": str, - "instance_xmlns": str, - "version": str, - "choices": dict, - "style": str, - "attribute": dict, - "namespaces": str, - constants.ENTITY_FEATURES: list, - } - ) + __slots__ = SURVEY_EXTRA_FIELDS + + @staticmethod + def get_slot_names() -> tuple[str, ...]: + return SURVEY_FIELDS + + def __init__(self, **kwargs): + # Internals + self._created: datetime.now = datetime.now() + self._search_lists: set = set() + self._translations: recursive_dict = recursive_dict() + self._xpath: dict[str, SurveyElement | None] = {} + + # Structure + # attribute is for custom instance attrs from settings e.g. attribute::abc:xyz + self.attribute: dict | None = None + self.choices: dict[str, tuple[Option, ...]] | None = None + self.entity_features: list[str] | None = None + self.setgeopoint_by_triggering_ref: dict[str, list[str]] = {} + self.setvalues_by_triggering_ref: dict[str, list[str]] = {} + + # Common / template settings + self.default_language: str = "" + self.id_string: str = "" + self.instance_name: str = "" + self.style: str | None = None + self.title: str = "" + self.version: str = "" + + # Other settings + self.add_none_option: bool = False + self.allow_choice_duplicates: bool = False + self.auto_delete: str | None = None + self.auto_send: str | None = None + self.clean_text_values: bool = False + self.instance_xmlns: str | None = None + self.namespaces: str | None = None + self.omit_instanceID: bool = False + self.public_key: str | None = None + self.submission_url: str | None = None + + # SMS / compact settings + self.delimiter: str | None = None + self.prefix: str | None = None + self.sms_allow_media: bool | None = None + self.sms_date_format: str | None = None + self.sms_datetime_format: str | None = None + self.sms_keyword: str | None = None + self.sms_response: str | None = None + self.sms_separator: str | None = None + + choices = kwargs.pop("choices", None) + if choices is not None: + self.choices = { + list_name: tuple( + c if isinstance(c, Option) else Option(**c) for c in values + ) + for list_name, values in choices.items() + } + kwargs[constants.TYPE] = constants.SURVEY + super().__init__(fields=SURVEY_EXTRA_FIELDS, **kwargs) + + def to_json_dict(self, delete_keys: Iterable[str] | None = None) -> dict: + to_delete = (k for k in self.get_slot_names() if k.startswith("_")) + if delete_keys is not None: + to_delete = chain(to_delete, delete_keys) + return super().to_json_dict(delete_keys=to_delete) def validate(self): if self.id_string in [None, "None"]: @@ -229,32 +305,34 @@ def validate(self): def _validate_uniqueness_of_section_names(self): root_node_name = self.name section_names = set() - for element in self.iter_descendants(): - if isinstance(element, Section): - if element.name in section_names: - if element.name == root_node_name: - # The root node name is rarely explictly set; explain - # the problem in a more helpful way (#510) - msg = ( - f"The name '{element.name}' is the same as the form name. " - "Use a different section name (or change the form name in " - "the 'name' column of the settings sheet)." - ) - raise PyXFormError(msg) - msg = f"There are two sections with the name {element.name}." + for element in self.iter_descendants(condition=lambda i: isinstance(i, Section)): + if element.name in section_names: + if element.name == root_node_name: + # The root node name is rarely explictly set; explain + # the problem in a more helpful way (#510) + msg = ( + f"The name '{element.name}' is the same as the form name. " + "Use a different section name (or change the form name in " + "the 'name' column of the settings sheet)." + ) raise PyXFormError(msg) - section_names.add(element.name) + msg = f"There are two sections with the name {element.name}." + raise PyXFormError(msg) + section_names.add(element.name) def get_nsmap(self): """Add additional namespaces""" - namespaces = getattr(self, constants.NAMESPACES, "") - if len(getattr(self, constants.ENTITY_FEATURES, [])) > 0: - namespaces += " entities=http://www.opendatakit.org/xforms/entities" + if self.entity_features: + entities_ns = " entities=http://www.opendatakit.org/xforms/entities" + if self.namespaces is None: + self.namespaces = entities_ns + else: + self.namespaces += entities_ns - if namespaces and isinstance(namespaces, str): + if self.namespaces: nslist = [ ns.split("=") - for ns in namespaces.split() + for ns in self.namespaces.split() if len(ns.split("=")) == 2 and ns.split("=")[0] != "" ] xmlns = "xmlns:" @@ -287,24 +365,22 @@ def xml(self): self.insert_xpaths(triggering_reference, self) body_kwargs = {} - if hasattr(self, constants.STYLE) and getattr(self, constants.STYLE): - body_kwargs["class"] = getattr(self, constants.STYLE) + if self.style: + body_kwargs["class"] = self.style nsmap = self.get_nsmap() return node( "h:html", node("h:head", node("h:title", self.title), self.xml_model()), - node("h:body", *self.xml_control(), **body_kwargs), + node("h:body", *self.xml_control(survey=self), **body_kwargs), **nsmap, ) - def get_trigger_values_for_question_name(self, question_name, trigger_type): - trigger_map = { - "setvalue": self.setvalues_by_triggering_ref, - "setgeopoint": self.setgeopoint_by_triggering_ref, - } - - return trigger_map.get(trigger_type, {}).get(f"${{{question_name}}}") + def get_trigger_values_for_question_name(self, question_name: str, trigger_type: str): + if trigger_type == "setvalue": + return self.setvalues_by_triggering_ref.get(f"${{{question_name}}}") + elif trigger_type == "setgeopoint": + return self.setgeopoint_by_triggering_ref.get(f"${{{question_name}}}") def _generate_static_instances(self, list_name, choice_list) -> InstanceInfo: """ @@ -315,34 +391,42 @@ def _generate_static_instances(self, list_name, choice_list) -> InstanceInfo: has_dyn_label = has_dynamic_label(choice_list) multi_language = False if isinstance(self._translations, dict): - choices = tuple( - k + choices = ( + True for items in self._translations.values() for k, v in items.items() if v.get(constants.TYPE, "") == constants.CHOICE and "-".join(k.split("-")[:-1]) == list_name ) - if 0 < len(choices): - multi_language = True + try: + if next(choices): + multi_language = True + except StopIteration: + pass for idx, choice in enumerate(choice_list): choice_element_list = [] # Add a unique id to the choice element in case there are itext references if multi_language or has_media or has_dyn_label: - itext_id = "-".join([list_name, str(idx)]) + itext_id = f"{list_name}-{idx}" choice_element_list.append(node("itextId", itext_id)) for name, value in choice.items(): - if isinstance(value, str) and name != "label": - choice_element_list.append(node(name, str(value))) - if ( + if not value: + continue + elif name != "label" and isinstance(value, str): + choice_element_list.append(node(name, value)) + elif name == "extra_data" and isinstance(value, dict): + for k, v in value.items(): + choice_element_list.append(node(k, v)) + elif ( not multi_language and not has_media and not has_dyn_label and isinstance(value, str) and name == "label" ): - choice_element_list.append(node(name, str(value))) + choice_element_list.append(node(name, value)) instance_element_list.append(node("item", *choice_element_list)) @@ -355,7 +439,7 @@ def _generate_static_instances(self, list_name, choice_list) -> InstanceInfo: ) @staticmethod - def _generate_external_instances(element) -> InstanceInfo | None: + def _generate_external_instances(element: SurveyElement) -> InstanceInfo | None: if isinstance(element, ExternalInstance): name = element["name"] extension = element["type"].split("-")[0] @@ -401,7 +485,7 @@ def _validate_external_instances(instances) -> None: raise ValidationError("\n".join(errors)) @staticmethod - def _generate_pulldata_instances(element) -> list[InstanceInfo] | None: + def _generate_pulldata_instances(element: SurveyElement) -> list[InstanceInfo] | None: def get_pulldata_functions(element): """ Returns a list of different pulldata(... function strings if @@ -412,11 +496,23 @@ def get_pulldata_functions(element): """ functions_present = [] for formula_name in constants.EXTERNAL_INSTANCES: - if "pulldata(" in str(element["bind"].get(formula_name)): + if ( + hasattr(element, "bind") + and element.bind is not None + and "pulldata(" in str(element["bind"].get(formula_name)) + ): functions_present.append(element["bind"][formula_name]) - if "pulldata(" in str(element["choice_filter"]): - functions_present.append(element["choice_filter"]) - if "pulldata(" in str(element["default"]): + if ( + hasattr(element, constants.CHOICE_FILTER) + and element.choice_filter is not None + and "pulldata(" in str(element[constants.CHOICE_FILTER]) + ): + functions_present.append(element[constants.CHOICE_FILTER]) + if ( + hasattr(element, "default") + and element.default is not None + and "pulldata(" in str(element["default"]) + ): functions_present.append(element["default"]) return functions_present @@ -434,6 +530,8 @@ def get_instance_info(element, file_id): instance=node("instance", id=file_id, src=uri), ) + if isinstance(element, Option | ExternalInstance | Tag | Survey): + return None pulldata_usages = get_pulldata_functions(element) if len(pulldata_usages) > 0: pulldata_instances = [] @@ -451,7 +549,9 @@ def get_instance_info(element, file_id): return None @staticmethod - def _generate_from_file_instances(element) -> InstanceInfo | None: + def _generate_from_file_instances(element: SurveyElement) -> InstanceInfo | None: + if not isinstance(element, MultipleChoiceQuestion) or element.itemset is None: + return None itemset = element.get("itemset") file_id, ext = os.path.splitext(itemset) if itemset and ext in EXTERNAL_INSTANCE_EXTENSIONS: @@ -474,16 +574,21 @@ def _generate_last_saved_instance(element) -> bool: """ True if a last-saved instance should be generated, false otherwise. """ + if not hasattr(element, "bind") or element.bind is None: + return False for expression_type in constants.EXTERNAL_INSTANCES: last_saved_expression = re.search( LAST_SAVED_REGEX, str(element["bind"].get(expression_type)) ) if last_saved_expression: return True - return bool( - re.search(LAST_SAVED_REGEX, str(element["choice_filter"])) - or re.search(LAST_SAVED_REGEX, str(element["default"])) + hasattr(element, constants.CHOICE_FILTER) + and element.choice_filter is not None + and re.search(LAST_SAVED_REGEX, str(element.choice_filter)) + or hasattr(element, "default") + and element.default is not None + and re.search(LAST_SAVED_REGEX, str(element.default)) ) @staticmethod @@ -499,7 +604,7 @@ def _get_last_saved_instance() -> InstanceInfo: instance=node("instance", id=name, src=uri), ) - def _generate_instances(self) -> Iterator[DetachableElement]: + def _generate_instances(self) -> Generator[DetachableElement, None, None]: """ Get instances from all the different ways that they may be generated. @@ -550,10 +655,12 @@ def _generate_instances(self) -> Iterator[DetachableElement]: instances += [self._get_last_saved_instance()] # Append last so the choice instance is excluded on a name clash. - for name, value in self.choices.items(): - instances += [ - self._generate_static_instances(list_name=name, choice_list=value) - ] + if self.choices: + for name, value in self.choices.items(): + if name not in self._search_lists: + instances += [ + self._generate_static_instances(list_name=name, choice_list=value) + ] # Check that external instances have unique names. if instances: @@ -580,6 +687,32 @@ def _generate_instances(self) -> Iterator[DetachableElement]: yield i.instance seen[i.name] = i + def xml_descendent_bindings(self) -> Generator[DetachableElement | None, None, None]: + """ + Yield bindings for this node and all its descendants. + """ + for e in self.iter_descendants( + condition=lambda i: not isinstance(i, Option | Tag) + ): + yield from e.xml_bindings(survey=self) + + # dynamic defaults for repeats go in the body. All other dynamic defaults (setvalue actions) go in the model + if not next( + e.iter_ancestors(condition=lambda i: i.type == constants.REPEAT), False + ): + dynamic_default = e.get_setvalue_node_for_dynamic_default(survey=self) + if dynamic_default: + yield dynamic_default + + def xml_actions(self) -> Generator[DetachableElement, None, None]: + """ + Yield xml_actions for this node and all its descendants. + """ + for e in self.iter_descendants(condition=lambda i: isinstance(i, Question)): + xml_action = e.xml_action() + if xml_action is not None: + yield xml_action + def xml_model(self): """ Generate the xform element @@ -590,13 +723,12 @@ def xml_model(self): model_kwargs = {"odk:xforms-version": constants.CURRENT_XFORMS_VERSION} - entity_features = getattr(self, constants.ENTITY_FEATURES, []) - if len(entity_features) > 0: - if "offline" in entity_features: + if self.entity_features: + if "offline" in self.entity_features: model_kwargs["entities:entities-version"] = ( constants.ENTITIES_OFFLINE_VERSION ) - elif "update" in entity_features: + elif "update" in self.entity_features: model_kwargs["entities:entities-version"] = ( constants.ENTITIES_UPDATE_VERSION ) @@ -608,10 +740,9 @@ def xml_model(self): model_children = [] if self._translations: model_children.append(self.itext()) - model_children += [node("instance", self.xml_instance())] - model_children += list(self._generate_instances()) - model_children += self.xml_descendent_bindings() - model_children += self.xml_actions() + model_children.append( + node("instance", self.xml_instance()), + ) if self.submission_url or self.public_key or self.auto_send or self.auto_delete: submission_attrs = {} @@ -627,14 +758,21 @@ def xml_model(self): submission_node = node("submission", **submission_attrs) model_children.insert(0, submission_node) - return node("model", *model_children, **model_kwargs) + def model_children_generator(): + yield from model_children + yield from self._generate_instances() + yield from self.xml_descendent_bindings() + yield from self.xml_actions() + + return node("model", model_children_generator(), **model_kwargs) def xml_instance(self, **kwargs): - result = Section.xml_instance(self, **kwargs) + result = Section.xml_instance(self, survey=self, **kwargs) # set these first to prevent overwriting id and version - for key, value in self.attribute.items(): - result.setAttribute(str(key), value) + if self.attribute: + for key, value in self.attribute.items(): + result.setAttribute(str(key), value) result.setAttribute("id", self.id_string) @@ -665,7 +803,7 @@ def _add_to_nested_dict(self, dicty, path, value): dicty[path[0]] = {} self._add_to_nested_dict(dicty[path[0]], path[1:], value) - def _redirect_is_search_itext(self, element: SurveyElement) -> bool: + def _redirect_is_search_itext(self, element: Question) -> bool: """ For selects using the "search()" function, redirect itext for in-line items. @@ -697,11 +835,12 @@ def _redirect_is_search_itext(self, element: SurveyElement) -> bool: "Remove the 'search()' usage, or change the select type." ) raise PyXFormError(msg) - itemset = element[constants.ITEMSET] - self.choices.pop(itemset, None) - element[constants.ITEMSET] = "" - for i, opt in enumerate(element.get(constants.CHILDREN, [])): - opt["_choice_itext_id"] = f"{element[constants.LIST_NAME_U]}-{i}" + if self.choices: + element.children = self.choices.get(element[constants.ITEMSET], None) + element[constants.ITEMSET] = "" + if element.children is not None: + for i, opt in enumerate(element.children): + opt["_choice_itext_id"] = f"{element[constants.LIST_NAME_U]}-{i}" return is_search def _setup_translations(self): @@ -735,15 +874,19 @@ def get_choices(): for idx, choice in enumerate(choice_list): for col_name, choice_value in choice.items(): lang_choice = None + if not choice_value: + continue if col_name == constants.MEDIA: has_media = True - if isinstance(choice_value, dict): lang_choice = choice_value - multi_language = True elif col_name == constants.LABEL: - lang_choice = {self.default_language: choice_value} - if is_label_dynamic(choice_value): - dyn_label = True + if isinstance(choice_value, dict): + lang_choice = choice_value + multi_language = True + else: + lang_choice = {self.default_language: choice_value} + if is_label_dynamic(choice_value): + dyn_label = True if lang_choice is not None: # e.g. (label, {"default": "Yes"}, "consent", 0) choices.append((col_name, lang_choice, list_name, idx)) @@ -759,27 +902,33 @@ def get_choices(): c[0], c[1], f"{c[2]}-{c[3]}" ) - for path, value in get_choices(): - last_path = path.pop() - leaf_value = {last_path: value, constants.TYPE: constants.CHOICE} - self._add_to_nested_dict(self._translations, path, leaf_value) + if self.choices: + for path, value in get_choices(): + last_path = path.pop() + leaf_value = {last_path: value, constants.TYPE: constants.CHOICE} + self._add_to_nested_dict(self._translations, path, leaf_value) select_types = set(aliases.select.keys()) search_lists = set() non_search_lists = set() - for element in self.iter_descendants(): - itemset = element.get("itemset") - if itemset is not None: - element._itemset_multi_language = itemset in itemsets_multi_language - element._itemset_has_media = itemset in itemsets_has_media - element._itemset_dyn_label = itemset in itemsets_has_dyn_label - - if element[constants.TYPE] in select_types: - select_ref = (element[constants.NAME], element[constants.LIST_NAME_U]) - if self._redirect_is_search_itext(element=element): - search_lists.add(select_ref) - else: - non_search_lists.add(select_ref) + for element in self.iter_descendants( + condition=lambda i: isinstance(i, Question | Section) + ): + if isinstance(element, MultipleChoiceQuestion): + if element.itemset is not None: + element._itemset_multi_language = ( + element.itemset in itemsets_multi_language + ) + element._itemset_has_media = element.itemset in itemsets_has_media + element._itemset_dyn_label = element.itemset in itemsets_has_dyn_label + + if element.type in select_types: + select_ref = (element[constants.NAME], element[constants.LIST_NAME_U]) + if self._redirect_is_search_itext(element=element): + search_lists.add(select_ref) + self._search_lists.add(element[constants.LIST_NAME_U]) + else: + non_search_lists.add(select_ref) # Skip creation of translations for choices in selects. The creation of these # translations is done above in this function. @@ -860,7 +1009,7 @@ def _set_up_media_translations(media_dict, translation_key): media_dict = media_dict_default for media_type, possibly_localized_media in media_dict.items(): - if media_type not in SurveyElement.SUPPORTED_MEDIA: + if media_type not in constants.SUPPORTED_MEDIA_TYPES: raise PyXFormError("Media type: " + media_type + " not supported") if isinstance(possibly_localized_media, dict): @@ -889,17 +1038,20 @@ def _set_up_media_translations(media_dict, translation_key): translations_trans_key[media_type] = media - for survey_element in self.iter_descendants(): + for survey_element in self.iter_descendants( + condition=lambda i: not isinstance( + i, Survey | EntityDeclaration | ExternalInstance | Tag | Option + ) + ): # Skip set up of media for choices in selects. Translations for their media # content should have been set up in _setup_translations, with one copy of # each choice translation per language (after _add_empty_translations). - if not isinstance(survey_element, Option): - media_dict = survey_element.get("media") - if isinstance(media_dict, dict) and 0 < len(media_dict): - translation_key = survey_element.get_xpath() + ":label" - _set_up_media_translations(media_dict, translation_key) + media_dict = survey_element.get("media") + if isinstance(media_dict, dict) and 0 < len(media_dict): + translation_key = survey_element.get_xpath() + ":label" + _set_up_media_translations(media_dict, translation_key) - def itext(self): + def itext(self) -> DetachableElement: """ This function creates the survey's itext nodes from _translations @see _setup_media _setup_translations @@ -995,13 +1147,11 @@ def __unicode__(self): return f"" def _setup_xpath_dictionary(self): - self._xpath = {} # pylint: disable=attribute-defined-outside-init - for element in self.iter_descendants(): - if isinstance(element, Question | Section): - if element.name in self._xpath: - self._xpath[element.name] = None - else: - self._xpath[element.name] = element.get_xpath() + for element in self.iter_descendants(lambda i: isinstance(i, Question | Section)): + if element.name in self._xpath: + self._xpath[element.name] = None + else: + self._xpath[element.name] = element def _var_repl_function( self, matchobj, context, use_current=False, reference_parent=False @@ -1036,7 +1186,7 @@ def _in_secondary_instance_predicate() -> bool: def _relative_path(ref_name: str, _use_current: bool) -> str | None: """Given name in ${name}, return relative xpath to ${name}.""" return_path = None - xpath = self._xpath[ref_name] + xpath = self._xpath[ref_name].get_xpath() context_xpath = context.get_xpath() # share same root i.e repeat_a from /data/repeat_a/... if ( @@ -1045,13 +1195,17 @@ def _relative_path(ref_name: str, _use_current: bool) -> str | None: ): # if context xpath and target xpath fall under the same # repeat use relative xpath referencing. - steps, ref_path = share_same_repeat_parent( - self, xpath, context_xpath, reference_parent - ) - if steps: - ref_path = ref_path if ref_path.endswith(ref_name) else f"/{name}" - prefix = " current()/" if _use_current else " " - return_path = prefix + "/".join([".."] * steps) + ref_path + " " + relation = context.has_common_repeat_parent(self._xpath[ref_name]) + if relation[0] == "Unrelated": + return return_path + else: + steps, ref_path = share_same_repeat_parent( + self, xpath, context_xpath, reference_parent + ) + if steps: + ref_path = ref_path if ref_path.endswith(ref_name) else f"/{name}" + prefix = " current()/" if _use_current else " " + return_path = prefix + "/".join([".."] * steps) + ref_path + " " return return_path @@ -1109,7 +1263,7 @@ def _is_return_relative_path() -> bool: raise PyXFormError(intro + " There is no survey element with this name.") if self._xpath[name] is None: raise PyXFormError( - intro + " There are multiple survey elements" " with this name." + intro + " There are multiple survey elements with this name." ) if _is_return_relative_path(): @@ -1122,9 +1276,15 @@ def _is_return_relative_path() -> bool: last_saved_prefix = ( "instance('" + LAST_SAVED_INSTANCE_NAME + "')" if last_saved else "" ) - return " " + last_saved_prefix + self._xpath[name] + " " + return " " + last_saved_prefix + self._xpath[name].get_xpath() + " " - def insert_xpaths(self, text, context, use_current=False, reference_parent=False): + def insert_xpaths( + self, + text: str, + context: SurveyElement, + use_current: bool = False, + reference_parent: bool = False, + ): """ Replace all instances of ${var} with the xpath to var. """ @@ -1134,6 +1294,7 @@ def _var_repl_function(matchobj): matchobj, context, use_current, reference_parent ) + # "text" may actually be a dict, e.g. for custom attributes. return re.sub(BRACKETED_TAG_REGEX, _var_repl_function, str(text)) def _var_repl_output_function(self, matchobj, context): @@ -1192,7 +1353,7 @@ def print_xform_to_file( if warnings is None: warnings = [] if not path: - path = self._print_name + ".xml" + path = self.id_string + ".xml" if pretty_print: xml = self._to_pretty_xml() else: @@ -1237,17 +1398,18 @@ def to_xml(self, validate=True, pretty_print=True, warnings=None, enketo=False): # So it must be explicitly created, opened, closed, and removed. tmp = tempfile.NamedTemporaryFile(delete=False) tmp.close() + tmp_path = Path(tmp.name) try: # this will throw an exception if the xml is not valid xml = self.print_xform_to_file( - path=tmp.name, + path=tmp_path, validate=validate, pretty_print=pretty_print, warnings=warnings, enketo=enketo, ) finally: - Path(tmp.name).unlink(missing_ok=True) + tmp_path.unlink(missing_ok=True) return xml def instantiate(self): diff --git a/pyxform/survey_element.py b/pyxform/survey_element.py index 3468f92f9..f72d4f747 100644 --- a/pyxform/survey_element.py +++ b/pyxform/survey_element.py @@ -4,78 +4,40 @@ import json import re -from collections import deque -from functools import lru_cache -from typing import TYPE_CHECKING, Any, ClassVar +from collections.abc import Callable, Generator, Iterable, Mapping +from itertools import chain +from typing import TYPE_CHECKING, Optional from pyxform import aliases as alias from pyxform import constants as const from pyxform.errors import PyXFormError from pyxform.parsing.expression import is_xml_tag -from pyxform.question_type_dictionary import QUESTION_TYPE_DICT from pyxform.utils import ( BRACKETED_TAG_REGEX, INVALID_XFORM_TAG_REGEXP, + DetachableElement, default_is_dynamic, node, ) from pyxform.xls2json import print_pyobj_to_json if TYPE_CHECKING: - from pyxform.utils import DetachableElement + from pyxform.survey import Survey + # The following are important keys for the underlying dict that describes SurveyElement -FIELDS = { - "name": str, - const.COMPACT_TAG: str, # used for compact (sms) representation - "sms_field": str, - "sms_option": str, - "label": str, - "hint": str, - "guidance_hint": str, - "default": str, - "type": str, - "appearance": str, - "parameters": dict, - "intent": str, - "jr:count": str, - "bind": dict, - "instance": dict, - "control": dict, - "media": dict, +SURVEY_ELEMENT_FIELDS = ( + "name", + "label", # this node will also have a parent and children, like a tree! - "parent": lambda: None, - "children": list, - "itemset": str, - "choice_filter": str, - "query": str, - "autoplay": str, - "flat": lambda: False, - "action": str, - const.LIST_NAME_U: str, - "trigger": str, -} - - -def _overlay(over, under): - if isinstance(under, dict): - result = under.copy() - result.update(over) - return result - return over if over else under - - -@lru_cache(maxsize=65536) -def any_repeat(survey_element: "SurveyElement", parent_xpath: str) -> bool: - """Return True if there ia any repeat in `parent_xpath`.""" - for item in survey_element.iter_descendants(): - if item.get_xpath() == parent_xpath and item.type == const.REPEAT: - return True - - return False + "parent", + "extra_data", +) +SURVEY_ELEMENT_EXTRA_FIELDS = ("_survey_element_xpath",) +SURVEY_ELEMENT_SLOTS = (*SURVEY_ELEMENT_FIELDS, *SURVEY_ELEMENT_EXTRA_FIELDS) -class SurveyElement(dict): +class SurveyElement(Mapping): """ SurveyElement is the base class we'll looks for the following keys in kwargs: name, label, hint, type, bind, control, parent, @@ -83,62 +45,95 @@ class SurveyElement(dict): """ __name__ = "SurveyElement" - FIELDS: ClassVar[dict[str, Any]] = FIELDS.copy() - - def _default(self): - # TODO: need way to override question type dictionary - defaults = QUESTION_TYPE_DICT - return defaults.get(self.get("type"), {}) - - def __getattr__(self, key): - """ - Get attributes from FIELDS rather than the class. - """ - if key in self.FIELDS: - question_type_dict = self._default() - under = question_type_dict.get(key, None) - over = self.get(key) - if not under: - return over - return _overlay(over, under) - raise AttributeError(key) + __slots__ = SURVEY_ELEMENT_SLOTS def __hash__(self): return hash(id(self)) - def __setattr__(self, key, value): - self[key] = value + def __getitem__(self, key): + return self.__getattribute__(key) + + @staticmethod + def get_slot_names() -> tuple[str, ...]: + """Each subclass must provide a list of slots from itself and all parents.""" + return SURVEY_ELEMENT_SLOTS + + def __len__(self): + return len(self.get_slot_names()) - def __init__(self, **kwargs): - for key, default in self.FIELDS.items(): - self[key] = kwargs.get(key, default()) - self._link_children() + def __iter__(self): + return iter(self.get_slot_names()) + + def __setitem__(self, key, value): + self.__setattr__(key, value) + + def __setattr__(self, key, value): + if key == "parent": + # If object graph position changes then invalidate cached. + self._survey_element_xpath = None + super().__setattr__(key, value) + + def __init__( + self, + name: str, + label: str | dict | None = None, + fields: tuple[str, ...] | None = None, + **kwargs, + ): + # Internals + self._survey_element_xpath: str | None = None + + # Structure + self.parent: SurveyElement | None = None + self.extra_data: dict | None = None + + # Settings + self.name: str = name + self.label: str | dict | None = label + + if fields is not None: + for key in fields: + if key not in SURVEY_ELEMENT_FIELDS: + value = kwargs.pop(key, None) + if value or not hasattr(self, key): + self[key] = value + if len(kwargs) > 0: + self.extra_data = kwargs + + if hasattr(self, const.CHILDREN): + self._link_children() # Create a space label for unlabeled elements with the label # appearance tag. # This is because such elements are used to label the # options for selects in a field-list and might want blank labels for # themselves. - if self.control.get("appearance") == "label" and not self.label: - self["label"] = " " + if ( + hasattr(self, "control") + and self.control + and self.control.get("appearance") == "label" + and not self.label + ): + self.label = " " + super().__init__() def _link_children(self): - for child in self.children: - child.parent = self + if self.children is not None: + for child in self.children: + child.parent = self def add_child(self, child): + if self.children is None: + self.children = [] self.children.append(child) child.parent = self def add_children(self, children): - if isinstance(children, list): + if isinstance(children, list | tuple): for child in children: self.add_child(child) else: self.add_child(children) - # Supported media types for attaching to questions - SUPPORTED_MEDIA = ("image", "big-image", "audio", "video") - def validate(self): if not is_xml_tag(self.name): invalid_char = re.search(INVALID_XFORM_TAG_REGEXP, self.name) @@ -147,53 +142,124 @@ def validate(self): ) # TODO: Make sure renaming this doesn't cause any problems - def iter_descendants(self): + def iter_descendants( + self, condition: Callable[["SurveyElement"], bool] | None = None + ) -> Generator["SurveyElement", None, None]: """ - A survey_element is a dictionary of survey_elements - This method does a preorder traversal over them. - For the time being this survery_element is included among its - descendants + Get each of self.children. + + :param condition: If this evaluates to True, yield the element. """ # it really seems like this method should not yield self - yield self - for e in self.children: - yield from e.iter_descendants() - - def any_repeat(self, parent_xpath: str) -> bool: - """Return True if there ia any repeat in `parent_xpath`.""" - return any_repeat(survey_element=self, parent_xpath=parent_xpath) + if condition is not None: + if condition(self): + yield self + else: + yield self + if hasattr(self, const.CHILDREN) and self.children is not None: + for e in self.children: + yield from e.iter_descendants(condition=condition) + + def iter_ancestors( + self, condition: Callable[["SurveyElement"], bool] | None = None + ) -> Generator[tuple["SurveyElement", int], None, None]: + """ + Get each self.parent with their distance from self (starting at 1). - def get_lineage(self): + :param condition: If this evaluates to True, yield the element. + """ + distance = 1 + current = self.parent + while current is not None: + if condition is not None: + if condition(current): + yield current, distance + else: + yield current, distance + current = current.parent + distance += 1 + + def has_common_repeat_parent( + self, other: "SurveyElement" + ) -> tuple[str, int | None, Optional["SurveyElement"]]: """ - Return a the list [root, ..., self._parent, self] + Get the relation type, steps (generations), and the common ancestor. """ - result = deque((self,)) - current_element = self - while current_element.parent: - current_element = current_element.parent - result.appendleft(current_element) - # For some reason the root element has a True flat property... - output = [result.popleft()] - output.extend([i for i in result if not i.get("flat")]) - return output - - def get_root(self): - return self.get_lineage()[0] + # Quick check for immediate relation. + if self.parent is other: + return "Parent (other)", 1, other + elif other.parent is self: + return "Parent (self)", 1, self + + # Traversal tracking + self_ancestors = {} + other_ancestors = {} + self_current = self + other_current = other + self_distance = 0 + other_distance = 0 + + # Traverse up both ancestor chains as far as necessary. + while self_current or other_current: + # Step up the self chain + if self_current: + self_distance += 1 + self_current = self_current.parent + if self_current: + self_ancestors[self_current] = self_distance + if ( + self_current.type == const.REPEAT + and self_current in other_ancestors + ): + max_steps = max(self_distance, other_ancestors[self_current]) + return "Common Ancestor Repeat", max_steps, self_current + + # Step up the other chain + if other_current: + other_distance += 1 + other_current = other_current.parent + if other_current: + other_ancestors[other_current] = other_distance + if ( + other_current.type == const.REPEAT + and other_current in self_ancestors + ): + max_steps = max(other_distance, self_ancestors[other_current]) + return "Common Ancestor Repeat", max_steps, other_current + + # No common ancestor found. + return "Unrelated", None, None def get_xpath(self): """ Return the xpath of this survey element. """ - return "/".join([""] + [n.name for n in self.get_lineage()]) + # Imported here to avoid circular references. + from pyxform.survey import Survey + + def condition(e): + # The "flat" setting was added in 2013 to support ODK Tables, and results in + # a data instance with no element nesting. Not sure if still needed. + return isinstance(e, Survey) or ( + not isinstance(e, Survey) and not (hasattr(e, "flat") and e.get("flat")) + ) - def get_abbreviated_xpath(self): - lineage = self.get_lineage() - if len(lineage) >= 2: - return "/".join([str(n.name) for n in lineage[1:]]) - else: - return lineage[0].name + current_value = self._survey_element_xpath + if current_value is None: + if condition(self): + self_element = (self,) + else: + self_element = () + lineage = chain( + reversed(tuple(i[0] for i in self.iter_ancestors(condition=condition))), + self_element, + ) + new_value = f'/{"/".join(n.name for n in lineage)}' + self._survey_element_xpath = new_value + return new_value + return current_value - def _delete_keys_from_dict(self, dictionary: dict, keys: list): + def _delete_keys_from_dict(self, dictionary: dict, keys: Iterable[str]): """ Deletes a list of keys from a dictionary. Credits: https://stackoverflow.com/a/49723101 @@ -206,21 +272,33 @@ def _delete_keys_from_dict(self, dictionary: dict, keys: list): if isinstance(value, dict): self._delete_keys_from_dict(value, keys) - def to_json_dict(self): + def copy(self): + return {k: self[k] for k in self} + + def to_json_dict(self, delete_keys: Iterable[str] | None = None) -> dict: """ Create a dict copy of this survey element by removing inappropriate attributes and converting its children to dicts """ self.validate() result = self.copy() - to_delete = ["parent", "question_type_dictionary", "_created"] + to_delete = chain(SURVEY_ELEMENT_EXTRA_FIELDS, ("extra_data",)) + if delete_keys is not None: + to_delete = chain(to_delete, delete_keys) # Delete all keys that may cause a "Circular Reference" # error while converting the result to JSON self._delete_keys_from_dict(result, to_delete) - children = result.pop("children") - result["children"] = [] - for child in children: - result["children"].append(child.to_json_dict()) + children = result.pop("children", None) + if children: + result["children"] = [ + c.to_json_dict(delete_keys=("parent",)) for c in children + ] + choices = result.pop("choices", None) + if choices: + result["choices"] = { + list_name: [o.to_json_dict(delete_keys=("parent",)) for o in options] + for list_name, options in choices.items() + } # Translation items with "output_context" have circular references. if "_translations" in result: for lang in result["_translations"].values(): @@ -305,7 +383,9 @@ def get_translations(self, default_language): } for display_element in ["label", "hint", "guidance_hint"]: - label_or_hint = self[display_element] + label_or_hint = None + if hasattr(self, display_element): + label_or_hint = self[display_element] if ( display_element == "label" @@ -319,6 +399,7 @@ def get_translations(self, default_language): # how they're defined - https://opendatakit.github.io/xforms-spec/#languages if ( display_element == "guidance_hint" + and label_or_hint is not None and not isinstance(label_or_hint, dict) and len(label_or_hint) > 0 ): @@ -328,8 +409,11 @@ def get_translations(self, default_language): if ( display_element == "hint" and not isinstance(label_or_hint, dict) + and hasattr(self, "hint") + and self.get("hint") is not None and len(label_or_hint) > 0 - and "guidance_hint" in self.keys() + and hasattr(self, "guidance_hint") + and self.get("guidance_hint") is not None and len(self["guidance_hint"]) > 0 ): label_or_hint = {default_language: label_or_hint} @@ -345,23 +429,20 @@ def get_translations(self, default_language): "text": text, } - def get_media_keys(self): - """ - @deprected - I'm leaving this in just in case it has outside references. - """ - return {"media": f"{self.get_xpath()}:media"} - def needs_itext_ref(self): return isinstance(self.label, dict) or ( - isinstance(self.media, dict) and len(self.media) > 0 + hasattr(self, const.MEDIA) and isinstance(self.media, dict) and self.media ) - def get_setvalue_node_for_dynamic_default(self, in_repeat=False): - if not self.default or not default_is_dynamic(self.default, self.type): + def get_setvalue_node_for_dynamic_default(self, survey: "Survey", in_repeat=False): + if ( + not hasattr(self, "default") + or not self.default + or not default_is_dynamic(self.default, self.type) + ): return None - default_with_xpath_paths = self.get_root().insert_xpaths(self.default, self) + default_with_xpath_paths = survey.insert_xpaths(self.default, self) triggering_events = "odk-instance-first-load" if in_repeat: @@ -375,26 +456,29 @@ def get_setvalue_node_for_dynamic_default(self, in_repeat=False): ) # XML generating functions, these probably need to be moved around. - def xml_label(self): + def xml_label(self, survey: "Survey"): if self.needs_itext_ref(): # If there is a dictionary label, or non-empty media dict, # then we need to make a label with an itext ref ref = f"""jr:itext('{self._translation_path("label")}')""" return node("label", ref=ref) - else: - survey = self.get_root() + elif self.label: label, output_inserted = survey.insert_output_values(self.label, self) return node("label", label, toParseString=output_inserted) + else: + return node("label") - def xml_hint(self): + def xml_hint(self, survey: "Survey"): if isinstance(self.hint, dict) or self.guidance_hint: path = self._translation_path("hint") return node("hint", ref=f"jr:itext('{path}')") - else: - hint, output_inserted = self.get_root().insert_output_values(self.hint, self) + elif self.hint: + hint, output_inserted = survey.insert_output_values(self.hint, self) return node("hint", hint, toParseString=output_inserted) + else: + return node("hint") - def xml_label_and_hint(self) -> list["DetachableElement"]: + def xml_label_and_hint(self, survey: "Survey") -> list["DetachableElement"]: """ Return a list containing one node for the label and if there is a hint one node for the hint. @@ -402,13 +486,13 @@ def xml_label_and_hint(self) -> list["DetachableElement"]: result = [] label_appended = False if self.label or self.media: - result.append(self.xml_label()) + result.append(self.xml_label(survey=survey)) label_appended = True if self.hint or self.guidance_hint: if not label_appended: - result.append(self.xml_label()) - result.append(self.xml_hint()) + result.append(self.xml_label(survey=survey)) + result.append(self.xml_hint(survey=survey)) msg = f"The survey element named '{self.name}' has no label or hint." if len(result) == 0: @@ -419,107 +503,62 @@ def xml_label_and_hint(self) -> list["DetachableElement"]: raise PyXFormError(msg) # big-image must combine with image - if "image" not in self.media and "big-image" in self.media: + if ( + self.media is not None + and "image" not in self.media + and "big-image" in self.media + ): raise PyXFormError( "To use big-image, you must also specify an image for the survey element named {self.name}." ) return result - def xml_bindings(self): + def xml_bindings( + self, survey: "Survey" + ) -> Generator[DetachableElement | None, None, None]: """ Return the binding(s) for this survey element. """ - survey = self.get_root() - bind_dict = self.bind.copy() - if self.get("flat"): + if not hasattr(self, "bind") or self.get("bind") is None: + return None + if hasattr(self, "flat") and self.get("flat"): # Don't generate bind element for flat groups. return None - if bind_dict: - # the expression goes in a setvalue action - if self.trigger and "calculate" in self.bind: - del bind_dict["calculate"] - - for k, v in bind_dict.items(): - # I think all the binding conversions should be happening on - # the xls2json side. - if ( - hashable(v) - and v in alias.BINDING_CONVERSIONS - and k in const.CONVERTIBLE_BIND_ATTRIBUTES - ): - v = alias.BINDING_CONVERSIONS[v] - if k == "jr:constraintMsg" and ( - isinstance(v, dict) or re.search(BRACKETED_TAG_REGEX, v) - ): - v = f"""jr:itext('{self._translation_path("jr:constraintMsg")}')""" - if k == "jr:requiredMsg" and ( - isinstance(v, dict) or re.search(BRACKETED_TAG_REGEX, v) - ): - v = f"""jr:itext('{self._translation_path("jr:requiredMsg")}')""" - if k == "jr:noAppErrorString" and isinstance(v, dict): - v = f"""jr:itext('{self._translation_path("jr:noAppErrorString")}')""" - bind_dict[k] = survey.insert_xpaths(v, context=self) - return [node("bind", nodeset=self.get_xpath(), **bind_dict)] - return None - - def xml_descendent_bindings(self): - """ - Return a list of bindings for this node and all its descendants. - """ - result = [] - for e in self.iter_descendants(): - xml_bindings = e.xml_bindings() - if xml_bindings is not None: - result.extend(xml_bindings) - # dynamic defaults for repeats go in the body. All other dynamic defaults (setvalue actions) go in the model + bind_dict = {} + for k, v in self.bind.items(): + # the expression goes in a setvalue action + if hasattr(self, "trigger") and self.trigger and k == "calculate": + continue + # I think all the binding conversions should be happening on + # the xls2json side. if ( - len( - [ - ancestor - for ancestor in e.get_lineage() - if ancestor.type == "repeat" - ] - ) - == 0 + hashable(v) + and v in alias.BINDING_CONVERSIONS + and k in const.CONVERTIBLE_BIND_ATTRIBUTES ): - dynamic_default = e.get_setvalue_node_for_dynamic_default() - if dynamic_default: - result.append(dynamic_default) - return result + v = alias.BINDING_CONVERSIONS[v] + elif k == "jr:constraintMsg" and ( + isinstance(v, dict) or re.search(BRACKETED_TAG_REGEX, v) + ): + v = f"""jr:itext('{self._translation_path("jr:constraintMsg")}')""" + elif k == "jr:requiredMsg" and ( + isinstance(v, dict) or re.search(BRACKETED_TAG_REGEX, v) + ): + v = f"""jr:itext('{self._translation_path("jr:requiredMsg")}')""" + elif k == "jr:noAppErrorString" and isinstance(v, dict): + v = f"""jr:itext('{self._translation_path("jr:noAppErrorString")}')""" + bind_dict[k] = survey.insert_xpaths(v, context=self) + yield node("bind", nodeset=self.get_xpath(), **bind_dict) - def xml_control(self): + def xml_control(self, survey: "Survey"): """ The control depends on what type of question we're asking, it doesn't make sense to implement here in the base class. """ raise NotImplementedError("Control not implemented") - def xml_action(self): - """ - Return the action for this survey element. - """ - if self.action: - action_dict = self.action.copy() - if action_dict: - name = action_dict["name"] - del action_dict["name"] - return node(name, ref=self.get_xpath(), **action_dict) - - return None - - def xml_actions(self): - """ - Return a list of actions for this node and all its descendants. - """ - result = [] - for e in self.iter_descendants(): - xml_action = e.xml_action() - if xml_action is not None: - result.append(xml_action) - return result - def hashable(v): """Determine whether `v` can be hashed.""" diff --git a/pyxform/utils.py b/pyxform/utils.py index 37e5a8495..66eb771b8 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -5,9 +5,10 @@ import copy import csv import json -import os import re +from collections.abc import Generator, Iterable from io import StringIO +from itertools import chain from json.decoder import JSONDecodeError from typing import Any from xml.dom import Node @@ -58,7 +59,7 @@ def writexml(self, writer, indent="", addindent="", newl=""): if self.childNodes: writer.write(">") # For text or mixed content, write without adding indents or newlines. - if 0 < len([c for c in self.childNodes if c.nodeType in NODE_TYPE_TEXT]): + if any(c.nodeType in NODE_TYPE_TEXT for c in self.childNodes): # Conditions to match old Survey.py regex for remaining whitespace. child_nodes = len(self.childNodes) for idx, cnode in enumerate(self.childNodes): @@ -94,39 +95,29 @@ def node(*args, **kwargs) -> DetachableElement: kwargs -- attributes returns a xml.dom.minidom.Element """ - blocked_attributes = ["tag"] + blocked_attributes = {"tag"} tag = args[0] if len(args) > 0 else kwargs["tag"] args = args[1:] result = DetachableElement(tag) - unicode_args = [u for u in args if isinstance(u, str)] + unicode_args = tuple(u for u in args if isinstance(u, str)) if len(unicode_args) > 1: raise PyXFormError("""Invalid value for `unicode_args`.""") parsed_string = False # Convert the kwargs xml attribute dictionary to a xml.dom.minidom.Element. - for k, v in iter(kwargs.items()): + for k, v in kwargs.items(): if k in blocked_attributes: continue if k == "toParseString": if v is True and len(unicode_args) == 1: parsed_string = True # Add this header string so parseString can be used? - s = ( - '<' - + tag - + ">" - + unicode_args[0] - + "" - ) + s = f"""<{tag}>{unicode_args[0]}""" parsed_node = parseString(s.encode("utf-8")).documentElement # Move node's children to the result Element # discarding node's root for child in parsed_node.childNodes: - # No tests seem to involve moving elements with children. - # Deep clone used anyway in case of unknown edge cases. - result.appendChild(child.cloneNode(deep=True)) + result.appendChild(child.cloneNode(deep=False)) else: result.setAttribute(k, v) @@ -139,6 +130,10 @@ def node(*args, **kwargs) -> DetachableElement: text_node = PatchedText() text_node.data = str(n) result.appendChild(text_node) + elif isinstance(n, Generator): + for e in n: + if e is not None: + result.appendChild(e) elif not isinstance(n, str): result.appendChild(n) return result @@ -214,27 +209,6 @@ def has_external_choices(json_struct): return False -def get_languages_with_bad_tags(languages): - """ - Returns languages with invalid or missing IANA subtags. - """ - path = os.path.join(os.path.dirname(__file__), "iana_subtags.txt") - with open(path, encoding="utf-8") as f: - iana_subtags = f.read().splitlines() - - lang_code_regex = re.compile(r"\((.*)\)$") - - languages_with_bad_tags = [] - for lang in languages: - lang_code = re.search(lang_code_regex, lang) - - if lang != "default" and ( - not lang_code or lang_code.group(1) not in iana_subtags - ): - languages_with_bad_tags.append(lang) - return languages_with_bad_tags - - def default_is_dynamic(element_default, element_type=None): """ Returns true if the default value is a dynamic value. @@ -335,3 +309,19 @@ def levenshtein_distance(a: str, b: str) -> int: def coalesce(*args): return next((a for a in args if a is not None), None) + + +def combine_lists( + a: Iterable | None = None, + b: Iterable | None = None, +) -> Iterable | None: + """Get the list that is not None, or both lists combined, or an empty list.""" + if b is None: + if a is None: + return None + else: + return a + elif a is None: + return b + else: + return chain(a, b) diff --git a/pyxform/validators/pyxform/choices.py b/pyxform/validators/pyxform/choices.py index c97298e75..3b347c353 100644 --- a/pyxform/validators/pyxform/choices.py +++ b/pyxform/validators/pyxform/choices.py @@ -37,7 +37,7 @@ def check(): return list(check()) -def validate_choices( +def validate_choice_list( options: list[dict], warnings: list[str], allow_duplicates: bool = False ) -> None: seen_options = set() @@ -57,3 +57,22 @@ def validate_choices( if 0 < len(duplicate_errors): raise PyXFormError("\n".join(duplicate_errors)) + + +def validate_choices( + choices: dict[str, list[dict]], + warnings: list[str], + headers: tuple[tuple[str, ...], ...], + allow_duplicates: bool = False, +): + invalid_headers = validate_headers(headers, warnings) + for options in choices.values(): + validate_choice_list( + options=options, + warnings=warnings, + allow_duplicates=allow_duplicates, + ) + for option in options: + for invalid_header in invalid_headers: + option.pop(invalid_header, None) + del option["__row"] diff --git a/pyxform/validators/pyxform/iana_subtags/__init__.py b/pyxform/validators/pyxform/iana_subtags/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyxform/validators/pyxform/iana_subtags/iana_subtags_2_characters.txt b/pyxform/validators/pyxform/iana_subtags/iana_subtags_2_characters.txt new file mode 100644 index 000000000..80e1e4318 --- /dev/null +++ b/pyxform/validators/pyxform/iana_subtags/iana_subtags_2_characters.txt @@ -0,0 +1,190 @@ +aa +ab +ae +af +ak +am +an +ar +as +av +ay +az +ba +be +bg +bh +bi +bm +bn +bo +br +bs +ca +ce +ch +co +cr +cs +cu +cv +cy +da +de +dv +dz +ee +el +en +eo +es +et +eu +fa +ff +fi +fj +fo +fr +fy +ga +gd +gl +gn +gu +gv +ha +he +hi +ho +hr +ht +hu +hy +hz +ia +id +ie +ig +ii +ik +in +io +is +it +iu +iw +ja +ji +jv +jw +ka +kg +ki +kj +kk +kl +km +kn +ko +kr +ks +ku +kv +kw +ky +la +lb +lg +li +ln +lo +lt +lu +lv +mg +mh +mi +mk +ml +mn +mo +mr +ms +mt +my +na +nb +nd +ne +ng +nl +nn +no +nr +nv +ny +oc +oj +om +or +os +pa +pi +pl +ps +pt +qu +rm +rn +ro +ru +rw +sa +sc +sd +se +sg +sh +si +sk +sl +sm +sn +so +sq +sr +ss +st +su +sv +sw +ta +te +tg +th +ti +tk +tl +tn +to +tr +ts +tt +tw +ty +ug +uk +ur +uz +ve +vi +vo +wa +wo +xh +yi +yo +za +zh +zu \ No newline at end of file diff --git a/pyxform/iana_subtags.txt b/pyxform/validators/pyxform/iana_subtags/iana_subtags_3_or_more_characters.txt similarity index 98% rename from pyxform/iana_subtags.txt rename to pyxform/validators/pyxform/iana_subtags/iana_subtags_3_or_more_characters.txt index 6d1ca078f..96a447498 100644 --- a/pyxform/iana_subtags.txt +++ b/pyxform/validators/pyxform/iana_subtags/iana_subtags_3_or_more_characters.txt @@ -1,193 +1,3 @@ -aa -ab -ae -af -ak -am -an -ar -as -av -ay -az -ba -be -bg -bh -bi -bm -bn -bo -br -bs -ca -ce -ch -co -cr -cs -cu -cv -cy -da -de -dv -dz -ee -el -en -eo -es -et -eu -fa -ff -fi -fj -fo -fr -fy -ga -gd -gl -gn -gu -gv -ha -he -hi -ho -hr -ht -hu -hy -hz -ia -id -ie -ig -ii -ik -in -io -is -it -iu -iw -ja -ji -jv -jw -ka -kg -ki -kj -kk -kl -km -kn -ko -kr -ks -ku -kv -kw -ky -la -lb -lg -li -ln -lo -lt -lu -lv -mg -mh -mi -mk -ml -mn -mo -mr -ms -mt -my -na -nb -nd -ne -ng -nl -nn -no -nr -nv -ny -oc -oj -om -or -os -pa -pi -pl -ps -pt -qu -rm -rn -ro -ru -rw -sa -sc -sd -se -sg -sh -si -sk -sl -sm -sn -so -sq -sr -ss -st -su -sv -sw -ta -te -tg -th -ti -tk -tl -tn -to -tr -ts -tt -tw -ty -ug -uk -ur -uz -ve -vi -vo -wa -wo -xh -yi -yo -za -zh -zu aaa aab aac diff --git a/pyxform/validators/pyxform/iana_subtags/validation.py b/pyxform/validators/pyxform/iana_subtags/validation.py new file mode 100644 index 000000000..4cd368426 --- /dev/null +++ b/pyxform/validators/pyxform/iana_subtags/validation.py @@ -0,0 +1,37 @@ +import re +from functools import lru_cache +from pathlib import Path + +LANG_CODE_REGEX = re.compile(r"\((.*)\)$") +HERE = Path(__file__).parent + + +@lru_cache(maxsize=2) # Expecting to read 2 files. +def read_tags(file_name: str) -> set[str]: + path = HERE / file_name + with open(path, encoding="utf-8") as f: + return {line.strip() for line in f} + + +def get_languages_with_bad_tags(languages): + """ + Returns languages with invalid or missing IANA subtags. + """ + languages_with_bad_tags = [] + for lang in languages: + # Minimum matchable lang code attempt requires 3 characters e.g. "a()". + if lang == "default" or len(lang) < 3: + continue + lang_code = LANG_CODE_REGEX.search(lang) + if not lang_code: + languages_with_bad_tags.append(lang) + else: + lang_match = lang_code.group(1) + # Check the short list first: 190 short codes vs 7948 long codes. + if lang_match in read_tags("iana_subtags_2_characters.txt"): + continue + elif lang_match in read_tags("iana_subtags_3_or_more_characters.txt"): + continue + else: + languages_with_bad_tags.append(lang) + return languages_with_bad_tags diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index 7e4e6ebe8..af281ef98 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -538,20 +538,15 @@ def workbook_to_json( # columns is run with Survey sheet below. # Warn and remove invalid headers in case the form uses headers for notes. - invalid_headers = vc.validate_headers(choices_sheet.headers, warnings) allow_duplicates = aliases.yes_no.get( settings.get("allow_choice_duplicates", False), False ) - for options in choices.values(): - vc.validate_choices( - options=options, - warnings=warnings, - allow_duplicates=allow_duplicates, - ) - for option in options: - for invalid_header in invalid_headers: - option.pop(invalid_header, None) - del option["__row"] + vc.validate_choices( + choices=choices, + warnings=warnings, + headers=choices_sheet.headers, + allow_duplicates=allow_duplicates, + ) if 0 < len(choices): json_dict[constants.CHOICES] = choices @@ -1137,13 +1132,57 @@ def workbook_to_json( specify_other_question = None if parse_dict.get("specify_other") is not None: sheet_translations.or_other_seen = True - select_type += constants.SELECT_OR_OTHER_SUFFIX if row.get(constants.CHOICE_FILTER): msg = ( ROW_FORMAT_STRING % row_number + " Choice filter not supported with or_other." ) raise PyXFormError(msg) + itemset_choices = choices.get(list_name, None) + if not itemset_choices: + msg = ( + ROW_FORMAT_STRING % row_number + + " Please specify choices for this 'or other' question." + ) + raise PyXFormError(msg) + if ( + itemset_choices is not None + and isinstance(itemset_choices, list) + and not any( + c[constants.NAME] == constants.OR_OTHER_CHOICE[constants.NAME] + for c in itemset_choices + ) + ): + if any( + isinstance(c.get(constants.LABEL), dict) + for c in itemset_choices + ): + itemset_choices.append( + { + constants.NAME: constants.OR_OTHER_CHOICE[ + constants.NAME + ], + constants.LABEL: { + lang: constants.OR_OTHER_CHOICE[constants.LABEL] + for lang in { + lang + for c in itemset_choices + for lang in c[constants.LABEL] + if isinstance(c.get(constants.LABEL), dict) + } + }, + } + ) + else: + itemset_choices.append(constants.OR_OTHER_CHOICE) + specify_other_question = { + constants.TYPE: "text", + constants.NAME: f"{row[constants.NAME]}_other", + constants.LABEL: "Specify other.", + constants.BIND: { + "relevant": f"selected(../{row[constants.NAME]}, 'other')" + }, + } new_json_dict = row.copy() new_json_dict[constants.TYPE] = select_type diff --git a/pyxform/xls2json_backends.py b/pyxform/xls2json_backends.py index 6a81a8117..276529156 100644 --- a/pyxform/xls2json_backends.py +++ b/pyxform/xls2json_backends.py @@ -5,7 +5,6 @@ import csv import datetime import re -from collections import OrderedDict from collections.abc import Callable, Iterator from dataclasses import dataclass from enum import Enum @@ -16,16 +15,17 @@ from typing import Any from zipfile import BadZipFile -import openpyxl -import xlrd +from openpyxl import open as pyxl_open from openpyxl.cell import Cell as pyxlCell from openpyxl.reader.excel import ExcelReader from openpyxl.workbook import Workbook as pyxlWorkbook from openpyxl.worksheet.worksheet import Worksheet as pyxlWorksheet +from xlrd import XL_CELL_BOOLEAN, XL_CELL_DATE, XL_CELL_NUMBER, XLRDError +from xlrd import open_workbook as xlrd_open from xlrd.book import Book as xlrdBook from xlrd.sheet import Cell as xlrdCell from xlrd.sheet import Sheet as xlrdSheet -from xlrd.xldate import XLDateAmbiguous +from xlrd.xldate import XLDateAmbiguous, xldate_as_tuple from pyxform import constants from pyxform.errors import PyXFormError, PyXFormReadError @@ -43,10 +43,7 @@ def _list_to_dict_list(list_items): Returns a list of the created dict or an empty list """ if list_items: - k = OrderedDict() - for item in list_items: - k[str(item)] = "" - return [k] + return [{str(i): "" for i in list_items}] return [] @@ -96,7 +93,7 @@ def get_excel_rows( adjacent_empty_rows = 0 result_rows = [] for row_n, row in enumerate(rows): - row_dict = OrderedDict() + row_dict = {} for col_n, key in col_header_enum: if key is None: continue @@ -168,7 +165,7 @@ def clean_func(cell: xlrdCell, row_n: int, col_key: str) -> str | None: return rows, _list_to_dict_list(column_header_list) def process_workbook(wb: xlrdBook): - result_book = OrderedDict() + result_book = {} for wb_sheet in wb.sheets(): # Note that the sheet exists but do no further processing here. result_book[wb_sheet.name] = [] @@ -190,12 +187,12 @@ def process_workbook(wb: xlrdBook): try: wb_file = get_definition_data(definition=path_or_file) - workbook = xlrd.open_workbook(file_contents=wb_file.data.getvalue()) + workbook = xlrd_open(file_contents=wb_file.data.getvalue()) try: return process_workbook(wb=workbook) finally: workbook.release_resources() - except (AttributeError, TypeError, xlrd.XLRDError) as read_err: + except (AttributeError, TypeError, XLRDError) as read_err: raise PyXFormReadError(f"Error reading .xls file: {read_err}") from read_err @@ -203,21 +200,21 @@ def xls_value_to_unicode(value, value_type, datemode) -> str: """ Take a xls formatted value and try to make a unicode string representation. """ - if value_type == xlrd.XL_CELL_BOOLEAN: + if value_type == XL_CELL_BOOLEAN: return "TRUE" if value else "FALSE" - elif value_type == xlrd.XL_CELL_NUMBER: + elif value_type == XL_CELL_NUMBER: # Try to display as an int if possible. int_value = int(value) if int_value == value: return str(int_value) else: return str(value) - elif value_type is xlrd.XL_CELL_DATE: + elif value_type is XL_CELL_DATE: # Warn that it is better to single quote as a string. # error_location = cellFormatString % (ss_row_idx, ss_col_idx) # raise Exception( # "Cannot handle excel formatted date at " + error_location) - datetime_or_time_only = xlrd.xldate_as_tuple(value, datemode) + datetime_or_time_only = xldate_as_tuple(value, datemode) if datetime_or_time_only[:3] == (0, 0, 0): # must be time only return str(datetime.time(*datetime_or_time_only[3:])) @@ -258,7 +255,7 @@ def xlsx_to_dict_normal_sheet(sheet: pyxlWorksheet): return rows, _list_to_dict_list(column_header_list) def process_workbook(wb: pyxlWorkbook): - result_book = OrderedDict() + result_book = {} for sheetname in wb.sheetnames: wb_sheet = wb[sheetname] # Note that the sheet exists but do no further processing here. @@ -372,7 +369,7 @@ def first_column_as_sheet_name(row): return s_or_c, content def process_csv_data(rd): - _dict = OrderedDict() + _dict = {} sheet_name = None current_headers = None for row in rd: @@ -387,7 +384,7 @@ def process_csv_data(rd): current_headers = content _dict[f"{sheet_name}_header"] = _list_to_dict_list(current_headers) else: - _d = OrderedDict() + _d = {} for key, val in zip(current_headers, content, strict=False): if val != "": # Slight modification so values are striped @@ -468,10 +465,10 @@ def sheet_to_csv(workbook_path, csv_path, sheet_name): def xls_sheet_to_csv(workbook_path, csv_path, sheet_name): - wb = xlrd.open_workbook(workbook_path) + wb = xlrd_open(workbook_path) try: sheet = wb.sheet_by_name(sheet_name) - except xlrd.biffh.XLRDError: + except XLRDError: return False if not sheet or sheet.nrows < 2: return False @@ -497,7 +494,7 @@ def xls_sheet_to_csv(workbook_path, csv_path, sheet_name): def xlsx_sheet_to_csv(workbook_path, csv_path, sheet_name): - wb = openpyxl.open(workbook_path, read_only=True, data_only=True) + wb = pyxl_open(workbook_path, read_only=True, data_only=True) try: sheet = wb[sheet_name] except KeyError: diff --git a/tests/parsing/test_expression.py b/tests/parsing/test_expression.py index 1fe3ad42c..ca8dc4c93 100644 --- a/tests/parsing/test_expression.py +++ b/tests/parsing/test_expression.py @@ -31,6 +31,9 @@ ("name with space", "Invalid character (space)"), ("na@me", "Invalid character (@)"), ("na#me", "Invalid character (#)"), + ("name:.name", "Invalid character (in local name)"), + ("-name:name", "Invalid character (in namespace)"), + ("$name:@name", "Invalid character (in both names)"), ("name:name:name", "Invalid structure (multiple colons)"), ] diff --git a/tests/pyxform_test_case.py b/tests/pyxform_test_case.py index df2fb5c75..0d9f3e633 100644 --- a/tests/pyxform_test_case.py +++ b/tests/pyxform_test_case.py @@ -161,12 +161,14 @@ def assertPyxformXform( if survey is None: result = convert( xlsform=coalesce(md, ss_structure), + pretty_print=True, form_name=coalesce(name, "test_name"), warnings=warnings, ) survey = result._survey - - xml = survey._to_pretty_xml() + xml = result.xform + else: + xml = survey._to_pretty_xml() root = etree.fromstring(xml.encode("utf-8")) # Ensure all namespaces are present, even if unused @@ -428,7 +430,7 @@ def assert_xpath_count( xpath=xpath, ) msg = ( - f"XPath found no matches (test case {case_num}):\n{xpath}" + f"XPath did not find the expected number of matches ({expected}, test case {case_num}):\n{xpath}" f"\n\nXForm content:\n{matcher_context.content_str}" ) self.assertEqual(expected, len(observed), msg=msg) diff --git a/tests/test_builder.py b/tests/test_builder.py index 83e10372a..2c5ded362 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -32,8 +32,8 @@ class BuilderTests(TestCase): # self.assertTrue(xml_compare(expected, result)) def test_unknown_question_type(self): - survey = utils.build_survey("unknown_question_type.xls") - self.assertRaises(PyXFormError, survey.to_xml) + with self.assertRaises(PyXFormError): + utils.build_survey("unknown_question_type.xls") def test_uniqueness_of_section_names(self): # Looking at the xls file, I think this test might be broken. @@ -43,9 +43,7 @@ def test_uniqueness_of_section_names(self): def setUp(self): self.this_directory = os.path.dirname(__file__) survey_out = Survey(name="age", sms_keyword="age", type="survey") - question = InputQuestion(name="age") - question.type = "integer" - question.label = "How old are you?" + question = InputQuestion(name="age", type="integer", label="How old are you?") survey_out.add_child(question) self.survey_out_dict = survey_out.to_json_dict() print_pyobj_to_json( @@ -113,8 +111,7 @@ def test_create_table_from_dict(self): }, ], } - - self.assertEqual(g.to_json_dict(), expected_dict) + self.assertEqual(expected_dict, g.to_json_dict()) def test_specify_other(self): survey = utils.create_survey_from_fixture( @@ -169,7 +166,6 @@ def test_specify_other(self): }, ], } - self.maxDiff = None self.assertEqual(survey.to_json_dict(), expected_dict) def test_select_one_question_with_identical_choice_name(self): @@ -211,8 +207,7 @@ def test_select_one_question_with_identical_choice_name(self): }, ], } - self.maxDiff = None - self.assertEqual(survey.to_json_dict(), expected_dict) + self.assertEqual(expected_dict, survey.to_json_dict()) def test_loop(self): survey = utils.create_survey_from_fixture("loop", filetype=FIXTURE_FILETYPE) @@ -351,8 +346,7 @@ def test_loop(self): }, ], } - self.maxDiff = None - self.assertEqual(survey.to_json_dict(), expected_dict) + self.assertEqual(expected_dict, survey.to_json_dict()) def test_sms_columns(self): survey = utils.create_survey_from_fixture("sms_info", filetype=FIXTURE_FILETYPE) @@ -502,7 +496,7 @@ def test_sms_columns(self): ], }, } - self.assertEqual(survey.to_json_dict(), expected_dict) + self.assertEqual(expected_dict, survey.to_json_dict()) def test_style_column(self): survey = utils.create_survey_from_fixture( @@ -541,7 +535,7 @@ def test_style_column(self): "title": "My Survey", "type": "survey", } - self.assertEqual(survey.to_json_dict(), expected_dict) + self.assertEqual(expected_dict, survey.to_json_dict()) STRIP_NS_FROM_TAG_RE = re.compile(r"\{.+\}") diff --git a/tests/test_choices_sheet.py b/tests/test_choices_sheet.py index 084b9b567..6ace48325 100644 --- a/tests/test_choices_sheet.py +++ b/tests/test_choices_sheet.py @@ -121,14 +121,14 @@ def test_choices_extra_columns_output_order_matches_xlsform(self): xml__xpath_match=[ """ /h:html/h:head/x:model/x:instance[@id='choices']/x:root/x:item[ - ./x:name = ./x:*[position() = 1 and text() = '1'] - and ./x:geometry = ./x:*[position() = 2 and text() = '46.5841618 7.0801379 0 0'] + ./x:name = ./x:*[text() = '1'] + and ./x:geometry = ./x:*[text() = '46.5841618 7.0801379 0 0'] ] """, """ /h:html/h:head/x:model/x:instance[@id='choices']/x:root/x:item[ - ./x:name = ./x:*[position() = 1 and text() = '2'] - and ./x:geometry = ./x:*[position() = 2 and text() = '35.8805082 76.515057 0 0'] + ./x:name = ./x:*[text() = '2'] + and ./x:geometry = ./x:*[text() = '35.8805082 76.515057 0 0'] ] """, ], diff --git a/tests/test_fieldlist_labels.py b/tests/test_fieldlist_labels.py index 469fd3a45..a824791de 100644 --- a/tests/test_fieldlist_labels.py +++ b/tests/test_fieldlist_labels.py @@ -44,6 +44,13 @@ def test_unlabeled_group_fieldlist(self): | | end_group | | | | """, warnings_count=0, + xml__xpath_match=[ + """ + /h:html/h:body/x:group[ + @ref = '/test_name/my-group' and @appearance='field-list' + ] + """ + ], ) def test_unlabeled_group_fieldlist_alternate_syntax(self): diff --git a/tests/test_fields.py b/tests/test_fields.py index 555915352..a3d679aac 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -80,7 +80,6 @@ def test_multiple_duplicate_choices_without_setting(self): vc.INVALID_DUPLICATE.format(row=3), vc.INVALID_DUPLICATE.format(row=5), ], - debug=True, ) def test_duplicate_choices_with_setting_not_set_to_yes(self): diff --git a/tests/test_groups.py b/tests/test_groups.py index 1d8d6efec..db44fb2b3 100644 --- a/tests/test_groups.py +++ b/tests/test_groups.py @@ -50,3 +50,24 @@ def test_group_intent(self): '' # nopep8 ], ) + + def test_group_relevant_included_in_bind(self): + """Should find the group relevance expression in the group binding.""" + md = """ + | survey | + | | type | name | label | relevant | + | | integer | q1 | Q1 | | + | | begin group | g1 | G1 | ${q1} = 1 | + | | text | q2 | Q2 | | + | | end group | | | | + """ + self.assertPyxformXform( + md=md, + xml__xpath_match=[ + """ + /h:html/h:head/x:model/x:bind[ + @nodeset = '/test_name/g1' and @relevant=' /test_name/q1 = 1' + ] + """ + ], + ) diff --git a/tests/test_image_app_parameter.py b/tests/test_image_app_parameter.py index 8d65db32f..3d8006e49 100644 --- a/tests/test_image_app_parameter.py +++ b/tests/test_image_app_parameter.py @@ -88,7 +88,7 @@ def test_ignoring_invalid_android_package_name_with_not_supported_appearances( name="data", md=md.format(case=case), xml__xpath_match=[ - "/h:html/h:body/x:upload[not(@intent) and @mediatype='image/*' and @ref='/data/my_image']" + f"/h:html/h:body/x:upload[not(@intent) and @mediatype='image/*' and @ref='/data/my_image' and @appearance='{case}']" ], ) @@ -105,7 +105,7 @@ def test_ignoring_android_package_name_in_image_with_not_supported_appearances(s name="data", md=md.format(case=case), xml__xpath_match=[ - "/h:html/h:body/x:upload[not(@intent) and @mediatype='image/*' and @ref='/data/my_image']" + f"/h:html/h:body/x:upload[not(@intent) and @mediatype='image/*' and @ref='/data/my_image' and @appearance='{case}']" ], ) diff --git a/tests/test_j2x_creation.py b/tests/test_j2x_creation.py index c0d7b41cd..f4b69989f 100644 --- a/tests/test_j2x_creation.py +++ b/tests/test_j2x_creation.py @@ -10,42 +10,21 @@ class Json2XformVerboseSurveyCreationTests(TestCase): - def test_survey_can_be_created_in_a_verbose_manner(self): - s = Survey() - s.name = "simple_survey" - - q = MultipleChoiceQuestion() - q.name = "cow_color" - q.type = "select one" - - q.add_choice(label="Green", name="green") - s.add_child(q) - - expected_dict = { - "name": "simple_survey", - "children": [ - { - "name": "cow_color", - "type": "select one", - "children": [{"label": "Green", "name": "green"}], - } - ], - } - self.maxDiff = None - self.assertEqual(s.to_json_dict(), expected_dict) - def test_survey_can_be_created_in_a_slightly_less_verbose_manner(self): option_dict_array = [ {"name": "red", "label": "Red"}, {"name": "blue", "label": "Blue"}, ] - q = MultipleChoiceQuestion(name="Favorite_Color", choices=option_dict_array) - q.type = "select one" - s = Survey(name="Roses_are_Red", children=[q]) + q = MultipleChoiceQuestion( + name="Favorite_Color", type="select one", choices=option_dict_array + ) + s = Survey(name="Roses_are_Red") + s.add_child(q) expected_dict = { "name": "Roses_are_Red", + "type": "survey", "children": [ { "name": "Favorite_Color", @@ -58,36 +37,38 @@ def test_survey_can_be_created_in_a_slightly_less_verbose_manner(self): ], } - self.assertEqual(s.to_json_dict(), expected_dict) + self.assertEqual(expected_dict, s.to_json_dict()) - def allow_surveys_with_comment_rows(self): + def test_allow_surveys_with_comment_rows(self): """assume that a survey with rows that don't have name, type, or label headings raise warning only""" path = utils.path_to_text_fixture("allow_comment_rows_test.xls") survey = create_survey_from_xls(path) expected_dict = { - "default_language": "default", - "id_string": "allow_comment_rows_test", "children": [ { - "name": "farmer_name", "label": {"English": "First and last name of farmer"}, + "name": "farmer_name", "type": "text", - } + }, + { + "children": [ + { + "bind": {"jr:preload": "uid", "readonly": "true()"}, + "name": "instanceID", + "type": "calculate", + } + ], + "control": {"bodyless": True}, + "name": "meta", + "type": "group", + }, ], - "name": "allow_comment_rows_test", - "_translations": { - "English": { - "/allow_comment_rows_test/farmer_name:label": { - "long": "First and last name of farmer" - } - } - }, + "default_language": "default", + "id_string": "allow_comment_rows_test", + "name": "data", + "sms_keyword": "allow_comment_rows_test", "title": "allow_comment_rows_test", - "_xpath": { - "allow_comment_rows_test": "/allow_comment_rows_test", - "farmer_name": "/allow_comment_rows_test/farmer_name", - }, "type": "survey", } - self.assertEqual(survey.to_json_dict(), expected_dict) + self.assertEqual(expected_dict, survey.to_json_dict()) diff --git a/tests/test_j2x_question.py b/tests/test_j2x_question.py index 8e3329957..c2229d90e 100644 --- a/tests/test_j2x_question.py +++ b/tests/test_j2x_question.py @@ -2,6 +2,7 @@ Testing creation of Surveys using verbose methods """ +from collections.abc import Generator from unittest import TestCase from pyxform import Survey @@ -19,6 +20,8 @@ def ctw(control): """ if isinstance(control, list) and len(control) == 1: control = control[0] + elif isinstance(control, Generator): + control = next(control) return control.toxml() @@ -55,10 +58,12 @@ def test_question_type_string(self): ) self.s.add_child(q) - self.assertEqual(ctw(q.xml_control()), expected_string_control_xml) + self.assertEqual(ctw(q.xml_control(survey=self.s)), expected_string_control_xml) if TESTING_BINDINGS: - self.assertEqual(ctw(q.xml_bindings()), expected_string_binding_xml) + self.assertEqual( + ctw(q.xml_bindings(survey=self.s)), expected_string_binding_xml + ) def test_select_one_question_multilingual(self): """ @@ -86,10 +91,14 @@ def test_select_one_question_multilingual(self): q = create_survey_element_from_dict(simple_select_one_json) self.s.add_child(q) - self.assertEqual(ctw(q.xml_control()), expected_select_one_control_xml) + self.assertEqual( + ctw(q.xml_control(survey=self.s)), expected_select_one_control_xml + ) if TESTING_BINDINGS: - self.assertEqual(ctw(q.xml_bindings()), expected_select_one_binding_xml) + self.assertEqual( + ctw(q.xml_bindings(survey=self.s)), expected_select_one_binding_xml + ) def test_simple_integer_question_type_multilingual(self): """ @@ -114,10 +123,12 @@ def test_simple_integer_question_type_multilingual(self): self.s.add_child(q) - self.assertEqual(ctw(q.xml_control()), expected_integer_control_xml) + self.assertEqual(ctw(q.xml_control(survey=self.s)), expected_integer_control_xml) if TESTING_BINDINGS: - self.assertEqual(ctw(q.xml_bindings()), expected_integer_binding_xml) + self.assertEqual( + ctw(q.xml_bindings(survey=self.s)), expected_integer_binding_xml + ) def test_simple_date_question_type_multilingual(self): """ @@ -140,10 +151,12 @@ def test_simple_date_question_type_multilingual(self): q = create_survey_element_from_dict(simple_date_question) self.s.add_child(q) - self.assertEqual(ctw(q.xml_control()), expected_date_control_xml) + self.assertEqual(ctw(q.xml_control(survey=self.s)), expected_date_control_xml) if TESTING_BINDINGS: - self.assertEqual(ctw(q.xml_bindings()), expected_date_binding_xml) + self.assertEqual( + ctw(q.xml_bindings(survey=self.s)), expected_date_binding_xml + ) def test_simple_phone_number_question_type_multilingual(self): """ @@ -159,7 +172,7 @@ def test_simple_phone_number_question_type_multilingual(self): self.s.add_child(q) # Inspect XML Control - observed = q.xml_control() + observed = q.xml_control(survey=self.s) self.assertEqual("input", observed.nodeName) self.assertEqual("/test/phone_number_q", observed.attributes["ref"].nodeValue) observed_label = observed.childNodes[0] @@ -178,7 +191,8 @@ def test_simple_phone_number_question_type_multilingual(self): "type": "string", "constraint": r"regex(., '^\d*$')", } - observed = {k: v for k, v in q.xml_bindings()[0].attributes.items()} # noqa: C416 + binding = next(q.xml_bindings(survey=self.s)) + observed = {k: v for k, v in binding.attributes.items()} # noqa: C416 self.assertDictEqual(expected, observed) def test_simple_select_all_question_multilingual(self): @@ -206,10 +220,14 @@ def test_simple_select_all_question_multilingual(self): q = create_survey_element_from_dict(simple_select_all_question) self.s.add_child(q) - self.assertEqual(ctw(q.xml_control()), expected_select_all_control_xml) + self.assertEqual( + ctw(q.xml_control(survey=self.s)), expected_select_all_control_xml + ) if TESTING_BINDINGS: - self.assertEqual(ctw(q.xml_bindings()), expected_select_all_binding_xml) + self.assertEqual( + ctw(q.xml_bindings(survey=self.s)), expected_select_all_binding_xml + ) def test_simple_decimal_question_multilingual(self): """ @@ -232,7 +250,9 @@ def test_simple_decimal_question_multilingual(self): q = create_survey_element_from_dict(simple_decimal_question) self.s.add_child(q) - self.assertEqual(ctw(q.xml_control()), expected_decimal_control_xml) + self.assertEqual(ctw(q.xml_control(survey=self.s)), expected_decimal_control_xml) if TESTING_BINDINGS: - self.assertEqual(ctw(q.xml_bindings()), expected_decimal_binding_xml) + self.assertEqual( + ctw(q.xml_bindings(survey=self.s)), expected_decimal_binding_xml + ) diff --git a/tests/test_j2x_xform_build_preparation.py b/tests/test_j2x_xform_build_preparation.py index 062a2280f..c83d7440e 100644 --- a/tests/test_j2x_xform_build_preparation.py +++ b/tests/test_j2x_xform_build_preparation.py @@ -15,10 +15,10 @@ def test_dictionary_consolidates_duplicate_entries(self): ] first_yesno_question = MultipleChoiceQuestion( - name="yn_q1", options=yes_or_no_dict_array, type="select one" + name="yn_q1", choices=yes_or_no_dict_array, type="select one" ) second_yesno_question = MultipleChoiceQuestion( - name="yn_q2", options=yes_or_no_dict_array, type="select one" + name="yn_q2", choices=yes_or_no_dict_array, type="select one" ) s = Survey(name="yes_or_no_tests") diff --git a/tests/test_survey.py b/tests/test_survey.py index cd2bec186..d38f3f82e 100644 --- a/tests/test_survey.py +++ b/tests/test_survey.py @@ -49,3 +49,19 @@ def test_many_xpath_references_do_not_hit_64_recursion_limit__many_to_many(self) n="\n".join(tmpl_n.format(i) for i in range(1, 250)), ), ) + + def test_autoplay_attribute_added_to_question_body_control(self): + """Should add the autoplay attribute when specified for a question.""" + md = """ + | survey | + | | type | name | label | audio | autoplay | + | | text | feel | Song feel? | amazing.mp3 | audio | + """ + self.assertPyxformXform( + md=md, + xml__xpath_match=[ + """ + /h:html/h:body/x:input[@ref='/test_name/feel' and @autoplay='audio'] + """ + ], + ) diff --git a/tests/test_translations.py b/tests/test_translations.py index 1f891ef3d..b644dde6e 100644 --- a/tests/test_translations.py +++ b/tests/test_translations.py @@ -1711,9 +1711,9 @@ def test_specify_other__with_translations_only__missing_first_translation(self): """Should add an "other" choice to the itemset instance and an itext label.""" # xls2json validation would raise an error if a choice has no label at all. md = """ - | survey | | | | | | - | | type | name | label | label::eng | label:fr | - | | select_one c1 or_other | q1 | Question 1 | Question A | QA fr | + | survey | | | | | | + | | type | name | label | label::eng | label::fr | + | | select_one c1 or_other | q1 | Question 1 | Question A | QA fr | | choices | | | | | | | | list name | name | label | label::eng | label::fr | | | c1 | na | la | la-e | la-f | diff --git a/tests/xform_test_case/test_xform_conversion.py b/tests/xform_test_case/test_xform_conversion.py index 95a0d9eb4..8e67451e7 100644 --- a/tests/xform_test_case/test_xform_conversion.py +++ b/tests/xform_test_case/test_xform_conversion.py @@ -35,8 +35,8 @@ def test_conversion_vs_expected(self): ) xlsform = Path(self.path_to_excel_file) if set_name: - result = convert(xlsform=xlsform, warnings=[], form_name=xlsform.stem) + result = convert(xlsform=xlsform, form_name=xlsform.stem) else: - result = convert(xlsform=xlsform, warnings=[]) + result = convert(xlsform=xlsform) with open(expected_output_path, encoding="utf-8") as expected: self.assertXFormEqual(expected.read(), result.xform)