diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index b3f475621..ae7362549 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1 +1 @@ -__version__ = "0.1.2" +__version__ = "0.1.3" diff --git a/openedx_tagging/core/tagging/import_export/__init__.py b/openedx_tagging/core/tagging/import_export/__init__.py new file mode 100644 index 000000000..55bcdaebd --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/__init__.py @@ -0,0 +1 @@ +from .parsers import ParserFormat diff --git a/openedx_tagging/core/tagging/import_export/actions.py b/openedx_tagging/core/tagging/import_export/actions.py new file mode 100644 index 000000000..da757cf4b --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/actions.py @@ -0,0 +1,443 @@ +""" +Actions for import tags +""" +from typing import List + +from django.utils.translation import gettext_lazy as _ + +from ..models import Taxonomy, Tag +from .exceptions import ImportActionError, ImportActionConflict + + +class ImportAction: + """ + Base class to create actions + + Each action is a simple operation to be performed on the database. + There are no compound actions or actions that have to do with each other. + + To create an Action you need to implement the following: + + Given a TagItem, the actions to be performed must be deduced + by comparing with the tag on the database. + Ex. The create action is inferred if the tag does not exist in the database. + This check is done in `applies_for` + + Then each action validates if the change is consistent with the database + or with previous actions. + Ex. Verify that when creating a tag, there is not a previous creation action + that has the same tag_id. + This checks is done in `validate` + + Then the actions are executed. Ex. Create the tag on the database + This is done in `execute` + """ + + name = "import_action" + + def __init__(self, taxonomy: Taxonomy, tag, index: int): + self.taxonomy = taxonomy + self.tag = tag + self.index = index + + def __repr__(self): + return str(_(f"Action {self.name} (index={self.index},id={self.tag.id})")) + + def __str__(self): + return self.__repr__() + + @classmethod + def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: + """ + Implement this to meet the conditions that a `TagItem` needs + to have for this action. If this function returns `True` for `tag` + then the action is created. + """ + raise NotImplementedError + + def validate(self, indexed_actions) -> List[ImportActionError]: + """ + Implement this to find inconsistencies with tags in the + database or with previous actions. + """ + raise NotImplementedError + + def execute(self): + """ + Implement this to execute the action. + """ + raise NotImplementedError + + def _get_tag(self): + """ + Returns the respective tag of this actions + """ + return self.taxonomy.tag_set.get(external_id=self.tag.id) + + def _search_action( + self, + indexed_actions: dict, + action_name: str, + attr: str, + search_value: str, + ): + """ + Use this function to find and action using an `attr` of `TagItem` + """ + for action in indexed_actions[action_name]: + if search_value == getattr(action.tag, attr): + return action + + return None + + def _validate_parent(self, indexed_actions) -> ImportActionError: + """ + Helper method to validate that the parent tag has already been defined. + """ + try: + # Validates that the parent exists on the taxonomy + self.taxonomy.tag_set.get(external_id=self.tag.parent_id) + except Tag.DoesNotExist: + # Or if the parent is created on previous actions + if not self._search_action( + indexed_actions, CreateTag.name, "id", self.tag.parent_id + ): + return ImportActionError( + action=self, + tag_id=self.tag.id, + message=_( + f"Unknown parent tag ({self.tag.parent_id}). " + "You need to add parent before the child in your file." + ), + ) + + def _validate_value(self, indexed_actions): + """ + Check for value duplicates in the models and in previous create/rename + actions + """ + try: + # Validates if exists a tag with the same value on the Taxonomy + taxonomy_tag = self.taxonomy.tag_set.get(value=self.tag.value) + return ImportActionError( + action=self, + tag_id=self.tag.id, + message=_( + f"Duplicated tag value with tag in database (external_id={taxonomy_tag.external_id})." + ), + ) + except Tag.DoesNotExist: + # Validates value duplication on create actions + action = self._search_action( + indexed_actions, + CreateTag.name, + "value", + self.tag.value, + ) + + if not action: + # Validates value duplication on rename actions + action = self._search_action( + indexed_actions, + RenameTag.name, + "value", + self.tag.value, + ) + + if action: + return ImportActionConflict( + action=self, + tag_id=self.tag.id, + conflict_action_index=action.index, + message=_("Duplicated tag value."), + ) + + +class CreateTag(ImportAction): + """ + Action for create a Tag + + Action created if the tag doesn't exist on the database + + Validations: + - Id duplicates with previous create actions. + - Value duplicates with tags on the database. + - Value duplicates with previous create and rename actions. + - Parent validation. If the parent is in the database or created + in previous actions. + """ + + name = "create" + + def __str__(self): + return str( + _( + "Create a new tag with values " + f"(external_id={self.tag.id}, value={self.tag.value}, " + f"parent_id={self.tag.parent_id})." + ) + ) + + @classmethod + def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: + """ + This action applies whenever the tag does not exist + """ + try: + taxonomy.tag_set.get(external_id=tag.id) + return False + except Tag.DoesNotExist: + return True + + def _validate_id(self, indexed_actions): + """ + Check for id duplicates in previous create actions + """ + action = self._search_action(indexed_actions, self.name, "id", self.tag.id) + if action: + return ImportActionConflict( + action=self, + tag_id=self.tag.id, + conflict_action_index=action.index, + message=_("Duplicated external_id tag."), + ) + + def validate(self, indexed_actions) -> List[ImportActionError]: + """ + Validates the creation action + """ + errors = [] + + # Duplicate id validation with previous create actions + error = self._validate_id(indexed_actions) + if error: + errors.append(error) + + # Duplicate value validation + error = self._validate_value(indexed_actions) + if error: + errors.append(error) + + # Parent validation + if self.tag.parent_id: + error = self._validate_parent(indexed_actions) + if error: + errors.append(error) + + return errors + + def execute(self): + """ + Creates a Tag + """ + parent = None + if self.tag.parent_id: + parent = self.taxonomy.tag_set.get(external_id=self.tag.parent_id) + taxonomy_tag = Tag( + taxonomy=self.taxonomy, + parent=parent, + value=self.tag.value, + external_id=self.tag.id, + ) + taxonomy_tag.save() + + +class UpdateParentTag(ImportAction): + """ + Action for update the parent of a Tag + + Action created if there is a change on the parent + + Validations: + - Parent validation. If the parent is in the database + or created in previous actions. + """ + + name = "update_parent" + + def __str__(self): + taxonomy_tag = self._get_tag() + if not taxonomy_tag.parent: + from_str = _("from empty parent") + else: + from_str = _(f"from parent (external_id={taxonomy_tag.parent.external_id})") + + return str( + _( + f"Update the parent of tag (external_id={taxonomy_tag.external_id}) " + f"{from_str} to parent (external_id={self.tag.parent_id})." + ) + ) + + @classmethod + def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: + """ + This action applies whenever there is a change on the parent + """ + try: + taxonomy_tag = taxonomy.tag_set.get(external_id=tag.id) + return ( + taxonomy_tag.parent is not None + and taxonomy_tag.parent.external_id != tag.parent_id + ) or (taxonomy_tag.parent is None and tag.parent_id is not None) + except Tag.DoesNotExist: + return False + + def validate(self, indexed_actions) -> List[ImportActionError]: + """ + Validates the update parent action + """ + errors = [] + + # Parent validation + if self.tag.parent_id: + error = self._validate_parent(indexed_actions) + if error: + errors.append(error) + + return errors + + def execute(self): + """ + Updates the parent of a tag + """ + taxonomy_tag = self._get_tag() + parent = None + if self.tag.parent_id: + parent = self.taxonomy.tag_set.get(external_id=self.tag.parent_id) + taxonomy_tag.parent = parent + taxonomy_tag.save() + + +class RenameTag(ImportAction): + """ + Action for rename a Tag + + Action created if there is a change on the tag value + + Validations: + - Value duplicates with tags on the database. + - Value duplicates with previous create and rename actions. + """ + + name = "rename" + + def __str__(self): + taxonomy_tag = self._get_tag() + return str( + _( + f"Rename tag value of tag (external_id={taxonomy_tag.external_id}) " + f"from '{taxonomy_tag.value}' to '{self.tag.value}'" + ) + ) + + @classmethod + def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: + """ + This action applies whenever there is a change on the tag value + """ + try: + taxonomy_tag = taxonomy.tag_set.get(external_id=tag.id) + return taxonomy_tag.value != tag.value + except Tag.DoesNotExist: + return False + + def validate(self, indexed_actions) -> List[ImportActionError]: + """ + Validates the rename action + """ + errors = [] + + # Duplicate value validation + error = self._validate_value(indexed_actions) + if error: + errors.append(error) + + return errors + + def execute(self): + """ + Rename a tag + """ + taxonomy_tag = self._get_tag() + taxonomy_tag.value = self.tag.value + taxonomy_tag.save() + + +class DeleteTag(ImportAction): + """ + Action for delete a Tag + + Action created if the action of the tag is 'delete' + + Does not require validations + """ + + def __str__(self): + taxonomy_tag = self._get_tag() + return str(_(f"Delete tag (external_id={taxonomy_tag.external_id})")) + + name = "delete" + + @classmethod + def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: + """ + This action is an exception. + These actions are created in `TagImportPlan.generate_actions` if `replace=True` + """ + return False + + def validate(self, indexed_actions) -> List[ImportActionError]: + """ + No validations necessary + """ + # TODO: Will it be necessary to check if this tag has children? + return [] + + def execute(self): + """ + Delete a tag + """ + taxonomy_tag = self._get_tag() + taxonomy_tag.delete() + + +class WithoutChanges(ImportAction): + """ + Action when there is no change on the Tag + + Does not require validations + """ + + name = "without_changes" + + def __str__(self): + return str(_(f"No changes needed for tag (external_id={self.tag.id})")) + + @classmethod + def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: + """ + No validations necessary + """ + return False + + def validate(self, indexed_actions) -> List[ImportActionError]: + """ + No validations necessary + """ + return [] + + def execute(self): + """ + Do nothing + """ + + +# Register actions here in the order in which you want to check. +available_actions = [ + UpdateParentTag, + RenameTag, + CreateTag, + DeleteTag, + WithoutChanges, +] diff --git a/openedx_tagging/core/tagging/import_export/api.py b/openedx_tagging/core/tagging/import_export/api.py new file mode 100644 index 000000000..4e26854ed --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/api.py @@ -0,0 +1,188 @@ +""" +Import/export API functions + +A modular implementation has been followed for both functionalities. + +Import +------------ + +In this functionality we have the following pipeline with the following classes: + +Parser.parse_import() -> TagImportPlan.generate_actions() -> [ImportActions] +-> TagImportPlan.plan() -> TagImportPlan.execute() + +Parsers are in charge of reading the input file, +making the respective verifications of its format and returning a list of TagItems. +You need to create parser for each format that the system will accept. +For more information see parsers.py + +TagImportPlan receives a list of TagItems. With this, it generates each +action that will be executed in the import. +Each Action are in charge of verifying and executing specific +and simple operations in the database, such as creating or rename tag. +For more information see actions.py + +In each action it is verified if there are no errors or inconsistencies +with taxonomy tags or with previous actions. +In the end, TagImportPlan contains all actions and possible errors. +You can run `plan()` to see the actions and errors or you can run `execute()` +to execute each action. + +Export +---------- + +The export only uses Parsers. Calls the respective function and +returns a string with the data. + + +TODO for next versions +--------- +- Function to force clean the status of an import task, or a way to avoid + a lock: a task not in SUCCESS or ERROR due to something unexpected + (ex. server crash) +- Join/reduce actions on TagImportPlan. See `generate_actions()` +""" +from io import BytesIO + +from django.utils.translation import gettext_lazy as _ + +from ..models import Taxonomy, TagImportTask, TagImportTaskState +from .parsers import get_parser, ParserFormat +from .import_plan import TagImportPlan, TagImportTask + + +def import_tags( + taxonomy: Taxonomy, + file: BytesIO, + parser_format: ParserFormat, + replace=False, +) -> bool: + """ + Execute the necessary actions to import the tags from `file` + + You can read the docstring of the top for more info about the + modular architecture. + + It creates an TagImportTask to keep logs of the execution + of each import step and the current status. + There can only be one task in progress at a time per taxonomy + + Set `replace` to True to delete all not readed Tag of the given taxonomy. + Ex. Given a taxonomy with `tag_1`, `tag_2` and `tag_3`. If there is only `tag_1` + in the file (regardless of action), then `tag_2` and `tag_3` will be deleted + if `replace=True` + """ + _import_export_validations(taxonomy) + + # Checks that exists only one task import in progress at a time per taxonomy + if not _check_unique_import_task(taxonomy): + raise ValueError( + _( + "There is an import task running. " + "Only one task per taxonomy can be created at a time." + ) + ) + + # Creating import task + task = TagImportTask.create(taxonomy) + + try: + # Get the parser and parse the file + task.log_parser_start() + parser = get_parser(parser_format) + tags, errors = parser.parse_import(file) + + # Check if there are errors in the parse + if errors: + task.handle_parser_errors(errors) + return False + + task.log_parser_end() + + # Generate actions + task.log_start_planning() + tag_import_plan = TagImportPlan(taxonomy) + tag_import_plan.generate_actions(tags, replace) + task.log_plan(tag_import_plan) + + if tag_import_plan.errors: + task.handle_plan_errors() + return False + + task.log_start_execute() + tag_import_plan.execute(task) + task.end_success() + return True + except Exception as exception: + # Log any exception + task.log_exception(exception) + return False + + +def get_last_import_status(taxonomy: Taxonomy) -> TagImportTaskState: + """ + Get status of the last import task of the given taxonomy + """ + task = _get_last_import_task(taxonomy) + return task.status + + +def get_last_import_log(taxonomy: Taxonomy) -> str: + """ + Get logs of the last import task of the given taxonomy + """ + task = _get_last_import_task(taxonomy) + return task.log + + +def export_tags(taxonomy: Taxonomy, output_format: ParserFormat) -> str: + """ + Returns a string with all tag data of the given taxonomy + """ + _import_export_validations(taxonomy) + parser = get_parser(output_format) + return parser.export(taxonomy) + + +def _check_unique_import_task(taxonomy: Taxonomy) -> bool: + """ + Verifies if there is another in progress import task for the + given taxonomy + """ + last_task = _get_last_import_task(taxonomy) + if not last_task: + return True + return ( + last_task.status == TagImportTaskState.SUCCESS.value + or last_task.status == TagImportTaskState.ERROR.value + ) + + +def _get_last_import_task(taxonomy: Taxonomy) -> TagImportTask: + """ + Get the last import task for the given taxonomy + """ + return ( + TagImportTask.objects.filter(taxonomy=taxonomy) + .order_by("-creation_date") + .first() + ) + + +def _import_export_validations(taxonomy: Taxonomy): + """ + Validates if the taxonomy is allowed to import or export tags + """ + taxonomy = taxonomy.cast() + if taxonomy.allow_free_text: + raise NotImplementedError( + _( + f"Import/export for free-form taxonomies will be implemented in the future." + ) + ) + if taxonomy.system_defined: + raise ValueError( + _( + f"Invalid taxonomy ({taxonomy.id}): You cannot import/export a system-defined taxonomy." + ) + ) diff --git a/openedx_tagging/core/tagging/import_export/exceptions.py b/openedx_tagging/core/tagging/import_export/exceptions.py new file mode 100644 index 000000000..2330cae6a --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/exceptions.py @@ -0,0 +1,98 @@ +""" +Exceptions for tag import/export actions +""" +from django.utils.translation import gettext_lazy as _ + + +class TagImportError(Exception): + """ + Base exception for import + """ + + def __init__(self, message: str = "", **kargs): + self.message = message + + def __str__(self): + return str(self.message) + + def __repr__(self): + return f"{self.__class__.__name__}({str(self)})" + + +class TagParserError(TagImportError): + """ + Base exception for parsers + """ + + def __init__(self, tag, **kargs): + self.message = _(f"Import parser error on {tag}") + + +class ImportActionError(TagImportError): + """ + Base exception for actions + """ + + def __init__(self, action: str, tag_id: str, message: str, **kargs): + self.message = _( + f"Action error in '{action.name}' (#{action.index}): {message}" + ) + + +class ImportActionConflict(ImportActionError): + """ + Exception used when exists a conflict between actions + """ + + def __init__( + self, + action: str, + tag_id: str, + conflict_action_index: int, + message: str, + **kargs, + ): + self.message = _( + f"Conflict with '{action.name}' (#{action.index}) " + f"and action #{conflict_action_index}: {message}" + ) + + +class InvalidFormat(TagParserError): + """ + Exception used when there is an error with the format + """ + + def __init__(self, tag: dict, format: str, message: str, **kargs): + self.tag = tag + self.message = _(f"Invalid '{format}' format: {message}") + + +class FieldJSONError(TagParserError): + """ + Exception used when missing a required field on the .json + """ + + def __init__(self, tag, field, **kargs): + self.tag = tag + self.message = _(f"Missing '{field}' field on {tag}") + + +class EmptyJSONField(TagParserError): + """ + Exception used when a required field is empty on the .json + """ + + def __init__(self, tag, field, **kargs): + self.tag = tag + self.message = _(f"Empty '{field}' field on {tag}") + + +class EmptyCSVField(TagParserError): + """ + Exception used when a required field is empty on the .csv + """ + + def __init__(self, tag, field, row, **kargs): + self.tag = tag + self.message = _(f"Empty '{field}' field on the row {row}") diff --git a/openedx_tagging/core/tagging/import_export/import_plan.py b/openedx_tagging/core/tagging/import_export/import_plan.py new file mode 100644 index 000000000..3af3597d6 --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/import_plan.py @@ -0,0 +1,198 @@ +""" +Classes and functions to create an import plan and execution. +""" +from attrs import define +from typing import List, Optional + +from django.db import transaction + +from ..models import Taxonomy, TagImportTask +from .actions import ( + DeleteTag, + ImportAction, + UpdateParentTag, + WithoutChanges, + available_actions, +) +from .exceptions import ImportActionError + + +@define +class TagItem: + """ + Tag representation on the tag import plan + """ + + id: str + value: str + index: Optional[int] = 0 + parent_id: Optional[str] = None + + +class TagImportPlan: + """ + Class with functions to build an import plan and excute the plan + """ + + actions: List[ImportAction] + errors: List[ImportActionError] + indexed_actions: dict + actions_dict: dict + taxonomy: Taxonomy + + def __init__(self, taxonomy: Taxonomy): + self.actions = [] + self.errors = [] + self.taxonomy = taxonomy + self.actions_dict = {} + self._init_indexed_actions() + + def _init_indexed_actions(self): + """ + Initialize the `indexed_actions` dict + """ + self.indexed_actions = {} + for action in available_actions: + self.indexed_actions[action.name] = [] + + def _build_action(self, action_cls, tag: TagItem): + """ + Build an action with `tag`. + + Run action validation and adds the errors to the errors lists + Add to the action list and the indexed actions + """ + action = action_cls(self.taxonomy, tag, len(self.actions) + 1) + + # We validate if there are no inconsistencies when executing this action + self.errors.extend(action.validate(self.indexed_actions)) + + # Add action + self.actions.append(action) + + # Index the actions for search + self.indexed_actions[action.name].append(action) + + def _search_parent_update( + self, + child_external_id, + parent_external_id, + ): + """ + Checks if there is a parent update in a child + """ + for action in self.indexed_actions["update_parent"]: + if ( + child_external_id == action.tag.id + and parent_external_id != action.tag.parent_id + ): + return True + + return False + + def _build_delete_actions(self, tags: dict): + """ + Adds delete actions for `tags` + """ + for tag in tags.values(): + for child in tag.children.all(): + # Verify if there is not a parent update before + if not self._search_parent_update(child.external_id, tag.external_id): + # Change parent to avoid delete childs + self._build_action( + UpdateParentTag, + TagItem( + id=child.external_id, + value=child.value, + parent_id=None, + ), + ) + + # Delete action + self._build_action( + DeleteTag, + TagItem( + id=tag.external_id, + value=tag.value, + ), + ) + + def generate_actions( + self, + tags: List[TagItem], + replace=False, + ): + """ + Reads each tag and generates the corresponding actions. + + Validates each action and create respective errors + If `replace` is True, then creates the delete action for tags + that are in the existing taxonomy but not the new tags list. + + TODO: Join/reduce actions. Ex. A tag may have no changes, + but then its parent needs to be updated because its parent is deleted. + Those two actions should be merged. + """ + self.actions.clear() + self.errors.clear() + self._init_indexed_actions() + tags_for_delete = {} + + if replace: + tags_for_delete = { + tag.external_id: tag for tag in self.taxonomy.tag_set.all() + } + + for tag in tags: + has_action = False + + # Check all available actions and add which ones should be executed + for action_cls in available_actions: + if action_cls.applies_for(self.taxonomy, tag): + self._build_action(action_cls, tag) + has_action = True + + if not has_action: + # If it doesn't find an action, a "without changes" is added + self._build_action(WithoutChanges, tag) + + if replace and tag.id in tags_for_delete: + tags_for_delete.pop(tag.id) + + if replace: + # Delete all not readed tags + self._build_delete_actions(tags_for_delete) + + def plan(self) -> str: + """ + Returns an string with the plan and errors + """ + result = ( + f"Import plan for {self.taxonomy.name}\n" + "--------------------------------\n" + ) + for action in self.actions: + result += f"#{action.index}: {str(action)}\n" + + if self.errors: + result += "\nOutput errors\n" "--------------------------------\n" + for error in self.errors: + result += f"{str(error)}\n" + + return result + + @transaction.atomic() + def execute(self, task: TagImportTask = None): + """ + Executes each action + + If task is set, creates logs for each action + """ + if self.errors: + return + for action in self.actions: + if task: + task.add_log(f"#{action.index}: {str(action)} [Started]") + action.execute() + if task: + task.add_log("Success") diff --git a/openedx_tagging/core/tagging/import_export/parsers.py b/openedx_tagging/core/tagging/import_export/parsers.py new file mode 100644 index 000000000..79171e7e5 --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/parsers.py @@ -0,0 +1,325 @@ +""" +Parsers to import and export tags +""" +import csv +import json +from enum import Enum +from io import BytesIO, TextIOWrapper, StringIO +from typing import List, Tuple + +from django.utils.translation import gettext_lazy as _ + +from .import_plan import TagItem +from .exceptions import ( + TagParserError, + InvalidFormat, + FieldJSONError, + EmptyJSONField, + EmptyCSVField, +) +from ..models import Taxonomy +from ..api import get_tags + + +class ParserFormat(Enum): + """ + Format of import tags to taxonomies + """ + + JSON = ".json" + CSV = ".csv" + + +class Parser: + """ + Base class to create a parser + + This contains the base functions to convert between + a simple file format like CSV/JSON and a list of TagItems. + It can convert in both directions, for use during import or export. + + If you want to add a new field, you can add it to + `required_fields` or `optional_fields` depending on the field type + + To create a new Parser you need to implement `_load_data` and `_export_data` + """ + + required_fields = ["id", "value"] + optional_fields = ["parent_id"] + + # Set the format associated to the parser + format = None + # We can change the error when is missing a required field + missing_field_error = TagParserError + # We can change the error when a required field is empty + empty_field_error = TagParserError + # We can change the initial row/index + inital_row = 1 + + @classmethod + def parse_import(cls, file: BytesIO) -> Tuple[List[TagItem], List[TagParserError]]: + """ + Parse tags in file an returns tags ready for use in TagImportPlan + + Top function that calls `_load_data` and `_parse_tags`. + Handle errors returned by both functions. + """ + try: + tags_data, load_errors = cls._load_data(file) + if load_errors: + return [], load_errors + except Exception as error: + raise error + finally: + file.close() + + return cls._parse_tags(tags_data) + + @classmethod + def export(cls, taxonomy: Taxonomy) -> str: + """ + Returns all tags in taxonomy. + The output file can be used to recreate the taxonomy with `parse_import` + """ + tags = cls._load_tags_for_export(taxonomy) + return cls._export_data(tags, taxonomy) + + @classmethod + def _load_data(cls, file: BytesIO) -> Tuple[List[dict], List[TagParserError]]: + """ + Each parser implements this function according to its format. + This function reads the file and returns a list with the values of each tag. + + This function does not do field validations, it only does validations of the + file structure in the parser format. Field validations are done in `_parse_tags` + """ + raise NotImplementedError + + @classmethod + def _export_data(cls, tags: List[dict], taxonomy: Taxonomy) -> str: + """ + Each parser implements this function according to its format. + Returns a string with tags data in the parser format. + Can use `taxonomy` to export taxonomy metadata. + + It must be implemented in such a way that the output of + this function works with _load_data + """ + raise NotImplementedError + + @classmethod + def _parse_tags(cls, tags_data: dict) -> Tuple[List[TagItem], List[TagParserError]]: + """ + Validate the required fields of each tag. + + Return a list of TagItems + and a list of validation errors. + """ + tags = [] + errors = [] + row = cls.inital_row + for tag in tags_data: + has_error = False + + # Verify the required fields + for req_field in cls.required_fields: + if req_field not in tag: + # Verify if the field exists + errors.append( + cls.missing_field_error( + tag, + field=req_field, + row=row, + ) + ) + has_error = True + elif not tag.get(req_field): + # Verify if the value of the field is not empty + errors.append( + cls.empty_field_error( + tag, + field=req_field, + row=row, + ) + ) + has_error = True + + tag["index"] = row + row += 1 + + # Skip parse if there is an error + if has_error: + continue + + # Updating any empty optional field to None + for opt_field in cls.optional_fields: + if opt_field in tag and not tag.get(opt_field): + tag[opt_field] = None + + tags.append(TagItem(**tag)) + + return tags, errors + + @classmethod + def _load_tags_for_export(cls, taxonomy: Taxonomy) -> List[dict]: + """ + Returns a list of taxonomy's tags in the form of a dictionary + with required and optional fields + + The tags are ordered by hierarchy, first, parents and then children. + `get_tags` is in charge of returning this in a hierarchical way. + """ + tags = get_tags(taxonomy) + result = [] + for tag in tags: + result_tag = { + "id": tag.external_id or tag.id, + "value": tag.value, + } + if tag.parent: + result_tag["parent_id"] = tag.parent.external_id or tag.parent.id + result.append(result_tag) + return result + + +class JSONParser(Parser): + """ + Parser used with .json files + + Valid file: + ``` + { + "tags": [ + { + "id": "tag_1", + "value": "tag 1", + "parent_id": "tag_2", + } + ] + } + ``` + """ + + format = ParserFormat.JSON + missing_field_error = FieldJSONError + empty_field_error = EmptyJSONField + inital_row = 0 + + @classmethod + def _load_data(cls, file: BytesIO) -> Tuple[List[dict], List[TagParserError]]: + """ + Read a .json file and validates the root structure of the json + """ + file.seek(0) + try: + tags_data = json.load(file) + except json.JSONDecodeError as error: + return None, [ + InvalidFormat(tag=None, format=cls.format.value, message=str(error)) + ] + if "tags" not in tags_data: + return None, [ + InvalidFormat( + tag=None, + format=cls.format.value, + message=_("Missing 'tags' field on the .json file"), + ) + ] + + tags_data = tags_data.get("tags") + return tags_data, [] + + @classmethod + def _export_data(cls, tags: List[dict], taxonomy: Taxonomy) -> str: + """ + Export tags and taxonomy metadata in JSON format + """ + json_result = { + "name": taxonomy.name, + "description": taxonomy.description, + "tags": tags, + } + return json.dumps(json_result) + + +class CSVParser(Parser): + """ + Parser used with .csv files + + Valid file: + ``` + id,value,parent_id + tag_1,tag 1, + tag_2,tag 2,tag_1 + ``` + """ + + format = ParserFormat.CSV + empty_field_error = EmptyCSVField + inital_row = 2 + + @classmethod + def _load_data(cls, file: BytesIO) -> Tuple[List[dict], List[TagParserError]]: + """ + Read a .csv file and validates the header fields + """ + file.seek(0) + text_tags = TextIOWrapper(file, encoding="utf-8") + csv_reader = csv.DictReader(text_tags) + header_fields = csv_reader.fieldnames + errors = cls._verify_header(header_fields) + if errors: + return None, errors + return list(csv_reader), [] + + @classmethod + def _export_data(cls, tags: List[dict], taxonomy: Taxonomy) -> str: + """ + Export tags in CSV format + + """ + fields = cls.required_fields + cls.optional_fields + + with StringIO() as csv_buffer: + csv_writer = csv.DictWriter(csv_buffer, fieldnames=fields) + csv_writer.writeheader() + + for tag in tags: + csv_writer.writerow(tag) + + csv_string = csv_buffer.getvalue() + return csv_string + + @classmethod + def _verify_header(cls, header_fields: List[str]) -> List[TagParserError]: + """ + Verify that the header contains the required fields + """ + errors = [] + for req_field in cls.required_fields: + if req_field not in header_fields: + errors.append( + InvalidFormat( + tag=None, + format=cls.format.value, + message=_(f"Missing '{req_field}' field on CSV headers"), + ) + ) + return errors + + +# Add parsers here +_parsers = [JSONParser, CSVParser] + + +def get_parser(parser_format: ParserFormat) -> Parser: + """ + Get the parser for the respective `format` + + Raise `ValueError` if no parser found + """ + for parser in _parsers: + if parser_format == parser.format: + return parser + + raise ValueError(_(f"Parser not found for format {parser_format}")) diff --git a/openedx_tagging/core/tagging/import_export/tasks.py b/openedx_tagging/core/tagging/import_export/tasks.py new file mode 100644 index 000000000..99d3cd49d --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/tasks.py @@ -0,0 +1,38 @@ +""" +Import and export celery tasks +""" +from io import BytesIO +from celery import shared_task + +import openedx_tagging.core.tagging.import_export.api as import_export_api +from ..models import Taxonomy +from .parsers import ParserFormat + + +@shared_task +def import_tags_task( + taxonomy: Taxonomy, + file: BytesIO, + parser_format: ParserFormat, + replace=False, +) -> bool: + """ + Runs import on a celery task + """ + return import_export_api.import_tags( + taxonomy, + file, + parser_format, + replace, + ) + + +@shared_task +def export_tags_task( + taxonomy: Taxonomy, + output_format: ParserFormat, +) -> str: + """ + Runs export on a celery task + """ + return import_export_api.export_tags(taxonomy, output_format) diff --git a/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py b/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py index 48bab478a..a6d4fd0cf 100644 --- a/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py +++ b/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py @@ -11,7 +11,7 @@ def load_language_taxonomy(apps, schema_editor): call_command("loaddata", "--app=oel_tagging", "language_taxonomy.yaml") -def revert(apps, schema_editor): +def revert(apps, schema_editor): # pragma: no cover """ Deletes language taxonomy an tags """ diff --git a/openedx_tagging/core/tagging/migrations/0006_auto_20230802_1631.py b/openedx_tagging/core/tagging/migrations/0006_auto_20230802_1631.py new file mode 100644 index 000000000..87bab9cd7 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0006_auto_20230802_1631.py @@ -0,0 +1,81 @@ +# Generated by Django 3.2.19 on 2023-08-02 21:31 + +from django.db import migrations, models +import django.db.models.deletion +import openedx_tagging.core.tagging.models.import_export + + +class Migration(migrations.Migration): + dependencies = [ + ("oel_tagging", "0006_alter_objecttag_unique_together"), + ] + + operations = [ + migrations.CreateModel( + name="TagImportTask", + fields=[ + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ( + "log", + models.TextField( + default=None, help_text="Action execution logs", null=True + ), + ), + ( + "status", + models.CharField( + choices=[ + ( + openedx_tagging.core.tagging.models.import_export.TagImportTaskState[ + "LOADING_DATA" + ], + "loading_data", + ), + ( + openedx_tagging.core.tagging.models.import_export.TagImportTaskState[ + "PLANNING" + ], + "planning", + ), + ( + openedx_tagging.core.tagging.models.import_export.TagImportTaskState[ + "EXECUTING" + ], + "executing", + ), + ( + openedx_tagging.core.tagging.models.import_export.TagImportTaskState[ + "SUCCESS" + ], + "success", + ), + ( + openedx_tagging.core.tagging.models.import_export.TagImportTaskState[ + "ERROR" + ], + "error", + ), + ], + help_text="Task status", + max_length=20, + ), + ), + ("creation_date", models.DateTimeField(auto_now_add=True)), + ( + "taxonomy", + models.ForeignKey( + help_text="Taxonomy associated with this import", + on_delete=django.db.models.deletion.CASCADE, + to="oel_tagging.taxonomy", + ), + ), + ], + ), + migrations.AddIndex( + model_name="tagimporttask", + index=models.Index( + fields=["taxonomy", "-creation_date"], + name="oel_tagging_taxonom_5e948c_idx", + ), + ), + ] diff --git a/openedx_tagging/core/tagging/models/__init__.py b/openedx_tagging/core/tagging/models/__init__.py index 90640ddfc..295e38bdb 100644 --- a/openedx_tagging/core/tagging/models/__init__.py +++ b/openedx_tagging/core/tagging/models/__init__.py @@ -9,3 +9,7 @@ UserSystemDefinedTaxonomy, LanguageTaxonomy, ) +from .import_export import ( + TagImportTask, + TagImportTaskState, +) diff --git a/openedx_tagging/core/tagging/models/import_export.py b/openedx_tagging/core/tagging/models/import_export.py new file mode 100644 index 000000000..1deaf9c9d --- /dev/null +++ b/openedx_tagging/core/tagging/models/import_export.py @@ -0,0 +1,107 @@ +from datetime import datetime +from enum import Enum +from django.db import models + +from django.utils.translation import gettext_lazy as _ + +from .base import Taxonomy + + +class TagImportTaskState(Enum): + LOADING_DATA = "loading_data" + PLANNING = "planning" + EXECUTING = "executing" + SUCCESS = "success" + ERROR = "error" + + +class TagImportTask(models.Model): + """ + Stores the state, plan and logs of a tag import task + """ + + id = models.BigAutoField(primary_key=True) + + taxonomy = models.ForeignKey( + "Taxonomy", + on_delete=models.CASCADE, + help_text=_("Taxonomy associated with this import"), + ) + + log = models.TextField( + null=True, default=None, help_text=_("Action execution logs") + ) + + status = models.CharField( + max_length=20, + choices=[(status, status.value) for status in TagImportTaskState], + help_text=_("Task status"), + ) + + creation_date = models.DateTimeField(auto_now_add=True) + + class Meta: + indexes = [ + models.Index(fields=["taxonomy", "-creation_date"]), + ] + + @classmethod + def create(cls, taxonomy: Taxonomy): + task = cls( + taxonomy=taxonomy, + status=TagImportTaskState.LOADING_DATA.value, + log="", + ) + task.add_log(_("Import task created"), save=False) + task.save() + return task + + def add_log(self, message: str, save=True): + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + log_message = f"[{timestamp}] {message}\n" + self.log += log_message + if save: + self.save() + + def log_exception(self, exception: Exception): + self.add_log(repr(exception), save=False) + self.status = TagImportTaskState.ERROR.value + self.save() + + def log_parser_start(self): + self.add_log(_("Starting to load data from file")) + + def log_parser_end(self): + self.add_log(_("Load data finished")) + + def handle_parser_errors(self, errors): + for error in errors: + self.add_log(f"{str(error)}", save=False) + self.status = TagImportTaskState.ERROR.value + self.save() + + def log_start_planning(self): + self.add_log(_("Starting plan actions"), save=False) + self.status = TagImportTaskState.PLANNING.value + self.save() + + def log_plan(self, plan): + self.add_log(_("Plan finished")) + plan_str = plan.plan() + self.log += f"\n{plan_str}\n" + self.save() + + def handle_plan_errors(self): + # Error are logged with plan + self.status = TagImportTaskState.ERROR.value + self.save() + + def log_start_execute(self): + self.add_log(_("Starting execute actions"), save=False) + self.status = TagImportTaskState.EXECUTING.value + self.save() + + def end_success(self): + self.add_log(_("Execution finished"), save=False) + self.status = TagImportTaskState.SUCCESS.value + self.save() diff --git a/requirements/base.in b/requirements/base.in index 94723505f..33b66281a 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,6 +1,10 @@ # Core requirements for using this application -c constraints.txt +attrs # Reduces boilerplate code involving class attributes + +celery # Asynchronous task execution library + Django<5.0 # Web application framework djangorestframework<4.0 # REST API diff --git a/requirements/base.txt b/requirements/base.txt index dc2d4c77f..3ad003afb 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,11 +1,19 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade # +amqp==5.1.1 + # via kombu asgiref==3.7.2 # via django +attrs==23.1.0 + # via -r requirements/base.in +billiard==4.1.0 + # via celery +celery==5.3.1 + # via -r requirements/base.in certifi==2023.7.22 # via requests cffi==1.15.1 @@ -15,7 +23,18 @@ cffi==1.15.1 charset-normalizer==3.2.0 # via requests click==8.1.6 - # via edx-django-utils + # via + # celery + # click-didyoumean + # click-plugins + # click-repl + # edx-django-utils +click-didyoumean==0.3.0 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.3.0 + # via celery cryptography==41.0.3 # via pyjwt django==3.2.19 @@ -49,10 +68,14 @@ edx-opaque-keys==2.4.0 # via edx-drf-extensions idna==3.4 # via requests +kombu==5.3.1 + # via celery newrelic==8.9.0 # via edx-django-utils pbr==5.11.1 # via stevedore +prompt-toolkit==3.0.39 + # via click-repl psutil==5.9.5 # via edx-django-utils pycparser==2.21 @@ -66,7 +89,9 @@ pymongo==3.13.0 pynacl==1.5.0 # via edx-django-utils python-dateutil==2.8.2 - # via edx-drf-extensions + # via + # celery + # edx-drf-extensions pytz==2023.3 # via # django @@ -89,5 +114,14 @@ stevedore==5.1.0 # edx-opaque-keys typing-extensions==4.6.3 # via asgiref +tzdata==2023.3 + # via celery urllib3==2.0.4 # via requests +vine==5.0.0 + # via + # amqp + # celery + # kombu +wcwidth==0.2.6 + # via prompt-toolkit diff --git a/requirements/ci.txt b/requirements/ci.txt index c03598b3e..f47e116e0 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade diff --git a/requirements/dev.txt b/requirements/dev.txt index 7c85fde2e..d5ad00f9d 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade # +amqp==5.1.1 + # via + # -r requirements/quality.txt + # kombu asgiref==3.7.2 # via # -r requirements/quality.txt @@ -13,6 +17,12 @@ astroid==2.15.5 # -r requirements/quality.txt # pylint # pylint-celery +attrs==23.1.0 + # via -r requirements/quality.txt +billiard==4.1.0 + # via + # -r requirements/quality.txt + # celery bleach==6.0.0 # via # -r requirements/quality.txt @@ -21,6 +31,8 @@ build==0.10.0 # via # -r requirements/pip-tools.txt # pip-tools +celery==5.3.1 + # via -r requirements/quality.txt certifi==2023.7.22 # via # -r requirements/quality.txt @@ -41,16 +53,32 @@ click==8.1.6 # -r requirements/ci.txt # -r requirements/pip-tools.txt # -r requirements/quality.txt + # celery + # click-didyoumean # click-log + # click-plugins + # click-repl # code-annotations # edx-django-utils # edx-lint # import-linter # pip-tools +click-didyoumean==0.3.0 + # via + # -r requirements/quality.txt + # celery click-log==0.4.0 # via # -r requirements/quality.txt # edx-lint +click-plugins==1.1.1 + # via + # -r requirements/quality.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/quality.txt + # celery code-annotations==1.3.0 # via # -r requirements/quality.txt @@ -155,10 +183,6 @@ importlib-metadata==6.7.0 # -r requirements/quality.txt # keyring # twine -importlib-resources==5.12.0 - # via - # -r requirements/quality.txt - # keyring iniconfig==2.0.0 # via # -r requirements/quality.txt @@ -185,6 +209,10 @@ keyring==24.0.0 # via # -r requirements/quality.txt # twine +kombu==5.3.1 + # via + # -r requirements/quality.txt + # celery lazy-object-proxy==1.9.0 # via # -r requirements/quality.txt @@ -252,6 +280,10 @@ pluggy==1.0.0 # tox polib==1.2.0 # via edx-i18n-tools +prompt-toolkit==3.0.39 + # via + # -r requirements/quality.txt + # click-repl psutil==5.9.5 # via # -r requirements/quality.txt @@ -323,6 +355,7 @@ pytest-django==4.5.2 python-dateutil==2.8.2 # via # -r requirements/quality.txt + # celery # edx-drf-extensions python-slugify==8.0.1 # via @@ -431,17 +464,29 @@ typing-extensions==4.6.3 # astroid # grimp # import-linter - # pylint - # rich +tzdata==2023.3 + # via + # -r requirements/quality.txt + # celery urllib3==2.0.4 # via # -r requirements/quality.txt # requests # twine +vine==5.0.0 + # via + # -r requirements/quality.txt + # amqp + # celery + # kombu virtualenv==20.23.1 # via # -r requirements/ci.txt # tox +wcwidth==0.2.6 + # via + # -r requirements/quality.txt + # prompt-toolkit webencodings==0.5.1 # via # -r requirements/quality.txt @@ -458,7 +503,6 @@ zipp==3.15.0 # via # -r requirements/quality.txt # importlib-metadata - # importlib-resources # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/doc.txt b/requirements/doc.txt index f997b86a7..06ae86ff7 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade @@ -8,18 +8,30 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx +amqp==5.1.1 + # via + # -r requirements/test.txt + # kombu asgiref==3.7.2 # via # -r requirements/test.txt # django +attrs==23.1.0 + # via -r requirements/test.txt babel==2.12.1 # via # pydata-sphinx-theme # sphinx beautifulsoup4==4.12.2 # via pydata-sphinx-theme +billiard==4.1.0 + # via + # -r requirements/test.txt + # celery bleach==6.0.0 # via readme-renderer +celery==5.3.1 + # via -r requirements/test.txt certifi==2023.7.22 # via # -r requirements/test.txt @@ -36,9 +48,25 @@ charset-normalizer==3.2.0 click==8.1.6 # via # -r requirements/test.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # code-annotations # edx-django-utils # import-linter +click-didyoumean==0.3.0 + # via + # -r requirements/test.txt + # celery +click-plugins==1.1.1 + # via + # -r requirements/test.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/test.txt + # celery code-annotations==1.3.0 # via -r requirements/test.txt coverage[toml]==7.2.7 @@ -118,8 +146,6 @@ imagesize==1.4.1 # via sphinx import-linter==1.9.0 # via -r requirements/test.txt -importlib-metadata==6.7.0 - # via sphinx iniconfig==2.0.0 # via # -r requirements/test.txt @@ -129,6 +155,10 @@ jinja2==3.1.2 # -r requirements/test.txt # code-annotations # sphinx +kombu==5.3.1 + # via + # -r requirements/test.txt + # celery markupsafe==2.1.3 # via # -r requirements/test.txt @@ -157,6 +187,10 @@ pluggy==1.0.0 # pytest pprintpp==0.4.0 # via sphinxcontrib-django +prompt-toolkit==3.0.39 + # via + # -r requirements/test.txt + # click-repl psutil==5.9.5 # via # -r requirements/test.txt @@ -199,6 +233,7 @@ pytest-django==4.5.2 python-dateutil==2.8.2 # via # -r requirements/test.txt + # celery # edx-drf-extensions python-slugify==8.0.1 # via @@ -207,7 +242,6 @@ python-slugify==8.0.1 pytz==2023.3 # via # -r requirements/test.txt - # babel # django # djangorestframework pyyaml==6.0 @@ -291,11 +325,23 @@ typing-extensions==4.6.3 # grimp # import-linter # pydata-sphinx-theme +tzdata==2023.3 + # via + # -r requirements/test.txt + # celery urllib3==2.0.4 # via # -r requirements/test.txt # requests +vine==5.0.0 + # via + # -r requirements/test.txt + # amqp + # celery + # kombu +wcwidth==0.2.6 + # via + # -r requirements/test.txt + # prompt-toolkit webencodings==0.5.1 # via bleach -zipp==3.15.0 - # via importlib-metadata diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 4b7fb4e53..9ce8ce227 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade diff --git a/requirements/quality.txt b/requirements/quality.txt index 0509905b3..22fb452e9 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade # +amqp==5.1.1 + # via + # -r requirements/test.txt + # kombu asgiref==3.7.2 # via # -r requirements/test.txt @@ -12,8 +16,16 @@ astroid==2.15.5 # via # pylint # pylint-celery +attrs==23.1.0 + # via -r requirements/test.txt +billiard==4.1.0 + # via + # -r requirements/test.txt + # celery bleach==6.0.0 # via readme-renderer +celery==5.3.1 + # via -r requirements/test.txt certifi==2023.7.22 # via # -r requirements/test.txt @@ -30,13 +42,29 @@ charset-normalizer==3.2.0 click==8.1.6 # via # -r requirements/test.txt + # celery + # click-didyoumean # click-log + # click-plugins + # click-repl # code-annotations # edx-django-utils # edx-lint # import-linter +click-didyoumean==0.3.0 + # via + # -r requirements/test.txt + # celery click-log==0.4.0 # via edx-lint +click-plugins==1.1.1 + # via + # -r requirements/test.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/test.txt + # celery code-annotations==1.3.0 # via # -r requirements/test.txt @@ -117,8 +145,6 @@ importlib-metadata==6.7.0 # via # keyring # twine -importlib-resources==5.12.0 - # via keyring iniconfig==2.0.0 # via # -r requirements/test.txt @@ -139,6 +165,10 @@ jinja2==3.1.2 # code-annotations keyring==24.0.0 # via twine +kombu==5.3.1 + # via + # -r requirements/test.txt + # celery lazy-object-proxy==1.9.0 # via astroid markdown-it-py==3.0.0 @@ -177,6 +207,10 @@ pluggy==1.0.0 # via # -r requirements/test.txt # pytest +prompt-toolkit==3.0.39 + # via + # -r requirements/test.txt + # click-repl psutil==5.9.5 # via # -r requirements/test.txt @@ -232,6 +266,7 @@ pytest-django==4.5.2 python-dateutil==2.8.2 # via # -r requirements/test.txt + # celery # edx-drf-extensions python-slugify==8.0.1 # via @@ -310,18 +345,28 @@ typing-extensions==4.6.3 # astroid # grimp # import-linter - # pylint - # rich +tzdata==2023.3 + # via + # -r requirements/test.txt + # celery urllib3==2.0.4 # via # -r requirements/test.txt # requests # twine +vine==5.0.0 + # via + # -r requirements/test.txt + # amqp + # celery + # kombu +wcwidth==0.2.6 + # via + # -r requirements/test.txt + # prompt-toolkit webencodings==0.5.1 # via bleach wrapt==1.15.0 # via astroid zipp==3.15.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata diff --git a/requirements/test.txt b/requirements/test.txt index e1240b21f..ba50193f0 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,13 +1,25 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade # +amqp==5.1.1 + # via + # -r requirements/base.txt + # kombu asgiref==3.7.2 # via # -r requirements/base.txt # django +attrs==23.1.0 + # via -r requirements/base.txt +billiard==4.1.0 + # via + # -r requirements/base.txt + # celery +celery==5.3.1 + # via -r requirements/base.txt certifi==2023.7.22 # via # -r requirements/base.txt @@ -24,9 +36,25 @@ charset-normalizer==3.2.0 click==8.1.6 # via # -r requirements/base.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # code-annotations # edx-django-utils # import-linter +click-didyoumean==0.3.0 + # via + # -r requirements/base.txt + # celery +click-plugins==1.1.1 + # via + # -r requirements/base.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/base.txt + # celery code-annotations==1.3.0 # via -r requirements/test.in coverage[toml]==7.2.7 @@ -93,6 +121,10 @@ iniconfig==2.0.0 # via pytest jinja2==3.1.2 # via code-annotations +kombu==5.3.1 + # via + # -r requirements/base.txt + # celery markupsafe==2.1.3 # via jinja2 mock==5.0.2 @@ -111,6 +143,10 @@ pbr==5.11.1 # stevedore pluggy==1.0.0 # via pytest +prompt-toolkit==3.0.39 + # via + # -r requirements/base.txt + # click-repl psutil==5.9.5 # via # -r requirements/base.txt @@ -144,6 +180,7 @@ pytest-django==4.5.2 python-dateutil==2.8.2 # via # -r requirements/base.txt + # celery # edx-drf-extensions python-slugify==8.0.1 # via code-annotations @@ -193,7 +230,21 @@ typing-extensions==4.6.3 # asgiref # grimp # import-linter +tzdata==2023.3 + # via + # -r requirements/base.txt + # celery urllib3==2.0.4 # via # -r requirements/base.txt # requests +vine==5.0.0 + # via + # -r requirements/base.txt + # amqp + # celery + # kombu +wcwidth==0.2.6 + # via + # -r requirements/base.txt + # prompt-toolkit diff --git a/tests/openedx_tagging/core/fixtures/tagging.yaml b/tests/openedx_tagging/core/fixtures/tagging.yaml index 5bb3936c9..3593a095c 100644 --- a/tests/openedx_tagging/core/fixtures/tagging.yaml +++ b/tests/openedx_tagging/core/fixtures/tagging.yaml @@ -173,6 +173,34 @@ parent: null value: System Tag 4 external_id: 'tag_4' +- model: oel_tagging.tag + pk: 26 + fields: + taxonomy: 5 + parent: null + value: Tag 1 + external_id: tag_1 +- model: oel_tagging.tag + pk: 27 + fields: + taxonomy: 5 + parent: 26 + value: Tag 2 + external_id: tag_2 +- model: oel_tagging.tag + pk: 28 + fields: + taxonomy: 5 + parent: null + value: Tag 3 + external_id: tag_3 +- model: oel_tagging.tag + pk: 29 + fields: + taxonomy: 5 + parent: 28 + value: Tag 4 + external_id: tag_4 - model: oel_tagging.taxonomy pk: 1 fields: @@ -202,3 +230,13 @@ allow_multiple: false allow_free_text: false _taxonomy_class: openedx_tagging.core.tagging.models.system_defined.SystemDefinedTaxonomy +- model: oel_tagging.taxonomy + pk: 5 + fields: + name: Import Taxonomy Test + description: null + enabled: true + required: false + allow_multiple: false + allow_free_text: false + diff --git a/tests/openedx_tagging/core/tagging/import_export/__init__.py b/tests/openedx_tagging/core/tagging/import_export/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_tagging/core/tagging/import_export/mixins.py b/tests/openedx_tagging/core/tagging/import_export/mixins.py new file mode 100644 index 000000000..045d3c180 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/import_export/mixins.py @@ -0,0 +1,16 @@ +""" +Mixins for ImportExport tests +""" +from openedx_tagging.core.tagging.models import Taxonomy + + +class TestImportExportMixin: + """ + Mixin that loads the base data for import/export tests + """ + + fixtures = ["tests/openedx_tagging/core/fixtures/tagging.yaml"] + + def setUp(self): + self.taxonomy = Taxonomy.objects.get(name="Import Taxonomy Test") + return super().setUp() diff --git a/tests/openedx_tagging/core/tagging/import_export/test_actions.py b/tests/openedx_tagging/core/tagging/import_export/test_actions.py new file mode 100644 index 000000000..1c0b14852 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/import_export/test_actions.py @@ -0,0 +1,500 @@ +""" +Tests for actions +""" +import ddt + +from django.test.testcases import TestCase + +from openedx_tagging.core.tagging.models import Tag +from openedx_tagging.core.tagging.import_export.import_plan import TagItem +from openedx_tagging.core.tagging.import_export.actions import ( + ImportAction, + CreateTag, + UpdateParentTag, + RenameTag, + DeleteTag, + WithoutChanges, +) +from .mixins import TestImportExportMixin + + +class TestImportActionMixin(TestImportExportMixin): + """ + Mixin for import action tests + """ + def setUp(self): + super().setUp() + self.indexed_actions = { + 'create': [ + CreateTag( + taxonomy=self.taxonomy, + tag=TagItem( + id='tag_10', + value='Tag 10', + index=0 + ), + index=0, + ) + ], + 'rename': [ + RenameTag( + taxonomy=self.taxonomy, + tag=TagItem( + id='tag_11', + value='Tag 11', + index=1 + ), + index=1, + ) + ] + } + + +@ddt.ddt +class TestImportAction(TestImportActionMixin, TestCase): + """ + Test for general function of the ImportAction class + """ + + def test_not_implemented_functions(self): + with self.assertRaises(NotImplementedError): + ImportAction.applies_for(None, None) + action = ImportAction(None, None, None) + with self.assertRaises(NotImplementedError): + action.validate(None) + with self.assertRaises(NotImplementedError): + action.execute() + + def test_str(self): + expected = "Action import_action (index=100,id=tag_1)" + action = ImportAction( + taxonomy=self.taxonomy, + tag=TagItem( + id='tag_1', + value='value', + ), + index=100, + ) + assert str(action) == expected + + @ddt.data( + ('create', 'id', 'tag_10', True), + ('rename', 'value', 'Tag 11', True), + ('rename', 'id', 'tag_10', False), + ('create', 'value', 'Tag 11', False), + ) + @ddt.unpack + def test_search_action(self, action_name, attr, search_value, expected): + import_action = ImportAction(self.taxonomy, None, None) + action = import_action._search_action( # pylint: disable=protected-access + self.indexed_actions, + action_name, + attr, + search_value, + ) + if expected: + self.assertEqual(getattr(action.tag, attr), search_value) + else: + self.assertIsNone(action) + + @ddt.data( + ('tag_1', True), + ('tag_10', True), + ('tag_100', False), + ) + @ddt.unpack + def test_validate_parent(self, parent_id, expected): + action = ImportAction( + self.taxonomy, + TagItem( + id='tag_110', + value='_', + parent_id=parent_id, + index=100 + ), + index=100, + ) + error = action._validate_parent(self.indexed_actions) # pylint: disable=protected-access + if expected: + self.assertIsNone(error) + else: + self.assertEqual( + str(error), + ( + "Action error in 'import_action' (#100): " + "Unknown parent tag (tag_100). " + "You need to add parent before the child in your file." + ) + ) + + @ddt.data( + ( + 'Tag 1', + ( + "Action error in 'import_action' (#100): " + "Duplicated tag value with tag in database (external_id=tag_1)." + ) + ), + ( + 'Tag 10', + ( + "Conflict with 'import_action' (#100) " + "and action #0: Duplicated tag value." + ) + ), + ( + 'Tag 11', + ( + "Conflict with 'import_action' (#100) " + "and action #1: Duplicated tag value." + ) + ), + ('Tag 20', None) + ) + @ddt.unpack + def test_validate_value(self, value, expected): + action = ImportAction( + self.taxonomy, + TagItem( + id='tag_110', + value=value, + index=100 + ), + index=100, + ) + error = action._validate_value(self.indexed_actions) # pylint: disable=protected-access + if not expected: + self.assertIsNone(error) + else: + self.assertEqual(str(error), expected) + + +@ddt.ddt +class TestCreateTag(TestImportActionMixin, TestCase): + """ + Test for 'create' action + """ + + @ddt.data( + ('tag_1', False), + ('tag_100', True), + ) + @ddt.unpack + def test_applies_for(self, tag_id, expected): + result = CreateTag.applies_for( + self.taxonomy, + TagItem( + id=tag_id, + value='_', + index=100, + ) + ) + self.assertEqual(result, expected) + + @ddt.data( + ('tag_10', False), + ('tag_100', True), + ) + @ddt.unpack + def test_validate_id(self, tag_id, expected): + action = CreateTag( + taxonomy=self.taxonomy, + tag=TagItem( + id=tag_id, + value='_', + index=100, + ), + index=100 + ) + error = action._validate_id(self.indexed_actions) # pylint: disable=protected-access + if expected: + self.assertIsNone(error) + else: + self.assertEqual( + str(error), + ( + "Conflict with 'create' (#100) " + "and action #0: Duplicated external_id tag." + ) + ) + + @ddt.data( + ('tag_10', "Tag 20", None, 1), # Invalid tag id + ('tag_20', "Tag 10", None, 1), # Invalid value, + ('tag_20', "Tag 20", 'tag_100', 1), # Invalid parent id, + ('tag_10', "Tag 10", None, 2), # Invalid tag id and value, + ('tag_10', "Tag 10", 'tag_100', 3), # Invalid tag id, value and parent, + ('tag_20', "Tag 20", 'tag_1', 0), # Valid + ) + @ddt.unpack + def test_validate(self, tag_id, tag_value, parent_id, expected): + action = CreateTag( + taxonomy=self.taxonomy, + tag=TagItem( + id=tag_id, + value=tag_value, + index=100, + parent_id=parent_id + ), + index=100 + ) + errors = action.validate(self.indexed_actions) + self.assertEqual(len(errors), expected) + + @ddt.data( + ('tag_30', 'Tag 30', None), # No parent + ('tag_31', 'Tag 31', 'tag_3'), # With parent + ) + @ddt.unpack + def test_execute(self, tag_id, value, parent_id): + tag = TagItem( + id=tag_id, + value=value, + parent_id=parent_id, + ) + action = CreateTag( + self.taxonomy, + tag, + index=100, + ) + with self.assertRaises(Tag.DoesNotExist): + self.taxonomy.tag_set.get(external_id=tag_id) + action.execute() + tag = self.taxonomy.tag_set.get(external_id=tag_id) + assert tag.value == value + if parent_id: + assert tag.parent.external_id == parent_id + else: + assert tag.parent is None + + +@ddt.ddt +class TestUpdateParentTag(TestImportActionMixin, TestCase): + """ + Test for 'update_parent' action + """ + + @ddt.data( + ( + "tag_4", + "tag_3", + ( + "Update the parent of tag (external_id=tag_4) from parent " + "(external_id=tag_3) to parent (external_id=tag_3)." + ) + ), + ( + "tag_3", + "tag_2", + ( + "Update the parent of tag (external_id=tag_3) from empty parent " + "to parent (external_id=tag_2)." + ) + ), + ) + @ddt.unpack + def test_str(self, tag_id, parent_id, expected): + tag_item = TagItem( + id=tag_id, + value='_', + parent_id=parent_id, + ) + action = UpdateParentTag( + taxonomy=self.taxonomy, + tag=tag_item, + index=100, + ) + assert str(action) == expected + + @ddt.data( + ('tag_100', None, False), # Tag doesn't exist on database + ('tag_2', 'tag_1', False), # Parent don't change + ('tag_2', 'tag_3', True), # Valid + ('tag_1', None, False), # Both parent id are None + ('tag_1', 'tag_3', True), # Valid + ) + @ddt.unpack + def test_applies_for(self, tag_id, parent_id, expected): + result = UpdateParentTag.applies_for( + taxonomy=self.taxonomy, + tag=TagItem( + id=tag_id, + value='_', + parent_id=parent_id, + index=100 + ) + ) + self.assertEqual(result, expected) + + @ddt.data( + ('tag_2', 'tag_30', 1), # Invalid parent + ('tag_2', None, 0), # Without parent + ('tag_2', 'tag_10', 0), # Valid + ) + @ddt.unpack + def test_validate(self, tag_id, parent_id, expected): + action = UpdateParentTag( + taxonomy=self.taxonomy, + tag=TagItem( + id=tag_id, + value='_', + parent_id=parent_id, + ), + index=100 + ) + errors = action.validate(self.indexed_actions) + self.assertEqual(len(errors), expected) + + @ddt.data( + ('tag_4', 'tag_2'), # Change parent + ('tag_4', None), # Set parent as None + ('tag_3', 'tag_1'), # Add parent + ) + @ddt.unpack + def test_execute(self, tag_id, parent_id): + tag_item = TagItem( + id=tag_id, + value='_', + parent_id=parent_id, + ) + action = UpdateParentTag( + taxonomy=self.taxonomy, + tag=tag_item, + index=100, + ) + tag = self.taxonomy.tag_set.get(external_id=tag_id) + if tag.parent: + assert tag.parent.external_id != parent_id + action.execute() + tag = self.taxonomy.tag_set.get(external_id=tag_id) + if not parent_id: + assert tag.parent is None + else: + assert tag.parent.external_id == parent_id + + +@ddt.ddt +class TestRenameTag(TestImportActionMixin, TestCase): + """ + Test for 'rename' action + """ + + @ddt.data( + ('tag_10', 'value', False), # Tag doesn't exist on database + ('tag_1', 'Tag 1', False), # Same value + ('tag_1', 'Tag 1 v2', True), # Valid + ) + @ddt.unpack + def test_applies_for(self, tag_id, value, expected): + result = RenameTag.applies_for( + taxonomy=self.taxonomy, + tag=TagItem( + id=tag_id, + value=value, + index=100, + ) + ) + self.assertEqual(result, expected) + + @ddt.data( + ('Tag 2', 1), # There is a tag with the same value on database + ('Tag 10', 1), # There is a tag with the same value on create action + ('Tag 11', 1), # There is a tag with the same value on rename action + ('Tag 12', 0), # Valid + ) + @ddt.unpack + def test_validate(self, value, expected): + action = RenameTag( + taxonomy=self.taxonomy, + tag=TagItem( + id='tag_1', + value=value, + index=100, + ), + index=100, + ) + errors = action.validate(self.indexed_actions) + self.assertEqual(len(errors), expected) + + def test_execute(self): + tag_id = 'tag_1' + value = 'Tag 1 V2' + tag_item = TagItem( + id=tag_id, + value=value, + ) + action = RenameTag( + taxonomy=self.taxonomy, + tag=tag_item, + index=100, + ) + tag = self.taxonomy.tag_set.get(external_id=tag_id) + assert tag.value != value + action.execute() + tag = self.taxonomy.tag_set.get(external_id=tag_id) + assert tag.value == value + + +class TestDeleteTag(TestImportActionMixin, TestCase): + """ + Test for 'delete' action + """ + + def test_applies_for(self): + assert not DeleteTag.applies_for(self.taxonomy, None) + + def test_validate(self): + action = DeleteTag( + taxonomy=self.taxonomy, + tag=TagItem( + id='_', + value='_', + index=100, + ), + index=100, + ) + assert not action.validate(self.indexed_actions) + + def test_execute(self): + tag_id = 'tag_3' + tag_item = TagItem( + id=tag_id, + value='_', + ) + action = DeleteTag( + taxonomy=self.taxonomy, + tag=tag_item, + index=100, + ) + assert self.taxonomy.tag_set.filter(external_id=tag_id).exists() + action.execute() + assert not self.taxonomy.tag_set.filter(external_id=tag_id).exists() + + +class TestWithoutChanges(TestImportActionMixin, TestCase): + """ + Test for 'without_changes' action + """ + def test_applies_for(self): + result = WithoutChanges.applies_for( + self.taxonomy, + tag=TagItem( + id='_', + value='_', + index=100, + ), + ) + self.assertFalse(result) + + def test_validate(self): + action = WithoutChanges( + taxonomy=self.taxonomy, + tag=TagItem( + id='_', + value='_', + index=100, + ), + index=100, + ) + result = action.validate(self.indexed_actions) + self.assertEqual(result, []) diff --git a/tests/openedx_tagging/core/tagging/import_export/test_api.py b/tests/openedx_tagging/core/tagging/import_export/test_api.py new file mode 100644 index 000000000..453979e01 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/import_export/test_api.py @@ -0,0 +1,219 @@ +""" +Test for import/export API +""" +import json +from io import BytesIO + +from django.test.testcases import TestCase + +from openedx_tagging.core.tagging.models import ( + TagImportTask, + TagImportTaskState, + Taxonomy, + LanguageTaxonomy, +) +from openedx_tagging.core.tagging.import_export import ParserFormat +import openedx_tagging.core.tagging.import_export.api as import_export_api + +from .mixins import TestImportExportMixin + + +class TestImportExportApi(TestImportExportMixin, TestCase): + """ + Test import/export API functions + """ + + def setUp(self): + self.tags = [ + {"id": "tag_31", "value": "Tag 31"}, + {"id": "tag_32", "value": "Tag 32"}, + {"id": "tag_33", "value": "Tag 33", "parent_id": "tag_31"}, + {"id": "tag_1", "value": "Tag 1 V2"}, + {"id": "tag_4", "value": "Tag 4", "parent_id": "tag_32"}, + ] + json_data = {"tags": self.tags} + self.file = BytesIO(json.dumps(json_data).encode()) + + json_data = {"invalid": [ + {"id": "tag_1", "name": "Tag 1"}, + ]} + self.invalid_parser_file = BytesIO(json.dumps(json_data).encode()) + json_data = {"tags": [ + {'id': 'tag_31', 'value': 'Tag 31',}, + {'id': 'tag_31', 'value': 'Tag 32',}, + ]} + self.invalid_plan_file = BytesIO(json.dumps(json_data).encode()) + + self.parser_format = ParserFormat.JSON + + self.open_taxonomy = Taxonomy( + name="Open taxonomy", + allow_free_text=True + ) + self.system_taxonomy = Taxonomy( + name="System taxonomy", + ) + self.system_taxonomy.taxonomy_class = LanguageTaxonomy + self.system_taxonomy = self.system_taxonomy.cast() + return super().setUp() + + def test_check_status(self): + TagImportTask.create(self.taxonomy) + status = import_export_api.get_last_import_status(self.taxonomy) + assert status == TagImportTaskState.LOADING_DATA.value + + def test_check_log(self): + TagImportTask.create(self.taxonomy) + log = import_export_api.get_last_import_log(self.taxonomy) + assert "Import task created" in log + + def test_invalid_import_tags(self): + TagImportTask.create(self.taxonomy) + with self.assertRaises(ValueError): + # Raise error if there is a current in progress task + import_export_api.import_tags( + self.taxonomy, + self.file, + self.parser_format, + ) + + def test_import_export_validations(self): + # Check that import is invalid with open taxonomy + with self.assertRaises(NotImplementedError): + import_export_api.import_tags( + self.open_taxonomy, + self.file, + self.parser_format, + ) + + # Check that import is invalid with system taxonomy + with self.assertRaises(ValueError): + import_export_api.import_tags( + self.system_taxonomy, + self.file, + self.parser_format, + ) + + def test_with_python_error(self): + self.file.close() + assert not import_export_api.import_tags( + self.taxonomy, + self.file, + self.parser_format, + ) + status = import_export_api.get_last_import_status(self.taxonomy) + log = import_export_api.get_last_import_log(self.taxonomy) + assert status == TagImportTaskState.ERROR.value + assert "ValueError('I/O operation on closed file.')" in log + + def test_with_parser_error(self): + assert not import_export_api.import_tags( + self.taxonomy, + self.invalid_parser_file, + self.parser_format, + ) + status = import_export_api.get_last_import_status(self.taxonomy) + log = import_export_api.get_last_import_log(self.taxonomy) + assert status == TagImportTaskState.ERROR.value + assert "Starting to load data from file" in log + assert "Invalid '.json' format" in log + + def test_with_plan_errors(self): + assert not import_export_api.import_tags( + self.taxonomy, + self.invalid_plan_file, + self.parser_format, + ) + status = import_export_api.get_last_import_status(self.taxonomy) + log = import_export_api.get_last_import_log(self.taxonomy) + assert status == TagImportTaskState.ERROR.value + assert "Starting to load data from file" in log + assert "Load data finished" in log + assert "Starting plan actions" in log + assert "Plan finished" in log + assert "Conflict with 'create'" in log + + def test_valid(self): + assert import_export_api.import_tags( + self.taxonomy, + self.file, + self.parser_format, + replace=True, + ) + status = import_export_api.get_last_import_status(self.taxonomy) + log = import_export_api.get_last_import_log(self.taxonomy) + assert status == TagImportTaskState.SUCCESS.value + assert "Starting to load data from file" in log + assert "Load data finished" in log + assert "Starting plan actions" in log + assert "Plan finished" in log + assert "Starting execute actions" in log + assert "Execution finished" in log + + def test_start_task_after_error(self): + assert not import_export_api.import_tags( + self.taxonomy, + self.invalid_parser_file, + self.parser_format, + ) + assert import_export_api.import_tags( + self.taxonomy, + self.file, + self.parser_format, + ) + + def test_start_task_after_success(self): + assert import_export_api.import_tags( + self.taxonomy, + self.file, + self.parser_format, + ) + + # Opening again the file + json_data = {"tags": self.tags} + self.file = BytesIO(json.dumps(json_data).encode()) + + assert import_export_api.import_tags( + self.taxonomy, + self.file, + self.parser_format, + ) + + def test_export_validations(self): + # Check that import is invalid with open taxonomy + with self.assertRaises(NotImplementedError): + import_export_api.export_tags( + self.open_taxonomy, + self.parser_format, + ) + + # Check that import is invalid with system taxonomy + with self.assertRaises(ValueError): + import_export_api.export_tags( + self.system_taxonomy, + self.parser_format, + ) + + def test_import_with_export_output(self): + for parser_format in ParserFormat: + output = import_export_api.export_tags( + self.taxonomy, + parser_format, + ) + file = BytesIO(output.encode()) + new_taxonomy = Taxonomy(name="New taxonomy") + new_taxonomy.save() + assert import_export_api.import_tags( + new_taxonomy, + file, + parser_format, + ) + old_tags = self.taxonomy.tag_set.all() + assert len(old_tags) == new_taxonomy.tag_set.count() + + for tag in old_tags: + new_tag = new_taxonomy.tag_set.get(external_id=tag.external_id) + assert new_tag.value == tag.value + if tag.parent: + assert tag.parent.external_id == new_tag.parent.external_id + \ No newline at end of file diff --git a/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py b/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py new file mode 100644 index 000000000..af1e55f30 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py @@ -0,0 +1,418 @@ +""" +Test for import_plan functions +""" +import ddt + +from django.test.testcases import TestCase + +from openedx_tagging.core.tagging.import_export.import_plan import TagItem, TagImportPlan +from openedx_tagging.core.tagging.import_export.actions import CreateTag +from openedx_tagging.core.tagging.import_export.exceptions import TagImportError +from .test_actions import TestImportActionMixin + +@ddt.ddt +class TestTagImportPlan(TestImportActionMixin, TestCase): + """ + Test for import plan functions + """ + + def setUp(self): + super().setUp() + self.import_plan = TagImportPlan(self.taxonomy) + + def test_tag_import_error(self): + message = "Error message" + expected_repr = f"TagImportError({message})" + error = TagImportError(message) + assert str(error) == message + assert repr(error) == expected_repr + + + @ddt.data( + ('tag_10', 1), # Test invalid + ('tag_30', 0), # Test valid + ) + @ddt.unpack + def test_build_action(self, tag_id, errors_expected): + self.import_plan.indexed_actions = self.indexed_actions + self.import_plan._build_action( # pylint: disable=protected-access + CreateTag, + TagItem( + id=tag_id, + value='_', + index=100 + ) + ) + assert len(self.import_plan.errors) == errors_expected + assert len(self.import_plan.actions) == 1 + assert self.import_plan.actions[0].name == 'create' + assert self.import_plan.indexed_actions['create'][1].tag.id == tag_id + + def test_build_delete_actions(self): + tags = { + tag.external_id: tag + for tag in self.taxonomy.tag_set.exclude(pk=25) + } + # Clear other actions to only have the delete ones + self.import_plan.actions.clear() + + self.import_plan._build_delete_actions(tags) # pylint: disable=protected-access + assert len(self.import_plan.errors) == 0 + + # Check actions in order + # #1 Update parent of 'tag_2' + assert self.import_plan.actions[0].name == 'update_parent' + assert self.import_plan.actions[0].tag.id == 'tag_2' + assert self.import_plan.actions[0].tag.parent_id is None + # #2 Delete 'tag_1' + assert self.import_plan.actions[1].name == 'delete' + assert self.import_plan.actions[1].tag.id == 'tag_1' + # #3 Delete 'tag_2' + assert self.import_plan.actions[2].name == 'delete' + assert self.import_plan.actions[2].tag.id == 'tag_2' + # #4 Update parent of 'tag_4' + assert self.import_plan.actions[3].name == 'update_parent' + assert self.import_plan.actions[3].tag.id == 'tag_4' + assert self.import_plan.actions[3].tag.parent_id is None + # #5 Delete 'tag_3' + assert self.import_plan.actions[4].name == 'delete' + assert self.import_plan.actions[4].tag.id == 'tag_3' + + @ddt.data( + ([ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_32', + 'value': 'Tag 32', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_2', + 'value': 'Tag 2 v2', + 'parent_id': 'tag_1' + }, + { + 'id': 'tag_4', + 'value': 'Tag 4 v2', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_1', + 'value': 'Tag 1', + }, + ], + False, + 0, + [ + { + 'name': 'create', + 'id': 'tag_31' + }, + { + 'name': 'create', + 'id': 'tag_32' + }, + { + 'name': 'rename', + 'id': 'tag_2' + }, + { + 'name': 'update_parent', + 'id': 'tag_4' + }, + { + 'name': 'rename', + 'id': 'tag_4' + }, + { + 'name': 'without_changes', + 'id': 'tag_1' + }, + ]), # Test valid actions + ([ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_31', + 'value': 'Tag 32', + }, + { + 'id': 'tag_1', + 'value': 'Tag 2', + }, + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_100', + }, + ], + False, + 3, + [ + { + 'name': 'create', + 'id': 'tag_31', + }, + { + 'name': 'create', + 'id': 'tag_31', + }, + { + 'name': 'rename', + 'id': 'tag_1', + }, + { + 'name': 'update_parent', + 'id': 'tag_4', + } + ]), # Test with errors in actions + ([ + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_3', + }, + ], + True, + 0, + [ + { + 'name': 'without_changes', + 'id': 'tag_4', + }, + { + 'name': 'update_parent', + 'id': 'tag_2', + }, + { + 'name': 'delete', + 'id': 'tag_1', + }, + { + 'name': 'delete', + 'id': 'tag_2', + }, + { + 'name': 'update_parent', + 'id': 'tag_4', + }, + { + 'name': 'delete', + 'id': 'tag_3', + }, + ]) # Test with deletes (replace=True) + ) + @ddt.unpack + def test_generate_actions(self, tags, replace, expected_errors, expected_actions): + tags = [TagItem(**tag) for tag in tags] + self.import_plan.generate_actions(tags=tags, replace=replace) + assert len(self.import_plan.errors) == expected_errors + assert len(self.import_plan.actions) == len(expected_actions) + + for index, action in enumerate(expected_actions): + assert self.import_plan.actions[index].name == action['name'] + assert self.import_plan.actions[index].tag.id == action['id'] + assert self.import_plan.actions[index].index == index + 1 + + @ddt.data( + ([ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_31', + 'value': 'Tag 32', + }, + { + 'id': 'tag_1', + 'value': 'Tag 2', + }, + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_100', + }, + { + 'id': 'tag_33', + 'value': 'Tag 32', + }, + { + 'id': 'tag_2', + 'value': 'Tag 31', + }, + ], + False, + "Import plan for Import Taxonomy Test\n" + "--------------------------------\n" + "#1: Create a new tag with values (external_id=tag_31, value=Tag 31, parent_id=None).\n" + "#2: Create a new tag with values (external_id=tag_31, value=Tag 32, parent_id=None).\n" + "#3: Rename tag value of tag (external_id=tag_1) from 'Tag 1' to 'Tag 2'\n" + "#4: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " + "to parent (external_id=tag_100).\n" + "#5: Create a new tag with values (external_id=tag_33, value=Tag 32, parent_id=None).\n" + "#6: Update the parent of tag (external_id=tag_2) from parent (external_id=tag_1) " + "to parent (external_id=None).\n" + "#7: Rename tag value of tag (external_id=tag_2) from 'Tag 2' to 'Tag 31'\n" + "\nOutput errors\n" + "--------------------------------\n" + "Conflict with 'create' (#2) and action #1: Duplicated external_id tag.\n" + "Action error in 'rename' (#3): Duplicated tag value with tag in database (external_id=tag_2).\n" + "Action error in 'update_parent' (#4): Unknown parent tag (tag_100). " + "You need to add parent before the child in your file.\n" + "Conflict with 'create' (#5) and action #2: Duplicated tag value.\n" + "Conflict with 'rename' (#7) and action #1: Duplicated tag value.\n" + ), # Testing plan with errors + ([ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_32', + 'value': 'Tag 32', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_2', + 'value': 'Tag 2 v2', + 'parent_id': 'tag_1' + }, + { + 'id': 'tag_4', + 'value': 'Tag 4 v2', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_1', + 'value': 'Tag 1', + }, + ], + False, + "Import plan for Import Taxonomy Test\n" + "--------------------------------\n" + "#1: Create a new tag with values (external_id=tag_31, value=Tag 31, parent_id=None).\n" + "#2: Create a new tag with values (external_id=tag_32, value=Tag 32, parent_id=tag_1).\n" + "#3: Rename tag value of tag (external_id=tag_2) from 'Tag 2' to 'Tag 2 v2'\n" + "#4: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " + "to parent (external_id=tag_1).\n" + "#5: Rename tag value of tag (external_id=tag_4) from 'Tag 4' to 'Tag 4 v2'\n" + "#6: No changes needed for tag (external_id=tag_1)\n" + ), # Testing valid plan + ([ + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_3', + }, + ], + True, + "Import plan for Import Taxonomy Test\n" + "--------------------------------\n" + "#1: No changes needed for tag (external_id=tag_4)\n" + "#2: Update the parent of tag (external_id=tag_2) from parent (external_id=tag_1) " + "to parent (external_id=None).\n" + "#3: Delete tag (external_id=tag_1)\n" + "#4: Delete tag (external_id=tag_2)\n" + "#5: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " + "to parent (external_id=None).\n" + "#6: Delete tag (external_id=tag_3)\n" + ) # Testing deletes (replace=True) + ) + @ddt.unpack + def test_plan(self, tags, replace, expected): + """ + Test the output of plan() function + + It has been decided to verify the output exactly to detect + any error when printing this information that the user is going to read. + """ + tags = [TagItem(**tag) for tag in tags] + self.import_plan.generate_actions(tags=tags, replace=replace) + plan = self.import_plan.plan() + print(plan) + assert plan == expected + + @ddt.data( + ([ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_32', + 'value': 'Tag 32', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_2', + 'value': 'Tag 2 v2', + 'parent_id': 'tag_1' + }, + { + 'id': 'tag_4', + 'value': 'Tag 4 v2', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_1', + 'value': 'Tag 1', + }, + ], False), # Testing all actions + ([ + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_3', + }, + ], True), # Testing deletes (replace=True) + ) + @ddt.unpack + def test_execute(self, tags, replace): + tags = [TagItem(**tag) for tag in tags] + self.import_plan.generate_actions(tags=tags, replace=replace) + self.import_plan.execute() + tag_external_ids = [] + for tag_item in tags: + # This checks any creation + tag = self.taxonomy.tag_set.get(external_id=tag_item.id) + + # Checks any rename + assert tag.value == tag_item.value + + # Checks any parent update + if not replace: + if not tag_item.parent_id: + assert tag.parent is None + else: + assert tag.parent.external_id == tag_item.parent_id + + tag_external_ids.append(tag_item.id) + + if replace: + # Checks deletions checking that exists the updated tags + external_ids = list(self.taxonomy.tag_set.values_list("external_id", flat=True)) + assert tag_external_ids == external_ids + + def test_error_in_execute(self): + created_tag = 'tag_31' + tags = [ + TagItem( + id=created_tag, + value='Tag 31' + ), # Valid tag (creation) + TagItem( + id='tag_32', + value='Tag 31' + ), # Invalid + ] + self.import_plan.generate_actions(tags=tags) + assert not self.taxonomy.tag_set.filter(external_id=created_tag).exists() + assert not self.import_plan.execute() + assert not self.taxonomy.tag_set.filter(external_id=created_tag).exists() + \ No newline at end of file diff --git a/tests/openedx_tagging/core/tagging/import_export/test_parsers.py b/tests/openedx_tagging/core/tagging/import_export/test_parsers.py new file mode 100644 index 000000000..5b77e3a54 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/import_export/test_parsers.py @@ -0,0 +1,310 @@ +""" +Test for import/export parsers +""" +from io import BytesIO +import json +import ddt + +from django.test.testcases import TestCase + +from openedx_tagging.core.tagging.import_export.parsers import ( + Parser, + get_parser, + JSONParser, + CSVParser, + ParserFormat, +) +from openedx_tagging.core.tagging.import_export.exceptions import ( + TagParserError, +) +from openedx_tagging.core.tagging.models import Taxonomy +from .mixins import TestImportExportMixin + + +class TestParser(TestCase): + """ + Test for general parser functions + """ + + def test_get_parser(self): + for parser_format in ParserFormat: + parser = get_parser(parser_format) + self.assertEqual(parser.format, parser_format) + + def test_parser_not_found(self): + with self.assertRaises(ValueError): + get_parser(None) + + def test_not_implemented(self): + taxonomy = Taxonomy(name="Test taxonomy") + taxonomy.save() + with self.assertRaises(NotImplementedError): + Parser.parse_import(BytesIO()) + with self.assertRaises(NotImplementedError): + Parser.export(taxonomy) + + def test_tag_parser_error(self): + tag = {"id": 'tag_id', "value": "tag_value"} + expected_str = f"Import parser error on {tag}" + expected_repr = f"TagParserError(Import parser error on {tag})" + error = TagParserError(tag) + assert str(error) == expected_str + assert repr(error) == expected_repr + + +@ddt.ddt +class TestJSONParser(TestImportExportMixin, TestCase): + """ + Test for .json parser + """ + + def test_invalid_json(self): + json_data = "{This is an invalid json}" + json_file = BytesIO(json_data.encode()) + tags, errors = JSONParser.parse_import(json_file) + assert len(tags) == 0 + assert len(errors) == 1 + assert "Expecting property name enclosed in double quotes" in str(errors[0]) + + def test_load_data_errors(self): + json_data = {"invalid": [ + {"id": "tag_1", "name": "Tag 1"}, + ]} + + json_file = BytesIO(json.dumps(json_data).encode()) + + tags, errors = JSONParser.parse_import(json_file) + self.assertEqual(len(tags), 0) + self.assertEqual(len(errors), 1) + self.assertEqual( + str(errors[0]), + "Invalid '.json' format: Missing 'tags' field on the .json file" + ) + + @ddt.data( + ( + {"tags": [ + {"id": "tag_1", "value": "Tag 1"}, # Valid + ]}, + [] + ), + ( + {"tags": [ + {"id": "tag_1"}, + {"value": "tag_1"}, + {}, + ]}, + [ + "Missing 'value' field on {'id': 'tag_1'}", + "Missing 'id' field on {'value': 'tag_1'}", + "Missing 'value' field on {}", + "Missing 'id' field on {}", + ] + ), + ( + {"tags": [ + {"id": "", "value": "tag 1"}, + {"id": "tag_2", "value": ""}, + {"id": "tag_3", "value": "tag 3", "parent_id": ""}, # Valid + ]}, + [ + "Empty 'id' field on {'id': '', 'value': 'tag 1'}", + "Empty 'value' field on {'id': 'tag_2', 'value': ''}", + ] + ) + ) + @ddt.unpack + def test_parse_tags_errors(self, json_data, expected_errors): + json_file = BytesIO(json.dumps(json_data).encode()) + + _, errors = JSONParser.parse_import(json_file) + self.assertEqual(len(errors), len(expected_errors)) + + for error in errors: + self.assertIn(str(error), expected_errors) + + def test_parse_tags(self): + expected_tags = [ + {"id": "tag_1", "value": "tag 1"}, + {"id": "tag_2", "value": "tag 2"}, + {"id": "tag_3", "value": "tag 3", "parent_id": "tag_1"}, + {"id": "tag_4", "value": "tag 4"}, + ] + json_data = {"tags": expected_tags} + + json_file = BytesIO(json.dumps(json_data).encode()) + + tags, errors = JSONParser.parse_import(json_file) + self.assertEqual(len(errors), 0) + self.assertEqual(len(tags), 4) + + # Result tags must be in the same order of the file + for index, expected_tag in enumerate(expected_tags): + self.assertEqual( + tags[index].id, + expected_tag.get('id') + ) + self.assertEqual( + tags[index].value, + expected_tag.get('value') + ) + self.assertEqual( + tags[index].parent_id, + expected_tag.get('parent_id') + ) + self.assertEqual( + tags[index].index, + index + JSONParser.inital_row + ) + + def test_export_data(self): + result = JSONParser.export(self.taxonomy) + tags = json.loads(result).get("tags") + assert len(tags) == self.taxonomy.tag_set.count() + for tag in tags: + taxonomy_tag = self.taxonomy.tag_set.get(external_id=tag.get("id")) + assert tag.get("value") == taxonomy_tag.value + if tag.get("parent_id"): + assert tag.get("parent_id") == taxonomy_tag.parent.external_id + + def test_import_with_export_output(self): + output = JSONParser.export(self.taxonomy) + json_file = BytesIO(output.encode()) + tags, errors = JSONParser.parse_import(json_file) + output_tags = json.loads(output).get("tags") + + self.assertEqual(len(errors), 0) + self.assertEqual(len(tags), len(output_tags)) + + + for tag in tags: + output_tag = None + for out_tag in output_tags: + if out_tag.get("id") == tag.id: + output_tag = out_tag + # Don't break because test coverage + assert output_tag + assert output_tag.get("value") == tag.value + if output_tag.get("parent_id"): + assert output_tag.get("parent_id") == tag.parent_id + + +@ddt.ddt +class TestCSVParser(TestImportExportMixin, TestCase): + """ + Test for .csv parser + """ + + @ddt.data( + ( + "value\n", + ["Invalid '.csv' format: Missing 'id' field on CSV headers"], + ), + ( + "id\n", + ["Invalid '.csv' format: Missing 'value' field on CSV headers"], + ), + ( + "id_name,value_name\n", + [ + "Invalid '.csv' format: Missing 'id' field on CSV headers", + "Invalid '.csv' format: Missing 'value' field on CSV headers" + ], + ), + ( + # Valid + "id,value\n", + [] + ) + ) + @ddt.unpack + def test_load_data_errors(self, csv_data, expected_errors): + csv_file = BytesIO(csv_data.encode()) + + tags, errors = CSVParser.parse_import(csv_file) + self.assertEqual(len(tags), 0) + self.assertEqual(len(errors), len(expected_errors)) + + for error in errors: + self.assertIn(str(error), expected_errors) + + @ddt.data( + ( + "id,value\ntag_1\ntag_2,\n", + [ + "Empty 'value' field on the row 2", + "Empty 'value' field on the row 3", + ] + ), + ( + "id,value\ntag_1,tag 1\n", # Valid + [] + ) + ) + @ddt.unpack + def test_parse_tags_errors(self, csv_data, expected_errors): + csv_file = BytesIO(csv_data.encode()) + + _, errors = CSVParser.parse_import(csv_file) + self.assertEqual(len(errors), len(expected_errors)) + + for error in errors: + self.assertIn(str(error), expected_errors) + + def _build_csv(self, tags): + """ + Builds a csv from 'tags' dict + """ + csv = "id,value,parent_id\n" + for tag in tags: + csv += ( + f"{tag.get('id')},{tag.get('value')}," + f"{tag.get('parent_id') or ''}\n" + ) + return csv + + def test_parse_tags(self): + expected_tags = [ + {"id": "tag_1", "value": "tag 1"}, + {"id": "tag_2", "value": "tag 2"}, + {"id": "tag_3", "value": "tag 3", "parent_id": "tag_1"}, + {"id": "tag_4", "value": "tag 4"}, + ] + csv_data = self._build_csv(expected_tags) + csv_file = BytesIO(csv_data.encode()) + tags, errors = CSVParser.parse_import(csv_file) + + self.assertEqual(len(errors), 0) + self.assertEqual(len(tags), 4) + + # Result tags must be in the same order of the file + for index, expected_tag in enumerate(expected_tags): + self.assertEqual( + tags[index].id, + expected_tag.get('id') + ) + self.assertEqual( + tags[index].value, + expected_tag.get('value') + ) + self.assertEqual( + tags[index].parent_id, + expected_tag.get('parent_id') + ) + self.assertEqual( + tags[index].index, + index + CSVParser.inital_row + ) + + def test_import_with_export_output(self): + output = CSVParser.export(self.taxonomy) + csv_file = BytesIO(output.encode()) + tags, errors = CSVParser.parse_import(csv_file) + self.assertEqual(len(errors), 0) + assert len(tags) == self.taxonomy.tag_set.count() + + for tag in tags: + taxonomy_tag = self.taxonomy.tag_set.get(external_id=tag.id) + assert tag.value == taxonomy_tag.value + if tag.parent_id: + assert tag.parent_id == taxonomy_tag.parent.external_id diff --git a/tests/openedx_tagging/core/tagging/import_export/test_tasks.py b/tests/openedx_tagging/core/tagging/import_export/test_tasks.py new file mode 100644 index 000000000..4b88fc6d6 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/import_export/test_tasks.py @@ -0,0 +1,42 @@ +""" +Test import/export celery tasks +""" +from io import BytesIO +from unittest.mock import patch + +from django.test.testcases import TestCase + +from openedx_tagging.core.tagging.import_export import ParserFormat +import openedx_tagging.core.tagging.import_export.tasks as import_export_tasks + +from .mixins import TestImportExportMixin + + +class TestImportExportCeleryTasks(TestImportExportMixin, TestCase): + """ + Test import/export celery tasks + """ + + def test_import_tags_task(self): + file = BytesIO(b"some_data") + parser_format = ParserFormat.CSV + replace = True + + with patch('openedx_tagging.core.tagging.import_export.api.import_tags') as mock_import_tags: + mock_import_tags.return_value = True + + result = import_export_tasks.import_tags_task(self.taxonomy, file, parser_format, replace) + + self.assertTrue(result) + mock_import_tags.assert_called_once_with(self.taxonomy, file, parser_format, replace) + + def test_export_tags_task(self): + output_format = ParserFormat.JSON + + with patch('openedx_tagging.core.tagging.import_export.api.export_tags') as mock_export_tags: + mock_export_tags.return_value = "exported_data" + + result = import_export_tasks.export_tags_task(self.taxonomy, output_format) + + self.assertEqual(result, "exported_data") + mock_export_tags.assert_called_once_with(self.taxonomy, output_format) diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index b6affec4a..5d6d4d0c6 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -4,7 +4,7 @@ from django.test.testcases import TestCase, override_settings import openedx_tagging.core.tagging.api as tagging_api -from openedx_tagging.core.tagging.models import ObjectTag, Tag +from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy from .test_models import TestTagTaxonomyMixin, get_tag @@ -55,20 +55,22 @@ def test_get_taxonomy(self): def test_get_taxonomies(self): tax1 = tagging_api.create_taxonomy("Enabled") tax2 = tagging_api.create_taxonomy("Disabled", enabled=False) + tax3 = Taxonomy.objects.get(name="Import Taxonomy Test") with self.assertNumQueries(1): enabled = list(tagging_api.get_taxonomies()) - assert enabled == [ tax1, + tax3, self.language_taxonomy, self.taxonomy, self.system_taxonomy, self.user_taxonomy, ] assert str(enabled[0]) == f" ({tax1.id}) Enabled" - assert str(enabled[1]) == " (-1) Languages" - assert str(enabled[2]) == " (1) Life on Earth" - assert str(enabled[3]) == " (4) System defined taxonomy" + assert str(enabled[1]) == " (5) Import Taxonomy Test" + assert str(enabled[2]) == " (-1) Languages" + assert str(enabled[3]) == " (1) Life on Earth" + assert str(enabled[4]) == " (4) System defined taxonomy" with self.assertNumQueries(1): disabled = list(tagging_api.get_taxonomies(enabled=False)) @@ -80,6 +82,7 @@ def test_get_taxonomies(self): assert both == [ tax2, tax1, + tax3, self.language_taxonomy, self.taxonomy, self.system_taxonomy,