From 61a19b593976a1226fe295a0af04ec79cca0d462 Mon Sep 17 00:00:00 2001 From: Sujai Kumar Gupta Date: Fri, 21 Feb 2025 01:08:39 +0530 Subject: [PATCH 1/5] move import_channel_by_id to channel_import utils --- kolibri/core/content/utils/channel_import.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/kolibri/core/content/utils/channel_import.py b/kolibri/core/content/utils/channel_import.py index 1cb19f8d90..07d3f9ae29 100644 --- a/kolibri/core/content/utils/channel_import.py +++ b/kolibri/core/content/utils/channel_import.py @@ -5,6 +5,7 @@ from itertools import islice from django.apps import apps +from django.core.management.base import CommandError from django.db.models.fields.related import ForeignKey from sqlalchemy import and_ from sqlalchemy import or_ @@ -39,6 +40,7 @@ from kolibri.core.content.models import LocalFile from kolibri.core.content.utils.annotation import set_channel_ancestors from kolibri.core.content.utils.search import annotate_label_bitmasks +from kolibri.core.errors import KolibriUpgradeError from kolibri.utils.time_utils import local_now logger = logging.getLogger(__name__) @@ -1248,3 +1250,18 @@ def import_channel_from_data(source_data, cancel_check=None, partial=False): ) return import_manager.run_and_annotate() + + +def import_channel_by_id(channel_id, cancel_check, contentfolder=None): + try: + return import_channel_from_local_db( + channel_id, cancel_check=cancel_check, contentfolder=contentfolder + ) + except InvalidSchemaVersionError: + raise CommandError( + "Database file had an invalid database schema, the file may be corrupted or have been modified." + ) + except FutureSchemaError: + raise KolibriUpgradeError( + "Database file uses a future database schema that this version of Kolibri does not support." + ) From 3a1785c67fdaefa6e8165b6b3f6ca90d0efd17be Mon Sep 17 00:00:00 2001 From: Sujai Kumar Gupta Date: Fri, 21 Feb 2025 01:11:50 +0530 Subject: [PATCH 2/5] extract channel_transfer logic from imporchannel command to a seperate module --- .../core/content/constants/transfer_types.py | 2 + .../management/commands/importchannel.py | 193 ++---------------- .../core/content/utils/channel_transfer.py | 174 ++++++++++++++++ 3 files changed, 190 insertions(+), 179 deletions(-) create mode 100644 kolibri/core/content/constants/transfer_types.py create mode 100644 kolibri/core/content/utils/channel_transfer.py diff --git a/kolibri/core/content/constants/transfer_types.py b/kolibri/core/content/constants/transfer_types.py new file mode 100644 index 0000000000..641d6ce627 --- /dev/null +++ b/kolibri/core/content/constants/transfer_types.py @@ -0,0 +1,2 @@ +DOWNLOAD_METHOD = "download" +COPY_METHOD = "copy" diff --git a/kolibri/core/content/management/commands/importchannel.py b/kolibri/core/content/management/commands/importchannel.py index 7cd28e599e..76f6366261 100644 --- a/kolibri/core/content/management/commands/importchannel.py +++ b/kolibri/core/content/management/commands/importchannel.py @@ -1,41 +1,16 @@ import logging -import os from django.core.management.base import CommandError -from le_utils.constants import content_kinds -from ...utils import channel_import from ...utils import paths -from ...utils.annotation import update_content_metadata -from kolibri.core.content.models import ContentNode -from kolibri.core.content.utils.importability_annotation import clear_channel_stats -from kolibri.core.device.models import ContentCacheKey -from kolibri.core.errors import KolibriUpgradeError +from kolibri.core.content.constants.transfer_types import COPY_METHOD +from kolibri.core.content.constants.transfer_types import DOWNLOAD_METHOD +from kolibri.core.content.utils.channel_transfer import transfer_channel from kolibri.core.tasks.management.commands.base import AsyncCommand from kolibri.utils import conf -from kolibri.utils import file_transfer as transfer logger = logging.getLogger(__name__) -# constants to specify the transfer method to be used -DOWNLOAD_METHOD = "download" -COPY_METHOD = "copy" - - -def import_channel_by_id(channel_id, cancel_check, contentfolder=None): - try: - return channel_import.import_channel_from_local_db( - channel_id, cancel_check=cancel_check, contentfolder=contentfolder - ) - except channel_import.InvalidSchemaVersionError: - raise CommandError( - "Database file had an invalid database schema, the file may be corrupted or have been modified." - ) - except channel_import.FutureSchemaError: - raise KolibriUpgradeError( - "Database file uses a future database schema that this version of Kolibri does not support." - ) - class Command(AsyncCommand): def add_arguments(self, parser): @@ -103,166 +78,26 @@ def add_arguments(self, parser): def download_channel(self, channel_id, baseurl, no_upgrade, content_dir): logger.info("Downloading data for channel id {}".format(channel_id)) - self._transfer( - DOWNLOAD_METHOD, - channel_id, - baseurl, + transfer_channel( + job=self, + channel_id=channel_id, + method=DOWNLOAD_METHOD, no_upgrade=no_upgrade, content_dir=content_dir, + baseurl=baseurl, ) - def copy_channel(self, channel_id, path, no_upgrade, content_dir): + def copy_channel(self, channel_id, source_path, no_upgrade, content_dir): logger.info("Copying in data for channel id {}".format(channel_id)) - self._transfer( - COPY_METHOD, - channel_id, - path=path, + transfer_channel( + job=self, + channel_id=channel_id, + method=COPY_METHOD, no_upgrade=no_upgrade, content_dir=content_dir, + source_path=source_path, ) - def _transfer( - self, - method, - channel_id, - baseurl=None, - path=None, - no_upgrade=False, - content_dir=None, - ): - - new_channel_dest = paths.get_upgrade_content_database_file_path( - channel_id, contentfolder=content_dir - ) - dest = ( - new_channel_dest - if no_upgrade - else paths.get_content_database_file_path( - channel_id, contentfolder=content_dir - ) - ) - - # if new channel version db has previously been downloaded, just copy it over - if os.path.exists(new_channel_dest) and not no_upgrade: - method = COPY_METHOD - # determine where we're downloading/copying from, and create appropriate transfer object - if method == DOWNLOAD_METHOD: - url = paths.get_content_database_file_url(channel_id, baseurl=baseurl) - logger.debug("URL to fetch: {}".format(url)) - filetransfer = transfer.FileDownload( - url, dest, cancel_check=self.is_cancelled - ) - elif method == COPY_METHOD: - # if there is a new channel version db, set that as source path - srcpath = ( - new_channel_dest - if os.path.exists(new_channel_dest) - else paths.get_content_database_file_path(channel_id, datafolder=path) - ) - filetransfer = transfer.FileCopy( - srcpath, dest, cancel_check=self.is_cancelled - ) - - logger.debug("Destination: {}".format(dest)) - - try: - self._start_file_transfer( - filetransfer, - channel_id, - dest, - no_upgrade=no_upgrade, - contentfolder=content_dir, - ) - except transfer.TransferCanceled: - pass - - if self.is_cancelled(): - try: - os.remove(dest) - except OSError as e: - logger.info( - "Tried to remove {}, but exception {} occurred.".format(dest, e) - ) - # Reraise any cancellation - self.check_for_cancel() - - # if we are trying to upgrade, remove new channel db - if os.path.exists(new_channel_dest) and not no_upgrade: - os.remove(new_channel_dest) - - def _start_file_transfer( - self, filetransfer, channel_id, dest, no_upgrade=False, contentfolder=None - ): - progress_extra_data = {"channel_id": channel_id} - - with filetransfer: - self.start_progress(total=filetransfer.transfer_size) - - def progress_callback(bytes): - self.update_progress(bytes, extra_data=progress_extra_data) - - filetransfer.run(progress_callback) - # if upgrading, import the channel - if not no_upgrade: - try: - # In each case we need to evaluate the queryset now, - # in order to get node ids as they currently are before - # the import. If we did not coerce each of these querysets - # to a list now, they would be lazily evaluated after the - # import, and would reflect the state of the database - # after the import. - - # evaluate list so we have the current node ids - node_ids = list( - ContentNode.objects.filter( - channel_id=channel_id, available=True - ) - .exclude(kind=content_kinds.TOPIC) - .values_list("id", flat=True) - ) - # evaluate list so we have the current node ids - admin_imported_ids = list( - ContentNode.objects.filter( - channel_id=channel_id, available=True, admin_imported=True - ) - .exclude(kind=content_kinds.TOPIC) - .values_list("id", flat=True) - ) - # evaluate list so we have the current node ids - not_admin_imported_ids = list( - ContentNode.objects.filter( - channel_id=channel_id, available=True, admin_imported=False - ) - .exclude(kind=content_kinds.TOPIC) - .values_list("id", flat=True) - ) - import_ran = import_channel_by_id( - channel_id, self.is_cancelled, contentfolder - ) - if import_ran: - if node_ids: - # annotate default channel db based on previously annotated leaf nodes - update_content_metadata(channel_id, node_ids=node_ids) - if admin_imported_ids: - # Reset admin_imported flag for nodes that were imported by admin - ContentNode.objects.filter_by_uuids( - admin_imported_ids - ).update(admin_imported=True) - if not_admin_imported_ids: - # Reset admin_imported flag for nodes that were not imported by admin - ContentNode.objects.filter_by_uuids( - not_admin_imported_ids - ).update(admin_imported=False) - else: - # ensure the channel is available to the frontend - ContentCacheKey.update_cache_key() - - # Clear any previously set channel availability stats for this channel - clear_channel_stats(channel_id) - except channel_import.ImportCancelError: - # This will only occur if is_cancelled is True. - pass - def handle_async(self, *args, **options): if options["command"] == "network": self.download_channel( diff --git a/kolibri/core/content/utils/channel_transfer.py b/kolibri/core/content/utils/channel_transfer.py new file mode 100644 index 0000000000..79c3e4452b --- /dev/null +++ b/kolibri/core/content/utils/channel_transfer.py @@ -0,0 +1,174 @@ +import logging +import os + +from le_utils.constants import content_kinds + +from kolibri.core.content.constants.transfer_types import COPY_METHOD +from kolibri.core.content.constants.transfer_types import DOWNLOAD_METHOD +from kolibri.core.content.models import ContentNode +from kolibri.core.content.utils import paths +from kolibri.core.content.utils.annotation import update_content_metadata +from kolibri.core.content.utils.channel_import import import_channel_by_id +from kolibri.core.content.utils.channel_import import ImportCancelError +from kolibri.core.content.utils.importability_annotation import clear_channel_stats +from kolibri.core.device.models import ContentCacheKey +from kolibri.utils import file_transfer as transfer + +logger = logging.getLogger(__name__) + + +def start_file_transfer(job, filetransfer, channel_id, dest, no_upgrade, contentfolder): + """ + Runs the file transfer and, if not in "no_upgrade" mode, imports the channel and updates metadata. + + :param job: The job instance; must have is_cancelled() and check_for_cancel() methods. + :param filetransfer: The file transfer object to execute. + :param channel_id: The channel id being transferred. + :param dest: The destination file path. + :param no_upgrade: If True, bypass the channel import. + :param contentfolder: The content folder used during import. + """ + progress_extra_data = {"channel_id": channel_id} + + with filetransfer: + job.start_progress(total=filetransfer.transfer_size) + + def progress_callback(bytes_transferred): + job.update_progress(bytes_transferred, extra_data=progress_extra_data) + + filetransfer.run(progress_callback) + + # if upgrading, import the channel + if not no_upgrade: + try: + # In each case we need to evaluate the queryset now, + # in order to get node ids as they currently are before + # the import. If we did not coerce each of these querysets + # to a list now, they would be lazily evaluated after the + # import, and would reflect the state of the database + # after the import. + + # evaluate list so we have the current node ids + node_ids = list( + ContentNode.objects.filter(channel_id=channel_id, available=True) + .exclude(kind=content_kinds.TOPIC) + .values_list("id", flat=True) + ) + admin_imported_ids = list( + ContentNode.objects.filter( + channel_id=channel_id, available=True, admin_imported=True + ) + .exclude(kind=content_kinds.TOPIC) + .values_list("id", flat=True) + ) + not_admin_imported_ids = list( + ContentNode.objects.filter( + channel_id=channel_id, available=True, admin_imported=False + ) + .exclude(kind=content_kinds.TOPIC) + .values_list("id", flat=True) + ) + import_ran = import_channel_by_id( + channel_id, job.is_cancelled, contentfolder + ) + if import_ran: + if node_ids: + # Annotate default channel DB based on previously annotated leaf nodes. + update_content_metadata(channel_id, node_ids=node_ids) + if admin_imported_ids: + # Reset admin_imported flag for nodes that were imported by admin. + ContentNode.objects.filter_by_uuids( + admin_imported_ids + ).update(admin_imported=True) + if not_admin_imported_ids: + # Reset admin_imported flag for nodes that were not imported by admin. + ContentNode.objects.filter_by_uuids( + not_admin_imported_ids + ).update(admin_imported=False) + else: + # Ensure the channel is available to the frontend. + ContentCacheKey.update_cache_key() + + # Clear any previously set channel availability stats for this channel. + clear_channel_stats(channel_id) + except ImportCancelError: + # This will only occur if job.is_cancelled() is True. + pass + + +def transfer_channel( + job, + channel_id, + method, + no_upgrade=False, + content_dir=None, + baseurl=None, + source_path=None, +): + """ + Transfers a channel database either by downloading or copying + + :param job: The job instance; must have is_cancelled() and check_for_cancel() methods. + :param channel_id: The channel id to transfer. + :param method: The transfer method (DOWNLOAD_METHOD or COPY_METHOD). + :param no_upgrade: If True, only download the database to an upgrade file path. + :param content_dir: The content directory. + :param baseurl: The base URL from which to download (if applicable). + :param source_path: The source path (if copying). + :return: The destination path of the transferred channel database. + """ + new_channel_dest = paths.get_upgrade_content_database_file_path( + channel_id, contentfolder=content_dir + ) + dest = ( + new_channel_dest + if no_upgrade + else paths.get_content_database_file_path(channel_id, contentfolder=content_dir) + ) + + # If a new channel version DB has previously been downloaded, just copy it over. + if os.path.exists(new_channel_dest) and not no_upgrade: + method = COPY_METHOD + + # Determine where we're downloading/copying from, and create the appropriate transfer object. + if method == DOWNLOAD_METHOD: + url = paths.get_content_database_file_url(channel_id, baseurl=baseurl) + logger.debug("URL to fetch: {}".format(url)) + filetransfer = transfer.FileDownload(url, dest, cancel_check=job.is_cancelled) + elif method == COPY_METHOD: + # If there is a new channel version DB, set that as source path. + srcpath = ( + new_channel_dest + if os.path.exists(new_channel_dest) + else paths.get_content_database_file_path( + channel_id, datafolder=source_path + ) + ) + filetransfer = transfer.FileCopy(srcpath, dest, cancel_check=job.is_cancelled) + else: + raise ValueError("Invalid transfer method specified: {}".format(method)) + + logger.debug("Destination: {}".format(dest)) + + try: + start_file_transfer( + job, filetransfer, channel_id, dest, no_upgrade, content_dir + ) + except transfer.TransferCanceled: + pass + + if job.is_cancelled(): + try: + os.remove(dest) + except OSError as e: + logger.info( + "Tried to remove {}, but exception {} occurred.".format(dest, e) + ) + # Reraise any cancellation. + job.check_for_cancel() + + # If we are trying to upgrade, remove the new channel DB. + if os.path.exists(new_channel_dest) and not no_upgrade: + os.remove(new_channel_dest) + + return dest From 3d8666b7acf33cd2dda9c02df2126ac98199ef68 Mon Sep 17 00:00:00 2001 From: Sujai Kumar Gupta Date: Fri, 21 Feb 2025 01:14:58 +0530 Subject: [PATCH 3/5] use the transfer_channel util for importchannel task instead of invoking management commands --- kolibri/core/content/tasks.py | 38 ++++++------------- .../core/content/test/test_import_export.py | 2 + 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/kolibri/core/content/tasks.py b/kolibri/core/content/tasks.py index 1a6fd52ba2..f50a725973 100644 --- a/kolibri/core/content/tasks.py +++ b/kolibri/core/content/tasks.py @@ -4,12 +4,15 @@ from rest_framework import serializers from kolibri.core.auth.models import FacilityDataset +from kolibri.core.content.constants.transfer_types import COPY_METHOD +from kolibri.core.content.constants.transfer_types import DOWNLOAD_METHOD from kolibri.core.content.models import ChannelMetadata from kolibri.core.content.models import ContentRequest from kolibri.core.content.models import ContentRequestReason from kolibri.core.content.models import ContentRequestStatus from kolibri.core.content.models import ContentRequestType from kolibri.core.content.utils.channel_import import import_channel_from_data +from kolibri.core.content.utils.channel_transfer import transfer_channel from kolibri.core.content.utils.channels import get_mounted_drive_by_id from kolibri.core.content.utils.channels import read_channel_metadata_from_db_file from kolibri.core.content.utils.content_request import incomplete_removals_queryset @@ -218,12 +221,8 @@ class RemoteChannelImportValidator(RemoteImportMixin, ChannelValidator): status_fn=get_status, ) def remotechannelimport(channel_id, baseurl=None, peer_id=None): - call_command( - "importchannel", - "network", - channel_id, - baseurl=baseurl, - ) + job = get_current_job() + transfer_channel(job, channel_id, DOWNLOAD_METHOD, baseurl=baseurl) class RemoteChannelResourcesImportValidator( @@ -538,15 +537,10 @@ def remoteimport( fail_on_error=False, all_thumbnails=False, ): - call_command( - "importchannel", - "network", - channel_id, - baseurl=baseurl, - ) + current_job = get_current_job() + transfer_channel(current_job, channel_id, DOWNLOAD_METHOD, baseurl=baseurl) if update: - current_job = get_current_job() current_job.update_metadata(database_ready=True) manager_class = ( @@ -587,15 +581,11 @@ def diskimport( drive = get_mounted_drive_by_id(drive_id) directory = drive.datafolder - call_command( - "importchannel", - "disk", - channel_id, - directory, - ) + current_job = get_current_job() + + transfer_channel(current_job, channel_id, COPY_METHOD, source_path=directory) if update: - current_job = get_current_job() current_job.update_metadata(database_ready=True) manager_class = ( @@ -633,12 +623,8 @@ def diskchannelimport( drive_id, ): drive = get_mounted_drive_by_id(drive_id) - call_command( - "importchannel", - "disk", - channel_id, - drive.datafolder, - ) + job = get_current_job() + transfer_channel(job, channel_id, COPY_METHOD, source_path=drive.datafolder) class RemoteChannelDiffStatsValidator(RemoteChannelImportValidator): diff --git a/kolibri/core/content/test/test_import_export.py b/kolibri/core/content/test/test_import_export.py index 857d3c6e63..097f17e67f 100644 --- a/kolibri/core/content/test/test_import_export.py +++ b/kolibri/core/content/test/test_import_export.py @@ -4,6 +4,7 @@ import sys import tempfile import time +import unittest import uuid from io import StringIO @@ -454,6 +455,7 @@ def test_empty_query(self): self.assertEqual(total_bytes_to_transfer, 0) +@unittest.skip(reason="TODO: New test case for channelimport") @patch( "kolibri.core.content.management.commands.importchannel.channel_import.import_channel_from_local_db" ) From 4d4beeaa4466d13b4ce09e5e7b234e0602416a71 Mon Sep 17 00:00:00 2001 From: Sujai Kumar Gupta Date: Fri, 21 Feb 2025 10:50:20 +0530 Subject: [PATCH 4/5] dont pass the job to channel_transfer --- .../management/commands/importchannel.py | 2 -- kolibri/core/content/tasks.py | 16 ++++----- .../core/content/utils/channel_transfer.py | 34 +++++++++++++++---- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/kolibri/core/content/management/commands/importchannel.py b/kolibri/core/content/management/commands/importchannel.py index 76f6366261..cf2aa86401 100644 --- a/kolibri/core/content/management/commands/importchannel.py +++ b/kolibri/core/content/management/commands/importchannel.py @@ -79,7 +79,6 @@ def add_arguments(self, parser): def download_channel(self, channel_id, baseurl, no_upgrade, content_dir): logger.info("Downloading data for channel id {}".format(channel_id)) transfer_channel( - job=self, channel_id=channel_id, method=DOWNLOAD_METHOD, no_upgrade=no_upgrade, @@ -90,7 +89,6 @@ def download_channel(self, channel_id, baseurl, no_upgrade, content_dir): def copy_channel(self, channel_id, source_path, no_upgrade, content_dir): logger.info("Copying in data for channel id {}".format(channel_id)) transfer_channel( - job=self, channel_id=channel_id, method=COPY_METHOD, no_upgrade=no_upgrade, diff --git a/kolibri/core/content/tasks.py b/kolibri/core/content/tasks.py index f50a725973..83e7c3cea3 100644 --- a/kolibri/core/content/tasks.py +++ b/kolibri/core/content/tasks.py @@ -221,8 +221,7 @@ class RemoteChannelImportValidator(RemoteImportMixin, ChannelValidator): status_fn=get_status, ) def remotechannelimport(channel_id, baseurl=None, peer_id=None): - job = get_current_job() - transfer_channel(job, channel_id, DOWNLOAD_METHOD, baseurl=baseurl) + transfer_channel(channel_id, DOWNLOAD_METHOD, baseurl=baseurl) class RemoteChannelResourcesImportValidator( @@ -538,9 +537,9 @@ def remoteimport( all_thumbnails=False, ): - current_job = get_current_job() - transfer_channel(current_job, channel_id, DOWNLOAD_METHOD, baseurl=baseurl) + transfer_channel(channel_id, DOWNLOAD_METHOD, baseurl=baseurl) if update: + current_job = get_current_job() current_job.update_metadata(database_ready=True) manager_class = ( @@ -581,17 +580,15 @@ def diskimport( drive = get_mounted_drive_by_id(drive_id) directory = drive.datafolder - current_job = get_current_job() - - transfer_channel(current_job, channel_id, COPY_METHOD, source_path=directory) + transfer_channel(channel_id, COPY_METHOD, source_path=directory) if update: + current_job = get_current_job() current_job.update_metadata(database_ready=True) manager_class = ( DiskChannelUpdateManager if update else DiskChannelResourceImportManager ) - drive = get_mounted_drive_by_id(drive_id) manager = manager_class( channel_id, path=drive.datafolder, @@ -623,8 +620,7 @@ def diskchannelimport( drive_id, ): drive = get_mounted_drive_by_id(drive_id) - job = get_current_job() - transfer_channel(job, channel_id, COPY_METHOD, source_path=drive.datafolder) + transfer_channel(channel_id, COPY_METHOD, source_path=drive.datafolder) class RemoteChannelDiffStatsValidator(RemoteChannelImportValidator): diff --git a/kolibri/core/content/utils/channel_transfer.py b/kolibri/core/content/utils/channel_transfer.py index 79c3e4452b..232fc40929 100644 --- a/kolibri/core/content/utils/channel_transfer.py +++ b/kolibri/core/content/utils/channel_transfer.py @@ -12,22 +12,44 @@ from kolibri.core.content.utils.channel_import import ImportCancelError from kolibri.core.content.utils.importability_annotation import clear_channel_stats from kolibri.core.device.models import ContentCacheKey +from kolibri.core.tasks.utils import get_current_job from kolibri.utils import file_transfer as transfer logger = logging.getLogger(__name__) -def start_file_transfer(job, filetransfer, channel_id, dest, no_upgrade, contentfolder): +class DummyJob: + def is_cancelled(self): + return False + + def check_for_cancel(self): + pass + + def start_progress(self, total): + pass + + def update_progress(self, bytes_transferred, extra_data=None): + pass + + +def get_job(): + job = get_current_job() + if job is None: + return DummyJob() + return job + + +def start_file_transfer(filetransfer, channel_id, dest, no_upgrade, contentfolder): """ Runs the file transfer and, if not in "no_upgrade" mode, imports the channel and updates metadata. - :param job: The job instance; must have is_cancelled() and check_for_cancel() methods. :param filetransfer: The file transfer object to execute. :param channel_id: The channel id being transferred. :param dest: The destination file path. :param no_upgrade: If True, bypass the channel import. :param contentfolder: The content folder used during import. """ + job = get_job() progress_extra_data = {"channel_id": channel_id} with filetransfer: @@ -97,7 +119,6 @@ def progress_callback(bytes_transferred): def transfer_channel( - job, channel_id, method, no_upgrade=False, @@ -108,7 +129,6 @@ def transfer_channel( """ Transfers a channel database either by downloading or copying - :param job: The job instance; must have is_cancelled() and check_for_cancel() methods. :param channel_id: The channel id to transfer. :param method: The transfer method (DOWNLOAD_METHOD or COPY_METHOD). :param no_upgrade: If True, only download the database to an upgrade file path. @@ -117,6 +137,8 @@ def transfer_channel( :param source_path: The source path (if copying). :return: The destination path of the transferred channel database. """ + job = get_job() + new_channel_dest = paths.get_upgrade_content_database_file_path( channel_id, contentfolder=content_dir ) @@ -151,9 +173,7 @@ def transfer_channel( logger.debug("Destination: {}".format(dest)) try: - start_file_transfer( - job, filetransfer, channel_id, dest, no_upgrade, content_dir - ) + start_file_transfer(filetransfer, channel_id, dest, no_upgrade, content_dir) except transfer.TransferCanceled: pass From b8baf27aadbf8f1c4deb99ed23a29e7462f11833 Mon Sep 17 00:00:00 2001 From: Sujai Kumar Gupta Date: Fri, 21 Feb 2025 10:51:46 +0530 Subject: [PATCH 5/5] update the importchannel tests with new mock paths --- .../core/content/test/test_import_export.py | 126 +++++++----------- 1 file changed, 50 insertions(+), 76 deletions(-) diff --git a/kolibri/core/content/test/test_import_export.py b/kolibri/core/content/test/test_import_export.py index 097f17e67f..8eb40ed6bf 100644 --- a/kolibri/core/content/test/test_import_export.py +++ b/kolibri/core/content/test/test_import_export.py @@ -4,7 +4,6 @@ import sys import tempfile import time -import unittest import uuid from io import StringIO @@ -455,10 +454,7 @@ def test_empty_query(self): self.assertEqual(total_bytes_to_transfer, 0) -@unittest.skip(reason="TODO: New test case for channelimport") -@patch( - "kolibri.core.content.management.commands.importchannel.channel_import.import_channel_from_local_db" -) +@patch("kolibri.core.content.utils.channel_import.import_channel_from_local_db") @patch( "kolibri.core.content.management.commands.importchannel.AsyncCommand.start_progress" ) @@ -470,33 +466,34 @@ class ImportChannelTestCase(TestCase): the_channel_id = "6199dde695db4ee4ab392222d5af1e5c" + def _create_dummy_job(self, is_cancelled=True, check_for_cancel_return=True): + dummy = MagicMock() + dummy.is_cancelled.return_value = is_cancelled + dummy.check_for_cancel.return_value = check_for_cancel_return + dummy.start_progress.return_value = None + dummy.update_progress.return_value = None + return dummy + @patch( - "kolibri.core.content.management.commands.importchannel.paths.get_content_database_file_url" - ) - @patch( - "kolibri.core.content.management.commands.importchannel.paths.get_content_database_file_path" - ) - @patch( - "kolibri.core.content.management.commands.importchannel.transfer.FileDownload" - ) - @patch( - "kolibri.core.content.management.commands.importchannel.AsyncCommand.check_for_cancel", - return_value=True, + "kolibri.core.content.utils.channel_transfer.paths.get_content_database_file_url" ) @patch( - "kolibri.core.content.management.commands.importchannel.AsyncCommand.is_cancelled", - return_value=True, + "kolibri.core.content.utils.channel_transfer.paths.get_content_database_file_path" ) + @patch("kolibri.core.content.utils.channel_transfer.transfer.FileDownload") + @patch("kolibri.core.content.utils.channel_transfer.get_current_job") def test_remote_cancel_during_transfer( self, - is_cancelled_mock, - cancel_mock, + get_current_job_mock, FileDownloadMock, local_path_mock, remote_path_mock, start_progress_mock, import_channel_mock, ): + + dummy_job = self._create_dummy_job() + get_current_job_mock.return_value = dummy_job fd, local_path = tempfile.mkstemp() os.close(fd) local_path_mock.return_value = local_path @@ -504,37 +501,31 @@ def test_remote_cancel_during_transfer( FileDownloadMock.return_value.run.side_effect = TransferCanceled() call_command("importchannel", "network", self.the_channel_id) # Check that is_cancelled was called - is_cancelled_mock.assert_called_with() + dummy_job.is_cancelled.assert_called_with() # Check that the FileDownload initiated FileDownloadMock.assert_called_with( - "notest", local_path, cancel_check=is_cancelled_mock + "notest", local_path, cancel_check=dummy_job.is_cancelled ) # Check that cancel was called - cancel_mock.assert_called_with() + dummy_job.check_for_cancel.assert_called_with() # Test that import channel cleans up database file if cancelled self.assertFalse(os.path.exists(local_path)) @patch( - "kolibri.core.content.management.commands.importchannel.paths.get_content_database_file_path" - ) - @patch("kolibri.core.content.management.commands.importchannel.transfer.FileCopy") - @patch( - "kolibri.core.content.management.commands.importchannel.AsyncCommand.check_for_cancel", - return_value=True, - ) - @patch( - "kolibri.core.content.management.commands.importchannel.AsyncCommand.is_cancelled", - return_value=True, + "kolibri.core.content.utils.channel_transfer.paths.get_content_database_file_path" ) + @patch("kolibri.core.content.utils.channel_transfer.transfer.FileCopy") + @patch("kolibri.core.content.utils.channel_transfer.get_current_job") def test_local_cancel_during_transfer( self, - is_cancelled_mock, - cancel_mock, + get_current_job_mock, FileCopyMock, local_path_mock, start_progress_mock, import_channel_mock, ): + dummy_job = self._create_dummy_job() + get_current_job_mock.return_value = dummy_job fd1, local_dest_path = tempfile.mkstemp() fd2, local_src_path = tempfile.mkstemp() os.close(fd1) @@ -543,26 +534,19 @@ def test_local_cancel_during_transfer( FileCopyMock.return_value.run.side_effect = TransferCanceled() call_command("importchannel", "disk", self.the_channel_id, tempfile.mkdtemp()) # Check that is_cancelled was called - is_cancelled_mock.assert_called_with() - # Check that the FileCopy initiated + dummy_job.is_cancelled.assert_called() FileCopyMock.assert_called_with( - local_src_path, local_dest_path, cancel_check=is_cancelled_mock + local_src_path, local_dest_path, cancel_check=dummy_job.is_cancelled ) - # Check that cancel was called - cancel_mock.assert_called_with() - # Test that import channel cleans up database file if cancelled + dummy_job.check_for_cancel.assert_called() self.assertFalse(os.path.exists(local_dest_path)) - @patch( - "kolibri.core.content.management.commands.importchannel.AsyncCommand.check_for_cancel" - ) - @patch( - "kolibri.core.content.management.commands.importchannel.AsyncCommand.is_cancelled", - return_value=True, - ) + @patch("kolibri.core.content.utils.channel_transfer.get_current_job") def test_remote_import_sslerror( - self, is_cancelled_mock, cancel_mock, start_progress_mock, import_channel_mock + self, get_current_job_mock, start_progress_mock, import_channel_mock ): + dummy_job = self._create_dummy_job() + get_current_job_mock.return_value = dummy_job SSLERROR = SSLError( ["SSL routines", "ssl3_get_record", "decryption failed or bad record mac"] ) @@ -582,56 +566,48 @@ def test_remote_import_sslerror( side_effect=SSLERROR, ): call_command("importchannel", "network", "197934f144305350b5820c7c4dd8e194") - cancel_mock.assert_called_with() + dummy_job.check_for_cancel.assert_called_with() import_channel_mock.assert_not_called() @patch( "kolibri.utils.file_transfer.FileDownload._run_download", side_effect=ReadTimeout("Read timed out."), ) - @patch( - "kolibri.core.content.management.commands.importchannel.AsyncCommand.check_for_cancel" - ) - @patch( - "kolibri.core.content.management.commands.importchannel.AsyncCommand.is_cancelled", - return_value=True, - ) + @patch("kolibri.core.content.utils.channel_transfer.get_current_job") def test_remote_import_readtimeout( self, - is_cancelled_mock, - cancel_mock, + get_current_job_mock, sslerror_mock, start_progress_mock, import_channel_mock, ): + dummy_job = self._create_dummy_job() + get_current_job_mock.return_value = dummy_job call_command("importchannel", "network", "197934f144305350b5820c7c4dd8e194") - cancel_mock.assert_called_with() + dummy_job.check_for_cancel.assert_called_with() import_channel_mock.assert_not_called() - @patch( - "kolibri.core.content.management.commands.importchannel.transfer.FileDownload" - ) - @patch( - "kolibri.core.content.management.commands.importchannel.AsyncCommand.is_cancelled", - return_value=False, - ) + @patch("kolibri.core.content.utils.channel_transfer.transfer.FileDownload") + @patch("kolibri.core.content.utils.channel_transfer.get_current_job") def test_remote_import_full_import( self, - is_cancelled_mock, + get_current_job_mock, FileDownloadMock, start_progress_mock, import_channel_mock, ): + dummy_job = self._create_dummy_job() + get_current_job_mock.return_value = dummy_job # Get the current content cache key and sleep a bit to ensure # time has elapsed before it's updated. cache_key_before = ContentCacheKey.get_cache_key() time.sleep(0.01) call_command("importchannel", "network", "197934f144305350b5820c7c4dd8e194") - is_cancelled_mock.assert_called() + dummy_job.is_cancelled.assert_called() import_channel_mock.assert_called_with( "197934f144305350b5820c7c4dd8e194", - cancel_check=is_cancelled_mock, + cancel_check=dummy_job.is_cancelled, contentfolder=paths.get_content_dir_path(), ) @@ -640,15 +616,13 @@ def test_remote_import_full_import( self.assertNotEqual(cache_key_before, cache_key_after) @patch( - "kolibri.core.content.management.commands.importchannel.paths.get_content_database_file_url" - ) - @patch( - "kolibri.core.content.management.commands.importchannel.paths.get_content_database_file_path" + "kolibri.core.content.utils.channel_transfer.paths.get_content_database_file_url" ) @patch( - "kolibri.core.content.management.commands.importchannel.transfer.FileDownload" + "kolibri.core.content.utils.channel_transfer.paths.get_content_database_file_path" ) - @patch("kolibri.core.content.management.commands.importchannel.clear_channel_stats") + @patch("kolibri.core.content.utils.channel_transfer.transfer.FileDownload") + @patch("kolibri.core.content.utils.channel_transfer.clear_channel_stats") def test_remote_successful_import_clears_stats_cache( self, channel_stats_clear_mock,