From 3c44b41cf1a38c49a267fc56a659e2b029f4c6a5 Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Wed, 24 Jan 2024 13:00:14 +0000 Subject: [PATCH 1/5] Add a helper to get the object store ids and the datasets size for every dataset in a job --- tpv/core/helpers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tpv/core/helpers.py b/tpv/core/helpers.py index 3ddf504..9c600a3 100644 --- a/tpv/core/helpers.py +++ b/tpv/core/helpers.py @@ -114,3 +114,13 @@ def tool_version_gte(tool, version): def tool_version_gt(tool, version): return parse_version(tool.version) > parse_version(version) + + +def object_store_ids_and_dataset_size(job): + # Return a dictionary of dataset ids and their object store ids + # and file sizes in bytes for all input datasets in a job + object_store_ids = {} + for i in job.get_input_datasets(): + object_store_ids[i.dataset.dataset.id] = (i.dataset.dataset.object_store_id, float(i.dataset.dataset.get_size(calculate_size=False))) + + return object_store_ids From 8aa024dfa7d7e021ee8d828b264a831790574a69 Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Wed, 24 Jan 2024 14:54:52 +0100 Subject: [PATCH 2/5] Update tpv/core/helpers.py Co-authored-by: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> --- tpv/core/helpers.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tpv/core/helpers.py b/tpv/core/helpers.py index 9c600a3..75543fe 100644 --- a/tpv/core/helpers.py +++ b/tpv/core/helpers.py @@ -119,8 +119,4 @@ def tool_version_gt(tool, version): def object_store_ids_and_dataset_size(job): # Return a dictionary of dataset ids and their object store ids # and file sizes in bytes for all input datasets in a job - object_store_ids = {} - for i in job.get_input_datasets(): - object_store_ids[i.dataset.dataset.id] = (i.dataset.dataset.object_store_id, float(i.dataset.dataset.get_size(calculate_size=False))) - - return object_store_ids + return {i.dataset.dataset.id: {'object_store_id': i.dataset.dataset.object_store_id, 'size': get_dataset_size(i.dataset.dataset)} for i in datasets or {}} From e92d23b45055ad813c2bf9f7dffc461c5a55b181 Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Wed, 24 Jan 2024 14:58:15 +0100 Subject: [PATCH 3/5] Update tpv/core/helpers.py Co-authored-by: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> --- tpv/core/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpv/core/helpers.py b/tpv/core/helpers.py index 75543fe..232e7ad 100644 --- a/tpv/core/helpers.py +++ b/tpv/core/helpers.py @@ -116,7 +116,7 @@ def tool_version_gt(tool, version): return parse_version(tool.version) > parse_version(version) -def object_store_ids_and_dataset_size(job): +def get_dataset_attributes(datasets): # Return a dictionary of dataset ids and their object store ids # and file sizes in bytes for all input datasets in a job return {i.dataset.dataset.id: {'object_store_id': i.dataset.dataset.object_store_id, 'size': get_dataset_size(i.dataset.dataset)} for i in datasets or {}} From a8221ea28b55760e953be40c671b6b8a75b3578f Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Wed, 24 Jan 2024 18:01:28 +0100 Subject: [PATCH 4/5] Update tpv/core/helpers.py Co-authored-by: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> --- tpv/core/helpers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tpv/core/helpers.py b/tpv/core/helpers.py index 232e7ad..343b79c 100644 --- a/tpv/core/helpers.py +++ b/tpv/core/helpers.py @@ -119,4 +119,9 @@ def tool_version_gt(tool, version): def get_dataset_attributes(datasets): # Return a dictionary of dataset ids and their object store ids # and file sizes in bytes for all input datasets in a job - return {i.dataset.dataset.id: {'object_store_id': i.dataset.dataset.object_store_id, 'size': get_dataset_size(i.dataset.dataset)} for i in datasets or {}} + return { + i.dataset.dataset.id: { + 'object_store_id': i.dataset.dataset.object_store_id, + 'size': get_dataset_size(i.dataset.dataset)} + for i in datasets or {} + } From 933a912f9206dad045bfbd9db18f414220c59835 Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Thu, 25 Jan 2024 08:52:44 +0000 Subject: [PATCH 5/5] Add test for the get_dataset_attributes helper function --- tests/test_helpers.py | 20 ++++++++++++++++++++ tpv/commands/test/mock_galaxy.py | 3 ++- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 tests/test_helpers.py diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..eee7492 --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,20 @@ +"""Unit tests module for the helper functions""" +import unittest +from tpv.commands.test import mock_galaxy +from tpv.core.helpers import get_dataset_attributes + + +class TestHelpers(unittest.TestCase): + """Tests for helper functions""" + def test_get_dataset_attributes(self): + """Test that the function returns a dictionary with the correct attributes""" + job = mock_galaxy.Job() + job.add_input_dataset( + mock_galaxy.DatasetAssociation( + "test", + mock_galaxy.Dataset("test.txt", file_size=7*1024**3, object_store_id="files1") + ) + ) + dataset_attributes = get_dataset_attributes(job.input_datasets) + expected_result = {0: {'object_store_id': 'files1', 'size': 7*1024**3}} + self.assertEqual(dataset_attributes, expected_result) diff --git a/tpv/commands/test/mock_galaxy.py b/tpv/commands/test/mock_galaxy.py index 08deb69..be3a82f 100644 --- a/tpv/commands/test/mock_galaxy.py +++ b/tpv/commands/test/mock_galaxy.py @@ -37,11 +37,12 @@ def __init__(self, name, dataset): class Dataset: counter = 0 - def __init__(self, file_name, file_size): + def __init__(self, file_name, file_size, object_store_id=None): self.id = self.counter self.counter += 1 self.file_name = file_name self.file_size = file_size + self.object_store_id = object_store_id def get_size(self, calculate_size=False): return self.file_size