Merge pull request #135 from NaveenKinnal/feature/task-353

Changes related to file and job deletion based on start and end dates

client/setup.cfg

@@ -7,11 +7,14 @@ universal = false
 
 [flake8]
 ignore =
-    E203,  # space before : (needed for how black formats slicing)
-    # E266,  # too many leading '#' for block comment
-    E501,  # line too long
-    E731,  # do not assign a lambda expression, use a def
-    W503,  # line break before binary operator
+    # space before : (needed for how black formats slicing)
+    E203,
+    # line too long
+    E501,
+    # do not assign a lambda expression, use a def
+    E731,
+    # line break before binary operator
+    W503,
 max-line-length = 88
 # select = B,C,E,F,W,T4,B9
 # max-complexity = 18

client/src/lab_client/environment.py

@@ -1,5 +1,6 @@
+from datetime import datetime
 import os
-from typing import Optional, Literal
+from typing import Optional, Literal, List
 
 from contaxy.clients import AuthClient, FileClient, ExtensionClient
 from contaxy.clients import DeploymentClient
@@ -301,6 +302,43 @@ def get_file_metadata(
         """
         return self.file_handler.get_file_metadata(project, key, version)
 
+    def delete_remote_file(
+        self, key: str, version: Optional[str] = None, keep_latest_version: bool = False
+    ) -> None:
+        """Deletes a file based on the specified 'key'.
+
+        Args:
+            key (str): Key or url of the requested file.
+            version (Optional[str], optional): Version of the file to be deleted. Defaults to None.
+            keep_latest_version (bool, optional): _description_. Defaults to False.
+        """
+        self.file_handler.delete_remote_file(key, version, keep_latest_version)
+
+    def delete_remote_files(
+        self, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None
+    ) -> None:
+        """Deletes all files (models/datasets) of a project.
+
+        Args:
+            date_from (Optional[datetime], optional): The start date (in UTC format) from which the file has to be deleted. If none, all files will be deleted.
+            date_to (Optional[datetime], optional): The end date (in UTC format) until which the file has to be deleted. If none, all files will be deleted.
+        """
+        self.file_handler.delete_remote_files(date_from, date_to)
+
+    def list_remote_files(
+        self, data_type: Literal['model', 'dataset'] = None, prefix: str = None
+    ) -> List[File]:
+        """Lists all the remote files.
+
+        Args:
+            data_type (Literal['model', 'dataset'], optional): The date type of the files to list. Defaults to None.
+            prefix (str, optional): The file name prefix. If 'None', all files would be listed.
+
+        Returns:
+            List[File]: List of the remote files.
+        """
+        return self.file_handler.list_remote_files(data_type, prefix)
+
     @property
     def mlflow_handler(self) -> MLFlowHandler:
         if self._mlflow_handler is None:

client/src/lab_client/handler/file_handler.py

@@ -12,6 +12,7 @@
 from lab_client.utils import file_handler_utils, request_utils
 from zipfile import ZipFile
 import shutil
+from datetime import datetime
 
 VALID_DATATYPES = ['dataset', 'model']
 
@@ -182,6 +183,21 @@ def delete_remote_file(
             keep_latest_version=keep_latest_version,
         )
 
+    def delete_remote_files(
+        self, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None
+    ) -> None:
+        """
+        Delete a file from remote storage.
+        # Arguments
+            key: Key of the file.
+            keep_latest_version: If `True` the latest file version will be kept.
+        """
+        self.file_client.delete_files(
+            project_id=self.env.project,
+            date_from=date_from,
+            date_to=date_to
+        )
+
     def resolve_path_from_key(self, key: str, version: Optional[str] = None) -> str:
         """
         Return the local path for a given key.
@@ -259,3 +275,4 @@ def get_file_metadata(
                                                            file_key, version)
 
         return metadata_file
+

client/src/lab_client/handler/job_handler.py

@@ -205,11 +205,13 @@ def delete_job(self, job_id: str) -> None:
             job_id=job_id
         )
 
-    def delete_jobs(self) -> None:
+    def delete_jobs(self, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None) -> None:
         """Deletes all jobs of a project.
         """
         self.deployment_client.delete_jobs(
-            project_id=self.env.project
+            project_id=self.env.project,
+            date_from=date_from,
+            date_to=date_to
         )
 
     def get_job_logs(

client/tests/test_deployment_handler.py

@@ -3,7 +3,7 @@
 from lab_client import Environment
 from .conftest import test_settings
 import pytest
-
+from datetime import datetime, timedelta, timezone
 
 @pytest.mark.integration
 class TestJob:
@@ -51,6 +51,52 @@ def test_list_jobs_get_job_metadata(self) -> None:
         job_list = env.job_handler.list_jobs()
         assert len(job_list) == 0
 
+    def test_delete_jobs_within_time_period(self) -> None:
+        env = Environment(lab_endpoint=test_settings.LAB_BACKEND,
+                          lab_api_token=test_settings.LAB_TOKEN,
+                          project=test_settings.LAB_PROJECT)
+
+        input = JobInput(
+            container_image='ubuntu:latest',
+            display_name='Job2',
+            command=['/bin/bash', '-c', '--'],
+            args=['sleep 5']
+        )
+        job_id = env.job_handler.deploy_job(input)
+
+        status = env.job_handler.wait_for_job_completion(job_id)
+        assert status == True
+
+        date_from = datetime.now(timezone.utc)
+        date_to = datetime.now(timezone.utc) + timedelta(days=1)
+
+        env.job_handler.delete_jobs(date_from, date_to)
+        job_list = env.job_handler.list_jobs()
+        assert len(job_list) == 0
+
+    def test_delete_jobs_outside_time_period(self) -> None:
+        env = Environment(lab_endpoint=test_settings.LAB_BACKEND,
+                          lab_api_token=test_settings.LAB_TOKEN,
+                          project=test_settings.LAB_PROJECT)
+
+        input = JobInput(
+            container_image='ubuntu:latest',
+            display_name='Job2',
+            command=['/bin/bash', '-c', '--'],
+            args=['sleep 5']
+        )
+        job_id = env.job_handler.deploy_job(input)
+
+        status = env.job_handler.wait_for_job_completion(job_id)
+        assert status == True
+
+        date_from = datetime.now(timezone.utc) - timedelta(days=3)
+        date_to = datetime.now(timezone.utc) - timedelta(days=2)
+
+        env.job_handler.delete_jobs(date_from, date_to)
+        job_list = env.job_handler.list_jobs()
+        assert len(job_list) == 1
+
     def test_job_logs(self) -> None:
         env = Environment(lab_endpoint=test_settings.LAB_BACKEND,
                           lab_api_token=test_settings.LAB_TOKEN,

client/tests/test_file_handler.py

@@ -5,7 +5,7 @@
 from .conftest import test_settings
 import requests
 import pytest
-
+from datetime import datetime, timedelta, timezone
 
 @pytest.mark.integration
 class TestFile:
@@ -184,3 +184,65 @@ def test_list_file_with_invalid_data_type(self) -> None:
         env.upload_file(tf.name, "dataset")
 
         env.file_handler.list_remote_files(data_type="datasets")
+
+    @pytest.mark.xfail(raises=Exception)
+    def test_delete_single_file(self) -> None:
+        env = Environment(lab_endpoint=test_settings.LAB_BACKEND,
+                          lab_api_token=test_settings.LAB_TOKEN,
+                          project=test_settings.LAB_PROJECT)
+
+        tf = tempfile.NamedTemporaryFile()
+        with open(tf.name, 'w') as f:
+            f.write("content 1")
+            f.close()
+        key = env.upload_file(tf.name, "dataset")
+
+        env.delete_remote_file(key)
+        # This should throw a 404 error as file will not be present.
+        env.get_file(key)
+
+    def test_delete_files_within_time_window(self) -> None:
+        env = Environment(lab_endpoint=test_settings.LAB_BACKEND,
+                          lab_api_token=test_settings.LAB_TOKEN,
+                          project=test_settings.LAB_PROJECT)
+
+        # Clear/Delete all files initially.
+        env.delete_remote_files()
+
+        # Test that files within the date window are deleted.
+        tf = tempfile.NamedTemporaryFile(prefix='abcd_')
+        with open(tf.name, 'w') as f:
+            f.write("content 1")
+            f.close()
+        env.upload_file(tf.name, "dataset")
+
+        date_from = datetime.now(timezone.utc)
+        date_to = datetime.now(timezone.utc) + timedelta(days=1)
+
+        env.delete_remote_files(date_from, date_to)
+
+        files = env.list_remote_files(data_type='dataset')
+        assert len(files) == 0
+
+    def test_delete_files_outside_time_window(self) -> None:
+        env = Environment(lab_endpoint=test_settings.LAB_BACKEND,
+                          lab_api_token=test_settings.LAB_TOKEN,
+                          project=test_settings.LAB_PROJECT)
+
+        # Clear/Delete all files initially.
+        env.delete_remote_files()
+
+        # Test that files outside the date window are not deleted.
+        tf = tempfile.NamedTemporaryFile(prefix='qwertz_')
+        with open(tf.name, 'w') as f:
+            f.write("content 1")
+            f.close()
+        env.upload_file(tf.name, "dataset")
+
+        date_from = datetime.now(timezone.utc) - timedelta(days=3)
+        date_to = datetime.now(timezone.utc) - timedelta(days=2)
+
+        env.delete_remote_files(date_from, date_to)
+
+        files = env.list_remote_files(data_type="dataset")
+        assert len(files) == 1

components/lab-job-scheduler/backend/setup.cfg

@@ -7,11 +7,14 @@ universal = false
 
 [flake8]
 ignore =
-    E203,  # space before : (needed for how black formats slicing)
-    # E266,  # too many leading '#' for block comment
-    E501,  # line too long
-    E731,  # do not assign a lambda expression, use a def
-    W503,  # line break before binary operator
+    # space before : (needed for how black formats slicing)
+    E203,
+    # line too long
+    E501,
+    # do not assign a lambda expression, use a def
+    E731,
+    # line break before binary operator
+    W503,
 max-line-length = 88
 # select = B,C,E,F,W,T4,B9
 # max-complexity = 18

components/lab-job-scheduler/backend/src/lab_job_scheduler/app.py

@@ -3,7 +3,7 @@
 import json
 import os
 import threading
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from contaxy.operations.components import ComponentOperations
 from contaxy.schema.exceptions import CREATE_RESOURCE_RESPONSES
@@ -231,7 +231,7 @@ def get_all_scheduled_jobs_from_db(
 
 
 def get_job_from_job_input(
-    job_schedule: ScheduledJobInput, job_id: str = None
+    job_schedule: ScheduledJobInput, job_id: Optional[str] = None
 ) -> ScheduledJob:
 
     return ScheduledJob(

components/lab-job-scheduler/backend/src/lab_job_scheduler/schema.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from contaxy.schema.deployment import JobInput
 from pydantic import BaseModel, Field
 
@@ -25,12 +27,12 @@ class ScheduledJob(ScheduledJobInput):
         description="The time the job was created in ISO format.",
         example="2022-09-29T11:39:52.441287",
     )
-    last_run: str = Field(
+    last_run: Optional[str] = Field(
         None,
         description="The last time the job was run in ISO format.",
         example="2022-09-29T11:39:52.441287",
     )
-    next_run: str = Field(
+    next_run: Optional[str] = Field(
         None,
         description="The next time the job will be run in ISO format.",
         example="2022-09-29T11:39:52.441287",

components/lab-job-scheduler/webapp/package.json

@@ -39,7 +39,7 @@
     "stylelint-config-standard": "^20.0.0"
   },
   "scripts": {
-    "start": "react-scripts start",
+    "start": "GENERATE_SOURCEMAP=false react-scripts start",
     "start-debug": "REACT_APP_CONTAXY_ENDPOINT=http://localhost:30010/api REACT_APP_EXTENSION_ENDPOINT=http://localhost:8080/ yarn start",
     "container": "run(){ docker build -t project-template-dev ../.github/actions/build-environment && echo 'Starting the container. That can take a moment...' && docker run -it --rm -p 3000:3000 -p 6006:6006 -v $(pwd):/workspace:delegated --entrypoint \"/bin/bash\" project-template-dev -c \"cd workspace/ && yarn $1\"; }; run",
     "build": "react-scripts build",

components/lab-mlflow-manager/backend/setup.cfg

@@ -7,11 +7,14 @@ universal = false
 
 [flake8]
 ignore =
-    E203,  # space before : (needed for how black formats slicing)
-    # E266,  # too many leading '#' for block comment
-    E501,  # line too long
-    E731,  # do not assign a lambda expression, use a def
-    W503,  # line break before binary operator
+    # space before : (needed for how black formats slicing)
+    E203,
+    # line too long
+    E501,
+    # do not assign a lambda expression, use a def
+    E731,
+    # line break before binary operator
+    W503,
 max-line-length = 88
 # select = B,C,E,F,W,T4,B9
 # max-complexity = 18

components/lab-workspace-manager/backend/setup.cfg

@@ -7,11 +7,14 @@ universal = false
 
 [flake8]
 ignore =
-    E203,  # space before : (needed for how black formats slicing)
-    # E266,  # too many leading '#' for block comment
-    E501,  # line too long
-    E731,  # do not assign a lambda expression, use a def
-    W503,  # line break before binary operator
+    # space before : (needed for how black formats slicing)
+    E203,
+    # line too long
+    E501,
+    # do not assign a lambda expression, use a def
+    E731,
+    # line break before binary operator
+    W503,
 max-line-length = 88
 # select = B,C,E,F,W,T4,B9
 # max-complexity = 18

components/template/backend/setup.cfg

@@ -7,11 +7,14 @@ universal = false
 
 [flake8]
 ignore =
-    E203,  # space before : (needed for how black formats slicing)
-    # E266,  # too many leading '#' for block comment
-    E501,  # line too long
-    E731,  # do not assign a lambda expression, use a def
-    W503,  # line break before binary operator
+    # space before : (needed for how black formats slicing)
+    E203,
+    # line too long
+    E501,
+    # do not assign a lambda expression, use a def
+    E731,
+    # line break before binary operator
+    W503,
 max-line-length = 88
 # select = B,C,E,F,W,T4,B9
 # max-complexity = 18