diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e53cf5cf00c..bc95ae9eb698 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ from online detectors & interactors) ( - Allowed trailing slashes in the SDK host address () - Adjusted initial camera position, enabled 'Reset zoom' option for 3D canvas () - Enabled authentication via email () +- In the SDK, functions taking paths as strings now also accept path-like objects + () ### Deprecated - TDB diff --git a/cvat-sdk/cvat_sdk/core/downloading.py b/cvat-sdk/cvat_sdk/core/downloading.py index 7831b63fa5e6..a7ba9a1391ca 100644 --- a/cvat-sdk/cvat_sdk/core/downloading.py +++ b/cvat-sdk/cvat_sdk/core/downloading.py @@ -5,8 +5,8 @@ from __future__ import annotations -import os.path as osp from contextlib import closing +from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, Optional from cvat_sdk.api_client.api_client import Endpoint @@ -28,7 +28,7 @@ def __init__(self, client: Client): def download_file( self, url: str, - output_path: str, + output_path: Path, *, timeout: int = 60, pbar: Optional[ProgressReporter] = None, @@ -39,7 +39,7 @@ def download_file( CHUNK_SIZE = 10 * 2**20 - assert not osp.exists(output_path) + assert not output_path.exists() response = self._client.api_client.rest_client.GET( url, @@ -70,7 +70,7 @@ def download_file( def prepare_and_download_file_from_endpoint( self, endpoint: Endpoint, - filename: str, + filename: Path, *, url_params: Optional[Dict[str, Any]] = None, query_params: Optional[Dict[str, Any]] = None, diff --git a/cvat-sdk/cvat_sdk/core/proxies/jobs.py b/cvat-sdk/cvat_sdk/core/proxies/jobs.py index 26985a91f040..e8259c427777 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/jobs.py +++ b/cvat-sdk/cvat_sdk/core/proxies/jobs.py @@ -6,9 +6,8 @@ import io import mimetypes -import os -import os.path as osp -from typing import List, Optional, Sequence +from pathlib import Path +from typing import TYPE_CHECKING, List, Optional, Sequence from PIL import Image @@ -26,6 +25,9 @@ ) from cvat_sdk.core.uploading import AnnotationUploader +if TYPE_CHECKING: + from _typeshed import StrPath + _JobEntityBase, _JobRepoBase = build_model_bases( models.JobRead, apis.JobsApi, api_member_name="jobs_api" ) @@ -43,7 +45,7 @@ class Job( def import_annotations( self, format_name: str, - filename: str, + filename: StrPath, *, status_check_period: Optional[int] = None, pbar: Optional[ProgressReporter] = None, @@ -52,6 +54,8 @@ def import_annotations( Upload annotations for a job in the specified format (e.g. 'YOLO ZIP 1.0'). """ + filename = Path(filename) + AnnotationUploader(self._client).upload_file_and_wait( self.api.create_annotations_endpoint, filename, @@ -66,7 +70,7 @@ def import_annotations( def export_dataset( self, format_name: str, - filename: str, + filename: StrPath, *, pbar: Optional[ProgressReporter] = None, status_check_period: Optional[int] = None, @@ -75,6 +79,9 @@ def export_dataset( """ Download annotations for a job in the specified format (e.g. 'YOLO ZIP 1.0'). """ + + filename = Path(filename) + if include_images: endpoint = self.api.retrieve_dataset_endpoint else: @@ -112,7 +119,7 @@ def download_frames( self, frame_ids: Sequence[int], *, - outdir: str = "", + outdir: StrPath = ".", quality: str = "original", filename_pattern: str = "frame_{frame_id:06d}{frame_ext}", ) -> Optional[List[Image.Image]]: @@ -120,7 +127,9 @@ def download_frames( Download the requested frame numbers for a job and save images as outdir/filename_pattern """ # TODO: add arg descriptions in schema - os.makedirs(outdir, exist_ok=True) + + outdir = Path(outdir) + outdir.mkdir(parents=True, exist_ok=True) for frame_id in frame_ids: frame_bytes = self.get_frame(frame_id, quality=quality) @@ -136,7 +145,7 @@ def download_frames( im_ext = ".jpg" outfile = filename_pattern.format(frame_id=frame_id, frame_ext=im_ext) - im.save(osp.join(outdir, outfile)) + im.save(outdir / outfile) def get_meta(self) -> models.IDataMetaRead: (meta, _) = self.api.retrieve_data_meta(self.id) diff --git a/cvat-sdk/cvat_sdk/core/proxies/projects.py b/cvat-sdk/cvat_sdk/core/proxies/projects.py index 5906a80af295..db77f9f179d1 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/projects.py +++ b/cvat-sdk/cvat_sdk/core/proxies/projects.py @@ -5,8 +5,8 @@ from __future__ import annotations import json -import os.path as osp -from typing import Optional +from pathlib import Path +from typing import TYPE_CHECKING, Optional from cvat_sdk.api_client import apis, models from cvat_sdk.core.downloading import Downloader @@ -21,6 +21,9 @@ ) from cvat_sdk.core.uploading import DatasetUploader, Uploader +if TYPE_CHECKING: + from _typeshed import StrPath + _ProjectEntityBase, _ProjectRepoBase = build_model_bases( models.ProjectRead, apis.ProjectsApi, api_member_name="projects_api" ) @@ -34,7 +37,7 @@ class Project( def import_dataset( self, format_name: str, - filename: str, + filename: StrPath, *, status_check_period: Optional[int] = None, pbar: Optional[ProgressReporter] = None, @@ -43,6 +46,8 @@ def import_dataset( Import dataset for a project in the specified format (e.g. 'YOLO ZIP 1.0'). """ + filename = Path(filename) + DatasetUploader(self._client).upload_file_and_wait( self.api.create_dataset_endpoint, filename, @@ -57,7 +62,7 @@ def import_dataset( def export_dataset( self, format_name: str, - filename: str, + filename: StrPath, *, pbar: Optional[ProgressReporter] = None, status_check_period: Optional[int] = None, @@ -66,6 +71,9 @@ def export_dataset( """ Download annotations for a project in the specified format (e.g. 'YOLO ZIP 1.0'). """ + + filename = Path(filename) + if include_images: endpoint = self.api.retrieve_dataset_endpoint else: @@ -84,7 +92,7 @@ def export_dataset( def download_backup( self, - filename: str, + filename: StrPath, *, status_check_period: int = None, pbar: Optional[ProgressReporter] = None, @@ -93,6 +101,8 @@ def download_backup( Download a project backup """ + filename = Path(filename) + Downloader(self._client).prepare_and_download_file_from_endpoint( self.api.retrieve_backup_endpoint, filename=filename, @@ -148,7 +158,7 @@ def create_from_dataset( def create_from_backup( self, - filename: str, + filename: StrPath, *, status_check_period: int = None, pbar: Optional[ProgressReporter] = None, @@ -156,10 +166,13 @@ def create_from_backup( """ Import a project from a backup file """ + + filename = Path(filename) + if status_check_period is None: status_check_period = self.config.status_check_period - params = {"filename": osp.basename(filename)} + params = {"filename": filename.name} url = self.api_map.make_endpoint_url(self.api.create_backup_endpoint.path) uploader = Uploader(self) diff --git a/cvat-sdk/cvat_sdk/core/proxies/tasks.py b/cvat-sdk/cvat_sdk/core/proxies/tasks.py index e42c585f177f..56d7bf5a54e6 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/tasks.py +++ b/cvat-sdk/cvat_sdk/core/proxies/tasks.py @@ -7,10 +7,9 @@ import io import json import mimetypes -import os -import os.path as osp import shutil from enum import Enum +from pathlib import Path from time import sleep from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence @@ -34,7 +33,7 @@ from cvat_sdk.core.utils import filter_dict if TYPE_CHECKING: - from _typeshed import SupportsWrite + from _typeshed import StrPath, SupportsWrite class ResourceType(Enum): @@ -67,7 +66,7 @@ class Task( def upload_data( self, resource_type: ResourceType, - resources: Sequence[str], + resources: Sequence[StrPath], *, pbar: Optional[ProgressReporter] = None, params: Optional[Dict[str, Any]] = None, @@ -77,15 +76,8 @@ def upload_data( """ params = params or {} - data = {} - if resource_type is ResourceType.LOCAL: - pass # handled later - elif resource_type is ResourceType.REMOTE: - data["remote_files"] = resources - elif resource_type is ResourceType.SHARE: - data["server_files"] = resources + data = {"image_quality": 70} - data["image_quality"] = 70 data.update( filter_dict( params, @@ -105,6 +97,15 @@ def upload_data( data["frame_filter"] = f"step={params.get('frame_step')}" if resource_type in [ResourceType.REMOTE, ResourceType.SHARE]: + for resource in resources: + if not isinstance(resource, str): + raise TypeError(f"resources: expected instances of str, got {type(resource)}") + + if resource_type is ResourceType.REMOTE: + data["remote_files"] = resources + elif resource_type is ResourceType.SHARE: + data["server_files"] = resources + self.api.create_data( self.id, data_request=models.DataRequest(**data), @@ -114,12 +115,14 @@ def upload_data( self.api.create_data_endpoint.path, kwsub={"id": self.id} ) - DataUploader(self._client).upload_files(url, resources, pbar=pbar, **data) + DataUploader(self._client).upload_files( + url, list(map(Path, resources)), pbar=pbar, **data + ) def import_annotations( self, format_name: str, - filename: str, + filename: StrPath, *, status_check_period: Optional[int] = None, pbar: Optional[ProgressReporter] = None, @@ -128,6 +131,8 @@ def import_annotations( Upload annotations for a task in the specified format (e.g. 'YOLO ZIP 1.0'). """ + filename = Path(filename) + AnnotationUploader(self._client).upload_file_and_wait( self.api.create_annotations_endpoint, filename, @@ -178,7 +183,7 @@ def download_frames( self, frame_ids: Sequence[int], *, - outdir: str = "", + outdir: StrPath = ".", quality: str = "original", filename_pattern: str = "frame_{frame_id:06d}{frame_ext}", ) -> Optional[List[Image.Image]]: @@ -186,7 +191,9 @@ def download_frames( Download the requested frame numbers for a task and save images as outdir/filename_pattern """ # TODO: add arg descriptions in schema - os.makedirs(outdir, exist_ok=True) + + outdir = Path(outdir) + outdir.mkdir(exist_ok=True) for frame_id in frame_ids: frame_bytes = self.get_frame(frame_id, quality=quality) @@ -202,12 +209,12 @@ def download_frames( im_ext = ".jpg" outfile = filename_pattern.format(frame_id=frame_id, frame_ext=im_ext) - im.save(osp.join(outdir, outfile)) + im.save(outdir / outfile) def export_dataset( self, format_name: str, - filename: str, + filename: StrPath, *, pbar: Optional[ProgressReporter] = None, status_check_period: Optional[int] = None, @@ -216,6 +223,9 @@ def export_dataset( """ Download annotations for a task in the specified format (e.g. 'YOLO ZIP 1.0'). """ + + filename = Path(filename) + if include_images: endpoint = self.api.retrieve_dataset_endpoint else: @@ -234,7 +244,7 @@ def export_dataset( def download_backup( self, - filename: str, + filename: StrPath, *, status_check_period: int = None, pbar: Optional[ProgressReporter] = None, @@ -243,6 +253,8 @@ def download_backup( Download a task backup """ + filename = Path(filename) + Downloader(self._client).prepare_and_download_file_from_endpoint( self.api.retrieve_backup_endpoint, filename=filename, @@ -370,7 +382,7 @@ def remove_by_ids(self, task_ids: Sequence[int]) -> None: def create_from_backup( self, - filename: str, + filename: StrPath, *, status_check_period: int = None, pbar: Optional[ProgressReporter] = None, @@ -378,10 +390,13 @@ def create_from_backup( """ Import a task from a backup file """ + + filename = Path(filename) + if status_check_period is None: status_check_period = self._client.config.status_check_period - params = {"filename": osp.basename(filename)} + params = {"filename": filename.name} url = self._client.api_map.make_endpoint_url(self.api.create_backup_endpoint.path) uploader = Uploader(self._client) response = uploader.upload_file( diff --git a/cvat-sdk/cvat_sdk/core/uploading.py b/cvat-sdk/cvat_sdk/core/uploading.py index 9747060a9440..721a2038a842 100644 --- a/cvat-sdk/cvat_sdk/core/uploading.py +++ b/cvat-sdk/cvat_sdk/core/uploading.py @@ -5,8 +5,8 @@ from __future__ import annotations import os -import os.path as osp from contextlib import ExitStack, closing +from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple import requests @@ -144,7 +144,7 @@ def __init__(self, client: Client): def upload_file( self, url: str, - filename: str, + filename: Path, *, meta: Dict[str, Any], query_params: Dict[str, Any] = None, @@ -207,15 +207,15 @@ def _wait_for_completion( ) def _split_files_by_requests( - self, filenames: List[str] - ) -> Tuple[List[Tuple[List[str], int]], List[str], int]: + self, filenames: List[Path] + ) -> Tuple[List[Tuple[List[Path], int]], List[Path], int]: bulk_files: Dict[str, int] = {} separate_files: Dict[str, int] = {} # sort by size for filename in filenames: - filename = os.path.abspath(filename) - file_size = os.stat(filename).st_size + filename = filename.resolve() + file_size = filename.stat().st_size if MAX_REQUEST_SIZE < file_size: separate_files[filename] = file_size else: @@ -252,7 +252,7 @@ def _make_tus_uploader(api_client: ApiClient, url: str, **kwargs): return _MyTusUploader(client=client, api_client=api_client, **kwargs) def _upload_file_data_with_tus(self, url, filename, *, meta=None, pbar=None, logger=None): - file_size = os.stat(filename).st_size + file_size = filename.stat().st_size if pbar is None: pbar = NullProgressReporter() @@ -299,7 +299,7 @@ class AnnotationUploader(Uploader): def upload_file_and_wait( self, endpoint: Endpoint, - filename: str, + filename: Path, format_name: str, *, url_params: Optional[Dict[str, Any]] = None, @@ -307,7 +307,7 @@ def upload_file_and_wait( status_check_period: Optional[int] = None, ): url = self._client.api_map.make_endpoint_url(endpoint.path, kwsub=url_params) - params = {"format": format_name, "filename": osp.basename(filename)} + params = {"format": format_name, "filename": filename.name} self.upload_file( url, filename, pbar=pbar, query_params=params, meta={"filename": params["filename"]} ) @@ -326,7 +326,7 @@ class DatasetUploader(Uploader): def upload_file_and_wait( self, endpoint: Endpoint, - filename: str, + filename: Path, format_name: str, *, url_params: Optional[Dict[str, Any]] = None, @@ -334,7 +334,7 @@ def upload_file_and_wait( status_check_period: Optional[int] = None, ): url = self._client.api_map.make_endpoint_url(endpoint.path, kwsub=url_params) - params = {"format": format_name, "filename": osp.basename(filename)} + params = {"format": format_name, "filename": filename.name} self.upload_file( url, filename, pbar=pbar, query_params=params, meta={"filename": params["filename"]} ) @@ -353,7 +353,7 @@ class DataUploader(Uploader): def upload_files( self, url: str, - resources: List[str], + resources: List[Path], *, pbar: Optional[ProgressReporter] = None, **kwargs, @@ -370,7 +370,7 @@ def upload_files( files = {} for i, filename in enumerate(group): files[f"client_files[{i}]"] = ( - filename, + os.fspath(filename), es.enter_context(closing(open(filename, "rb"))).read(), ) response = self._client.api_client.rest_client.POST( @@ -392,7 +392,7 @@ def upload_files( self._upload_file_data_with_tus( url, filename, - meta={"filename": osp.basename(filename)}, + meta={"filename": filename.name}, pbar=pbar, logger=self._client.logger.debug, ) diff --git a/tests/python/cli/test_cli.py b/tests/python/cli/test_cli.py index f2a0c1acca35..50b0d3d43eaf 100644 --- a/tests/python/cli/test_cli.py +++ b/tests/python/cli/test_cli.py @@ -65,7 +65,7 @@ def fxt_backup_file(self, fxt_new_task: Task, fxt_coco_file: str): backup_path = self.tmp_path / "backup.zip" fxt_new_task.import_annotations("COCO 1.0", filename=fxt_coco_file) - fxt_new_task.download_backup(str(backup_path)) + fxt_new_task.download_backup(backup_path) yield backup_path @@ -79,7 +79,7 @@ def fxt_new_task(self): "labels": [{"name": "car"}, {"name": "person"}], }, resource_type=ResourceType.LOCAL, - resources=list(map(os.fspath, files)), + resources=files, ) return task diff --git a/tests/python/sdk/test_issues_comments.py b/tests/python/sdk/test_issues_comments.py index 5316d0bca64c..4f3b4eecfb6e 100644 --- a/tests/python/sdk/test_issues_comments.py +++ b/tests/python/sdk/test_issues_comments.py @@ -42,7 +42,7 @@ def fxt_new_task(self, fxt_image_file: Path): "labels": [{"name": "car"}, {"name": "person"}], }, resource_type=ResourceType.LOCAL, - resources=[str(fxt_image_file)], + resources=[fxt_image_file], data_params={"image_quality": 80}, ) @@ -162,7 +162,7 @@ def fxt_new_task(self, fxt_image_file: Path): "labels": [{"name": "car"}, {"name": "person"}], }, resource_type=ResourceType.LOCAL, - resources=[str(fxt_image_file)], + resources=[fxt_image_file], data_params={"image_quality": 80}, ) diff --git a/tests/python/sdk/test_jobs.py b/tests/python/sdk/test_jobs.py index 51edf2addf90..bf876807be2f 100644 --- a/tests/python/sdk/test_jobs.py +++ b/tests/python/sdk/test_jobs.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: MIT import io -import os from logging import Logger from pathlib import Path from typing import Tuple @@ -46,7 +45,7 @@ def fxt_new_task(self, fxt_image_file: Path): "labels": [{"name": "car"}, {"name": "person"}], }, resource_type=ResourceType.LOCAL, - resources=[str(fxt_image_file)], + resources=[fxt_image_file], data_params={"image_quality": 80}, ) @@ -108,7 +107,7 @@ def test_can_download_dataset(self, fxt_new_task: Task, include_images: bool): job = self.client.jobs.retrieve(job_id) job.export_dataset( format_name="CVAT for images 1.1", - filename=os.fspath(path), + filename=path, pbar=pbar, include_images=include_images, ) @@ -135,7 +134,7 @@ def test_can_download_frames(self, fxt_new_task: Task, quality: str): fxt_new_task.get_jobs()[0].download_frames( [0], quality=quality, - outdir=str(self.tmp_path), + outdir=self.tmp_path, filename_pattern="frame-{frame_id}{frame_ext}", ) @@ -147,7 +146,7 @@ def test_can_upload_annotations(self, fxt_new_task: Task, fxt_coco_file: Path): pbar = make_pbar(file=pbar_out) fxt_new_task.get_jobs()[0].import_annotations( - format_name="COCO 1.0", filename=str(fxt_coco_file), pbar=pbar + format_name="COCO 1.0", filename=fxt_coco_file, pbar=pbar ) assert "uploaded" in self.logger_stream.getvalue() diff --git a/tests/python/sdk/test_tasks.py b/tests/python/sdk/test_tasks.py index 0a4868406e63..291ed38cc389 100644 --- a/tests/python/sdk/test_tasks.py +++ b/tests/python/sdk/test_tasks.py @@ -4,7 +4,6 @@ import io import json -import os import zipfile from logging import Logger from pathlib import Path @@ -48,7 +47,7 @@ def fxt_backup_file(self, fxt_new_task: Task, fxt_coco_file: str): backup_path = self.tmp_path / "backup.zip" fxt_new_task.import_annotations("COCO 1.0", filename=fxt_coco_file) - fxt_new_task.download_backup(str(backup_path)) + fxt_new_task.download_backup(backup_path) yield backup_path @@ -60,7 +59,7 @@ def fxt_new_task(self, fxt_image_file: Path): "labels": [{"name": "car"}, {"name": "person"}], }, resource_type=ResourceType.LOCAL, - resources=[str(fxt_image_file)], + resources=[fxt_image_file], data_params={"image_quality": 80}, ) @@ -113,9 +112,8 @@ def test_can_create_task_with_local_data(self): task_files = generate_image_files(7) for i, f in enumerate(task_files): fname = self.tmp_path / f.name - with fname.open("wb") as fd: - fd.write(f.getvalue()) - task_files[i] = str(fname) + fname.write_bytes(f.getvalue()) + task_files[i] = fname task = self.client.tasks.create_from_data( spec=task_spec, @@ -184,7 +182,7 @@ def test_can_create_task_with_git_repo(self, fxt_image_file: Path): task = self.client.tasks.create_from_data( spec=task_spec, resource_type=ResourceType.LOCAL, - resources=[str(fxt_image_file)], + resources=[fxt_image_file], pbar=pbar, dataset_repository_url=repository_url, ) @@ -256,7 +254,7 @@ def test_can_download_dataset(self, fxt_new_task: Task, include_images: bool): task = self.client.tasks.retrieve(task_id) task.export_dataset( format_name="CVAT for images 1.1", - filename=os.fspath(path), + filename=path, pbar=pbar, include_images=include_images, ) @@ -272,7 +270,7 @@ def test_can_download_backup(self, fxt_new_task: Task): task_id = fxt_new_task.id path = self.tmp_path / f"task_{task_id}-backup.zip" task = self.client.tasks.retrieve(task_id) - task.download_backup(filename=os.fspath(path), pbar=pbar) + task.download_backup(filename=path, pbar=pbar) assert "100%" in pbar_out.getvalue().strip("\r").split("\r")[-1] assert path.is_file() @@ -296,7 +294,7 @@ def test_can_download_frames(self, fxt_new_task: Task, quality: str): fxt_new_task.download_frames( [0], quality=quality, - outdir=str(self.tmp_path), + outdir=self.tmp_path, filename_pattern="frame-{frame_id}{frame_ext}", ) @@ -319,9 +317,7 @@ def test_can_upload_annotations(self, fxt_new_task: Task, fxt_coco_file: Path): pbar_out = io.StringIO() pbar = make_pbar(file=pbar_out) - fxt_new_task.import_annotations( - format_name="COCO 1.0", filename=str(fxt_coco_file), pbar=pbar - ) + fxt_new_task.import_annotations(format_name="COCO 1.0", filename=fxt_coco_file, pbar=pbar) assert "uploaded" in self.logger_stream.getvalue() assert "100%" in pbar_out.getvalue().strip("\r").split("\r")[-1] @@ -331,7 +327,7 @@ def _test_can_create_from_backup(self, fxt_new_task: Task, fxt_backup_file: Path pbar_out = io.StringIO() pbar = make_pbar(file=pbar_out) - task = self.client.tasks.create_from_backup(str(fxt_backup_file), pbar=pbar) + task = self.client.tasks.create_from_backup(fxt_backup_file, pbar=pbar) assert task.id assert task.id != fxt_new_task.id