diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index cab6e202e484..c20ca4fc6099 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -18,16 +18,11 @@ jobs: CHANGED_FILES="${{steps.files.outputs.all_changed_files}}" if [[ ! -z $CHANGED_FILES ]]; then - sudo apt-get --no-install-recommends install -y build-essential curl python3-dev python3-pip python3-venv - python3 -m venv .env - . .env/bin/activate - pip install -U pip wheel setuptools - pip install bandit + pipx install bandit echo "Bandit version: "$(bandit --version | head -1) echo "The files will be checked: "$(echo $CHANGED_FILES) bandit -a file --ini .bandit $CHANGED_FILES - deactivate else echo "No files with the \"py\" extension found" fi diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 460dc102e044..5270e185edc0 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -28,11 +28,7 @@ jobs: UPDATED_DIRS="${{steps.files.outputs.all_changed_files}}" if [[ ! -z $UPDATED_DIRS ]]; then - sudo apt-get --no-install-recommends install -y build-essential curl python3-dev python3-pip python3-venv - python3 -m venv .env - . .env/bin/activate - pip install -U pip wheel setuptools - pip install $(egrep "black.*" ./cvat-cli/requirements/development.txt) + pipx install $(egrep "black.*" ./cvat-cli/requirements/development.txt) echo "Black version: "$(black --version) echo "The dirs will be checked: $UPDATED_DIRS" @@ -40,7 +36,6 @@ jobs: for DIR in $UPDATED_DIRS; do black --check --diff $DIR || EXIT_CODE=$(($? | $EXIT_CODE)) || true done - deactivate exit $EXIT_CODE else echo "No files with the \"py\" extension found" diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml index d2f0a23a3c32..58c3b63113a5 100644 --- a/.github/workflows/full.yml +++ b/.github/workflows/full.yml @@ -152,16 +152,19 @@ jobs: name: expected_schema path: cvat/schema-expected.yml - - name: Running REST API and SDK tests - id: run_tests + - name: Generate SDK run: | pip3 install -r cvat-sdk/gen/requirements.txt ./cvat-sdk/gen/generate.sh - pip3 install -r ./tests/python/requirements.txt - pip3 install -e ./cvat-sdk - pip3 install -e ./cvat-cli + - name: Install SDK + run: | + pip3 install -r ./tests/python/requirements.txt \ + -e './cvat-sdk[pytorch]' -e ./cvat-cli + - name: Running REST API and SDK tests + id: run_tests + run: | pytest tests/python/ - name: Creating a log file from cvat containers diff --git a/.github/workflows/isort.yml b/.github/workflows/isort.yml index f3157b446c75..b5c2b4921aa9 100644 --- a/.github/workflows/isort.yml +++ b/.github/workflows/isort.yml @@ -25,11 +25,7 @@ jobs: UPDATED_DIRS="${{steps.files.outputs.all_changed_files}}" if [[ ! -z $UPDATED_DIRS ]]; then - sudo apt-get --no-install-recommends install -y build-essential curl python3-dev python3-pip python3-venv - python3 -m venv .env - . .env/bin/activate - pip install -U pip wheel setuptools - pip install $(egrep "isort.*" ./cvat-cli/requirements/development.txt) + pipx install $(egrep "isort.*" ./cvat-cli/requirements/development.txt) echo "isort version: $(isort --version-number)" echo "The dirs will be checked: $UPDATED_DIRS" @@ -37,7 +33,6 @@ jobs: for DIR in $UPDATED_DIRS; do isort --check $DIR || EXIT_CODE=$(($? | $EXIT_CODE)) || true done - deactivate exit $EXIT_CODE else echo "No files with the \"py\" extension found" diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index f54623bc2984..b890f02517d3 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -19,17 +19,15 @@ jobs: CHANGED_FILES="${{steps.files.outputs.all_changed_files}}" if [[ ! -z $CHANGED_FILES ]]; then - sudo apt-get --no-install-recommends install -y build-essential curl python3-dev python3-pip python3-venv - python3 -m venv .env - . .env/bin/activate - pip install -U pip wheel setuptools - pip install $(egrep "pylint.*==.*" ./cvat/requirements/development.txt) - pip install $(egrep "django==.*" ./cvat/requirements/base.txt) + pipx install $(egrep "^pylint==" ./cvat/requirements/development.txt) + + pipx inject pylint \ + $(egrep "^pylint-.+==" ./cvat/requirements/development.txt) \ + $(egrep "^django==" ./cvat/requirements/base.txt) echo "Pylint version: "$(pylint --version | head -1) echo "The files will be checked: "$(echo $CHANGED_FILES) pylint $CHANGED_FILES - deactivate else echo "No files with the \"py\" extension found" fi diff --git a/.vscode/launch.json b/.vscode/launch.json index 366bd49a3fa1..4d0f09c55257 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -520,6 +520,7 @@ "type": "node", "request": "launch", "name": "jest debug", + "cwd": "${workspaceFolder}/cvat-core", "program": "${workspaceFolder}/node_modules/.bin/jest", "args": [ "--config", diff --git a/CHANGELOG.md b/CHANGELOG.md index 933d5468e80d..497654b88760 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,44 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## \[2.6.0\] - 2023-08-11 + +### Added + +- \[SDK\] Introduced the `DeferredTqdmProgressReporter` class, + which avoids the glitchy output seen with the `TqdmProgressReporter` under certain circumstances + () +- \[SDK, CLI\] Added the `cvat_sdk.auto_annotation` + module, providing functionality to automatically annotate tasks + by executing a user-provided function on the local machine. + A corresponding CLI command (`auto-annotate`) is also available. + Some predefined functions using torchvision are also available. + (, + ) +- Included an indication for cached frames in the interface + () + +### Changed + +- Raised the default guide assets limitations to 30 assets, + with a maximum size of 10MB each + () +- \[SDK\] Custom `ProgressReporter` implementations should now override `start2` instead of `start` + The old implementation is still supported. + () +- Improved memory optimization and code in the decoding module () + +### Removed + +- Removed the YOLOv5 serverless function + () + +### Fixed + +- Corrected an issue where the prebuilt FFmpeg bundled in PyAV + was being used instead of the custom build. +- Fixed the filename for labels in the CamVid format () + ## \[2.5.2\] - 2023-07-27 ### Added @@ -42,7 +80,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 () - \[API\] Fixed API issue related to file downloading failures for filenames with special characters () - \[Helm\] In Helm, we've resolved an issue with multiple caches - in the same RWX volume, which was preventing db migration from starting () + in the same RWX volume, which was preventing db migration from starting () ## \[2.5.1\] - 2023-07-19 diff --git a/Dockerfile b/Dockerfile index d62b85d4980a..75d2f0bf497b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,8 +56,11 @@ COPY utils/dataset_manifest/requirements.txt /tmp/utils/dataset_manifest/require RUN grep -q '^av==' /tmp/utils/dataset_manifest/requirements.txt RUN sed -i '/^av==/!d' /tmp/utils/dataset_manifest/requirements.txt +# Work around https://github.com/PyAV-Org/PyAV/issues/1140 +RUN pip install setuptools wheel 'cython<3' + RUN --mount=type=cache,target=/root/.cache/pip/http \ - python3 -m pip wheel \ + python3 -m pip wheel --no-binary=av --no-build-isolation \ -r /tmp/utils/dataset_manifest/requirements.txt \ -w /tmp/wheelhouse diff --git a/Dockerfile.ui b/Dockerfile.ui index 30bf27b68f7d..a2e235b2a2a5 100644 --- a/Dockerfile.ui +++ b/Dockerfile.ui @@ -3,6 +3,7 @@ FROM node:lts-slim AS cvat-ui ARG WA_PAGE_VIEW_HIT ARG UI_APP_CONFIG ARG CLIENT_PLUGINS +ARG SOURCE_MAPS_TOKEN ENV TERM=xterm \ LANG='C.UTF-8' \ @@ -27,7 +28,9 @@ COPY cvat-core/ /tmp/cvat-core/ COPY cvat-canvas3d/ /tmp/cvat-canvas3d/ COPY cvat-canvas/ /tmp/cvat-canvas/ COPY cvat-ui/ /tmp/cvat-ui/ -RUN CLIENT_PLUGINS="${CLIENT_PLUGINS}" UI_APP_CONFIG="${UI_APP_CONFIG}" yarn run build:cvat-ui +RUN CLIENT_PLUGINS="${CLIENT_PLUGINS}" \ +UI_APP_CONFIG="${UI_APP_CONFIG}" \ +SOURCE_MAPS_TOKEN="${SOURCE_MAPS_TOKEN}" yarn run build:cvat-ui FROM nginx:mainline-alpine # Replace default.conf configuration to remove unnecessary rules diff --git a/README.md b/README.md index 3192c62a898a..b6d28199bde6 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,6 @@ up to 10x. Here is a list of the algorithms we support, and the platforms they c | [Object reidentification](/serverless/openvino/omz/intel/person-reidentification-retail-0277/nuclio) | reid | OpenVINO | ✔️ | | | [Semantic segmentation for ADAS](/serverless/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio) | detector | OpenVINO | ✔️ | | | [Text detection v4](/serverless/openvino/omz/intel/text-detection-0004/nuclio) | detector | OpenVINO | ✔️ | | -| [YOLO v5](/serverless/pytorch/ultralytics/yolov5/nuclio) | detector | PyTorch | ✔️ | | | [SiamMask](/serverless/pytorch/foolwood/siammask/nuclio) | tracker | PyTorch | ✔️ | ✔️ | | [TransT](/serverless/pytorch/dschoerk/transt/nuclio) | tracker | PyTorch | ✔️ | ✔️ | | [f-BRS](/serverless/pytorch/saic-vul/fbrs/nuclio) | interactor | PyTorch | ✔️ | | diff --git a/cvat-canvas/package.json b/cvat-canvas/package.json index 64ac4c6e81f7..29b52cce8248 100644 --- a/cvat-canvas/package.json +++ b/cvat-canvas/package.json @@ -1,6 +1,6 @@ { "name": "cvat-canvas", - "version": "2.17.1", + "version": "2.17.3", "description": "Part of Computer Vision Annotation Tool which presents its canvas library", "main": "src/canvas.ts", "scripts": { diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts index dcc3deca173e..86e28b3fccb0 100644 --- a/cvat-canvas/src/typescript/canvasModel.ts +++ b/cvat-canvas/src/typescript/canvasModel.ts @@ -14,7 +14,7 @@ export interface Size { export interface Image { renderWidth: number; renderHeight: number; - imageData: ImageData | CanvasImageSource; + imageData: ImageBitmap; } export interface Position { @@ -569,7 +569,7 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { .catch((exception: any): void => { this.data.exception = exception; // don't notify when the frame is no longer needed - if (typeof exception !== 'number' || exception === this.data.imageID) { + if (typeof exception !== 'number') { this.notify(UpdateReasons.DATA_FAILED); } }); diff --git a/cvat-canvas/src/typescript/canvasView.ts b/cvat-canvas/src/typescript/canvasView.ts index 1a6d442d614e..593ea962bf00 100644 --- a/cvat-canvas/src/typescript/canvasView.ts +++ b/cvat-canvas/src/typescript/canvasView.ts @@ -1420,19 +1420,7 @@ export class CanvasViewImpl implements CanvasView, Listener { this.background.setAttribute('height', `${image.renderHeight}px`); if (ctx) { - if (image.imageData instanceof ImageData) { - ctx.scale( - image.renderWidth / image.imageData.width, - image.renderHeight / image.imageData.height, - ); - ctx.putImageData(image.imageData, 0, 0); - // Transformation matrix must not affect the putImageData() method. - // By this reason need to redraw the image to apply scale. - // https://www.w3.org/TR/2dcontext/#dom-context-2d-putimagedata - ctx.drawImage(this.background, 0, 0); - } else { - ctx.drawImage(image.imageData, 0, 0); - } + ctx.drawImage(image.imageData, 0, 0, image.renderWidth, image.renderHeight); } if (model.imageIsDeleted) { diff --git a/cvat-cli/requirements/base.txt b/cvat-cli/requirements/base.txt index 612076fc5c72..923c04ae2ae1 100644 --- a/cvat-cli/requirements/base.txt +++ b/cvat-cli/requirements/base.txt @@ -1,3 +1,3 @@ -cvat-sdk~=2.5.0 +cvat-sdk~=2.6.0 Pillow>=6.2.0 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/cvat-cli/setup.py b/cvat-cli/setup.py index 58567b65e3d8..454ce2f00956 100644 --- a/cvat-cli/setup.py +++ b/cvat-cli/setup.py @@ -56,7 +56,7 @@ def parse_requirements(filename=BASE_REQUIREMENTS_FILE): "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - python_requires=">=3.7", + python_requires=">=3.8", install_requires=BASE_REQUIREMENTS, entry_points={ "console_scripts": [ diff --git a/cvat-cli/src/cvat_cli/__main__.py b/cvat-cli/src/cvat_cli/__main__.py index 673adf3e6ae7..2448587245f9 100755 --- a/cvat-cli/src/cvat_cli/__main__.py +++ b/cvat-cli/src/cvat_cli/__main__.py @@ -59,6 +59,7 @@ def main(args: List[str] = None): "upload": CLI.tasks_upload, "export": CLI.tasks_export, "import": CLI.tasks_import, + "auto-annotate": CLI.tasks_auto_annotate, } parser = make_cmdline_parser() parsed_args = parser.parse_args(args) diff --git a/cvat-cli/src/cvat_cli/cli.py b/cvat-cli/src/cvat_cli/cli.py index 9e5389acf868..114e5bed8945 100644 --- a/cvat-cli/src/cvat_cli/cli.py +++ b/cvat-cli/src/cvat_cli/cli.py @@ -4,12 +4,15 @@ from __future__ import annotations +import importlib +import importlib.util import json -from typing import Dict, List, Sequence, Tuple +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple -import tqdm +import cvat_sdk.auto_annotation as cvataa from cvat_sdk import Client, models -from cvat_sdk.core.helpers import TqdmProgressReporter +from cvat_sdk.core.helpers import DeferredTqdmProgressReporter from cvat_sdk.core.proxies.tasks import ResourceType @@ -67,7 +70,7 @@ def tasks_create( status_check_period=status_check_period, dataset_repository_url=dataset_repository_url, use_lfs=lfs, - pbar=self._make_pbar(), + pbar=DeferredTqdmProgressReporter(), ) print("Created task id", task.id) @@ -109,7 +112,7 @@ def tasks_dump( self.client.tasks.retrieve(obj_id=task_id).export_dataset( format_name=fileformat, filename=filename, - pbar=self._make_pbar(), + pbar=DeferredTqdmProgressReporter(), status_check_period=status_check_period, include_images=include_images, ) @@ -123,22 +126,56 @@ def tasks_upload( format_name=fileformat, filename=filename, status_check_period=status_check_period, - pbar=self._make_pbar(), + pbar=DeferredTqdmProgressReporter(), ) def tasks_export(self, task_id: str, filename: str, *, status_check_period: int = 2) -> None: """Download a task backup""" self.client.tasks.retrieve(obj_id=task_id).download_backup( - filename=filename, status_check_period=status_check_period, pbar=self._make_pbar() + filename=filename, + status_check_period=status_check_period, + pbar=DeferredTqdmProgressReporter(), ) def tasks_import(self, filename: str, *, status_check_period: int = 2) -> None: """Import a task from a backup file""" self.client.tasks.create_from_backup( - filename=filename, status_check_period=status_check_period, pbar=self._make_pbar() + filename=filename, + status_check_period=status_check_period, + pbar=DeferredTqdmProgressReporter(), ) - def _make_pbar(self, title: str = None) -> TqdmProgressReporter: - return TqdmProgressReporter( - tqdm.tqdm(unit_scale=True, unit="B", unit_divisor=1024, desc=title) + def tasks_auto_annotate( + self, + task_id: int, + *, + function_module: Optional[str] = None, + function_file: Optional[Path] = None, + function_parameters: Dict[str, Any], + clear_existing: bool = False, + allow_unmatched_labels: bool = False, + ) -> None: + if function_module is not None: + function = importlib.import_module(function_module) + elif function_file is not None: + module_spec = importlib.util.spec_from_file_location("__cvat_function__", function_file) + function = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(function) + else: + assert False, "function identification arguments missing" + + if hasattr(function, "create"): + # this is actually a function factory + function = function.create(**function_parameters) + else: + if function_parameters: + raise TypeError("function takes no parameters") + + cvataa.annotate_task( + self.client, + task_id, + function, + pbar=DeferredTqdmProgressReporter(), + clear_existing=clear_existing, + allow_unmatched_labels=allow_unmatched_labels, ) diff --git a/cvat-cli/src/cvat_cli/parser.py b/cvat-cli/src/cvat_cli/parser.py index 32630baaad8a..f03a52f9b41a 100644 --- a/cvat-cli/src/cvat_cli/parser.py +++ b/cvat-cli/src/cvat_cli/parser.py @@ -10,6 +10,8 @@ import os import textwrap from distutils.util import strtobool +from pathlib import Path +from typing import Any, Tuple from cvat_sdk.core.proxies.tasks import ResourceType @@ -40,6 +42,40 @@ def parse_resource_type(s: str) -> ResourceType: return s +def parse_function_parameter(s: str) -> Tuple[str, Any]: + key, sep, type_and_value = s.partition("=") + + if not sep: + raise argparse.ArgumentTypeError("parameter value not specified") + + type_, sep, value = type_and_value.partition(":") + + if not sep: + raise argparse.ArgumentTypeError("parameter type not specified") + + if type_ == "int": + value = int(value) + elif type_ == "float": + value = float(value) + elif type_ == "str": + pass + elif type_ == "bool": + value = bool(strtobool(value)) + else: + raise argparse.ArgumentTypeError(f"unsupported parameter type {type_!r}") + + return (key, value) + + +class BuildDictAction(argparse.Action): + def __init__(self, option_strings, dest, default=None, **kwargs): + super().__init__(option_strings, dest, default=default or {}, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + key, value = values + getattr(namespace, self.dest)[key] = value + + def make_cmdline_parser() -> argparse.ArgumentParser: ####################################################################### # Command line interface definition @@ -369,6 +405,50 @@ def make_cmdline_parser() -> argparse.ArgumentParser: help="time interval between checks if archive processing was finished, in seconds", ) + ####################################################################### + # Auto-annotate + ####################################################################### + auto_annotate_task_parser = task_subparser.add_parser( + "auto-annotate", + description="Automatically annotate a CVAT task by running a function on the local machine.", + ) + auto_annotate_task_parser.add_argument("task_id", type=int, help="task ID") + + function_group = auto_annotate_task_parser.add_mutually_exclusive_group(required=True) + + function_group.add_argument( + "--function-module", + metavar="MODULE", + help="qualified name of a module to use as the function", + ) + + function_group.add_argument( + "--function-file", + metavar="PATH", + type=Path, + help="path to a Python source file to use as the function", + ) + + auto_annotate_task_parser.add_argument( + "--function-parameter", + "-p", + metavar="NAME=TYPE:VALUE", + type=parse_function_parameter, + action=BuildDictAction, + dest="function_parameters", + help="parameter for the function", + ) + + auto_annotate_task_parser.add_argument( + "--clear-existing", action="store_true", help="Remove existing annotations from the task" + ) + + auto_annotate_task_parser.add_argument( + "--allow-unmatched-labels", + action="store_true", + help="Allow the function to declare labels not configured in the task", + ) + return parser diff --git a/cvat-cli/src/cvat_cli/version.py b/cvat-cli/src/cvat_cli/version.py index 84e6495dd8d9..840688e3f56d 100644 --- a/cvat-cli/src/cvat_cli/version.py +++ b/cvat-cli/src/cvat_cli/version.py @@ -1 +1 @@ -VERSION = "2.5.0" +VERSION = "2.6.0" diff --git a/cvat-core/package.json b/cvat-core/package.json index d8970e107443..d9712f79ea66 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "9.3.0", + "version": "11.0.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "src/api.ts", "scripts": { @@ -27,7 +27,6 @@ "dependencies": { "@types/lodash": "^4.14.191", "axios": "^0.27.2", - "browser-or-node": "^2.0.0", "cvat-data": "link:./../cvat-data", "detect-browser": "^5.2.1", "error-stack-parser": "^2.0.2", diff --git a/cvat-core/src/api.ts b/cvat-core/src/api.ts index 4ff47db6ad12..9a99daa1f3f8 100644 --- a/cvat-core/src/api.ts +++ b/cvat-core/src/api.ts @@ -326,16 +326,23 @@ function build() { cvat.server = Object.freeze(cvat.server); cvat.projects = Object.freeze(cvat.projects); cvat.tasks = Object.freeze(cvat.tasks); + cvat.assets = Object.freeze(cvat.assets); cvat.jobs = Object.freeze(cvat.jobs); + cvat.frames = Object.freeze(cvat.frames); cvat.users = Object.freeze(cvat.users); cvat.plugins = Object.freeze(cvat.plugins); cvat.lambda = Object.freeze(cvat.lambda); + // logger: todo: logger storage implemented other way + cvat.config = Object.freeze(cvat.config); cvat.client = Object.freeze(cvat.client); cvat.enums = Object.freeze(cvat.enums); + cvat.exceptions = Object.freeze(cvat.exceptions); cvat.cloudStorages = Object.freeze(cvat.cloudStorages); cvat.organizations = Object.freeze(cvat.organizations); + cvat.webhooks = Object.freeze(cvat.webhooks); cvat.analytics = Object.freeze(cvat.analytics); - cvat.frames = Object.freeze(cvat.frames); + cvat.storage = Object.freeze(cvat.storage); + cvat.classes = Object.freeze(cvat.classes); const implemented = Object.freeze(implementAPI(cvat)); return implemented; diff --git a/cvat-core/src/cloud-storage.ts b/cvat-core/src/cloud-storage.ts index bfc39f999d00..50655edad7d4 100644 --- a/cvat-core/src/cloud-storage.ts +++ b/cvat-core/src/cloud-storage.ts @@ -254,8 +254,8 @@ export default class CloudStorage { return result; } - public async getPreview(): Promise { - const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getPreview); + public async preview(): Promise { + const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.preview); return result; } @@ -375,20 +375,14 @@ Object.defineProperties(CloudStorage.prototype.getContent, { }, }); -Object.defineProperties(CloudStorage.prototype.getPreview, { +Object.defineProperties(CloudStorage.prototype.preview, { implementation: { writable: false, enumerable: false, - value: async function implementation(): Promise { - return new Promise((resolve, reject) => { - serverProxy.cloudStorages - .getPreview(this.id) - .then((result) => ((result) ? decodePreview(result) : Promise.resolve(result))) - .then((decoded) => resolve(decoded)) - .catch((error) => { - reject(error); - }); - }); + value: async function implementation(this: CloudStorage): Promise { + const preview = await serverProxy.cloudStorages.getPreview(this.id); + if (!preview) return ''; + return decodePreview(preview); }, }, }); diff --git a/cvat-core/src/frames.ts b/cvat-core/src/frames.ts index 247bafafb443..2f5b328f81bf 100644 --- a/cvat-core/src/frames.ts +++ b/cvat-core/src/frames.ts @@ -3,28 +3,35 @@ // // SPDX-License-Identifier: MIT -import { isBrowser, isNode } from 'browser-or-node'; - -import * as cvatData from 'cvat-data'; -import { DimensionType } from 'enums'; +import _ from 'lodash'; +import { + FrameDecoder, BlockType, DimensionType, ChunkQuality, decodeContextImages, RequestOutdatedError, +} from 'cvat-data'; import PluginRegistry from './plugins'; import serverProxy, { RawFramesMetaData } from './server-proxy'; -import { - Exception, ArgumentError, DataError, ServerError, -} from './exceptions'; +import { Exception, ArgumentError, DataError } from './exceptions'; // frame storage by job id const frameDataCache: Record & { deleted_frames: Record }; chunkSize: number; mode: 'annotation' | 'interpolation'; startFrame: number; stopFrame: number; - provider: cvatData.FrameProvider; - frameBuffer: FrameBuffer; + decodeForward: boolean; + forwardStep: number; + latestFrameDecodeRequest: number | null; + latestContextImagesRequest: number | null; + provider: FrameDecoder; decodedBlocksCacheSize: number; - activeChunkRequest: null; - nextChunkRequest: null; + activeChunkRequest: Promise | null; + activeContextRequest: Promise> | null; + contextCache: Record; + timestamp: number; + size: number; + }>; + getChunk: (chunkNumber: number, quality: ChunkQuality) => Promise; }> = {}; export class FramesMetaData { @@ -98,15 +105,20 @@ export class FramesMetaData { } export class FrameData { + public readonly filename: string; + public readonly width: number; + public readonly height: number; + public readonly number: number; + public readonly relatedFiles: number; + public readonly deleted: boolean; + public readonly jobID: number; + constructor({ width, height, name, jobID, frameNumber, - startFrame, - stopFrame, - decodeForward, deleted, related_files: relatedFiles, }) { @@ -125,7 +137,7 @@ export class FrameData { value: height, writable: false, }, - jid: { + jobID: { value: jobID, writable: false, }, @@ -137,18 +149,6 @@ export class FrameData { value: relatedFiles, writable: false, }, - startFrame: { - value: startFrame, - writable: false, - }, - stopFrame: { - value: stopFrame, - writable: false, - }, - decodeForward: { - value: decodeForward, - writable: false, - }, deleted: { value: deleted, writable: false, @@ -157,233 +157,167 @@ export class FrameData { ); } - async data(onServerRequest = () => {}) { + async data(onServerRequest = () => {}): Promise { const result = await PluginRegistry.apiWrapper.call(this, FrameData.prototype.data, onServerRequest); return result; } - - get imageData() { - return this._data.imageData; - } - - set imageData(imageData) { - this._data.imageData = imageData; - } } -FrameData.prototype.data.implementation = async function (onServerRequest) { - return new Promise((resolve, reject) => { - const resolveWrapper = (data) => { - this._data = { - imageData: data, - renderWidth: this.width, - renderHeight: this.height, - }; - return resolve(this._data); - }; +Object.defineProperty(FrameData.prototype.data, 'implementation', { + value(this: FrameData, onServerRequest) { + return new Promise<{ + renderWidth: number; + renderHeight: number; + imageData: ImageBitmap | Blob; + } | Blob>((resolve, reject) => { + const { + provider, chunkSize, stopFrame, decodeForward, forwardStep, decodedBlocksCacheSize, + } = frameDataCache[this.jobID]; + + const requestId = +_.uniqueId(); + const chunkNumber = Math.floor(this.number / chunkSize); + const frame = provider.frame(this.number); + + function findTheNextNotDecodedChunk(searchFrom: number): number { + let firstFrameInNextChunk = searchFrom + forwardStep; + let nextChunkNumber = Math.floor(firstFrameInNextChunk / chunkSize); + while (nextChunkNumber === chunkNumber) { + firstFrameInNextChunk += forwardStep; + nextChunkNumber = Math.floor(firstFrameInNextChunk / chunkSize); + } - if (this._data) { - resolve(this._data); - return; - } + if (provider.isChunkCached(nextChunkNumber)) { + return findTheNextNotDecodedChunk(firstFrameInNextChunk); + } + + return nextChunkNumber; + } - const { provider } = frameDataCache[this.jid]; - const { chunkSize } = frameDataCache[this.jid]; - const start = parseInt(this.number / chunkSize, 10) * chunkSize; - const stop = Math.min(this.stopFrame, (parseInt(this.number / chunkSize, 10) + 1) * chunkSize - 1); - const chunkNumber = Math.floor(this.number / chunkSize); - - const onDecodeAll = async (frameNumber) => { - if ( - frameDataCache[this.jid].activeChunkRequest && - chunkNumber === frameDataCache[this.jid].activeChunkRequest.chunkNumber - ) { - const callbackArray = frameDataCache[this.jid].activeChunkRequest.callbacks; - for (let i = callbackArray.length - 1; i >= 0; --i) { - if (callbackArray[i].frameNumber === frameNumber) { - const callback = callbackArray[i]; - callbackArray.splice(i, 1); - callback.resolve(await provider.frame(callback.frameNumber)); + if (frame) { + if (decodeForward && decodedBlocksCacheSize > 1 && !frameDataCache[this.jobID].activeChunkRequest) { + const nextChunkNumber = findTheNextNotDecodedChunk(this.number); + const predecodeChunksMax = Math.floor(decodedBlocksCacheSize / 2); + if (nextChunkNumber * chunkSize <= stopFrame && + nextChunkNumber <= chunkNumber + predecodeChunksMax) { + provider.cleanup(1); + frameDataCache[this.jobID].activeChunkRequest = new Promise((resolveForward) => { + const releasePromise = (): void => { + resolveForward(); + frameDataCache[this.jobID].activeChunkRequest = null; + }; + + frameDataCache[this.jobID].getChunk( + nextChunkNumber, ChunkQuality.COMPRESSED, + ).then((chunk: ArrayBuffer) => { + provider.requestDecodeBlock( + chunk, + nextChunkNumber * chunkSize, + Math.min(stopFrame, (nextChunkNumber + 1) * chunkSize - 1), + () => {}, + releasePromise, + releasePromise, + ); + }).catch(() => { + releasePromise(); + }); + }); } } - if (callbackArray.length === 0) { - frameDataCache[this.jid].activeChunkRequest = null; - } + + resolve({ + renderWidth: this.width, + renderHeight: this.height, + imageData: frame, + }); + return; } - }; - const rejectRequestAll = () => { - if ( - frameDataCache[this.jid].activeChunkRequest && - chunkNumber === frameDataCache[this.jid].activeChunkRequest.chunkNumber - ) { - for (const r of frameDataCache[this.jid].activeChunkRequest.callbacks) { - r.reject(r.frameNumber); + onServerRequest(); + frameDataCache[this.jobID].latestFrameDecodeRequest = requestId; + (frameDataCache[this.jobID].activeChunkRequest || Promise.resolve()).finally(() => { + if (frameDataCache[this.jobID].latestFrameDecodeRequest !== requestId) { + // not relevant request anymore + reject(this.number); + return; } - frameDataCache[this.jid].activeChunkRequest = null; - } - }; - const makeActiveRequest = () => { - const taskDataCache = frameDataCache[this.jid]; - const activeChunk = taskDataCache.activeChunkRequest; - activeChunk.request = serverProxy.frames - .getData(null, this.jid, activeChunk.chunkNumber) - .then((chunk) => { - frameDataCache[this.jid].activeChunkRequest.completed = true; - if (!taskDataCache.nextChunkRequest) { - provider.requestDecodeBlock( - chunk, - taskDataCache.activeChunkRequest.start, - taskDataCache.activeChunkRequest.stop, - taskDataCache.activeChunkRequest.onDecodeAll, - taskDataCache.activeChunkRequest.rejectRequestAll, - ); - } - }) - .catch((exception) => { - if (exception instanceof Exception) { - reject(exception); - } else { - reject(new Exception(exception.message)); - } - }) - .finally(() => { - if (taskDataCache.nextChunkRequest) { - if (taskDataCache.activeChunkRequest) { - for (const r of taskDataCache.activeChunkRequest.callbacks) { - r.reject(r.frameNumber); - } - } - taskDataCache.activeChunkRequest = taskDataCache.nextChunkRequest; - taskDataCache.nextChunkRequest = null; - makeActiveRequest(); - } - }); - }; + // it might appear during decoding, so, check again + const currentFrame = provider.frame(this.number); + if (currentFrame) { + resolve({ + renderWidth: this.width, + renderHeight: this.height, + imageData: currentFrame, + }); + return; + } - if (isNode) { - resolve('Dummy data'); - } else if (isBrowser) { - provider - .frame(this.number) - .then((frame) => { - if (frame === null) { - onServerRequest(); - const activeRequest = frameDataCache[this.jid].activeChunkRequest; - if (!provider.isChunkCached(start, stop)) { - if ( - !activeRequest || - (activeRequest && - activeRequest.completed && - activeRequest.chunkNumber !== chunkNumber) - ) { - if (activeRequest && activeRequest.rejectRequestAll) { - activeRequest.rejectRequestAll(); - } - frameDataCache[this.jid].activeChunkRequest = { - request: null, - chunkNumber, - start, - stop, - onDecodeAll, - rejectRequestAll, - completed: false, - callbacks: [ - { - resolve: resolveWrapper, - reject, - frameNumber: this.number, - }, - ], - }; - makeActiveRequest(); - } else if (activeRequest.chunkNumber === chunkNumber) { - if (!activeRequest.onDecodeAll && !activeRequest.rejectRequestAll) { - activeRequest.onDecodeAll = onDecodeAll; - activeRequest.rejectRequestAll = rejectRequestAll; - } - activeRequest.callbacks.push({ - resolve: resolveWrapper, - reject, - frameNumber: this.number, - }); - } else { - if (frameDataCache[this.jid].nextChunkRequest) { - const { callbacks } = frameDataCache[this.jid].nextChunkRequest; - for (const r of callbacks) { - r.reject(r.frameNumber); - } - } - frameDataCache[this.jid].nextChunkRequest = { - request: null, - chunkNumber, - start, - stop, - onDecodeAll, - rejectRequestAll, - completed: false, - callbacks: [ - { - resolve: resolveWrapper, - reject, - frameNumber: this.number, - }, - ], - }; - } - } else { - activeRequest.callbacks.push({ - resolve: resolveWrapper, - reject, - frameNumber: this.number, - }); - provider.requestDecodeBlock(null, start, stop, onDecodeAll, rejectRequestAll); - } - } else { - if ( - this.number % chunkSize > chunkSize / 4 && - provider.decodedBlocksCacheSize > 1 && - this.decodeForward && - !provider.isNextChunkExists(this.number) - ) { - const nextChunkNumber = Math.floor(this.number / chunkSize) + 1; - if (nextChunkNumber * chunkSize < this.stopFrame) { - provider.setReadyToLoading(nextChunkNumber); - const nextStart = nextChunkNumber * chunkSize; - const nextStop = Math.min(this.stopFrame, (nextChunkNumber + 1) * chunkSize - 1); - if (!provider.isChunkCached(nextStart, nextStop)) { - if (!frameDataCache[this.jid].activeChunkRequest) { - frameDataCache[this.jid].activeChunkRequest = { - request: null, - chunkNumber: nextChunkNumber, - start: nextStart, - stop: nextStop, - onDecodeAll: null, - rejectRequestAll: null, - completed: false, - callbacks: [], - }; - makeActiveRequest(); - } - } else { - provider.requestDecodeBlock(null, nextStart, nextStop, null, null); - } - } + frameDataCache[this.jobID].activeChunkRequest = new Promise(( + resolveLoadAndDecode, + ) => { + let wasResolved = false; + frameDataCache[this.jobID].getChunk( + chunkNumber, ChunkQuality.COMPRESSED, + ).then((chunk: ArrayBuffer) => { + try { + provider + .requestDecodeBlock( + chunk, + chunkNumber * chunkSize, + Math.min(stopFrame, (chunkNumber + 1) * chunkSize - 1), + (_frame: number, bitmap: ImageBitmap | Blob) => { + if (decodeForward) { + // resolve immediately only if is not playing + return; + } + + if (frameDataCache[this.jobID].latestFrameDecodeRequest === requestId && + this.number === _frame + ) { + wasResolved = true; + resolve({ + renderWidth: this.width, + renderHeight: this.height, + imageData: bitmap, + }); + } + }, () => { + frameDataCache[this.jobID].activeChunkRequest = null; + resolveLoadAndDecode(); + const decodedFrame = provider.frame(this.number); + if (decodeForward) { + // resolve after decoding everything if playing + resolve({ + renderWidth: this.width, + renderHeight: this.height, + imageData: decodedFrame, + }); + } else if (!wasResolved) { + reject(this.number); + } + }, (error: Error | RequestOutdatedError) => { + frameDataCache[this.jobID].activeChunkRequest = null; + resolveLoadAndDecode(); + if (error instanceof RequestOutdatedError) { + reject(this.number); + } else { + reject(error); + } + }, + ); + } catch (error) { + reject(error); } - resolveWrapper(frame); - } - }) - .catch((exception) => { - if (exception instanceof Exception) { - reject(exception); - } else { - reject(new Exception(exception.message)); - } + }).catch((error) => { + reject(error); + resolveLoadAndDecode(error); + }); }); - } - }); -}; + }); + }); + }, + writable: false, +}); function getFrameMeta(jobID, frame): RawFramesMetaData['frames'][0] { const { meta, mode, startFrame } = frameDataCache[jobID]; @@ -403,305 +337,93 @@ function getFrameMeta(jobID, frame): RawFramesMetaData['frames'][0] { return frameMeta; } -class FrameBuffer { - constructor(size, chunkSize, stopFrame, jobID) { - this._size = size; - this._buffer = {}; - this._contextImage = {}; - this._requestedChunks = {}; - this._chunkSize = chunkSize; - this._stopFrame = stopFrame; - this._activeFillBufferRequest = false; - this._jobID = jobID; - } - - addContextImage(frame, data): void { - const promise = new Promise((resolve, reject) => { - data.then((resolvedData) => { - const meta = getFrameMeta(this._jobID, frame); - return cvatData - .decodeZip(resolvedData, 0, meta.related_files, cvatData.DimensionType.DIMENSION_2D); - }).then((decodedData) => { - this._contextImage[frame] = decodedData; - resolve(); - }).catch((error: Error) => { - if (error instanceof ServerError && (error as any).code === 404) { - this._contextImage[frame] = {}; - resolve(); - } else { - reject(error); - } - }); - }); - - this._contextImage[frame] = promise; - } - - isContextImageAvailable(frame): boolean { - return frame in this._contextImage; - } - - getContextImage(frame): Promise { - return new Promise((resolve) => { - if (frame in this._contextImage) { - if (this._contextImage[frame] instanceof Promise) { - this._contextImage[frame].then(() => { - resolve(this.getContextImage(frame)); - }); - } else { - resolve({ ...this._contextImage[frame] }); - } - } else { - resolve([]); - } - }); - } - - getFreeBufferSize() { - let requestedFrameCount = 0; - for (const chunk of Object.values(this._requestedChunks)) { - requestedFrameCount += chunk.requestedFrames.size; +export function getContextImage(jobID: number, frame: number): Promise> { + return new Promise>((resolve, reject) => { + if (!(jobID in frameDataCache)) { + reject(new Error( + 'Frame data was not initialized for this job. Try first requesting any frame.', + )); } + const frameData = frameDataCache[jobID]; + const requestId = frame; + const { startFrame } = frameData; + const { related_files: relatedFiles } = frameData.meta.frames[frame - startFrame]; + + if (relatedFiles === 0) { + resolve({}); + } else if (frame in frameData.contextCache) { + resolve(frameData.contextCache[frame].data); + } else { + frameData.latestContextImagesRequest = requestId; + const executor = (): void => { + if (frameData.latestContextImagesRequest !== requestId) { + reject(frame); + } else if (frame in frameData.contextCache) { + resolve(frameData.contextCache[frame].data); + } else { + frameData.activeContextRequest = serverProxy.frames.getImageContext(jobID, frame) + .then((encodedImages) => decodeContextImages(encodedImages, 0, relatedFiles)); + frameData.activeContextRequest.then((images) => { + const size = Object.values(images) + .reduce((acc, image) => acc + image.width * image.height * 4, 0); + const totalSize = Object.values(frameData.contextCache) + .reduce((acc, item) => acc + item.size, 0); + if (totalSize > 512 * 1024 * 1024) { + const [leastTimestampFrame] = Object.entries(frameData.contextCache) + .sort(([, item1], [, item2]) => item1.timestamp - item2.timestamp)[0]; + delete frameData.contextCache[leastTimestampFrame]; + } - return this._size - Object.keys(this._buffer).length - requestedFrameCount; - } - - requestOneChunkFrames(chunkIdx) { - return new Promise((resolve, reject) => { - this._requestedChunks[chunkIdx] = { - ...this._requestedChunks[chunkIdx], - resolve, - reject, - }; - for (const frame of this._requestedChunks[chunkIdx].requestedFrames.entries()) { - const requestedFrame = frame[1]; - const frameMeta = getFrameMeta(this._jobID, requestedFrame); - const frameData = new FrameData({ - ...frameMeta, - jobID: this._jobID, - frameNumber: requestedFrame, - startFrame: frameDataCache[this._jobID].startFrame, - stopFrame: frameDataCache[this._jobID].stopFrame, - decodeForward: false, - deleted: requestedFrame in frameDataCache[this._jobID].meta, - }); + frameData.contextCache[frame] = { + data: images, + timestamp: Date.now(), + size, + }; - frameData - .data() - .then(() => { - if ( - !(chunkIdx in this._requestedChunks) || - !this._requestedChunks[chunkIdx].requestedFrames.has(requestedFrame) - ) { - reject(chunkIdx); + if (frameData.latestContextImagesRequest !== requestId) { + reject(frame); } else { - this._requestedChunks[chunkIdx].requestedFrames.delete(requestedFrame); - this._requestedChunks[chunkIdx].buffer[requestedFrame] = frameData; - if (this._requestedChunks[chunkIdx].requestedFrames.size === 0) { - const bufferedframes = Object.keys(this._requestedChunks[chunkIdx].buffer).map( - (f) => +f, - ); - this._requestedChunks[chunkIdx].resolve(new Set(bufferedframes)); - } + resolve(images); } - }) - .catch(() => { - reject(chunkIdx); + }).finally(() => { + frameData.activeContextRequest = null; }); - } - }); - } - - fillBuffer(startFrame, frameStep = 1, count = null) { - const freeSize = this.getFreeBufferSize(); - const requestedFrameCount = count ? count * frameStep : freeSize * frameStep; - const stopFrame = Math.min(startFrame + requestedFrameCount, this._stopFrame + 1); - - for (let i = startFrame; i < stopFrame; i += frameStep) { - const chunkIdx = Math.floor(i / this._chunkSize); - if (!(chunkIdx in this._requestedChunks)) { - this._requestedChunks[chunkIdx] = { - requestedFrames: new Set(), - resolve: null, - reject: null, - buffer: {}, - }; - } - this._requestedChunks[chunkIdx].requestedFrames.add(i); - } - - let bufferedFrames = new Set(); - - // if we send one request to get frame 1 with filling the buffer - // then quicky send one more request to get frame 1 - // frame 1 will be already decoded and written to buffer - // the second request gets frame 1 from the buffer, removes it from there and returns - // after the first request finishes decoding it tries to get frame 1, but failed - // because frame 1 was already removed from the buffer by the second request - // to prevent this behavior we do not write decoded frames to buffer till the end of decoding all chunks - const buffersToBeCommited = []; - const commitBuffers = () => { - for (const buffer of buffersToBeCommited) { - this._buffer = { - ...this._buffer, - ...buffer, - }; - } - }; + } + }; - // Need to decode chunks in sequence - // eslint-disable-next-line no-async-promise-executor - return new Promise(async (resolve, reject) => { - for (const chunkIdx of Object.keys(this._requestedChunks)) { - try { - const chunkFrames = await this.requestOneChunkFrames(chunkIdx); - if (chunkIdx in this._requestedChunks) { - bufferedFrames = new Set([...bufferedFrames, ...chunkFrames]); - - buffersToBeCommited.push(this._requestedChunks[chunkIdx].buffer); - delete this._requestedChunks[chunkIdx]; - if (Object.keys(this._requestedChunks).length === 0) { - commitBuffers(); - resolve(bufferedFrames); - } + if (!frameData.activeContextRequest) { + executor(); + } else { + const checkAndExecute = (): void => { + if (frameData.activeContextRequest) { + // if we just execute in finally + // it might raise multiple server requests for context images + // if the promise was pending before and several requests came for the same frame + // all these requests will stuck on "finally" + // and when the promise fullfilled, it will run all the microtasks + // since they all have the same request id, all they will perform in executor() + frameData.activeContextRequest.finally(() => setTimeout(checkAndExecute)); } else { - commitBuffers(); - reject(chunkIdx); - break; + executor(); } - } catch (error) { - commitBuffers(); - reject(error); - break; - } - } - }); - } - - async makeFillRequest(start, step, count = null) { - if (!this._activeFillBufferRequest) { - this._activeFillBufferRequest = true; - try { - await this.fillBuffer(start, step, count); - this._activeFillBufferRequest = false; - } catch (error) { - if (typeof error === 'number' && error in this._requestedChunks) { - this._activeFillBufferRequest = false; - } - throw error; - } - } - } - - async require(frameNumber: number, jobID: number, fillBuffer: boolean, frameStep: number): FrameData { - for (const frame in this._buffer) { - if (+frame < frameNumber || +frame >= frameNumber + this._size * frameStep) { - delete this._buffer[frame]; - } - } - - this._required = frameNumber; - const frameMeta = getFrameMeta(jobID, frameNumber); - let frame = new FrameData({ - ...frameMeta, - jobID, - frameNumber, - startFrame: frameDataCache[jobID].startFrame, - stopFrame: frameDataCache[jobID].stopFrame, - decodeForward: !fillBuffer, - deleted: frameNumber in frameDataCache[jobID].meta.deleted_frames, - }); - - if (frameNumber in this._buffer) { - frame = this._buffer[frameNumber]; - delete this._buffer[frameNumber]; - const cachedFrames = this.cachedFrames(); - if ( - fillBuffer && - !this._activeFillBufferRequest && - this._size > this._chunkSize && - cachedFrames.length < (this._size * 3) / 4 - ) { - const maxFrame = cachedFrames ? Math.max(...cachedFrames) : frameNumber; - if (maxFrame < this._stopFrame) { - this.makeFillRequest(maxFrame + 1, frameStep).catch((e) => { - if (e !== 'not needed') { - throw e; - } - }); - } - } - } else if (fillBuffer) { - this.clear(); - await this.makeFillRequest(frameNumber, frameStep, fillBuffer ? null : 1); - frame = this._buffer[frameNumber]; - } else { - this.clear(); - } - - return frame; - } + }; - clear() { - for (const chunkIdx in this._requestedChunks) { - if ( - Object.prototype.hasOwnProperty.call(this._requestedChunks, chunkIdx) && - this._requestedChunks[chunkIdx].reject - ) { - this._requestedChunks[chunkIdx].reject('not needed'); + setTimeout(checkAndExecute); } } - this._activeFillBufferRequest = false; - this._requestedChunks = {}; - this._buffer = {}; - } - - cachedFrames() { - return Object.keys(this._buffer).map((f) => +f); - } -} - -async function getImageContext(jobID, frame) { - return new Promise((resolve, reject) => { - serverProxy.frames - .getImageContext(jobID, frame) - .then((result) => { - if (isNode) { - // eslint-disable-next-line no-undef - resolve(global.Buffer.from(result, 'binary').toString('base64')); - } else if (isBrowser) { - resolve(result); - } - }) - .catch((error) => { - reject(error); - }); }); } -export async function getContextImage(jobID, frame) { - if (frameDataCache[jobID].frameBuffer.isContextImageAvailable(frame)) { - return frameDataCache[jobID].frameBuffer.getContextImage(frame); - } - const response = getImageContext(jobID, frame); - await frameDataCache[jobID].frameBuffer.addContextImage(frame, response); - return frameDataCache[jobID].frameBuffer.getContextImage(frame); -} - export function decodePreview(preview: Blob): Promise { return new Promise((resolve, reject) => { - if (isNode) { - resolve(global.Buffer.from(preview, 'binary').toString('base64')); - } else if (isBrowser) { - const reader = new FileReader(); - reader.onload = () => { - resolve(reader.result as string); - }; - reader.onerror = (error) => { - reject(error); - }; - reader.readAsDataURL(preview); - } + const reader = new FileReader(); + reader.onload = () => { + resolve(reader.result as string); + }; + reader.onerror = (error) => { + reject(error); + }; + reader.readAsDataURL(preview); }); } @@ -716,60 +438,73 @@ export async function getFrame( isPlaying: boolean, step: number, dimension: DimensionType, -) { + getChunk: (chunkNumber: number, quality: ChunkQuality) => Promise, +): Promise { if (!(jobID in frameDataCache)) { - const blockType = chunkType === 'video' ? cvatData.BlockType.MP4VIDEO : cvatData.BlockType.ARCHIVE; + const blockType = chunkType === 'video' ? BlockType.MP4VIDEO : BlockType.ARCHIVE; const meta = await serverProxy.frames.getMeta('job', jobID); - meta.deleted_frames = Object.fromEntries(meta.deleted_frames.map((_frame) => [_frame, true])); - const mean = meta.frames.reduce((a, b) => a + b.width * b.height, 0) / meta.frames.length; + const updatedMeta = { + ...meta, + deleted_frames: Object.fromEntries(meta.deleted_frames.map((_frame) => [_frame, true])), + }; + const mean = updatedMeta.frames.reduce((a, b) => a + b.width * b.height, 0) / updatedMeta.frames.length; const stdDev = Math.sqrt( - meta.frames.map((x) => (x.width * x.height - mean) ** 2).reduce((a, b) => a + b) / - meta.frames.length, + updatedMeta.frames.map((x) => (x.width * x.height - mean) ** 2).reduce((a, b) => a + b) / + updatedMeta.frames.length, ); // limit of decoded frames cache by 2GB - const decodedBlocksCacheSize = Math.floor(2147483648 / (mean + stdDev) / 4 / chunkSize) || 1; - + const decodedBlocksCacheSize = Math.min( + Math.floor((2048 * 1024 * 1024) / ((mean + stdDev) * 4 * chunkSize)) || 1, 10, + ); frameDataCache[jobID] = { - meta, + meta: updatedMeta, chunkSize, mode, startFrame, stopFrame, - provider: new cvatData.FrameProvider( + decodeForward: isPlaying, + forwardStep: step, + provider: new FrameDecoder( blockType, chunkSize, - Math.max(decodedBlocksCacheSize, 9), decodedBlocksCacheSize, - 1, dimension, ), - frameBuffer: new FrameBuffer( - Math.min(180, decodedBlocksCacheSize * chunkSize), - chunkSize, - stopFrame, - jobID, - ), decodedBlocksCacheSize, activeChunkRequest: null, - nextChunkRequest: null, + activeContextRequest: null, + latestFrameDecodeRequest: null, + latestContextImagesRequest: null, + contextCache: {}, + getChunk, }; - - const frameMeta = getFrameMeta(jobID, frame); - frameDataCache[jobID].provider.setRenderSize(frameMeta.width, frameMeta.height); } - return frameDataCache[jobID].frameBuffer.require(frame, jobID, isPlaying, step); + const frameMeta = getFrameMeta(jobID, frame); + frameDataCache[jobID].provider.setRenderSize(frameMeta.width, frameMeta.height); + frameDataCache[jobID].decodeForward = isPlaying; + frameDataCache[jobID].forwardStep = step; + + return new FrameData({ + width: frameMeta.width, + height: frameMeta.height, + name: frameMeta.name, + related_files: frameMeta.related_files, + frameNumber: frame, + deleted: frame in frameDataCache[jobID].meta.deleted_frames, + jobID, + }); } -export async function getDeletedFrames(instanceType, id) { +export async function getDeletedFrames(instanceType: 'job' | 'task', id) { if (instanceType === 'job') { const { meta } = frameDataCache[id]; return meta.deleted_frames; } if (instanceType === 'task') { - const meta = await serverProxy.frames.getMeta('job', id); + const meta = await serverProxy.frames.getMeta('task', id); meta.deleted_frames = Object.fromEntries(meta.deleted_frames.map((_frame) => [_frame, true])); return meta; } @@ -777,19 +512,19 @@ export async function getDeletedFrames(instanceType, id) { throw new Exception(`getDeletedFrames is not implemented for ${instanceType}`); } -export function deleteFrame(jobID, frame) { +export function deleteFrame(jobID: number, frame: number): void { const { meta } = frameDataCache[jobID]; meta.deleted_frames[frame] = true; } -export function restoreFrame(jobID, frame) { +export function restoreFrame(jobID: number, frame: number): void { const { meta } = frameDataCache[jobID]; if (frame in meta.deleted_frames) { delete meta.deleted_frames[frame]; } } -export async function patchMeta(jobID) { +export async function patchMeta(jobID: number): Promise { const { meta } = frameDataCache[jobID]; const newMeta = await serverProxy.frames.saveMeta('job', jobID, { deleted_frames: Object.keys(meta.deleted_frames), @@ -808,7 +543,9 @@ export async function patchMeta(jobID) { frameDataCache[jobID].meta.deleted_frames = prevDeletedFrames; } -export async function findFrame(jobID, frameFrom, frameTo, filters) { +export async function findFrame( + jobID: number, frameFrom: number, frameTo: number, filters: { offset?: number, notDeleted: boolean }, +): Promise { const offset = filters.offset || 1; let meta; if (!frameDataCache[jobID]) { @@ -845,23 +582,16 @@ export async function findFrame(jobID, frameFrom, frameTo, filters) { return lastUndeletedFrame; } -export function getRanges(jobID) { +export function getCachedChunks(jobID): number[] { if (!(jobID in frameDataCache)) { - return { - decoded: [], - buffered: [], - }; + return []; } - return { - decoded: frameDataCache[jobID].provider.cachedFrames, - buffered: frameDataCache[jobID].frameBuffer.cachedFrames(), - }; + return frameDataCache[jobID].provider.cachedChunks(true); } -export function clear(jobID) { +export function clear(jobID: number): void { if (jobID in frameDataCache) { - frameDataCache[jobID].frameBuffer.clear(); delete frameDataCache[jobID]; } } diff --git a/cvat-core/src/ml-model.ts b/cvat-core/src/ml-model.ts index 2d70f6aace33..04952d153f08 100644 --- a/cvat-core/src/ml-model.ts +++ b/cvat-core/src/ml-model.ts @@ -3,9 +3,9 @@ // // SPDX-License-Identifier: MIT -import { isBrowser, isNode } from 'browser-or-node'; import serverProxy from './server-proxy'; import PluginRegistry from './plugins'; +import { decodePreview } from './frames'; import { ModelProviders, ModelKind, ModelReturnType } from './enums'; import { SerializedModel, ModelAttribute, ModelParams, ModelTip, @@ -117,8 +117,8 @@ export default class MLModel { return result; } - public async getPreview(): Promise { - const result = await PluginRegistry.apiWrapper.call(this, MLModel.prototype.getPreview); + public async preview(): Promise { + const result = await PluginRegistry.apiWrapper.call(this, MLModel.prototype.preview); return result; } } @@ -127,7 +127,7 @@ Object.defineProperties(MLModel.prototype.save, { implementation: { writable: false, enumerable: false, - value: async function implementation(): Promise { + value: async function implementation(this: MLModel): Promise { const modelData = { provider: this.provider, url: this.serialized.url, @@ -144,7 +144,7 @@ Object.defineProperties(MLModel.prototype.delete, { implementation: { writable: false, enumerable: false, - value: async function implementation(): Promise { + value: async function implementation(this: MLModel): Promise { if (this.isDeletable) { await serverProxy.functions.delete(this.id); } @@ -152,32 +152,15 @@ Object.defineProperties(MLModel.prototype.delete, { }, }); -Object.defineProperties(MLModel.prototype.getPreview, { +Object.defineProperties(MLModel.prototype.preview, { implementation: { writable: false, enumerable: false, - value: async function implementation(): Promise { - if (this.provider === ModelProviders.CVAT) { - return ''; - } - return new Promise((resolve, reject) => { - serverProxy.functions - .getPreview(this.id) - .then((result) => { - if (isNode) { - resolve(global.Buffer.from(result, 'binary').toString('base64')); - } else if (isBrowser) { - const reader = new FileReader(); - reader.onload = () => { - resolve(reader.result); - }; - reader.readAsDataURL(result); - } - }) - .catch((error) => { - reject(error); - }); - }); + value: async function implementation(this: MLModel): Promise { + if (this.provider === ModelProviders.CVAT) return ''; + const preview = await serverProxy.functions.getPreview(this.id); + if (!preview) return ''; + return decodePreview(preview); }, }, }); diff --git a/cvat-core/src/organization.ts b/cvat-core/src/organization.ts index 166fa790d58d..fcc3379d2150 100644 --- a/cvat-core/src/organization.ts +++ b/cvat-core/src/organization.ts @@ -3,31 +3,15 @@ // // SPDX-License-Identifier: MIT +import { SerializedOrganization, SerializedOrganizationContact } from './server-response-types'; import { checkObjectType, isEnum } from './common'; import config from './config'; import { MembershipRole } from './enums'; -import { ArgumentError, ServerError } from './exceptions'; +import { ArgumentError, DataError } from './exceptions'; import PluginRegistry from './plugins'; import serverProxy from './server-proxy'; import User from './user'; -interface RawOrganizationData { - id?: number, - slug?: string, - name?: string, - description?: string, - created_date?: string, - updated_date?: string, - owner?: any, - contact?: OrganizationContact, -} - -interface OrganizationContact { - email?: string; - location?: string; - phoneNumber?: string -} - interface Membership { user: User; is_active: boolean; @@ -45,12 +29,12 @@ export default class Organization { public readonly createdDate: string; public readonly updatedDate: string; public readonly owner: User; - public contact: OrganizationContact; + public contact: SerializedOrganizationContact; public name: string; public description: string; - constructor(initialData: RawOrganizationData) { - const data: RawOrganizationData = { + constructor(initialData: SerializedOrganization) { + const data: SerializedOrganization = { id: undefined, slug: undefined, name: undefined, @@ -354,7 +338,7 @@ Object.defineProperties(Organization.prototype.leave, { }); const [membership] = result.results; if (!membership) { - throw new ServerError( + throw new DataError( `Could not find membership for user ${user.username} in organization ${this.slug}`, ); } diff --git a/cvat-core/src/project-implementation.ts b/cvat-core/src/project-implementation.ts index 17e0c953d1e1..8a274d4ee308 100644 --- a/cvat-core/src/project-implementation.ts +++ b/cvat-core/src/project-implementation.ts @@ -83,10 +83,11 @@ export default function implementProject(projectClass) { return result; }; - projectClass.prototype.preview.implementation = async function () { + projectClass.prototype.preview.implementation = async function (this: Project): Promise { + if (this.id === null) return ''; const preview = await serverProxy.projects.getPreview(this.id); - const decoded = await decodePreview(preview); - return decoded; + if (!preview) return ''; + return decodePreview(preview); }; projectClass.prototype.annotations.exportDataset.implementation = async function ( diff --git a/cvat-core/src/quality-settings.ts b/cvat-core/src/quality-settings.ts index 2ff4ab9001e6..73f9245d131a 100644 --- a/cvat-core/src/quality-settings.ts +++ b/cvat-core/src/quality-settings.ts @@ -2,26 +2,10 @@ // // SPDX-License-Identifier: MIT +import { SerializedQualitySettingsData } from './server-response-types'; import PluginRegistry from './plugins'; import serverProxy from './server-proxy'; -export interface SerializedQualitySettingsData { - id?: number; - task?: number; - iou_threshold?: number; - oks_sigma?: number; - line_thickness?: number; - low_overlap_threshold?: number; - compare_line_orientation?: boolean; - line_orientation_threshold?: number; - compare_groups?: boolean; - group_match_threshold?: number; - check_covered_annotations?: boolean; - object_visibility_threshold?: number; - panoptic_comparison?: boolean; - compare_attributes?: boolean; -} - export default class QualitySettings { #id: number; #task: number; diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 6b438d6ca168..0fabf6ffb20b 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -7,14 +7,15 @@ import FormData from 'form-data'; import store from 'store'; import Axios, { AxiosError, AxiosResponse } from 'axios'; import * as tus from 'tus-js-client'; +import { ChunkQuality } from 'cvat-data'; + import { SerializedLabel, SerializedAnnotationFormats, ProjectsFilter, - SerializedProject, SerializedTask, TasksFilter, SerializedUser, - SerializedAbout, SerializedRemoteFile, SerializedUserAgreement, + SerializedProject, SerializedTask, TasksFilter, SerializedUser, SerializedOrganization, + SerializedAbout, SerializedRemoteFile, SerializedUserAgreement, FunctionsResponseBody, SerializedRegister, JobsFilter, SerializedJob, SerializedGuide, SerializedAsset, -} from 'server-response-types'; -import { SerializedQualityReportData } from 'quality-report'; -import { SerializedQualitySettingsData } from 'quality-settings'; +} from './server-response-types'; +import { SerializedQualityReportData } from './quality-report'; import { SerializedAnalyticsReport } from './analytics-report'; import { Storage } from './storage'; import { StorageLocation, WebhookSourceType } from './enums'; @@ -22,7 +23,6 @@ import { isEmail, isResourceURL } from './common'; import config from './config'; import DownloadWorker from './download.worker'; import { ServerError } from './exceptions'; -import { FunctionsResponseBody } from './server-response-types'; import { SerializedQualityConflictData } from './quality-conflict'; type Params = { @@ -1402,7 +1402,7 @@ async function deleteJob(jobID: number): Promise { } } -async function getUsers(filter = { page_size: 'all' }) { +async function getUsers(filter = { page_size: 'all' }): Promise { const { backendAPI } = config; let response = null; @@ -1419,8 +1419,8 @@ async function getUsers(filter = { page_size: 'all' }) { return response.data.results; } -function getPreview(instance: 'projects' | 'tasks' | 'jobs' | 'cloudstorages') { - return async function (id: number) { +function getPreview(instance: 'projects' | 'tasks' | 'jobs' | 'cloudstorages' | 'functions') { + return async function (id: number | string): Promise { const { backendAPI } = config; let response = null; @@ -1429,21 +1429,23 @@ function getPreview(instance: 'projects' | 'tasks' | 'jobs' | 'cloudstorages') { response = await Axios.get(url, { responseType: 'blob', }); + + return response.data; } catch (errorData) { const code = errorData.response ? errorData.response.status : errorData.code; + if (code === 404) { + return null; + } throw new ServerError(`Could not get preview for "${instance}/${id}"`, code); } - - return (response.status === 200) ? response.data : ''; }; } -async function getImageContext(jid, frame) { +async function getImageContext(jid: number, frame: number): Promise { const { backendAPI } = config; - let response = null; try { - response = await Axios.get(`${backendAPI}/jobs/${jid}/data`, { + const response = await Axios.get(`${backendAPI}/jobs/${jid}/data`, { params: { quality: 'original', type: 'context_image', @@ -1451,29 +1453,28 @@ async function getImageContext(jid, frame) { }, responseType: 'arraybuffer', }); + + return response.data; } catch (errorData) { throw generateError(errorData); } - - return response.data; } -async function getData(tid, jid, chunk) { +async function getData(jid: number, chunk: number, quality: ChunkQuality): Promise { const { backendAPI } = config; - const url = jid === null ? `tasks/${tid}/data` : `jobs/${jid}/data`; - - let response = null; try { - response = await workerAxios.get(`${backendAPI}/${url}`, { + const response = await workerAxios.get(`${backendAPI}/jobs/${jid}/data`, { params: { ...enableOrganization(), - quality: 'compressed', + quality, type: 'chunk', number: chunk, }, responseType: 'arraybuffer', }); + + return response; } catch (errorData) { throw generateError({ message: '', @@ -1483,8 +1484,6 @@ async function getData(tid, jid, chunk) { }, }); } - - return response; } export interface RawFramesMetaData { @@ -1560,23 +1559,6 @@ async function getFunctions(): Promise { } } -async function getFunctionPreview(modelID) { - const { backendAPI } = config; - - let response = null; - try { - const url = `${backendAPI}/functions/${modelID}/preview`; - response = await Axios.get(url, { - responseType: 'blob', - }); - } catch (errorData) { - const code = errorData.response ? errorData.response.status : errorData.code; - throw new ServerError(`Could not get preview for the model ${modelID} from the server`, code); - } - - return response.data; -} - async function getFunctionProviders() { const { backendAPI } = config; @@ -1973,7 +1955,7 @@ async function getOrganizations() { return response.results; } -async function createOrganization(data) { +async function createOrganization(data: SerializedOrganization): Promise { const { backendAPI } = config; let response = null; @@ -1988,7 +1970,9 @@ async function createOrganization(data) { return response.data; } -async function updateOrganization(id, data) { +async function updateOrganization( + id: number, data: Partial, +): Promise { const { backendAPI } = config; let response = null; @@ -2001,7 +1985,7 @@ async function updateOrganization(id, data) { return response.data; } -async function deleteOrganization(id) { +async function deleteOrganization(id: number): Promise { const { backendAPI } = config; try { @@ -2058,7 +2042,7 @@ async function updateOrganizationMembership(membershipId, data) { return response.data; } -async function deleteOrganizationMembership(membershipId) { +async function deleteOrganizationMembership(membershipId: number): Promise { const { backendAPI } = config; try { @@ -2435,7 +2419,7 @@ export default Object.freeze({ providers: getFunctionProviders, delete: deleteFunction, cancel: cancelFunctionRequest, - getPreview: getFunctionPreview, + getPreview: getPreview('functions'), }), issues: Object.freeze({ diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index 5b96ee21e8a3..ef4cfadef61b 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -198,3 +198,37 @@ export interface SerializedAsset { created_date: string; owner: SerializedUser; } + +export interface SerializedOrganizationContact { + email?: string; + location?: string; + phoneNumber?: string +} + +export interface SerializedOrganization { + id?: number, + slug?: string, + name?: string, + description?: string, + created_date?: string, + updated_date?: string, + owner?: any, + contact?: SerializedOrganizationContact, +} + +export interface SerializedQualitySettingsData { + id?: number; + task?: number; + iou_threshold?: number; + oks_sigma?: number; + line_thickness?: number; + low_overlap_threshold?: number; + compare_line_orientation?: boolean; + line_orientation_threshold?: number; + compare_groups?: boolean; + group_match_threshold?: number; + check_covered_annotations?: boolean; + object_visibility_threshold?: number; + panoptic_comparison?: boolean; + compare_attributes?: boolean; +} diff --git a/cvat-core/src/session-implementation.ts b/cvat-core/src/session-implementation.ts index 1cae70abd298..5a873ae823c3 100644 --- a/cvat-core/src/session-implementation.ts +++ b/cvat-core/src/session-implementation.ts @@ -6,13 +6,14 @@ import { ArgumentError } from './exceptions'; import { HistoryActions, JobType } from './enums'; import { Storage } from './storage'; +import { Task as TaskClass, Job as JobClass } from './session'; import loggerStorage from './logger-storage'; import serverProxy from './server-proxy'; import { getFrame, deleteFrame, restoreFrame, - getRanges, + getCachedChunks, clear as clearFrames, findFrame, getContextImage, @@ -128,6 +129,7 @@ export function implementJob(Job) { isPlaying, step, this.dimension, + (chunkNumber, quality) => this.frames.chunk(chunkNumber, quality), ); return frameData; }; @@ -161,19 +163,16 @@ export function implementJob(Job) { return result; }; - Job.prototype.frames.ranges.implementation = async function () { - const rangesData = await getRanges(this.id); - return rangesData; + Job.prototype.frames.cachedChunks.implementation = async function () { + const cachedChunks = await getCachedChunks(this.id); + return cachedChunks; }; - Job.prototype.frames.preview.implementation = async function () { - if (this.id === null || this.taskId === null) { - return ''; - } - + Job.prototype.frames.preview.implementation = async function (this: JobClass): Promise { + if (this.id === null || this.taskId === null) return ''; const preview = await serverProxy.jobs.getPreview(this.id); - const decoded = await decodePreview(preview); - return decoded; + if (!preview) return ''; + return decodePreview(preview); }; Job.prototype.frames.contextImage.implementation = async function (frameId) { @@ -181,6 +180,11 @@ export function implementJob(Job) { return result; }; + Job.prototype.frames.chunk.implementation = async function (chunkNumber, quality) { + const result = await serverProxy.frames.getData(this.id, chunkNumber, quality); + return result; + }; + Job.prototype.frames.search.implementation = async function (filters, frameFrom, frameTo) { if (typeof filters !== 'object') { throw new ArgumentError('Filters should be an object'); @@ -566,31 +570,25 @@ export function implementTask(Task) { isPlaying, step, this.dimension, + (chunkNumber, quality) => job.frames.chunk(chunkNumber, quality), ); return result; }; - Task.prototype.frames.ranges.implementation = async function () { - const rangesData = { - decoded: [], - buffered: [], - }; + Task.prototype.frames.cachedChunks.implementation = async function () { + let chunks = []; for (const job of this.jobs) { - const { decoded, buffered } = await getRanges(job.id); - rangesData.decoded.push(decoded); - rangesData.buffered.push(buffered); + const cachedChunks = await getCachedChunks(job.id); + chunks = chunks.concat(cachedChunks); } - return rangesData; + return Array.from(new Set(chunks)); }; - Task.prototype.frames.preview.implementation = async function () { - if (this.id === null) { - return ''; - } - + Task.prototype.frames.preview.implementation = async function (this: TaskClass): Promise { + if (this.id === null) return ''; const preview = await serverProxy.tasks.getPreview(this.id); - const decoded = await decodePreview(preview); - return decoded; + if (!preview) return ''; + return decodePreview(preview); }; Task.prototype.frames.delete.implementation = async function (frame) { @@ -661,6 +659,14 @@ export function implementTask(Task) { return null; }; + Task.prototype.frames.contextImage.implementation = async function () { + throw new Error('Not implemented'); + }; + + Task.prototype.frames.chunk.implementation = async function () { + throw new Error('Not implemented'); + }; + // TODO: Check filter for annotations Task.prototype.annotations.get.implementation = async function (frame, allTracks, filters) { if (!Array.isArray(filters) || filters.some((filter) => typeof filter !== 'string')) { diff --git a/cvat-core/src/session.ts b/cvat-core/src/session.ts index 96e355e64d74..cad8b773ae21 100644 --- a/cvat-core/src/session.ts +++ b/cvat-core/src/session.ts @@ -210,8 +210,8 @@ function buildDuplicatedAPI(prototype) { prototype.frames.save, ); }, - async ranges() { - const result = await PluginRegistry.apiWrapper.call(this, prototype.frames.ranges); + async cachedChunks() { + const result = await PluginRegistry.apiWrapper.call(this, prototype.frames.cachedChunks); return result; }, async preview() { @@ -236,6 +236,15 @@ function buildDuplicatedAPI(prototype) { ); return result; }, + async chunk(chunkNumber, quality) { + const result = await PluginRegistry.apiWrapper.call( + this, + prototype.frames.chunk, + chunkNumber, + quality, + ); + return result; + }, }, writable: true, }), @@ -320,6 +329,7 @@ export class Job extends Session { public readonly taskId: number; public readonly dimension: DimensionType; public readonly dataChunkType: ChunkType; + public readonly dataChunkSize: number; public readonly bugTracker: string | null; public readonly mode: TaskMode; public readonly labels: Label[]; @@ -360,10 +370,11 @@ export class Job extends Session { delete: CallableFunction; restore: CallableFunction; save: CallableFunction; - ranges: CallableFunction; + cachedChunks: CallableFunction; preview: CallableFunction; contextImage: CallableFunction; search: CallableFunction; + chunk: CallableFunction; }; public logger: { @@ -563,10 +574,11 @@ export class Job extends Session { delete: Object.getPrototypeOf(this).frames.delete.bind(this), restore: Object.getPrototypeOf(this).frames.restore.bind(this), save: Object.getPrototypeOf(this).frames.save.bind(this), - ranges: Object.getPrototypeOf(this).frames.ranges.bind(this), + cachedChunks: Object.getPrototypeOf(this).frames.cachedChunks.bind(this), preview: Object.getPrototypeOf(this).frames.preview.bind(this), search: Object.getPrototypeOf(this).frames.search.bind(this), contextImage: Object.getPrototypeOf(this).frames.contextImage.bind(this), + chunk: Object.getPrototypeOf(this).frames.chunk.bind(this), }; this.logger = { @@ -673,10 +685,11 @@ export class Task extends Session { delete: CallableFunction; restore: CallableFunction; save: CallableFunction; - ranges: CallableFunction; + cachedChunks: CallableFunction; preview: CallableFunction; contextImage: CallableFunction; search: CallableFunction; + chunk: CallableFunction; }; public logger: { @@ -1089,10 +1102,11 @@ export class Task extends Session { delete: Object.getPrototypeOf(this).frames.delete.bind(this), restore: Object.getPrototypeOf(this).frames.restore.bind(this), save: Object.getPrototypeOf(this).frames.save.bind(this), - ranges: Object.getPrototypeOf(this).frames.ranges.bind(this), + cachedChunks: Object.getPrototypeOf(this).frames.cachedChunks.bind(this), preview: Object.getPrototypeOf(this).frames.preview.bind(this), contextImage: Object.getPrototypeOf(this).frames.contextImage.bind(this), search: Object.getPrototypeOf(this).frames.search.bind(this), + chunk: Object.getPrototypeOf(this).frames.chunk.bind(this), }; this.logger = { diff --git a/cvat-core/tests/api/frames.js b/cvat-core/tests/api/frames.js index 8f9299ab2895..0ee20eb9fda5 100644 --- a/cvat-core/tests/api/frames.js +++ b/cvat-core/tests/api/frames.js @@ -88,22 +88,6 @@ describe('Feature: delete/restore frame', () => { }); }); -describe('Feature: get frame data', () => { - test('get frame data for a task', async () => { - const task = (await window.cvat.tasks.get({ id: 100 }))[0]; - const frame = await task.frames.get(0); - const frameData = await frame.data(); - expect(typeof frameData).toBe('string'); - }); - - test('get frame data for a job', async () => { - const job = (await window.cvat.jobs.get({ jobID: 100 }))[0]; - const frame = await job.frames.get(0); - const frameData = await frame.data(); - expect(typeof frameData).toBe('string'); - }); -}); - describe('Feature: get frame preview', () => { test('get frame preview for a task', async () => { const task = (await window.cvat.tasks.get({ id: 100 }))[0]; diff --git a/cvat-core/tests/mocks/server-proxy.mock.js b/cvat-core/tests/mocks/server-proxy.mock.js index d5587a42082d..06020c20a840 100644 --- a/cvat-core/tests/mocks/server-proxy.mock.js +++ b/cvat-core/tests/mocks/server-proxy.mock.js @@ -373,19 +373,36 @@ class ServerProxy { } async function getPreview() { - return 'DUMMY_IMAGE'; + return null; } async function getData() { return 'DUMMY_IMAGE'; } - async function getMeta(session, jid) { + async function getMeta(session, id) { if (session !== 'job') { - throw new Error('not implemented test'); + const task = tasksDummyData.results.find((task) => task.id === id); + const jobs = jobsDummyData.results.filter((job) => job.task_id === id); + const jobsMeta = jobs.map((job) => frameMetaDummyData[job.id]).flat(); + let framesMeta = jobsMeta.map((jobMeta) => jobMeta.frames); + if (task.mode === 'interpolation') { + framesMeta = [framesMeta[0]]; + } + + return { + chunk_size: jobsMeta[0].chunk_size , + size: task.size, + image_quality: task.image_quality, + start_frame: task.start_frame, + stop_frame: task.stop_frame, + frames: framesMeta, + deleted_frames: [], + included_frames: [], + }; } - return JSON.parse(JSON.stringify(frameMetaDummyData[jid])); + return JSON.parse(JSON.stringify(frameMetaDummyData[id])); } async function saveMeta(session, jid, meta) { diff --git a/cvat-data/package.json b/cvat-data/package.json index 63c1be63d625..375a1f23cee1 100644 --- a/cvat-data/package.json +++ b/cvat-data/package.json @@ -1,6 +1,6 @@ { "name": "cvat-data", - "version": "1.1.0", + "version": "2.0.0", "description": "", "main": "src/ts/cvat-data.ts", "scripts": { diff --git a/cvat-data/src/ts/cvat-data.ts b/cvat-data/src/ts/cvat-data.ts index b60d0a82384d..8d50fe64083d 100644 --- a/cvat-data/src/ts/cvat-data.ts +++ b/cvat-data/src/ts/cvat-data.ts @@ -8,35 +8,43 @@ import { MP4Reader, Bytestream } from './3rdparty/mp4'; import ZipDecoder from './unzip_imgs.worker'; import H264Decoder from './3rdparty/Decoder.worker'; +export class RequestOutdatedError extends Error {} + export enum BlockType { MP4VIDEO = 'mp4video', ARCHIVE = 'archive', } +export enum ChunkQuality { + ORIGINAL = 'original', + COMPRESSED = 'compressed', +} + export enum DimensionType { DIMENSION_3D = '3d', DIMENSION_2D = '2d', } -export function decodeZip( - block: any, start: number, end: number, dimension: any, +export function decodeContextImages( + block: any, start: number, end: number, ): Promise> { + const decodeZipWorker = ((decodeContextImages as any).zipWorker || new (ZipDecoder as any)()) as Worker; + (decodeContextImages as any).zipWorker = decodeZipWorker; return new Promise((resolve, reject) => { - decodeZip.mutex.acquire().then((release) => { - const worker = new ZipDecoder(); + decodeContextImages.mutex.acquire().then((release) => { const result: Record = {}; let decoded = 0; - worker.onerror = (e: ErrorEvent) => { + decodeZipWorker.onerror = (e: ErrorEvent) => { release(); - worker.terminate(); reject(new Error(`Archive can not be decoded. ${e.message}`)); }; - worker.onmessage = async (event) => { + decodeZipWorker.onmessage = async (event) => { const { error, fileName } = event.data; if (error) { - worker.onerror(new ErrorEvent('error', { message: error.toString() })); + decodeZipWorker.onerror(new ErrorEvent('error', { message: error.toString() })); + return; } const { data } = event.data; @@ -45,175 +53,136 @@ export function decodeZip( if (decoded === end) { release(); - worker.terminate(); resolve(result); } }; - worker.postMessage({ + decodeZipWorker.postMessage({ block, start, end, - dimension, + dimension: DimensionType.DIMENSION_2D, dimension2D: DimensionType.DIMENSION_2D, }); }); }); } -decodeZip.mutex = new Mutex(); +decodeContextImages.mutex = new Mutex(); interface BlockToDecode { start: number; end: number; block: ArrayBuffer; - resolveCallback: (frame: number) => void; - rejectCallback: (e: ErrorEvent) => void; + onDecodeAll(): void; + onDecode(frame: number, bitmap: ImageBitmap | Blob): void; + onReject(e: Error): void; } -export class FrameProvider { - private blocksRanges: string[]; - private blockSize: number; +export class FrameDecoder { private blockType: BlockType; - + private chunkSize: number; /* - ImageBitmap when decode zip chunks - ImageData when decode video chunks + ImageBitmap when decode zip or video chunks Blob when 3D dimension null when not decoded yet */ - private frames: Record; - private requestedBlockToDecode: null | BlockToDecode; - private blocksAreBeingDecoded: Record; - private promisedFrames: Record void; - reject: () => void; - }>; - private currentDecodingThreads: number; - private currentFrame: number; + private decodedChunks: Record>; + private chunkIsBeingDecoded: BlockToDecode | null; + private requestedChunkToDecode: BlockToDecode | null; + private orderedStack: number[]; private mutex: Mutex; - private dimension: DimensionType; - private workerThreadsLimit: number; - private cachedEncodedBlocksLimit: number; - private cachedDecodedBlocksLimit: number; - - // used for video chunks to resize after decoding + private cachedChunksLimit: number; + // used for video chunks to get correct side after decoding private renderWidth: number; private renderHeight: number; + private zipWorker: Worker; constructor( blockType: BlockType, - blockSize: number, + chunkSize: number, cachedBlockCount: number, - decodedBlocksCacheSize = 5, - maxWorkerThreadCount = 2, dimension: DimensionType = DimensionType.DIMENSION_2D, ) { this.mutex = new Mutex(); - this.blocksRanges = []; - this.frames = {}; - this.promisedFrames = {}; - this.currentDecodingThreads = 0; - this.currentFrame = -1; - - this.cachedEncodedBlocksLimit = Math.max(1, cachedBlockCount); // number of stored blocks - this.cachedDecodedBlocksLimit = decodedBlocksCacheSize; - this.workerThreadsLimit = maxWorkerThreadCount; + this.orderedStack = []; + + this.cachedChunksLimit = Math.max(1, cachedBlockCount); this.dimension = dimension; this.renderWidth = 1920; this.renderHeight = 1080; - this.blockSize = blockSize; + this.chunkSize = chunkSize; this.blockType = blockType; - // todo: sort out with logic of blocks - this._blocks = {}; - this.requestedBlockToDecode = null; - this.blocksAreBeingDecoded = {}; - - setTimeout(this._checkDecodeRequests.bind(this), 100); + this.decodedChunks = {}; + this.requestedChunkToDecode = null; + this.chunkIsBeingDecoded = null; } - _checkDecodeRequests(): void { - if (this.requestedBlockToDecode !== null && this.currentDecodingThreads < this.workerThreadsLimit) { - this.startDecode().then(() => { - setTimeout(this._checkDecodeRequests.bind(this), 100); - }); - } else { - setTimeout(this._checkDecodeRequests.bind(this), 100); - } + isChunkCached(chunkNumber: number): boolean { + return chunkNumber in this.decodedChunks; } - isChunkCached(start: number, end: number): boolean { - // todo: always returns false because this.blocksRanges is Array, not dictionary - // but if try to correct other errors happens, need to debug.. - return `${start}:${end}` in this.blocksRanges; + hasFreeSpace(): boolean { + return Object.keys(this.decodedChunks).length < this.cachedChunksLimit; } - /* This method removes extra data from a cache when memory overflow */ - async _cleanup(): Promise { - if (this.blocksRanges.length > this.cachedEncodedBlocksLimit) { - const shifted = this.blocksRanges.shift(); // get the oldest block - const [start, end] = shifted.split(':').map((el) => +el); - delete this._blocks[Math.floor(start / this.blockSize)]; - for (let i = start; i <= end; i++) { - delete this.frames[i]; - } - } - - // delete frames whose are not in areas of current frame - const distance = Math.floor(this.cachedDecodedBlocksLimit / 2); - for (let i = 0; i < this.blocksRanges.length; i++) { - const [start, end] = this.blocksRanges[i].split(':').map((el) => +el); - if ( - end < this.currentFrame - distance * this.blockSize || - start > this.currentFrame + distance * this.blockSize - ) { - for (let j = start; j <= end; j++) { - delete this.frames[j]; - } + cleanup(extra = 1): void { + // argument allows us to specify how many chunks we want to write after clear + const chunks = Object.keys(this.decodedChunks).map((chunk: string) => +chunk); + let { length } = chunks; + while (length > this.cachedChunksLimit - Math.min(extra, this.cachedChunksLimit)) { + const lastChunk = this.orderedStack.pop(); + if (typeof lastChunk === 'undefined') { + return; } + delete this.decodedChunks[lastChunk]; + length--; } } - async requestDecodeBlock( + requestDecodeBlock( block: ArrayBuffer, start: number, end: number, - resolveCallback: () => void, - rejectCallback: () => void, - ): Promise { - const release = await this.mutex.acquire(); - try { - if (this.requestedBlockToDecode !== null) { - if (start === this.requestedBlockToDecode.start && end === this.requestedBlockToDecode.end) { - // only rewrite callbacks if the same block was requested again - this.requestedBlockToDecode.resolveCallback = resolveCallback; - this.requestedBlockToDecode.rejectCallback = rejectCallback; - - // todo: should we reject the previous request here? - } else if (this.requestedBlockToDecode.rejectCallback) { - // if another block requested, the previous request should be rejected - this.requestedBlockToDecode.rejectCallback(); - } + onDecode: (frame: number, bitmap: ImageBitmap | Blob) => void, + onDecodeAll: () => void, + onReject: (e: Error) => void, + ): void { + if (this.requestedChunkToDecode !== null) { + // a chunk was already requested to be decoded, but decoding didn't start yet + if (start === this.requestedChunkToDecode.start && end === this.requestedChunkToDecode.end) { + // it was the same chunk + this.requestedChunkToDecode.onReject(new RequestOutdatedError()); + + this.requestedChunkToDecode.onDecode = onDecode; + this.requestedChunkToDecode.onReject = onReject; + } else if (this.requestedChunkToDecode.onReject) { + // it was other chunk + this.requestedChunkToDecode.onReject(new RequestOutdatedError()); } + } else if (this.chunkIsBeingDecoded === null || this.chunkIsBeingDecoded.start !== start) { + // everything was decoded or decoding other chunk is in process + this.requestedChunkToDecode = { + block, + start, + end, + onDecode, + onDecodeAll, + onReject, + }; + } else { + // the same chunk is being decoded right now + // reject previous decoding request + this.chunkIsBeingDecoded.onReject(new RequestOutdatedError()); - if (!(`${start}:${end}` in this.blocksAreBeingDecoded)) { - this.requestedBlockToDecode = { - block: block || this._blocks[Math.floor(start / this.blockSize)], - start, - end, - resolveCallback, - rejectCallback, - }; - } else { - this.blocksAreBeingDecoded[`${start}:${end}`].rejectCallback = rejectCallback; - this.blocksAreBeingDecoded[`${start}:${end}`].resolveCallback = resolveCallback; - } - } finally { - release(); + this.chunkIsBeingDecoded.onReject = onReject; + this.chunkIsBeingDecoded.onDecode = onDecode; } + + this.startDecode(); } setRenderSize(width: number, height: number): void { @@ -221,85 +190,37 @@ export class FrameProvider { this.renderHeight = height; } - /* Method returns frame from collection. Else method returns null */ - async frame(frameNumber: number): Promise { - this.currentFrame = frameNumber; - return new Promise((resolve, reject) => { - if (frameNumber in this.frames) { - if (this.frames[frameNumber] !== null) { - resolve(this.frames[frameNumber]); - } else { - this.promisedFrames[frameNumber] = { resolve, reject }; - } - } else { - resolve(null); - } - }); - } - - isNextChunkExists(frameNumber: number): boolean { - const nextChunkNum = Math.floor(frameNumber / this.blockSize) + 1; - return nextChunkNum in this._blocks; - } - - setReadyToLoading(chunkNumber: number): void { - this._blocks[chunkNumber] = 'loading'; - } - - static cropImage( - imageBuffer: ArrayBuffer, - imageWidth: number, - imageHeight: number, - xOffset: number, - yOffset: number, - width: number, - height: number, - ): ImageData { - if (xOffset === 0 && width === imageWidth && yOffset === 0 && height === imageHeight) { - return new ImageData(new Uint8ClampedArray(imageBuffer), width, height); - } - const source = new Uint32Array(imageBuffer); - - const bufferSize = width * height * 4; - const buffer = new ArrayBuffer(bufferSize); - const rgbaInt32 = new Uint32Array(buffer); - const rgbaInt8Clamped = new Uint8ClampedArray(buffer); - - if (imageWidth === width) { - return new ImageData(new Uint8ClampedArray(imageBuffer, yOffset * 4, bufferSize), width, height); + frame(frameNumber: number): ImageBitmap | Blob | null { + const chunkNumber = Math.floor(frameNumber / this.chunkSize); + if (chunkNumber in this.decodedChunks) { + return this.decodedChunks[chunkNumber][frameNumber]; } - let writeIdx = 0; - for (let row = yOffset; row < height; row++) { - const start = row * imageWidth + xOffset; - rgbaInt32.set(source.subarray(start, start + width), writeIdx); - writeIdx += width; - } - - return new ImageData(rgbaInt8Clamped, width, height); + return null; } async startDecode(): Promise { + const blockToDecode = { ...this.requestedChunkToDecode }; const release = await this.mutex.acquire(); try { - const height = this.renderHeight; - const width = this.renderWidth; - const { start, end, block } = this.requestedBlockToDecode; - - this.blocksRanges.push(`${start}:${end}`); - this.blocksAreBeingDecoded[`${start}:${end}`] = this.requestedBlockToDecode; - this.requestedBlockToDecode = null; - this._blocks[Math.floor((start + 1) / this.blockSize)] = block; - - for (let i = start; i <= end; i++) { - this.frames[i] = null; + const { start, end, block } = this.requestedChunkToDecode; + if (start !== blockToDecode.start) { + // request is not relevant, another block was already requested + // it happens when A is being decoded, B comes and wait for mutex, C comes and wait for mutex + // B is not necessary anymore, because C already was requested + blockToDecode.onReject(new RequestOutdatedError()); + throw new RequestOutdatedError(); } - this._cleanup(); - this.currentDecodingThreads++; + const chunkNumber = Math.floor(start / this.chunkSize); + this.orderedStack = [chunkNumber, ...this.orderedStack]; + this.cleanup(); + const decodedFrames: Record = {}; + this.chunkIsBeingDecoded = this.requestedChunkToDecode; + this.requestedChunkToDecode = null; if (this.blockType === BlockType.MP4VIDEO) { - const worker = new H264Decoder(); + const worker = new H264Decoder() as any as Worker; let index = start; worker.onmessage = (e) => { @@ -307,56 +228,36 @@ export class FrameProvider { // ignore initialization message return; } - - const scaleFactor = Math.ceil(height / e.data.height); - this.frames[index] = FrameProvider.cropImage( - e.data.buf, - e.data.width, - e.data.height, - 0, - 0, - Math.floor(width / scaleFactor), - Math.floor(height / scaleFactor), - ); - - const { resolveCallback } = this.blocksAreBeingDecoded[`${start}:${end}`]; - if (resolveCallback) { - resolveCallback(index); - } - - if (index in this.promisedFrames) { - const { resolve } = this.promisedFrames[index]; - delete this.promisedFrames[index]; - resolve(this.frames[index]); - } - - if (index === end) { - worker.terminate(); - this.currentDecodingThreads--; - delete this.blocksAreBeingDecoded[`${start}:${end}`]; - } + const keptIndex = index; + + // do not use e.data.height and e.data.width because they might be not correct + // instead, try to understand real height and width of decoded image via scale factor + const scaleFactor = Math.ceil(this.renderHeight / e.data.height); + const height = Math.round(this.renderHeight / scaleFactor); + const width = Math.round(this.renderWidth / scaleFactor); + + const array = new Uint8ClampedArray(e.data.buf.slice(0, width * height * 4)); + createImageBitmap(new ImageData(array, width)).then((bitmap) => { + decodedFrames[keptIndex] = bitmap; + this.chunkIsBeingDecoded.onDecode(keptIndex, decodedFrames[keptIndex]); + + if (keptIndex === end) { + this.decodedChunks[chunkNumber] = decodedFrames; + this.chunkIsBeingDecoded.onDecodeAll(); + this.chunkIsBeingDecoded = null; + worker.terminate(); + release(); + } + }); index++; }; - worker.onerror = (e: ErrorEvent) => { + worker.onerror = () => { + release(); worker.terminate(); - this.currentDecodingThreads--; - - for (let i = index; i <= end; i++) { - // reject all the following frames - if (i in this.promisedFrames) { - const { reject } = this.promisedFrames[i]; - delete this.promisedFrames[i]; - reject(); - } - } - - if (this.blocksAreBeingDecoded[`${start}:${end}`].rejectCallback) { - this.blocksAreBeingDecoded[`${start}:${end}`].rejectCallback(e); - } - - delete this.blocksAreBeingDecoded[`${start}:${end}`]; + this.chunkIsBeingDecoded.onReject(new Error('Error occured during decode')); + this.chunkIsBeingDecoded = null; }; worker.postMessage({ @@ -375,58 +276,44 @@ export class FrameProvider { const sps = avc.sps[0]; const pps = avc.pps[0]; - /* Decode Sequence & Picture Parameter Sets */ worker.postMessage({ buf: sps, offset: 0, length: sps.length }); worker.postMessage({ buf: pps, offset: 0, length: pps.length }); - /* Decode Pictures */ for (let sample = 0; sample < video.getSampleCount(); sample++) { video.getSampleNALUnits(sample).forEach((nal) => { worker.postMessage({ buf: nal, offset: 0, length: nal.length }); }); } } else { - const worker = new ZipDecoder(); + this.zipWorker = this.zipWorker || new (ZipDecoder as any)() as any as Worker; let index = start; - worker.onmessage = async (event) => { - this.frames[event.data.index] = event.data.data; - - const { resolveCallback } = this.blocksAreBeingDecoded[`${start}:${end}`]; - if (resolveCallback) { - resolveCallback(event.data.index); + this.zipWorker.onmessage = async (event) => { + if (event.data.error) { + this.zipWorker.onerror(new ErrorEvent('error', { message: event.data.error.toString() })); + return; } - if (event.data.index in this.promisedFrames) { - const { resolve } = this.promisedFrames[event.data.index]; - delete this.promisedFrames[event.data.index]; - resolve(this.frames[event.data.index]); - } + decodedFrames[event.data.index] = event.data.data as ImageBitmap | Blob; + this.chunkIsBeingDecoded.onDecode(event.data.index, decodedFrames[event.data.index]); if (index === end) { - worker.terminate(); - this.currentDecodingThreads--; - delete this.blocksAreBeingDecoded[`${start}:${end}`]; + this.decodedChunks[chunkNumber] = decodedFrames; + this.chunkIsBeingDecoded.onDecodeAll(); + this.chunkIsBeingDecoded = null; + release(); } index++; }; - worker.onerror = (e: ErrorEvent) => { - for (let i = start; i <= end; i++) { - if (i in this.promisedFrames) { - const { reject } = this.promisedFrames[i]; - delete this.promisedFrames[i]; - reject(); - } - } - if (this.blocksAreBeingDecoded[`${start}:${end}`].rejectCallback) { - this.blocksAreBeingDecoded[`${start}:${end}`].rejectCallback(e); - } - this.currentDecodingThreads--; - worker.terminate(); + this.zipWorker.onerror = () => { + release(); + + this.chunkIsBeingDecoded.onReject(new Error('Error occured during decode')); + this.chunkIsBeingDecoded = null; }; - worker.postMessage({ + this.zipWorker.postMessage({ block, start, end, @@ -434,20 +321,17 @@ export class FrameProvider { dimension2D: DimensionType.DIMENSION_2D, }); } - } finally { + } catch (error) { + this.chunkIsBeingDecoded = null; release(); } } - get decodedBlocksCacheSize(): number { - return this.cachedDecodedBlocksLimit; - } - - /* - Method returns a list of cached ranges - Is an array of strings like "start:end" - */ - get cachedFrames(): string[] { - return [...this.blocksRanges].sort((a, b) => +a.split(':')[0] - +b.split(':')[0]); + public cachedChunks(includeInProgress = false): number[] { + const chunkIsBeingDecoded = includeInProgress && this.chunkIsBeingDecoded ? + Math.floor(this.chunkIsBeingDecoded.start / this.chunkSize) : null; + return Object.keys(this.decodedChunks).map((chunkNumber: string) => +chunkNumber).concat( + ...(chunkIsBeingDecoded !== null ? [chunkIsBeingDecoded] : []), + ).sort((a, b) => a - b); } } diff --git a/cvat-sdk/README.md b/cvat-sdk/README.md index 8fa7b4379741..cf5732aaeba7 100644 --- a/cvat-sdk/README.md +++ b/cvat-sdk/README.md @@ -8,6 +8,7 @@ The SDK API includes several layers: - Server API wrappers (`ApiClient`). Located in at `cvat_sdk.api_client`. - High-level tools (`Core`). Located at `cvat_sdk.core`. - PyTorch adapter. Located at `cvat_sdk.pytorch`. +* Auto-annotation support. Located at `cvat_sdk.auto_annotation`. Package documentation is available [here](https://opencv.github.io/cvat/docs/api_sdk/sdk). diff --git a/cvat-sdk/cvat_sdk/auto_annotation/__init__.py b/cvat-sdk/cvat_sdk/auto_annotation/__init__.py new file mode 100644 index 000000000000..e5dbdf9fcc42 --- /dev/null +++ b/cvat-sdk/cvat_sdk/auto_annotation/__init__.py @@ -0,0 +1,17 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from .driver import BadFunctionError, annotate_task +from .interface import ( + DetectionFunction, + DetectionFunctionContext, + DetectionFunctionSpec, + keypoint, + keypoint_spec, + label_spec, + rectangle, + shape, + skeleton, + skeleton_label_spec, +) diff --git a/cvat-sdk/cvat_sdk/auto_annotation/driver.py b/cvat-sdk/cvat_sdk/auto_annotation/driver.py new file mode 100644 index 000000000000..8c1c71b46e8b --- /dev/null +++ b/cvat-sdk/cvat_sdk/auto_annotation/driver.py @@ -0,0 +1,303 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import logging +from typing import List, Mapping, Optional, Sequence + +import attrs + +import cvat_sdk.models as models +from cvat_sdk.core import Client +from cvat_sdk.core.progress import NullProgressReporter, ProgressReporter +from cvat_sdk.datasets.task_dataset import TaskDataset + +from .interface import DetectionFunction, DetectionFunctionContext, DetectionFunctionSpec + + +class BadFunctionError(Exception): + """ + An exception that signifies that an auto-detection function has violated some constraint + set by its interface. + """ + + +class _AnnotationMapper: + @attrs.frozen + class _MappedLabel: + id: int + sublabel_mapping: Mapping[int, Optional[int]] + expected_num_elements: int = 0 + + _label_mapping: Mapping[int, Optional[_MappedLabel]] + + def _build_mapped_label( + self, fun_label: models.ILabel, ds_labels_by_name: Mapping[str, models.ILabel] + ) -> Optional[_MappedLabel]: + if getattr(fun_label, "attributes", None): + raise BadFunctionError(f"label attributes are currently not supported") + + ds_label = ds_labels_by_name.get(fun_label.name) + if ds_label is None: + if not self._allow_unmatched_labels: + raise BadFunctionError(f"label {fun_label.name!r} is not in dataset") + + self._logger.info( + "label %r is not in dataset; any annotations using it will be ignored", + fun_label.name, + ) + return None + + sl_map = {} + + if getattr(fun_label, "sublabels", []): + fun_label_type = getattr(fun_label, "type", "any") + if fun_label_type != "skeleton": + raise BadFunctionError( + f"label {fun_label.name!r} with sublabels has type {fun_label_type!r} (should be 'skeleton')" + ) + + ds_sublabels_by_name = {ds_sl.name: ds_sl for ds_sl in ds_label.sublabels} + + for fun_sl in fun_label.sublabels: + if not hasattr(fun_sl, "id"): + raise BadFunctionError( + f"sublabel {fun_sl.name!r} of label {fun_label.name!r} has no ID" + ) + + if fun_sl.id in sl_map: + raise BadFunctionError( + f"sublabel {fun_sl.name!r} of label {fun_label.name!r} has same ID as another sublabel ({fun_sl.id})" + ) + + ds_sl = ds_sublabels_by_name.get(fun_sl.name) + if not ds_sl: + if not self._allow_unmatched_labels: + raise BadFunctionError( + f"sublabel {fun_sl.name!r} of label {fun_label.name!r} is not in dataset" + ) + + self._logger.info( + "sublabel %r of label %r is not in dataset; any annotations using it will be ignored", + fun_sl.name, + fun_label.name, + ) + sl_map[fun_sl.id] = None + continue + + sl_map[fun_sl.id] = ds_sl.id + + return self._MappedLabel( + ds_label.id, sublabel_mapping=sl_map, expected_num_elements=len(ds_label.sublabels) + ) + + def __init__( + self, + logger: logging.Logger, + fun_labels: Sequence[models.ILabel], + ds_labels: Sequence[models.ILabel], + *, + allow_unmatched_labels: bool, + ) -> None: + self._logger = logger + self._allow_unmatched_labels = allow_unmatched_labels + + ds_labels_by_name = {ds_label.name: ds_label for ds_label in ds_labels} + + self._label_mapping = {} + + for fun_label in fun_labels: + if not hasattr(fun_label, "id"): + raise BadFunctionError(f"label {fun_label.name!r} has no ID") + + if fun_label.id in self._label_mapping: + raise BadFunctionError( + f"label {fun_label.name} has same ID as another label ({fun_label.id})" + ) + + self._label_mapping[fun_label.id] = self._build_mapped_label( + fun_label, ds_labels_by_name + ) + + def validate_and_remap(self, shapes: List[models.LabeledShapeRequest], ds_frame: int) -> None: + new_shapes = [] + + for shape in shapes: + if hasattr(shape, "id"): + raise BadFunctionError("function output shape with preset id") + + if hasattr(shape, "source"): + raise BadFunctionError("function output shape with preset source") + shape.source = "auto" + + if shape.frame != 0: + raise BadFunctionError( + f"function output shape with unexpected frame number ({shape.frame})" + ) + + shape.frame = ds_frame + + try: + mapped_label = self._label_mapping[shape.label_id] + except KeyError: + raise BadFunctionError( + f"function output shape with unknown label ID ({shape.label_id})" + ) + + if not mapped_label: + continue + + shape.label_id = mapped_label.id + + if getattr(shape, "attributes", None): + raise BadFunctionError( + "function output shape with attributes, which is not yet supported" + ) + + new_shapes.append(shape) + + if shape.type.value == "skeleton": + new_elements = [] + seen_sl_ids = set() + + for element in shape.elements: + if hasattr(element, "id"): + raise BadFunctionError("function output shape element with preset id") + + if hasattr(element, "source"): + raise BadFunctionError("function output shape element with preset source") + element.source = "auto" + + if element.frame != 0: + raise BadFunctionError( + f"function output shape element with unexpected frame number ({element.frame})" + ) + + element.frame = ds_frame + + if element.type.value != "points": + raise BadFunctionError( + f"function output skeleton with element type other than 'points' ({element.type.value})" + ) + + try: + mapped_sl_id = mapped_label.sublabel_mapping[element.label_id] + except KeyError: + raise BadFunctionError( + f"function output shape with unknown sublabel ID ({element.label_id})" + ) + + if not mapped_sl_id: + continue + + if mapped_sl_id in seen_sl_ids: + raise BadFunctionError( + "function output skeleton with multiple elements with same sublabel" + ) + + element.label_id = mapped_sl_id + + seen_sl_ids.add(mapped_sl_id) + + new_elements.append(element) + + if len(new_elements) != mapped_label.expected_num_elements: + # new_elements could only be shorter than expected, + # because the reverse would imply that there are more distinct sublabel IDs + # than are actually defined in the dataset. + assert len(new_elements) < mapped_label.expected_num_elements + + raise BadFunctionError( + f"function output skeleton with fewer elements than expected ({len(new_elements)} vs {mapped_label.expected_num_elements})" + ) + + shape.elements[:] = new_elements + else: + if getattr(shape, "elements", None): + raise BadFunctionError("function output non-skeleton shape with elements") + + shapes[:] = new_shapes + + +@attrs.frozen +class _DetectionFunctionContextImpl(DetectionFunctionContext): + frame_name: str + + +def annotate_task( + client: Client, + task_id: int, + function: DetectionFunction, + *, + pbar: Optional[ProgressReporter] = None, + clear_existing: bool = False, + allow_unmatched_labels: bool = False, +) -> None: + """ + Downloads data for the task with the given ID, applies the given function to it + and uploads the resulting annotations back to the task. + + Only tasks with 2D image (not video) data are supported at the moment. + + client is used to make all requests to the CVAT server. + + Currently, the only type of auto-annotation function supported is the detection function. + A function of this type is applied independently to each image in the task. + The resulting annotations are then combined and modified as follows: + + * The label IDs are replaced with the IDs of the corresponding labels in the task. + * The frame numbers are replaced with the frame number of the image. + * The sources are set to "auto". + + See the documentation for DetectionFunction for more details. + + If the function is found to violate any constraints set in its interface, BadFunctionError + is raised. + + pbar, if supplied, is used to report progress information. + + If clear_existing is true, any annotations already existing in the tesk are removed. + Otherwise, they are kept, and the new annotations are added to them. + + The allow_unmatched_labels parameter controls the behavior in the case when a detection + function declares a label in its spec that has no corresponding label in the task. + If it's set to true, then such labels are allowed, and any annotations returned by the + function that refer to this label are ignored. Otherwise, BadFunctionError is raised. + """ + + if pbar is None: + pbar = NullProgressReporter() + + dataset = TaskDataset(client, task_id) + + assert isinstance(function.spec, DetectionFunctionSpec) + + mapper = _AnnotationMapper( + client.logger, + function.spec.labels, + dataset.labels, + allow_unmatched_labels=allow_unmatched_labels, + ) + + shapes = [] + + with pbar.task(total=len(dataset.samples), unit="samples"): + for sample in pbar.iter(dataset.samples): + frame_shapes = function.detect( + _DetectionFunctionContextImpl(sample.frame_name), sample.media.load_image() + ) + mapper.validate_and_remap(frame_shapes, sample.frame_index) + shapes.extend(frame_shapes) + + client.logger.info("Uploading annotations to task %d", task_id) + + if clear_existing: + client.tasks.api.update_annotations( + task_id, task_annotations_update_request=models.LabeledDataRequest(shapes=shapes) + ) + else: + client.tasks.api.partial_update_annotations( + "create", + task_id, + patched_labeled_data_request=models.PatchedLabeledDataRequest(shapes=shapes), + ) diff --git a/cvat-sdk/cvat_sdk/auto_annotation/functions/__init__.py b/cvat-sdk/cvat_sdk/auto_annotation/functions/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cvat-sdk/cvat_sdk/auto_annotation/functions/torchvision_detection.py b/cvat-sdk/cvat_sdk/auto_annotation/functions/torchvision_detection.py new file mode 100644 index 000000000000..57457d742256 --- /dev/null +++ b/cvat-sdk/cvat_sdk/auto_annotation/functions/torchvision_detection.py @@ -0,0 +1,41 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from functools import cached_property +from typing import List + +import PIL.Image +import torchvision.models + +import cvat_sdk.auto_annotation as cvataa +import cvat_sdk.models as models + + +class _TorchvisionDetectionFunction: + def __init__(self, model_name: str, weights_name: str = "DEFAULT", **kwargs) -> None: + weights_enum = torchvision.models.get_model_weights(model_name) + self._weights = weights_enum[weights_name] + self._transforms = self._weights.transforms() + self._model = torchvision.models.get_model(model_name, weights=self._weights, **kwargs) + self._model.eval() + + @cached_property + def spec(self) -> cvataa.DetectionFunctionSpec: + return cvataa.DetectionFunctionSpec( + labels=[ + cvataa.label_spec(cat, i) for i, cat in enumerate(self._weights.meta["categories"]) + ] + ) + + def detect(self, context, image: PIL.Image.Image) -> List[models.LabeledShapeRequest]: + results = self._model([self._transforms(image)]) + + return [ + cvataa.rectangle(label.item(), [x.item() for x in box]) + for result in results + for box, label in zip(result["boxes"], result["labels"]) + ] + + +create = _TorchvisionDetectionFunction diff --git a/cvat-sdk/cvat_sdk/auto_annotation/functions/torchvision_keypoint_detection.py b/cvat-sdk/cvat_sdk/auto_annotation/functions/torchvision_keypoint_detection.py new file mode 100644 index 000000000000..b4eb47d476d3 --- /dev/null +++ b/cvat-sdk/cvat_sdk/auto_annotation/functions/torchvision_keypoint_detection.py @@ -0,0 +1,59 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from functools import cached_property +from typing import List + +import PIL.Image +import torchvision.models + +import cvat_sdk.auto_annotation as cvataa +import cvat_sdk.models as models + + +class _TorchvisionKeypointDetectionFunction: + def __init__(self, model_name: str, weights_name: str = "DEFAULT", **kwargs) -> None: + weights_enum = torchvision.models.get_model_weights(model_name) + self._weights = weights_enum[weights_name] + self._transforms = self._weights.transforms() + self._model = torchvision.models.get_model(model_name, weights=self._weights, **kwargs) + self._model.eval() + + @cached_property + def spec(self) -> cvataa.DetectionFunctionSpec: + return cvataa.DetectionFunctionSpec( + labels=[ + cvataa.skeleton_label_spec( + cat, + i, + [ + cvataa.keypoint_spec(name, j) + for j, name in enumerate(self._weights.meta["keypoint_names"]) + ], + ) + for i, cat in enumerate(self._weights.meta["categories"]) + ] + ) + + def detect(self, context, image: PIL.Image.Image) -> List[models.LabeledShapeRequest]: + results = self._model([self._transforms(image)]) + + return [ + cvataa.skeleton( + label.item(), + elements=[ + cvataa.keypoint( + keypoint_id, + [keypoint[0].item(), keypoint[1].item()], + occluded=not keypoint[2].item(), + ) + for keypoint_id, keypoint in enumerate(keypoints) + ], + ) + for result in results + for keypoints, label in zip(result["keypoints"], result["labels"]) + ] + + +create = _TorchvisionKeypointDetectionFunction diff --git a/cvat-sdk/cvat_sdk/auto_annotation/interface.py b/cvat-sdk/cvat_sdk/auto_annotation/interface.py new file mode 100644 index 000000000000..67313a7da6e5 --- /dev/null +++ b/cvat-sdk/cvat_sdk/auto_annotation/interface.py @@ -0,0 +1,165 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import abc +from typing import List, Protocol, Sequence + +import attrs +import PIL.Image + +import cvat_sdk.models as models + + +@attrs.frozen(kw_only=True) +class DetectionFunctionSpec: + """ + Static information about an auto-annotation detection function. + """ + + labels: Sequence[models.PatchedLabelRequest] + """ + Information about labels that the function supports. + + The members of the sequence must follow the same constraints as if they were being + used to create a CVAT project, and the following additional constraints: + + * The id attribute must be set to a distinct integer. + + * The id attribute of any sublabels must be set to an integer, distinct between all + sublabels of the same parent label. + + * There must not be any attributes (attribute support may be added in a future version). + + It's recommented to use the helper factory functions (label_spec, skeleton_label_spec, + keypoint_spec) to create the label objects, as they are more concise than the model + constructors and help to follow some of the constraints. + """ + + +class DetectionFunctionContext(metaclass=abc.ABCMeta): + """ + Information that is supplied to an auto-annotation detection function. + """ + + @property + @abc.abstractmethod + def frame_name(self) -> str: + """ + The file name of the frame that the current image corresponds to in + the dataset. + """ + ... + + +class DetectionFunction(Protocol): + """ + The interface that an auto-annotation detection function must implement. + + A detection function is supposed to accept an image and return a list of shapes + describing objects in that image. + + Since the same function could be used with multiple datasets, it needs some way + to refer to labels without using dataset-specific label IDs. The way this is + accomplished is that the function declares its own labels via the spec attribute, + and then refers to those labels in the returned annotations. The caller then matches + up the labels from the function's spec with the labels in the actual dataset, and + replaces the label IDs in the returned annotations with IDs of the corresponding + labels in the dataset. + + The matching of labels between the function and the dataset is done by name. + Therefore, a function can be used with a dataset if they have (at least some) labels + that have the same name. + """ + + @property + def spec(self) -> DetectionFunctionSpec: + """Returns the function's spec.""" + ... + + def detect( + self, context: DetectionFunctionContext, image: PIL.Image.Image + ) -> List[models.LabeledShapeRequest]: + """ + Detects objects on the supplied image and returns the results. + + The supplied context will contain information about the current image. + + The returned LabeledShapeRequest objects must follow general constraints + imposed by the data model (such as the number of points in a shape), + as well as the following additional constraints: + + * The id attribute must not be set. + + * The source attribute must not be set. + + * The frame_id attribute must be set to 0. + + * The label_id attribute must equal one of the label IDs + in the function spec. + + * There must not be any attributes (attribute support may be added in a + future version). + + * The above constraints also apply to each sub-shape (element of a shape), + except that the label_id of a sub-shape must equal one of the sublabel IDs + of the label of its parent shape. + + It's recommented to use the helper factory functions (shape, rectangle, skeleton, + keypoint) to create the shape objects, as they are more concise than the model + constructors and help to follow some of the constraints. + + The function must not retain any references to the returned objects, + so that the caller may freely modify them. + """ + ... + + +# spec factories + + +# pylint: disable-next=redefined-builtin +def label_spec(name: str, id: int, **kwargs) -> models.PatchedLabelRequest: + """Helper factory function for PatchedLabelRequest.""" + return models.PatchedLabelRequest(name=name, id=id, **kwargs) + + +# pylint: disable-next=redefined-builtin +def skeleton_label_spec( + name: str, id: int, sublabels: Sequence[models.SublabelRequest], **kwargs +) -> models.PatchedLabelRequest: + """Helper factory function for PatchedLabelRequest with type="skeleton".""" + return models.PatchedLabelRequest(name=name, id=id, type="skeleton", sublabels=sublabels) + + +# pylint: disable-next=redefined-builtin +def keypoint_spec(name: str, id: int, **kwargs) -> models.SublabelRequest: + """Helper factory function for SublabelRequest.""" + return models.SublabelRequest(name=name, id=id, **kwargs) + + +# annotation factories + + +def shape(label_id: int, **kwargs) -> models.LabeledShapeRequest: + """Helper factory function for LabeledShapeRequest with frame=0.""" + return models.LabeledShapeRequest(label_id=label_id, frame=0, **kwargs) + + +def rectangle(label_id: int, points: Sequence[float], **kwargs) -> models.LabeledShapeRequest: + """Helper factory function for LabeledShapeRequest with frame=0 and type="rectangle".""" + return shape(label_id, type="rectangle", points=points, **kwargs) + + +def skeleton( + label_id: int, elements: Sequence[models.SubLabeledShapeRequest], **kwargs +) -> models.LabeledShapeRequest: + """Helper factory function for LabeledShapeRequest with frame=0 and type="skeleton".""" + return shape(label_id, type="skeleton", elements=elements, **kwargs) + + +def keypoint(label_id: int, points: Sequence[float], **kwargs) -> models.SubLabeledShapeRequest: + """Helper factory function for SubLabeledShapeRequest with frame=0 and type="points".""" + return models.SubLabeledShapeRequest( + label_id=label_id, frame=0, type="points", points=points, **kwargs + ) diff --git a/cvat-sdk/cvat_sdk/core/downloading.py b/cvat-sdk/cvat_sdk/core/downloading.py index 3dc338f1468a..fdde84304385 100644 --- a/cvat-sdk/cvat_sdk/core/downloading.py +++ b/cvat-sdk/cvat_sdk/core/downloading.py @@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional from cvat_sdk.api_client.api_client import Endpoint -from cvat_sdk.core.progress import ProgressReporter +from cvat_sdk.core.progress import NullProgressReporter, ProgressReporter from cvat_sdk.core.utils import atomic_writer if TYPE_CHECKING: @@ -41,6 +41,9 @@ def download_file( assert not output_path.exists() + if pbar is None: + pbar = NullProgressReporter() + response = self._client.api_client.rest_client.GET( url, _request_timeout=timeout, @@ -53,18 +56,15 @@ def download_file( except ValueError: file_size = None - with atomic_writer(output_path, "wb") as fd: - if pbar is not None: - pbar.start(file_size, desc="Downloading") - + with atomic_writer(output_path, "wb") as fd, pbar.task( + total=file_size, desc="Downloading", unit_scale=True, unit="B", unit_divisor=1024 + ): while True: chunk = response.read(amt=CHUNK_SIZE, decode_content=False) if not chunk: break - if pbar is not None: - pbar.advance(len(chunk)) - + pbar.advance(len(chunk)) fd.write(chunk) def prepare_and_download_file_from_endpoint( diff --git a/cvat-sdk/cvat_sdk/core/helpers.py b/cvat-sdk/cvat_sdk/core/helpers.py index 36b739bebab5..b04e33e4c687 100644 --- a/cvat-sdk/cvat_sdk/core/helpers.py +++ b/cvat-sdk/cvat_sdk/core/helpers.py @@ -6,6 +6,7 @@ import io import json +import warnings from typing import Any, Dict, Iterable, List, Optional, Union import tqdm @@ -13,7 +14,7 @@ from cvat_sdk import exceptions from cvat_sdk.api_client.api_client import Endpoint -from cvat_sdk.core.progress import ProgressReporter +from cvat_sdk.core.progress import BaseProgressReporter, ProgressReporter def get_paginated_collection( @@ -46,39 +47,83 @@ def get_paginated_collection( return results -class TqdmProgressReporter(ProgressReporter): +class _BaseTqdmProgressReporter(BaseProgressReporter): + tqdm: Optional[tqdm.tqdm] + + def report_status(self, progress: int): + super().report_status(progress) + self.tqdm.update(progress - self.tqdm.n) + + def advance(self, delta: int): + super().advance(delta) + self.tqdm.update(delta) + + +class TqdmProgressReporter(_BaseTqdmProgressReporter): def __init__(self, instance: tqdm.tqdm) -> None: super().__init__() + warnings.warn(f"use {DeferredTqdmProgressReporter.__name__} instead", DeprecationWarning) + self.tqdm = instance - @property - def period(self) -> float: - return 0 + def start2(self, total: int, *, desc: Optional[str] = None, **kwargs) -> None: + super().start2(total=total, desc=desc, **kwargs) - def start(self, total: int, *, desc: Optional[str] = None): self.tqdm.reset(total) self.tqdm.set_description_str(desc) - def report_status(self, progress: int): - self.tqdm.update(progress - self.tqdm.n) + def finish(self): + self.tqdm.refresh() + super().finish() - def advance(self, delta: int): - self.tqdm.update(delta) + +class DeferredTqdmProgressReporter(_BaseTqdmProgressReporter): + def __init__(self, tqdm_args: Optional[dict] = None) -> None: + super().__init__() + self.tqdm_args = tqdm_args or {} + self.tqdm = None + + def start2( + self, + total: int, + *, + desc: Optional[str] = None, + unit: str = "it", + unit_scale: bool = False, + unit_divisor: int = 1000, + **kwargs, + ) -> None: + super().start2( + total=total, + desc=desc, + unit=unit, + unit_scale=unit_scale, + unit_divisor=unit_divisor, + **kwargs, + ) + assert not self.tqdm + + self.tqdm = tqdm.tqdm( + **self.tqdm_args, + total=total, + desc=desc, + unit=unit, + unit_scale=unit_scale, + unit_divisor=unit_divisor, + ) def finish(self): - self.tqdm.refresh() + self.tqdm.close() + self.tqdm = None + super().finish() class StreamWithProgress: - def __init__(self, stream: io.RawIOBase, pbar: ProgressReporter, length: Optional[int] = None): + def __init__(self, stream: io.RawIOBase, pbar: ProgressReporter): self.stream = stream self.pbar = pbar - if hasattr(stream, "__len__"): - length = len(stream) - - self.length = length - pbar.start(length) + assert self.stream.tell() == 0 def read(self, size=-1): chunk = self.stream.read(size) @@ -86,22 +131,15 @@ def read(self, size=-1): self.pbar.advance(len(chunk)) return chunk - def __len__(self): - return self.length + def seek(self, pos: int, whence: int = io.SEEK_SET) -> None: + old_pos = self.stream.tell() + new_pos = self.stream.seek(pos, whence) + self.pbar.advance(new_pos - old_pos) + return new_pos - def seek(self, pos, start=0): - self.stream.seek(pos, start) - self.pbar.report_status(pos) - - def tell(self): + def tell(self) -> int: return self.stream.tell() - def __enter__(self) -> StreamWithProgress: - return self - - def __exit__(self, exc_type, exc_value, exc_traceback) -> None: - self.pbar.finish() - def expect_status(codes: Union[int, Iterable[int]], response: urllib3.HTTPResponse) -> None: if not hasattr(codes, "__iter__"): diff --git a/cvat-sdk/cvat_sdk/core/progress.py b/cvat-sdk/cvat_sdk/core/progress.py index f620e13c50c1..7fd2d13a2cd2 100644 --- a/cvat-sdk/cvat_sdk/core/progress.py +++ b/cvat-sdk/cvat_sdk/core/progress.py @@ -5,36 +5,69 @@ from __future__ import annotations -import math -from typing import Iterable, Optional, Tuple, TypeVar +import contextlib +from typing import ContextManager, Iterable, Optional, TypeVar T = TypeVar("T") class ProgressReporter: """ - Only one set of methods must be called: - - start - report_status / advance - finish - - iter - - split + Use as follows: - This class is supposed to manage the state of children progress bars - and release of their resources, if necessary. + with r.task(...): + r.report_status(...) + r.advance(...) + + for x in r.iter(...): + ... + + Implementations must override start2, finish, report_status and advance. """ - @property - def period(self) -> float: + @contextlib.contextmanager + def task(self, **kwargs) -> ContextManager[None]: """ - Returns reporting period. + Returns a context manager that represents a long-running task + for which progress can be reported. + + Entering it creates a progress bar, and exiting it destroys it. - For example, 0.1 would mean every 10%. + kwargs will be passed to `start()`. """ - raise NotImplementedError + self.start2(**kwargs) + + try: + yield None + finally: + self.finish() - def start(self, total: int, *, desc: Optional[str] = None): - """Initializes the progress bar""" + def start(self, total: int, *, desc: Optional[str] = None) -> None: + """ + This is a compatibility method. Override start2 instead. + """ raise NotImplementedError + def start2( + self, + total: int, + *, + desc: Optional[str] = None, + unit: str = "it", + unit_scale: bool = False, + unit_divisor: int = 1000, + **kwargs, + ) -> None: + """ + Initializes the progress bar. + + total, desc, unit, unit_scale, unit_divisor have the same meaning as in tqdm. + + kwargs is included for future extension; implementations of this method + must ignore it. + """ + self.start(total=total, desc=desc) + def report_status(self, progress: int): """Updates the progress bar""" raise NotImplementedError @@ -50,74 +83,52 @@ def finish(self): def iter( self, iterable: Iterable[T], - *, - total: Optional[int] = None, - desc: Optional[str] = None, ) -> Iterable[T]: """ Traverses the iterable and reports progress simultaneously. - Starts and finishes the progress bar automatically. - Args: iterable: An iterable to be traversed - total: The expected number of iterations. If not provided, will - try to use iterable.__len__. - desc: The status message Returns: An iterable over elements of the input sequence """ - if total is None and hasattr(iterable, "__len__"): - total = len(iterable) - - self.start(total, desc=desc) - - if total: - display_step = math.ceil(total * self.period) - - for i, elem in enumerate(iterable): - if not total or i % display_step == 0: - self.report_status(i) - + for elem in iterable: yield elem - - self.finish() - - def split(self, count: int) -> Tuple[ProgressReporter, ...]: - """ - Splits the progress bar into few independent parts. - In case of 0 must return an empty tuple. - - This class is supposed to manage the state of children progress bars - and release of their resources, if necessary. - """ - raise NotImplementedError + self.advance(1) -class NullProgressReporter(ProgressReporter): - @property - def period(self) -> float: - return 0 +class BaseProgressReporter(ProgressReporter): + def __init__(self) -> None: + self._in_progress = False - def start(self, total: int, *, desc: Optional[str] = None): - pass + def start2( + self, + total: int, + *, + desc: Optional[str] = None, + unit: str = "it", + unit_scale: bool = False, + unit_divisor: int = 1000, + **kwargs, + ) -> None: + assert not self._in_progress + self._in_progress = True def report_status(self, progress: int): - pass + assert self._in_progress def advance(self, delta: int): - pass + assert self._in_progress + + def finish(self) -> None: + assert self._in_progress + self._in_progress = False + + def __del__(self): + assert not self._in_progress, "Unfinished task!" - def iter( - self, - iterable: Iterable[T], - *, - total: Optional[int] = None, - desc: Optional[str] = None, - ) -> Iterable[T]: - yield from iterable - def split(self, count: int) -> Tuple[ProgressReporter]: - return (self,) * count +class NullProgressReporter(BaseProgressReporter): + pass diff --git a/cvat-sdk/cvat_sdk/core/proxies/model_proxy.py b/cvat-sdk/cvat_sdk/core/proxies/model_proxy.py index 9f71fdd93658..9a761771af73 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/model_proxy.py +++ b/cvat-sdk/cvat_sdk/core/proxies/model_proxy.py @@ -13,6 +13,7 @@ Dict, Generic, List, + Literal, Optional, Tuple, Type, @@ -21,7 +22,7 @@ overload, ) -from typing_extensions import Literal, Self +from typing_extensions import Self from cvat_sdk.api_client.model_utils import IModelData, ModelNormal, to_json from cvat_sdk.core.helpers import get_paginated_collection diff --git a/cvat-sdk/cvat_sdk/core/uploading.py b/cvat-sdk/cvat_sdk/core/uploading.py index 85815ea36286..f13f862e91cd 100644 --- a/cvat-sdk/cvat_sdk/core/uploading.py +++ b/cvat-sdk/cvat_sdk/core/uploading.py @@ -7,7 +7,7 @@ import json import os from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple +from typing import TYPE_CHECKING, Any, ContextManager, Dict, List, Optional, Sequence, Tuple import requests import urllib3 @@ -180,13 +180,25 @@ def upload_file( # query params are used only in the extra messages assert meta["filename"] + if pbar is None: + pbar = NullProgressReporter() + + file_size = filename.stat().st_size + self._tus_start_upload(url, query_params=query_params) - real_filename = self._upload_file_data_with_tus( - url=url, filename=filename, meta=meta, pbar=pbar, logger=logger - ) + with self._uploading_task(pbar, file_size): + real_filename = self._upload_file_data_with_tus( + url=url, filename=filename, meta=meta, pbar=pbar, logger=logger + ) query_params["filename"] = real_filename return self._tus_finish_upload(url, query_params=query_params, fields=fields) + @staticmethod + def _uploading_task(pbar: ProgressReporter, total_size: int) -> ContextManager[None]: + return pbar.task( + total=total_size, desc="Uploading data", unit_scale=True, unit="B", unit_divisor=1024 + ) + def _wait_for_completion( self, url: str, @@ -219,21 +231,13 @@ def _make_tus_uploader(api_client: ApiClient, url: str, **kwargs): return _MyTusUploader(client=client, api_client=api_client, **kwargs) - def _upload_file_data_with_tus( - self, url, filename, *, meta=None, pbar=None, logger=None - ) -> str: - file_size = filename.stat().st_size - if pbar is None: - pbar = NullProgressReporter() - - with open(filename, "rb") as input_file, StreamWithProgress( - input_file, pbar, length=file_size - ) as input_file_with_progress: + def _upload_file_data_with_tus(self, url, filename, *, meta=None, pbar, logger=None) -> str: + with open(filename, "rb") as input_file: tus_uploader = self._make_tus_uploader( self._client.api_client, url=url.rstrip("/") + "/", metadata=meta, - file_stream=input_file_with_progress, + file_stream=StreamWithProgress(input_file, pbar), chunk_size=Uploader._CHUNK_SIZE, log_func=logger, ) @@ -347,44 +351,44 @@ def upload_files( ): bulk_file_groups, separate_files, total_size = self._split_files_by_requests(resources) - if pbar is not None: - pbar.start(total_size, desc="Uploading data") + if pbar is None: + pbar = NullProgressReporter() if str(kwargs.get("sorting_method")).lower() == "predefined": # Request file ordering, because we reorder files to send more efficiently kwargs.setdefault("upload_file_order", [p.name for p in resources]) - self._tus_start_upload(url) - - for group, group_size in bulk_file_groups: - files = {} - for i, filename in enumerate(group): - files[f"client_files[{i}]"] = ( - os.fspath(filename), - filename.read_bytes(), + with self._uploading_task(pbar, total_size): + self._tus_start_upload(url) + + for group, group_size in bulk_file_groups: + files = {} + for i, filename in enumerate(group): + files[f"client_files[{i}]"] = ( + os.fspath(filename), + filename.read_bytes(), + ) + response = self._client.api_client.rest_client.POST( + url, + post_params={"image_quality": kwargs["image_quality"], **files}, + headers={ + "Content-Type": "multipart/form-data", + "Upload-Multiple": "", + **self._client.api_client.get_common_headers(), + }, ) - response = self._client.api_client.rest_client.POST( - url, - post_params={"image_quality": kwargs["image_quality"], **files}, - headers={ - "Content-Type": "multipart/form-data", - "Upload-Multiple": "", - **self._client.api_client.get_common_headers(), - }, - ) - expect_status(200, response) + expect_status(200, response) - if pbar is not None: pbar.advance(group_size) - for filename in separate_files: - self._upload_file_data_with_tus( - url, - filename, - meta={"filename": filename.name}, - pbar=pbar, - logger=self._client.logger.debug, - ) + for filename in separate_files: + self._upload_file_data_with_tus( + url, + filename, + meta={"filename": filename.name}, + pbar=pbar, + logger=self._client.logger.debug, + ) self._tus_finish_upload(url, fields=kwargs) diff --git a/cvat-sdk/cvat_sdk/core/utils.py b/cvat-sdk/cvat_sdk/core/utils.py index e7c28e90e9f9..1708dfd5779a 100644 --- a/cvat-sdk/cvat_sdk/core/utils.py +++ b/cvat-sdk/cvat_sdk/core/utils.py @@ -14,14 +14,13 @@ ContextManager, Dict, Iterator, + Literal, Sequence, TextIO, Union, overload, ) -from typing_extensions import Literal - def filter_dict( d: Dict[str, Any], *, keep: Sequence[str] = None, drop: Sequence[str] = None diff --git a/cvat-sdk/cvat_sdk/datasets/common.py b/cvat-sdk/cvat_sdk/datasets/common.py index 2b8269dbd567..c621a2d2ed33 100644 --- a/cvat-sdk/cvat_sdk/datasets/common.py +++ b/cvat-sdk/cvat_sdk/datasets/common.py @@ -50,6 +50,9 @@ class Sample: frame_index: int """Index of the corresponding frame in its task.""" + frame_name: str + """File name of the frame in its task.""" + annotations: FrameAnnotations """Annotations belonging to the frame.""" diff --git a/cvat-sdk/cvat_sdk/datasets/task_dataset.py b/cvat-sdk/cvat_sdk/datasets/task_dataset.py index 586070457934..111528d43715 100644 --- a/cvat-sdk/cvat_sdk/datasets/task_dataset.py +++ b/cvat-sdk/cvat_sdk/datasets/task_dataset.py @@ -126,7 +126,12 @@ def ensure_chunk(chunk_index): # TODO: tracks? self._samples = [ - Sample(frame_index=k, annotations=v, media=self._TaskMediaElement(self, k)) + Sample( + frame_index=k, + frame_name=data_meta.frames[k].name, + annotations=v, + media=self._TaskMediaElement(self, k), + ) for k, v in self._frame_annotations.items() ] diff --git a/cvat-sdk/cvat_sdk/pytorch/transforms.py b/cvat-sdk/cvat_sdk/pytorch/transforms.py index d63fdba65f68..1fb99362defc 100644 --- a/cvat-sdk/cvat_sdk/pytorch/transforms.py +++ b/cvat-sdk/cvat_sdk/pytorch/transforms.py @@ -2,13 +2,12 @@ # # SPDX-License-Identifier: MIT -from typing import FrozenSet +from typing import FrozenSet, TypedDict import attrs import attrs.validators import torch import torch.utils.data -from typing_extensions import TypedDict from cvat_sdk.datasets.common import UnsupportedDatasetError from cvat_sdk.pytorch.common import Target diff --git a/cvat-sdk/gen/generate.sh b/cvat-sdk/gen/generate.sh index 1b3e8bcb7789..0f1b1f997fd5 100755 --- a/cvat-sdk/gen/generate.sh +++ b/cvat-sdk/gen/generate.sh @@ -8,7 +8,7 @@ set -e GENERATOR_VERSION="v6.0.1" -VERSION="2.5.0" +VERSION="2.6.0" LIB_NAME="cvat_sdk" LAYER1_LIB_NAME="${LIB_NAME}/api_client" DST_DIR="$(cd "$(dirname -- "$0")/.." && pwd)" diff --git a/cvat-sdk/gen/generator-config.yml b/cvat-sdk/gen/generator-config.yml index 80bc80a5e4f6..26e78cb8a3a9 100644 --- a/cvat-sdk/gen/generator-config.yml +++ b/cvat-sdk/gen/generator-config.yml @@ -4,7 +4,7 @@ additionalProperties: packageName: "cvat_sdk.api_client" initRequiredVars: true generateSourceCodeOnly: false - generatorLanguageVersion: '>=3.7' + generatorLanguageVersion: '>=3.8' globalProperties: generateAliasAsModel: true apiTests: false diff --git a/cvat-ui/package.json b/cvat-ui/package.json index 5e9569c05029..90ab58642008 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.54.1", + "version": "1.55.0", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/plugins/sam_plugin/src/ts/index.tsx b/cvat-ui/plugins/sam/src/ts/index.tsx similarity index 97% rename from cvat-ui/plugins/sam_plugin/src/ts/index.tsx rename to cvat-ui/plugins/sam/src/ts/index.tsx index fe030c035652..e4f358ab77e9 100644 --- a/cvat-ui/plugins/sam_plugin/src/ts/index.tsx +++ b/cvat-ui/plugins/sam/src/ts/index.tsx @@ -132,7 +132,7 @@ function modelData( const samPlugin: SAMPlugin = { name: 'Segment Anything', - description: 'Plugin handles non-default SAM serverless function output', + description: 'Handles non-default SAM serverless function output', cvat: { jobs: { get: { @@ -287,7 +287,7 @@ const samPlugin: SAMPlugin = { }, }; -const SAMModelPlugin: ComponentBuilder = ({ core }) => { +const builder: ComponentBuilder = ({ core }) => { samPlugin.data.core = core; core.plugins.register(samPlugin); InferenceSession.create(samPlugin.data.modelURL).then((session) => { @@ -295,7 +295,7 @@ const SAMModelPlugin: ComponentBuilder = ({ core }) => { }); return { - name: 'Segment Anything model', + name: samPlugin.name, destructor: () => {}, }; }; @@ -303,7 +303,7 @@ const SAMModelPlugin: ComponentBuilder = ({ core }) => { function register(): void { if (Object.prototype.hasOwnProperty.call(window, 'cvatUI')) { (window as any as { cvatUI: { registerComponent: PluginEntryPoint } }) - .cvatUI.registerComponent(SAMModelPlugin); + .cvatUI.registerComponent(builder); } } diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index 8b7a899b9ec3..24c3cc63c175 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -581,10 +581,41 @@ export function switchPlay(playing: boolean): AnyAction { }; } -export function confirmCanvasReady(): AnyAction { +export function confirmCanvasReady(ranges?: string): AnyAction { return { type: AnnotationActionTypes.CONFIRM_CANVAS_READY, - payload: {}, + payload: { ranges }, + }; +} + +export function confirmCanvasReadyAsync(): ThunkAction { + return async (dispatch: ActionCreator, getState: () => CombinedState): Promise => { + try { + const state: CombinedState = getState(); + const { instance: job } = state.annotation.job; + const chunks = await job.frames.cachedChunks() as number[]; + const { startFrame, stopFrame, dataChunkSize } = job; + + const ranges = chunks.map((chunk) => ( + [ + Math.max(startFrame, chunk * dataChunkSize), + Math.min(stopFrame, (chunk + 1) * dataChunkSize - 1), + ] + )).reduce>((acc, val) => { + if (acc.length && acc[acc.length - 1][1] + 1 === val[0]) { + const newMax = val[1]; + acc[acc.length - 1][1] = newMax; + } else { + acc.push(val as [number, number]); + } + return acc; + }, []).map(([start, end]) => `${start}:${end}`).join(';'); + + dispatch(confirmCanvasReady(ranges)); + } catch (error) { + // even if error happens here, do not need to notify the users + dispatch(confirmCanvasReady()); + } }; } diff --git a/cvat-ui/src/actions/cloud-storage-actions.ts b/cvat-ui/src/actions/cloud-storage-actions.ts index f25c6d2a5626..89bd3b72095d 100644 --- a/cvat-ui/src/actions/cloud-storage-actions.ts +++ b/cvat-ui/src/actions/cloud-storage-actions.ts @@ -198,7 +198,7 @@ export function getCloudStoragePreviewAsync(cloudStorage: CloudStorage): ThunkAc return async (dispatch: ActionCreator): Promise => { dispatch(cloudStoragesActions.getCloudStoragePreview(cloudStorage.id)); try { - const result = await cloudStorage.getPreview(); + const result = await cloudStorage.preview(); dispatch(cloudStoragesActions.getCloudStoragePreviewSuccess(cloudStorage.id, result)); } catch (error) { dispatch(cloudStoragesActions.getCloudStoragePreviewFailed(cloudStorage.id, error)); diff --git a/cvat-ui/src/actions/models-actions.ts b/cvat-ui/src/actions/models-actions.ts index 4e42a8308e6b..7cc374750e86 100644 --- a/cvat-ui/src/actions/models-actions.ts +++ b/cvat-ui/src/actions/models-actions.ts @@ -271,7 +271,7 @@ export function getModelProvidersAsync(): ThunkAction { export const getModelPreviewAsync = (model: MLModel): ThunkAction => async (dispatch) => { dispatch(modelsActions.getModelPreview(model.id)); try { - const result = await model.getPreview(); + const result = await model.preview(); dispatch(modelsActions.getModelPreviewSuccess(model.id, result)); } catch (error) { dispatch(modelsActions.getModelPreviewFailed(model.id, error)); diff --git a/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx index 4493ba014f04..df2d3d63381d 100644 --- a/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx +++ b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx @@ -24,7 +24,7 @@ import config from 'config'; import CVATTooltip from 'components/common/cvat-tooltip'; import FrameTags from 'components/annotation-page/tag-annotation-workspace/frame-tags'; import { - confirmCanvasReady, + confirmCanvasReadyAsync, dragCanvas, zoomCanvas, resetCanvas, @@ -259,7 +259,7 @@ function mapStateToProps(state: CombinedState): StateToProps { function mapDispatchToProps(dispatch: any): DispatchToProps { return { onSetupCanvas(): void { - dispatch(confirmCanvasReady()); + dispatch(confirmCanvasReadyAsync()); }, onDragCanvas(enabled: boolean): void { dispatch(dragCanvas(enabled)); diff --git a/cvat-ui/src/components/annotation-page/canvas/views/canvas3d/canvas-wrapper3D.tsx b/cvat-ui/src/components/annotation-page/canvas/views/canvas3d/canvas-wrapper3D.tsx index e21a4a138d99..479ad283ede5 100644 --- a/cvat-ui/src/components/annotation-page/canvas/views/canvas3d/canvas-wrapper3D.tsx +++ b/cvat-ui/src/components/annotation-page/canvas/views/canvas3d/canvas-wrapper3D.tsx @@ -16,7 +16,7 @@ import Spin from 'antd/lib/spin'; import { activateObject, - confirmCanvasReady, + confirmCanvasReadyAsync, createAnnotationsAsync, dragCanvas, editShape, @@ -131,7 +131,7 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { dispatch(dragCanvas(enabled)); }, onSetupCanvas(): void { - dispatch(confirmCanvasReady()); + dispatch(confirmCanvasReadyAsync()); }, onResetCanvas(): void { dispatch(resetCanvas()); diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx index 5a97f23cf74d..ef1ce7e6a4fd 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx @@ -397,8 +397,20 @@ class OpenCVControlComponent extends React.PureComponent activeImageModifier .modifier.processImage(oldImageData, frame), imageData); const imageBitmap = await createImageBitmap(newImageData); - frameData.imageData = imageBitmap; - canvasInstance.setup(frameData, states, curZOrder); + const proxy = new Proxy(frameData, { + get: (_frameData, prop, receiver) => { + if (prop === 'data') { + return async () => ({ + renderWidth: imageData.width, + renderHeight: imageData.height, + imageData: imageBitmap, + }); + } + + return Reflect.get(_frameData, prop, receiver); + }, + }); + canvasInstance.setup(proxy, states, curZOrder); } } catch (error: any) { notification.error({ diff --git a/cvat-ui/src/components/annotation-page/styles.scss b/cvat-ui/src/components/annotation-page/styles.scss index 7d75f57d8f7e..9d31599fe3d5 100644 --- a/cvat-ui/src/components/annotation-page/styles.scss +++ b/cvat-ui/src/components/annotation-page/styles.scss @@ -3,7 +3,7 @@ // // SPDX-License-Identifier: MIT -@import '../../base.scss'; +@import '../../base'; .cvat-annotation-page.ant-layout { height: 100%; @@ -126,15 +126,39 @@ } } -.cvat-player-slider { +.cvat-player-slider.ant-slider { width: 350px; margin: 0; + margin-top: $grid-unit-size * -0.5; + + > .ant-slider-handle { + z-index: 100; + margin-top: -3.5px; + } + + > .ant-slider-track { + background: none; + } > .ant-slider-rail { + height: $grid-unit-size; background-color: $player-slider-color; } } +.cvat-player-slider-progress { + width: 350px; + height: $grid-unit-size; + position: absolute; + top: 0; + pointer-events: none; + + > rect { + transition: width 0.5s; + fill: #1890ff; + } +} + .cvat-player-filename-wrapper { max-width: $grid-unit-size * 30; max-height: $grid-unit-size * 3; @@ -221,7 +245,7 @@ .ant-table-thead { > tr > th { - padding: 5px 5px; + padding: $grid-unit-size 0 $grid-unit-size $grid-unit-size * 0.5; } } } @@ -446,7 +470,7 @@ } .group { - background: rgba(216, 233, 250, 0.5); + background: rgba(216, 233, 250, 50%); border: 1px solid #d3e0ec; } } diff --git a/cvat-ui/src/components/annotation-page/top-bar/player-navigation.tsx b/cvat-ui/src/components/annotation-page/top-bar/player-navigation.tsx index e95d15cf33d0..4a05b4444a1f 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/player-navigation.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/player-navigation.tsx @@ -21,6 +21,7 @@ interface Props { startFrame: number; stopFrame: number; playing: boolean; + ranges: string; frameNumber: number; frameFilename: string; frameDeleted: boolean; @@ -47,6 +48,7 @@ function PlayerNavigation(props: Props): JSX.Element { deleteFrameShortcut, focusFrameInputShortcut, inputFrameRef, + ranges, onSliderChange, onInputChange, onURLIconClick, @@ -105,6 +107,23 @@ function PlayerNavigation(props: Props): JSX.Element { value={frameNumber || 0} onChange={onSliderChange} /> + {!!ranges && ( + + {ranges.split(';').map((range) => { + const [start, end] = range.split(':').map((num) => +num); + const adjustedStart = Math.max(0, start - 1); + let totalSegments = stopFrame - startFrame; + if (totalSegments === 0) { + // corner case for jobs with one image + totalSegments = 1; + } + const segmentWidth = 1000 / totalSegments; + const width = Math.max((end - adjustedStart), 1) * segmentWidth; + const offset = (Math.max((adjustedStart - startFrame), 0) / totalSegments) * 1000; + return (); + })} + + )} diff --git a/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx b/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx index ee800ce68619..7c88063bf217 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx @@ -69,6 +69,7 @@ interface Props { onRestoreFrame(): void; switchNavigationBlocked(blocked: boolean): void; jobInstance: any; + ranges: string; } export default function AnnotationTopBarComponent(props: Props): JSX.Element { @@ -77,6 +78,7 @@ export default function AnnotationTopBarComponent(props: Props): JSX.Element { undoAction, redoAction, playing, + ranges, frameNumber, frameFilename, frameDeleted, @@ -168,6 +170,7 @@ export default function AnnotationTopBarComponent(props: Props): JSX.Element { startFrame={startFrame} stopFrame={stopFrame} playing={playing} + ranges={ranges} frameNumber={frameNumber} frameFilename={frameFilename} frameDeleted={frameDeleted} diff --git a/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx b/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx index 4796af1c1079..30ce8a552db7 100644 --- a/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx +++ b/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx @@ -449,8 +449,8 @@ class AdvancedConfigurationForm extends React.PureComponent { > - Use zip/video chunks - + Prefer zip chunks + diff --git a/cvat-ui/src/components/header/settings-modal/player-settings.tsx b/cvat-ui/src/components/header/settings-modal/player-settings.tsx index 324243f3afb6..ab8dbab42c01 100644 --- a/cvat-ui/src/components/header/settings-modal/player-settings.tsx +++ b/cvat-ui/src/components/header/settings-modal/player-settings.tsx @@ -1,4 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -18,6 +19,7 @@ import { clamp } from 'utils/math'; import { BackJumpIcon, ForwardJumpIcon } from 'icons'; import { FrameSpeed } from 'reducers'; import config from 'config'; +import { usePlugins } from 'utils/hooks'; interface Props { frameStep: number; @@ -54,177 +56,201 @@ export default function PlayerSettingsComponent(props: Props): JSX.Element { onSwitchShowingDeletedFrames, } = props; + const plugins = usePlugins((state) => state.plugins.components.settings.player, props); + const minFrameStep = 2; const maxFrameStep = 1000; - return ( -
- - - Player step - { - if (typeof value !== 'undefined' && value !== null) { - onChangeFrameStep(Math.floor(clamp(+value, minFrameStep, maxFrameStep))); - } - }} - /> - - - - Number of frames skipped when selecting - - or - - - - - - - Player speed - { + onChangeFrameSpeed(speed); + }} + > + - - Fastest - - - Fast - - - Usual - - - Slow - - - Slower - - - Slowest - - - - - - - onChangeCanvasBackgroundColor(e.hex)} - /> - )} - overlayClassName='canvas-background-color-picker-popover' - trigger='click' + Fastest + + + Fast + + + Usual + + + Slow + + + Slower + + + Slowest + + + + + ), 10]); + + items.push([( + + + onChangeCanvasBackgroundColor(e.hex)} + /> + )} + overlayClassName='canvas-background-color-picker-popover' + trigger='click' + > + - - - - - - - - { - onSwitchResetZoom(event.target.checked); - }} - > - Reset zoom - - - - Fit image after changing frame - - - - - - - { - onSwitchRotateAll(event.target.checked); - }} - > - Rotate all images - - - - Rotate all images simultaneously - - - - - - - - - { - onSwitchSmoothImage(event.target.checked); - }} - > - Smooth image - - - - Smooth image when zoom-in it - - - - - + Rotate all images + + + + Rotate all images simultaneously + + + + + ), 30]); + + items.push([( + + + + { - onSwitchShowingDeletedFrames(event.target.checked); + onSwitchSmoothImage(event.target.checked); }} > - Show deleted frames + Smooth image - - - You will be able to navigate and restore deleted frames - - - + + + Smooth image when zoom-in it + + + + + + { + onSwitchShowingDeletedFrames(event.target.checked); + }} + > + Show deleted frames + + + + You will be able to navigate and restore deleted frames + + + + ), 40]); + + items.push(...plugins.map(({ component: Component, weight }, index: number) => ( + [, weight] as [JSX.Element, number] + ))); + + return ( +
+ { items.sort((item1, item2) => item1[1] - item2[1]) + .map((item) => item[0]) }
); } diff --git a/cvat-ui/src/components/header/settings-modal/styles.scss b/cvat-ui/src/components/header/settings-modal/styles.scss index 8f4ce00edabd..4d6c496b6422 100644 --- a/cvat-ui/src/components/header/settings-modal/styles.scss +++ b/cvat-ui/src/components/header/settings-modal/styles.scss @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -@import '../../../base.scss'; +@import '../../../base'; .cvat-settings-tabs { height: 100%; @@ -24,24 +24,6 @@ padding: 24px; } -.cvat-workspace-settings-auto-save, -.cvat-workspace-settings-autoborders, -.cvat-workspace-settings-intelligent-polygon-cropping, -.cvat-workspace-settings-show-text-always, -.cvat-workspace-settings-show-interpolated, -.cvat-workspace-settings-show-deleted, -.cvat-workspace-settings-approx-poly-threshold, -.cvat-workspace-settings-aam-zoom-margin, -.cvat-workspace-settings-show-frame-tags, -.cvat-workspace-settings-text-settings, -.cvat-workspace-settings-control-points-size { - margin-bottom: $grid-unit-size * 3; - - > div:first-child { - margin-bottom: $grid-unit-size; - } -} - .cvat-workspace-settings-text-content { width: 100%; } @@ -50,13 +32,7 @@ user-select: none; } -.cvat-player-settings-step, -.cvat-player-settings-speed, -.cvat-player-settings-reset-zoom, -.cvat-player-settings-rotate-all, -.cvat-player-settings-canvas-background, -.cvat-workspace-settings-aam-zoom-margin, -.cvat-workspace-settings-auto-save-interval { +.cvat-player-setting { margin-bottom: $grid-unit-size * 3; } diff --git a/cvat-ui/src/components/header/settings-modal/workspace-settings.tsx b/cvat-ui/src/components/header/settings-modal/workspace-settings.tsx index a55d6ae8ec7a..e624e7c4adf0 100644 --- a/cvat-ui/src/components/header/settings-modal/workspace-settings.tsx +++ b/cvat-ui/src/components/header/settings-modal/workspace-settings.tsx @@ -85,8 +85,8 @@ function WorkspaceSettingsComponent(props: Props): JSX.Element { return (
- - + + - - Auto save every minutes - + - + - + Content of a text @@ -172,7 +171,7 @@ function WorkspaceSettingsComponent(props: Props): JSX.Element { - + Position of a text @@ -199,7 +198,7 @@ function WorkspaceSettingsComponent(props: Props): JSX.Element { /> - + - + Try to crop polygons automatically when editing - + Show frame tags in the corner of the workspace - + Attribute annotation mode (AAM) zoom margin - + Control points size - + Default number of points in polygon approximation @@ -296,7 +295,7 @@ function WorkspaceSettingsComponent(props: Props): JSX.Element { marks={marks} /> - + Works for serverless interactors and OpenCV scissors diff --git a/cvat-ui/src/containers/annotation-page/top-bar/top-bar.tsx b/cvat-ui/src/containers/annotation-page/top-bar/top-bar.tsx index f53c5f61cdbe..067d85fe1ad0 100644 --- a/cvat-ui/src/containers/annotation-page/top-bar/top-bar.tsx +++ b/cvat-ui/src/containers/annotation-page/top-bar/top-bar.tsx @@ -65,6 +65,7 @@ interface StateToProps { normalizedKeyMap: Record; canvasInstance: Canvas | Canvas3d; forceExit: boolean; + ranges: string; activeControl: ActiveControl; } @@ -91,6 +92,7 @@ function mapStateToProps(state: CombinedState): StateToProps { annotation: { player: { playing, + ranges, frame: { data: { deleted: frameIsDeleted }, filename: frameFilename, @@ -142,6 +144,7 @@ function mapStateToProps(state: CombinedState): StateToProps { canvasInstance, forceExit, activeControl, + ranges, }; } @@ -638,6 +641,7 @@ class AnnotationTopBarContainer extends React.PureComponent { workspace, canvasIsReady, keyMap, + ranges, normalizedKeyMap, activeControl, searchAnnotations, @@ -766,6 +770,7 @@ class AnnotationTopBarContainer extends React.PureComponent { workspace={workspace} playing={playing} saving={saving} + ranges={ranges} startFrame={startFrame} stopFrame={stopFrame} frameNumber={frameNumber} diff --git a/cvat-ui/src/cvat-core-wrapper.ts b/cvat-ui/src/cvat-core-wrapper.ts index 888c4413e549..ae5ff60e9e2d 100644 --- a/cvat-ui/src/cvat-core-wrapper.ts +++ b/cvat-ui/src/cvat-core-wrapper.ts @@ -7,6 +7,7 @@ import _cvat from 'cvat-core/src/api'; import ObjectState from 'cvat-core/src/object-state'; import Webhook from 'cvat-core/src/webhook'; import MLModel from 'cvat-core/src/ml-model'; +import CloudStorage from 'cvat-core/src/cloud-storage'; import { ModelProvider } from 'cvat-core/src/lambda-manager'; import { Label, Attribute, @@ -60,6 +61,7 @@ export { Webhook, Issue, User, + CloudStorage, Organization, Comment, MLModel, diff --git a/cvat-ui/src/reducers/annotation-reducer.ts b/cvat-ui/src/reducers/annotation-reducer.ts index 71524879f8bd..d7d395adb6f1 100644 --- a/cvat-ui/src/reducers/annotation-reducer.ts +++ b/cvat-ui/src/reducers/annotation-reducer.ts @@ -55,6 +55,7 @@ const defaultState: AnnotationState = { job: { openTime: null, labels: [], + groundTruthJobFramesMeta: null, requestedId: null, groundTruthJobId: null, instance: null, @@ -72,6 +73,7 @@ const defaultState: AnnotationState = { delay: 0, changeTime: null, }, + ranges: '', playing: false, frameAngles: [], navigationBlocked: false, @@ -417,8 +419,13 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { }; } case AnnotationActionTypes.CONFIRM_CANVAS_READY: { + const { ranges } = action.payload; return { ...state, + player: { + ...state.player, + ranges: ranges || state.player.ranges, + }, canvas: { ...state.canvas, ready: true, diff --git a/cvat-ui/src/reducers/index.ts b/cvat-ui/src/reducers/index.ts index af2c8d9ab1d4..2207d73b807d 100644 --- a/cvat-ui/src/reducers/index.ts +++ b/cvat-ui/src/reducers/index.ts @@ -312,6 +312,9 @@ export interface PluginsState { player: PluginComponent[]; }; }; + settings: { + player: PluginComponent[], + } router: PluginComponent[]; loggedInModals: PluginComponent[]; } @@ -692,6 +695,7 @@ export interface AnnotationState { delay: number; changeTime: number | null; }; + ranges: string; navigationBlocked: boolean; playing: boolean; frameAngles: number[]; diff --git a/cvat-ui/src/reducers/plugins-reducer.ts b/cvat-ui/src/reducers/plugins-reducer.ts index 49ff3f77cb80..aaedad46dcbd 100644 --- a/cvat-ui/src/reducers/plugins-reducer.ts +++ b/cvat-ui/src/reducers/plugins-reducer.ts @@ -44,6 +44,9 @@ const defaultState: PluginsState = { }, router: [], loggedInModals: [], + settings: { + player: [], + }, }, }; diff --git a/cvat-ui/webpack.config.js b/cvat-ui/webpack.config.js index f933c2cab7ec..467233f97908 100644 --- a/cvat-ui/webpack.config.js +++ b/cvat-ui/webpack.config.js @@ -8,16 +8,18 @@ */ const path = require('path'); +const webpack = require('webpack'); const HtmlWebpackPlugin = require('html-webpack-plugin'); const Dotenv = require('dotenv-webpack'); const CopyPlugin = require('copy-webpack-plugin'); module.exports = (env) => { const defaultAppConfig = path.join(__dirname, 'src/config.tsx'); - const defaultPlugins = ['plugins/sam_plugin']; + const defaultPlugins = ['plugins/sam']; const appConfigFile = process.env.UI_APP_CONFIG ? process.env.UI_APP_CONFIG : defaultAppConfig; const pluginsList = process.env.CLIENT_PLUGINS ? [...defaultPlugins, ...process.env.CLIENT_PLUGINS.split(':')] - .map((s) => s.trim()).filter((s) => !!s) : defaultPlugins + .map((s) => s.trim()).filter((s) => !!s) : defaultPlugins; + const sourceMapsToken = process.env.SOURCE_MAPS_TOKEN || ''; const transformedPlugins = pluginsList .filter((plugin) => !!plugin).reduce((acc, _path, index) => ({ @@ -209,6 +211,10 @@ module.exports = (env) => { }, ], }), + ...(sourceMapsToken ? [new webpack.SourceMapDevToolPlugin({ + append: '\n', + filename: `${sourceMapsToken}/[file].map`, + })] : []), ], } }; diff --git a/cvat/__init__.py b/cvat/__init__.py index 2b4479680b23..43f1aba73a8e 100644 --- a/cvat/__init__.py +++ b/cvat/__init__.py @@ -4,6 +4,6 @@ from cvat.utils.version import get_version -VERSION = (2, 5, 2, 'final', 0) +VERSION = (2, 6, 0, 'final', 0) __version__ = get_version(VERSION) diff --git a/cvat/apps/analytics_report/report/primary_metrics/objects.py b/cvat/apps/analytics_report/report/primary_metrics/objects.py index d7d369ca146e..8004fd2e9f9e 100644 --- a/cvat/apps/analytics_report/report/primary_metrics/objects.py +++ b/cvat/apps/analytics_report/report/primary_metrics/objects.py @@ -12,38 +12,42 @@ class JobObjects(PrimaryMetricBase): _default_view = ViewChoice.HISTOGRAM _key = "objects" # Raw SQL queries are used to execute ClickHouse queries, as there is no ORM available here - _query = "SELECT toStartOfDay(timestamp) as day, sum(JSONLength(JSONExtractString(payload, {object_type:String}))) as s FROM events WHERE scope = {scope:String} AND job_id = {job_id:UInt64} GROUP BY day ORDER BY day ASC" + _query = "SELECT toStartOfDay(timestamp) as day, scope, sum(count) FROM events WHERE scope IN ({scopes:Array(String)}) AND job_id = {job_id:UInt64} GROUP BY scope, day ORDER BY day ASC" _granularity = GranularityChoice.DAY def calculate(self): statistics = {} - - for action in ["create", "update", "delete"]: - action_data = statistics.setdefault(f"{action}d", {}) - for obj_type in ["tracks", "shapes", "tags"]: - result = self._make_clickhouse_query( - { - "scope": f"{action}:{obj_type}", - "object_type": obj_type, - "job_id": self._db_obj.id, - } - ) - action_data[obj_type] = {entry[0]: entry[1] for entry in result.result_rows} + actions = ("create", "update", "delete") + obj_types = ("tracks", "shapes", "tags") + scopes = [f"{action}:{obj_type}" for action in actions for obj_type in obj_types] + for action in actions: + statistics[action] = {} + for obj_type in obj_types: + statistics[action][obj_type] = {} + + result = self._make_clickhouse_query( + { + "scopes": scopes, + "job_id": self._db_obj.id, + } + ) + + for day, scope, count in result.result_rows: + action, obj_type = scope.split(":") + statistics[action][obj_type][day] = count objects_statistics = self.get_empty() dates = set() - for action in ["created", "updated", "deleted"]: - for obj in ["tracks", "shapes", "tags"]: + for action in actions: + for obj in obj_types: dates.update(statistics[action][obj].keys()) - for action in ["created", "updated", "deleted"]: + for action in actions: for date in sorted(dates): - objects_statistics[action].append( + objects_statistics[f"{action}d"].append( { - "value": sum( - statistics[action][t].get(date, 0) for t in ["tracks", "shapes", "tags"] - ), + "value": sum(statistics[action][t].get(date, 0) for t in obj_types), "datetime": date.isoformat() + "Z", } ) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index a89ebb773a13..220502aaa505 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -6,6 +6,7 @@ import io import os import os.path as osp +from PIL import Image from types import SimpleNamespace from typing import Optional import pytz @@ -2720,9 +2721,16 @@ def create(self, request, *args, **kwargs): self.perform_create(serializer) path = os.path.join(settings.ASSETS_ROOT, str(serializer.instance.uuid)) os.makedirs(path) - with open(os.path.join(path, file.name), 'wb+') as destination: - for chunk in file.chunks(): - destination.write(chunk) + if file.content_type in ('image/jpeg', 'image/png'): + image = Image.open(file) + if any(map(lambda x: x > settings.ASSET_MAX_IMAGE_SIZE, image.size)): + scale_factor = settings.ASSET_MAX_IMAGE_SIZE / max(image.size) + image = image.resize((map(lambda x: int(x * scale_factor), image.size))) + image.save(os.path.join(path, file.name)) + else: + with open(os.path.join(path, file.name), 'wb+') as destination: + for chunk in file.chunks(): + destination.write(chunk) headers = self.get_success_headers(serializer.data) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) diff --git a/cvat/requirements/base.in b/cvat/requirements/base.in index 7c773219336f..32e152e1e91e 100644 --- a/cvat/requirements/base.in +++ b/cvat/requirements/base.in @@ -58,7 +58,7 @@ dnspython==2.2.0 setuptools==65.5.1 django-health-check==3.17.0 psutil==5.9.4 -clickhouse-connect==0.5.10 +clickhouse-connect==0.6.8 django-crum==0.7.9 wheel>=0.38.0 # not directly required, pinned by Snyk to avoid a vulnerability psycopg2-binary==2.9.5 diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 2f18e19e7dcf..669962ee68e9 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:d1435558d66ec49d0c691492b2f3798960ca3bba +# SHA1:47a46c0f57bd02f1446db65b3d107ca9e7927d76 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -12,7 +12,7 @@ absl-py==1.4.0 # via # tensorboard # tensorflow -asgiref==3.7.1 +asgiref==3.7.2 # via django astunparse==1.6.3 # via tensorflow @@ -23,7 +23,7 @@ attrs==21.4.0 # -r cvat/requirements/base.in # datumaro # jsonschema -azure-core==1.26.4 +azure-core==1.28.0 # via # azure-storage-blob # msrest @@ -35,9 +35,9 @@ botocore==1.20.112 # via # boto3 # s3transfer -cachetools==5.3.0 +cachetools==5.3.1 # via google-auth -certifi==2023.5.7 +certifi==2023.7.22 # via # clickhouse-connect # msrest @@ -50,17 +50,17 @@ click==8.1.3 # via # -r cvat/requirements/base.in # rq -clickhouse-connect==0.5.10 +clickhouse-connect==0.6.8 # via -r cvat/requirements/base.in -contourpy==1.0.7 +contourpy==1.1.0 # via matplotlib coreapi==2.3.3 # via -r cvat/requirements/base.in coreschema==0.0.4 # via coreapi -croniter==1.3.15 +croniter==1.4.1 # via rq-scheduler -cryptography==41.0.0 +cryptography==41.0.2 # via # azure-storage-blob # pyjwt @@ -72,7 +72,7 @@ defusedxml==0.7.1 # via # datumaro # python3-openid -deprecated==1.2.13 +deprecated==1.2.14 # via limits diskcache==5.4.0 # via -r cvat/requirements/base.in @@ -80,7 +80,7 @@ dj-pagination==2.5.0 # via -r cvat/requirements/base.in dj-rest-auth[with_social]==2.2.7 # via -r cvat/requirements/base.in -django==4.2.1 +django==4.2.3 # via # -r cvat/requirements/base.in # dj-rest-auth @@ -132,9 +132,9 @@ easyprocess==0.3 # pyunpack entrypoint2==1.1 # via pyunpack -flatbuffers==23.5.9 +flatbuffers==23.5.26 # via tensorflow -fonttools==4.39.4 +fonttools==4.41.1 # via matplotlib furl==2.1.0 # via -r cvat/requirements/base.in @@ -144,11 +144,11 @@ gitdb==4.0.10 # via gitpython gitpython==3.1.30 # via -r cvat/requirements/base.in -google-api-core==2.11.0 +google-api-core==2.11.1 # via # google-cloud-core # google-cloud-storage -google-auth==2.18.1 +google-auth==2.22.0 # via # google-api-core # google-auth-oauthlib @@ -157,7 +157,7 @@ google-auth==2.18.1 # tensorboard google-auth-oauthlib==0.4.6 # via tensorboard -google-cloud-core==2.3.2 +google-cloud-core==2.3.3 # via google-cloud-storage google-cloud-storage==1.42.0 # via -r cvat/requirements/base.in @@ -167,9 +167,9 @@ google-pasta==0.2.0 # via tensorflow google-resumable-media==2.5.0 # via google-cloud-storage -googleapis-common-protos==1.59.0 +googleapis-common-protos==1.59.1 # via google-api-core -grpcio==1.54.2 +grpcio==1.56.2 # via # tensorboard # tensorflow @@ -180,7 +180,9 @@ h5py==3.6.0 # tensorflow idna==3.4 # via requests -importlib-resources==5.12.0 +importlib-metadata==6.8.0 + # via clickhouse-connect +importlib-resources==6.0.0 # via limits inflection==0.5.1 # via drf-spectacular @@ -200,11 +202,11 @@ keras==2.11.0 # via tensorflow kiwisolver==1.4.4 # via matplotlib -libclang==16.0.0 +libclang==16.0.6 # via tensorflow limits==3.5.0 # via python-logstash-async -lxml==4.9.2 +lxml==4.9.3 # via datumaro lz4==4.3.2 # via clickhouse-connect @@ -212,15 +214,15 @@ markdown==3.2.2 # via # -r cvat/requirements/base.in # tensorboard -markupsafe==2.1.2 +markupsafe==2.1.3 # via # jinja2 # werkzeug -matplotlib==3.7.1 +matplotlib==3.7.2 # via # datumaro # pycocotools -mistune==2.0.5 +mistune==3.0.1 # via -r cvat/requirements/base.in msrest==0.7.1 # via azure-storage-blob @@ -234,7 +236,7 @@ opt-einsum==3.3.0 # via tensorflow orderedmultidict==1.0.1 # via furl -orjson==3.8.13 +orjson==3.9.2 # via datumaro packaging==23.1 # via @@ -243,7 +245,7 @@ packaging==23.1 # nibabel # tensorboardx # tensorflow -pandas==2.0.1 +pandas==2.0.3 # via datumaro patool==1.12 # via -r cvat/requirements/base.in @@ -275,7 +277,7 @@ pycparser==2.21 # via cffi pygments==2.7.4 # via -r cvat/requirements/base.in -pyjwt[crypto]==2.7.0 +pyjwt[crypto]==2.8.0 # via django-allauth pylogbeat==2.0.0 # via python-logstash-async @@ -305,7 +307,7 @@ pytz==2020.1 # pandas pyunpack==0.2.1 # via -r cvat/requirements/base.in -pyyaml==6.0 +pyyaml==6.0.1 # via # datumaro # drf-spectacular @@ -345,7 +347,7 @@ rq-scheduler==0.10.0 # via -r cvat/requirements/base.in rsa==4.9 # via google-auth -ruamel-yaml==0.17.26 +ruamel-yaml==0.17.32 # via datumaro ruamel-yaml-clib==0.2.7 # via ruamel-yaml @@ -353,7 +355,7 @@ rules==3.3 # via -r cvat/requirements/base.in s3transfer==0.4.2 # via boto3 -scipy==1.10.1 +scipy==1.11.1 # via datumaro shapely==1.7.1 # via -r cvat/requirements/base.in @@ -390,7 +392,7 @@ tensorflow-io-gcs-filesystem==0.32.0 # via tensorflow termcolor==2.3.0 # via tensorflow -typing-extensions==4.6.2 +typing-extensions==4.7.1 # via # asgiref # azure-core @@ -410,9 +412,9 @@ urllib3==1.26.16 # clickhouse-connect # google-auth # requests -werkzeug==2.3.4 +werkzeug==2.3.6 # via tensorboard -wheel==0.40.0 +wheel==0.41.0 # via # -r cvat/requirements/base.in # astunparse @@ -421,6 +423,8 @@ wrapt==1.15.0 # via # deprecated # tensorflow +zipp==3.16.2 + # via importlib-metadata zstandard==0.21.0 # via clickhouse-connect diff --git a/cvat/requirements/development.txt b/cvat/requirements/development.txt index 65019acf8a51..61e3939f3993 100644 --- a/cvat/requirements/development.txt +++ b/cvat/requirements/development.txt @@ -14,7 +14,7 @@ autopep8==2.0.2 # via django-silk black==23.3.0 # via -r cvat/requirements/development.in -dill==0.3.6 +dill==0.3.7 # via pylint django-extensions==3.0.8 # via -r cvat/requirements/development.in @@ -30,9 +30,9 @@ mccabe==0.7.0 # via pylint mypy-extensions==1.0.0 # via black -pathspec==0.11.1 +pathspec==0.11.2 # via black -platformdirs==3.5.1 +platformdirs==3.9.1 # via # black # pylint @@ -58,7 +58,7 @@ tomli==2.0.1 # autopep8 # black # pylint -tomlkit==0.11.8 +tomlkit==0.12.1 # via pylint tornado==6.3.2 # via snakeviz diff --git a/cvat/requirements/production.txt b/cvat/requirements/production.txt index 3899e8286174..a903a1c2a637 100644 --- a/cvat/requirements/production.txt +++ b/cvat/requirements/production.txt @@ -1,4 +1,4 @@ -# SHA1:d3d4b2262fd87a700593e22be8811e6d04230e40 +# SHA1:784a6a811263fa11d49da152d9840f92b650d6fd # # This file is autogenerated by pip-compile-multi # To update, run: @@ -8,11 +8,15 @@ -r base.txt --no-binary av -anyio==3.6.2 +anyio==3.7.1 # via watchfiles +coverage==7.2.3 + # via -r cvat/requirements/production.in +exceptiongroup==1.1.2 + # via anyio h11==0.14.0 # via uvicorn -httptools==0.5.0 +httptools==0.6.0 # via uvicorn python-dotenv==1.0.0 # via uvicorn @@ -26,7 +30,5 @@ watchfiles==0.19.0 # via uvicorn websockets==11.0.3 # via uvicorn -coverage==7.2.3 - # via -r cvat/requirements/production.in # The following packages are considered to be unsafe in a requirements file: diff --git a/cvat/requirements/testing.txt b/cvat/requirements/testing.txt index a868cb1c6b0c..7e2da3371e85 100644 --- a/cvat/requirements/testing.txt +++ b/cvat/requirements/testing.txt @@ -1,4 +1,4 @@ -# SHA1:910e8edd8fcfdbe7c9a7278ba499bcfad1313c19 +# SHA1:429cfd9ce2f6b66fbb7c898a5c6279d9d8a61335 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -8,11 +8,11 @@ -r development.txt --no-binary av +coverage==7.2.3 + # via -r cvat/requirements/testing.in fakeredis==2.10.3 # via -r cvat/requirements/testing.in sortedcontainers==2.4.0 # via fakeredis -coverage==7.2.3 - # via -r cvat/requirements/testing.in # The following packages are considered to be unsafe in a requirements file: diff --git a/cvat/schema.yml b/cvat/schema.yml index 091b33daa052..8daba12a02b0 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: CVAT REST API - version: 2.5.2 + version: '2.6' description: REST API for Computer Vision Annotation Tool (CVAT) termsOfService: https://www.google.com/policies/terms/ contact: diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 82b198d36426..0664e6cd8e70 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -420,17 +420,21 @@ class CVAT_QUEUES(Enum): # Make sure to update other config files when updating these directories DATA_ROOT = os.path.join(BASE_DIR, 'data') -EVENTS_LOCAL_DB = os.path.join(DATA_ROOT, 'events.db') -os.makedirs(DATA_ROOT, exist_ok=True) -if not os.path.exists(EVENTS_LOCAL_DB): - open(EVENTS_LOCAL_DB, 'w').close() - MEDIA_DATA_ROOT = os.path.join(DATA_ROOT, 'data') os.makedirs(MEDIA_DATA_ROOT, exist_ok=True) CACHE_ROOT = os.path.join(DATA_ROOT, 'cache') os.makedirs(CACHE_ROOT, exist_ok=True) +EVENTS_LOCAL_DB_ROOT = os.path.join(CACHE_ROOT, 'events') +os.makedirs(EVENTS_LOCAL_DB_ROOT, exist_ok=True) +EVENTS_LOCAL_DB_FILE = os.path.join( + EVENTS_LOCAL_DB_ROOT, + os.getenv('CVAT_EVENTS_LOCAL_DB_FILENAME', 'events.db'), +) +if not os.path.exists(EVENTS_LOCAL_DB_FILE): + open(EVENTS_LOCAL_DB_FILE, 'w').close() + JOBS_ROOT = os.path.join(DATA_ROOT, 'jobs') os.makedirs(JOBS_ROOT, exist_ok=True) @@ -504,7 +508,7 @@ class CVAT_QUEUES(Enum): 'port': os.getenv('DJANGO_LOG_SERVER_PORT', 8282), 'version': 1, 'message_type': 'django', - 'database_path': EVENTS_LOCAL_DB, + 'database_path': EVENTS_LOCAL_DB_FILE, } }, 'loggers': { @@ -695,8 +699,9 @@ class CVAT_QUEUES(Enum): IMPORT_CACHE_SUCCESS_TTL = timedelta(hours=1) IMPORT_CACHE_CLEAN_DELAY = timedelta(hours=2) -ASSET_MAX_SIZE_MB = 2 +ASSET_MAX_SIZE_MB = 10 ASSET_SUPPORTED_TYPES = ('image/jpeg', 'image/png', 'image/webp', 'image/gif', 'application/pdf', ) -ASSET_MAX_COUNT_PER_GUIDE = 10 +ASSET_MAX_IMAGE_SIZE = 1920 +ASSET_MAX_COUNT_PER_GUIDE = 30 SMOKESCREEN_ENABLED = True diff --git a/cvat/settings/testing.py b/cvat/settings/testing.py index 74703f86ad8a..bba64d94dc36 100644 --- a/cvat/settings/testing.py +++ b/cvat/settings/testing.py @@ -18,11 +18,6 @@ DATA_ROOT = os.path.join(BASE_DIR, 'data') os.makedirs(DATA_ROOT, exist_ok=True) -EVENTS_LOCAL_DB = os.path.join(DATA_ROOT, 'events.db') -os.makedirs(DATA_ROOT, exist_ok=True) -if not os.path.exists(EVENTS_LOCAL_DB): - open(EVENTS_LOCAL_DB, 'w').close() - MEDIA_DATA_ROOT = os.path.join(DATA_ROOT, 'data') os.makedirs(MEDIA_DATA_ROOT, exist_ok=True) diff --git a/docker-compose.yml b/docker-compose.yml index 0bf34777f604..bd9d6db66c46 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: cvat_server: container_name: cvat_server - image: cvat/server:${CVAT_VERSION:-v2.5.2} + image: cvat/server:${CVAT_VERSION:-v2.6.0} restart: always depends_on: - cvat_redis @@ -64,7 +64,7 @@ services: cvat_utils: container_name: cvat_utils - image: cvat/server:${CVAT_VERSION:-v2.5.2} + image: cvat/server:${CVAT_VERSION:-v2.6.0} restart: always depends_on: - cvat_redis @@ -89,7 +89,7 @@ services: cvat_worker_import: container_name: cvat_worker_import - image: cvat/server:${CVAT_VERSION:-v2.5.2} + image: cvat/server:${CVAT_VERSION:-v2.6.0} restart: always depends_on: - cvat_redis @@ -112,7 +112,7 @@ services: cvat_worker_export: container_name: cvat_worker_export - image: cvat/server:${CVAT_VERSION:-v2.5.2} + image: cvat/server:${CVAT_VERSION:-v2.6.0} restart: always depends_on: - cvat_redis @@ -135,7 +135,7 @@ services: cvat_worker_annotation: container_name: cvat_worker_annotation - image: cvat/server:${CVAT_VERSION:-v2.5.2} + image: cvat/server:${CVAT_VERSION:-v2.6.0} restart: always depends_on: - cvat_redis @@ -158,7 +158,7 @@ services: cvat_worker_webhooks: container_name: cvat_worker_webhooks - image: cvat/server:${CVAT_VERSION:-v2.5.2} + image: cvat/server:${CVAT_VERSION:-v2.6.0} restart: always depends_on: - cvat_redis @@ -182,7 +182,7 @@ services: cvat_worker_quality_reports: container_name: cvat_worker_quality_reports - image: cvat/server:${CVAT_VERSION:-v2.5.2} + image: cvat/server:${CVAT_VERSION:-v2.6.0} restart: always depends_on: - cvat_redis @@ -204,7 +204,7 @@ services: cvat_worker_analytics_reports: container_name: cvat_worker_analytics_reports - image: cvat/server:${CVAT_VERSION:-v2.5.2} + image: cvat/server:${CVAT_VERSION:-v2.6.0} restart: always depends_on: - cvat_redis @@ -227,7 +227,7 @@ services: cvat_ui: container_name: cvat_ui - image: cvat/ui:${CVAT_VERSION:-v2.5.2} + image: cvat/ui:${CVAT_VERSION:-v2.6.0} restart: always depends_on: - cvat_server diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index aa63541a37da..0338799e72e2 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -95,7 +95,7 @@ cvat: additionalVolumeMounts: [] replicas: 1 image: cvat/server - tag: v2.5.2 + tag: v2.6.0 imagePullPolicy: Always permissionFix: enabled: true @@ -119,7 +119,7 @@ cvat: frontend: replicas: 1 image: cvat/ui - tag: v2.5.2 + tag: v2.6.0 imagePullPolicy: Always labels: {} # test: test diff --git a/serverless/onnx/WongKinYiu/yolov7/nuclio/function-gpu.yaml b/serverless/onnx/WongKinYiu/yolov7/nuclio/function-gpu.yaml index 45f410aeb7ee..199c0d0c2f86 100644 --- a/serverless/onnx/WongKinYiu/yolov7/nuclio/function-gpu.yaml +++ b/serverless/onnx/WongKinYiu/yolov7/nuclio/function-gpu.yaml @@ -96,18 +96,18 @@ spec: eventTimeout: 30s build: image: cvat.onnx.wongkinyiu.yolov7 - baseImage: ultralytics/yolov5:latest + baseImage: nvidia/cuda:12.2.0-runtime-ubuntu22.04 directives: preCopy: - kind: USER value: root - kind: RUN - value: apt update && apt install --no-install-recommends -y libglib2.0-0 wget + value: apt update && apt install --no-install-recommends -y wget python3-pip - kind: WORKDIR value: /opt/nuclio - kind: RUN - value: pip install onnxruntime + value: pip install onnxruntime opencv-python-headless pillow pyyaml - kind: WORKDIR value: /opt/nuclio - kind: RUN diff --git a/serverless/onnx/WongKinYiu/yolov7/nuclio/function.yaml b/serverless/onnx/WongKinYiu/yolov7/nuclio/function.yaml index 328e2bbd64fc..5642ce58fb1b 100644 --- a/serverless/onnx/WongKinYiu/yolov7/nuclio/function.yaml +++ b/serverless/onnx/WongKinYiu/yolov7/nuclio/function.yaml @@ -95,16 +95,16 @@ spec: eventTimeout: 30s build: image: cvat.onnx.wongkinyiu.yolov7 - baseImage: ultralytics/yolov5:latest-cpu + baseImage: ubuntu:22.04 directives: preCopy: - kind: USER value: root - kind: RUN - value: apt update && apt install --no-install-recommends -y libglib2.0-0 && apt install wget + value: apt update && apt install --no-install-recommends -y wget python3-pip - kind: RUN - value: pip install onnxruntime + value: pip install onnxruntime opencv-python-headless pillow pyyaml - kind: WORKDIR value: /opt/nuclio - kind: RUN diff --git a/serverless/pytorch/ultralytics/yolov5/nuclio/function-gpu.yaml b/serverless/pytorch/ultralytics/yolov5/nuclio/function-gpu.yaml deleted file mode 100644 index d2caf2c97833..000000000000 --- a/serverless/pytorch/ultralytics/yolov5/nuclio/function-gpu.yaml +++ /dev/null @@ -1,127 +0,0 @@ -metadata: - name: pth-ultralytics-yolov5 - namespace: cvat - annotations: - name: YOLO v5 - type: detector - framework: pytorch - spec: | - [ - { "id": 0, "name": "person" }, - { "id": 1, "name": "bicycle" }, - { "id": 2, "name": "car" }, - { "id": 3, "name": "motorbike" }, - { "id": 4, "name": "aeroplane" }, - { "id": 5, "name": "bus" }, - { "id": 6, "name": "train" }, - { "id": 7, "name": "truck" }, - { "id": 8, "name": "boat" }, - { "id": 9, "name": "traffic light" }, - { "id": 10, "name": "fire hydrant" }, - { "id": 11, "name": "stop sign" }, - { "id": 12, "name": "parking meter" }, - { "id": 13, "name": "bench" }, - { "id": 14, "name": "bird" }, - { "id": 15, "name": "cat" }, - { "id": 16, "name": "dog" }, - { "id": 17, "name": "horse" }, - { "id": 18, "name": "sheep" }, - { "id": 19, "name": "cow" }, - { "id": 20, "name": "elephant" }, - { "id": 21, "name": "bear" }, - { "id": 22, "name": "zebra" }, - { "id": 23, "name": "giraffe" }, - { "id": 24, "name": "backpack" }, - { "id": 25, "name": "umbrella" }, - { "id": 26, "name": "handbag" }, - { "id": 27, "name": "tie" }, - { "id": 28, "name": "suitcase" }, - { "id": 29, "name": "frisbee" }, - { "id": 30, "name": "skis" }, - { "id": 31, "name": "snowboard" }, - { "id": 32, "name": "sports ball" }, - { "id": 33, "name": "kite" }, - { "id": 34, "name": "baseball bat" }, - { "id": 35, "name": "baseball glove" }, - { "id": 36, "name": "skateboard" }, - { "id": 37, "name": "surfboard" }, - { "id": 38, "name": "tennis racket" }, - { "id": 39, "name": "bottle" }, - { "id": 40, "name": "wine glass" }, - { "id": 41, "name": "cup" }, - { "id": 42, "name": "fork" }, - { "id": 43, "name": "knife" }, - { "id": 44, "name": "spoon" }, - { "id": 45, "name": "bowl" }, - { "id": 46, "name": "banana" }, - { "id": 47, "name": "apple" }, - { "id": 48, "name": "sandwich" }, - { "id": 49, "name": "orange" }, - { "id": 50, "name": "broccoli" }, - { "id": 51, "name": "carrot" }, - { "id": 52, "name": "hot dog" }, - { "id": 53, "name": "pizza" }, - { "id": 54, "name": "donut" }, - { "id": 55, "name": "cake" }, - { "id": 56, "name": "chair" }, - { "id": 57, "name": "sofa" }, - { "id": 58, "name": "pottedplant" }, - { "id": 59, "name": "bed" }, - { "id": 60, "name": "diningtable" }, - { "id": 61, "name": "toilet" }, - { "id": 62, "name": "tvmonitor" }, - { "id": 63, "name": "laptop" }, - { "id": 64, "name": "mouse" }, - { "id": 65, "name": "remote" }, - { "id": 66, "name": "keyboard" }, - { "id": 67, "name": "cell phone" }, - { "id": 68, "name": "microwave" }, - { "id": 69, "name": "oven" }, - { "id": 70, "name": "toaster" }, - { "id": 71, "name": "sink" }, - { "id": 72, "name": "refrigerator" }, - { "id": 73, "name": "book" }, - { "id": 74, "name": "clock" }, - { "id": 75, "name": "vase" }, - { "id": 76, "name": "scissors" }, - { "id": 77, "name": "teddy bear" }, - { "id": 78, "name": "hair drier" }, - { "id": 79, "name": "toothbrush" } - ] - -spec: - description: YOLO v5 via pytorch hub - runtime: 'python:3.6' - handler: main:handler - eventTimeout: 30s - build: - image: cvat.pth.ultralytics.yolov5 - baseImage: ultralytics/yolov5:latest - - directives: - preCopy: - - kind: USER - value: root - - kind: RUN - value: apt update && apt install --no-install-recommends -y libglib2.0-0 - - kind: WORKDIR - value: /opt/nuclio - - triggers: - myHttpTrigger: - maxWorkers: 1 - kind: 'http' - workerAvailabilityTimeoutMilliseconds: 10000 - attributes: - maxRequestBodySize: 33554432 # 32MB - - resources: - limits: - nvidia.com/gpu: 1 - - platform: - attributes: - restartPolicy: - name: always - maximumRetryCount: 3 - mountMode: volume diff --git a/serverless/pytorch/ultralytics/yolov5/nuclio/function.yaml b/serverless/pytorch/ultralytics/yolov5/nuclio/function.yaml deleted file mode 100644 index 22e794d1d86e..000000000000 --- a/serverless/pytorch/ultralytics/yolov5/nuclio/function.yaml +++ /dev/null @@ -1,123 +0,0 @@ -metadata: - name: pth-ultralytics-yolov5 - namespace: cvat - annotations: - name: YOLO v5 - type: detector - framework: pytorch - spec: | - [ - { "id": 0, "name": "person" }, - { "id": 1, "name": "bicycle" }, - { "id": 2, "name": "car" }, - { "id": 3, "name": "motorbike" }, - { "id": 4, "name": "aeroplane" }, - { "id": 5, "name": "bus" }, - { "id": 6, "name": "train" }, - { "id": 7, "name": "truck" }, - { "id": 8, "name": "boat" }, - { "id": 9, "name": "traffic light" }, - { "id": 10, "name": "fire hydrant" }, - { "id": 11, "name": "stop sign" }, - { "id": 12, "name": "parking meter" }, - { "id": 13, "name": "bench" }, - { "id": 14, "name": "bird" }, - { "id": 15, "name": "cat" }, - { "id": 16, "name": "dog" }, - { "id": 17, "name": "horse" }, - { "id": 18, "name": "sheep" }, - { "id": 19, "name": "cow" }, - { "id": 20, "name": "elephant" }, - { "id": 21, "name": "bear" }, - { "id": 22, "name": "zebra" }, - { "id": 23, "name": "giraffe" }, - { "id": 24, "name": "backpack" }, - { "id": 25, "name": "umbrella" }, - { "id": 26, "name": "handbag" }, - { "id": 27, "name": "tie" }, - { "id": 28, "name": "suitcase" }, - { "id": 29, "name": "frisbee" }, - { "id": 30, "name": "skis" }, - { "id": 31, "name": "snowboard" }, - { "id": 32, "name": "sports ball" }, - { "id": 33, "name": "kite" }, - { "id": 34, "name": "baseball bat" }, - { "id": 35, "name": "baseball glove" }, - { "id": 36, "name": "skateboard" }, - { "id": 37, "name": "surfboard" }, - { "id": 38, "name": "tennis racket" }, - { "id": 39, "name": "bottle" }, - { "id": 40, "name": "wine glass" }, - { "id": 41, "name": "cup" }, - { "id": 42, "name": "fork" }, - { "id": 43, "name": "knife" }, - { "id": 44, "name": "spoon" }, - { "id": 45, "name": "bowl" }, - { "id": 46, "name": "banana" }, - { "id": 47, "name": "apple" }, - { "id": 48, "name": "sandwich" }, - { "id": 49, "name": "orange" }, - { "id": 50, "name": "broccoli" }, - { "id": 51, "name": "carrot" }, - { "id": 52, "name": "hot dog" }, - { "id": 53, "name": "pizza" }, - { "id": 54, "name": "donut" }, - { "id": 55, "name": "cake" }, - { "id": 56, "name": "chair" }, - { "id": 57, "name": "sofa" }, - { "id": 58, "name": "pottedplant" }, - { "id": 59, "name": "bed" }, - { "id": 60, "name": "diningtable" }, - { "id": 61, "name": "toilet" }, - { "id": 62, "name": "tvmonitor" }, - { "id": 63, "name": "laptop" }, - { "id": 64, "name": "mouse" }, - { "id": 65, "name": "remote" }, - { "id": 66, "name": "keyboard" }, - { "id": 67, "name": "cell phone" }, - { "id": 68, "name": "microwave" }, - { "id": 69, "name": "oven" }, - { "id": 70, "name": "toaster" }, - { "id": 71, "name": "sink" }, - { "id": 72, "name": "refrigerator" }, - { "id": 73, "name": "book" }, - { "id": 74, "name": "clock" }, - { "id": 75, "name": "vase" }, - { "id": 76, "name": "scissors" }, - { "id": 77, "name": "teddy bear" }, - { "id": 78, "name": "hair drier" }, - { "id": 79, "name": "toothbrush" } - ] - -spec: - description: YOLO v5 via pytorch hub - runtime: 'python:3.6' - handler: main:handler - eventTimeout: 30s - build: - image: cvat.pth.ultralytics.yolov5 - baseImage: ultralytics/yolov5:latest-cpu - - directives: - preCopy: - - kind: USER - value: root - - kind: RUN - value: apt update && apt install --no-install-recommends -y libglib2.0-0 - - kind: WORKDIR - value: /opt/nuclio - - triggers: - myHttpTrigger: - maxWorkers: 2 - kind: 'http' - workerAvailabilityTimeoutMilliseconds: 10000 - attributes: - maxRequestBodySize: 33554432 # 32MB - - platform: - attributes: - restartPolicy: - name: always - maximumRetryCount: 3 - mountMode: volume diff --git a/serverless/pytorch/ultralytics/yolov5/nuclio/main.py b/serverless/pytorch/ultralytics/yolov5/nuclio/main.py deleted file mode 100644 index 92bcf2e02a32..000000000000 --- a/serverless/pytorch/ultralytics/yolov5/nuclio/main.py +++ /dev/null @@ -1,40 +0,0 @@ -import json -import base64 -from PIL import Image -import io -import torch - -def init_context(context): - context.logger.info("Init context... 0%") - - # Read the DL model - model = torch.hub.load('ultralytics/yolov5', 'yolov5s') # or yolov5m, yolov5l, yolov5x, custom - context.user_data.model = model - - context.logger.info("Init context...100%") - -def handler(context, event): - context.logger.info("Run yolo-v5 model") - data = event.body - buf = io.BytesIO(base64.b64decode(data["image"])) - threshold = float(data.get("threshold", 0.5)) - context.user_data.model.conf = threshold - image = Image.open(buf) - yolo_results_json = context.user_data.model(image).pandas().xyxy[0].to_dict(orient='records') - - encoded_results = [] - for result in yolo_results_json: - encoded_results.append({ - 'confidence': result['confidence'], - 'label': result['name'], - 'points': [ - result['xmin'], - result['ymin'], - result['xmax'], - result['ymax'] - ], - 'type': 'rectangle' - }) - - return context.Response(body=json.dumps(encoded_results), headers={}, - content_type='application/json', status_code=200) diff --git a/site/content/en/docs/api_sdk/cli/_index.md b/site/content/en/docs/api_sdk/cli/_index.md index c44596a8983e..bca0a0702642 100644 --- a/site/content/en/docs/api_sdk/cli/_index.md +++ b/site/content/en/docs/api_sdk/cli/_index.md @@ -30,7 +30,7 @@ To install an [official release of CVAT CLI](https://pypi.org/project/cvat-cli/) pip install cvat-cli ``` -We support Python versions 3.7 - 3.9. +We support Python versions 3.8 and higher. ## Usage @@ -39,12 +39,12 @@ You can get help with `cvat-cli --help`. ``` usage: cvat-cli [-h] [--version] [--insecure] [--auth USER:[PASS]] [--server-host SERVER_HOST] [--server-port SERVER_PORT] [--organization SLUG] [--debug] - {create,delete,ls,frames,dump,upload,export,import} ... + {create,delete,ls,frames,dump,upload,export,import,auto-annotate} ... Perform common operations related to CVAT tasks. positional arguments: - {create,delete,ls,frames,dump,upload,export,import} + {create,delete,ls,frames,dump,upload,export,import,auto-annotate} options: -h, --help show this help message and exit @@ -230,3 +230,71 @@ by using the [label constructor](/docs/manual/basics/creating_an_annotation_task ```bash cvat-cli import task_backup.zip ``` + +### Auto-annotate + +This command provides a command-line interface +to the [auto-annotation API](/docs/api_sdk/sdk/auto-annotation). + +It can auto-annotate using AA functions implemented in one of the following ways: + +1. As a Python module directly implementing the AA function protocol. + Such a module must define the required attributes at the module level. + + For example: + + ```python + import cvat_sdk.auto_annotation as cvataa + + spec = cvataa.DetectionFunctionSpec(...) + + def detect(context, image): + ... + ``` + +1. As a Python module implementing a factory function named `create`. + This function must return an object implementing the AA function protocol. + Any parameters specified on the command line using the `-p` option + will be passed to `create`. + + For example: + + ```python + import cvat_sdk.auto_annotation as cvataa + + class _MyFunction: + def __init__(...): + ... + + spec = cvataa.DetectionFunctionSpec(...) + + def detect(context, image): + ... + + def create(...) -> cvataa.DetectionFunction: + return _MyFunction(...) + ``` + +- Annotate the task with id 137 with the predefined torchvision detection function, + which is parameterized: + ```bash + cvat-cli auto-annotate 137 --function-module cvat_sdk.auto_annotation.functions.torchvision_detection \ + -p model_name=str:fasterrcnn_resnet50_fpn_v2 -p box_score_thresh=float:0.5 + ``` + +- Annotate the task with id 138 with an AA function defined in `my_func.py`: + ```bash + cvat-cli auto-annotate 138 --function-file path/to/my_func.py + ``` + +Note that this command does not modify the Python module search path. +If your function module needs to import other local modules, +you must add your module directory to the search path +if it isn't there already. + +- Annotate the task with id 139 with a function defined in the `my_func` module + located in the `my-project` directory, + letting it import other modules from that directory. + ```bash + PYTHONPATH=path/to/my-project cvat-cli auto-annotate 139 --function-module my_func + ``` diff --git a/site/content/en/docs/api_sdk/sdk/_index.md b/site/content/en/docs/api_sdk/sdk/_index.md index 1b9695ea7f21..025130ba3963 100644 --- a/site/content/en/docs/api_sdk/sdk/_index.md +++ b/site/content/en/docs/api_sdk/sdk/_index.md @@ -15,6 +15,7 @@ SDK API includes several layers: - Low-level API with REST API wrappers. Located at `cvat_sdk.api_client`. [Read more](/docs/api_sdk/sdk/lowlevel-api) - High-level API. Located at `cvat_sdk.core`. [Read more](/docs/api_sdk/sdk/highlevel-api) - PyTorch adapter. Located at `cvat_sdk.pytorch`. [Read more](/docs/api_sdk/sdk/pytorch-adapter) +- Auto-annotation API. Located at `cvat_sdk.auto_annotation.` [Read more](/docs/api_sdk/sdk/auto-annotation) In general, the low-level API provides single-request operations, while the high-level one implements composite, multi-request operations, and provides local proxies for server objects. @@ -25,6 +26,11 @@ The PyTorch adapter is a specialized layer that represents datasets stored in CVAT as PyTorch `Dataset` objects. This enables direct use of such datasets in PyTorch-based machine learning pipelines. +The auto-annotation API is a specialized layer +that lets you automatically annotate CVAT datasets +by running a custom function on the local machine. +See also the `auto-annotate` command in the CLI. + ## Installation To install an [official release of CVAT SDK](https://pypi.org/project/cvat-sdk/) use this command: @@ -38,7 +44,7 @@ To use the PyTorch adapter, request the `pytorch` extra: pip install "cvat-sdk[pytorch]" ``` -We support Python versions 3.7 - 3.9. +We support Python versions 3.8 and higher. ## Usage diff --git a/site/content/en/docs/api_sdk/sdk/auto-annotation.md b/site/content/en/docs/api_sdk/sdk/auto-annotation.md new file mode 100644 index 000000000000..b85ab7b067b9 --- /dev/null +++ b/site/content/en/docs/api_sdk/sdk/auto-annotation.md @@ -0,0 +1,253 @@ +--- +title: 'Auto-annotation API' +linkTitle: 'Auto-annotation API' +weight: 6 +--- + +## Overview + +This layer provides functionality that allows you to automatically annotate a CVAT dataset +by running a custom function on your local machine. +A function, in this context, is a Python object that implements a particular protocol +defined by this layer. +To avoid confusion with Python functions, +auto-annotation functions will be referred to as "AA functions" in the following text. +A typical AA function will be based on a machine learning model +and consist of the following basic elements: + +- Code to load the ML model. + +- A specification describing the annotations that the AA function can produce. + +- Code to convert data from CVAT to a format the ML model can understand. + +- Code to run the ML model. + +- Code to convert resulting annotations to a format CVAT can understand. + +The layer can be divided into several parts: + +- The interface, containing the protocol that an AA function must implement. + +- The driver, containing functionality to annotate a CVAT dataset using an AA function. + +- The predefined AA function based on Ultralytics YOLOv8n. + +The `auto-annotate` CLI command provides a way to use an AA function from the command line +rather than from a Python program. +See [the CLI documentation](/docs/api_sdk/cli/) for details. + +## Example + +```python +from typing import List +import PIL.Image + +import torchvision.models + +from cvat_sdk import make_client +import cvat_sdk.models as models +import cvat_sdk.auto_annotation as cvataa + +class TorchvisionDetectionFunction: + def __init__(self, model_name: str, weights_name: str, **kwargs) -> None: + # load the ML model + weights_enum = torchvision.models.get_model_weights(model_name) + self._weights = weights_enum[weights_name] + self._transforms = self._weights.transforms() + self._model = torchvision.models.get_model(model_name, weights=self._weights, **kwargs) + self._model.eval() + + @property + def spec(self) -> cvataa.DetectionFunctionSpec: + # describe the annotations + return cvataa.DetectionFunctionSpec( + labels=[ + cvataa.label_spec(cat, i) + for i, cat in enumerate(self._weights.meta['categories']) + ] + ) + + def detect(self, context, image: PIL.Image.Image) -> List[models.LabeledShapeRequest]: + # convert the input into a form the model can understand + transformed_image = [self._transforms(image)] + + # run the ML model + results = self._model(transformed_image) + + # convert the results into a form CVAT can understand + return [ + cvataa.rectangle(label.item(), [x.item() for x in box]) + for result in results + for box, label in zip(result['boxes'], result['labels']) + ] + +# log into the CVAT server +with make_client(host="localhost", credentials=("user", "password")) as client: + # annotate task 12345 using Faster R-CNN + cvataa.annotate_task(client, 41617, + TorchvisionDetectionFunction("fasterrcnn_resnet50_fpn_v2", "DEFAULT", box_score_thresh=0.5), + ) +``` + +## Auto-annotation interface + +Currently, the only type of AA function supported by this layer is the detection function. +Therefore, all of the following information will pertain to detection functions. + +A detection function accepts an image and returns a list of shapes found in that image. +When it is applied to a dataset, the AA function is run for every image, +and the resulting lists of shapes are combined and uploaded to CVAT. + +A detection function must have two attributes, `spec` and `detect`. + +`spec` must contain the AA function's specification, +which is an instance of `DetectionFunctionSpec`. + +`DetectionFunctionSpec` must be initialized with a sequence of `PatchedLabelRequest` objects +that represent the labels that the AA function knows about. +See the docstring of `DetectionFunctionSpec` for more information on the constraints +that these objects must follow. + +`detect` must be a function/method accepting two parameters: + +- `context` (`DetectionFunctionContext`). + Contains information about the current image. + Currently `DetectionFunctionContext` only contains a single field, `frame_name`, + which contains the file name of the frame on the CVAT server. + +- `image` (`PIL.Image.Image`). + Contains image data. + +`detect` must return a list of `LabeledShapeRequest` objects, +representing shapes found in the image. +See the docstring of `DetectionFunctionSpec` for more information on the constraints +that these objects must follow. + +The same AA function may be used with any dataset that contain labels with the same name +as the AA function's specification. +The way it works is that the driver matches labels between the spec and the dataset, +and replaces the label IDs in the shape objects with those defined in the dataset. + +For example, suppose the AA function's spec defines the following labels: + +| Name | ID | +|-------|----| +| `bat` | 0 | +| `rat` | 1 | + +And the dataset defines the following labels: + +| Name | ID | +|-------|-----| +| `bat` | 100 | +| `cat` | 101 | +| `rat` | 102 | + +Then suppose `detect` returns a shape with `label_id` equal to 1. +The driver will see that it refers to the `rat` label, and replace it with 102, +since that's the ID this label has in the dataset. + +The same logic is used for sub-label IDs. + +### Helper factory functions + +The CVAT API model types used in the AA function protocol are somewhat unwieldy to work with, +so it's recommented to use the helper factory functions provided by this layer. +These helpers instantiate an object of their corresponding model type, +passing their arguments to the model constructor +and sometimes setting some attributes to fixed values. + +The following helpers are available for building specifications: + +| Name | Model type | Fixed attributes | +|-----------------------|-----------------------|-------------------| +| `label_spec` | `PatchedLabelRequest` | - | +| `skeleton_label_spec` | `PatchedLabelRequest` | `type="skeleton"` | +| `keypoint_spec` | `SublabelRequest` | - | + +The following helpers are available for use in `detect`: + +| Name | Model type | Fixed attributes | +|-------------|--------------------------|-------------------------------| +| `shape` | `LabeledShapeRequest` | `frame=0` | +| `rectangle` | `LabeledShapeRequest` | `frame=0`, `type="rectangle"` | +| `skeleton` | `LabeledShapeRequest` | `frame=0`, `type="skeleton"` | +| `keypoint` | `SubLabeledShapeRequest` | `frame=0`, `type="points"` | + +## Auto-annotation driver + +The `annotate_task` function uses an AA function to annotate a CVAT task. +It must be called as follows: + +```python +annotate_task(, , , ) +``` + +The supplied client will be used to make all API calls. + +By default, new annotations will be appended to the old ones. +Use `clear_existing=True` to remove old annotations instead. + +If a detection function declares a label that has no matching label in the task, +then by default, `BadFunctionError` is raised, and auto-annotation is aborted. +If you use `allow_unmatched_label=True`, then such labels will be ignored, +and any shapes referring to them will be dropped. +Same logic applies to sub-label IDs. + +`annotate_task` will raise a `BadFunctionError` exception +if it detects that the function violated the AA function protocol. + +## Predefined AA functions + +This layer includes several predefined AA functions. +You can use them as-is, or as a base on which to build your own. + +Each function is implemented as a module +to allow usage via the CLI `auto-annotate` command. +Therefore, in order to use it from the SDK, +you'll need to import the corresponding module. + +### `cvat_sdk.auto_annotation.functions.torchvision_detection` + +This AA function uses object detection models from +the [torchvision](https://pytorch.org/vision/stable/index.html) library. +It produces rectangle annotations. + +To use it, install CVAT SDK with the `pytorch` extra: + +``` +$ pip install "cvat-sdk[pytorch]" +``` + +Usage from Python: + +```python +from cvat_sdk.auto_annotation.functions.torchvision_detection import create as create_torchvision +annotate_task(, , create_torchvision(, ...)) +``` + +Usage from the CLI: + +```bash +cvat-cli auto-annotate "" --function-module cvat_sdk.auto_annotation.functions.torchvision_detection \ + -p model_name=str:"" ... +``` + +The `create` function accepts the following parameters: + +- `model_name` (`str`) - the name of the model, such as `fasterrcnn_resnet50_fpn_v2`. + This parameter is required. +- `weights_name` (`str`) - the name of a weights enum value for the model, such as `COCO_V1`. + Defaults to `DEFAULT`. + +It also accepts arbitrary additional parameters, +which are passed directly to the model constructor. + +### `cvat_sdk.auto_annotation.functions.torchvision_keypoint_detection` + +This AA function is analogous to `torchvision_detection`, +except it uses torchvision's keypoint detection models and produces skeleton annotations. +Keypoints which the model marks as invisible will be marked as occluded in CVAT. + +Refer to the previous section for usage instructions and parameter information. diff --git a/site/content/en/docs/enterprise/subscription-managment.md b/site/content/en/docs/enterprise/subscription-managment.md index 2dcfca9130b3..4d14acfd6acb 100644 --- a/site/content/en/docs/enterprise/subscription-managment.md +++ b/site/content/en/docs/enterprise/subscription-managment.md @@ -17,21 +17,22 @@ See: - [Billing](#billing) - [Pro plan](#pro-plan) - [Team plan](#team-plan) -- [Change payment method](#change-payment-method) +- [Payment methods](#payment-methods) + - [Paying with bank transfer](#paying-with-bank-transfer) + - [Change payment method on Pro plan](#change-payment-method-on-pro-plan) + - [Change payment method on Team plan](#change-payment-method-on-team-plan) +- [Adding and removing team members](#adding-and-removing-team-members) - [Pro plan](#pro-plan-1) - [Team plan](#team-plan-1) -- [Adding and removing team members](#adding-and-removing-team-members) - - [Pro plan](#pro-plan-2) - - [Team plan](#team-plan-2) - [Change plan](#change-plan) - [Can I subscribe to several plans?](#can-i-subscribe-to-several-plans) - [Cancel plan](#cancel-plan) - [What will happen to my data?](#what-will-happen-to-my-data) + - [Pro plan](#pro-plan-2) + - [Team plan](#team-plan-2) +- [Plan renewal](#plan-renewal) - [Pro plan](#pro-plan-3) - [Team plan](#team-plan-3) -- [Plan renewal](#plan-renewal) - - [Pro plan](#pro-plan-4) - - [Team plan](#team-plan-4) ## Billing @@ -44,28 +45,50 @@ see: [Pricing Plans](https://www.cvat.ai/post/new-pricing-plans) ### Pro plan **Account/Month**: The **Pro** plan has a fixed price and is -designed for personal use only. It doesn't allow collaboration with team members, +designed **for personal use only**. It doesn't allow collaboration with team members, but removes all the other limits of the **Free** plan. +> **Note**: Although it allows the creation of an organization and +> access for up to 3 members -- it is _for trial purposes_ only, +> organization and members _will have all the limitations of the **Free** plan_. ### Team plan -**Member/month**: The **Team** plan allows you to create +**Member/ month**: The **Team** plan allows you to create an organization and add team members who can collaborate on projects. The **monthly payment for the plan depends on the number of team members you've added**. All limits of the **Free** plan will be removed. -## Change payment method +> **Note**: The organization owner is also part of the team. +> So, if you have three annotators working, you'll need to pay +> for 4 seats (3 annotators + 1 organization owner). + +## Payment methods This section describes how to change or add payment methods. -### Pro plan +### Paying with bank transfer + +> **Note** at the moment this method of payment +> work only with US banks. + +To pay with bank transfer: + +1. Go to the **Upgrade to Pro**/**Team plan**> **Get started**. +2. Click **US Bank Transfer**. +3. Upon successful completion of the payment, the you will receive a receipt via email. + +> **Note** that the completion of the payment process may take up to three banking days. + +![Bank Transfer Payment](/images/bank_transfer_payment.jpg) + +### Change payment method on Pro plan Access Manage **Pro Plan** > **Manage** and click **+Add Payment Method** ![Payment pro](/images/update_payment_pro.png) -### Team plan +### Change payment method on Team plan Access **Manage Team Plan** > **Manage** and click **+Add Payment Method**. diff --git a/site/content/en/docs/manual/advanced/ai-tools.md b/site/content/en/docs/manual/advanced/ai-tools.md index 3c355eb745dd..feab483e901d 100644 --- a/site/content/en/docs/manual/advanced/ai-tools.md +++ b/site/content/en/docs/manual/advanced/ai-tools.md @@ -204,7 +204,6 @@ see [Automatic annotation](/docs/manual/advanced/automatic-annotation/). | Mask RCNN | The model generates polygons for each instance of an object in the image.

For more information, see:
  • [GitHub: Mask RCNN](https://github.com/matterport/Mask_RCNN)
  • [Paper: Mask RCNN](https://arxiv.org/pdf/1703.06870.pdf) | | Faster RCNN | The model generates bounding boxes for each instance of an object in the image.
    In this model, RPN and Fast R-CNN are combined into a single network.

    For more information, see:
  • [GitHub: Faster RCNN](https://github.com/ShaoqingRen/faster_rcnn)
  • [Paper: Faster RCNN](https://arxiv.org/pdf/1506.01497.pdf) | | YOLO v3 | YOLO v3 is a family of object detection architectures and models pre-trained on the COCO dataset.

    For more information, see:
  • [GitHub: YOLO v3](https://github.com/ultralytics/yolov3)
  • [Site: YOLO v3](https://docs.ultralytics.com/#yolov3)
  • [Paper: YOLO v3](https://arxiv.org/pdf/1804.02767v1.pdf) | -| YOLO v5 | YOLO v5 is a family of object detection architectures and models based on the Pytorch framework.

    For more information, see:
  • [GitHub: YOLO v5](https://github.com/ultralytics/yolov5)
  • [Site: YOLO v5](https://docs.ultralytics.com/#yolov5) | | Semantic segmentation for ADAS | This is a segmentation network to classify each pixel into 20 classes.

    For more information, see:
  • [Site: ADAS](https://docs.openvino.ai/2019_R1/_semantic_segmentation_adas_0001_description_semantic_segmentation_adas_0001.html) | | Mask RCNN with Tensorflow | Mask RCNN version with Tensorflow. The model generates polygons for each instance of an object in the image.

    For more information, see:
  • [GitHub: Mask RCNN](https://github.com/matterport/Mask_RCNN)
  • [Paper: Mask RCNN](https://arxiv.org/pdf/1703.06870.pdf) | | Faster RCNN with Tensorflow | Faster RCNN version with Tensorflow. The model generates bounding boxes for each instance of an object in the image.
    In this model, RPN and Fast R-CNN are combined into a single network.

    For more information, see:
  • [Site: Faster RCNN with Tensorflow](https://docs.openvino.ai/2021.4/omz_models_model_faster_rcnn_inception_v2_coco.html)
  • [Paper: Faster RCNN](https://arxiv.org/pdf/1506.01497.pdf) | diff --git a/site/content/en/docs/manual/advanced/analytics-and-monitoring/_index.md b/site/content/en/docs/manual/advanced/analytics-and-monitoring/_index.md new file mode 100644 index 000000000000..3f054436977f --- /dev/null +++ b/site/content/en/docs/manual/advanced/analytics-and-monitoring/_index.md @@ -0,0 +1,6 @@ +--- +title: 'CVAT Analytics and quality assessment in Cloud' +linkTitle: 'Analytics and quality assessment' +weight: 14 +description: 'Analytics and quality assessment in CVAT Cloud' +--- diff --git a/site/content/en/docs/manual/advanced/analytics-and-monitoring/analytics-in-cloud.md b/site/content/en/docs/manual/advanced/analytics-and-monitoring/analytics-in-cloud.md new file mode 100644 index 000000000000..4bc28cd8e264 --- /dev/null +++ b/site/content/en/docs/manual/advanced/analytics-and-monitoring/analytics-in-cloud.md @@ -0,0 +1,54 @@ +--- +title: 'CVAT Performance & Monitoring' +linkTitle: 'Performance & Monitoring' +weight: 2 +description: 'How to monitor team activity and performance in CVAT' +--- + +In CVAT Cloud, you can track a variety of metrics +reflecting the team's productivity and the pace of annotation with +the **Performance** feature. + +See: + +- [Performance dashboard](#performance-dashboard) +- [Performance video tutorial](#performance-video-tutorial) + +## Performance dashboard + +To open the **Performance** dashboard, do the following: + +1. In the top menu click on **Projects**/ **Tasks**/ **Jobs**. +2. Select an item from the list, and click on three dots (![Open menu](/images/openmenu.jpg)). +3. From the menu, select **View analytics** > **Performance** tab. + +![Open menu](/images/viewanalytics.jpg) + +The following dashboard will open: + +![Open menu](/images/performance_dashboard.jpg) + +The **Performance** dashboard has the following elements: + + + +| Element | Description | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------- | +| Analytics for | **Object**/ **Task**/ **Job** number. | +| Created | Time when the dashboard was updated last time. | +| Objects | Graph, showing the number of annotated, updated, and deleted objects by day. | +| Annotation speed (objects per hour) | Number of objects annotated per hour. | +| Time | A drop-down list with various periods for the graph. Currently affects only the histogram data. | +| Annotation time (hours) | Shows for how long the **Project**/**Task**/**Job** is in **In progress** state. | +| Total objects count | Shows the total objects count in the task. Interpolated objects are counted. | +| Total annotation speed (objects per hour) | Shows the annotation speed in the **Project**/**Task**/**Job**. Interpolated objects are counted. | + + + +You can rearrange elements of the dashboard by dragging and dropping each of them. + +## Performance video tutorial + +This video demonstrates the process: + + diff --git a/site/content/en/docs/manual/advanced/annotation-quality.md b/site/content/en/docs/manual/advanced/analytics-and-monitoring/annotation-quality.md similarity index 76% rename from site/content/en/docs/manual/advanced/annotation-quality.md rename to site/content/en/docs/manual/advanced/analytics-and-monitoring/annotation-quality.md index dd8cee145491..a3d613c06f7b 100644 --- a/site/content/en/docs/manual/advanced/annotation-quality.md +++ b/site/content/en/docs/manual/advanced/analytics-and-monitoring/annotation-quality.md @@ -1,7 +1,7 @@ --- title: 'Annotation quality & Honeypot' linkTitle: 'Annotation quality' -weight: 14 +weight: 1 description: 'How to check the quality of annotation in CVAT' --- @@ -187,20 +187,19 @@ Annotation quality settings have the following parameters: -| Field | Description | -| ---------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Min overlap threshold | Min overlap threshold(IoU) is used for the distinction between matched / unmatched shapes. | -| Low overlap threshold | Low overlap threshold is used for the distinction between strong/weak (low overlap) matches. | -| OKS Sigma | IoU threshold for points. The percent of the box area, used as the radius of the circle around the GT point, where the checked point is expected to be. | -| Relative thickness (frame side %) | Thickness of polylines, relative to the (image area) ^ 0.5. The distance to the boundary around the GT line inside of which the checked line points should be. | -| Check orientation | Indicates that polylines have direction. | -| Min similarity gain (%) | The minimal gain in the GT IoU between the given and reversed line directions to consider the line inverted. Only useful with the Check orientation parameter. | -| Compare groups | Enables or disables annotation group checks. | -| Min group match threshold | Minimal IoU for groups to be considered matching, used when the Compare groups are enabled. | -| Check object visibility | Check for partially-covered annotations. Masks and polygons will be compared to each other. | -| Min visibility threshold | Minimal visible area percent of the spatial annotations (polygons, masks) | -| For reporting covered annotations, useful with the Check object visibility option. | -| Match only visible parts | Use only the visible part of the masks and polygons in comparisons. | +| Field | Description | +| --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Min overlap threshold | Min overlap threshold(IoU) is used for the distinction between matched / unmatched shapes. | +| Low overlap threshold | Low overlap threshold is used for the distinction between strong/weak (low overlap) matches. | +| OKS Sigma | IoU threshold for points. The percent of the box area, used as the radius of the circle around the GT point, where the checked point is expected to be. | +| Relative thickness (frame side %) | Thickness of polylines, relative to the (image area) ^ 0.5. The distance to the boundary around the GT line inside of which the checked line points should be. | +| Check orientation | Indicates that polylines have direction. | +| Min similarity gain (%) | The minimal gain in the GT IoU between the given and reversed line directions to consider the line inverted. Only useful with the Check orientation parameter. | +| Compare groups | Enables or disables annotation group checks. | +| Min group match threshold | Minimal IoU for groups to be considered matching, used when the Compare groups are enabled. | +| Check object visibility | Check for partially-covered annotations. Masks and polygons will be compared to each other. | +| Min visibility threshold | Minimal visible area percent of the spatial annotations (polygons, masks). For reporting covered annotations, useful with the Check object visibility option. | +| Match only visible parts | Use only the visible part of the masks and polygons in comparisons. | diff --git a/site/content/en/docs/manual/advanced/automatic-annotation.md b/site/content/en/docs/manual/advanced/automatic-annotation.md index 3805509d7ad5..9f3c7955fad1 100644 --- a/site/content/en/docs/manual/advanced/automatic-annotation.md +++ b/site/content/en/docs/manual/advanced/automatic-annotation.md @@ -93,7 +93,6 @@ List of pre-installed models: | RetinaNet R101 | RetinaNet is a one-stage object detection model that utilizes a focal loss function to address class imbalance during training. Focal loss applies a modulating term to the cross entropy loss to focus learning on hard negative examples. RetinaNet is a single, unified network composed of a backbone network and two task-specific subnetworks.

    For more information, see:
  • [Site: RetinaNET](https://paperswithcode.com/lib/detectron2/retinanet) | | Text detection | Text detector based on PixelLink architecture with MobileNetV2, depth_multiplier=1.4 as a backbone for indoor/outdoor scenes.

    For more information, see:
  • [Site: OpenVINO Text detection 004](https://docs.openvino.ai/2022.3/omz_models_model_text_detection_0004.html) | | YOLO v3 | YOLO v3 is a family of object detection architectures and models pre-trained on the COCO dataset.

    For more information, see:
  • [Site: YOLO v3](https://docs.openvino.ai/2022.3/omz_models_model_yolo_v3_tf.html) | -| YOLO v5 | YOLO v5 is a family of object detection architectures and models based on the Pytorch framework.

    For more information, see:
  • [GitHub: YOLO v5](https://github.com/ultralytics/yolov5)
  • [Site: YOLO v5](https://docs.ultralytics.com/#yolov5) | | YOLO v7 | YOLOv7 is an advanced object detection model that outperforms other detectors in terms of both speed and accuracy. It can process frames at a rate ranging from 5 to 160 frames per second (FPS) and achieves the highest accuracy with 56.8% average precision (AP) among real-time object detectors running at 30 FPS or higher on the V100 graphics processing unit (GPU).

    For more information, see:
  • [GitHub: YOLO v7](https://github.com/WongKinYiu/yolov7)
  • [Paper: YOLO v7](https://arxiv.org/pdf/2207.02696.pdf) | diff --git a/site/content/en/docs/manual/advanced/formats/format-camvid.md b/site/content/en/docs/manual/advanced/formats/format-camvid.md index 1485188a5bc6..301d32b8fcdb 100644 --- a/site/content/en/docs/manual/advanced/formats/format-camvid.md +++ b/site/content/en/docs/manual/advanced/formats/format-camvid.md @@ -13,7 +13,7 @@ Downloaded file: a zip archive of the following structure: ```bash taskname.zip/ -├── labelmap.txt # optional, required for non-CamVid labels +├── label_colors.txt # optional, required for non-CamVid labels ├── / | ├── image1.png | └── image2.png @@ -22,13 +22,23 @@ taskname.zip/ | └── image2.png └── .txt -# labelmap.txt +# label_colors.txt (with color value type) +# if you want to manually set the color for labels, configure label_colors.txt as follows: # color (RGB) label 0 0 0 Void 64 128 64 Animal 192 0 128 Archway 0 128 192 Bicyclist 0 128 64 Bridge + +# label_colors.txt (without color value type) +# if you do not manually set the color for labels, it will be set automatically: +# label +Void +Animal +Archway +Bicyclist +Bridge ``` Mask is a `png` image with 1 or 3 channels where each pixel diff --git a/site/content/en/docs/manual/basics/create_an_annotation_task.md b/site/content/en/docs/manual/basics/create_an_annotation_task.md index 0275b9367b38..a756753a2f0b 100644 --- a/site/content/en/docs/manual/basics/create_an_annotation_task.md +++ b/site/content/en/docs/manual/basics/create_an_annotation_task.md @@ -153,6 +153,11 @@ To add an attribute, do the following: 4. In the **Attribute values** field, add attribute values.
    To separate values use **Enter**.
    To delete value, use **Backspace** or click **x** next to the value name. 5. (Optional) For mutable attributes, select **Mutable**. +6. (Optional) To set the default attribute, hover over it with mouse cursor and + click on it. The default attribute will change color to blue. + + ![Default attribute](/images/default_attribute.jpg) + To delete an attribute, click **Delete attribute**. diff --git a/site/content/en/images/bank_transfer_payment.jpg b/site/content/en/images/bank_transfer_payment.jpg new file mode 100644 index 000000000000..cb4b730d53dc Binary files /dev/null and b/site/content/en/images/bank_transfer_payment.jpg differ diff --git a/site/content/en/images/default_attribute.jpg b/site/content/en/images/default_attribute.jpg new file mode 100644 index 000000000000..ffa8871c07c8 Binary files /dev/null and b/site/content/en/images/default_attribute.jpg differ diff --git a/site/content/en/images/openmenu.jpg b/site/content/en/images/openmenu.jpg new file mode 100644 index 000000000000..c3b38dce4040 Binary files /dev/null and b/site/content/en/images/openmenu.jpg differ diff --git a/site/content/en/images/performance_dashboard.jpg b/site/content/en/images/performance_dashboard.jpg new file mode 100644 index 000000000000..174aaf066d6a Binary files /dev/null and b/site/content/en/images/performance_dashboard.jpg differ diff --git a/site/content/en/images/viewanalytics.jpg b/site/content/en/images/viewanalytics.jpg new file mode 100644 index 000000000000..08360b12efb7 Binary files /dev/null and b/site/content/en/images/viewanalytics.jpg differ diff --git a/supervisord/server.conf b/supervisord/server.conf index 88707249f2f3..5fba7a5e029c 100644 --- a/supervisord/server.conf +++ b/supervisord/server.conf @@ -39,7 +39,8 @@ process_name=%(program_name)s-%(process_num)s socket=unix:///tmp/uvicorn.sock command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_POSTGRES_HOST)s:5432 -t 0 -- python3 -m uvicorn --fd 0 --forwarded-allow-ips='*' cvat.asgi:application -environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock" +autorestart=true +environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock",CVAT_EVENTS_LOCAL_DB_FILENAME="events_%(process_num)03d.db" numprocs=%(ENV_NUMPROCS)s process_name=%(program_name)s-%(process_num)s stdout_logfile=/dev/stdout diff --git a/tests/python/cli/example_function.py b/tests/python/cli/example_function.py new file mode 100644 index 000000000000..4b1b41857825 --- /dev/null +++ b/tests/python/cli/example_function.py @@ -0,0 +1,23 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from typing import List + +import cvat_sdk.auto_annotation as cvataa +import cvat_sdk.models as models +import PIL.Image + +spec = cvataa.DetectionFunctionSpec( + labels=[ + cvataa.label_spec("car", 0), + ], +) + + +def detect( + context: cvataa.DetectionFunctionContext, image: PIL.Image.Image +) -> List[models.LabeledShapeRequest]: + return [ + cvataa.rectangle(0, [1, 2, 3, 4]), + ] diff --git a/tests/python/cli/example_parameterized_function.py b/tests/python/cli/example_parameterized_function.py new file mode 100644 index 000000000000..29d9038e78b4 --- /dev/null +++ b/tests/python/cli/example_parameterized_function.py @@ -0,0 +1,32 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from types import SimpleNamespace as namespace +from typing import List + +import cvat_sdk.auto_annotation as cvataa +import cvat_sdk.models as models +import PIL.Image + + +def create(s: str, i: int, f: float, b: bool) -> cvataa.DetectionFunction: + assert s == "string" + assert i == 123 + assert f == 5.5 + assert b is False + + spec = cvataa.DetectionFunctionSpec( + labels=[ + cvataa.label_spec("car", 0), + ], + ) + + def detect( + context: cvataa.DetectionFunctionContext, image: PIL.Image.Image + ) -> List[models.LabeledShapeRequest]: + return [ + cvataa.rectangle(0, [1, 2, 3, 4]), + ] + + return namespace(spec=spec, detect=detect) diff --git a/tests/python/cli/test_cli.py b/tests/python/cli/test_cli.py index 6dbcbb5241fd..66749f992aa6 100644 --- a/tests/python/cli/test_cli.py +++ b/tests/python/cli/test_cli.py @@ -302,3 +302,46 @@ def test_can_control_organization_context(self): all_task_ids = list(map(int, self.run_cli("ls").split())) assert personal_task_id in all_task_ids assert org_task_id in all_task_ids + + def test_auto_annotate_with_module(self, fxt_new_task: Task): + annotations = fxt_new_task.get_annotations() + assert not annotations.shapes + + self.run_cli( + "auto-annotate", + str(fxt_new_task.id), + f"--function-module={__package__}.example_function", + ) + + annotations = fxt_new_task.get_annotations() + assert annotations.shapes + + def test_auto_annotate_with_file(self, fxt_new_task: Task): + annotations = fxt_new_task.get_annotations() + assert not annotations.shapes + + self.run_cli( + "auto-annotate", + str(fxt_new_task.id), + f"--function-file={Path(__file__).with_name('example_function.py')}", + ) + + annotations = fxt_new_task.get_annotations() + assert annotations.shapes + + def test_auto_annotate_with_parameters(self, fxt_new_task: Task): + annotations = fxt_new_task.get_annotations() + assert not annotations.shapes + + self.run_cli( + "auto-annotate", + str(fxt_new_task.id), + f"--function-module={__package__}.example_parameterized_function", + "-ps=str:string", + "-pi=int:123", + "-pf=float:5.5", + "-pb=bool:false", + ) + + annotations = fxt_new_task.get_annotations() + assert annotations.shapes diff --git a/tests/python/pytest.ini b/tests/python/pytest.ini index 05cda52273da..775758a29697 100644 --- a/tests/python/pytest.ini +++ b/tests/python/pytest.ini @@ -8,3 +8,6 @@ timeout = 15 markers = with_external_services: The test requires services extrernal to the default CVAT deployment, e.g. a Git server etc. + +filterwarnings = + ignore::DeprecationWarning:cvat_sdk.core diff --git a/tests/python/sdk/test_auto_annotation.py b/tests/python/sdk/test_auto_annotation.py new file mode 100644 index 000000000000..142c4354c4d1 --- /dev/null +++ b/tests/python/sdk/test_auto_annotation.py @@ -0,0 +1,714 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import io +from logging import Logger +from pathlib import Path +from types import SimpleNamespace as namespace +from typing import List, Tuple + +import cvat_sdk.auto_annotation as cvataa +import PIL.Image +import pytest +from cvat_sdk import Client, models +from cvat_sdk.core.proxies.tasks import ResourceType + +from shared.utils.helpers import generate_image_file + +from .util import make_pbar + +try: + import torchvision.models as torchvision_models +except ModuleNotFoundError: + torchvision_models = None + + +@pytest.fixture(autouse=True) +def _common_setup( + tmp_path: Path, + fxt_login: Tuple[Client, str], + fxt_logger: Tuple[Logger, io.StringIO], +): + logger = fxt_logger[0] + client = fxt_login[0] + client.logger = logger + client.config.cache_dir = tmp_path / "cache" + + api_client = client.api_client + for k in api_client.configuration.logger: + api_client.configuration.logger[k] = logger + + +class TestTaskAutoAnnotation: + @pytest.fixture(autouse=True) + def setup( + self, + tmp_path: Path, + fxt_login: Tuple[Client, str], + ): + self.client = fxt_login[0] + self.images = [ + generate_image_file("1.png", size=(333, 333), color=(0, 0, 0)), + generate_image_file("2.png", size=(333, 333), color=(100, 100, 100)), + ] + + image_dir = tmp_path / "images" + image_dir.mkdir() + + image_paths = [] + for image in self.images: + image_path = image_dir / image.name + image_path.write_bytes(image.getbuffer()) + image_paths.append(image_path) + + self.task = self.client.tasks.create_from_data( + models.TaskWriteRequest( + "Auto-annotation test task", + labels=[ + models.PatchedLabelRequest(name="person"), + models.PatchedLabelRequest(name="car"), + models.PatchedLabelRequest( + name="cat", + type="skeleton", + sublabels=[ + models.SublabelRequest(name="head"), + models.SublabelRequest(name="tail"), + ], + ), + ], + ), + resource_type=ResourceType.LOCAL, + resources=image_paths, + ) + + task_labels = self.task.get_labels() + self.task_labels_by_id = {label.id: label for label in task_labels} + self.cat_sublabels_by_id = { + sl.id: sl + for sl in next(label for label in task_labels if label.name == "cat").sublabels + } + + # The initial annotation is just to check that it gets erased after auto-annotation + self.task.update_annotations( + models.PatchedLabeledDataRequest( + shapes=[ + models.LabeledShapeRequest( + frame=0, + label_id=next(iter(self.task_labels_by_id)), + type="rectangle", + points=[1.0, 2.0, 3.0, 4.0], + ), + ], + ) + ) + + def test_detection_rectangle(self): + spec = cvataa.DetectionFunctionSpec( + labels=[ + cvataa.label_spec("car", 123), + cvataa.label_spec("bicycle (should be ignored)", 456), + ], + ) + + def detect( + context: cvataa.DetectionFunctionContext, image: PIL.Image.Image + ) -> List[models.LabeledShapeRequest]: + assert context.frame_name in {"1.png", "2.png"} + assert image.width == image.height == 333 + return [ + cvataa.rectangle( + 123, # car + # produce different coordinates for different images + [*image.getpixel((0, 0)), 300 + int(context.frame_name[0])], + ), + cvataa.shape( + 456, # ignored + type="points", + points=[1, 1], + ), + ] + + cvataa.annotate_task( + self.client, + self.task.id, + namespace(spec=spec, detect=detect), + clear_existing=True, + allow_unmatched_labels=True, + ) + + annotations = self.task.get_annotations() + + shapes = sorted(annotations.shapes, key=lambda shape: shape.frame) + + assert len(shapes) == 2 + + for i, shape in enumerate(shapes): + assert shape.frame == i + assert shape.type.value == "rectangle" + assert self.task_labels_by_id[shape.label_id].name == "car" + assert shape.points[3] in {301, 302} + + assert shapes[0].points[0] != shapes[1].points[0] + assert shapes[0].points[3] != shapes[1].points[3] + + def test_detection_skeleton(self): + spec = cvataa.DetectionFunctionSpec( + labels=[ + cvataa.skeleton_label_spec( + "cat", + 123, + [ + cvataa.keypoint_spec("head", 10), + cvataa.keypoint_spec("torso (should be ignored)", 20), + cvataa.keypoint_spec("tail", 30), + ], + ), + ], + ) + + def detect(context, image: PIL.Image.Image) -> List[models.LabeledShapeRequest]: + assert image.width == image.height == 333 + return [ + cvataa.skeleton( + 123, # cat + [ + # ignored + cvataa.keypoint(20, [20, 20]), + # tail + cvataa.keypoint(30, [30, 30]), + # head + cvataa.keypoint(10, [10, 10]), + ], + ), + ] + + cvataa.annotate_task( + self.client, + self.task.id, + namespace(spec=spec, detect=detect), + clear_existing=True, + allow_unmatched_labels=True, + ) + + annotations = self.task.get_annotations() + + shapes = sorted(annotations.shapes, key=lambda shape: shape.frame) + + assert len(shapes) == 2 + + for i, shape in enumerate(shapes): + assert shape.frame == i + assert shape.type.value == "skeleton" + assert self.task_labels_by_id[shape.label_id].name == "cat" + assert len(shape.elements) == 2 + + elements = sorted( + shape.elements, key=lambda s: self.cat_sublabels_by_id[s.label_id].name + ) + + for element in elements: + assert element.frame == i + assert element.type.value == "points" + + assert self.cat_sublabels_by_id[elements[0].label_id].name == "head" + assert elements[0].points == [10, 10] + assert self.cat_sublabels_by_id[elements[1].label_id].name == "tail" + assert elements[1].points == [30, 30] + + def test_progress_reporting(self): + spec = cvataa.DetectionFunctionSpec(labels=[]) + + def detect(context, image): + return [] + + file = io.StringIO() + + cvataa.annotate_task( + self.client, + self.task.id, + namespace(spec=spec, detect=detect), + pbar=make_pbar(file), + ) + + assert "100%" in file.getvalue() + + def test_detection_without_clearing(self): + spec = cvataa.DetectionFunctionSpec( + labels=[ + cvataa.label_spec("car", 123), + ], + ) + + def detect(context, image: PIL.Image.Image) -> List[models.LabeledShapeRequest]: + return [ + cvataa.rectangle( + 123, # car + [5, 6, 7, 8], + rotation=10, + ), + ] + + cvataa.annotate_task( + self.client, + self.task.id, + namespace(spec=spec, detect=detect), + clear_existing=False, + ) + + annotations = self.task.get_annotations() + + shapes = sorted(annotations.shapes, key=lambda shape: (shape.frame, shape.rotation)) + + # original annotation + assert shapes[0].points == [1, 2, 3, 4] + assert shapes[0].rotation == 0 + + # new annotations + for i in (1, 2): + assert shapes[i].points == [5, 6, 7, 8] + assert shapes[i].rotation == 10 + + def _test_bad_function_spec(self, spec: cvataa.DetectionFunctionSpec, exc_match: str) -> None: + def detect(context, image): + assert False + + with pytest.raises(cvataa.BadFunctionError, match=exc_match): + cvataa.annotate_task(self.client, self.task.id, namespace(spec=spec, detect=detect)) + + def test_attributes(self): + self._test_bad_function_spec( + cvataa.DetectionFunctionSpec( + labels=[ + cvataa.label_spec( + "car", + 123, + attributes=[ + models.AttributeRequest( + "age", + mutable=False, + input_type="number", + values=["0", "100", "1"], + default_value="0", + ) + ], + ), + ], + ), + "currently not supported", + ) + + def test_label_not_in_dataset(self): + self._test_bad_function_spec( + cvataa.DetectionFunctionSpec( + labels=[cvataa.label_spec("dog", 123)], + ), + "not in dataset", + ) + + def test_label_without_id(self): + self._test_bad_function_spec( + cvataa.DetectionFunctionSpec( + labels=[ + models.PatchedLabelRequest( + name="car", + ), + ], + ), + "label .+ has no ID", + ) + + def test_duplicate_label_id(self): + self._test_bad_function_spec( + cvataa.DetectionFunctionSpec( + labels=[ + cvataa.label_spec("car", 123), + cvataa.label_spec("bicycle", 123), + ], + ), + "same ID as another label", + ) + + def test_non_skeleton_sublabels(self): + self._test_bad_function_spec( + cvataa.DetectionFunctionSpec( + labels=[ + cvataa.label_spec( + "car", + 123, + sublabels=[models.SublabelRequest("wheel", id=1)], + ), + ], + ), + "should be 'skeleton'", + ) + + def test_sublabel_without_id(self): + self._test_bad_function_spec( + cvataa.DetectionFunctionSpec( + labels=[ + cvataa.skeleton_label_spec( + "car", + 123, + [models.SublabelRequest("wheel")], + ), + ], + ), + "sublabel .+ of label .+ has no ID", + ) + + def test_duplicate_sublabel_id(self): + self._test_bad_function_spec( + cvataa.DetectionFunctionSpec( + labels=[ + cvataa.skeleton_label_spec( + "cat", + 123, + [ + cvataa.keypoint_spec("head", 1), + cvataa.keypoint_spec("tail", 1), + ], + ), + ], + ), + "same ID as another sublabel", + ) + + def test_sublabel_not_in_dataset(self): + self._test_bad_function_spec( + cvataa.DetectionFunctionSpec( + labels=[ + cvataa.skeleton_label_spec("cat", 123, [cvataa.keypoint_spec("nose", 1)]), + ], + ), + "not in dataset", + ) + + def _test_bad_function_detect(self, detect, exc_match: str) -> None: + spec = cvataa.DetectionFunctionSpec( + labels=[ + cvataa.label_spec("car", 123), + cvataa.skeleton_label_spec( + "cat", + 456, + [ + cvataa.keypoint_spec("head", 12), + cvataa.keypoint_spec("tail", 34), + ], + ), + ], + ) + + with pytest.raises(cvataa.BadFunctionError, match=exc_match): + cvataa.annotate_task(self.client, self.task.id, namespace(spec=spec, detect=detect)) + + def test_preset_shape_id(self): + self._test_bad_function_detect( + lambda context, image: [ + models.LabeledShapeRequest( + type="rectangle", frame=0, label_id=123, id=1111, points=[1, 2, 3, 4] + ), + ], + "shape with preset id", + ) + + def test_preset_shape_source(self): + self._test_bad_function_detect( + lambda context, image: [ + models.LabeledShapeRequest( + type="rectangle", frame=0, label_id=123, source="manual", points=[1, 2, 3, 4] + ), + ], + "shape with preset source", + ) + + def test_bad_shape_frame_number(self): + self._test_bad_function_detect( + lambda context, image: [ + models.LabeledShapeRequest( + type="rectangle", + frame=1, + label_id=123, + points=[1, 2, 3, 4], + ), + ], + "unexpected frame number", + ) + + def test_unknown_label_id(self): + self._test_bad_function_detect( + lambda context, image: [ + cvataa.rectangle(111, [1, 2, 3, 4]), + ], + "unknown label ID", + ) + + def test_shape_with_attributes(self): + self._test_bad_function_detect( + lambda context, image: [ + cvataa.rectangle( + 123, + [1, 2, 3, 4], + attributes=[ + models.AttributeValRequest(spec_id=1, value="asdf"), + ], + ), + ], + "shape with attributes", + ) + + def test_preset_element_id(self): + self._test_bad_function_detect( + lambda context, image: [ + cvataa.skeleton( + 456, + [ + models.SubLabeledShapeRequest( + type="points", frame=0, label_id=12, id=1111, points=[1, 2] + ), + ], + ), + ], + "element with preset id", + ) + + def test_preset_element_source(self): + self._test_bad_function_detect( + lambda context, image: [ + cvataa.skeleton( + 456, + [ + models.SubLabeledShapeRequest( + type="points", frame=0, label_id=12, source="manual", points=[1, 2] + ), + ], + ), + ], + "element with preset source", + ) + + def test_bad_element_frame_number(self): + self._test_bad_function_detect( + lambda context, image: [ + cvataa.skeleton( + 456, + [ + models.SubLabeledShapeRequest( + type="points", frame=1, label_id=12, points=[1, 2] + ), + ], + ), + ], + "element with unexpected frame number", + ) + + def test_non_points_element(self): + self._test_bad_function_detect( + lambda context, image: [ + cvataa.skeleton( + 456, + [ + models.SubLabeledShapeRequest( + type="rectangle", frame=0, label_id=12, points=[1, 2, 3, 4] + ), + ], + ), + ], + "element type other than 'points'", + ) + + def test_unknown_sublabel_id(self): + self._test_bad_function_detect( + lambda context, image: [ + cvataa.skeleton(456, [cvataa.keypoint(56, [1, 2])]), + ], + "unknown sublabel ID", + ) + + def test_multiple_elements_with_same_sublabel(self): + self._test_bad_function_detect( + lambda context, image: [ + cvataa.skeleton( + 456, + [ + cvataa.keypoint(12, [1, 2]), + cvataa.keypoint(12, [3, 4]), + ], + ), + ], + "multiple elements with same sublabel", + ) + + def test_not_enough_elements(self): + self._test_bad_function_detect( + lambda context, image: [ + cvataa.skeleton(456, [cvataa.keypoint(12, [1, 2])]), + ], + "with fewer elements than expected", + ) + + def test_non_skeleton_with_elements(self): + self._test_bad_function_detect( + lambda context, image: [ + cvataa.shape( + 456, + type="rectangle", + elements=[cvataa.keypoint(12, [1, 2])], + ), + ], + "non-skeleton shape with elements", + ) + + +if torchvision_models is not None: + import torch + import torch.nn as nn + + class FakeTorchvisionDetector(nn.Module): + def __init__(self, label_id: int) -> None: + super().__init__() + self._label_id = label_id + + def forward(self, images: List[torch.Tensor]) -> List[dict]: + assert isinstance(images, list) + assert all(isinstance(t, torch.Tensor) for t in images) + + return [ + { + "boxes": torch.tensor([[1, 2, 3, 4]]), + "labels": torch.tensor([self._label_id]), + } + ] + + def fake_get_detection_model(name: str, weights, test_param): + assert test_param == "expected_value" + + car_label_id = weights.meta["categories"].index("car") + + return FakeTorchvisionDetector(label_id=car_label_id) + + class FakeTorchvisionKeypointDetector(nn.Module): + def __init__(self, label_id: int, keypoint_names: List[str]) -> None: + super().__init__() + self._label_id = label_id + self._keypoint_names = keypoint_names + + def forward(self, images: List[torch.Tensor]) -> List[dict]: + assert isinstance(images, list) + assert all(isinstance(t, torch.Tensor) for t in images) + + return [ + { + "labels": torch.tensor([self._label_id]), + "keypoints": torch.tensor( + [ + [ + [hash(name) % 100, 0, 1 if name.startswith("right_") else 0] + for i, name in enumerate(self._keypoint_names) + ] + ] + ), + } + ] + + def fake_get_keypoint_detection_model(name: str, weights, test_param): + assert test_param == "expected_value" + + person_label_id = weights.meta["categories"].index("person") + + return FakeTorchvisionKeypointDetector( + label_id=person_label_id, keypoint_names=weights.meta["keypoint_names"] + ) + + +@pytest.mark.skipif(torchvision_models is None, reason="torchvision is not installed") +class TestAutoAnnotationFunctions: + @pytest.fixture(autouse=True) + def setup( + self, + tmp_path: Path, + fxt_login: Tuple[Client, str], + ): + self.client = fxt_login[0] + self.image = generate_image_file("1.png", size=(100, 100)) + + image_dir = tmp_path / "images" + image_dir.mkdir() + + image_path = image_dir / self.image.name + image_path.write_bytes(self.image.getbuffer()) + + self.task = self.client.tasks.create_from_data( + models.TaskWriteRequest( + "Auto-annotation test task", + labels=[ + models.PatchedLabelRequest( + name="person", + type="skeleton", + sublabels=[ + models.SublabelRequest(name="left_eye"), + models.SublabelRequest(name="right_eye"), + ], + ), + models.PatchedLabelRequest(name="car"), + ], + ), + resources=[image_path], + ) + + task_labels = self.task.get_labels() + self.task_labels_by_id = {label.id: label for label in task_labels} + + person_label = next(label for label in task_labels if label.name == "person") + self.person_sublabels_by_id = {sl.id: sl for sl in person_label.sublabels} + + def test_torchvision_detection(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(torchvision_models, "get_model", fake_get_detection_model) + + import cvat_sdk.auto_annotation.functions.torchvision_detection as td + + cvataa.annotate_task( + self.client, + self.task.id, + td.create("fasterrcnn_resnet50_fpn_v2", "COCO_V1", test_param="expected_value"), + allow_unmatched_labels=True, + ) + + annotations = self.task.get_annotations() + + assert len(annotations.shapes) == 1 + assert self.task_labels_by_id[annotations.shapes[0].label_id].name == "car" + assert annotations.shapes[0].type.value == "rectangle" + assert annotations.shapes[0].points == [1, 2, 3, 4] + + def test_torchvision_keypoint_detection(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(torchvision_models, "get_model", fake_get_keypoint_detection_model) + + import cvat_sdk.auto_annotation.functions.torchvision_keypoint_detection as tkd + + cvataa.annotate_task( + self.client, + self.task.id, + tkd.create("keypointrcnn_resnet50_fpn", "COCO_V1", test_param="expected_value"), + allow_unmatched_labels=True, + ) + + annotations = self.task.get_annotations() + + assert len(annotations.shapes) == 1 + assert self.task_labels_by_id[annotations.shapes[0].label_id].name == "person" + assert annotations.shapes[0].type.value == "skeleton" + assert len(annotations.shapes[0].elements) == 2 + + elements = sorted( + annotations.shapes[0].elements, + key=lambda e: self.person_sublabels_by_id[e.label_id].name, + ) + + assert self.person_sublabels_by_id[elements[0].label_id].name == "left_eye" + assert elements[0].points[0] == hash("left_eye") % 100 + assert elements[0].occluded + + assert self.person_sublabels_by_id[elements[1].label_id].name == "right_eye" + assert elements[1].points[0] == hash("right_eye") % 100 + assert not elements[1].occluded diff --git a/tests/python/sdk/test_datasets.py b/tests/python/sdk/test_datasets.py index 67204e4c26c9..35b2339ec67e 100644 --- a/tests/python/sdk/test_datasets.py +++ b/tests/python/sdk/test_datasets.py @@ -101,6 +101,7 @@ def test_basic(self): for index, sample in enumerate(dataset.samples): assert sample.frame_index == index + assert sample.frame_name == self.images[index].name actual_image = sample.media.load_image() expected_image = PIL.Image.open(self.images[index]) diff --git a/tests/python/sdk/test_progress.py b/tests/python/sdk/test_progress.py new file mode 100644 index 000000000000..a8f2fc10c6a5 --- /dev/null +++ b/tests/python/sdk/test_progress.py @@ -0,0 +1,82 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import io +import warnings +from typing import Optional + +import tqdm +from cvat_sdk.core.helpers import DeferredTqdmProgressReporter, TqdmProgressReporter +from cvat_sdk.core.progress import NullProgressReporter, ProgressReporter + + +def _exercise_reporter(r: ProgressReporter) -> None: + with r.task(total=5, desc="Test task", unit="parrots"): + r.advance(1) + r.report_status(4) + + for x in r.iter(["x"]): + assert x == "x" + + +def test_null_reporter(): + _exercise_reporter(NullProgressReporter()) + # NPR doesn't do anything, so there's nothing to assert + + +def test_tqdm_reporter(): + f = io.StringIO() + + instance = tqdm.tqdm(file=f) + + with warnings.catch_warnings(): + r = TqdmProgressReporter(instance) + + _exercise_reporter(r) + + output = f.getvalue() + + assert "100%" in output + assert "Test task" in output + # TPR doesn't support parameters other than "total" and "desc", + # so there won't be any parrots in the output. + + +def test_deferred_tqdm_reporter(): + f = io.StringIO() + + _exercise_reporter(DeferredTqdmProgressReporter({"file": f})) + + output = f.getvalue() + + assert "100%" in output + assert "Test task" in output + assert "parrots" in output + + +class _LegacyProgressReporter(ProgressReporter): + # overriding start instead of start2 + def start(self, total: int, *, desc: Optional[str] = None) -> None: + self.total = total + self.desc = desc + self.progress = 0 + + def report_status(self, progress: int): + self.progress = progress + + def advance(self, delta: int): + self.progress += delta + + def finish(self): + self.finished = True + + +def test_legacy_progress_reporter(): + r = _LegacyProgressReporter() + + _exercise_reporter(r) + + assert r.total == 5 + assert r.desc == "Test task" + assert r.progress == 5 diff --git a/tests/python/sdk/util.py b/tests/python/sdk/util.py index 5861c658111a..1686330ad9f1 100644 --- a/tests/python/sdk/util.py +++ b/tests/python/sdk/util.py @@ -9,12 +9,11 @@ import pytest from cvat_sdk.api_client.rest import RESTClientObject -from cvat_sdk.core.helpers import TqdmProgressReporter -from tqdm import tqdm +from cvat_sdk.core.helpers import DeferredTqdmProgressReporter def make_pbar(file, **kwargs): - return TqdmProgressReporter(tqdm(file=file, mininterval=0, **kwargs)) + return DeferredTqdmProgressReporter({"file": file, "mininterval": 0, **kwargs}) def generate_coco_json(filename: Path, img_info: Tuple[Path, int, int]): diff --git a/yarn.lock b/yarn.lock index b48ea52b3183..0c1047660b82 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3356,11 +3356,6 @@ braces@^3.0.2, braces@~3.0.2: dependencies: fill-range "^7.0.1" -browser-or-node@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/browser-or-node/-/browser-or-node-2.1.1.tgz#738790b3a86a8fc020193fa581273fbe65eaea0f" - integrity sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg== - browser-process-hrtime@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626" @@ -4155,7 +4150,7 @@ custom-error-instance@2.1.1: three "^0.126.1" "cvat-canvas@link:./cvat-canvas": - version "2.17.1" + version "2.17.3" dependencies: "@types/fabric" "^4.5.7" "@types/polylabel" "^1.0.5" @@ -4168,11 +4163,10 @@ custom-error-instance@2.1.1: svg.select.js "3.0.1" "cvat-core@link:./cvat-core": - version "9.3.0" + version "10.0.1" dependencies: "@types/lodash" "^4.14.191" axios "^0.27.2" - browser-or-node "^2.0.0" cvat-data "link:./cvat-data" detect-browser "^5.2.1" error-stack-parser "^2.0.2" @@ -4186,7 +4180,7 @@ custom-error-instance@2.1.1: tus-js-client "^3.0.1" "cvat-data@link:./cvat-data": - version "1.1.0" + version "2.0.0" dependencies: async-mutex "^0.4.0" jszip "3.10.1"