Merge annotations and dataset_manager apps

.travis.yml

@@ -21,11 +21,12 @@ before_script:
   - mkdir -m a=rwx -p ${HOST_COVERAGE_DATA_DIR}
 
 script:
-  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli && coverage run -a --source cvat/apps/ manage.py test --pattern="_tests.py" cvat/apps/dataset_manager && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
+  # FIXME: Git package and application name conflict in PATH
+  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps && coverage run -a manage.py test --pattern="_test*.py" cvat/apps/dataset_manager/tests cvat/apps/engine/tests utils/cli && coverage run -a manage.py test datumaro/ && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
   - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm install && cd ../cvat-core && npm install && npm run test && mv ./reports/coverage/lcov.info ${CONTAINER_COVERAGE_DATA_DIR}'
 
 after_success:
   # https://coveralls-python.readthedocs.io/en/latest/usage/multilang.html
   - mv ${HOST_COVERAGE_DATA_DIR}/* .
   - coveralls-lcov -v -n lcov.info > coverage.json
-  - coveralls --merge=coverage.json
+  - coveralls --merge=coverage.json

.vscode/settings.json

@@ -33,5 +33,6 @@
         "./datumaro",
     ],
     "licenser.license": "Custom",
-    "licenser.customHeader": "Copyright (C) @YEAR@ Intel Corporation\n\nSPDX-License-Identifier: MIT"
+    "licenser.customHeader": "Copyright (C) @YEAR@ Intel Corporation\n\nSPDX-License-Identifier: MIT",
+    "files.trimTrailingWhitespace": true
 }

CHANGELOG.md

@@ -6,18 +6,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [1.0.0] - Unreleased
 ### Added
--
+- Added `datumaro_project` export format (https://github.com/opencv/cvat/pull/1352)
 
 ### Changed
-- cvat-core: session.annotations.put() now returns identificators of added objects (<https://github.com/opencv/cvat/pull/1493>)
+- Downloaded file name in annotations export became more informative (https://github.com/opencv/cvat/pull/1352)
+- Added auto trimming for trailing whitespaces style enforsement (https://github.com/opencv/cvat/pull/1352)
+- REST API: updated `GET /task/<id>/annotations`: parameters are `format`, `filename` (now optional), `action` (optional) (https://github.com/opencv/cvat/pull/1352)
+- REST API: removed `dataset/formats`, changed format of `annotation/formats` (https://github.com/opencv/cvat/pull/1352)
+- Exported annotations are stored for N hours instead of indefinitely (https://github.com/opencv/cvat/pull/1352)
+- Formats: CVAT format now accepts ZIP and XML (https://github.com/opencv/cvat/pull/1352)
+- Formats: COCO format now accepts ZIP and JSON (https://github.com/opencv/cvat/pull/1352)
+- Formats: most of formats renamed, no extension in title (https://github.com/opencv/cvat/pull/1352)
+- Formats: definitions are changed, are not stored in DB anymore (https://github.com/opencv/cvat/pull/1352)
+- cvat-core: session.annotations.put() now returns identificators of added objects (https://github.com/opencv/cvat/pull/1493)
 
 ### Deprecated
 -
 
 ### Removed
--
+- `annotation` application is replaced with `dataset_manager` (https://github.com/opencv/cvat/pull/1352)
 
 ### Fixed
+- Categories for empty projects with no sources are taken from own dataset (https://github.com/opencv/cvat/pull/1352)
+- Added directory removal on error during `extract` command (https://github.com/opencv/cvat/pull/1352)
+- Added debug error message on incorrect XPath (https://github.com/opencv/cvat/pull/1352)
+- Exporting frame stepped task (https://github.com/opencv/cvat/issues/1294, https://github.com/opencv/cvat/issues/1334)
+- Fixed broken command line interface for `cvat` export format in Datumaro (https://github.com/opencv/cvat/issues/1494)
 - Updated Rest API document, Swagger document serving instruction issue (https://github.com/opencv/cvat/issues/1495)
 
 ### Security

CONTRIBUTING.md

@@ -75,6 +75,7 @@ for development
     - [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint)
     - [vscode-remark-lint](https://marketplace.visualstudio.com/items?itemName=drewbourne.vscode-remark-lint)
     - [licenser](https://marketplace.visualstudio.com/items?itemName=ymotongpoo.licenser)
+    - [Trailing Spaces](https://marketplace.visualstudio.com/items?itemName=shardulm94.trailing-spaces)
 
 -   Reload Visual Studio Code from virtual environment
 

README.md

@@ -7,7 +7,10 @@
 [![codebeat badge](https://codebeat.co/badges/53cd0d16-fddc-46f8-903c-f43ed9abb6dd)](https://codebeat.co/projects/github-com-opencv-cvat-develop)
 [![DOI](https://zenodo.org/badge/139156354.svg)](https://zenodo.org/badge/latestdoi/139156354)
 
-CVAT is free, online, interactive video and image annotation tool for computer vision. It is being used by our team to annotate million of objects with different properties. Many UI and UX decisions are based on feedbacks from professional data annotation team.
+CVAT is free, online, interactive video and image annotation
+tool for computer vision. It is being used by our team to
+annotate million of objects with different properties. Many UI
+and UX decisions are based on feedbacks from professional data annotation team.
 
 ![CVAT screenshot](cvat/apps/documentation/static/documentation/images/cvat.jpg)
 
@@ -34,21 +37,23 @@ CVAT is free, online, interactive video and image annotation tool for computer v
 
 ## Supported annotation formats
 
-Format selection is possible after clicking on the Upload annotation / Dump annotation button.
-[Datumaro](datumaro/README.md) dataset framework allows additional dataset transformations
-via its command line tool.
+Format selection is possible after clicking on the Upload annotation
+and Dump annotation buttons. [Datumaro](datumaro/README.md) dataset
+framework allows additional dataset transformations
+via its command line tool and Python library.
 
-| Annotation format                                                                          | Dumper | Loader |
+| Annotation format                                                                          | Import | Export |
 | ------------------------------------------------------------------------------------------ | ------ | ------ |
-| [CVAT XML v1.1 for images](cvat/apps/documentation/xml_format.md#annotation)               | X      | X      |
-| [CVAT XML v1.1 for a video](cvat/apps/documentation/xml_format.md#interpolation)           | X      | X      |
-| [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/)                                      | X      | X      |
+| [CVAT for images](cvat/apps/documentation/xml_format.md#annotation)                        | X      | X      |
+| [CVAT for a video](cvat/apps/documentation/xml_format.md#interpolation)                    | X      | X      |
+| [Datumaro](datumaro/README.md)                                                             |        | X      |
+| [PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/)                                      | X      | X      |
+| Segmentation masks from [PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/)              | X      | X      |
 | [YOLO](https://pjreddie.com/darknet/yolo/)                                                 | X      | X      |
 | [MS COCO Object Detection](http://cocodataset.org/#format-data)                            | X      | X      |
-| PNG class mask + instance mask as in [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/) | X      | X      |
 | [TFrecord](https://www.tensorflow.org/tutorials/load_data/tf_records)                      | X      | X      |
 | [MOT](https://motchallenge.net/)                                                           | X      | X      |
-| [LabelMe](http://labelme.csail.mit.edu/Release3.0)                                         | X      | X      |
+| [LabelMe 3.0](http://labelme.csail.mit.edu/Release3.0)                                     | X      | X      |
 
 ## Links
 - [Intel AI blog: New Computer Vision Tool Accelerates Annotation of Digital Images and Video](https://www.intel.ai/introducing-cvat)
@@ -57,14 +62,19 @@ via its command line tool.
 
 ## Online Demo
 
-[Onepanel](https://www.onepanel.io/) has added CVAT as an environment into their platform and a running demo of CVAT can be accessed at [CVAT Public Demo](https://c.onepanel.io/onepanel-demo/projects/cvat-public-demo/workspaces?utm_source=cvat).
+[Onepanel](https://www.onepanel.io/) has added CVAT as an environment
+into their platform and a running demo of CVAT can be accessed at
+[CVAT Public Demo](https://c.onepanel.io/onepanel-demo/projects/cvat-public-demo/workspaces?utm_source=cvat).
 
-If you have any questions, please contact Onepanel directly at support@onepanel.io. If you are in the Onepanel application, you can also use the chat icon in the bottom right corner.
+If you have any questions, please contact Onepanel directly at
+support@onepanel.io. If you are in the Onepanel application, you can also
+use the chat icon in the bottom right corner.
 
 ## REST API
 
 Automatically generated Swagger documentation for Django REST API is
-available on ``<cvat_origin>/api/swagger`` (default: ``localhost:8080/api/swagger``).
+available on ``<cvat_origin>/api/swagger``
+(default: ``localhost:8080/api/swagger``).
 
 Swagger documentation is visiable on allowed hostes, Update environement variable in docker-compose.yml file with cvat hosted machine IP or domain name. Example - ``ALLOWED_HOSTS: 'localhost, 127.0.0.1'``)
 

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "1.0.0",
+  "version": "2.0.0",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "babel.config.js",
   "scripts": {

cvat-core/src/annotation-format.js → cvat-core/src/annotation-formats.js

@@ -12,9 +12,8 @@
     class Loader {
         constructor(initialData) {
             const data = {
-                display_name: initialData.display_name,
-                format: initialData.format,
-                handler: initialData.handler,
+                name: initialData.name,
+                format: initialData.ext,
                 version: initialData.version,
             };
 
@@ -27,7 +26,7 @@
                         * @readonly
                         * @instance
                     */
-                    get: () => data.display_name,
+                    get: () => data.name,
                 },
                 format: {
                     /**
@@ -39,16 +38,6 @@
                     */
                     get: () => data.format,
                 },
-                handler: {
-                    /**
-                        * @name handler
-                        * @type {string}
-                        * @memberof module:API.cvat.classes.Loader
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.handler,
-                },
                 version: {
                     /**
                         * @name version
@@ -71,9 +60,8 @@
     class Dumper {
         constructor(initialData) {
             const data = {
-                display_name: initialData.display_name,
-                format: initialData.format,
-                handler: initialData.handler,
+                name: initialData.name,
+                format: initialData.ext,
                 version: initialData.version,
             };
 
@@ -86,7 +74,7 @@
                         * @readonly
                         * @instance
                     */
-                    get: () => data.display_name,
+                    get: () => data.name,
                 },
                 format: {
                     /**
@@ -98,16 +86,6 @@
                     */
                     get: () => data.format,
                 },
-                handler: {
-                    /**
-                        * @name handler
-                        * @type {string}
-                        * @memberof module:API.cvat.classes.Dumper
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.handler,
-                },
                 version: {
                     /**
                         * @name version
@@ -127,108 +105,41 @@
         * @memberof module:API.cvat.classes
         * @hideconstructor
     */
-    class AnnotationFormat {
+    class AnnotationFormats {
         constructor(initialData) {
             const data = {
-                created_date: initialData.created_date,
-                updated_date: initialData.updated_date,
-                id: initialData.id,
-                owner: initialData.owner,
-                name: initialData.name,
-                handler_file: initialData.handler_file,
+                exporters: initialData.exporters.map((el) => new Dumper(el)),
+                importers: initialData.importers.map((el) => new Loader(el)),
             };
 
-            data.dumpers = initialData.dumpers.map((el) => new Dumper(el));
-            data.loaders = initialData.loaders.map((el) => new Loader(el));
-
             // Now all fields are readonly
             Object.defineProperties(this, {
-                id: {
-                    /**
-                        * @name id
-                        * @type {integer}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.id,
-                },
-                owner: {
-                    /**
-                        * @name owner
-                        * @type {integer}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.owner,
-                },
-                name: {
-                    /**
-                        * @name name
-                        * @type {string}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.name,
-                },
-                createdDate: {
-                    /**
-                        * @name createdDate
-                        * @type {string}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.created_date,
-                },
-                updatedDate: {
-                    /**
-                        * @name updatedDate
-                        * @type {string}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.updated_date,
-                },
-                handlerFile: {
-                    /**
-                        * @name handlerFile
-                        * @type {string}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.handler_file,
-                },
                 loaders: {
                     /**
                         * @name loaders
                         * @type {module:API.cvat.classes.Loader[]}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
+                        * @memberof module:API.cvat.classes.AnnotationFormats
                         * @readonly
                         * @instance
                     */
-                    get: () => [...data.loaders],
+                    get: () => [...data.importers],
                 },
                 dumpers: {
                     /**
                         * @name dumpers
                         * @type {module:API.cvat.classes.Dumper[]}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
+                        * @memberof module:API.cvat.classes.AnnotationFormats
                         * @readonly
                         * @instance
                     */
-                    get: () => [...data.dumpers],
+                    get: () => [...data.exporters],
                 },
             });
         }
     }
 
     module.exports = {
-        AnnotationFormat,
+        AnnotationFormats,
         Loader,
         Dumper,
     };

cvat-core/src/annotations.js

@@ -17,7 +17,7 @@
     const {
         Loader,
         Dumper,
-    } = require('./annotation-format.js');
+    } = require('./annotation-formats.js');
     const {
         ScriptingError,
         DataError,

cvat-core/src/api-implementation.js

@@ -26,7 +26,7 @@
     } = require('./enums');
 
     const User = require('./user');
-    const { AnnotationFormat } = require('./annotation-format.js');
+    const { AnnotationFormats } = require('./annotation-formats.js');
     const { ArgumentError } = require('./exceptions');
     const { Task } = require('./session');
 
@@ -66,12 +66,7 @@
 
         cvat.server.formats.implementation = async () => {
             const result = await serverProxy.server.formats();
-            return result.map((el) => new AnnotationFormat(el));
-        };
-
-        cvat.server.datasetFormats.implementation = async () => {
-            const result = await serverProxy.server.datasetFormats();
-            return result;
+            return new AnnotationFormats(result);
         };
 
         cvat.server.register.implementation = async (username, firstName, lastName,

cvat-core/src/api.js

@@ -109,7 +109,7 @@ function build() {
                 * @method formats
                 * @async
                 * @memberof module:API.cvat.server
-                * @returns {module:API.cvat.classes.AnnotationFormat[]}
+                * @returns {module:API.cvat.classes.AnnotationFormats}
                 * @throws {module:API.cvat.exceptions.PluginError}
                 * @throws {module:API.cvat.exceptions.ServerError}
             */
@@ -118,20 +118,6 @@ function build() {
                     .apiWrapper(cvat.server.formats);
                 return result;
             },
-            /**
-                * Method returns available dataset export formats
-                * @method exportFormats
-                * @async
-                * @memberof module:API.cvat.server
-                * @returns {module:String[]}
-                * @throws {module:API.cvat.exceptions.PluginError}
-                * @throws {module:API.cvat.exceptions.ServerError}
-            */
-            async datasetFormats() {
-                const result = await PluginRegistry
-                    .apiWrapper(cvat.server.datasetFormats);
-                return result;
-            },
             /**
                 * Method allows to register on a server
                 * @method register