From 7ebf2e5691b954dcff616c63c18a3b7dae291846 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 22 May 2023 19:20:00 +0300 Subject: [PATCH 01/62] Initial commit --- cvat-core/src/api-implementation.ts | 12 + cvat-core/src/api.ts | 13 +- cvat-core/src/guide.ts | 103 +++ cvat-core/src/project.ts | 5 + cvat-core/src/server-proxy.ts | 41 +- cvat-core/src/server-response-types.ts | 11 +- cvat-core/src/session.ts | 20 +- cvat-ui/package.json | 1 + cvat-ui/src/components/cvat-app.tsx | 4 + .../src/components/md-guide/guide-page.tsx | 126 ++++ .../components/md-guide/md-guide-control.tsx | 39 + cvat-ui/src/components/md-guide/styles.scss | 11 + cvat-ui/src/components/task-page/details.tsx | 2 + cvat-ui/src/cvat-core-wrapper.ts | 2 + .../migrations/0068_annotationguide_asset.py | 38 + .../0069_annotationguide_markdown.py | 18 + cvat/apps/engine/models.py | 28 + cvat/apps/engine/serializers.py | 45 +- cvat/apps/engine/urls.py | 2 + cvat/apps/engine/views.py | 65 +- cvat/settings/base.py | 53 +- yarn.lock | 671 +++++++++++++++++- 22 files changed, 1260 insertions(+), 50 deletions(-) create mode 100644 cvat-core/src/guide.ts create mode 100644 cvat-ui/src/components/md-guide/guide-page.tsx create mode 100644 cvat-ui/src/components/md-guide/md-guide-control.tsx create mode 100644 cvat-ui/src/components/md-guide/styles.scss create mode 100644 cvat/apps/engine/migrations/0068_annotationguide_asset.py create mode 100644 cvat/apps/engine/migrations/0069_annotationguide_markdown.py diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts index 4c9ed843737..d643d7cf813 100644 --- a/cvat-core/src/api-implementation.ts +++ b/cvat-core/src/api-implementation.ts @@ -25,6 +25,8 @@ import Project from './project'; import CloudStorage from './cloud-storage'; import Organization from './organization'; import Webhook from './webhook'; +import { ArgumentError } from './exceptions'; +import AnnotationGuide from './guide'; export default function implementAPI(cvat) { cvat.plugins.list.implementation = PluginRegistry.list; @@ -133,6 +135,16 @@ export default function implementAPI(cvat) { return result; }; + cvat.guides.get.implementation = async (filter: { id: number }) => { + if (!('id' in filter)) { + throw new ArgumentError('Guide id was not provided'); + } + checkFilter(filter, { id: isInteger }); + + const result = await serverProxy.guides.get(filter.id); + return new AnnotationGuide(result); + }; + cvat.users.get.implementation = async (filter) => { checkFilter(filter, { id: isInteger, diff --git a/cvat-core/src/api.ts b/cvat-core/src/api.ts index b899b3c96e8..4002ea0f166 100644 --- a/cvat-core/src/api.ts +++ b/cvat-core/src/api.ts @@ -3,11 +3,6 @@ // // SPDX-License-Identifier: MIT -/** - * External API which should be used by for development - * @module API - */ - import PluginRegistry from './plugins'; import loggerStorage from './logger-storage'; import { EventLogger } from './log'; @@ -25,6 +20,7 @@ import { FrameData } from './frames'; import CloudStorage from './cloud-storage'; import Organization from './organization'; import Webhook from './webhook'; +import AnnotationGuide from './guide'; import * as enums from './enums'; @@ -147,6 +143,12 @@ function build() { return result; }, }, + guides: { + async get(filter: { id: number }) { + const result = await PluginRegistry.apiWrapper(cvat.guides.get, filter); + return result; + }, + }, jobs: { async get(filter = {}) { const result = await PluginRegistry.apiWrapper(cvat.jobs.get, filter); @@ -281,6 +283,7 @@ function build() { CloudStorage, Organization, Webhook, + AnnotationGuide, }, }; diff --git a/cvat-core/src/guide.ts b/cvat-core/src/guide.ts new file mode 100644 index 00000000000..f5f60957cf7 --- /dev/null +++ b/cvat-core/src/guide.ts @@ -0,0 +1,103 @@ +// Copyright (C) 2023 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import User from './user'; +import { SerializedGuide } from './server-response-types'; +import { ArgumentError, DataError } from './exceptions'; +import PluginRegistry from './plugins'; +import serverProxy from './server-proxy'; + +class AnnotationGuide { + public readonly id?: number; + public readonly taskId: number; + public readonly projectId: number; + public readonly owner?: User; + public readonly createdDate?: string; + public readonly updatedDate?: string; + public markdown: string; + + constructor(initialData: SerializedGuide) { + const data = { + id: undefined, + task_id: null, + project_id: null, + owner: undefined, + created_date: undefined, + updated_date: undefined, + markdown: '', + }; + + for (const property in data) { + if (Object.prototype.hasOwnProperty.call(data, property) && property in initialData) { + data[property] = initialData[property]; + } + } + if (data.owner) data.owner = new User(data.owner); + + Object.defineProperties(this, Object.freeze({ + id: { + get: () => data.id, + }, + taskId: { + get: () => data.task_id, + }, + projectId: { + get: () => data.project_id, + }, + owner: { + get: () => data.owner, + }, + createdDate: { + get: () => data.created_date, + }, + updatedData: { + get: () => data.updated_date, + }, + markdown: { + get: () => data.updated_date, + set: (value: string) => { + if (typeof value !== 'string') { + throw new ArgumentError(`Markdown value must be a string, ${typeof value} received`); + } + data.markdown = value; + }, + }, + })); + } + + async save(): Promise { + const result = await PluginRegistry.apiWrapper.call(this, AnnotationGuide.prototype.save); + return result; + } +} + +Object.defineProperties(AnnotationGuide.prototype.save, { + implementation: { + writable: false, + enumerable: false, + value: async function implementation(this: AnnotationGuide) { + if (Number.isInteger(this.id)) { + const result = await serverProxy.guides.update(this.id, { markdown: this.markdown }); + return new AnnotationGuide(result); + } + + if (this.projectId === null && this.taskId === null) { + throw new DataError('One of projectId or taskId must be presented for a guide'); + } + + if (this.projectId !== null && this.taskId !== null) { + throw new DataError('Both projectId and taskId must not be presented for a guide'); + } + + const result = await serverProxy.guides.update(this.id, { + task_id: this.taskId, + project_id: this.projectId, + markdown: this.markdown, + }); + return new AnnotationGuide(result); + }, + }, +}); + +export default AnnotationGuide; diff --git a/cvat-core/src/project.ts b/cvat-core/src/project.ts index 92be144179a..270786e1833 100644 --- a/cvat-core/src/project.ts +++ b/cvat-core/src/project.ts @@ -19,6 +19,7 @@ export default class Project { public assignee: User; public bugTracker: string; public readonly status: ProjectStatus; + public readonly guideId: number | null; public readonly organization: string | null; public readonly owner: User; public readonly createdDate: string; @@ -39,6 +40,7 @@ export default class Project { name: undefined, status: undefined, assignee: undefined, + guide_id: undefined, organization: undefined, owner: undefined, bug_tracker: undefined, @@ -98,6 +100,9 @@ export default class Project { owner: { get: () => data.owner, }, + guideId: { + get: () => data.guide_id, + }, organization: { get: () => data.organization, }, diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 0410f630898..b2925357b90 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -11,7 +11,7 @@ import { SerializedLabel, SerializedAnnotationFormats, ProjectsFilter, SerializedProject, SerializedTask, TasksFilter, SerializedUser, SerializedAbout, SerializedRemoteFile, SerializedUserAgreement, - SerializedRegister, JobsFilter, SerializedJob, + SerializedRegister, JobsFilter, SerializedJob, SerializedGuide, } from 'server-response-types'; import { Storage } from './storage'; import { StorageLocation, WebhookSourceType } from './enums'; @@ -2146,6 +2146,39 @@ async function receiveWebhookEvents(type: WebhookSourceType): Promise } } +async function getGuide(id: number): Promise { + const { backendAPI } = config; + + try { + const response = await Axios.get(`${backendAPI}/guides/${id}`); + return response.data; + } catch (errorData) { + throw generateError(errorData); + } +} + +async function createGuide(data: SerializedGuide): Promise { + const { backendAPI } = config; + + try { + const response = await Axios.post(`${backendAPI}/guides`, data); + return response.data; + } catch (errorData) { + throw generateError(errorData); + } +} + +async function updateGuide(id: number, data: Partial): Promise { + const { backendAPI } = config; + + try { + const response = await Axios.patch(`${backendAPI}/guides/${id}`, data); + return response.data; + } catch (errorData) { + throw generateError(errorData); + } +} + export default Object.freeze({ server: Object.freeze({ setAuthData, @@ -2289,4 +2322,10 @@ export default Object.freeze({ ping: pingWebhook, events: receiveWebhookEvents, }), + + guides: Object.freeze({ + get: getGuide, + create: createGuide, + update: updateGuide, + }), }); diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index c0d80eda159..e9d52e67c06 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -61,6 +61,7 @@ export interface SerializedProject { dimension: DimensionType; name: string; organization: number | null; + guide_id: number | null; owner: SerializedUser; source_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; target_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; @@ -94,6 +95,7 @@ export interface SerializedTask { overlap: number | null; owner: SerializedUser; project_id: number | null; + guide_id: number | null; segment_size: number; size: number; source_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; @@ -115,10 +117,11 @@ export interface SerializedJob { labels: { count: number; url: string }; mode: TaskMode; project_id: number | null; + guide_id: number | null; stage: JobStage; state: JobState; - startFrame: number; - stopFrame: number; + start_frame: number; + stop_frame: number; task_id: number; updated_date: string; url: string; @@ -174,3 +177,7 @@ export interface SerializedRegister { last_name: string; username: string; } + +export interface SerializedGuide { + [index: string]: any; +} diff --git a/cvat-core/src/session.ts b/cvat-core/src/session.ts index 4a01202de8c..77d4d5f2e2d 100644 --- a/cvat-core/src/session.ts +++ b/cvat-core/src/session.ts @@ -15,6 +15,7 @@ import { ArgumentError } from './exceptions'; import { Label } from './labels'; import User from './user'; import { FieldUpdateTrigger } from './common'; +import { SerializedJob, SerializedTask } from 'server-response-types'; function buildDuplicatedAPI(prototype) { Object.defineProperties(prototype, { @@ -312,6 +313,7 @@ export class Job extends Session { public readonly startFrame: number; public readonly stopFrame: number; public readonly projectId: number | null; + public readonly guideId: number | null; public readonly taskId: number; public readonly dimension: DimensionType; public readonly dataCompressedChunkType: ChunkType; @@ -361,7 +363,7 @@ export class Job extends Session { log: CallableFunction; }; - constructor(initialData) { + constructor(initialData: SerializedJob) { super(); const data = { id: undefined, @@ -370,7 +372,8 @@ export class Job extends Session { state: undefined, start_frame: undefined, stop_frame: undefined, - project_id: null, + project_id: undefined, + guide_id: null, task_id: undefined, labels: [], dimension: undefined, @@ -476,6 +479,9 @@ export class Job extends Session { projectId: { get: () => data.project_id, }, + guideId: { + get: () => data.guide_id, + }, taskId: { get: () => data.task_id, }, @@ -577,6 +583,7 @@ export class Task extends Session { public bugTracker: string; public subset: string; public labels: Label[]; + public readonly guideId: number | null; public readonly id: number; public readonly status: TaskStatus; public readonly size: number; @@ -648,13 +655,14 @@ export class Task extends Session { log: CallableFunction; }; - constructor(initialData) { + constructor(initialData: SerializedTask) { super(); const data = { id: undefined, name: undefined, project_id: null, + guide_id: undefined, status: undefined, size: undefined, mode: undefined, @@ -729,6 +737,9 @@ export class Task extends Session { stage: job.stage, start_frame: job.start_frame, stop_frame: job.stop_frame, + guide_id: job.guide_id, + issues: job.issues, + updated_date: job.updated_date, // following fields also returned when doing API request /jobs/ // here we know them from task and append to constructor @@ -773,6 +784,9 @@ export class Task extends Session { data.project_id = projectId; }, }, + guideId: { + get: () => data.guide_id, + }, status: { get: () => data.status, }, diff --git a/cvat-ui/package.json b/cvat-ui/package.json index 9e4994a994b..6e781c6a60c 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -34,6 +34,7 @@ "@types/react-share": "^3.0.3", "@types/redux-logger": "^3.0.9", "@types/resize-observer-browser": "^0.1.6", + "@uiw/react-md-editor": "^3.22.0", "antd": "~4.18.9", "copy-to-clipboard": "^3.3.1", "cvat-canvas": "link:./../cvat-canvas", diff --git a/cvat-ui/src/components/cvat-app.tsx b/cvat-ui/src/components/cvat-app.tsx index 4ca5a6a331f..e540899bb21 100644 --- a/cvat-ui/src/components/cvat-app.tsx +++ b/cvat-ui/src/components/cvat-app.tsx @@ -56,6 +56,8 @@ import WebhooksPage from 'components/webhooks-page/webhooks-page'; import CreateWebhookPage from 'components/setup-webhook-pages/create-webhook-page'; import UpdateWebhookPage from 'components/setup-webhook-pages/update-webhook-page'; +import GuidePage from 'components/md-guide/guide-page'; + import AnnotationPageContainer from 'containers/annotation-page/annotation-page'; import { getCore } from 'cvat-core-wrapper'; import GlobalHotKeys, { KeyMap } from 'utils/mousetrap-react'; @@ -457,9 +459,11 @@ class CVATApplication extends React.PureComponent + + diff --git a/cvat-ui/src/components/md-guide/guide-page.tsx b/cvat-ui/src/components/md-guide/guide-page.tsx new file mode 100644 index 00000000000..a0637bd0e44 --- /dev/null +++ b/cvat-ui/src/components/md-guide/guide-page.tsx @@ -0,0 +1,126 @@ +// Copyright (C) 2023 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import './styles.scss'; + +import React, { useState, useEffect } from 'react'; +import { useLocation, useParams } from 'react-router'; +import { Row, Col } from 'antd/lib/grid'; +import notification from 'antd/lib/notification'; +import Text from 'antd/lib/typography/Text'; +import Button from 'antd/lib/button'; + +import MDEditor from '@uiw/react-md-editor'; +import { + getCore, Task, Project, AnnotationGuide, +} from 'cvat-core-wrapper'; +import { useIsMounted } from 'utils/hooks'; + +const core = getCore(); + +function GuidePage(): JSX.Element { + // добавить эндпоинты для получения гайда + // добавить guides пространство имен в cvat-core + + // создать эндпоинт для загрузки ассетов на стороне клиента + // завершить эндпоинт для загрузки ассетов на стороне сервера + // сделать загрузку ассетов с сервера + + // сделать нормальную страничку с редактированием гайда + // todo: add working with a local storage + + const location = useLocation(); + const isMounted = useIsMounted(); + const [value, setValue] = useState(''); + const [guide, setGuide] = useState(null); + const [fetching, setFetching] = useState(false); + const id = +useParams<{ id: string }>().id; + const instanceType = location.pathname.includes('projects') ? 'project' : 'task'; + + useEffect(() => { + setFetching(true); + const promise = instanceType === 'project' ? core.projects.get({ id }) : core.tasks.get({ id }); + promise.then(([instance]: [Task | Project]) => { + const { guideId } = instance; + if (guideId !== null) { + return core.guides.get(guideId); + } + return Promise.resolve(null); + }).then((guideInstance: AnnotationGuide | null) => { + if (guideInstance && isMounted()) { + setValue(guideInstance.markdown); + setGuide(guideInstance); + } + }).catch((error: any) => { + if (isMounted()) { + notification.error({ + message: `Could not receive guide for the ${instanceType} ${id}`, + description: error.toString(), + }); + } + console.log(error.toString()); + }).finally(() => { + if (isMounted()) { + setFetching(false); + } + }); + }, []); + + // todo: add fetching overlay + + return ( + <> +
+ Header + {/* add back arrow */} +
+
+ { + // todo: debounce + setValue(val || ''); + }} + style={{ whiteSpace: 'pre-wrap' }} + /> +
+
+ {/* add submit arrow */} + +
+ + ); +} + +export default React.memo(GuidePage); diff --git a/cvat-ui/src/components/md-guide/md-guide-control.tsx b/cvat-ui/src/components/md-guide/md-guide-control.tsx new file mode 100644 index 00000000000..9d2bc599d2f --- /dev/null +++ b/cvat-ui/src/components/md-guide/md-guide-control.tsx @@ -0,0 +1,39 @@ +// Copyright (C) 2023 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import './styles.scss'; + +import React from 'react'; +import { useHistory } from 'react-router'; +import { Row, Col } from 'antd/lib/grid'; +import Text from 'antd/lib/typography/Text'; +import Button from 'antd/lib/button'; + +interface Props { + instanceType: 'task' | 'project'; + id: number; +} + +function MdGuideControl(props: Props): JSX.Element { + const { instanceType, id } = props; + const history = useHistory(); + + return ( + + + Task description + + + + ); +} + +export default React.memo(MdGuideControl); diff --git a/cvat-ui/src/components/md-guide/styles.scss b/cvat-ui/src/components/md-guide/styles.scss new file mode 100644 index 00000000000..7b3dd711722 --- /dev/null +++ b/cvat-ui/src/components/md-guide/styles.scss @@ -0,0 +1,11 @@ +// Copyright (C) 2023 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +@import '../../base.scss'; + +.cvat-md-guide-control-wrapper { + button { + margin-left: $grid-unit-size; + } +} \ No newline at end of file diff --git a/cvat-ui/src/components/task-page/details.tsx b/cvat-ui/src/components/task-page/details.tsx index debfcc580b7..ef987ab9971 100644 --- a/cvat-ui/src/components/task-page/details.tsx +++ b/cvat-ui/src/components/task-page/details.tsx @@ -23,6 +23,7 @@ import Space from 'antd/lib/space'; import { getCore, Task } from 'cvat-core-wrapper'; import { getReposData, syncRepos, changeRepo } from 'utils/git-utils'; import AutomaticAnnotationProgress from 'components/tasks-page/automatic-annotation-progress'; +import MdGuideControl from 'components/md-guide/md-guide-control'; import Preview from 'components/common/preview'; import { cancelInferenceAsync } from 'actions/models-actions'; import { CombinedState, ActiveInference, PluginComponent } from 'reducers'; @@ -454,6 +455,7 @@ class DetailsComponent extends React.PureComponent { {this.renderDescription()} + Date: Tue, 23 May 2023 11:14:19 +0300 Subject: [PATCH 02/62] Some fixes, guide works for tasks --- cvat-core/src/guide.ts | 4 +- cvat-core/src/server-proxy.ts | 2 +- cvat-core/src/server-response-types.ts | 8 +++- .../src/components/md-guide/guide-page.tsx | 8 ++-- cvat/apps/engine/serializers.py | 37 +++++++++++++++++-- cvat/apps/engine/views.py | 16 +++++--- 6 files changed, 60 insertions(+), 15 deletions(-) diff --git a/cvat-core/src/guide.ts b/cvat-core/src/guide.ts index f5f60957cf7..1000604f7ac 100644 --- a/cvat-core/src/guide.ts +++ b/cvat-core/src/guide.ts @@ -55,7 +55,7 @@ class AnnotationGuide { get: () => data.updated_date, }, markdown: { - get: () => data.updated_date, + get: () => data.markdown, set: (value: string) => { if (typeof value !== 'string') { throw new ArgumentError(`Markdown value must be a string, ${typeof value} received`); @@ -90,7 +90,7 @@ Object.defineProperties(AnnotationGuide.prototype.save, { throw new DataError('Both projectId and taskId must not be presented for a guide'); } - const result = await serverProxy.guides.update(this.id, { + const result = await serverProxy.guides.create({ task_id: this.taskId, project_id: this.projectId, markdown: this.markdown, diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index b2925357b90..f57e4c21da1 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -2157,7 +2157,7 @@ async function getGuide(id: number): Promise { } } -async function createGuide(data: SerializedGuide): Promise { +async function createGuide(data: Partial): Promise { const { backendAPI } = config; try { diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index e9d52e67c06..cf3ed60a2d9 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -179,5 +179,11 @@ export interface SerializedRegister { } export interface SerializedGuide { - [index: string]: any; + id?: number; + task_id: number | null; + project_id: number | null; + owner: SerializedUser; + created_date: string; + updated_date: string; + markdown: string; } diff --git a/cvat-ui/src/components/md-guide/guide-page.tsx b/cvat-ui/src/components/md-guide/guide-page.tsx index a0637bd0e44..bbc3a47d8ab 100644 --- a/cvat-ui/src/components/md-guide/guide-page.tsx +++ b/cvat-ui/src/components/md-guide/guide-page.tsx @@ -44,7 +44,7 @@ function GuidePage(): JSX.Element { promise.then(([instance]: [Task | Project]) => { const { guideId } = instance; if (guideId !== null) { - return core.guides.get(guideId); + return core.guides.get({ id: guideId }); } return Promise.resolve(null); }).then((guideInstance: AnnotationGuide | null) => { @@ -100,8 +100,10 @@ function GuidePage(): JSX.Element { setFetching(true); guideInstance.save().then((result: AnnotationGuide) => { - setValue(result.markdown); - setGuide(result); + if (isMounted()) { + setValue(result.markdown); + setGuide(result); + } }).catch((error: any) => { if (isMounted()) { notification.error({ diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 042ce78ff0c..8c0dc7df0f8 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -901,7 +901,7 @@ class TaskReadSerializer(serializers.ModelSerializer): owner = BasicUserSerializer(required=False) assignee = BasicUserSerializer(allow_null=True, required=False) project_id = serializers.IntegerField(required=False, allow_null=True) - guide_id = serializers.IntegerField(required=False, allow_null=True) + guide_id = serializers.IntegerField(source='annotation_guide.id', required=False, allow_null=True) dimension = serializers.CharField(allow_blank=True, required=False) target_storage = StorageSerializer(required=False, allow_null=True) source_storage = StorageSerializer(required=False, allow_null=True) @@ -1116,7 +1116,7 @@ def validate(self, attrs): class ProjectReadSerializer(serializers.ModelSerializer): owner = BasicUserSerializer(required=False, read_only=True) assignee = BasicUserSerializer(allow_null=True, required=False, read_only=True) - guide_id = serializers.IntegerField(required=False, allow_null=True) + guide_id = serializers.IntegerField(source='annotation_guide.id', required=False, allow_null=True) task_subsets = serializers.ListField(child=serializers.CharField(), required=False, read_only=True) dimension = serializers.CharField(max_length=16, required=False, read_only=True, allow_null=True) target_storage = StorageSerializer(required=False, allow_null=True, read_only=True) @@ -1862,8 +1862,39 @@ class Meta: class AnnotationGuideWriteSerializer(WriteOnceMixin, serializers.ModelSerializer): owner = BasicUserSerializer(required=False) + project_id = serializers.IntegerField(required=False, allow_null=True) + task_id = serializers.IntegerField(required=False, allow_null=True) + + @transaction.atomic + def create(self, validated_data): + project_id = validated_data.get("project_id", None) + task_id = validated_data.get("task_id", None) + if project_id is None and task_id is None: + raise serializers.ValidationError('One of project_id or task_id must be specified') + if project_id is not None and task_id is not None: + raise serializers.ValidationError('Both project_id and task_id must not be specified') + + project = None + task = None + if project_id is not None: + try: + project = models.Project.objects.get(id=project_id) + except models.Project.DoesNotExist: + raise serializers.ValidationError(f'The specified project #{project_id} does not exist.') + print(project) + + # todo: check patch project permissions + + if task_id is not None: + try: + task = models.Task.objects.get(id=task_id) + except models.Task.DoesNotExist: + raise serializers.ValidationError(f'The specified task #{task_id} does not exist.') + print(task) + # todo: check patch task permissions + db_data = models.AnnotationGuide.objects.create(**validated_data, project = project, task = task) + return db_data class Meta: model = models.AnnotationGuide fields = ('id', 'task_id', 'project_id', 'owner', 'markdown', ) - write_once_fields = ('id', 'task_id', 'project_id', 'owner', ) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 41de79a7b25..454cceeba67 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -229,7 +229,7 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, PartialUpdateModelMixin, UploadMixin, AnnotationMixin, SerializeMixin ): queryset = models.Project.objects.select_related( - 'assignee', 'owner', 'target_storage', 'source_storage' + 'assignee', 'owner', 'target_storage', 'source_storage', 'annotation_guide', ).prefetch_related( 'tasks', 'label_set__sublabels__attributespec_set', 'label_set__attributespec_set' @@ -663,7 +663,7 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, ): queryset = Task.objects.select_related( 'data', 'assignee', 'owner', - 'target_storage', 'source_storage' + 'target_storage', 'source_storage', 'annotation_guide', ).prefetch_related( 'segment_set__job_set', 'segment_set__job_set__assignee', 'label_set__attributespec_set', @@ -2320,15 +2320,21 @@ class AnnotationGuidesViewset( search_fields = () ordering = "-id" + # NOTE: This filter works incorrectly for this view + # it requires task__organization OR project__organization check. + # Thus, we rely on permission-based filtering + iam_organization_field = None + def get_serializer_class(self): if self.request.method in SAFE_METHODS: return AnnotationGuideReadSerializer else: return AnnotationGuideWriteSerializer - def perform_create(self, instance): - # todo: update all guide_id for assets - pass + def perform_create(self, serializer): + serializer.save( + owner=self.request.user, + ) def perform_update(self, instance): # todo: update all guide_id for assets From 2d4ea90671924b014b8c30078f4330eb49545b83 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 23 May 2023 11:23:49 +0300 Subject: [PATCH 03/62] Create/update for a project --- cvat-ui/src/components/md-guide/guide-page.tsx | 2 ++ cvat-ui/src/components/project-page/details.tsx | 2 ++ cvat/apps/engine/views.py | 6 +----- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cvat-ui/src/components/md-guide/guide-page.tsx b/cvat-ui/src/components/md-guide/guide-page.tsx index bbc3a47d8ab..aabbd7b88fe 100644 --- a/cvat-ui/src/components/md-guide/guide-page.tsx +++ b/cvat-ui/src/components/md-guide/guide-page.tsx @@ -96,6 +96,8 @@ function GuidePage(): JSX.Element { ...(instanceType === 'project' ? { project_id: id } : { task_id: id }), markdown: value, }); + } else { + guideInstance.markdown = value; } setFetching(true); diff --git a/cvat-ui/src/components/project-page/details.tsx b/cvat-ui/src/components/project-page/details.tsx index 30481044877..24d29a10f99 100644 --- a/cvat-ui/src/components/project-page/details.tsx +++ b/cvat-ui/src/components/project-page/details.tsx @@ -13,6 +13,7 @@ import { getCore, Project } from 'cvat-core-wrapper'; import LabelsEditor from 'components/labels-editor/labels-editor'; import BugTrackerEditor from 'components/task-page/bug-tracker-editor'; import UserSelector from 'components/task-page/user-selector'; +import MdGuideControl from 'components/md-guide/md-guide-control'; const core = getCore(); @@ -51,6 +52,7 @@ export default function DetailsComponent(props: DetailsComponentProps): JSX.Elem {project.owner ? ` by ${project.owner.username}` : null} {` on ${moment(project.createdDate).format('MMMM Do YYYY')}`} + { diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 454cceeba67..437b0150a9d 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -2313,7 +2313,7 @@ def perform_destroy(self, instance): class AnnotationGuidesViewset( viewsets.GenericViewSet, mixins.RetrieveModelMixin, - mixins.CreateModelMixin, mixins.DestroyModelMixin + mixins.CreateModelMixin, mixins.DestroyModelMixin, mixins.UpdateModelMixin ): # todo: prefetch related for guide? queryset = AnnotationGuide.objects.order_by('-id').select_related('owner').prefetch_related('assets').all() @@ -2336,10 +2336,6 @@ def perform_create(self, serializer): owner=self.request.user, ) - def perform_update(self, instance): - # todo: update all guide_id for assets - pass - def perform_destroy(self, instance): # todo: remove all related assets from filesystem resources pass From 08a085d04e34b859ad09fd4436b4282c9e2423bc Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 23 May 2023 15:10:27 +0300 Subject: [PATCH 04/62] Attaching assets basic implementation --- cvat-core/src/api-implementation.ts | 10 ++++ cvat-core/src/api.ts | 6 +++ cvat-core/src/server-proxy.ts | 23 +++++++- cvat-core/src/server-response-types.ts | 8 +++ .../src/components/md-guide/guide-page.tsx | 52 +++++++++++++++++-- cvat/apps/engine/serializers.py | 2 +- cvat/apps/engine/views.py | 30 +++++++++-- 7 files changed, 120 insertions(+), 11 deletions(-) diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts index d643d7cf813..b92607cc2e1 100644 --- a/cvat-core/src/api-implementation.ts +++ b/cvat-core/src/api-implementation.ts @@ -27,6 +27,7 @@ import Organization from './organization'; import Webhook from './webhook'; import { ArgumentError } from './exceptions'; import AnnotationGuide from './guide'; +import { SerializedAsset } from 'server-response-types'; export default function implementAPI(cvat) { cvat.plugins.list.implementation = PluginRegistry.list; @@ -145,6 +146,15 @@ export default function implementAPI(cvat) { return new AnnotationGuide(result); }; + cvat.assets.create.implementation = async (file: File): Promise => { + if (!(file instanceof File)) { + throw new ArgumentError('Assets expect a file'); + } + + const result = await serverProxy.assets.create(file); + return result; + }; + cvat.users.get.implementation = async (filter) => { checkFilter(filter, { id: isInteger, diff --git a/cvat-core/src/api.ts b/cvat-core/src/api.ts index 4002ea0f166..e9c743c5a1e 100644 --- a/cvat-core/src/api.ts +++ b/cvat-core/src/api.ts @@ -149,6 +149,12 @@ function build() { return result; }, }, + assets: { + async create(file: File) { + const result = await PluginRegistry.apiWrapper(cvat.assets.create, file); + return result; + }, + }, jobs: { async get(filter = {}) { const result = await PluginRegistry.apiWrapper(cvat.jobs.get, filter); diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index f57e4c21da1..f92cd540fcd 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -11,7 +11,7 @@ import { SerializedLabel, SerializedAnnotationFormats, ProjectsFilter, SerializedProject, SerializedTask, TasksFilter, SerializedUser, SerializedAbout, SerializedRemoteFile, SerializedUserAgreement, - SerializedRegister, JobsFilter, SerializedJob, SerializedGuide, + SerializedRegister, JobsFilter, SerializedJob, SerializedGuide, SerializedAsset, } from 'server-response-types'; import { Storage } from './storage'; import { StorageLocation, WebhookSourceType } from './enums'; @@ -2179,6 +2179,23 @@ async function updateGuide(id: number, data: Partial): Promise< } } +async function createAsset(file: File): Promise { + const { backendAPI } = config; + const form = new FormData(); + form.append('file', file); + + try { + const response = await Axios.post(`${backendAPI}/assets`, form, { + headers: { + 'Content-Type': 'multipart/form-data', + }, + }); + return response.data; + } catch (errorData) { + throw generateError(errorData); + } +} + export default Object.freeze({ server: Object.freeze({ setAuthData, @@ -2328,4 +2345,8 @@ export default Object.freeze({ create: createGuide, update: updateGuide, }), + + assets: Object.freeze({ + create: createAsset, + }), }); diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index cf3ed60a2d9..021f0ae93ac 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -187,3 +187,11 @@ export interface SerializedGuide { updated_date: string; markdown: string; } + +export interface SerializedAsset { + uuid?: string; + guide?: number; + filename: string; + created_date: string; + owner: SerializedUser; +} diff --git a/cvat-ui/src/components/md-guide/guide-page.tsx b/cvat-ui/src/components/md-guide/guide-page.tsx index aabbd7b88fe..58277e745ff 100644 --- a/cvat-ui/src/components/md-guide/guide-page.tsx +++ b/cvat-ui/src/components/md-guide/guide-page.tsx @@ -4,7 +4,7 @@ import './styles.scss'; -import React, { useState, useEffect } from 'react'; +import React, { useState, useEffect, useRef } from 'react'; import { useLocation, useParams } from 'react-router'; import { Row, Col } from 'antd/lib/grid'; import notification from 'antd/lib/notification'; @@ -20,16 +20,17 @@ import { useIsMounted } from 'utils/hooks'; const core = getCore(); function GuidePage(): JSX.Element { - // добавить эндпоинты для получения гайда - // добавить guides пространство имен в cvat-core - // создать эндпоинт для загрузки ассетов на стороне клиента // завершить эндпоинт для загрузки ассетов на стороне сервера // сделать загрузку ассетов с сервера // сделать нормальную страничку с редактированием гайда + + // refactoring, check db performance // todo: add working with a local storage + // добавить обработку через rego файлы + const mdEditorRef = useRef(); const location = useLocation(); const isMounted = useIsMounted(); const [value, setValue] = useState(''); @@ -77,11 +78,54 @@ function GuidePage(): JSX.Element {
{ // todo: debounce setValue(val || ''); }} + onPaste={async (event: React.ClipboardEvent) => { + const { clipboardData } = event; + const { files } = clipboardData; + if (files.length) { + const selection = window.getSelection(); + if (!selection || !selection.rangeCount) return false; + event.preventDefault(); + for (const file of files) { + const { uuid } = await core.assets.create(file); + const { selectionStart, selectionEnd } = mdEditorRef.current.textarea; + let text = ''; + if (file.type.startsWith('image/')) { + text = `![image](/api/assets/${uuid})`; + } else { + text = `[${file.name}](/api/assets/${uuid})`; + } + + setValue(`${value.slice(0, selectionStart)}${text}${value.slice(selectionEnd)}`); + } + } + }} + onDrop={async (event: React.DragEvent) => { + const { dataTransfer } = event; + const { files } = dataTransfer; + if (files.length) { + const selection = window.getSelection(); + if (!selection || !selection.rangeCount) return false; + event.preventDefault(); + for (const file of files) { + const { uuid } = await core.assets.create(file); + const { selectionStart, selectionEnd } = mdEditorRef.current.textarea; + let text = ''; + if (file.type.startsWith('image/')) { + text = `![image](/api/assets/${uuid})`; + } else { + text = `[${file.name}](/api/assets/${uuid})`; + } + + setValue(`${value.slice(0, selectionStart)}${text}${value.slice(selectionEnd)}`); + } + } + }} style={{ whiteSpace: 'pre-wrap' }} />
diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 8c0dc7df0f8..5b355df8380 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -1842,7 +1842,7 @@ class Meta: read_only_fields = fields class AssetWriteSerializer(WriteOnceMixin, serializers.ModelSerializer): - uuid = serializers.CharField() + uuid = serializers.CharField(required=False) filename = serializers.CharField(required=True, max_length=1024) guide_id = serializers.IntegerField(required=False, allow_null=True) owner = BasicUserSerializer(required=False) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 437b0150a9d..4dab83d69aa 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -2290,6 +2290,7 @@ class AssetsViewset( ): # todo: prefetch related for guide? queryset = Asset.objects.select_related('owner').all() + parser_classes=_UPLOAD_PARSER_CLASSES search_fields = () ordering = "uuid" @@ -2300,10 +2301,29 @@ def get_serializer_class(self): return AssetWriteSerializer def create(self, request, *args, **kwargs): - response = super().create(request, *args, **kwargs) - # get filename, create instance, save it - # save file from request to uuid/filename - return response + file = request.data.get('file') + serializer = self.get_serializer(data={ + 'filename': file.name, + }) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + path = os.path.join(settings.ASSETS_ROOT, str(serializer.instance.uuid)) + os.makedirs(path) + with open(os.path.join(path, file.name), 'wb+') as destination: + for chunk in file.chunks(): + destination.write(chunk) + + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + + def perform_create(self, serializer): + serializer.save( + owner=self.request.user, + ) + + def retrieve(self, request, *args, **kwargs): + instance = self.get_object() + return sendfile(request, os.path.join(settings.ASSETS_ROOT, str(instance.uuid), instance.filename)) def perform_destroy(self, instance): full_path = os.path.join(instance.get_asset_dir(), instance.filename) @@ -2337,7 +2357,7 @@ def perform_create(self, serializer): ) def perform_destroy(self, instance): - # todo: remove all related assets from filesystem resources + # todo: remove all related assets from filesystem if necessary pass From a0d863ec46995ceda1962e8400c7d43d414d91b5 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 23 May 2023 19:50:46 +0300 Subject: [PATCH 05/62] Assignment assets to guides and removing unnecessary assets --- .../src/components/md-guide/guide-page.tsx | 7 +-- cvat/apps/engine/models.py | 2 +- cvat/apps/engine/signals.py | 7 ++- cvat/apps/engine/views.py | 49 ++++++++++++++++--- 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/cvat-ui/src/components/md-guide/guide-page.tsx b/cvat-ui/src/components/md-guide/guide-page.tsx index 58277e745ff..7bc40ddba30 100644 --- a/cvat-ui/src/components/md-guide/guide-page.tsx +++ b/cvat-ui/src/components/md-guide/guide-page.tsx @@ -20,11 +20,8 @@ import { useIsMounted } from 'utils/hooks'; const core = getCore(); function GuidePage(): JSX.Element { - // создать эндпоинт для загрузки ассетов на стороне клиента - // завершить эндпоинт для загрузки ассетов на стороне сервера - // сделать загрузку ассетов с сервера - - // сделать нормальную страничку с редактированием гайда + // сделать нормальную страничку с редактированием гайда\ + // отображение гайда на странице джобы // refactoring, check db performance // todo: add working with a local storage diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index f7b4ae3b771..342977631e3 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -898,4 +898,4 @@ class Asset(models.Model): guide = models.ForeignKey(AnnotationGuide, null=True, blank=True, on_delete=models.CASCADE, related_name="assets") def get_asset_dir(self): - return os.path.join(settings.ASSETS_ROOT, self.uuid) + return os.path.join(settings.ASSETS_ROOT, str(self.uuid)) diff --git a/cvat/apps/engine/signals.py b/cvat/apps/engine/signals.py index b3d3693bc6e..f6ea421abc6 100644 --- a/cvat/apps/engine/signals.py +++ b/cvat/apps/engine/signals.py @@ -8,7 +8,7 @@ from django.dispatch import receiver from .models import (CloudStorage, Data, Job, Profile, Project, - StatusChoice, Task) + StatusChoice, Task, Asset) # TODO: need to log any problems reported by shutil.rmtree when the new # analytics feature is available. Now the log system can write information @@ -46,6 +46,11 @@ def __save_user_handler(instance, **kwargs): def __delete_project_handler(instance, **kwargs): shutil.rmtree(instance.get_dirname(), ignore_errors=True) +@receiver(post_delete, sender=Asset, + dispatch_uid=__name__ + ".__delete_asset_handler") +def __delete_asset_handler(instance, **kwargs): + shutil.rmtree(instance.get_asset_dir(), ignore_errors=True) + @receiver(post_delete, sender=Task, dispatch_uid=__name__ + ".delete_task_handler") def __delete_task_handler(instance, **kwargs): diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 4dab83d69aa..1061cb8072b 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -7,6 +7,7 @@ import os import os.path as osp import pytz +import uuid import traceback from datetime import datetime from distutils.util import strtobool @@ -2305,6 +2306,7 @@ def create(self, request, *args, **kwargs): serializer = self.get_serializer(data={ 'filename': file.name, }) + serializer.is_valid(raise_exception=True) self.perform_create(serializer) path = os.path.join(settings.ASSETS_ROOT, str(serializer.instance.uuid)) @@ -2335,7 +2337,6 @@ class AnnotationGuidesViewset( viewsets.GenericViewSet, mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, mixins.UpdateModelMixin ): - # todo: prefetch related for guide? queryset = AnnotationGuide.objects.order_by('-id').select_related('owner').prefetch_related('assets').all() search_fields = () ordering = "-id" @@ -2345,6 +2346,40 @@ class AnnotationGuidesViewset( # Thus, we rely on permission-based filtering iam_organization_field = None + @staticmethod + def _update_assets(guide): + UUID_LEN = 36 + TEMPLATE = '/api/assets/' + new_assets = [] + current_assets = list(guide.assets.all()) + markdown = guide.markdown + idx = markdown.find(TEMPLATE) + while idx != -1: + _from = idx + len(TEMPLATE) + _to = _from + UUID_LEN + try: + pk = uuid.UUID(markdown[_from : _to]) + try: + asset = models.Asset.objects.get(pk=pk) + if asset not in current_assets and asset.owner != guide.owner: + raise PermissionDenied('Asset owner and guide owner are different') + new_assets.append(asset) + except models.Asset.DoesNotExist: + pass + except ValueError: + continue + finally: + idx = markdown.find(TEMPLATE, _to) + + for asset in current_assets: + if asset not in new_assets: + asset.delete() + + for asset in new_assets: + # todo: handle case when asset is already assigned to another guide + asset.guide = guide + asset.save() + def get_serializer_class(self): if self.request.method in SAFE_METHODS: return AnnotationGuideReadSerializer @@ -2352,14 +2387,12 @@ def get_serializer_class(self): return AnnotationGuideWriteSerializer def perform_create(self, serializer): - serializer.save( - owner=self.request.user, - ) - - def perform_destroy(self, instance): - # todo: remove all related assets from filesystem if necessary - pass + serializer.save(owner=self.request.user) + AnnotationGuidesViewset._update_assets(serializer.instance) + def perform_update(self, serializer): + super().perform_update(serializer) + AnnotationGuidesViewset._update_assets(serializer.instance) def rq_exception_handler(rq_job, exc_type, exc_value, tb): rq_job.exc_info = "".join( From 6313e5142929b0c1a8e041690ed6bfffe9ca6797 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 23 May 2023 19:55:21 +0300 Subject: [PATCH 06/62] Aborted unnecessary changes --- .../src/components/md-guide/guide-page.tsx | 10 +++- cvat/settings/base.py | 50 +++++++++---------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/cvat-ui/src/components/md-guide/guide-page.tsx b/cvat-ui/src/components/md-guide/guide-page.tsx index 7bc40ddba30..6c9a38b6eb2 100644 --- a/cvat-ui/src/components/md-guide/guide-page.tsx +++ b/cvat-ui/src/components/md-guide/guide-page.tsx @@ -20,12 +20,18 @@ import { useIsMounted } from 'utils/hooks'; const core = getCore(); function GuidePage(): JSX.Element { - // сделать нормальную страничку с редактированием гайда\ // отображение гайда на странице джобы + // сделать нормальную страничку с редактированием гайда + // копирование гайда с ассетами с проверкой доступа + // production + + // удаление ассетов без гайда через какой то время // refactoring, check db performance + // добавить обработку правил через rego файлы, iam организации // todo: add working with a local storage - // добавить обработку через rego файлы + // todo: Ctrl + S добавить шорткат + // добавить ассеты и гайды в админ панель const mdEditorRef = useRef(); const location = useLocation(); diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 2ccd4b879e0..baebc6180e1 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -444,16 +444,16 @@ class CVAT_QUEUES(Enum): os.makedirs(Path(IAM_OPA_BUNDLE_PATH).parent, exist_ok=True) # logging is known to be unreliable with RQ when using async transports -# vector_log_handler = os.getenv('VECTOR_EVENT_HANDLER', 'AsynchronousLogstashHandler') +vector_log_handler = os.getenv('VECTOR_EVENT_HANDLER', 'AsynchronousLogstashHandler') logstash_async_constants.QUEUED_EVENTS_FLUSH_INTERVAL = 2.0 LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'formatters': { - # 'vector': { - # 'format': '%(message)s', - # }, + 'vector': { + 'format': '%(message)s', + }, 'standard': { 'format': '[%(asctime)s] %(levelname)s %(name)s: %(message)s' } @@ -472,19 +472,19 @@ class CVAT_QUEUES(Enum): 'maxBytes': 1024*1024*50, # 50 MB 'backupCount': 5, }, - # 'vector': { - # 'level': 'INFO', - # 'class': f'logstash_async.handler.{vector_log_handler}', - # 'formatter': 'vector', - # 'transport': 'logstash_async.transport.HttpTransport', - # 'ssl_enable': False, - # 'ssl_verify': False, - # 'host': os.getenv('DJANGO_LOG_SERVER_HOST', 'localhost'), - # 'port': os.getenv('DJANGO_LOG_SERVER_PORT', 8282), - # 'version': 1, - # 'message_type': 'django', - # 'database_path': EVENTS_LOCAL_DB, - # } + 'vector': { + 'level': 'INFO', + 'class': f'logstash_async.handler.{vector_log_handler}', + 'formatter': 'vector', + 'transport': 'logstash_async.transport.HttpTransport', + 'ssl_enable': False, + 'ssl_verify': False, + 'host': os.getenv('DJANGO_LOG_SERVER_HOST', 'localhost'), + 'port': os.getenv('DJANGO_LOG_SERVER_PORT', 8282), + 'version': 1, + 'message_type': 'django', + 'database_path': EVENTS_LOCAL_DB, + } }, 'loggers': { 'cvat.server': { @@ -497,17 +497,17 @@ class CVAT_QUEUES(Enum): 'level': 'INFO', 'propagate': True }, - # 'vector': { - # 'handlers': [], - # 'level': 'INFO', - # # set True for debug - # 'propagate': False - # } + 'vector': { + 'handlers': [], + 'level': 'INFO', + # set True for debug + 'propagate': False + } }, } -# if os.getenv('DJANGO_LOG_SERVER_HOST'): -# LOGGING['loggers']['vector']['handlers'] += ['vector'] +if os.getenv('DJANGO_LOG_SERVER_HOST'): + LOGGING['loggers']['vector']['handlers'] += ['vector'] DATA_UPLOAD_MAX_MEMORY_SIZE = 100 * 1024 * 1024 # 100 MB DATA_UPLOAD_MAX_NUMBER_FIELDS = None # this django check disabled From ecab18502e3b492a2abf25dfdea8f4e671d878bd Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 24 May 2023 09:03:18 +0300 Subject: [PATCH 07/62] Added icons on annotation view --- cvat-ui/src/assets/filter-icon.svg | 8 ++++++++ cvat-ui/src/assets/fullscreen-icon.svg | 8 +++++++- cvat-ui/src/assets/guide-icon.svg | 7 +++++++ cvat-ui/src/assets/info-icon.svg | 16 +++++++++++++++- cvat-ui/src/assets/object-filter-icon.svg | 1 - .../src/components/annotation-page/styles.scss | 7 ++++--- .../annotation-page/top-bar/right-group.tsx | 12 +++++++++++- cvat-ui/src/components/md-guide/guide-page.tsx | 1 + cvat-ui/src/icons.tsx | 4 +++- 9 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 cvat-ui/src/assets/filter-icon.svg create mode 100644 cvat-ui/src/assets/guide-icon.svg delete mode 100644 cvat-ui/src/assets/object-filter-icon.svg diff --git a/cvat-ui/src/assets/filter-icon.svg b/cvat-ui/src/assets/filter-icon.svg new file mode 100644 index 00000000000..d11d1462bd0 --- /dev/null +++ b/cvat-ui/src/assets/filter-icon.svg @@ -0,0 +1,8 @@ + + + + + diff --git a/cvat-ui/src/assets/fullscreen-icon.svg b/cvat-ui/src/assets/fullscreen-icon.svg index e620d72cdf9..b5e6a28e9ea 100644 --- a/cvat-ui/src/assets/fullscreen-icon.svg +++ b/cvat-ui/src/assets/fullscreen-icon.svg @@ -1 +1,7 @@ - + + + + diff --git a/cvat-ui/src/assets/guide-icon.svg b/cvat-ui/src/assets/guide-icon.svg new file mode 100644 index 00000000000..04b8247c488 --- /dev/null +++ b/cvat-ui/src/assets/guide-icon.svg @@ -0,0 +1,7 @@ + + + + diff --git a/cvat-ui/src/assets/info-icon.svg b/cvat-ui/src/assets/info-icon.svg index 96ca1120b01..2a95977a9eb 100644 --- a/cvat-ui/src/assets/info-icon.svg +++ b/cvat-ui/src/assets/info-icon.svg @@ -1 +1,15 @@ - + + + + + + + + + + + + diff --git a/cvat-ui/src/assets/object-filter-icon.svg b/cvat-ui/src/assets/object-filter-icon.svg deleted file mode 100644 index 7ab2b729b33..00000000000 --- a/cvat-ui/src/assets/object-filter-icon.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/cvat-ui/src/components/annotation-page/styles.scss b/cvat-ui/src/components/annotation-page/styles.scss index 7b00ff9acb9..e611f404596 100644 --- a/cvat-ui/src/components/annotation-page/styles.scss +++ b/cvat-ui/src/components/annotation-page/styles.scss @@ -57,7 +57,7 @@ } > span[role='img'] { - font-size: 24px; + font-size: 20px; } &:hover > span[role='img'] { @@ -102,7 +102,7 @@ margin-right: 10px; > span { - font-size: 25px; + font-size: 20px; margin: 0 4px; color: $player-buttons-color; @@ -119,9 +119,10 @@ .cvat-player-controls { height: 100%; line-height: 27px; + padding-top: 16px; > div { - height: 50%; + height: 25%; } } diff --git a/cvat-ui/src/components/annotation-page/top-bar/right-group.tsx b/cvat-ui/src/components/annotation-page/top-bar/right-group.tsx index ab826dc1e60..645608e8c1d 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/right-group.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/right-group.tsx @@ -10,7 +10,7 @@ import Select from 'antd/lib/select'; import Button from 'antd/lib/button'; import { useSelector } from 'react-redux'; -import { FilterIcon, FullscreenIcon, InfoIcon } from 'icons'; +import { FilterIcon, FullscreenIcon, GuideIcon, InfoIcon } from 'icons'; import { DimensionType } from 'cvat-core-wrapper'; import { CombinedState, Workspace } from 'reducers'; @@ -51,6 +51,16 @@ function RightGroup(props: Props): JSX.Element { Fullscreen + - + { jobInstance.guideId !== null && ( + + )} + Back + + ); +} + +export default React.memo(GoBackButton); diff --git a/cvat-ui/src/components/md-guide/guide-page.tsx b/cvat-ui/src/components/md-guide/guide-page.tsx index 2091f5ae8da..2a36abfe853 100644 --- a/cvat-ui/src/components/md-guide/guide-page.tsx +++ b/cvat-ui/src/components/md-guide/guide-page.tsx @@ -8,31 +8,20 @@ import React, { useState, useEffect, useRef } from 'react'; import { useLocation, useParams } from 'react-router'; import { Row, Col } from 'antd/lib/grid'; import notification from 'antd/lib/notification'; -import Text from 'antd/lib/typography/Text'; import Button from 'antd/lib/button'; - +import Space from 'antd/lib/space'; import MDEditor from '@uiw/react-md-editor'; + import { getCore, Task, Project, AnnotationGuide, } from 'cvat-core-wrapper'; import { useIsMounted } from 'utils/hooks'; +import CVATLoadingSpinner from 'components/common/loading-spinner'; +import GoBackButton from 'components/common/go-back-button'; const core = getCore(); function GuidePage(): JSX.Element { - // сделать нормальную страничку с редактированием гайда - // копирование гайда с ассетами с проверкой доступа - - // production - - // удаление ассетов без гайда через какой то время - // refactoring, check db performance - // добавить обработку правил через rego файлы, iam организации - // todo: add working with a local storage - // todo: Ctrl + S добавить шорткат - // добавить ассеты и гайды в админ панель - // merge migration files - const mdEditorRef = useRef(); const location = useLocation(); const isMounted = useIsMounted(); @@ -71,106 +60,113 @@ function GuidePage(): JSX.Element { }); }, []); - // todo: add fetching overlay - return ( - <> -
- Header - {/* add back arrow */} -
-
- { - // todo: debounce - setValue(val || ''); - }} - onPaste={async (event: React.ClipboardEvent) => { - const { clipboardData } = event; - const { files } = clipboardData; - if (files.length) { - const selection = window.getSelection(); - if (!selection || !selection.rangeCount) return false; - event.preventDefault(); - for (const file of files) { - const { uuid } = await core.assets.create(file); - const { selectionStart, selectionEnd } = mdEditorRef.current.textarea; - let text = ''; - if (file.type.startsWith('image/')) { - text = `![image](/api/assets/${uuid})`; - } else { - text = `[${file.name}](/api/assets/${uuid})`; - } + + { fetching && } + +
+ +
+
+ { + // todo: debounce + setValue(val || ''); + }} + onPaste={async (event: React.ClipboardEvent) => { + const { clipboardData } = event; + const { files } = clipboardData; + if (files.length) { + const selection = window.getSelection(); + if (!selection || !selection.rangeCount) return false; + event.preventDefault(); + for (const file of files) { + const { uuid } = await core.assets.create(file); + const { selectionStart, selectionEnd } = mdEditorRef.current.textarea; + let text = ''; + if (file.type.startsWith('image/')) { + text = `![image](/api/assets/${uuid})`; + } else { + text = `[${file.name}](/api/assets/${uuid})`; + } - setValue(`${value.slice(0, selectionStart)}${text}${value.slice(selectionEnd)}`); - } - } - }} - onDrop={async (event: React.DragEvent) => { - const { dataTransfer } = event; - const { files } = dataTransfer; - if (files.length) { - const selection = window.getSelection(); - if (!selection || !selection.rangeCount) return false; - event.preventDefault(); - for (const file of files) { - const { uuid } = await core.assets.create(file); - const { selectionStart, selectionEnd } = mdEditorRef.current.textarea; - let text = ''; - if (file.type.startsWith('image/')) { - text = `![image](/api/assets/${uuid})`; - } else { - text = `[${file.name}](/api/assets/${uuid})`; + setValue(`${value.slice(0, selectionStart)}${text}${value.slice(selectionEnd)}`); } - - setValue(`${value.slice(0, selectionStart)}${text}${value.slice(selectionEnd)}`); } - } - }} - style={{ whiteSpace: 'pre-wrap' }} - /> -
-
- {/* add submit arrow */} -
+ + -
- + + setFetching(true); + guideInstance.save().then((result: AnnotationGuide) => { + if (isMounted()) { + setValue(result.markdown); + setGuide(result); + } + }).catch((error: any) => { + if (isMounted()) { + notification.error({ + message: 'Could not save guide on the server', + description: error.toString(), + }); + } + }).finally(() => { + if (isMounted()) { + setFetching(false); + } + }); + }} + > + Submit + + + + ); } diff --git a/cvat-ui/src/components/md-guide/styles.scss b/cvat-ui/src/components/md-guide/styles.scss index 7b3dd711722..e8516b124d4 100644 --- a/cvat-ui/src/components/md-guide/styles.scss +++ b/cvat-ui/src/components/md-guide/styles.scss @@ -8,4 +8,29 @@ button { margin-left: $grid-unit-size; } -} \ No newline at end of file +} + +.cvat-guide-page { + height: 100%; + + > div { + display: flex; + flex-direction: column; + height: 100%; + + .cvat-guide-page-top { + margin-top: $grid-unit-size * 2; + margin-bottom: $grid-unit-size * 2; + } + + .cvat-guide-page-editor-wrapper { + flex: 1; + } + + .cvat-guide-page-bottom { + margin-top: $grid-unit-size * 2; + margin-bottom: $grid-unit-size * 2; + justify-content: flex-end; + } + } +} diff --git a/cvat-ui/src/utils/hooks.ts b/cvat-ui/src/utils/hooks.ts index d466b354f7d..dd278d03826 100644 --- a/cvat-ui/src/utils/hooks.ts +++ b/cvat-ui/src/utils/hooks.ts @@ -8,6 +8,7 @@ import { useRef, useEffect, useState, useCallback, } from 'react'; import { useSelector } from 'react-redux'; +import { useHistory } from 'react-router'; import { CombinedState, PluginComponent } from 'reducers'; // eslint-disable-next-line import/prefer-default-export @@ -60,6 +61,19 @@ export function usePlugins( return ref.current; } +export default function useGoBack(): () => void { + const history = useHistory(); + const goBack = useCallback(() => { + if (history.location !== 'default') { + history.goBack(); + } else { + history.push('/'); + } + }, []); + + return goBack; +}; + export interface ICardHeightHOC { numberOfRows: number; paddings: number; From 388771fd69466565cb7fe3db8fe3613cbdfa316c Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 24 May 2023 11:19:59 +0300 Subject: [PATCH 12/62] Some code refactoring --- cvat-core/src/guide.ts | 2 +- .../src/components/common/go-back-button.tsx | 2 +- .../src/components/md-guide/guide-page.tsx | 62 ++++++++----------- cvat/apps/engine/models.py | 2 +- 4 files changed, 28 insertions(+), 40 deletions(-) diff --git a/cvat-core/src/guide.ts b/cvat-core/src/guide.ts index 1000604f7ac..81ac3f3084b 100644 --- a/cvat-core/src/guide.ts +++ b/cvat-core/src/guide.ts @@ -17,7 +17,7 @@ class AnnotationGuide { public readonly updatedDate?: string; public markdown: string; - constructor(initialData: SerializedGuide) { + constructor(initialData: Partial) { const data = { id: undefined, task_id: null, diff --git a/cvat-ui/src/components/common/go-back-button.tsx b/cvat-ui/src/components/common/go-back-button.tsx index 50e4a7d3df9..ff10e96e930 100644 --- a/cvat-ui/src/components/common/go-back-button.tsx +++ b/cvat-ui/src/components/common/go-back-button.tsx @@ -11,7 +11,7 @@ function GoBackButton(): JSX.Element { - Back + Back ); } diff --git a/cvat-ui/src/components/md-guide/guide-page.tsx b/cvat-ui/src/components/md-guide/guide-page.tsx index 2a36abfe853..32ab5de6de6 100644 --- a/cvat-ui/src/components/md-guide/guide-page.tsx +++ b/cvat-ui/src/components/md-guide/guide-page.tsx @@ -10,7 +10,7 @@ import { Row, Col } from 'antd/lib/grid'; import notification from 'antd/lib/notification'; import Button from 'antd/lib/button'; import Space from 'antd/lib/space'; -import MDEditor from '@uiw/react-md-editor'; +import MDEditor, { commands } from '@uiw/react-md-editor'; import { getCore, Task, Project, AnnotationGuide, @@ -22,7 +22,7 @@ import GoBackButton from 'components/common/go-back-button'; const core = getCore(); function GuidePage(): JSX.Element { - const mdEditorRef = useRef(); + const mdEditorRef = useRef(null); const location = useLocation(); const isMounted = useIsMounted(); const [value, setValue] = useState(''); @@ -60,6 +60,27 @@ function GuidePage(): JSX.Element { }); }, []); + const handleInsert = async (event: React.ClipboardEvent | React.DragEvent, files: FileList): Promise => { + if (files.length) { + event.preventDefault(); + const addedAssets = []; + if (mdEditorRef.current) { + const { textArea } = mdEditorRef.current.commandOrchestrator; + const { selectionStart, selectionEnd } = textArea; + for (const file of files) { + const { uuid } = await core.assets.create(file); + if (file.type.startsWith('image/')) { + addedAssets.push(`![image](/api/assets/${uuid})`); + } else { + addedAssets.push(`[${file.name}](/api/assets/${uuid})`); + } + } + + setValue(`${value.slice(0, selectionStart)}\n${addedAssets.join('\n')}\n${value.slice(selectionEnd)}`); + } + } + }; + return ( { - // todo: debounce setValue(val || ''); }} onPaste={async (event: React.ClipboardEvent) => { const { clipboardData } = event; const { files } = clipboardData; - if (files.length) { - const selection = window.getSelection(); - if (!selection || !selection.rangeCount) return false; - event.preventDefault(); - for (const file of files) { - const { uuid } = await core.assets.create(file); - const { selectionStart, selectionEnd } = mdEditorRef.current.textarea; - let text = ''; - if (file.type.startsWith('image/')) { - text = `![image](/api/assets/${uuid})`; - } else { - text = `[${file.name}](/api/assets/${uuid})`; - } - - setValue(`${value.slice(0, selectionStart)}${text}${value.slice(selectionEnd)}`); - } - } + handleInsert(event, files); }} onDrop={async (event: React.DragEvent) => { const { dataTransfer } = event; const { files } = dataTransfer; - if (files.length) { - const selection = window.getSelection(); - if (!selection || !selection.rangeCount) return false; - event.preventDefault(); - for (const file of files) { - const { uuid } = await core.assets.create(file); - const { selectionStart, selectionEnd } = mdEditorRef.current.textarea; - let text = ''; - if (file.type.startsWith('image/')) { - text = `![image](/api/assets/${uuid})`; - } else { - text = `[${file.name}](/api/assets/${uuid})`; - } - - setValue(`${value.slice(0, selectionStart)}${text}${value.slice(selectionEnd)}`); - } - } + handleInsert(event, files); }} style={{ whiteSpace: 'pre-wrap' }} /> diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index 342977631e3..dca5db5694f 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -886,7 +886,7 @@ class AnnotationGuide(models.Model): task = models.OneToOneField(Task, null=True, blank=True, on_delete=models.CASCADE, related_name="annotation_guide") project = models.OneToOneField(Project, null=True, blank=True, on_delete=models.CASCADE, related_name="annotation_guide") owner = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL, related_name="annotation_guides") - markdown = models.TextField(default='') + markdown = models.TextField(blank=True, default='') created_date = models.DateTimeField(auto_now_add=True) updated_date = models.DateTimeField(auto_now=True) From a9cd369d08cb00c3590017fc31a525c094882d6d Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 26 May 2023 10:48:57 +0300 Subject: [PATCH 13/62] Minor fixes --- .../components/annotation-page/top-bar/statistics-modal.tsx | 4 +--- cvat-ui/src/components/common/go-back-button.tsx | 2 +- cvat-ui/src/styles.scss | 2 ++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cvat-ui/src/components/annotation-page/top-bar/statistics-modal.tsx b/cvat-ui/src/components/annotation-page/top-bar/statistics-modal.tsx index 506a6f2e140..61edfda6ced 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/statistics-modal.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/statistics-modal.tsx @@ -142,9 +142,7 @@ function StatisticsModalComponent(props: StateToProps & DispatchToProps): JSX.El const makeShapesTracksTitle = (title: string): JSX.Element => ( - - {title} - + {title} ); diff --git a/cvat-ui/src/components/common/go-back-button.tsx b/cvat-ui/src/components/common/go-back-button.tsx index ff10e96e930..feb5aff50a1 100644 --- a/cvat-ui/src/components/common/go-back-button.tsx +++ b/cvat-ui/src/components/common/go-back-button.tsx @@ -8,7 +8,7 @@ function GoBackButton(): JSX.Element { const goBack = useGoBack(); return ( <> - Back diff --git a/cvat-ui/src/styles.scss b/cvat-ui/src/styles.scss index 78536d98e39..4f45b2e5051 100644 --- a/cvat-ui/src/styles.scss +++ b/cvat-ui/src/styles.scss @@ -69,6 +69,8 @@ hr { .cvat-info-circle-icon { color: $info-icon-color; + margin-left: $grid-unit-size; + margin-right: $grid-unit-size; } .cvat-danger-circle-icon { From e7268b3d4240412246af0153c60dc41c480b1f9d Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Sun, 28 May 2023 20:33:20 +0300 Subject: [PATCH 14/62] Added more plugins --- cvat-core/src/server-proxy.ts | 2 +- .../top-bar/player-navigation.tsx | 2 +- .../annotation-page/top-bar/top-bar.tsx | 98 ++++++++++++------- cvat-ui/src/reducers/index.ts | 5 + cvat-ui/src/reducers/plugins-reducer.ts | 5 + 5 files changed, 72 insertions(+), 40 deletions(-) diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index f92cd540fcd..c9c223b5a64 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -250,7 +250,7 @@ Axios.defaults.xsrfHeaderName = 'X-CSRFTOKEN'; Axios.defaults.xsrfCookieName = 'csrftoken'; const workerAxios = new WorkerWrappedAxios(); Axios.interceptors.request.use((reqConfig) => { - if ('params' in reqConfig && 'org' in reqConfig.params) { + if (reqConfig.url.includes('toloka') || ('params' in reqConfig && 'org' in reqConfig.params)) { return reqConfig; } diff --git a/cvat-ui/src/components/annotation-page/top-bar/player-navigation.tsx b/cvat-ui/src/components/annotation-page/top-bar/player-navigation.tsx index 9054ea023e7..ab59dbc738d 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/player-navigation.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/player-navigation.tsx @@ -10,11 +10,11 @@ import Slider from 'antd/lib/slider'; import InputNumber from 'antd/lib/input-number'; import Input from 'antd/lib/input'; import Text from 'antd/lib/typography/Text'; +import modal from 'antd/lib/modal'; import { RestoreIcon } from 'icons'; import CVATTooltip from 'components/common/cvat-tooltip'; import { clamp } from 'utils/math'; -import modal from 'antd/lib/modal'; interface Props { startFrame: number; diff --git a/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx b/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx index c231f9b6fdd..f69286d4aba 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx @@ -7,7 +7,10 @@ import React from 'react'; import Input from 'antd/lib/input'; import { Col, Row } from 'antd/lib/grid'; -import { ActiveControl, ToolsBlockerState, Workspace } from 'reducers'; +import { + ActiveControl, CombinedState, ToolsBlockerState, Workspace, +} from 'reducers'; +import { usePlugins } from 'utils/hooks'; import LeftGroup from './left-group'; import PlayerButtons from './player-buttons'; import PlayerNavigation from './player-navigation'; @@ -122,6 +125,60 @@ export default function AnnotationTopBarComponent(props: Props): JSX.Element { jobInstance, } = props; + const playerPlugins = usePlugins( + (state: CombinedState) => state.plugins.components.annotationPage.header.player, props, + ); + const playerItems: [JSX.Element, number][] = []; + playerItems.push( + ...playerPlugins.map(({ component: Component, weight }, index) => { + const component = ; + return [component, weight] as [JSX.Element, number]; + }), + ); + + playerItems.push([( + + ), 0]); + + playerItems.push([( + + ), 10]); + return ( - - + { playerItems.sort((menuItem1, menuItem2) => menuItem1[1] - menuItem2[1]) + .map((menuItem) => menuItem[0]) } Date: Tue, 30 May 2023 05:04:47 +0300 Subject: [PATCH 15/62] added missed migration file --- .../0070_alter_annotationguide_markdown.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 cvat/apps/engine/migrations/0070_alter_annotationguide_markdown.py diff --git a/cvat/apps/engine/migrations/0070_alter_annotationguide_markdown.py b/cvat/apps/engine/migrations/0070_alter_annotationguide_markdown.py new file mode 100644 index 00000000000..947e1d30121 --- /dev/null +++ b/cvat/apps/engine/migrations/0070_alter_annotationguide_markdown.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.18 on 2023-05-30 00:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0069_annotationguide_markdown'), + ] + + operations = [ + migrations.AlterField( + model_name='annotationguide', + name='markdown', + field=models.TextField(blank=True, default=''), + ), + ] From 33d466eb181bef129f7388738a8732b5b6a6ce33 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 8 Jun 2023 10:14:38 +0300 Subject: [PATCH 16/62] Implemented basic rules for sandbox --- cvat-core/src/server-proxy.ts | 2 +- .../migrations/0071_asset_access_type.py | 19 +++ cvat/apps/engine/models.py | 16 +++ cvat/apps/engine/serializers.py | 7 +- cvat/apps/engine/views.py | 13 +- cvat/apps/iam/permissions.py | 54 ++++++++ cvat/apps/iam/rules/annotationguides.rego | 124 ++++++++++++++++++ 7 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 cvat/apps/engine/migrations/0071_asset_access_type.py create mode 100644 cvat/apps/iam/rules/annotationguides.rego diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index c9c223b5a64..f92cd540fcd 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -250,7 +250,7 @@ Axios.defaults.xsrfHeaderName = 'X-CSRFTOKEN'; Axios.defaults.xsrfCookieName = 'csrftoken'; const workerAxios = new WorkerWrappedAxios(); Axios.interceptors.request.use((reqConfig) => { - if (reqConfig.url.includes('toloka') || ('params' in reqConfig && 'org' in reqConfig.params)) { + if ('params' in reqConfig && 'org' in reqConfig.params) { return reqConfig; } diff --git a/cvat/apps/engine/migrations/0071_asset_access_type.py b/cvat/apps/engine/migrations/0071_asset_access_type.py new file mode 100644 index 00000000000..ef6178fbc3e --- /dev/null +++ b/cvat/apps/engine/migrations/0071_asset_access_type.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.18 on 2023-06-07 13:27 + +import cvat.apps.engine.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0070_alter_annotationguide_markdown'), + ] + + operations = [ + migrations.AddField( + model_name='asset', + name='access_type', + field=models.CharField(choices=[('ANONYMOUS_ACCESS', 'ANONYMOUS_ACCESS'), ('PRIVATE_ACCESS', 'PRIVATE_ACCESS')], default=cvat.apps.engine.models.AssetAccessType['PRIVATE_ACCESS'], max_length=16), + ), + ] diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index dca5db5694f..952be2dc5da 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -890,8 +890,24 @@ class AnnotationGuide(models.Model): created_date = models.DateTimeField(auto_now_add=True) updated_date = models.DateTimeField(auto_now=True) +class AssetAccessType(str, Enum): + ANONYMOUS_ACCESS = 'ANONYMOUS_ACCESS' + PRIVATE_ACCESS = 'PRIVATE_ACCESS' + + @classmethod + def choices(cls): + return tuple((x.value, x.name) for x in cls) + + @classmethod + def list(cls): + return list(map(lambda x: x.value, cls)) + + def __str__(self): + return self.value + class Asset(models.Model): uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + access_type = models.CharField(max_length=16, default=AssetAccessType.PRIVATE_ACCESS, choices=AssetAccessType.choices()) filename = models.CharField(max_length=1024) created_date = models.DateTimeField(auto_now_add=True) owner = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL, related_name="assets") diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 5b355df8380..54a3d2b80e4 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -1838,19 +1838,20 @@ class AssetReadSerializer(WriteOnceMixin, serializers.ModelSerializer): class Meta: model = models.Asset - fields = ('uuid', 'filename', 'created_date', 'owner', 'guide_id',) + fields = ('uuid', 'filename', 'created_date', 'owner', 'guide_id', 'access_type', ) read_only_fields = fields class AssetWriteSerializer(WriteOnceMixin, serializers.ModelSerializer): uuid = serializers.CharField(required=False) filename = serializers.CharField(required=True, max_length=1024) + access_type = serializers.CharField(required=True, max_length=16) guide_id = serializers.IntegerField(required=False, allow_null=True) owner = BasicUserSerializer(required=False) class Meta: model = models.Asset - fields = ('uuid', 'filename', 'created_date', 'owner', 'guide_id', ) - write_once_fields = ('uuid', 'filename', 'created_date', 'owner', ) + fields = ('uuid', 'filename', 'created_date', 'owner', 'access_type', 'guide_id', ) + write_once_fields = ('uuid', 'filename', 'created_date', 'owner', 'access_type', ) class AnnotationGuideReadSerializer(WriteOnceMixin, serializers.ModelSerializer): owner = BasicUserSerializer(required=False) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 1061cb8072b..c4313dee93f 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -78,7 +78,7 @@ from .log import slogger from cvat.apps.iam.permissions import (CloudStoragePermission, CommentPermission, IssuePermission, JobPermission, LabelPermission, ProjectPermission, - TaskPermission, UserPermission) + TaskPermission, UserPermission, AnnotationGuidePermission) from cvat.apps.engine.cache import MediaCache from cvat.apps.events.handlers import handle_annotations_patch from cvat.apps.engine.view_utils import tus_chunk_action @@ -2337,7 +2337,7 @@ class AnnotationGuidesViewset( viewsets.GenericViewSet, mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, mixins.UpdateModelMixin ): - queryset = AnnotationGuide.objects.order_by('-id').select_related('owner').prefetch_related('assets').all() + queryset = AnnotationGuide.objects.order_by('-id').select_related('owner', 'project', 'project__owner', 'project__organization', 'task', 'task__owner', 'task__organization').prefetch_related('assets').all() search_fields = () ordering = "-id" @@ -2346,6 +2346,15 @@ class AnnotationGuidesViewset( # Thus, we rely on permission-based filtering iam_organization_field = None + def get_queryset(self): + queryset = super().get_queryset() + + if self.action == 'list': + perm = AnnotationGuidePermission.create_scope_list(self.request) + queryset = perm.filter(queryset) + + return queryset + @staticmethod def _update_assets(guide): UUID_LEN = 36 diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index 5e336a344c5..0445f994c3f 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -1554,6 +1554,60 @@ def get_resource(self): return data +class AnnotationGuidePermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + VIEW = 'view' + UPDATE = 'update' + DELETE = 'delete' + CREATE = 'create' + + @classmethod + def create(cls, request, view, obj): + permissions = [] + if view.basename == 'annotationguide': + for scope in cls.get_scopes(request, view, obj): + self = cls.create_base_perm(request, view, scope, obj) + permissions.append(self) + + return permissions + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.url = settings.IAM_OPA_DATA_URL + '/annotationguides/allow' + + @staticmethod + def get_scopes(request, view, obj): + Scopes = __class__.Scopes + return [{ + 'create': Scopes.CREATE, + 'destroy': Scopes.DELETE, + 'partial_update': Scopes.UPDATE, + 'retrieve': Scopes.VIEW, + }.get(view.action, None)] + + def get_resource(self): + data = {} + if self.obj: + db_project = getattr(self.obj, 'project', {}) + db_task = getattr(self.obj, 'task', {}) + db_organization = getattr(db_project, 'organization', None) or getattr(db_task, 'organization', None) or {} + data.update({ + 'id': self.obj.id, + 'owner': { 'id': getattr(self.obj.owner, 'id', None) }, + 'project': { + 'owner': { 'id': getattr(getattr(db_project, 'owner', {}), 'id', None) }, + 'assignee': { 'id': getattr(getattr(db_project, 'assignee', {}), 'id', None) } + }, + 'task': { + 'owner': { 'id': getattr(getattr(db_task, 'owner', {}), 'id', None) }, + 'assignee': { 'id': getattr(getattr(db_task, 'assignee', {}), 'id', None) } + }, + 'organization': { 'id': getattr(db_organization, 'id', None) } + }) + elif self.scope == __class__.Scopes.CREATE: + pass + + return data class PolicyEnforcer(BasePermission): # pylint: disable=no-self-use diff --git a/cvat/apps/iam/rules/annotationguides.rego b/cvat/apps/iam/rules/annotationguides.rego new file mode 100644 index 00000000000..cceb24b8a6a --- /dev/null +++ b/cvat/apps/iam/rules/annotationguides.rego @@ -0,0 +1,124 @@ +package annotationguides + +import data.utils +import data.organizations + +# input: { +# "scope": <"view"|"update"|"delete"|"create"> or null, +# "auth": { +# "user": { +# "id": , +# "privilege": <"admin"|"business"|"user"|"worker"> or null +# }, +# "organization": { +# "id": , +# "owner": { +# "id": +# }, +# "user": { +# "role": <"owner"|"maintainer"|"supervisor"|"worker"> or null +# } +# } or null, +# }, +# "resource": { +# "id": , +# "owner": { "id": }, +# "organization": { "id": or null }, +# "task": { +# "owner": { "id": or null }, +# "assignee": { "id": or null }, +# }, +# "project": { +# "owner": { "id": or null }, +# "assignee": { "id": or null }, +# }, +# } +# } + +is_task_owner { + input.resource.owner.id == input.auth.user.id +} + +is_task_assignee { + input.resource.assignee.id == input.auth.user.id +} + +is_project_owner { + input.resource.project.owner.id == input.auth.user.id +} + +is_project_assignee { + input.resource.project.assignee.id == input.auth.user.id +} + +is_project_staff { + is_project_owner +} + +is_project_staff { + is_project_assignee +} + +is_task_staff { + is_task_owner +} + +is_task_staff { + is_task_assignee +} + +is_target_staff { + is_project_staff +} + +is_target_staff { + is_task_staff +} + +default allow = false + +allow { + utils.is_admin +} + +allow { + input.scope == utils.CREATE + utils.is_sandbox + is_target_staff +} + +allow { + input.scope == utils.DELETE + utils.is_sandbox + is_target_staff +} + +allow { + input.scope == utils.DELETE + utils.is_sandbox + utils.is_resource_owner +} + +allow { + input.scope == utils.UPDATE + utils.is_sandbox + is_target_staff +} + +allow { + input.scope == utils.UPDATE + utils.is_sandbox + utils.is_resource_owner +} + +allow { + input.scope == utils.VIEW + utils.is_sandbox + is_target_staff +} + +allow { + input.scope == utils.VIEW + utils.is_sandbox + utils.is_resource_owner +} From 32d29c477c45659c2966aa890576fb25594596df Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 8 Jun 2023 10:38:44 +0300 Subject: [PATCH 17/62] Added create permissions --- cvat/apps/engine/serializers.py | 4 --- cvat/apps/iam/permissions.py | 45 +++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 54a3d2b80e4..190fa86285d 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -1882,9 +1882,6 @@ def create(self, validated_data): project = models.Project.objects.get(id=project_id) except models.Project.DoesNotExist: raise serializers.ValidationError(f'The specified project #{project_id} does not exist.') - print(project) - - # todo: check patch project permissions if task_id is not None: try: @@ -1892,7 +1889,6 @@ def create(self, validated_data): except models.Task.DoesNotExist: raise serializers.ValidationError(f'The specified task #{task_id} does not exist.') print(task) - # todo: check patch task permissions db_data = models.AnnotationGuide.objects.create(**validated_data, project = project, task = task) return db_data diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index 0445f994c3f..434c6aebf79 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -1565,8 +1565,12 @@ class Scopes(StrEnum): def create(cls, request, view, obj): permissions = [] if view.basename == 'annotationguide': + project_id = request.data.get('project_id') + task_id = request.data.get('task_id') + params = { 'project_id': project_id, 'task_id': task_id } + for scope in cls.get_scopes(request, view, obj): - self = cls.create_base_perm(request, view, scope, obj) + self = cls.create_base_perm(request, view, scope, obj, **params) permissions.append(self) return permissions @@ -1605,7 +1609,44 @@ def get_resource(self): 'organization': { 'id': getattr(db_organization, 'id', None) } }) elif self.scope == __class__.Scopes.CREATE: - pass + data.update({ + 'project': { + 'owner': { 'id': None }, + 'assignee': { 'id': None } + }, + 'task': { + 'owner': { 'id': None }, + 'assignee': { 'id': None } + }, + 'organization': { 'id': None } + }) + + if self.project_id is not None: + try: + db_project = Project.objects.get(id=self.project_id) + db_organization = getattr(db_project, 'organization', {}) + data.update({ + 'project': { + 'owner': { 'id': db_project.owner.id }, + 'assignee': { 'id': getattr(db_project.assignee, 'id', None) } + }, + 'organization': { 'id': getattr(db_organization, 'id', None) } + }) + except Project.DoesNotExist as ex: + raise ValidationError(str(ex)) + elif self.task_id is not None: + try: + db_task = Task.objects.get(id=self.project_id) + db_organization = getattr(db_task, 'organization', {}) + data.update({ + 'task': { + 'owner': { 'id': db_task.owner.id }, + 'assignee': { 'id': getattr(db_task.assignee, 'id', None) } + }, + 'organization': { 'id': getattr(db_organization, 'id', None) } + }) + except Task.DoesNotExist as ex: + raise ValidationError(str(ex)) return data From 14654289e50650241f973c110a3e5520b32697c8 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 8 Jun 2023 11:57:40 +0300 Subject: [PATCH 18/62] Implemented permissions for annotator --- cvat-core/src/api-implementation.ts | 10 --------- cvat-core/src/api.ts | 6 ----- cvat-core/src/project-implementation.ts | 10 +++++++++ cvat-core/src/project.ts | 6 +++++ cvat-core/src/server-proxy.ts | 4 ++-- cvat-core/src/session-implementation.ts | 19 ++++++++++++++++ cvat-core/src/session.ts | 11 ++++++++++ .../annotation-page/top-bar/right-group.tsx | 2 +- .../src/components/md-guide/guide-page.tsx | 10 +++------ cvat/apps/engine/views.py | 22 +++++++++++++------ cvat/apps/iam/permissions.py | 11 ++++++++-- 11 files changed, 76 insertions(+), 35 deletions(-) diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts index b92607cc2e1..5469d2a1e52 100644 --- a/cvat-core/src/api-implementation.ts +++ b/cvat-core/src/api-implementation.ts @@ -136,16 +136,6 @@ export default function implementAPI(cvat) { return result; }; - cvat.guides.get.implementation = async (filter: { id: number }) => { - if (!('id' in filter)) { - throw new ArgumentError('Guide id was not provided'); - } - checkFilter(filter, { id: isInteger }); - - const result = await serverProxy.guides.get(filter.id); - return new AnnotationGuide(result); - }; - cvat.assets.create.implementation = async (file: File): Promise => { if (!(file instanceof File)) { throw new ArgumentError('Assets expect a file'); diff --git a/cvat-core/src/api.ts b/cvat-core/src/api.ts index e9c743c5a1e..84fddfb55e0 100644 --- a/cvat-core/src/api.ts +++ b/cvat-core/src/api.ts @@ -143,12 +143,6 @@ function build() { return result; }, }, - guides: { - async get(filter: { id: number }) { - const result = await PluginRegistry.apiWrapper(cvat.guides.get, filter); - return result; - }, - }, assets: { async create(file: File) { const result = await PluginRegistry.apiWrapper(cvat.assets.create, file); diff --git a/cvat-core/src/project-implementation.ts b/cvat-core/src/project-implementation.ts index 202f56ab701..17e0c953d1e 100644 --- a/cvat-core/src/project-implementation.ts +++ b/cvat-core/src/project-implementation.ts @@ -10,6 +10,7 @@ import Project from './project'; import { exportDataset, importDataset } from './annotations'; import { SerializedLabel } from './server-response-types'; import { Label } from './labels'; +import AnnotationGuide from './guide'; export default function implementProject(projectClass) { projectClass.prototype.save.implementation = async function () { @@ -125,5 +126,14 @@ export default function implementProject(projectClass) { return result; }; + projectClass.prototype.guide.implementation = async function guide() { + if (this.guideId === null) { + return null; + } + + const result = await serverProxy.guides.get(this.guideId); + return new AnnotationGuide(result); + }; + return projectClass; } diff --git a/cvat-core/src/project.ts b/cvat-core/src/project.ts index 270786e1833..71ac6a07a7f 100644 --- a/cvat-core/src/project.ts +++ b/cvat-core/src/project.ts @@ -12,6 +12,7 @@ import { ArgumentError } from './exceptions'; import { Label } from './labels'; import User from './user'; import { FieldUpdateTrigger } from './common'; +import AnnotationGuide from './guide'; export default class Project { public readonly id: number; @@ -235,6 +236,11 @@ export default class Project { const result = await PluginRegistry.apiWrapper.call(this, Project.restore, storage, file); return result; } + + async guide(): Promise { + const result = await PluginRegistry.apiWrapper.call(this, Project.prototype.guide); + return result; + } } Object.defineProperties( diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index f92cd540fcd..0323f002ce1 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -2146,11 +2146,11 @@ async function receiveWebhookEvents(type: WebhookSourceType): Promise } } -async function getGuide(id: number): Promise { +async function getGuide(id: number, params = {}): Promise { const { backendAPI } = config; try { - const response = await Axios.get(`${backendAPI}/guides/${id}`); + const response = await Axios.get(`${backendAPI}/guides/${id}`, { params }); return response.data; } catch (errorData) { throw generateError(errorData); diff --git a/cvat-core/src/session-implementation.ts b/cvat-core/src/session-implementation.ts index bdde1c7c21d..767d559e457 100644 --- a/cvat-core/src/session-implementation.ts +++ b/cvat-core/src/session-implementation.ts @@ -34,6 +34,7 @@ import { freezeHistory, clearActions, getActions, clearCache, getHistory, } from './annotations'; +import AnnotationGuide from './guide'; // must be called with task/job context async function deleteFrameWrapper(jobID, frame) { @@ -371,6 +372,15 @@ export function implementJob(Job) { return this; }; + Job.prototype.guide.implementation = async function guide() { + if (this.guideId === null) { + return null; + } + + const result = await serverProxy.guides.get(this.guideId, { job_id: this.id }); + return new AnnotationGuide(result); + }; + return Job; } @@ -810,5 +820,14 @@ export function implementTask(Task) { return result; }; + Task.prototype.guide.implementation = async function guide() { + if (this.guideId === null) { + return null; + } + + const result = await serverProxy.guides.get(this.guideId); + return new AnnotationGuide(result); + }; + return Task; } diff --git a/cvat-core/src/session.ts b/cvat-core/src/session.ts index 77d4d5f2e2d..ac092988f9b 100644 --- a/cvat-core/src/session.ts +++ b/cvat-core/src/session.ts @@ -16,6 +16,7 @@ import { Label } from './labels'; import User from './user'; import { FieldUpdateTrigger } from './common'; import { SerializedJob, SerializedTask } from 'server-response-types'; +import AnnotationGuide from './guide'; function buildDuplicatedAPI(prototype) { Object.defineProperties(prototype, { @@ -565,6 +566,11 @@ export class Job extends Session { return result; } + async guide(): Promise { + const result = await PluginRegistry.apiWrapper.call(this, Job.prototype.guide); + return result; + } + async openIssue(issue, message) { const result = await PluginRegistry.apiWrapper.call(this, Job.prototype.openIssue, issue, message); return result; @@ -1099,6 +1105,11 @@ export class Task extends Session { const result = await PluginRegistry.apiWrapper.call(this, Task.restore, storage, file); return result; } + + async guide(): Promise { + const result = await PluginRegistry.apiWrapper.call(this, Task.prototype.guide); + return result; + } } buildDuplicatedAPI(Job.prototype); diff --git a/cvat-ui/src/components/annotation-page/top-bar/right-group.tsx b/cvat-ui/src/components/annotation-page/top-bar/right-group.tsx index 2cd6bd6d0c8..c9dbc40cd92 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/right-group.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/right-group.tsx @@ -64,7 +64,7 @@ function RightGroup(props: Props): JSX.Element { className='cvat-annotation-header-guide-button cvat-annotation-header-button' onClick={async (): Promise => { const PADDING = Math.min(window.screen.availHeight, window.screen.availWidth) * 0.4; - const guide = await core.guides.get({ id: jobInstance.guideId }); + const guide = await jobInstance.guide(); Modal.info({ icon: null, width: window.screen.availWidth - PADDING, diff --git a/cvat-ui/src/components/md-guide/guide-page.tsx b/cvat-ui/src/components/md-guide/guide-page.tsx index 32ab5de6de6..07bdca52eba 100644 --- a/cvat-ui/src/components/md-guide/guide-page.tsx +++ b/cvat-ui/src/components/md-guide/guide-page.tsx @@ -34,13 +34,9 @@ function GuidePage(): JSX.Element { useEffect(() => { setFetching(true); const promise = instanceType === 'project' ? core.projects.get({ id }) : core.tasks.get({ id }); - promise.then(([instance]: [Task | Project]) => { - const { guideId } = instance; - if (guideId !== null) { - return core.guides.get({ id: guideId }); - } - return Promise.resolve(null); - }).then((guideInstance: AnnotationGuide | null) => { + promise.then(([instance]: [Task | Project]) => ( + instance.guide() + )).then((guideInstance: AnnotationGuide | null) => { if (guideInstance && isMounted()) { setValue(guideInstance.markdown); setGuide(guideInstance); diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index c4313dee93f..bffb99b8f57 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -2346,14 +2346,22 @@ class AnnotationGuidesViewset( # Thus, we rely on permission-based filtering iam_organization_field = None - def get_queryset(self): - queryset = super().get_queryset() - - if self.action == 'list': - perm = AnnotationGuidePermission.create_scope_list(self.request) - queryset = perm.filter(queryset) + def check_object_permissions(self, request, obj): + if self.action == 'retrieve': + job_id = self.request.GET.get('job_id', None) + if job_id is not None: + # NOTE: This filter is too complex to be implemented by other means + # It requires the following filter query: + # ( + # project__task__segment__job__id = job_id + # OR + # task__segment__job__id = job_id + # ) + db_job = Job.objects.select_related('segment', 'segment__task', 'segment__task__project').get(id=job_id) + super().check_object_permissions(request, db_job) + return - return queryset + super().check_object_permissions(request, obj) @staticmethod def _update_assets(guide): diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index 434c6aebf79..59f0bf0968b 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -1563,15 +1563,22 @@ class Scopes(StrEnum): @classmethod def create(cls, request, view, obj): + Scopes = __class__.Scopes permissions = [] + if view.basename == 'annotationguide': project_id = request.data.get('project_id') task_id = request.data.get('task_id') params = { 'project_id': project_id, 'task_id': task_id } for scope in cls.get_scopes(request, view, obj): - self = cls.create_base_perm(request, view, scope, obj, **params) - permissions.append(self) + if scope == Scopes.VIEW and isinstance(obj, Job): + permissions.append(JobPermission.create_base_perm( + request, view, scope=JobPermission.Scopes.VIEW, obj=obj, + )) + else: + self = cls.create_base_perm(request, view, scope, obj, **params) + permissions.append(self) return permissions From 6643533011bcc10db3430a713526c9f927784e89 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 8 Jun 2023 12:09:22 +0300 Subject: [PATCH 19/62] Update mixing -> PartialUpdate --- cvat/apps/engine/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index bffb99b8f57..87403920c35 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -2335,7 +2335,7 @@ def perform_destroy(self, instance): class AnnotationGuidesViewset( viewsets.GenericViewSet, mixins.RetrieveModelMixin, - mixins.CreateModelMixin, mixins.DestroyModelMixin, mixins.UpdateModelMixin + mixins.CreateModelMixin, mixins.DestroyModelMixin, PartialUpdateModelMixin ): queryset = AnnotationGuide.objects.order_by('-id').select_related('owner', 'project', 'project__owner', 'project__organization', 'task', 'task__owner', 'task__organization').prefetch_related('assets').all() search_fields = () From bb19139473c44e90beddc94a7f3572df821dab19 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 8 Jun 2023 13:12:14 +0300 Subject: [PATCH 20/62] Squashed migrations --- .../migrations/0068_annotationguide_asset.py | 5 +++-- .../0069_annotationguide_markdown.py | 18 ------------------ .../0070_alter_annotationguide_markdown.py | 18 ------------------ .../migrations/0071_asset_access_type.py | 19 ------------------- cvat/apps/engine/models.py | 18 +----------------- cvat/apps/engine/serializers.py | 9 ++++----- 6 files changed, 8 insertions(+), 79 deletions(-) delete mode 100644 cvat/apps/engine/migrations/0069_annotationguide_markdown.py delete mode 100644 cvat/apps/engine/migrations/0070_alter_annotationguide_markdown.py delete mode 100644 cvat/apps/engine/migrations/0071_asset_access_type.py diff --git a/cvat/apps/engine/migrations/0068_annotationguide_asset.py b/cvat/apps/engine/migrations/0068_annotationguide_asset.py index bcfa4a133e6..1f70e42eba6 100644 --- a/cvat/apps/engine/migrations/0068_annotationguide_asset.py +++ b/cvat/apps/engine/migrations/0068_annotationguide_asset.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.18 on 2023-05-22 11:46 +# Generated by Django 3.2.18 on 2023-06-08 10:10 from django.conf import settings from django.db import migrations, models @@ -18,6 +18,7 @@ class Migration(migrations.Migration): name='AnnotationGuide', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('markdown', models.TextField(blank=True, default='')), ('created_date', models.DateTimeField(auto_now_add=True)), ('updated_date', models.DateTimeField(auto_now=True)), ('owner', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='annotation_guides', to=settings.AUTH_USER_MODEL)), @@ -31,7 +32,7 @@ class Migration(migrations.Migration): ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ('filename', models.CharField(max_length=1024)), ('created_date', models.DateTimeField(auto_now_add=True)), - ('guide', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='assets', to='engine.annotationguide')), + ('guide', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='assets', to='engine.annotationguide')), ('owner', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='assets', to=settings.AUTH_USER_MODEL)), ], ), diff --git a/cvat/apps/engine/migrations/0069_annotationguide_markdown.py b/cvat/apps/engine/migrations/0069_annotationguide_markdown.py deleted file mode 100644 index 123a770d55c..00000000000 --- a/cvat/apps/engine/migrations/0069_annotationguide_markdown.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.18 on 2023-05-22 14:55 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('engine', '0068_annotationguide_asset'), - ] - - operations = [ - migrations.AddField( - model_name='annotationguide', - name='markdown', - field=models.TextField(default=''), - ), - ] diff --git a/cvat/apps/engine/migrations/0070_alter_annotationguide_markdown.py b/cvat/apps/engine/migrations/0070_alter_annotationguide_markdown.py deleted file mode 100644 index 947e1d30121..00000000000 --- a/cvat/apps/engine/migrations/0070_alter_annotationguide_markdown.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.18 on 2023-05-30 00:15 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('engine', '0069_annotationguide_markdown'), - ] - - operations = [ - migrations.AlterField( - model_name='annotationguide', - name='markdown', - field=models.TextField(blank=True, default=''), - ), - ] diff --git a/cvat/apps/engine/migrations/0071_asset_access_type.py b/cvat/apps/engine/migrations/0071_asset_access_type.py deleted file mode 100644 index ef6178fbc3e..00000000000 --- a/cvat/apps/engine/migrations/0071_asset_access_type.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.2.18 on 2023-06-07 13:27 - -import cvat.apps.engine.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('engine', '0070_alter_annotationguide_markdown'), - ] - - operations = [ - migrations.AddField( - model_name='asset', - name='access_type', - field=models.CharField(choices=[('ANONYMOUS_ACCESS', 'ANONYMOUS_ACCESS'), ('PRIVATE_ACCESS', 'PRIVATE_ACCESS')], default=cvat.apps.engine.models.AssetAccessType['PRIVATE_ACCESS'], max_length=16), - ), - ] diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index 952be2dc5da..a5e6ade9545 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -890,28 +890,12 @@ class AnnotationGuide(models.Model): created_date = models.DateTimeField(auto_now_add=True) updated_date = models.DateTimeField(auto_now=True) -class AssetAccessType(str, Enum): - ANONYMOUS_ACCESS = 'ANONYMOUS_ACCESS' - PRIVATE_ACCESS = 'PRIVATE_ACCESS' - - @classmethod - def choices(cls): - return tuple((x.value, x.name) for x in cls) - - @classmethod - def list(cls): - return list(map(lambda x: x.value, cls)) - - def __str__(self): - return self.value - class Asset(models.Model): uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - access_type = models.CharField(max_length=16, default=AssetAccessType.PRIVATE_ACCESS, choices=AssetAccessType.choices()) filename = models.CharField(max_length=1024) created_date = models.DateTimeField(auto_now_add=True) owner = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL, related_name="assets") - guide = models.ForeignKey(AnnotationGuide, null=True, blank=True, on_delete=models.CASCADE, related_name="assets") + guide = models.ForeignKey(AnnotationGuide, on_delete=models.CASCADE, related_name="assets") def get_asset_dir(self): return os.path.join(settings.ASSETS_ROOT, str(self.uuid)) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 190fa86285d..dc45a086000 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -1838,20 +1838,19 @@ class AssetReadSerializer(WriteOnceMixin, serializers.ModelSerializer): class Meta: model = models.Asset - fields = ('uuid', 'filename', 'created_date', 'owner', 'guide_id', 'access_type', ) + fields = ('uuid', 'filename', 'created_date', 'owner', 'guide_id', ) read_only_fields = fields class AssetWriteSerializer(WriteOnceMixin, serializers.ModelSerializer): uuid = serializers.CharField(required=False) filename = serializers.CharField(required=True, max_length=1024) - access_type = serializers.CharField(required=True, max_length=16) - guide_id = serializers.IntegerField(required=False, allow_null=True) + guide_id = serializers.IntegerField(required=True) owner = BasicUserSerializer(required=False) class Meta: model = models.Asset - fields = ('uuid', 'filename', 'created_date', 'owner', 'access_type', 'guide_id', ) - write_once_fields = ('uuid', 'filename', 'created_date', 'owner', 'access_type', ) + fields = ('uuid', 'filename', 'created_date', 'owner', 'guide_id', ) + write_once_fields = ('uuid', 'filename', 'created_date', 'owner', 'guide_id', ) class AnnotationGuideReadSerializer(WriteOnceMixin, serializers.ModelSerializer): owner = BasicUserSerializer(required=False) From 1c5e3ee28f0cb92521915c0af2f43e76272193c4 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 8 Jun 2023 13:27:26 +0300 Subject: [PATCH 21/62] Assets always associated with a guide --- cvat-core/src/api-implementation.ts | 4 ++-- cvat-core/src/api.ts | 4 ++-- cvat-core/src/server-proxy.ts | 3 ++- .../src/components/md-guide/guide-page.tsx | 21 +++++++++++++------ cvat/apps/engine/views.py | 2 ++ 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts index 5469d2a1e52..58246e0c9d8 100644 --- a/cvat-core/src/api-implementation.ts +++ b/cvat-core/src/api-implementation.ts @@ -136,12 +136,12 @@ export default function implementAPI(cvat) { return result; }; - cvat.assets.create.implementation = async (file: File): Promise => { + cvat.assets.create.implementation = async (file: File, guideId: number): Promise => { if (!(file instanceof File)) { throw new ArgumentError('Assets expect a file'); } - const result = await serverProxy.assets.create(file); + const result = await serverProxy.assets.create(file, guideId); return result; }; diff --git a/cvat-core/src/api.ts b/cvat-core/src/api.ts index 84fddfb55e0..253b51980ea 100644 --- a/cvat-core/src/api.ts +++ b/cvat-core/src/api.ts @@ -144,8 +144,8 @@ function build() { }, }, assets: { - async create(file: File) { - const result = await PluginRegistry.apiWrapper(cvat.assets.create, file); + async create(file: File, guideId: number) { + const result = await PluginRegistry.apiWrapper(cvat.assets.create, file, guideId); return result; }, }, diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 0323f002ce1..3421a12fb0d 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -2179,10 +2179,11 @@ async function updateGuide(id: number, data: Partial): Promise< } } -async function createAsset(file: File): Promise { +async function createAsset(file: File, guideId: number): Promise { const { backendAPI } = config; const form = new FormData(); form.append('file', file); + form.append('guide_id', guideId); try { const response = await Axios.post(`${backendAPI}/assets`, form, { diff --git a/cvat-ui/src/components/md-guide/guide-page.tsx b/cvat-ui/src/components/md-guide/guide-page.tsx index 07bdca52eba..eade3c07770 100644 --- a/cvat-ui/src/components/md-guide/guide-page.tsx +++ b/cvat-ui/src/components/md-guide/guide-page.tsx @@ -26,10 +26,15 @@ function GuidePage(): JSX.Element { const location = useLocation(); const isMounted = useIsMounted(); const [value, setValue] = useState(''); - const [guide, setGuide] = useState(null); - const [fetching, setFetching] = useState(false); - const id = +useParams<{ id: string }>().id; const instanceType = location.pathname.includes('projects') ? 'project' : 'task'; + const id = +useParams<{ id: string }>().id; + const [guide, setGuide] = useState( + new AnnotationGuide({ + ...(instanceType === 'project' ? { project_id: id } : { task_id: id }), + markdown: value, + }), + ); + const [fetching, setFetching] = useState(false); useEffect(() => { setFetching(true); @@ -37,7 +42,12 @@ function GuidePage(): JSX.Element { promise.then(([instance]: [Task | Project]) => ( instance.guide() )).then((guideInstance: AnnotationGuide | null) => { - if (guideInstance && isMounted()) { + if (guideInstance) { + return (async () => guideInstance)(); + } + return guide.save(); + }).then((guideInstance: AnnotationGuide) => { + if (isMounted()) { setValue(guideInstance.markdown); setGuide(guideInstance); } @@ -48,7 +58,6 @@ function GuidePage(): JSX.Element { description: error.toString(), }); } - console.log(error.toString()); }).finally(() => { if (isMounted()) { setFetching(false); @@ -64,7 +73,7 @@ function GuidePage(): JSX.Element { const { textArea } = mdEditorRef.current.commandOrchestrator; const { selectionStart, selectionEnd } = textArea; for (const file of files) { - const { uuid } = await core.assets.create(file); + const { uuid } = await core.assets.create(file, guide.id); if (file.type.startsWith('image/')) { addedAssets.push(`![image](/api/assets/${uuid})`); } else { diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 87403920c35..6fea6625317 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -2303,8 +2303,10 @@ def get_serializer_class(self): def create(self, request, *args, **kwargs): file = request.data.get('file') + guide_id = request.data.get('guide_id') serializer = self.get_serializer(data={ 'filename': file.name, + 'guide_id': guide_id, }) serializer.is_valid(raise_exception=True) From 348f0cabda38a9a5009161c93d081fa98466db06 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 8 Jun 2023 14:02:23 +0300 Subject: [PATCH 22/62] Assets permissions --- cvat/apps/engine/views.py | 6 +++-- cvat/apps/iam/permissions.py | 44 +++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 6fea6625317..d5b93df0fc1 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -2289,12 +2289,14 @@ class AssetsViewset( viewsets.GenericViewSet, mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin ): - # todo: prefetch related for guide? - queryset = Asset.objects.select_related('owner').all() + queryset = Asset.objects.select_related('owner', 'guide').all() parser_classes=_UPLOAD_PARSER_CLASSES search_fields = () ordering = "uuid" + def check_object_permissions(self, request, obj): + super().check_object_permissions(request, obj.guide) + def get_serializer_class(self): if self.request.method in SAFE_METHODS: return AssetReadSerializer diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index 59f0bf0968b..2a847cd9925 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -18,7 +18,7 @@ from rest_framework.permissions import BasePermission from cvat.apps.organizations.models import Membership, Organization -from cvat.apps.engine.models import CloudStorage, Label, Project, Task, Job, Issue +from cvat.apps.engine.models import CloudStorage, Label, Project, Task, Job, Issue, AnnotationGuide from cvat.apps.webhooks.models import WebhookTypeChoice from cvat.utils.http import make_requests_session @@ -1657,6 +1657,48 @@ def get_resource(self): return data +class GuideAssetPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + VIEW = 'view' + DELETE = 'delete' + CREATE = 'create' + + @classmethod + def create(cls, request, view, obj): + Scopes = __class__.Scopes + permissions = [] + + if view.basename == 'asset': + for scope in cls.get_scopes(request, view, obj): + if scope == Scopes.VIEW and isinstance(obj, AnnotationGuide): + permissions.append(AnnotationGuidePermission.create_base_perm( + request, view, scope=AnnotationGuidePermission.Scopes.VIEW, obj=obj) + ) + if scope == Scopes.DELETE and isinstance(obj, AnnotationGuide): + permissions.append(AnnotationGuidePermission.create_base_perm( + request, view, scope=AnnotationGuidePermission.Scopes.UPDATE, obj=obj) + ) + if scope == Scopes.CREATE: + guide_id = request.data.get('guide_id') + try: + obj = AnnotationGuide.objects.get(id=guide_id) + permissions.append(AnnotationGuidePermission.create_base_perm( + request, view, scope=AnnotationGuidePermission.Scopes.UPDATE, obj=obj) + ) + except AnnotationGuide.DoesNotExist as ex: + raise ValidationError(str(ex)) + + return permissions + + @staticmethod + def get_scopes(request, view, obj): + Scopes = __class__.Scopes + return [{ + 'create': Scopes.CREATE, + 'destroy': Scopes.DELETE, + 'retrieve': Scopes.VIEW, + }.get(view.action, None)] + class PolicyEnforcer(BasePermission): # pylint: disable=no-self-use def check_permission(self, request, view, obj): From 5eb216620604c0c94435687df83b5e4fe5cea7f6 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 8 Jun 2023 14:19:45 +0300 Subject: [PATCH 23/62] Removed guide owner (useless field), added opa rules for organizations --- cvat-core/src/guide.ts | 6 ---- cvat/apps/engine/models.py | 1 - cvat/apps/engine/serializers.py | 5 ++- cvat/apps/iam/rules/annotationguides.rego | 37 +++++------------------ 4 files changed, 10 insertions(+), 39 deletions(-) diff --git a/cvat-core/src/guide.ts b/cvat-core/src/guide.ts index 81ac3f3084b..c5ed94bef5e 100644 --- a/cvat-core/src/guide.ts +++ b/cvat-core/src/guide.ts @@ -12,7 +12,6 @@ class AnnotationGuide { public readonly id?: number; public readonly taskId: number; public readonly projectId: number; - public readonly owner?: User; public readonly createdDate?: string; public readonly updatedDate?: string; public markdown: string; @@ -22,7 +21,6 @@ class AnnotationGuide { id: undefined, task_id: null, project_id: null, - owner: undefined, created_date: undefined, updated_date: undefined, markdown: '', @@ -33,7 +31,6 @@ class AnnotationGuide { data[property] = initialData[property]; } } - if (data.owner) data.owner = new User(data.owner); Object.defineProperties(this, Object.freeze({ id: { @@ -45,9 +42,6 @@ class AnnotationGuide { projectId: { get: () => data.project_id, }, - owner: { - get: () => data.owner, - }, createdDate: { get: () => data.created_date, }, diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index a5e6ade9545..00ef0ae3e2a 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -885,7 +885,6 @@ class Meta: class AnnotationGuide(models.Model): task = models.OneToOneField(Task, null=True, blank=True, on_delete=models.CASCADE, related_name="annotation_guide") project = models.OneToOneField(Project, null=True, blank=True, on_delete=models.CASCADE, related_name="annotation_guide") - owner = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL, related_name="annotation_guides") markdown = models.TextField(blank=True, default='') created_date = models.DateTimeField(auto_now_add=True) updated_date = models.DateTimeField(auto_now=True) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index dc45a086000..a4cc7eec832 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -1857,11 +1857,10 @@ class AnnotationGuideReadSerializer(WriteOnceMixin, serializers.ModelSerializer) class Meta: model = models.AnnotationGuide - fields = ('id', 'task_id', 'project_id', 'owner', 'created_date', 'updated_date', 'markdown', ) + fields = ('id', 'task_id', 'project_id', 'created_date', 'updated_date', 'markdown', ) read_only_fields = fields class AnnotationGuideWriteSerializer(WriteOnceMixin, serializers.ModelSerializer): - owner = BasicUserSerializer(required=False) project_id = serializers.IntegerField(required=False, allow_null=True) task_id = serializers.IntegerField(required=False, allow_null=True) @@ -1893,4 +1892,4 @@ def create(self, validated_data): class Meta: model = models.AnnotationGuide - fields = ('id', 'task_id', 'project_id', 'owner', 'markdown', ) + fields = ('id', 'task_id', 'project_id', 'markdown', ) diff --git a/cvat/apps/iam/rules/annotationguides.rego b/cvat/apps/iam/rules/annotationguides.rego index cceb24b8a6a..0b4e89e5e7d 100644 --- a/cvat/apps/iam/rules/annotationguides.rego +++ b/cvat/apps/iam/rules/annotationguides.rego @@ -82,43 +82,22 @@ allow { } allow { + { utils.CREATE, utils.DELETE, utils.UPDATE, utils.VIEW }[input.scope] input.scope == utils.CREATE utils.is_sandbox is_target_staff } allow { - input.scope == utils.DELETE - utils.is_sandbox - is_target_staff -} - -allow { - input.scope == utils.DELETE - utils.is_sandbox - utils.is_resource_owner -} - -allow { - input.scope == utils.UPDATE - utils.is_sandbox - is_target_staff -} - -allow { - input.scope == utils.UPDATE - utils.is_sandbox - utils.is_resource_owner + { utils.CREATE, utils.DELETE, utils.UPDATE, utils.VIEW }[input.scope] + input.auth.organization.id == input.resource.organization.id + utils.has_perm(utils.USER) + organizations.has_perm(organizations.MAINTAINER) } allow { - input.scope == utils.VIEW - utils.is_sandbox + { utils.CREATE, utils.DELETE, utils.UPDATE, utils.VIEW }[input.scope] + input.auth.organization.id == input.resource.organization.id + organizations.is_member is_target_staff } - -allow { - input.scope == utils.VIEW - utils.is_sandbox - utils.is_resource_owner -} From 5080b38e0cc30ca8a04777d4f86feb9d6cfa32ed Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 8 Jun 2023 15:33:26 +0300 Subject: [PATCH 24/62] Improved some code --- cvat/apps/engine/serializers.py | 2 - cvat/apps/engine/views.py | 70 +++++++++-------------- cvat/apps/iam/permissions.py | 1 - cvat/apps/iam/rules/annotationguides.rego | 2 - 4 files changed, 28 insertions(+), 47 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index a4cc7eec832..7e291491b25 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -1853,8 +1853,6 @@ class Meta: write_once_fields = ('uuid', 'filename', 'created_date', 'owner', 'guide_id', ) class AnnotationGuideReadSerializer(WriteOnceMixin, serializers.ModelSerializer): - owner = BasicUserSerializer(required=False) - class Meta: model = models.AnnotationGuide fields = ('id', 'task_id', 'project_id', 'created_date', 'updated_date', 'markdown', ) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index d5b93df0fc1..82b7812b803 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -5,6 +5,7 @@ import io import os +import re import os.path as osp import pytz import uuid @@ -2341,66 +2342,51 @@ class AnnotationGuidesViewset( viewsets.GenericViewSet, mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, PartialUpdateModelMixin ): - queryset = AnnotationGuide.objects.order_by('-id').select_related('owner', 'project', 'project__owner', 'project__organization', 'task', 'task__owner', 'task__organization').prefetch_related('assets').all() + queryset = AnnotationGuide.objects.order_by('-id').select_related( + 'project', 'project__owner', 'project__organization', 'task', 'task__owner', 'task__organization' + ).prefetch_related('assets').all() search_fields = () ordering = "-id" # NOTE: This filter works incorrectly for this view # it requires task__organization OR project__organization check. - # Thus, we rely on permission-based filtering + # Anyway we do not need any filtering in this class since it does not have 'list' method iam_organization_field = None + @staticmethod + def _update_assets(guide): + new_assets = [] + current_assets = list(guide.assets.all()) + markdown = guide.markdown + + # pylint: disable=anomalous-backslash-in-string + pattern = re.compile('\!\[image\]\(\/api\/assets\/([0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12})\)') + results = re.findall(pattern, markdown) + + for asset_id in results: + new_asset = models.Asset.objects.get(pk=asset_id) + if new_asset.guide_id != guide.id: + raise ValidationError('Asset is already related to another guide') + new_assets.append(new_asset) + + for current_asset in current_assets: + if current_asset not in new_assets: + current_asset.delete() + def check_object_permissions(self, request, obj): if self.action == 'retrieve': job_id = self.request.GET.get('job_id', None) if job_id is not None: # NOTE: This filter is too complex to be implemented by other means - # It requires the following filter query: - # ( - # project__task__segment__job__id = job_id - # OR - # task__segment__job__id = job_id - # ) + # Otherwise it would require difficult query project__task__segment__job__assignee = user + # OR + # task__segment__job__assignee = user db_job = Job.objects.select_related('segment', 'segment__task', 'segment__task__project').get(id=job_id) super().check_object_permissions(request, db_job) return super().check_object_permissions(request, obj) - @staticmethod - def _update_assets(guide): - UUID_LEN = 36 - TEMPLATE = '/api/assets/' - new_assets = [] - current_assets = list(guide.assets.all()) - markdown = guide.markdown - idx = markdown.find(TEMPLATE) - while idx != -1: - _from = idx + len(TEMPLATE) - _to = _from + UUID_LEN - try: - pk = uuid.UUID(markdown[_from : _to]) - try: - asset = models.Asset.objects.get(pk=pk) - if asset not in current_assets and asset.owner != guide.owner: - raise PermissionDenied('Asset owner and guide owner are different') - new_assets.append(asset) - except models.Asset.DoesNotExist: - pass - except ValueError: - continue - finally: - idx = markdown.find(TEMPLATE, _to) - - for asset in current_assets: - if asset not in new_assets: - asset.delete() - - for asset in new_assets: - # todo: handle case when asset is already assigned to another guide - asset.guide = guide - asset.save() - def get_serializer_class(self): if self.request.method in SAFE_METHODS: return AnnotationGuideReadSerializer diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index 2a847cd9925..b5e3da44fc7 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -1604,7 +1604,6 @@ def get_resource(self): db_organization = getattr(db_project, 'organization', None) or getattr(db_task, 'organization', None) or {} data.update({ 'id': self.obj.id, - 'owner': { 'id': getattr(self.obj.owner, 'id', None) }, 'project': { 'owner': { 'id': getattr(getattr(db_project, 'owner', {}), 'id', None) }, 'assignee': { 'id': getattr(getattr(db_project, 'assignee', {}), 'id', None) } diff --git a/cvat/apps/iam/rules/annotationguides.rego b/cvat/apps/iam/rules/annotationguides.rego index 0b4e89e5e7d..28a2b92c296 100644 --- a/cvat/apps/iam/rules/annotationguides.rego +++ b/cvat/apps/iam/rules/annotationguides.rego @@ -22,7 +22,6 @@ import data.organizations # }, # "resource": { # "id": , -# "owner": { "id": }, # "organization": { "id": or null }, # "task": { # "owner": { "id": or null }, @@ -83,7 +82,6 @@ allow { allow { { utils.CREATE, utils.DELETE, utils.UPDATE, utils.VIEW }[input.scope] - input.scope == utils.CREATE utils.is_sandbox is_target_staff } From 655ea29d65b3dda269cc8c1831a5b68690b0b077 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 8 Jun 2023 15:48:52 +0300 Subject: [PATCH 25/62] Some code refactored --- cvat/apps/engine/views.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 82b7812b803..e88ad49ea81 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -2374,17 +2374,15 @@ def _update_assets(guide): current_asset.delete() def check_object_permissions(self, request, obj): - if self.action == 'retrieve': - job_id = self.request.GET.get('job_id', None) - if job_id is not None: - # NOTE: This filter is too complex to be implemented by other means - # Otherwise it would require difficult query project__task__segment__job__assignee = user - # OR - # task__segment__job__assignee = user - db_job = Job.objects.select_related('segment', 'segment__task', 'segment__task__project').get(id=job_id) - super().check_object_permissions(request, db_job) - return - + job_id = self.request.GET.get('job_id', None) + if self.action == 'retrieve' and job_id is not None: + # NOTE: This filter is too complex to be implemented by other means + # Otherwise it would require difficult query project__task__segment__job__assignee = user + # OR + # task__segment__job__assignee = user + db_job = Job.objects.select_related('segment', 'segment__task', 'segment__task__project').get(id=job_id) + super().check_object_permissions(request, db_job) + return super().check_object_permissions(request, obj) def get_serializer_class(self): From 4fb885057c5aa683022d9a92c840431efbe478c9 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 8 Jun 2023 16:22:16 +0300 Subject: [PATCH 26/62] Org permissions --- cvat/apps/engine/views.py | 13 +++++++++++-- cvat/apps/iam/permissions.py | 16 +++++++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 35683cd7eb6..e7b3f2e47dc 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -2548,12 +2548,17 @@ class AssetsViewset( viewsets.GenericViewSet, mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin ): - queryset = Asset.objects.select_related('owner', 'guide').all() + queryset = Asset.objects.select_related( + 'owner', 'guide', 'guide__project', 'guide__task', 'guide__project__organization', 'guide__task__organization', + ).all() parser_classes=_UPLOAD_PARSER_CLASSES search_fields = () ordering = "uuid" def check_object_permissions(self, request, obj): + setattr(obj.guide, 'organization_id', getattr( + (obj.guide.project if obj.guide.project else obj.guide.task).organization + , 'id', None)) super().check_object_permissions(request, obj.guide) def get_serializer_class(self): @@ -2641,6 +2646,10 @@ def check_object_permissions(self, request, obj): db_job = Job.objects.select_related('segment', 'segment__task', 'segment__task__project').get(id=job_id) super().check_object_permissions(request, db_job) return + + setattr(obj, 'organization_id', getattr( + (obj.project if obj.project else obj.task).organization + , 'id', None)) super().check_object_permissions(request, obj) def get_serializer_class(self): @@ -2650,7 +2659,7 @@ def get_serializer_class(self): return AnnotationGuideWriteSerializer def perform_create(self, serializer): - serializer.save(owner=self.request.user) + serializer.save() AnnotationGuidesViewset._update_assets(serializer.instance) def perform_update(self, serializer): diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index 6a36e762df4..d6361c2e8fb 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -56,6 +56,8 @@ def get_organization(request, obj): # Try to get organization from an object otherwise, return the organization that is specified in query parameters if obj is not None and isinstance(obj, Organization): return obj + if obj is not None and isinstance(obj, AnnotationGuide): + return (obj.project if obj.project else obj.task).organization if obj: try: @@ -1604,7 +1606,7 @@ class Scopes(StrEnum): CREATE = 'create' @classmethod - def create(cls, request, view, obj): + def create(cls, request, view, obj, iam_context): Scopes = __class__.Scopes permissions = [] @@ -1616,10 +1618,10 @@ def create(cls, request, view, obj): for scope in cls.get_scopes(request, view, obj): if scope == Scopes.VIEW and isinstance(obj, Job): permissions.append(JobPermission.create_base_perm( - request, view, scope=JobPermission.Scopes.VIEW, obj=obj, + request, view, JobPermission.Scopes.VIEW, iam_context, obj=obj, )) else: - self = cls.create_base_perm(request, view, scope, obj, **params) + self = cls.create_base_perm(request, view, scope, iam_context, obj, **params) permissions.append(self) return permissions @@ -1705,7 +1707,7 @@ class Scopes(StrEnum): CREATE = 'create' @classmethod - def create(cls, request, view, obj): + def create(cls, request, view, obj, iam_context): Scopes = __class__.Scopes permissions = [] @@ -1713,18 +1715,18 @@ def create(cls, request, view, obj): for scope in cls.get_scopes(request, view, obj): if scope == Scopes.VIEW and isinstance(obj, AnnotationGuide): permissions.append(AnnotationGuidePermission.create_base_perm( - request, view, scope=AnnotationGuidePermission.Scopes.VIEW, obj=obj) + request, view, AnnotationGuidePermission.Scopes.VIEW, iam_context, obj=obj) ) if scope == Scopes.DELETE and isinstance(obj, AnnotationGuide): permissions.append(AnnotationGuidePermission.create_base_perm( - request, view, scope=AnnotationGuidePermission.Scopes.UPDATE, obj=obj) + request, view, AnnotationGuidePermission.Scopes.UPDATE, iam_context, obj=obj) ) if scope == Scopes.CREATE: guide_id = request.data.get('guide_id') try: obj = AnnotationGuide.objects.get(id=guide_id) permissions.append(AnnotationGuidePermission.create_base_perm( - request, view, scope=AnnotationGuidePermission.Scopes.UPDATE, obj=obj) + request, view, AnnotationGuidePermission.Scopes.UPDATE, iam_context, obj=obj) ) except AnnotationGuide.DoesNotExist as ex: raise ValidationError(str(ex)) From 05f249643003555301e87d7f69cd77633e4cc3ad Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 13 Jun 2023 17:15:12 +0300 Subject: [PATCH 27/62] Added limits, updated migrations --- .../src/components/md-guide/guide-page.tsx | 101 +++++++++++------- ...asset.py => 0071_annotationguide_asset.py} | 4 +- cvat/apps/engine/views.py | 15 ++- 3 files changed, 78 insertions(+), 42 deletions(-) rename cvat/apps/engine/migrations/{0069_annotationguide_asset.py => 0071_annotationguide_asset.py} (94%) diff --git a/cvat-ui/src/components/md-guide/guide-page.tsx b/cvat-ui/src/components/md-guide/guide-page.tsx index eade3c07770..9b77ec7a03c 100644 --- a/cvat-ui/src/components/md-guide/guide-page.tsx +++ b/cvat-ui/src/components/md-guide/guide-page.tsx @@ -4,7 +4,7 @@ import './styles.scss'; -import React, { useState, useEffect, useRef } from 'react'; +import React, { useState, useEffect, useRef, useCallback } from 'react'; import { useLocation, useParams } from 'react-router'; import { Row, Col } from 'antd/lib/grid'; import notification from 'antd/lib/notification'; @@ -65,23 +65,75 @@ function GuidePage(): JSX.Element { }); }, []); + const submit = useCallback((currentValue: string) => { + guide.markdown = currentValue; + setFetching(true); + guide.save().then((result: AnnotationGuide) => { + if (isMounted()) { + setValue(result.markdown); + setGuide(result); + } + }).catch((error: any) => { + if (isMounted()) { + notification.error({ + message: 'Could not save guide on the server', + description: error.toString(), + }); + } + }).finally(() => { + if (isMounted()) { + setFetching(false); + } + }); + }, [guide, fetching]); + const handleInsert = async (event: React.ClipboardEvent | React.DragEvent, files: FileList): Promise => { if (files.length) { event.preventDefault(); - const addedAssets = []; + const assetsToAdd = Array.from(files); + const addedAssets: [File, string][] = []; + if (mdEditorRef.current) { const { textArea } = mdEditorRef.current.commandOrchestrator; const { selectionStart, selectionEnd } = textArea; - for (const file of files) { - const { uuid } = await core.assets.create(file, guide.id); - if (file.type.startsWith('image/')) { - addedAssets.push(`![image](/api/assets/${uuid})`); - } else { - addedAssets.push(`[${file.name}](/api/assets/${uuid})`); + const computeNewValue = (): string => { + const addedStrings = addedAssets.map(([file, uuid]) => { + if (file.type.startsWith('image/')) { + return (`![image](/api/assets/${uuid})`); + } + return (`[${file.name}](/api/assets/${uuid})`); + }); + + const stringsToAdd = assetsToAdd.map((file: File) => { + if (file.type.startsWith('image/')) { + return '![image](Loading...)'; + } + return `![${file.name}](Loading...)`; + }); + + return `${value.slice(0, selectionStart)}\n${addedStrings.concat(stringsToAdd).join('\n')}\n${value.slice(selectionEnd)}`; + }; + + setValue(computeNewValue()); + let file = assetsToAdd.shift(); + while (file) { + try { + const { uuid } = await core.assets.create(file, guide.id); + addedAssets.push([file, uuid]); + setValue(computeNewValue()); + } catch (error: any) { + notification.error({ + message: 'Could not create a server asset', + description: error.toString(), + }); + } finally { + file = assetsToAdd.shift(); } } - setValue(`${value.slice(0, selectionStart)}\n${addedAssets.join('\n')}\n${value.slice(selectionEnd)}`); + const finalValue = computeNewValue(); + setValue(finalValue); + submit(finalValue); } } }; @@ -124,36 +176,7 @@ function GuidePage(): JSX.Element { diff --git a/cvat/apps/engine/migrations/0069_annotationguide_asset.py b/cvat/apps/engine/migrations/0071_annotationguide_asset.py similarity index 94% rename from cvat/apps/engine/migrations/0069_annotationguide_asset.py rename to cvat/apps/engine/migrations/0071_annotationguide_asset.py index 2be067ba43c..1060c4576ab 100644 --- a/cvat/apps/engine/migrations/0069_annotationguide_asset.py +++ b/cvat/apps/engine/migrations/0071_annotationguide_asset.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.18 on 2023-06-08 12:52 +# Generated by Django 3.2.18 on 2023-06-13 13:14 from django.conf import settings from django.db import migrations, models @@ -10,7 +10,7 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('engine', '0068_auto_20230418_0901'), + ('engine', '0070_add_job_type_created_date'), ] operations = [ diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 45dc838395b..3b1e105181a 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -2688,8 +2688,21 @@ def get_serializer_class(self): return AssetWriteSerializer def create(self, request, *args, **kwargs): - file = request.data.get('file') + file = request.data.get('file', None) + if not file: + raise ValidationError('Asset file was not provided') + + if file.size / (1024 * 1024) > settings.ASSET_MAX_SIZE_MB: + raise ValidationError(f'Maximum size of asset is {settings.ASSET_MAX_SIZE_MB} MB') + + if file.content_type not in settings.ASSET_SUPPORTED_TYPES: + raise ValidationError(f'File is not supported as an asset. Supported are {settings.ASSET_SUPPORTED_TYPES}') + guide_id = request.data.get('guide_id') + db_guide = AnnotationGuide.objects.get(pk=guide_id) + if db_guide.assets.count() > settings.ASSET_MAX_COUNT_PER_GUIDE: + raise ValidationError(f'Maximum number of assets per guide reached') + serializer = self.get_serializer(data={ 'filename': file.name, 'guide_id': guide_id, From 802cc437f08cc8dfdcd5f7d643bbf67c268ff128 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 13 Jun 2023 23:15:36 +0300 Subject: [PATCH 28/62] Implemented plugin for tags --- .../components/project-page/project-page.tsx | 4 +- .../components/projects-page/project-item.tsx | 92 ++++++++------- .../src/components/projects-page/styles.scss | 106 ++++++++++-------- cvat-ui/src/components/task-page/details.tsx | 8 +- cvat-ui/src/components/tasks-page/styles.scss | 9 ++ .../src/components/tasks-page/task-item.tsx | 38 +++++-- .../src/containers/tasks-page/task-item.tsx | 4 +- cvat-ui/src/reducers/index.ts | 5 +- cvat-ui/src/reducers/plugins-reducer.ts | 5 +- 9 files changed, 162 insertions(+), 109 deletions(-) diff --git a/cvat-ui/src/components/project-page/project-page.tsx b/cvat-ui/src/components/project-page/project-page.tsx index 5cf8aa92df9..c0764c2af39 100644 --- a/cvat-ui/src/components/project-page/project-page.tsx +++ b/cvat-ui/src/components/project-page/project-page.tsx @@ -60,7 +60,7 @@ export default function ProjectPageComponent(): JSX.Element { const [updatingProject, setUpdatingProject] = useState(false); const mounted = useRef(false); - const taskNamePlugins = useSelector((state: CombinedState) => state.plugins.components.taskItem.name); + const ribbonPlugins = useSelector((state: CombinedState) => state.plugins.components.taskItem.ribbon); const deletes = useSelector((state: CombinedState) => state.projects.activities.deletes); const taskDeletes = useSelector((state: CombinedState) => state.tasks.activities.deletes); const tasksActiveInferences = useSelector((state: CombinedState) => state.models.inferences); @@ -153,10 +153,10 @@ export default function ProjectPageComponent(): JSX.Element { .map((task: Task) => ( diff --git a/cvat-ui/src/components/md-guide/styles.scss b/cvat-ui/src/components/md-guide/styles.scss index e8516b124d4..ee6a8ce3929 100644 --- a/cvat-ui/src/components/md-guide/styles.scss +++ b/cvat-ui/src/components/md-guide/styles.scss @@ -5,8 +5,10 @@ @import '../../base.scss'; .cvat-md-guide-control-wrapper { + margin-top: $grid-unit-size; + button { - margin-left: $grid-unit-size; + margin-top: $grid-unit-size; } } diff --git a/cvat-ui/src/components/project-page/styles.scss b/cvat-ui/src/components/project-page/styles.scss index f11c2ec9569..edbed3b04f5 100644 --- a/cvat-ui/src/components/project-page/styles.scss +++ b/cvat-ui/src/components/project-page/styles.scss @@ -61,11 +61,6 @@ align-items: center; justify-content: flex-end; } - - .cvat-issue-tracker { - margin-top: $grid-unit-size * 2; - margin-bottom: $grid-unit-size * 2; - } } .ant-menu.cvat-project-actions-menu { diff --git a/cvat-ui/src/components/task-page/bug-tracker-editor.tsx b/cvat-ui/src/components/task-page/bug-tracker-editor.tsx index 4534d0e7c1b..24f9b9ebd9b 100644 --- a/cvat-ui/src/components/task-page/bug-tracker-editor.tsx +++ b/cvat-ui/src/components/task-page/bug-tracker-editor.tsx @@ -52,13 +52,12 @@ export default function BugTrackerEditorComponent(props: Props): JSX.Element { Issue Tracker -
{bugTracker} +