From 7a2bde2ae67ad1854dc5ca8b5eb1a05db016e142 Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Wed, 6 Sep 2023 14:44:47 +0300 Subject: [PATCH] Add gamma correction filter (#6771) ### Motivation and context Adds gamma correction filter ![debounced](https://github.com/opencv/cvat/assets/50956430/c1748100-355c-4dd7-a0fc-b994dc42e8de) ### How has this been tested? ### Checklist - [x] I submit my changes into the `develop` branch - [x] I have added a description of my changes into the [CHANGELOG](https://github.com/opencv/cvat/blob/develop/CHANGELOG.md) file - ~~[ ] I have updated the documentation accordingly~~ - ~~[ ] I have added tests to cover my changes~~ - [x] I have linked related issues (see [GitHub docs]( https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)) - [x] I have increased versions of npm packages if it is necessary ([cvat-canvas](https://github.com/opencv/cvat/tree/develop/cvat-canvas#versioning), [cvat-core](https://github.com/opencv/cvat/tree/develop/cvat-core#versioning), [cvat-data](https://github.com/opencv/cvat/tree/develop/cvat-data#versioning) and [cvat-ui](https://github.com/opencv/cvat/tree/develop/cvat-ui#versioning)) ### License - [x] I submit _my code changes_ under the same [MIT License]( https://github.com/opencv/cvat/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. --------- Co-authored-by: Boris Sekachev --- CHANGELOG.md | 1 + cvat-canvas/.eslintrc.js | 11 ++ cvat-canvas/package.json | 2 - cvat-ui/.eslintrc.js | 8 ++ cvat-ui/package.json | 2 +- cvat-ui/src/actions/settings-actions.ts | 31 ++++ .../canvas/views/canvas2d/canvas-wrapper.tsx | 69 ++++++++- .../canvas/views/canvas2d/gamma-filter.tsx | 74 ++++++++++ .../views/canvas2d/image-setups-content.tsx | 35 +++-- .../canvas/views/canvas2d/image-setups.scss | 9 ++ .../controls-side-bar/opencv-control.tsx | 122 +++------------- cvat-ui/src/reducers/index.ts | 2 + cvat-ui/src/reducers/settings-reducer.ts | 45 ++++++ .../utils/fabric-wrapper/fabric-wrapper.ts | 26 ++++ .../utils/fabric-wrapper/gamma-correciton.ts | 26 ++++ cvat-ui/src/utils/image-processing.tsx | 44 ++++++ .../opencv-wrapper/histogram-equalization.ts | 133 ++++++------------ .../utils/opencv-wrapper/opencv-interfaces.ts | 6 +- package.json | 5 +- .../case_101_opencv_basic_actions.js | 2 +- 20 files changed, 433 insertions(+), 220 deletions(-) create mode 100644 cvat-ui/src/components/annotation-page/canvas/views/canvas2d/gamma-filter.tsx create mode 100644 cvat-ui/src/components/annotation-page/canvas/views/canvas2d/image-setups.scss create mode 100644 cvat-ui/src/utils/fabric-wrapper/fabric-wrapper.ts create mode 100644 cvat-ui/src/utils/fabric-wrapper/gamma-correciton.ts create mode 100644 cvat-ui/src/utils/image-processing.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index acd696255ebb..e26042a7f5d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## \[Unreleased\] ### Added - Ability to hide/show an object in review mode () +- Gamma correcton filter () ### Changed - \[Helm\] Database migrations now run in a separate job instead of the server pod, diff --git a/cvat-canvas/.eslintrc.js b/cvat-canvas/.eslintrc.js index ab68a0338f42..23b211a9a3fd 100644 --- a/cvat-canvas/.eslintrc.js +++ b/cvat-canvas/.eslintrc.js @@ -1,7 +1,10 @@ // Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT +const { join } = require('path'); + module.exports = { ignorePatterns: [ '.eslintrc.js', @@ -13,4 +16,12 @@ module.exports = { project: './tsconfig.json', tsconfigRootDir: __dirname, }, + rules: { + 'import/no-extraneous-dependencies': [ + 'error', + { + packageDir: [__dirname, join(__dirname, '../')] + }, + ], + } }; diff --git a/cvat-canvas/package.json b/cvat-canvas/package.json index 10d3fc875304..19404cdedda0 100644 --- a/cvat-canvas/package.json +++ b/cvat-canvas/package.json @@ -16,8 +16,6 @@ ], "dependencies": { "@types/polylabel": "^1.0.5", - "@types/fabric": "^4.5.7", - "fabric": "^5.2.1", "polylabel": "^1.1.0", "svg.draggable.js": "2.2.2", "svg.draw.js": "^2.0.4", diff --git a/cvat-ui/.eslintrc.js b/cvat-ui/.eslintrc.js index 22c20d3c9c43..0f89242e3d23 100644 --- a/cvat-ui/.eslintrc.js +++ b/cvat-ui/.eslintrc.js @@ -1,8 +1,10 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT const globalConfig = require('../.eslintrc.js'); +const { join } = require('path'); module.exports = { parserOptions: { @@ -29,5 +31,11 @@ module.exports = { 'react/jsx-indent-props': ['warn', 4], 'react/jsx-props-no-spreading': 0, 'jsx-quotes': ['error', 'prefer-single'], + 'import/no-extraneous-dependencies': [ + 'error', + { + packageDir: [__dirname, join(__dirname, '../')] + }, + ], }, }; diff --git a/cvat-ui/package.json b/cvat-ui/package.json index cba98fd3168c..b1f86c253d1a 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.55.8", + "version": "1.56.0", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/actions/settings-actions.ts b/cvat-ui/src/actions/settings-actions.ts index 0e37fe7d3da5..5cbf1f187947 100644 --- a/cvat-ui/src/actions/settings-actions.ts +++ b/cvat-ui/src/actions/settings-actions.ts @@ -1,4 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -6,6 +7,7 @@ import { AnyAction } from 'redux'; import { GridColor, ColorBy, SettingsState, ToolsBlockerState, } from 'reducers'; +import { ImageFilter, ImageFilterAlias } from 'utils/image-processing'; export enum SettingsActionTypes { SWITCH_ROTATE_ALL = 'SWITCH_ROTATE_ALL', @@ -45,6 +47,9 @@ export enum SettingsActionTypes { SWITCH_TOOLS_BLOCKER_STATE = 'SWITCH_TOOLS_BLOCKER_STATE', SWITCH_SHOWING_DELETED_FRAMES = 'SWITCH_SHOWING_DELETED_FRAMES', SWITCH_SHOWING_TAGS_ON_FRAME = 'SWITCH_SHOWING_TAGS_ON_FRAME', + ENABLE_IMAGE_FILTER = 'ENABLE_IMAGE_FILTER', + DISABLE_IMAGE_FILTER = 'DISABLE_IMAGE_FILTER', + RESET_IMAGE_FILTERS = 'RESET_IMAGE_FILTERS', } export function changeShapesOpacity(opacity: number): AnyAction { @@ -380,3 +385,29 @@ export function switchShowingTagsOnFrame(showTagsOnFrame: boolean): AnyAction { }, }; } + +export function enableImageFilter(filter: ImageFilter, options: object | null = null): AnyAction { + return { + type: SettingsActionTypes.ENABLE_IMAGE_FILTER, + payload: { + filter, + options, + }, + }; +} + +export function disableImageFilter(filterAlias: ImageFilterAlias): AnyAction { + return { + type: SettingsActionTypes.DISABLE_IMAGE_FILTER, + payload: { + filterAlias, + }, + }; +} + +export function resetImageFilters(): AnyAction { + return { + type: SettingsActionTypes.RESET_IMAGE_FILTERS, + payload: {}, + }; +} diff --git a/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx index df2d3d63381d..9ecd1efd0a68 100644 --- a/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx +++ b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx @@ -9,6 +9,8 @@ import Slider from 'antd/lib/slider'; import Spin from 'antd/lib/spin'; import Dropdown from 'antd/lib/dropdown'; import { PlusCircleOutlined, UpOutlined } from '@ant-design/icons'; +import notification from 'antd/lib/notification'; +import debounce from 'lodash/debounce'; import GlobalHotKeys, { KeyMap } from 'utils/mousetrap-react'; import { @@ -57,6 +59,7 @@ import { import { reviewActions } from 'actions/review-actions'; import { filterAnnotations } from 'utils/filter-annotations'; +import { ImageFilter } from 'utils/image-processing'; import ImageSetupsContent from './image-setups-content'; import BrushTools from './brush-tools'; @@ -113,6 +116,7 @@ interface StateToProps { showGroundTruth: boolean; highlightedConflict: QualityConflict | null; groundTruthJobFramesMeta: FramesMetaData | null; + imageFilters: ImageFilter[]; } interface DispatchToProps { @@ -194,6 +198,7 @@ function mapStateToProps(state: CombinedState): StateToProps { shapes: { opacity, colorBy, selectedOpacity, outlined, outlineColor, showBitmap, showProjections, showGroundTruth, }, + imageFilters, }, shortcuts: { keyMap }, review: { conflicts }, @@ -253,6 +258,7 @@ function mapStateToProps(state: CombinedState): StateToProps { showGroundTruth, highlightedConflict, groundTruthJobFramesMeta, + imageFilters, }; } @@ -358,6 +364,8 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { type Props = StateToProps & DispatchToProps; class CanvasWrapperComponent extends React.PureComponent { + private debouncedUpdate = debounce(this.updateCanvas.bind(this), 250, { leading: true }); + public componentDidMount(): void { const { automaticBordering, @@ -445,6 +453,7 @@ class CanvasWrapperComponent extends React.PureComponent { statesSources, showGroundTruth, highlightedConflict, + imageFilters, } = this.props; const { canvasInstance } = this.props as { canvasInstance: Canvas }; @@ -530,6 +539,10 @@ class CanvasWrapperComponent extends React.PureComponent { }); } + if (prevProps.imageFilters !== imageFilters) { + canvasInstance.configure({ forceFrameUpdate: true }); + } + if ( prevProps.annotations !== annotations || prevProps.statesSources !== statesSources || @@ -537,6 +550,10 @@ class CanvasWrapperComponent extends React.PureComponent { prevProps.curZLayer !== curZLayer ) { this.updateCanvas(); + } else if (prevProps.imageFilters !== imageFilters) { + // In case of frequent image filters changes, we apply debounced canvas update + // that makes UI smoother + this.debouncedUpdate(); } if (prevProps.showBitmap !== showBitmap) { @@ -897,9 +914,11 @@ class CanvasWrapperComponent extends React.PureComponent { private updateCanvas(): void { const { - curZLayer, annotations, frameData, canvasInstance, statesSources, - workspace, groundTruthJobFramesMeta, frame, + curZLayer, annotations, frameData, statesSources, + workspace, groundTruthJobFramesMeta, frame, imageFilters, } = this.props; + const { canvasInstance } = this.props as { canvasInstance: Canvas }; + if (frameData !== null && canvasInstance) { const filteredAnnotations = filterAnnotations(annotations, { statesSources, @@ -908,12 +927,54 @@ class CanvasWrapperComponent extends React.PureComponent { workspace, exclude: [ObjectType.TAG], }); - + const proxy = new Proxy(frameData, { + get: (_frameData, prop, receiver) => { + if (prop === 'data') { + return async () => { + const originalImage = await _frameData.data(); + const imageIsNotProcessed = imageFilters.some((imageFilter: ImageFilter) => ( + imageFilter.modifier.currentProcessedImage !== frame + )); + + if (imageIsNotProcessed) { + try { + const { renderWidth, renderHeight, imageData: imageBitmap } = originalImage; + + const offscreen = new OffscreenCanvas(renderWidth, renderHeight); + const ctx = offscreen.getContext('2d') as OffscreenCanvasRenderingContext2D; + ctx.drawImage(imageBitmap, 0, 0); + const imageData = ctx.getImageData(0, 0, renderWidth, renderHeight); + + const newImageData = imageFilters + .reduce((oldImageData, activeImageModifier) => activeImageModifier + .modifier.processImage(oldImageData, frame), imageData); + const newImageBitmap = await createImageBitmap(newImageData); + return { + renderWidth, + renderHeight, + imageData: newImageBitmap, + }; + } catch (error: any) { + notification.error({ + description: error.toString(), + message: 'Image processing error occurred', + className: 'cvat-notification-notice-image-processing-error', + }); + } + } + + return originalImage; + }; + } + return Reflect.get(_frameData, prop, receiver); + }, + }); canvasInstance.setup( - frameData, + proxy, frameData.deleted ? [] : filteredAnnotations, curZLayer, ); + canvasInstance.configure({ forceFrameUpdate: false }); } } diff --git a/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/gamma-filter.tsx b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/gamma-filter.tsx new file mode 100644 index 000000000000..cdd2d887f2ac --- /dev/null +++ b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/gamma-filter.tsx @@ -0,0 +1,74 @@ +// Copyright (C) 2023 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import React, { useEffect, useState, useCallback } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { Row, Col } from 'antd/lib/grid'; +import { CombinedState } from 'reducers'; +import Text from 'antd/lib/typography/Text'; +import Slider from 'antd/lib/slider'; + +import { + enableImageFilter, + disableImageFilter, +} from 'actions/settings-actions'; +import GammaCorrection from 'utils/fabric-wrapper/gamma-correciton'; +import { ImageFilterAlias, hasFilter } from 'utils/image-processing'; + +import './image-setups.scss'; + +export default function GammaFilter(): JSX.Element { + const dispatch = useDispatch(); + const [gamma, setGamma] = useState(1); + const filters = useSelector((state: CombinedState) => state.settings.imageFilters); + const gammaFilter = hasFilter(filters, ImageFilterAlias.GAMMA_CORRECTION); + + const onChangeGamma = useCallback((newGamma: number): void => { + setGamma(newGamma); + if (newGamma === 1) { + if (gammaFilter) { + dispatch(disableImageFilter(ImageFilterAlias.GAMMA_CORRECTION)); + } + } else { + const convertedGamma = [newGamma, newGamma, newGamma]; + if (gammaFilter) { + dispatch(enableImageFilter(gammaFilter, { gamma: convertedGamma })); + } else { + dispatch(enableImageFilter({ + modifier: new GammaCorrection({ gamma: convertedGamma }), + alias: ImageFilterAlias.GAMMA_CORRECTION, + })); + } + } + }, [gammaFilter]); + + useEffect(() => { + if (filters.length === 0) { + setGamma(1); + } + }, [filters]); + + return ( +
+ + + + + Gamma + + + + + + + +
+ ); +} diff --git a/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/image-setups-content.tsx b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/image-setups-content.tsx index 07884b1262e0..5fefabfe8deb 100644 --- a/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/image-setups-content.tsx +++ b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/image-setups-content.tsx @@ -1,4 +1,5 @@ // Copyright (C) 2021-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -20,9 +21,11 @@ import { changeContrastLevel, changeSaturationLevel, changeGridSize, + resetImageFilters, } from 'actions/settings-actions'; import { clamp } from 'utils/math'; import { GridColor, CombinedState, PlayerSettingsState } from 'reducers'; +import GammaFilter from './gamma-filter'; const minGridSize = 5; const maxGridSize = 1000; @@ -168,21 +171,23 @@ export default function ImageSetupsContent(): JSX.Element { /> - - - - - + + + + + + diff --git a/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/image-setups.scss b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/image-setups.scss new file mode 100644 index 000000000000..c6229a4f072f --- /dev/null +++ b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/image-setups.scss @@ -0,0 +1,9 @@ +// Copyright (C) 2023 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +@import 'base'; + +.cvat-image-setups-filters { + margin-bottom: $grid-unit-size * 3; +} diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx index ef1ce7e6a4fd..7f9acd45d187 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx @@ -1,4 +1,5 @@ // Copyright (C) 2021-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -36,8 +37,9 @@ import CVATTooltip from 'components/common/cvat-tooltip'; import ApproximationAccuracy, { thresholdFromAccuracy, } from 'components/annotation-page/standard-workspace/controls-side-bar/approximation-accuracy'; -import { ImageProcessing, OpenCVTracker, TrackerModel } from 'utils/opencv-wrapper/opencv-interfaces'; -import { switchToolsBlockerState } from 'actions/settings-actions'; +import { OpenCVTracker, TrackerModel } from 'utils/opencv-wrapper/opencv-interfaces'; +import { enableImageFilter as enableImageFilterAction, disableImageFilter as disableImageFilterAction, switchToolsBlockerState } from 'actions/settings-actions'; +import { ImageFilter, ImageFilterAlias, hasFilter } from 'utils/image-processing'; import withVisibilityHandling from './handle-popover-visibility'; interface Props { @@ -52,6 +54,7 @@ interface Props { frameData: any; toolsBlockerState: ToolsBlockerState; activeControl: ActiveControl; + filters: ImageFilter[]; } interface DispatchToProps { @@ -62,6 +65,8 @@ interface DispatchToProps { changeFrame(toFrame: number, fillBuffer?: boolean, frameStep?: number, forceUpdate?: boolean):void; onSwitchToolsBlockerState(toolsBlockerState: ToolsBlockerState):void; switchNavigationBlocked(navigationBlocked: boolean): void; + enableImageFilter(filter: ImageFilter): void; + disableImageFilter(filterAlias: string): void; } interface TrackedShape { @@ -76,16 +81,10 @@ interface State { initializationProgress: number; activeLabelID: number; approxPolyAccuracy: number; - activeImageModifiers: ImageModifier[]; mode: 'interaction' | 'tracking'; trackedShapes: TrackedShape[]; activeTracker: OpenCVTracker | null; - trackers: OpenCVTracker[] -} - -interface ImageModifier { - modifier: ImageProcessing, - alias: string + trackers: OpenCVTracker[]; } const core = getCore(); @@ -106,6 +105,7 @@ function mapStateToProps(state: CombinedState): Props { }, settings: { workspace: { defaultApproxPolyAccuracy, toolsBlockerState }, + imageFilters: filters, }, } = state; @@ -121,6 +121,7 @@ function mapStateToProps(state: CombinedState): Props { frame, frameData, toolsBlockerState, + filters, }; } @@ -132,19 +133,19 @@ const mapDispatchToProps = { changeFrame: changeFrameAsync, onSwitchToolsBlockerState: switchToolsBlockerState, switchNavigationBlocked: switchNavigationBlockedAction, + enableImageFilter: enableImageFilterAction, + disableImageFilter: disableImageFilterAction, }; class OpenCVControlComponent extends React.PureComponent { private activeTool: IntelligentScissors | null; private latestPoints: number[]; - private canvasForceUpdateWasEnabled: boolean; public constructor(props: Props & DispatchToProps) { super(props); const { labels, defaultApproxPolyAccuracy } = props; this.activeTool = null; this.latestPoints = []; - this.canvasForceUpdateWasEnabled = false; this.state = { libraryInitialized: openCVWrapper.isInitialized, @@ -152,7 +153,6 @@ class OpenCVControlComponent extends React.PureComponent => { - const { activeImageModifiers } = this.state; - const { - frameData, states, curZOrder, canvasInstance, frame, - } = this.props; - - try { - if (activeImageModifiers.length !== 0 && activeImageModifiers[0].modifier.currentProcessedImage !== frame) { - this.enableCanvasForceUpdate(); - const imageData = this.getCanvasImageData(); - const newImageData = activeImageModifiers - .reduce((oldImageData, activeImageModifier) => activeImageModifier - .modifier.processImage(oldImageData, frame), imageData); - const imageBitmap = await createImageBitmap(newImageData); - const proxy = new Proxy(frameData, { - get: (_frameData, prop, receiver) => { - if (prop === 'data') { - return async () => ({ - renderWidth: imageData.width, - renderHeight: imageData.height, - imageData: imageBitmap, - }); - } - - return Reflect.get(_frameData, prop, receiver); - }, - }); - canvasInstance.setup(proxy, states, curZOrder); - } - } catch (error: any) { - notification.error({ - description: error.toString(), - message: 'OpenCV.js processing error occurred', - className: 'cvat-notification-notice-opencv-processing-error', - }); - } finally { - this.disableCanvasForceUpdate(); - } - }; - private applyTracking = (imageData: ImageData, shape: TrackedShape, objectState: any): Promise => new Promise((resolve, reject) => { setTimeout(() => { @@ -566,45 +524,6 @@ class OpenCVControlComponent extends React.PureComponent imageModifier.alias === alias)?.modifier || null; - } - - private disableImageModifier(alias: string):void { - const { activeImageModifiers } = this.state; - const index = activeImageModifiers.findIndex((imageModifier) => imageModifier.alias === alias); - if (index !== -1) { - activeImageModifiers.splice(index, 1); - this.setState({ - activeImageModifiers: [...activeImageModifiers], - }); - } - } - - private enableImageModifier(modifier: ImageProcessing, alias: string): void { - this.setState((prev: State) => ({ - ...prev, - activeImageModifiers: [...prev.activeImageModifiers, { modifier, alias }], - }), () => { - this.runImageModifier(); - }); - } - - private enableCanvasForceUpdate():void { - const { canvasInstance } = this.props; - canvasInstance.configure({ forceFrameUpdate: true }); - this.canvasForceUpdateWasEnabled = true; - } - - private disableCanvasForceUpdate():void { - if (this.canvasForceUpdateWasEnabled) { - const { canvasInstance } = this.props; - canvasInstance.configure({ forceFrameUpdate: false }); - this.canvasForceUpdateWasEnabled = false; - } - } - private async initializeOpenCV():Promise { try { this.setState({ @@ -675,27 +594,26 @@ class OpenCVControlComponent extends React.PureComponent