Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Export only annotated images option. #8340

Closed
wants to merge 54 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
0097fd1
added an option to export only annotated images when saving images du…
Aug 22, 2024
186c928
removed git ignore additions
Aug 22, 2024
721f887
moved nested conditional expression into separate statements
Aug 22, 2024
f0728d7
missed adding all_images parameter to one of the build_annotations_fi…
Aug 22, 2024
691af9d
changed export download filename to match previous behavior
Aug 27, 2024
8050b98
sync with upstream
Aug 27, 2024
e51a1ca
added test for downloading only annotated images
Sep 11, 2024
144c28b
Merge branch 'cvat-ai:develop' into develop
bschultz96 Sep 12, 2024
f3c6944
Job validations (public part) (#8321)
zhiltsov-max Aug 23, 2024
4fdc2df
YOLOv8 support (#8240)
Eldies Aug 23, 2024
f69b197
Prepare auto-annotation RQ jobs for migration to the general request …
SpecLad Aug 23, 2024
b0eaaa4
Propagate filtered shapes Documentation (#8181)
mdacoca Aug 27, 2024
74b9710
Upgrade Silk (#8357)
SpecLad Aug 27, 2024
11a8c27
Prepare release v2.17.0
cvat-bot[bot] Aug 27, 2024
9e141a0
Update develop after v2.17.0
cvat-bot[bot] Aug 27, 2024
8cce693
[GSoC'24] Dependent on #8007, UI, Working Conflict Detector and Short…
tahamukhtar20 Aug 29, 2024
a11e6b5
Support `Target metric` in quality UI (#8347)
klakhov Aug 29, 2024
c6a14cb
updating datumaro (#8370)
Eldies Aug 29, 2024
d6ebad6
Bump webpack from 5.91.0 to 5.94.0 (#8374)
dependabot[bot] Aug 29, 2024
6c1e6cb
Move `--no-binary=lxml,xmlsec` into the Dockerfile (#8361)
SpecLad Aug 29, 2024
eb11d75
Bump micromatch from 4.0.5 to 4.0.8 in /tests (#8359)
dependabot[bot] Aug 29, 2024
acc7a8c
Don't check permissions for the target resource when canceling reques…
SpecLad Aug 30, 2024
afa769c
Update high-level SDK used for exporting `project`|`task`|`job` `data…
Marishka17 Aug 30, 2024
9db8817
update docs for yolov8 (#8379)
Eldies Aug 30, 2024
16c8414
[GSoC'24] Keyboard Shortcut Customization: Merge and Group handler fi…
tahamukhtar20 Aug 31, 2024
906d2b8
[GSoC'24] Keyboard Shortcuts Customization, Documentation (#8345)
tahamukhtar20 Aug 31, 2024
712d284
[GSoC'24] Keyboard Shortcuts Customization, Cypress Tests (#8343)
tahamukhtar20 Aug 31, 2024
7a90095
Switch to v4 of artifact actions (#8342)
SpecLad Sep 2, 2024
d1e94b8
Remove the cleanup workflow (#8390)
SpecLad Sep 3, 2024
bcdaf75
[Datumaro] Import Memory optimisation (#8382)
Bobronium Sep 4, 2024
c000a1d
Log events for function calls (#8395)
SpecLad Sep 4, 2024
7c06022
fixing problems with skeleton tracks with missing points (#8377)
Eldies Sep 5, 2024
5a5b8f4
lambda_manager: revamp function parsing error handling (#8406)
SpecLad Sep 6, 2024
6fcc060
Fixed undefined shortcut in single shape mode (#8414)
bsekachev Sep 8, 2024
49d2e33
Display shortcuts correctly in attribute annotation mode (#8415)
bsekachev Sep 8, 2024
1597f12
Display the ID for non-system models (#8412)
SpecLad Sep 9, 2024
1d32ce3
Removed legacy component to setup shortcuts (#8416)
bsekachev Sep 9, 2024
072f3fb
Rename and move `LambdaType` (#8418)
SpecLad Sep 9, 2024
3f5e4e1
Merge commit from fork
SpecLad Sep 10, 2024
4a12beb
Prepare release v2.18.0
cvat-bot[bot] Sep 10, 2024
69437ac
Update develop after v2.18.0
cvat-bot[bot] Sep 10, 2024
f92bde6
Added trigger dependent repo update workflow (#8389)
azhavoro Sep 10, 2024
b749b68
Update backup guide (#8405)
azhavoro Sep 10, 2024
395bfeb
`*Permission.get_scopes`: don't tolerate unknown actions (#8426)
SpecLad Sep 10, 2024
b40b0d2
Upgrade GitHub Actions to remove dependencies on Node 12/16 (#7459)
SpecLad Sep 10, 2024
53a129b
Quality control page (#8329)
klakhov Sep 11, 2024
0242a3f
Bump axios from 1.6.8 to 1.7.4 (#8311)
dependabot[bot] Sep 11, 2024
c10efd3
Fixing skeleton track project export (#8423)
Eldies Sep 12, 2024
38e07f4
merged main in
Sep 12, 2024
f28c3c5
changed all_images to only_annotated, added new tests for exporting o…
Sep 12, 2024
ef48c7e
fixed some negations that got missed when changing all_images to only…
Sep 12, 2024
06239fa
removed empty lines
Sep 12, 2024
bcd2f57
removed duplicate code by adding a helper method to common.py
Sep 12, 2024
cabf203
missed changing an all_images to only_annotated
Sep 12, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions cvat-core/src/annotations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ export async function exportDataset(
instance,
format: string,
saveImages: boolean,
onlyAnnotated: boolean,
useDefaultSettings: boolean,
targetStorage: Storage,
name?: string,
Expand All @@ -177,13 +178,13 @@ export async function exportDataset(
let result = null;
if (instance instanceof Task) {
result = await serverProxy.tasks
.exportDataset(instance.id, format, saveImages, useDefaultSettings, targetStorage, name);
.exportDataset(instance.id, format, saveImages, onlyAnnotated, useDefaultSettings, targetStorage, name);
} else if (instance instanceof Job) {
result = await serverProxy.jobs
.exportDataset(instance.id, format, saveImages, useDefaultSettings, targetStorage, name);
.exportDataset(instance.id, format, saveImages, onlyAnnotated, useDefaultSettings, targetStorage, name);
} else {
result = await serverProxy.projects
.exportDataset(instance.id, format, saveImages, useDefaultSettings, targetStorage, name);
.exportDataset(instance.id, format, saveImages, onlyAnnotated, useDefaultSettings, targetStorage, name);
}

return result;
Expand Down
17 changes: 13 additions & 4 deletions cvat-core/src/project-implementation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,11 +111,20 @@ export default function implementProject(Project: typeof ProjectClass): typeof P
this: ProjectClass,
format: Parameters<typeof ProjectClass.prototype.annotations.exportDataset>[0],
saveImages: Parameters<typeof ProjectClass.prototype.annotations.exportDataset>[1],
useDefaultSettings: Parameters<typeof ProjectClass.prototype.annotations.exportDataset>[2],
targetStorage: Parameters<typeof ProjectClass.prototype.annotations.exportDataset>[3],
customName: Parameters<typeof ProjectClass.prototype.annotations.exportDataset>[4],
onlyAnnotated: Parameters<typeof ProjectClass.prototype.annotations.exportDataset>[2],
useDefaultSettings: Parameters<typeof ProjectClass.prototype.annotations.exportDataset>[3],
targetStorage: Parameters<typeof ProjectClass.prototype.annotations.exportDataset>[4],
customName: Parameters<typeof ProjectClass.prototype.annotations.exportDataset>[5],
): ReturnType<typeof ProjectClass.prototype.annotations.exportDataset> {
const rqID = await exportDataset(this, format, saveImages, useDefaultSettings, targetStorage, customName);
const rqID = await exportDataset(
this,
format,
saveImages,
onlyAnnotated,
useDefaultSettings,
targetStorage,
customName,
);
return rqID;
},
});
Expand Down
9 changes: 6 additions & 3 deletions cvat-core/src/project.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ export default class Project {
exportDataset: (
format: string,
saveImages: boolean,
onlyAnnotated: boolean,
useDefaultSettings: boolean,
targetStorage: Storage,
name?: string,
Expand Down Expand Up @@ -265,15 +266,17 @@ Object.defineProperties(
async exportDataset(
format: Parameters<typeof Project.prototype.annotations.exportDataset>[0],
saveImages: Parameters<typeof Project.prototype.annotations.exportDataset>[1],
useDefaultSettings: Parameters<typeof Project.prototype.annotations.exportDataset>[2],
targetStorage: Parameters<typeof Project.prototype.annotations.exportDataset>[3],
customName: Parameters<typeof Project.prototype.annotations.exportDataset>[4],
onlyAnnotated: Parameters<typeof Project.prototype.annotations.exportDataset>[2],
useDefaultSettings: Parameters<typeof Project.prototype.annotations.exportDataset>[3],
targetStorage: Parameters<typeof Project.prototype.annotations.exportDataset>[4],
customName: Parameters<typeof Project.prototype.annotations.exportDataset>[5],
) {
const result = await PluginRegistry.apiWrapper.call(
this,
Project.prototype.annotations.exportDataset,
format,
saveImages,
onlyAnnotated,
useDefaultSettings,
targetStorage,
customName,
Expand Down
3 changes: 3 additions & 0 deletions cvat-core/src/server-proxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type Params = {
filename?: string,
action?: string,
save_images?: boolean,
only_annotated?: boolean,
};

tus.defaultOptions.storeFingerprintForResuming = false;
Expand Down Expand Up @@ -805,6 +806,7 @@ function exportDataset(instanceType: 'projects' | 'jobs' | 'tasks') {
id: number,
format: string,
saveImages: boolean,
onlyAnnotated: boolean,
useDefaultSettings: boolean,
targetStorage: Storage,
name?: string,
Expand All @@ -817,6 +819,7 @@ function exportDataset(instanceType: 'projects' | 'jobs' | 'tasks') {
...(name ? { filename: name } : {}),
format,
save_images: saveImages,
only_annotated: onlyAnnotated,
};
return new Promise<string | void>((resolve, reject) => {
async function request() {
Expand Down
34 changes: 26 additions & 8 deletions cvat-core/src/session-implementation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -513,11 +513,20 @@ export function implementJob(Job: typeof JobClass): typeof JobClass {
this: JobClass,
format: Parameters<typeof JobClass.prototype.annotations.exportDataset>[0],
saveImages: Parameters<typeof JobClass.prototype.annotations.exportDataset>[1],
useDefaultSettings: Parameters<typeof JobClass.prototype.annotations.exportDataset>[2],
targetStorage: Parameters<typeof JobClass.prototype.annotations.exportDataset>[3],
customName?: Parameters<typeof JobClass.prototype.annotations.exportDataset>[4],
onlyAnnotated: Parameters<typeof JobClass.prototype.annotations.exportDataset>[2],
useDefaultSettings: Parameters<typeof JobClass.prototype.annotations.exportDataset>[3],
targetStorage: Parameters<typeof JobClass.prototype.annotations.exportDataset>[4],
customName?: Parameters<typeof JobClass.prototype.annotations.exportDataset>[5],
): ReturnType<typeof JobClass.prototype.annotations.exportDataset> {
const rqID = await exportDataset(this, format, saveImages, useDefaultSettings, targetStorage, customName);
const rqID = await exportDataset(
this,
format,
saveImages,
onlyAnnotated,
useDefaultSettings,
targetStorage,
customName,
);
return rqID;
},
});
Expand Down Expand Up @@ -1169,11 +1178,20 @@ export function implementTask(Task: typeof TaskClass): typeof TaskClass {
this: TaskClass,
format: Parameters<typeof TaskClass.prototype.annotations.exportDataset>[0],
saveImages: Parameters<typeof TaskClass.prototype.annotations.exportDataset>[1],
useDefaultSettings: Parameters<typeof TaskClass.prototype.annotations.exportDataset>[2],
targetStorage: Parameters<typeof TaskClass.prototype.annotations.exportDataset>[3],
customName: Parameters<typeof TaskClass.prototype.annotations.exportDataset>[4],
onlyAnnotated: Parameters<typeof TaskClass.prototype.annotations.exportDataset>[2],
useDefaultSettings: Parameters<typeof TaskClass.prototype.annotations.exportDataset>[3],
targetStorage: Parameters<typeof TaskClass.prototype.annotations.exportDataset>[4],
customName: Parameters<typeof TaskClass.prototype.annotations.exportDataset>[5],
): ReturnType<typeof TaskClass.prototype.annotations.exportDataset> {
const rqID = await exportDataset(this, format, saveImages, useDefaultSettings, targetStorage, customName);
const rqID = await exportDataset(
this,
format,
saveImages,
onlyAnnotated,
useDefaultSettings,
targetStorage,
customName,
);
return rqID;
},
});
Expand Down
3 changes: 3 additions & 0 deletions cvat-core/src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ function buildDuplicatedAPI(prototype) {
async exportDataset(
format: string,
saveImages: boolean,
onlyAnnotated: boolean,
useDefaultSettings: boolean,
targetStorage: Storage,
customName?: string,
Expand All @@ -182,6 +183,7 @@ function buildDuplicatedAPI(prototype) {
prototype.annotations.exportDataset,
format,
saveImages,
onlyAnnotated,
useDefaultSettings,
targetStorage,
customName,
Expand Down Expand Up @@ -360,6 +362,7 @@ export class Session {
exportDataset: (
format: string,
saveImages: boolean,
onlyAnnotated: boolean,
useDefaultSettings: boolean,
targetStorage: Storage,
name?: string,
Expand Down
8 changes: 8 additions & 0 deletions cvat-sdk/cvat_sdk/core/proxies/model_proxy.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,8 @@ def export(
status_check_period: Optional[int] = None,
location: Optional[Location] = None,
cloud_storage_id: Optional[int] = None,
save_images: Optional[bool] = None,
only_annotated: Optional[bool] = None,
**query_params,
) -> None:
query_params = {
Expand All @@ -254,6 +256,10 @@ def export(
if not local_downloading:
query_params["filename"] = str(filename)

if save_images and only_annotated:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How these 2 parameters are related?

query_params["save_images"] = save_images
query_params["only_annotated"] = only_annotated

downloader = Downloader(self._client)
export_request = downloader.prepare_file(
endpoint,
Expand Down Expand Up @@ -300,6 +306,7 @@ def export_dataset(
pbar: Optional[ProgressReporter] = None,
status_check_period: Optional[int] = None,
include_images: bool = True,
only_annotated: bool = False,
location: Optional[Location] = None,
cloud_storage_id: Optional[int] = None,
) -> None:
Expand Down Expand Up @@ -332,6 +339,7 @@ def export_dataset(
cloud_storage_id=cloud_storage_id,
format=format_name,
save_images=include_images,
only_annotated=only_annotated,
)

self._client.logger.info(
Expand Down
3 changes: 2 additions & 1 deletion cvat-ui/src/actions/export-actions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ export const exportDatasetAsync = (
instance: ProjectOrTaskOrJob,
format: string,
saveImages: boolean,
onlyAnnotated: boolean,
useDefaultSettings: boolean,
targetStorage: Storage,
name?: string,
Expand All @@ -114,7 +115,7 @@ export const exportDatasetAsync = (

try {
const rqID = await instance.annotations
.exportDataset(format, saveImages, useDefaultSettings, targetStorage, name);
.exportDataset(format, saveImages, onlyAnnotated, useDefaultSettings, targetStorage, name);
if (shouldListenForProgress(rqID, state.requests)) {
await listenExportDatasetAsync(rqID, dispatch, {
instance, format, saveImages,
Expand Down
27 changes: 25 additions & 2 deletions cvat-ui/src/components/export-dataset/export-dataset-modal.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import Input from 'antd/lib/input';
import Form from 'antd/lib/form';
import Switch from 'antd/lib/switch';
import Space from 'antd/lib/space';
import Radio from 'antd/lib/radio';
import TargetStorageField from 'components/storage/target-storage-field';
import CVATMarkdown from 'components/common/cvat-markdown';
import { CombinedState, StorageLocation } from 'reducers';
Expand All @@ -27,6 +28,7 @@ import {
type FormValues = {
selectedFormat: string | undefined;
saveImages: boolean;
onlyAnnotated: boolean;
customName: string | undefined;
targetStorage: StorageData;
useProjectTargetStorage: boolean;
Expand All @@ -35,6 +37,7 @@ type FormValues = {
const initialValues: FormValues = {
selectedFormat: undefined,
saveImages: false,
onlyAnnotated: false,
customName: undefined,
targetStorage: {
location: StorageLocation.LOCAL,
Expand All @@ -52,6 +55,7 @@ function ExportDatasetModal(props: StateToProps): JSX.Element {
const [instanceType, setInstanceType] = useState('');

const [useDefaultTargetStorage, setUseDefaultTargetStorage] = useState(true);
const [saveImages, setSaveImages] = useState(false);
const [form] = Form.useForm();
const [targetStorage, setTargetStorage] = useState<StorageData>({
location: StorageLocation.LOCAL,
Expand Down Expand Up @@ -94,6 +98,7 @@ function ExportDatasetModal(props: StateToProps): JSX.Element {
const closeModal = (): void => {
setUseDefaultTargetStorage(true);
setTargetStorage({ location: StorageLocation.LOCAL });
setSaveImages(false);
form.resetFields();
if (instance) {
dispatch(exportActions.closeExportDatasetModal(instance));
Expand All @@ -108,6 +113,7 @@ function ExportDatasetModal(props: StateToProps): JSX.Element {
instance as ProjectOrTaskOrJob,
values.selectedFormat as string,
values.saveImages,
values.onlyAnnotated,
useDefaultTargetStorage,
useDefaultTargetStorage ? new Storage({
location: defaultStorageLocation,
Expand Down Expand Up @@ -180,11 +186,28 @@ function ExportDatasetModal(props: StateToProps): JSX.Element {
name='saveImages'
valuePropName='checked'
>
<Switch className='cvat-modal-export-save-images' />
</Form.Item>
<Switch
className='cvat-modal-export-save-images'
onChange={(checked: boolean) => {
setSaveImages(checked);
form.setFieldsValue({ saveImages: checked });
}}
/> </Form.Item>
<Text strong>Save images</Text>
</Space>

{saveImages && (
<Form.Item
name='onlyAnnotated'
label={<Text strong>Image saving option</Text>}
>
<Radio.Group className='cvat-modal-export-image-saving-options'>
<Radio value={false}>All images</Radio>
<Radio value={true}>Images with annotations only</Radio>
</Radio.Group>
</Form.Item>
)}

<Form.Item label={<Text strong>Custom name</Text>} name='customName'>
<Input
placeholder='Custom name for a dataset'
Expand Down
13 changes: 10 additions & 3 deletions cvat/apps/dataset_manager/bindings.py
Original file line number Diff line number Diff line change
Expand Up @@ -1574,6 +1574,7 @@ def __init__(
instance_data: CommonData,
*,
include_images: bool = False,
only_annotated: bool = False,
format_type: str = None,
dimension: DimensionType = DimensionType.DIM_2D,
**kwargs
Expand Down Expand Up @@ -1640,7 +1641,8 @@ def __init__(
attributes=attributes, subset=frame_data.subset,
)

dm_items.append(dm_item)
if not only_annotated or len(dm_anno) > 0:
dm_items.append(dm_item)

self._items = dm_items

Expand All @@ -1662,6 +1664,7 @@ def __init__(
project_data: ProjectData,
*,
include_images: bool = False,
only_annotated: bool = False,
format_type: str = None,
dimension: DimensionType = DimensionType.DIM_2D,
**kwargs
Expand Down Expand Up @@ -1729,8 +1732,10 @@ def __init__(
annotations=dm_anno, media=PointCloud(dm_image[0]), related_images=dm_image[1],
attributes=attributes, subset=frame_data.subset
)
dm_items.append(dm_item)


if not only_annotated or len(dm_anno) > 0:
dm_items.append(dm_item)

self._items = dm_items

def categories(self):
Expand All @@ -1746,12 +1751,14 @@ def __len__(self):
def GetCVATDataExtractor(
instance_data: Union[ProjectData, CommonData],
include_images: bool = False,
only_annotated: bool = False,
format_type: str = None,
dimension: DimensionType = DimensionType.DIM_2D,
**kwargs
):
kwargs.update({
'include_images': include_images,
'only_annotated': only_annotated,
'format_type': format_type,
'dimension': dimension,
})
Expand Down
4 changes: 2 additions & 2 deletions cvat/apps/dataset_manager/formats/camvid.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@


@exporter(name='CamVid', ext='ZIP', version='1.0')
def _export(dst_file, temp_dir, instance_data, save_images=False):
with GetCVATDataExtractor(instance_data, include_images=save_images) as extractor:
def _export(dst_file, temp_dir, instance_data, save_images=False, only_annotated=False):
with GetCVATDataExtractor(instance_data, include_images=save_images, only_annotated=only_annotated) as extractor:
dataset = Dataset.from_extractors(extractor, env=dm_env)
dataset.transform(RotatedBoxesToPolygons)
dataset.transform('polygons_to_masks')
Expand Down
4 changes: 2 additions & 2 deletions cvat/apps/dataset_manager/formats/cityscapes.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@


@exporter(name='Cityscapes', ext='ZIP', version='1.0')
def _export(dst_file, temp_dir, instance_data, save_images=False):
with GetCVATDataExtractor(instance_data, include_images=save_images) as extractor:
def _export(dst_file, temp_dir, instance_data, save_images=False, only_annotated=False):
with GetCVATDataExtractor(instance_data, include_images=save_images, only_annotated=only_annotated) as extractor:
dataset = Dataset.from_extractors(extractor, env=dm_env)
dataset.transform(RotatedBoxesToPolygons)
dataset.transform('polygons_to_masks')
Expand Down
Loading