diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index d513dcd86..8488aad8c 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -598,6 +598,7 @@ export default defineComponent({ progress.total = tracks.length + groups.length; const trackStore = cameraStore.camMap.value.get(camera)?.trackStore; const groupStore = cameraStore.camMap.value.get(camera)?.groupStore; + console.log(`Loading Camera: ${camera}`); if (trackStore && groupStore) { // We can start sorting if our total tracks are less than 20000 // If greater we do one sort at the end instead to speed loading. diff --git a/client/dive-common/constants.ts b/client/dive-common/constants.ts index 04cae3de3..b3f3ff792 100644 --- a/client/dive-common/constants.ts +++ b/client/dive-common/constants.ts @@ -13,7 +13,7 @@ const MediaTypes: Record = { [LargeImageType]: 'tiled image', }; -const DefaultVideoFPS = 10; +const DefaultVideoFPS = -1; const FPSOptions = [ { text: 1, value: 1 }, { text: 5, value: 5 }, diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 9ddac5a44..aaa885da6 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -430,9 +430,9 @@ async function getPipelineList(settings: Settings): Promise { */ async function getTrainingConfigs(settings: Settings): Promise { const pipelinePath = npath.join(settings.viamePath, 'configs/pipelines'); - const defaultTrainingConfiguration = 'train_detector_default.viame_csv.conf'; - const allowedPatterns = /\.viame_csv\.conf$/; - const disallowedPatterns = /.*(_nf|\.continue)\.viame_csv\.conf$/; + const defaultTrainingConfiguration = 'train_detector_default.conf'; + const allowedPatterns = /train_.*\.conf$/; + const disallowedPatterns = /.*(_nf|\.continue)\.viame_csv\.conf$|.*\.kw18\.conf$|.*\.habcam\.conf$|.*\.continue\.conf$/; const exists = await fs.pathExists(pipelinePath); if (!exists) { throw new Error(`Path does not exist: ${pipelinePath}`); diff --git a/client/platform/desktop/backend/serializers/viame.ts b/client/platform/desktop/backend/serializers/viame.ts index 0c8e604a1..6a729ea81 100644 --- a/client/platform/desktop/backend/serializers/viame.ts +++ b/client/platform/desktop/backend/serializers/viame.ts @@ -281,8 +281,11 @@ async function parse(input: Readable, imageMap?: Map): Promise - export detections + VIAME CSV diff --git a/client/platform/desktop/frontend/components/NavigationBar.vue b/client/platform/desktop/frontend/components/NavigationBar.vue index d8fcb754b..8ec1958ce 100644 --- a/client/platform/desktop/frontend/components/NavigationBar.vue +++ b/client/platform/desktop/frontend/components/NavigationBar.vue @@ -21,7 +21,7 @@ export default defineComponent({ color="accent" > - Recents + Library mdi-folder-open diff --git a/client/platform/desktop/frontend/components/Settings.vue b/client/platform/desktop/frontend/components/Settings.vue index e8cb36990..34d6304b6 100644 --- a/client/platform/desktop/frontend/components/Settings.vue +++ b/client/platform/desktop/frontend/components/Settings.vue @@ -282,7 +282,7 @@ export default defineComponent({ > mdi-check-circle - Sync recents with Project Data + Sync library with Project Data diff --git a/client/platform/desktop/frontend/components/ViewerLoader.vue b/client/platform/desktop/frontend/components/ViewerLoader.vue index 37429a173..cb4e9e18f 100644 --- a/client/platform/desktop/frontend/components/ViewerLoader.vue +++ b/client/platform/desktop/frontend/components/ViewerLoader.vue @@ -129,7 +129,7 @@ export default defineComponent({ style="flex-basis:0; flex-grow:0;" > - Recents + Library mdi-folder-open diff --git a/client/platform/web-girder/views/Export.vue b/client/platform/web-girder/views/Export.vue index ce63c505d..52779c18a 100644 --- a/client/platform/web-girder/views/Export.vue +++ b/client/platform/web-girder/views/Export.vue @@ -298,13 +298,8 @@ export default defineComponent({ - - - - - - - TrackJSON - - - - - + + DIVE TrackJSON + detections unavailable + + + diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 450cb85e4..4295e989a 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -278,7 +278,11 @@ export default defineComponent({ } else { tailLayer.disable(); } - pointLayer.changeData(frameData); + if (visibleModes.includes('LineString')) { + pointLayer.changeData(frameData); + } else { + pointLayer.disable(); + } if (visibleModes.includes('text')) { textLayer.changeData(frameData); attributeBoxLayer.changeData(frameData); diff --git a/client/src/track.ts b/client/src/track.ts index 8610aef58..94e93414d 100644 --- a/client/src/track.ts +++ b/client/src/track.ts @@ -517,6 +517,9 @@ export default class Track extends BaseAnnotation { }; }); // accept either number or string, convert to number + if (json.id === null) { + console.log(json); + } const intTrackId = parseInt(json.id.toString(), 10); const track = new Track(intTrackId, { features: sparseFeatures, diff --git a/docs/Dive-Desktop.md b/docs/Dive-Desktop.md index 44923394c..80b065c52 100644 --- a/docs/Dive-Desktop.md +++ b/docs/Dive-Desktop.md @@ -73,7 +73,7 @@ DIVE Desktop requires a local installation of the VIAME toolkit to run ML pipeli * Use ==Choose :material-folder:== to choose the base installation path, then click ==:material-content-save: Save==. * **Project Data Storage Path** defaults to a subfolder in your user workspace and should generally not be changed. * **Read only mode** disables the ability to save when using the annotator. -* **Synchronize Recents** - The ==:material-sync: Synchronize Recents with Project Data== button is useful if data in the Project Data Storage Path gets out of sync with what appears in the ==:material-folder-open: Recents== list. +* **Synchronize Library** - The ==:material-sync: Synchronize Library with Project Data== button is useful if data in the Project Data Storage Path gets out of sync with what appears in the ==:material-folder-open: Library== list. ### Data Storage Path diff --git a/docs/Pipeline-Documentation.md b/docs/Pipeline-Documentation.md index 1aef0bb65..56b54b18b 100644 --- a/docs/Pipeline-Documentation.md +++ b/docs/Pipeline-Documentation.md @@ -112,15 +112,15 @@ By default, training runs include all frames from the chosen input datasets, and | Configuration | Availability | Use Case | | ------------- | ------------ | -------- | | detector_default | both | alias: train detector netharn cfrnn | -| [detector_netharn_cfrnn](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_netharn_cfrnn.viame_csv.conf) | both | | -| [detector_netharn_mask_rcnn](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_netharn_mask_rcnn.viame_csv.conf) | both | | -| [detector_svm_over_generic_detections](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_svm_over_generic_detections.viame_csv.conf) | both | general purpose svm | -| [detector_svm_over_fish_detections](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_svm_over_fish_detections.viame_csv.conf) | both | fish svm | +| [detector_netharn_cfrnn](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_netharn_cfrnn.conf) | both | | +| [detector_netharn_mask_rcnn](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_netharn_mask_rcnn.conf) | both | | +| [detector_svm_over_generic_detections](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_svm_over_generic_detections.conf) | both | general purpose svm | +| [detector_svm_over_fish_detections](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_svm_over_fish_detections.conf) | both | fish svm | | frame_classifier_default | both | alias: frame classifier netharn resnet | -| [frame_classifier_netharn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_frame_classifier_netharn_resnet.viame_csv.conf) | both | full-frame | -| [frame_classifier_svm_overn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_frame_classifier_svm_over_resnet.viame_csv.conf) | both | full-frame | +| [frame_classifier_netharn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_frame_classifier_netharn_resnet.conf) | both | full-frame | +| [frame_classifier_svm_overn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_frame_classifier_svm_over_resnet.conf) | both | full-frame | | object_classifier_default | both | alias: netharn resnet object classifier | -| [object_classifier_netharn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_object_classifier_netharn_resnet.viame_csv.conf) | both | | +| [object_classifier_netharn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_object_classifier_netharn_resnet.conf) | both | | | yolo | desktop only | can train, but resulting model **is not runnable with desktop** yet | ## Pipeline Import and Export diff --git a/samples/rename.py b/samples/rename.py new file mode 100644 index 000000000..b98f045da --- /dev/null +++ b/samples/rename.py @@ -0,0 +1,47 @@ +import json +import os + +import click +import girder_client +import csv +from os.path import basename, splitext + + +apiURL = "viame.kitware.com" +rootFolder = "65a19f18cf5a99794ea99cdb" # Sample folder girder Id +limit = 10 # for testing purposes keep lower then increase + + +# Login to the girder client, interactive means it will prompt for username and password +def login(): + gc = girder_client.GirderClient(apiURL, port=443, apiRoot="girder/api/v1", ) + gc.authenticate(interactive=True) + return gc + + +def get_dive_datasets(gc: girder_client.GirderClient, rootFolderId: str,): + folders = gc.listFolder(rootFolderId) + dive_datasets = {} + for item in folders: + if item.get('meta', {}).get('annotate', False) is True: + # then it's a dive Dataset + dive_datasets[item['name'].replace('Video ', '').replace('.mp4', '')] = item + if 'Video ' in item['name']: + print(f'Renaming: {item["name"]}') + newname = item['name'].replace('Video ', '').replace('.mp4', '') + gc.put(f"folder/{item['_id']}", data={'name': newname}) + items = gc.listItem(rootFolderId) + for item in items: + if item.get('name').endswith('csv'): + print(f'Deleting: {item["name"]}') + gc.delete(f"item/{item['_id']}") + + +@click.command(name="process NoCopy Imports", help="Load in ") +def load_data(): + gc = login() + get_dive_datasets(gc, rootFolder) + + +if __name__ == "__main__": + load_data() diff --git a/server/dive_tasks/pipeline_discovery.py b/server/dive_tasks/pipeline_discovery.py index 8716704d5..baf20d819 100644 --- a/server/dive_tasks/pipeline_discovery.py +++ b/server/dive_tasks/pipeline_discovery.py @@ -9,9 +9,9 @@ TrainingConfigurationSummary, ) -DefaultTrainingConfiguration = "train_detector_default.viame_csv.conf" -AllowedTrainingConfigs = r".*\.viame_csv\.conf$" -DisallowedTrainingConfigs = r".*(_nf|\.continue)\.viame_csv\.conf$" +DefaultTrainingConfiguration = "train_detector_default.conf" +AllowedTrainingConfigs = r"train_.*\.conf$" +DisallowedTrainingConfigs = r".*(_nf|\.continue)\.viame_csv\.conf$|.*\.continue\.conf$|.*\.habcam\.conf$|.*\.kw18\.conf$" AllowedStaticPipelines = r"^detector_.+|^tracker_.+|^utility_.+|^generate_.+" DisallowedStaticPipelines = ( # Remove utilities pipes which hold no meaning in web diff --git a/server/dive_tasks/utils.py b/server/dive_tasks/utils.py index f8ab0f549..4d3b27082 100644 --- a/server/dive_tasks/utils.py +++ b/server/dive_tasks/utils.py @@ -1,6 +1,7 @@ from datetime import datetime, timedelta import json import os +import time from pathlib import Path import shutil import signal @@ -91,6 +92,8 @@ def stream_subprocess( if process.stdout is None: raise RuntimeError("Stdout must not be none") + last_refresh_time = time.time() + # call readline until it returns empty bytes for line in iter(process.stdout.readline, b''): line_str = line.decode('utf-8') @@ -99,8 +102,13 @@ def stream_subprocess( stdout += line_str # Cancel the subprocess if the status is cancelling - # note this only checks when there is stdout from the subprocess - manager.refreshStatus() + # note this only checks when there is stdout from the subprocess every 5 minutes + # refreshStatus I believe is an expensive tas + current_time = time.time() + if current_time - last_refresh_time >= 300: + last_refresh_time = current_time + manager.refreshStatus() + if check_canceled(task, context, force=False) or manager.status == JobStatus.CANCELING: # Can never be sure what signal a process will respond to. process.send_signal(signal.SIGTERM) diff --git a/server/dive_utils/constants.py b/server/dive_utils/constants.py index 5abef6ecf..02590783a 100644 --- a/server/dive_utils/constants.py +++ b/server/dive_utils/constants.py @@ -7,7 +7,7 @@ ImageSequenceType = "image-sequence" VideoType = "video" LargeImageType = "large-image" -DefaultVideoFPS = 10 +DefaultVideoFPS = -1 JsonMetaCurrentVersion = 1 SettingsCurrentVersion = 1 AnnotationsCurrentVersion = 2