Aggregate updates (#1400)

* intial updates

* utils task canceling update

* training config updates

* regex update, download JSON menu, tail point hiding

client/dive-common/components/Viewer.vue

@@ -598,6 +598,7 @@ export default defineComponent({
  progress.total = tracks.length + groups.length;
  const trackStore = cameraStore.camMap.value.get(camera)?.trackStore;
  const groupStore = cameraStore.camMap.value.get(camera)?.groupStore;
+ console.log(`Loading Camera: ${camera}`);
  if (trackStore && groupStore) {
  // We can start sorting if our total tracks are less than 20000
  // If greater we do one sort at the end instead to speed loading.

client/dive-common/constants.ts

@@ -13,7 +13,7 @@ const MediaTypes: Record<DatasetType, string> = {
  [LargeImageType]: 'tiled image',
 };
 
-const DefaultVideoFPS = 10;
+const DefaultVideoFPS = -1;
 const FPSOptions = [
  { text: 1, value: 1 },
  { text: 5, value: 5 },

client/platform/desktop/backend/native/common.ts

@@ -430,9 +430,9 @@ async function getPipelineList(settings: Settings): Promise<Pipelines> {
  */
 async function getTrainingConfigs(settings: Settings): Promise<TrainingConfigs> {
  const pipelinePath = npath.join(settings.viamePath, 'configs/pipelines');
- const defaultTrainingConfiguration = 'train_detector_default.viame_csv.conf';
- const allowedPatterns = /\.viame_csv\.conf$/;
- const disallowedPatterns = /.*(_nf|\.continue)\.viame_csv\.conf$/;
+ const defaultTrainingConfiguration = 'train_detector_default.conf';
+ const allowedPatterns = /train_.*\.conf$/;
+ const disallowedPatterns = /.*(_nf|\.continue)\.viame_csv\.conf$|.*\.kw18\.conf$|.*\.habcam\.conf$|.*\.continue\.conf$/;
  const exists = await fs.pathExists(pipelinePath);
  if (!exists) {
  throw new Error(`Path does not exist: ${pipelinePath}`);

client/platform/desktop/backend/serializers/viame.ts

@@ -281,8 +281,11 @@ async function parse(input: Readable, imageMap?: Map<string, number>): Promise<A
  const k = i + 1;
  if (k < filteredImages.length) {
  if (filteredImages[i].csvFrame + 1 !== filteredImages[k].csvFrame || filteredImages[i].frame + 1 !== filteredImages[k].frame) {
- // We have misaligned video sequences so we error out
- error = new Error('A subsampling of images were used with the CSV but they were not sequential');
+ // We have misaligned image sequences so we error out
+ error = new Error(`A subsampling of images were used with the CSV but they were not sequential\n 
+ ${filteredImages[i].csvFrame + 1} !== ${filteredImages[k].csvFrame} || ${filteredImages[i].frame + 1} !== ${filteredImages[k].frame}\n
+ image1: ${filteredImages[i].image} image2: ${filteredImages[k].image} - these should be sequential in the CSV
+ \n`);
  }
  }
  frameMapper[filteredImages[i].csvFrame] = i;

client/platform/desktop/frontend/components/Export.vue

@@ -210,7 +210,7 @@ export default defineComponent({
  block
  @click="doExport({ type: 'dataset' })"
  >
- <span>export detections</span>
+ <span>VIAME CSV</span>
  </v-btn>
  </v-card-actions>
  <v-card-text class="pb-0">

client/platform/desktop/frontend/components/NavigationBar.vue

@@ -21,7 +21,7 @@ export default defineComponent({
  color="accent"
  >
  <v-tab :to="{ name: 'recent' }">
- Recents
+ Library
  <v-icon>mdi-folder-open</v-icon>
  </v-tab>
  <job-tab />

client/platform/desktop/frontend/components/Settings.vue

@@ -282,7 +282,7 @@ export default defineComponent({
  >
  mdi-check-circle
  </v-icon>
- Sync recents with Project Data
+ Sync library with Project Data
  </v-btn>
 
  <v-card-text>

client/platform/desktop/frontend/components/ViewerLoader.vue

@@ -129,7 +129,7 @@ export default defineComponent({
  style="flex-basis:0; flex-grow:0;"
  >
  <v-tab :to="{ name: 'recent' }">
- Recents
+ Library
  <v-icon>mdi-folder-open</v-icon>
  </v-tab>
  <job-tab />

client/platform/web-girder/views/Export.vue

@@ -298,13 +298,8 @@ export default defineComponent({
  </v-card-text>
 
  <v-card-actions>
- <v-menu
- offset-y
- offset-x
- nudge-left="180"
- max-width="180"
- >
- <template v-slot:activator="{ on }">
+ <v-row>
+ <v-col>
  <v-btn
  depressed
  block
@@ -313,35 +308,25 @@ export default defineComponent({
  >
  <span
  v-if="exportUrls.exportDetectionsUrl"
- class="col-11"
- >annotations</span>
+ >VIAME CSV</span>
  <span
  v-else
- class="col-11"
  >detections unavailable</span>
- <v-icon
- v-if="exportUrls.exportDetectionsUrl"
- class="button-dropdown col-1"
- v-on="on"
- >
- mdi-chevron-down
- </v-icon>
  </v-btn>
- </template>
- <v-card outlined>
- <v-list dense>
- <v-list-item
- style="align-items':'center"
- @click="doExport({ url: exportUrls
- && exportUrls.exportDetectionsUrlTrackJSON })"
- >
- <v-list-item-content>
- <v-list-item-title>TrackJSON</v-list-item-title>
- </v-list-item-content>
- </v-list-item>
- </v-list>
- </v-card>
- </v-menu>
+ <v-btn
+ depressed
+ block
+ class="mt-2"
+ :disabled="!exportUrls.exportDetectionsUrl"
+ @click="doExport({ url: exportUrls
+ && exportUrls.exportDetectionsUrlTrackJSON })" >
+ <span
+ v-if="exportUrls.exportDetectionsUrl"
+ >DIVE TrackJSON</span>
+ <span
+ v-else
+ >detections unavailable</span>
+ </v-btn>
  <!-- <v-btn
  depressed
  block
@@ -351,6 +336,8 @@ export default defineComponent({
  <span v-if="exportUrls.exportDetectionsUrl">annotations</span>
  <span v-else>detections unavailable</span>
  </v-btn> -->
+ </v-col>
+ </v-row>
  </v-card-actions>
 
  <v-card-text class="pb-0">

client/src/components/LayerManager.vue

@@ -278,7 +278,11 @@ export default defineComponent({
  } else {
  tailLayer.disable();
  }
- pointLayer.changeData(frameData);
+ if (visibleModes.includes('LineString')) {
+ pointLayer.changeData(frameData);
+ } else {
+ pointLayer.disable();
+ }
  if (visibleModes.includes('text')) {
  textLayer.changeData(frameData);
  attributeBoxLayer.changeData(frameData);

client/src/track.ts

@@ -517,6 +517,9 @@ export default class Track extends BaseAnnotation {
  };
  });
  // accept either number or string, convert to number
+ if (json.id === null) {
+ console.log(json);
+ }
  const intTrackId = parseInt(json.id.toString(), 10);
  const track = new Track(intTrackId, {
  features: sparseFeatures,

docs/Dive-Desktop.md

@@ -73,7 +73,7 @@ DIVE Desktop requires a local installation of the VIAME toolkit to run ML pipeli
  * Use ==Choose :material-folder:== to choose the base installation path, then click ==:material-content-save: Save==.
 * **Project Data Storage Path** defaults to a subfolder in your user workspace and should generally not be changed.
 * **Read only mode** disables the ability to save when using the annotator.
-* **Synchronize Recents** - The ==:material-sync: Synchronize Recents with Project Data== button is useful if data in the Project Data Storage Path gets out of sync with what appears in the ==:material-folder-open: Recents== list.
+* **Synchronize Library** - The ==:material-sync: Synchronize Library with Project Data== button is useful if data in the Project Data Storage Path gets out of sync with what appears in the ==:material-folder-open: Library== list.
 
 ### Data Storage Path
 

docs/Pipeline-Documentation.md

@@ -112,15 +112,15 @@ By default, training runs include all frames from the chosen input datasets, and
 | Configuration | Availability | Use Case |
 | ------------- | ------------ | -------- |
 | detector_default | both | alias: train detector netharn cfrnn |
-| [detector_netharn_cfrnn](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_netharn_cfrnn.viame_csv.conf) | both | |
-| [detector_netharn_mask_rcnn](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_netharn_mask_rcnn.viame_csv.conf) | both | |
-| [detector_svm_over_generic_detections](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_svm_over_generic_detections.viame_csv.conf) | both | general purpose svm |
-| [detector_svm_over_fish_detections](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_svm_over_fish_detections.viame_csv.conf) | both | fish svm |
+| [detector_netharn_cfrnn](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_netharn_cfrnn.conf) | both | |
+| [detector_netharn_mask_rcnn](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_netharn_mask_rcnn.conf) | both | |
+| [detector_svm_over_generic_detections](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_svm_over_generic_detections.conf) | both | general purpose svm |
+| [detector_svm_over_fish_detections](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_svm_over_fish_detections.conf) | both | fish svm |
 | frame_classifier_default | both | alias: frame classifier netharn resnet |
-| [frame_classifier_netharn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_frame_classifier_netharn_resnet.viame_csv.conf) | both | full-frame |
-| [frame_classifier_svm_overn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_frame_classifier_svm_over_resnet.viame_csv.conf) | both | full-frame |
+| [frame_classifier_netharn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_frame_classifier_netharn_resnet.conf) | both | full-frame |
+| [frame_classifier_svm_overn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_frame_classifier_svm_over_resnet.conf) | both | full-frame |
 | object_classifier_default | both | alias: netharn resnet object classifier |
-| [object_classifier_netharn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_object_classifier_netharn_resnet.viame_csv.conf) | both | |
+| [object_classifier_netharn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_object_classifier_netharn_resnet.conf) | both | |
 | yolo | desktop only | can train, but resulting model **is not runnable with desktop** yet |
 
 ## Pipeline Import and Export

samples/rename.py

@@ -0,0 +1,47 @@
+import json
+import os
+
+import click
+import girder_client
+import csv
+from os.path import basename, splitext
+
+
+apiURL = "viame.kitware.com"
+rootFolder = "65a19f18cf5a99794ea99cdb" # Sample folder girder Id
+limit = 10 # for testing purposes keep lower then increase
+
+
+# Login to the girder client, interactive means it will prompt for username and password
+def login():
+ gc = girder_client.GirderClient(apiURL, port=443, apiRoot="girder/api/v1", )
+ gc.authenticate(interactive=True)
+ return gc
+
+
+def get_dive_datasets(gc: girder_client.GirderClient, rootFolderId: str,):
+ folders = gc.listFolder(rootFolderId)
+ dive_datasets = {}
+ for item in folders:
+ if item.get('meta', {}).get('annotate', False) is True:
+ # then it's a dive Dataset
+ dive_datasets[item['name'].replace('Video ', '').replace('.mp4', '')] = item
+ if 'Video ' in item['name']:
+ print(f'Renaming: {item["name"]}')
+ newname = item['name'].replace('Video ', '').replace('.mp4', '')
+ gc.put(f"folder/{item['_id']}", data={'name': newname})
+ items = gc.listItem(rootFolderId)
+ for item in items:
+ if item.get('name').endswith('csv'):
+ print(f'Deleting: {item["name"]}')
+ gc.delete(f"item/{item['_id']}")
+
+
+@click.command(name="process NoCopy Imports", help="Load in ")
+def load_data():
+ gc = login()
+ get_dive_datasets(gc, rootFolder)
+
+
+if __name__ == "__main__":
+ load_data()

server/dive_tasks/pipeline_discovery.py

@@ -9,9 +9,9 @@
  TrainingConfigurationSummary,
 )
 
-DefaultTrainingConfiguration = "train_detector_default.viame_csv.conf"
-AllowedTrainingConfigs = r".*\.viame_csv\.conf$"
-DisallowedTrainingConfigs = r".*(_nf|\.continue)\.viame_csv\.conf$"
+DefaultTrainingConfiguration = "train_detector_default.conf"
+AllowedTrainingConfigs = r"train_.*\.conf$"
+DisallowedTrainingConfigs = r".*(_nf|\.continue)\.viame_csv\.conf$|.*\.continue\.conf$|.*\.habcam\.conf$|.*\.kw18\.conf$"
 AllowedStaticPipelines = r"^detector_.+|^tracker_.+|^utility_.+|^generate_.+"
 DisallowedStaticPipelines = (
  # Remove utilities pipes which hold no meaning in web

server/dive_tasks/utils.py

@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta
 import json
 import os
+import time
 from pathlib import Path
 import shutil
 import signal
@@ -91,6 +92,8 @@ def stream_subprocess(
  if process.stdout is None:
  raise RuntimeError("Stdout must not be none")
 
+ last_refresh_time = time.time()
+
  # call readline until it returns empty bytes
  for line in iter(process.stdout.readline, b''):
  line_str = line.decode('utf-8')
@@ -99,8 +102,13 @@ def stream_subprocess(
  stdout += line_str
 
  # Cancel the subprocess if the status is cancelling
- # note this only checks when there is stdout from the subprocess
- manager.refreshStatus()
+ # note this only checks when there is stdout from the subprocess every 5 minutes
+ # refreshStatus I believe is an expensive tas
+ current_time = time.time()
+ if current_time - last_refresh_time >= 300:
+ last_refresh_time = current_time
+ manager.refreshStatus()
+
  if check_canceled(task, context, force=False) or manager.status == JobStatus.CANCELING:
  # Can never be sure what signal a process will respond to.
  process.send_signal(signal.SIGTERM)

server/dive_utils/constants.py

@@ -7,7 +7,7 @@
 ImageSequenceType = "image-sequence"
 VideoType = "video"
 LargeImageType = "large-image"
-DefaultVideoFPS = 10
+DefaultVideoFPS = -1
 JsonMetaCurrentVersion = 1
 SettingsCurrentVersion = 1
 AnnotationsCurrentVersion = 2