Add experimental queue experiments from csv command

demo/queue.csv

@@ -0,0 +1,4 @@
+lr,weight_decay
+0.0001,0.02
+0.00075,0.01
+0.0005,

extension/package.json

@@ -182,6 +182,15 @@
         "category": "DVC",
         "icon": "$(cloud-upload)"
       },
+      {
+        "title": "%command.queueExperimentsFromCsv%",
+        "command": "dvc.queueExperimentsFromCsv",
+        "category": "DVC",
+        "icon": {
+          "dark": "resources/dark/queue-experiment.svg",
+          "light": "resources/light/queue-experiment.svg"
+        }
+      },
       {
         "title": "%command.queueExperiment%",
         "command": "dvc.queueExperiment",
@@ -462,6 +471,10 @@
           "command": "dvc.pushTarget",
           "when": "false"
         },
+        {
+          "command": "dvc.queueExperimentsFromCsv",
+          "when": "dvc.commands.available && dvc.project.available"
+        },
         {
           "command": "dvc.queueExperiment",
           "when": "dvc.commands.available && dvc.project.available"
@@ -1042,6 +1055,7 @@
   "dependencies": {
     "@hediet/std": "^0.6.0",
     "chokidar": "^3.5.2",
+    "csv-parse": "^5.0.3",
     "execa": "^5.1.1",
     "fs-extra": "^10.0.0",
     "lodash.get": "^4.4.2",

extension/package.nls.json

@@ -27,6 +27,7 @@
   "command.pullTarget": "Pull",
   "command.push": "Push",
   "command.pushTarget": "Push",
+  "command.queueExperimentsFromCsv": "Queue Experiments From CSV",
   "command.queueExperiment": "Queue Experiment",
   "command.removeExperiment": "Remove Experiment",
   "command.removeExperimentsTableFilters": "Remove Filter(s) From Experiments Table",

extension/src/cli/args.ts

@@ -26,7 +26,8 @@ export enum Flag {
   HELP = '-h',
   RECURSIVE = '-R',
   SHOW_JSON = '--show-json',
-  SUBDIRECTORY = '--subdir'
+  SUBDIRECTORY = '--subdir',
+  SET_PARAM = '-S'
 }
 
 export enum ExperimentSubCommand {

extension/src/cli/executor.ts

@@ -83,11 +83,12 @@ export class CliExecutor extends Cli {
     )
   }
 
-  public experimentRunQueue(cwd: string) {
+  public experimentRunQueue(cwd: string, ...args: Args) {
     return this.executeExperimentProcess(
       cwd,
       ExperimentSubCommand.RUN,
-      ExperimentFlag.QUEUE
+      ExperimentFlag.QUEUE,
+      ...args
     )
   }
 

extension/src/commands/external.ts

@@ -37,6 +37,7 @@ export enum RegisteredCommands {
   EXPERIMENT_SORTS_REMOVE = 'dvc.removeExperimentsTableSorts',
   EXPERIMENT_SORTS_REMOVE_ALL = 'dvc.views.experimentsSortByTree.removeAllSorts',
   EXPERIMENT_TOGGLE = 'dvc.views.experimentsTree.toggleStatus',
+  QUEUE_EXPERIMENTS_FROM_CSV = 'dvc.queueExperimentsFromCsv',
   STOP_EXPERIMENT = 'dvc.stopRunningExperiment',
 
   PLOTS_SHOW = 'dvc.showPlots',

extension/src/experiments/commands/register.ts

@@ -9,12 +9,18 @@ import {
 const registerExperimentCwdCommands = (
   experiments: WorkspaceExperiments,
   internalCommands: InternalCommands
-): void =>
+): void => {
   internalCommands.registerExternalCliCommand(
     RegisteredCliCommands.QUEUE_EXPERIMENT,
     () => experiments.getCwdThenReport(AvailableCommands.EXPERIMENT_QUEUE)
   )
 
+  internalCommands.registerExternalCommand(
+    RegisteredCommands.QUEUE_EXPERIMENTS_FROM_CSV,
+    () => experiments.queueExperimentsFromCsv()
+  )
+}
+
 const registerExperimentNameCommands = (
   experiments: WorkspaceExperiments,
   internalCommands: InternalCommands

extension/src/experiments/queue.ts

@@ -0,0 +1,43 @@
+import { Flag } from '../cli/args'
+import { exists, readCsv } from '../fileSystem'
+import { join } from '../test/util/path'
+import { definedAndNonEmpty } from '../util/array'
+import { delay } from '../util/time'
+
+const collectParamsToVary = (csvRow: Record<string, unknown>): string[] =>
+  Object.entries(csvRow).reduce((acc, [k, v]) => {
+    const key = k.trim()
+    const value = (v as string).trim()
+
+    if (key !== '' && value !== '') {
+      acc.push(Flag.SET_PARAM)
+      const str = [key, value].join('=')
+      acc.push(str)
+    }
+
+    return acc
+  }, [] as string[])
+
+export const readToQueueFromCsv = (path: string): Promise<string[][]> =>
+  new Promise(resolve => {
+    const toQueue: string[][] = []
+    readCsv(path)
+      .on('data', row => {
+        const paramsToVary = collectParamsToVary(row)
+        if (definedAndNonEmpty(paramsToVary)) {
+          toQueue.push(paramsToVary)
+        }
+      })
+      .on('end', () => {
+        resolve(toQueue)
+      })
+  })
+
+export const waitForLock = async (cwd: string): Promise<void> => {
+  const lock = join(cwd, '.dvc', 'rwlock')
+
+  if (exists(lock)) {
+    await delay(2000)
+    return waitForLock(cwd)
+  }
+}

extension/src/experiments/workspace.ts

@@ -1,5 +1,6 @@
 import { EventEmitter, Memento } from 'vscode'
 import { Experiments } from '.'
+import { readToQueueFromCsv, waitForLock } from './queue'
 import { pickExperimentName } from './quickPick'
 import { TableData } from './webview/contract'
 import {
@@ -12,6 +13,7 @@ import { reportOutput } from '../vscode/reporting'
 import { getInput } from '../vscode/inputBox'
 import { BaseWorkspaceWebviews } from '../webview/workspace'
 import { WorkspacePlots } from '../plots/workspace'
+import { pickCsv } from '../vscode/resourcePicker'
 
 export class WorkspaceExperiments extends BaseWorkspaceWebviews<
   Experiments,
@@ -100,6 +102,31 @@ export class WorkspaceExperiments extends BaseWorkspaceWebviews<
     return this.getRepository(dvcRoot).autoApplyFilters(enable)
   }
 
+  public async queueExperimentsFromCsv() {
+    const cwd = await this.getFocusedOrOnlyOrPickProject()
+    if (!cwd) {
+      return
+    }
+
+    const csv = await pickCsv('Select a CSV to queue experiments from')
+    if (!csv) {
+      return
+    }
+
+    const toQueue = await readToQueueFromCsv(csv)
+
+    for (const params of toQueue) {
+      await waitForLock(cwd)
+      await reportOutput(
+        this.internalCommands.executeCommand(
+          AvailableCommands.EXPERIMENT_QUEUE,
+          cwd,
+          ...params
+        )
+      )
+    }
+  }
+
   public async getCwdThenRun(commandId: CommandId) {
     const cwd = await this.getFocusedOrOnlyOrPickProject()
     if (!cwd) {

extension/src/fileSystem/index.ts

@@ -1,6 +1,7 @@
 import { basename, extname, join, relative, resolve } from 'path'
-import { existsSync, lstatSync, readdir } from 'fs-extra'
+import { createReadStream, existsSync, lstatSync, readdir } from 'fs-extra'
 import { Uri } from 'vscode'
+import { parse, Parser } from 'csv-parse'
 import { definedAndNonEmpty } from '../util/array'
 
 export const exists = (path: string): boolean => existsSync(path)
@@ -64,3 +65,6 @@ export const isAnyDvcYaml = (path?: string): boolean =>
 
 export const relativeWithUri = (dvcRoot: string, uri: Uri) =>
   relative(dvcRoot, uri.fsPath)
+
+export const readCsv = (path: string): Parser =>
+  createReadStream(path).pipe(parse({ columns: true, delimiter: ',' }))

extension/src/telemetry/constants.ts

@@ -91,6 +91,7 @@ export interface IEventNamePropertyMapping {
   [EventName.EXPERIMENT_SORTS_REMOVE_ALL]: undefined
   [EventName.EXPERIMENT_TOGGLE]: undefined
   [EventName.QUEUE_EXPERIMENT]: undefined
+  [EventName.QUEUE_EXPERIMENTS_FROM_CSV]: undefined
   [EventName.STOP_EXPERIMENT]: { stopped: boolean; wasRunning: boolean }
 
   [EventName.PLOTS_SHOW]: undefined

extension/src/test/suite/experiments/workspace.test.ts

@@ -1,7 +1,7 @@
 import { afterEach, beforeEach, describe, it, suite } from 'mocha'
 import { expect } from 'chai'
 import { stub, restore } from 'sinon'
-import { window, commands, QuickPickItem } from 'vscode'
+import { window, commands, QuickPickItem, Uri } from 'vscode'
 import { buildMultiRepoExperiments, buildSingleRepoExperiments } from './util'
 import { Disposable } from '../../../extension'
 import { CliReader } from '../../../cli/reader'
@@ -11,9 +11,14 @@ import * as QuickPick from '../../../vscode/quickPick'
 import { CliExecutor } from '../../../cli/executor'
 import { closeAllEditors, mockDuration } from '../util'
 import { dvcDemoPath } from '../../util'
-import { RegisteredCliCommands } from '../../../commands/external'
+import {
+  RegisteredCliCommands,
+  RegisteredCommands
+} from '../../../commands/external'
 import * as Telemetry from '../../../telemetry'
+import * as Time from '../../../util/time'
 import { CliRunner } from '../../../cli/runner'
+import { join } from '../../util/path'
 
 suite('Workspace Experiments Test Suite', () => {
   const disposable = Disposable.fn()
@@ -80,6 +85,51 @@ suite('Workspace Experiments Test Suite', () => {
     })
   }).timeout(8000)
 
+  describe('dvc.queueExperimentsFromCsv', () => {
+    it('should be able to queue multiple experiments from a csv', async () => {
+      const mockExperimentRunQueue = stub(
+        CliExecutor.prototype,
+        'experimentRunQueue'
+      ).resolves('true')
+
+      stub(
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (WorkspaceExperiments as any).prototype,
+        'getOnlyOrPickProject'
+      ).returns(dvcDemoPath)
+
+      const mockUri = Uri.file(join(dvcDemoPath, 'queue.csv'))
+
+      stub(window, 'showOpenDialog').resolves([mockUri])
+      stub(Time, 'delay').resolves(undefined)
+
+      await commands.executeCommand(
+        RegisteredCommands.QUEUE_EXPERIMENTS_FROM_CSV
+      )
+
+      expect(mockExperimentRunQueue).to.be.calledThrice
+      expect(mockExperimentRunQueue).to.be.calledWith(
+        dvcDemoPath,
+        '-S',
+        'lr=0.0001',
+        '-S',
+        'weight_decay=0.02'
+      )
+      expect(mockExperimentRunQueue).to.be.calledWith(
+        dvcDemoPath,
+        '-S',
+        'lr=0.00075',
+        '-S',
+        'weight_decay=0.01'
+      )
+      expect(mockExperimentRunQueue).to.be.calledWith(
+        dvcDemoPath,
+        '-S',
+        'lr=0.0005'
+      )
+    })
+  })
+
   describe('dvc.queueExperiment', () => {
     it('should be able to queue an experiment', async () => {
       const mockExperimentRunQueue = stub(

extension/src/vscode/resourcePicker.ts

@@ -13,6 +13,21 @@ export const pickFile = async (title: string): Promise<string | undefined> => {
   }
 }
 
+export const pickCsv = async (title: string): Promise<string | undefined> => {
+  const uris = await window.showOpenDialog({
+    canSelectFiles: true,
+    canSelectFolders: false,
+    canSelectMany: false,
+    filters: { CSV: ['csv'] },
+    title
+  })
+
+  if (uris) {
+    const [{ fsPath }] = uris
+    return fsPath
+  }
+}
+
 export const pickResources = (title: string): Thenable<Uri[] | undefined> => {
   return window.showOpenDialog({
     canSelectFiles: true,

yarn.lock

@@ -6087,6 +6087,11 @@ csstype@^3.0.2:
   resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.0.9.tgz#6410af31b26bd0520933d02cbc64fce9ce3fbf0b"
   integrity sha512-rpw6JPxK6Rfg1zLOYCSwle2GFOOsnjmDYDaBwEcwoOg4qlsIVCN789VkBZDJAGi4T07gI4YSutR43t9Zz4Lzuw==
 
+csv-parse@^5.0.3:
+  version "5.0.3"
+  resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-5.0.3.tgz#eeb0b1ac8bf2970f703176c8f54d313325b6d2e8"
+  integrity sha512-86R0WU4aEEF/1fPZKxP3NmDAYC4Ce1t9iFgKPZogG5Lvk4m9WZQkCEsDANktG29jppejwclTtEOzubN2ieCJqw==
+
 cyclist@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9"