Add Commands for exp queue, exp run --run-all, and exp gc (#251)

* Add a "Queue Experiments" command

No tests yet, but there is info message functionality which seems to work well.

* Try to make a test for the new queue command

* Re-add failing test

* Move individual test file into index test file

* Use commands enum for queue experiment

* Rename `queue_experiment` enum to follow new allcaps convention

* Experiments Commands #2: Add "Run All Queued Experiments" Command (#268)

* Add "Run Queued Experiments" command

* Add test for "runQueuedExperiments"

* Experiment Commands #3: Add command for `exp gc` with selectable flags (#269)

* Add command for GC

* Change gc to exp gc, and add leading `--` to enum flags

* Add GC command tests and export GC QuickPickItem interface

* Rename exp gc enum entry and reader command

* Change gcExperiments to experimentGarbageCollect

* Replace test() with it() for consistency

* Experiment Commands #4: Addressing comments from 1-3 (#271)

* Change execCommand to resolve to stdout and simplify its consumers

* runDvcCommand => runCommand

* Reorganize command enum to distinguish exp commands with a prefix

* DvcGc => Gc

Co-authored-by: mattseddon <37993418+mattseddon@users.noreply.github.com>

extension/package.json

@@ -55,6 +55,21 @@
         "command": "dvc.runExperiment",
         "category": "DVC"
       },
+      {
+        "title": "%command.queueExperiment%",
+        "command": "dvc.queueExperiment",
+        "category": "DVC"
+      },
+      {
+        "title": "%command.runQueuedExperiments%",
+        "command": "dvc.runQueuedExperiments",
+        "category": "DVC"
+      },
+      {
+        "title": "%command.experimentGarbageCollect%",
+        "command": "dvc.experimentGarbageCollect",
+        "category": "DVC"
+      },
       {
         "title": "%command.selectDvcPath%",
         "command": "dvc.selectDvcPath",

extension/package.nls.json

@@ -12,6 +12,9 @@
   "command.push": "Push",
   "command.pushTarget": "Push Target",
   "command.runExperiment": "Run Experiment",
+  "command.queueExperiment": "Queue Experiment",
+  "command.runQueuedExperiments": "Run Queued Experiments",
+  "command.experimentGarbageCollect": "Garbage Collect Experiments",
   "command.selectDvcPath": "Select DVC CLI Path",
   "command.showExperiments": "Show Experiments",
   "config.dvcPath.description": "Call DVC from this path. Follows Python Extension when blank.",

extension/src/IntegratedTerminal.test.ts

@@ -1,4 +1,8 @@
-import { IntegratedTerminal, runExperiment } from './IntegratedTerminal'
+import {
+  IntegratedTerminal,
+  runExperiment,
+  runQueuedExperiments
+} from './IntegratedTerminal'
 
 describe('runExperiment', () => {
   it('should run the correct command in the IntegratedTerminal', async () => {
@@ -9,6 +13,19 @@ describe('runExperiment', () => {
     const undef = await runExperiment()
     expect(undef).toBeUndefined()
 
-    expect(terminalSpy).toBeCalledWith('dvc exp run ')
+    expect(terminalSpy).toBeCalledWith('dvc exp run')
+  })
+})
+
+describe('runQueuedExperiments', () => {
+  it('should run the correct command in the IntegratedTerminal', async () => {
+    const terminalSpy = jest
+      .spyOn(IntegratedTerminal, 'run')
+      .mockResolvedValueOnce(undefined)
+
+    const returnValue = await runQueuedExperiments()
+    expect(returnValue).toBeUndefined()
+
+    expect(terminalSpy).toBeCalledWith('dvc exp run --run-all')
   })
 })

extension/src/IntegratedTerminal.ts

@@ -1,5 +1,5 @@
 import { Terminal, window, workspace } from 'vscode'
-import { getRunExperimentCommand } from './cli/commands'
+import { Commands } from './cli/commands'
 import { getReadyPythonExtension } from './extensions/python'
 import { delay } from './util'
 
@@ -21,6 +21,10 @@ export class IntegratedTerminal {
     return currentTerminal?.sendText(command, true)
   }
 
+  static runCommand = async (command: string): Promise<void> => {
+    return IntegratedTerminal.run(`dvc ${command}`)
+  }
+
   static dispose = (): void => {
     const currentTerminal = IntegratedTerminal.instance
     if (currentTerminal) {
@@ -65,6 +69,9 @@ export class IntegratedTerminal {
 }
 
 export const runExperiment = (): Promise<void> => {
-  const runExperimentCommand = getRunExperimentCommand()
-  return IntegratedTerminal.run(runExperimentCommand)
+  return IntegratedTerminal.runCommand(Commands.EXPERIMENT_RUN)
+}
+
+export const runQueuedExperiments = (): Promise<void> => {
+  return IntegratedTerminal.runCommand(Commands.EXPERIMENT_RUN_ALL)
 }

extension/src/__mocks__/vscode.ts

@@ -6,7 +6,11 @@ export const Extension = jest.fn()
 export const extensions = jest.fn()
 export const ThemeColor = jest.fn()
 export const Terminal = jest.fn()
-export const window = jest.fn()
+export const window = {
+  showInformationMessage: jest.fn(),
+  showErrorMessage: jest.fn(),
+  showQuickPick: jest.fn()
+}
 export const workspace = {
   workspaceFolders: [
     {

extension/src/cli/commands.ts

@@ -2,20 +2,22 @@ export enum Commands {
   ADD = 'add',
   CHECKOUT = 'checkout',
   CHECKOUT_RECURSIVE = 'checkout --recursive',
-  EXPERIMENT_RUN = 'exp run',
-  EXPERIMENT_SHOW = 'exp show --show-json',
   INITIALIZE_SUBDIRECTORY = 'init --subdir',
   PULL = 'pull',
   PUSH = 'push',
-  STATUS = 'status --show-json'
-}
-
-const getCliCommand = (command: string, ...options: string[]): string => {
-  return `dvc ${command} ${options.join(' ')}`
+  STATUS = 'status --show-json',
+  EXPERIMENT_RUN = 'exp run',
+  EXPERIMENT_SHOW = 'exp show --show-json',
+  EXPERIMENT_QUEUE = 'exp run --queue',
+  EXPERIMENT_RUN_ALL = 'exp run --run-all',
+  EXPERIMENT_GC = 'exp gc -f -w'
 }
 
-export const getRunExperimentCommand = (): string => {
-  return getCliCommand(Commands.EXPERIMENT_RUN)
+export enum GcPreserveFlag {
+  ALL_BRANCHES = '--all-branches',
+  ALL_TAGS = '--all-tags',
+  ALL_COMMITS = '--all-commits',
+  QUEUED = '--queued'
 }
 
 export const getCommandWithTarget = (

extension/src/cli/index.test.ts

@@ -1,13 +1,29 @@
+import { Config } from '../Config'
+import {
+  GcQuickPickItem,
+  experimentGcCommand,
+  queueExperimentCommand
+} from './index'
 import { mocked } from 'ts-jest/utils'
 import { execPromise } from '../util'
 import { basename, resolve } from 'path'
 import { addTarget } from '.'
+import { QuickPickOptions, window } from 'vscode'
+import { GcPreserveFlag } from './commands'
 
 jest.mock('fs')
 jest.mock('../util')
 jest.mock('vscode')
 
 const mockedExecPromise = mocked(execPromise)
+const mockedShowErrorMessage = mocked(window.showErrorMessage)
+const mockedShowInformationMessage = mocked(window.showInformationMessage)
+const mockedShowQuickPick = mocked<
+  (
+    items: GcQuickPickItem[],
+    options: QuickPickOptions
+  ) => Thenable<GcQuickPickItem[] | undefined>
+>(window.showQuickPick)
 
 beforeEach(() => {
   jest.resetAllMocks()
@@ -42,3 +58,133 @@ describe('add', () => {
     })
   })
 })
+
+describe('queueExperimentCommand', () => {
+  const exampleConfig = ({
+    dvcPath: 'dvc',
+    cwd: resolve()
+  } as unknown) as Config
+
+  it('displays an info message with the contents of stdout when the command succeeds', async () => {
+    const stdout = 'Example stdout that will be resolved literally\n'
+    mockedExecPromise.mockResolvedValue({ stdout, stderr: '' })
+    await queueExperimentCommand(exampleConfig)
+    expect(mockedShowInformationMessage).toBeCalledWith(stdout)
+  })
+
+  it('displays an error message with the contents of stderr when the command fails', async () => {
+    const stderr = 'Example stderr that will be resolved literally\n'
+    mockedExecPromise.mockRejectedValue({ stderr, stdout: '' })
+    await queueExperimentCommand(exampleConfig)
+    expect(mockedShowErrorMessage).toBeCalledWith(stderr)
+  })
+})
+
+describe('experimentGcCommand', () => {
+  const exampleConfig = ({
+    dvcPath: 'dvc',
+    cwd: resolve()
+  } as unknown) as Config
+
+  it('invokes a QuickPick with snapshotted options', async () => {
+    await experimentGcCommand(exampleConfig)
+    expect(mockedShowQuickPick.mock.calls).toMatchInlineSnapshot(`
+      Array [
+        Array [
+          Array [
+            Object {
+              "detail": "Preserve Experiments derived from all Git branches",
+              "flag": "--all-branches",
+              "label": "All Branches",
+            },
+            Object {
+              "detail": "Preserve Experiments derived from all Git tags",
+              "flag": "--all-tags",
+              "label": "All Tags",
+            },
+            Object {
+              "detail": "Preserve Experiments derived from all Git commits",
+              "flag": "--all-commits",
+              "label": "All Commits",
+            },
+            Object {
+              "detail": "Preserve all queued Experiments",
+              "flag": "--queued",
+              "label": "Queued Experiments",
+            },
+          ],
+          Object {
+            "canPickMany": true,
+            "placeHolder": "Select which Experiments to preserve",
+          },
+        ],
+      ]
+    `)
+  })
+
+  it('executes the proper command given a mocked selection', async () => {
+    mockedShowQuickPick.mockResolvedValue([
+      {
+        detail: 'Preserve Experiments derived from all Git tags',
+        flag: GcPreserveFlag.ALL_TAGS,
+        label: 'All Tags'
+      },
+      {
+        detail: 'Preserve Experiments derived from all Git commits',
+        flag: GcPreserveFlag.ALL_COMMITS,
+        label: 'All Commits'
+      }
+    ])
+
+    await experimentGcCommand(exampleConfig)
+
+    expect(mockedExecPromise).toBeCalledWith(
+      'dvc exp gc -f -w --all-tags --all-commits',
+      {
+        cwd: exampleConfig.workspaceRoot
+      }
+    )
+  })
+
+  it('reports stdout from the executed command via showInformationMessage', async () => {
+    const stdout = 'example stdout that will be passed on'
+    mockedShowQuickPick.mockResolvedValue([])
+    mockedExecPromise.mockResolvedValue({ stdout, stderr: '' })
+    await experimentGcCommand(exampleConfig)
+    expect(mockedShowInformationMessage).toBeCalledWith(stdout)
+  })
+
+  it('reports stderr from the executed command via showInformationMessage', async () => {
+    const stderr = 'example stderr that will be passed on'
+    mockedShowQuickPick.mockResolvedValue([])
+    mockedExecPromise.mockRejectedValue({ stderr, stdout: '' })
+    await experimentGcCommand(exampleConfig)
+    expect(mockedShowErrorMessage).toBeCalledWith(stderr)
+  })
+
+  it('reports the message from a non-shell Exception', async () => {
+    const message = 'example message that will be passed on'
+    mockedShowQuickPick.mockResolvedValue([])
+    mockedExecPromise.mockImplementation(() => {
+      throw new Error(message)
+    })
+    await experimentGcCommand(exampleConfig)
+    expect(mockedShowErrorMessage).toBeCalledWith(message)
+  })
+
+  it('executes the proper default command given no selections', async () => {
+    mockedShowQuickPick.mockResolvedValue([])
+
+    await experimentGcCommand(exampleConfig)
+
+    expect(mockedExecPromise).toBeCalledWith('dvc exp gc -f -w', {
+      cwd: exampleConfig.workspaceRoot
+    })
+  })
+
+  it('does not execute a command if the QuickPick is dismissed', async () => {
+    mockedShowQuickPick.mockResolvedValue(undefined)
+    await experimentGcCommand(exampleConfig)
+    expect(mockedExecPromise).not.toBeCalled()
+  })
+})