From f2b18cbd4573c93e8dcbf1762e06fd3981b2b453 Mon Sep 17 00:00:00 2001 From: Matt Seddon Date: Tue, 1 Feb 2022 15:00:03 +1100 Subject: [PATCH] remove queue from csv --- demo/queue.csv | 4 -- extension/package.json | 14 ----- extension/package.nls.json | 1 - extension/src/commands/external.ts | 1 - .../src/experiments/commands/register.ts | 8 --- .../src/experiments/model/queue/index.ts | 32 ----------- extension/src/experiments/workspace.ts | 28 ---------- extension/src/fileSystem/index.ts | 12 +---- extension/src/telemetry/constants.ts | 1 - .../test/suite/experiments/workspace.test.ts | 54 +------------------ extension/src/vscode/resourcePicker.ts | 15 ------ yarn.lock | 5 -- 12 files changed, 2 insertions(+), 173 deletions(-) delete mode 100644 demo/queue.csv delete mode 100644 extension/src/experiments/model/queue/index.ts diff --git a/demo/queue.csv b/demo/queue.csv deleted file mode 100644 index 82392b3ddb..0000000000 --- a/demo/queue.csv +++ /dev/null @@ -1,4 +0,0 @@ -lr,weight_decay -0.0001,0.02 -0.00075,0.01 -0.0005, diff --git a/extension/package.json b/extension/package.json index 71854e0ab9..0b3d2db312 100644 --- a/extension/package.json +++ b/extension/package.json @@ -188,15 +188,6 @@ "category": "DVC", "icon": "$(cloud-upload)" }, - { - "title": "%command.queueExperimentsFromCsv%", - "command": "dvc.queueExperimentsFromCsv", - "category": "DVC", - "icon": { - "dark": "resources/dark/queue-experiment.svg", - "light": "resources/light/queue-experiment.svg" - } - }, { "title": "%command.queueExperimentFromExisting%", "command": "dvc.queueExperimentFromExisting", @@ -486,10 +477,6 @@ "command": "dvc.pushTarget", "when": "false" }, - { - "command": "dvc.queueExperimentsFromCsv", - "when": "dvc.commands.available && dvc.project.available" - }, { "command": "dvc.queueExperimentFromExisting", "when": "dvc.commands.available && dvc.project.available" @@ -1074,7 +1061,6 @@ "dependencies": { "@hediet/std": "^0.6.0", "chokidar": "^3.5.2", - "csv-parse": "^5.0.3", "execa": "^5.1.1", "fs-extra": "^10.0.0", "lodash.clonedeep": "^4.5.0", diff --git a/extension/package.nls.json b/extension/package.nls.json index 29be822e4b..ebb1e5316a 100644 --- a/extension/package.nls.json +++ b/extension/package.nls.json @@ -27,7 +27,6 @@ "command.pullTarget": "Pull", "command.push": "Push", "command.pushTarget": "Push", - "command.queueExperimentsFromCsv": "Queue Experiments From CSV", "command.queueExperimentFromExisting": "Queue Experiment From Existing", "command.queueExperiment": "Queue Experiment", "command.removeExperiment": "Remove Experiment", diff --git a/extension/src/commands/external.ts b/extension/src/commands/external.ts index aa642252dd..43f52c5b82 100644 --- a/extension/src/commands/external.ts +++ b/extension/src/commands/external.ts @@ -37,7 +37,6 @@ export enum RegisteredCommands { EXPERIMENT_SORTS_REMOVE = 'dvc.removeExperimentsTableSorts', EXPERIMENT_SORTS_REMOVE_ALL = 'dvc.views.experimentsSortByTree.removeAllSorts', EXPERIMENT_TOGGLE = 'dvc.views.experimentsTree.toggleStatus', - QUEUE_EXPERIMENTS_FROM_CSV = 'dvc.queueExperimentsFromCsv', QUEUE_EXPERIMENT_FROM_EXISTING = 'dvc.queueExperimentFromExisting', STOP_EXPERIMENT = 'dvc.stopRunningExperiment', diff --git a/extension/src/experiments/commands/register.ts b/extension/src/experiments/commands/register.ts index a7831c0d9b..6d890749f0 100644 --- a/extension/src/experiments/commands/register.ts +++ b/extension/src/experiments/commands/register.ts @@ -18,14 +18,6 @@ const registerExperimentCwdCommands = ( ) ) - internalCommands.registerExternalCommand( - RegisteredCommands.QUEUE_EXPERIMENTS_FROM_CSV, - () => - experiments.pauseUpdatesThenRun(() => - experiments.queueExperimentsFromCsv() - ) - ) - internalCommands.registerExternalCommand( RegisteredCommands.QUEUE_EXPERIMENT_FROM_EXISTING, () => diff --git a/extension/src/experiments/model/queue/index.ts b/extension/src/experiments/model/queue/index.ts deleted file mode 100644 index fc0d91bef7..0000000000 --- a/extension/src/experiments/model/queue/index.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Flag } from '../../../cli/args' -import { readCsv } from '../../../fileSystem' -import { definedAndNonEmpty } from '../../../util/array' - -const collectParamsToVary = (csvRow: Record): string[] => - Object.entries(csvRow).reduce((acc, [k, v]) => { - const key = k.trim() - const value = (v as string).trim() - - if (key !== '' && value !== '') { - acc.push(Flag.SET_PARAM) - const str = [key, value].join('=') - acc.push(str) - } - - return acc - }, [] as string[]) - -export const readToQueueFromCsv = (path: string): Promise => - new Promise(resolve => { - const toQueue: string[][] = [] - readCsv(path) - .on('data', row => { - const paramsToVary = collectParamsToVary(row) - if (definedAndNonEmpty(paramsToVary)) { - toQueue.push(paramsToVary) - } - }) - .on('end', () => { - resolve(toQueue) - }) - }) diff --git a/extension/src/experiments/workspace.ts b/extension/src/experiments/workspace.ts index e3a89298d1..91d2acbf1f 100644 --- a/extension/src/experiments/workspace.ts +++ b/extension/src/experiments/workspace.ts @@ -1,6 +1,5 @@ import { EventEmitter, Memento } from 'vscode' import { Experiments } from '.' -import { readToQueueFromCsv } from './model/queue' import { TableData } from './webview/contract' import { CommandId, @@ -12,7 +11,6 @@ import { reportOutput } from '../vscode/reporting' import { getInput } from '../vscode/inputBox' import { BaseWorkspaceWebviews } from '../webview/workspace' import { WorkspacePlots } from '../plots/workspace' -import { pickCsv } from '../vscode/resourcePicker' export class WorkspaceExperiments extends BaseWorkspaceWebviews< Experiments, @@ -103,32 +101,6 @@ export class WorkspaceExperiments extends BaseWorkspaceWebviews< return this.getRepository(dvcRoot).autoApplyFilters(enable) } - public async queueExperimentsFromCsv() { - const cwd = await this.getFocusedOrOnlyOrPickProject() - if (!cwd) { - return - } - const repository = this.getRepository(cwd) - - const csv = await pickCsv('Select a CSV to queue experiments from') - if (!csv) { - return - } - - const toQueue = await readToQueueFromCsv(csv) - - for (const params of toQueue) { - await repository.forceUpdate() - await reportOutput( - this.internalCommands.executeCommand( - AvailableCommands.EXPERIMENT_QUEUE, - cwd, - ...params - ) - ) - } - } - public async queueExperimentFromExisting() { const cwd = await this.getFocusedOrOnlyOrPickProject() if (!cwd) { diff --git a/extension/src/fileSystem/index.ts b/extension/src/fileSystem/index.ts index 23dcaf16df..4fff187cd6 100644 --- a/extension/src/fileSystem/index.ts +++ b/extension/src/fileSystem/index.ts @@ -1,13 +1,6 @@ import { basename, extname, join, relative, resolve } from 'path' -import { - createReadStream, - existsSync, - lstatSync, - readdir, - removeSync -} from 'fs-extra' +import { existsSync, lstatSync, readdir, removeSync } from 'fs-extra' import { Uri } from 'vscode' -import { parse, Parser } from 'csv-parse' import { definedAndNonEmpty } from '../util/array' export const exists = (path: string): boolean => existsSync(path) @@ -72,7 +65,4 @@ export const isAnyDvcYaml = (path?: string): boolean => export const relativeWithUri = (dvcRoot: string, uri: Uri) => relative(dvcRoot, uri.fsPath) -export const readCsv = (path: string): Parser => - createReadStream(path).pipe(parse({ columns: true, delimiter: ',' })) - export const removeDir = (path: string): void => removeSync(path) diff --git a/extension/src/telemetry/constants.ts b/extension/src/telemetry/constants.ts index f8281dd172..46b9c96eec 100644 --- a/extension/src/telemetry/constants.ts +++ b/extension/src/telemetry/constants.ts @@ -91,7 +91,6 @@ export interface IEventNamePropertyMapping { [EventName.EXPERIMENT_SORTS_REMOVE_ALL]: undefined [EventName.EXPERIMENT_TOGGLE]: undefined [EventName.QUEUE_EXPERIMENT]: undefined - [EventName.QUEUE_EXPERIMENTS_FROM_CSV]: undefined [EventName.QUEUE_EXPERIMENT_FROM_EXISTING]: undefined [EventName.STOP_EXPERIMENT]: { stopped: boolean; wasRunning: boolean } diff --git a/extension/src/test/suite/experiments/workspace.test.ts b/extension/src/test/suite/experiments/workspace.test.ts index f9ea94309d..c90b9f3f56 100644 --- a/extension/src/test/suite/experiments/workspace.test.ts +++ b/extension/src/test/suite/experiments/workspace.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, it, suite } from 'mocha' import { expect } from 'chai' import { stub, restore, SinonStub } from 'sinon' -import { window, commands, QuickPickItem, Uri, QuickPickOptions } from 'vscode' +import { window, commands, QuickPickItem, QuickPickOptions } from 'vscode' import { buildExperiments, buildMultiRepoExperiments, @@ -20,7 +20,6 @@ import { } from '../../../commands/external' import * as Telemetry from '../../../telemetry' import { CliRunner } from '../../../cli/runner' -import { join } from '../../util/path' import { Param } from '../../../experiments/model/queue/collect' import { QuickPickItemWithValue } from '../../../vscode/quickPick' @@ -89,57 +88,6 @@ suite('Workspace Experiments Test Suite', () => { }) }).timeout(8000) - describe('dvc.queueExperimentsFromCsv', () => { - it('should be able to queue multiple experiments from a csv', async () => { - const mockExperimentRunQueue = stub( - CliExecutor.prototype, - 'experimentRunQueue' - ).resolves('true') - - stub( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (WorkspaceExperiments as any).prototype, - 'getOnlyOrPickProject' - ).returns(dvcDemoPath) - - const mockForceUpdate = stub() - - stub(WorkspaceExperiments.prototype, 'getRepository').returns({ - forceUpdate: mockForceUpdate - } as unknown as Experiments) - - const mockUri = Uri.file(join(dvcDemoPath, 'queue.csv')) - - stub(window, 'showOpenDialog').resolves([mockUri]) - - await commands.executeCommand( - RegisteredCommands.QUEUE_EXPERIMENTS_FROM_CSV - ) - - expect(mockForceUpdate).to.be.calledThrice - expect(mockExperimentRunQueue).to.be.calledThrice - expect(mockExperimentRunQueue).to.be.calledWith( - dvcDemoPath, - '-S', - 'lr=0.0001', - '-S', - 'weight_decay=0.02' - ) - expect(mockExperimentRunQueue).to.be.calledWith( - dvcDemoPath, - '-S', - 'lr=0.00075', - '-S', - 'weight_decay=0.01' - ) - expect(mockExperimentRunQueue).to.be.calledWith( - dvcDemoPath, - '-S', - 'lr=0.0005' - ) - }) - }) - describe('dvc.queueExperimentsFromExisting', () => { it('should be able to queue an experiment using an existing one as a base', async () => { const { experiments } = buildExperiments(disposable) diff --git a/extension/src/vscode/resourcePicker.ts b/extension/src/vscode/resourcePicker.ts index c29146b01f..cef03e8c10 100644 --- a/extension/src/vscode/resourcePicker.ts +++ b/extension/src/vscode/resourcePicker.ts @@ -13,21 +13,6 @@ export const pickFile = async (title: string): Promise => { } } -export const pickCsv = async (title: string): Promise => { - const uris = await window.showOpenDialog({ - canSelectFiles: true, - canSelectFolders: false, - canSelectMany: false, - filters: { CSV: ['csv'] }, - title - }) - - if (uris) { - const [{ fsPath }] = uris - return fsPath - } -} - export const pickResources = (title: string): Thenable => { return window.showOpenDialog({ canSelectFiles: true, diff --git a/yarn.lock b/yarn.lock index 66611a2547..521f40aa6c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6383,11 +6383,6 @@ csstype@^3.0.2: resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.0.9.tgz#6410af31b26bd0520933d02cbc64fce9ce3fbf0b" integrity sha512-rpw6JPxK6Rfg1zLOYCSwle2GFOOsnjmDYDaBwEcwoOg4qlsIVCN789VkBZDJAGi4T07gI4YSutR43t9Zz4Lzuw== -csv-parse@^5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-5.0.3.tgz#eeb0b1ac8bf2970f703176c8f54d313325b6d2e8" - integrity sha512-86R0WU4aEEF/1fPZKxP3NmDAYC4Ce1t9iFgKPZogG5Lvk4m9WZQkCEsDANktG29jppejwclTtEOzubN2ieCJqw== - cyclist@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9"