diff --git a/bin/deepforge b/bin/deepforge index 7d04d44c7..6d6bbdacd 100755 --- a/bin/deepforge +++ b/bin/deepforge @@ -1,6 +1,9 @@ #!/usr/bin/env node const childProcess = require('child_process'); const Conda = require('../utils/conda-utils'); +const {promisify} = require('util'); +const fs = require('fs'); +const writeFile = promisify(fs.writeFile); const os = require('os'), IS_WINDOWS = os.type() === 'WINDOWS_NT', SHELL = IS_WINDOWS ? true : '/bin/bash', @@ -14,7 +17,6 @@ var Command = require('commander').Command, Q = require('q'), execSync = childProcess.execSync, path = require('path'), - fs = require('fs'), pkgJson = require('../package.json'), version = pkgJson.version, exists = require('exists-file'), @@ -413,7 +415,8 @@ program const extensionData = extender.getExtensionsConfig(); const libraries = Object.values(extensionData.Library); const dirs = libraries.map(lib => lib.project.root); - const name = typeof cmd.name === 'string' ? cmd.name : 'deepforge'; + const isUpdatingDefault = typeof cmd.name !== 'string'; + const name = isUpdatingDefault ? 'deepforge' : cmd.name; try { if (cmd.server) { @@ -422,6 +425,9 @@ program } if (cmd.worker) { await createEnvFromDirs(name, dirs, 'worker', cmd.force); + if (isUpdatingDefault) { + await exportWorkerEnvForJobs(name); + } } } catch (errOrExitCode) { const msg = '\n\nUnable to create environment.'; @@ -454,6 +460,13 @@ function getCondaEnvFiles(dirs, type) { return envFiles; } +async function exportWorkerEnvForJobs(name) { + const contents = await Conda.export(name); + const filename = path.join(__dirname, '..', 'src', 'plugins', 'GenerateJob', + 'templates', 'environment.worker.yml'); + await writeFile(filename, contents); +} + // user-management program.command('users', 'Manage deepforge users.'); diff --git a/src/plugins/GenerateJob/GenerateJob.js b/src/plugins/GenerateJob/GenerateJob.js index 76f74abd8..81a92ecad 100644 --- a/src/plugins/GenerateJob/GenerateJob.js +++ b/src/plugins/GenerateJob/GenerateJob.js @@ -363,6 +363,7 @@ define([ if (condaEnv) { files.addFile('environment.yml', condaEnv); } + files.addFile('environment.worker.yml', Templates.WORKER_ENV); const filename = GenerateJob.toSnakeCase(content.name); files.addFile(`operations/${filename}.py`, content.code); diff --git a/src/plugins/GenerateJob/templates/environment.worker.yml b/src/plugins/GenerateJob/templates/environment.worker.yml new file mode 100644 index 000000000..e69de29bb diff --git a/src/plugins/GenerateJob/templates/index.js b/src/plugins/GenerateJob/templates/index.js index 5d3d12fd8..7e3aea1a3 100644 --- a/src/plugins/GenerateJob/templates/index.js +++ b/src/plugins/GenerateJob/templates/index.js @@ -9,6 +9,7 @@ define([ 'text!./serialize.ejs', 'text!./deserialize.ejs', 'text!./utils.build.js', + 'text!./environment.worker.yml', ], function( START, RUN_DEBUG, @@ -19,6 +20,7 @@ define([ SERIALIZE, DESERIALIZE, UTILS, + WORKER_ENV, ) { return { @@ -31,5 +33,6 @@ define([ DEEPFORGE_INIT, DESERIALIZE, UTILS, + WORKER_ENV, }; }); diff --git a/src/plugins/GenerateJob/templates/start.js b/src/plugins/GenerateJob/templates/start.js index 63d308c11..5a4436ad7 100644 --- a/src/plugins/GenerateJob/templates/start.js +++ b/src/plugins/GenerateJob/templates/start.js @@ -71,8 +71,7 @@ requirejs([ cleanup(); }); - const envFile = path.join(__dirname, 'environment.yml'); - const envName = await updateCondaEnvironment(envFile); + const envName = await prepareCondaEnvironment(__dirname); const workerCacheDir = await prepareCache(process.env.DEEPFORGE_WORKER_CACHE); await prepareInputsOutputs(); try { @@ -106,6 +105,34 @@ requirejs([ }); } + async function prepareCondaEnvironment(jobDir) { + if (!await hasConda()) { + return null; + } + const envs = await getCondaEnvironments(); + if (!envs.includes('deepforge')) { + await createBaseEnvironment(jobDir); + } + const jobEnvFile = path.join(jobDir, 'environment.yml'); + await updateCondaEnvironment(jobEnvFile); + const envName = await updateCondaEnvironment(jobEnvFile); + return envName; + } + + async function hasConda() { + try { + await conda('-V'); + return true; + } catch (err) { + return false; + } + } + + async function createBaseEnvironment(jobDir) { + const envFile = path.join(jobDir, 'environment.worker.yml'); + await conda(`create -n ${name} -f ${envFile}`); + } + function getJobStartCommand(envName) { if (envName) { return [ @@ -187,7 +214,7 @@ requirejs([ if (exists) { let name; try { - name = await getCondaEnvironmentName(); + name = await getCondaEnvironmentName('deepforge'); await conda(`create -n ${name} --clone deepforge`); await conda(`env update -n ${name} --file ${filepath}`); return name; @@ -198,8 +225,7 @@ requirejs([ } } - async function getCondaEnvironmentName() { - const basename = 'deepforge'; + async function getCondaEnvironmentName(basename) { const envs = await getCondaEnvironments(); let newEnvName = basename; let counter = Math.floor(1000*Math.random());