diff --git a/.gitignore b/.gitignore index 53d9d5b0..85e60616 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,6 @@ admin.env #PYTHON STUFF *.py[co] __pycache__ + +#NODE STUFF +package-lock.json diff --git a/README.md b/README.md index 16547837..119553fe 100644 --- a/README.md +++ b/README.md @@ -140,25 +140,40 @@ custom: ``` ## Extra Config Options -### extra pip arguments -You can specify extra arguments to be passed to pip like this: +### Caching +You can enable two kinds of caching with this plugin which are currently both DISABLED by default. First, a download cache that will cache downloads that pip needs to compile the packages. And second, a what we call "static caching" which caches output of pip after compiling everything for your requirements file. Since generally requirements.txt files rarely change, you will often see large amounts of speed improvements when enabling the static cache feature. These caches will be shared between all your projects if no custom cacheLocation is specified (see below). + + _**Please note:** This has replaced the previously recommended usage of "--cache-dir" in the pipCmdExtraArgs_ ```yaml custom: pythonRequirements: - dockerizePip: true - pipCmdExtraArgs: - - --cache-dir - - .requirements-cache + useDownloadCache: true + useStaticCache: true ``` +_Additionally, In future versions of this plugin, both caching features will probably be enabled by default_ -When using `--cache-dir` don't forget to also exclude it from the package. +### Other caching options... +There are two additional options related to caching. You can specify where in your system that this plugin caches with the `cacheLocation` option. By default it will figure out automatically where based on your username and your OS to store the cache via the [appdirectory](https://www.npmjs.com/package/appdirectory) module. Additionally, you can specify how many max static caches to store with `staticCacheMaxVersions`, as a simple attempt to limit disk space usage for caching. This is DISABLED (set to 0) by default. Example: +```yaml +custom: + pythonRequirements: + useStaticCache: true + useDownloadCache: true + cacheLocation: '/home/user/.my_cache_goes_here' + staticCacheMaxVersions: 10 + +``` +### Extra pip arguments +You can specify extra arguments [supported by pip](https://pip.pypa.io/en/stable/reference/pip_install/#options) to be passed to pip like this: ```yaml -package: - exclude: - - .requirements-cache/** +custom: + pythonRequirements: + pipCmdExtraArgs: + - --compile ``` + ### Customize requirements file name [Some `pip` workflows involve using requirements files not named `requirements.txt`](https://www.kennethreitz.org/essays/a-better-pip-workflow). @@ -350,4 +365,4 @@ zipinfo .serverless/xxx.zip improved pip chache support when using docker. * [@dee-me-tree-or-love](https://github.com/dee-me-tree-or-love) - the `slim` package option * [@alexjurkiewicz](https://github.com/alexjurkiewicz) - [docs about docker workflows](#native-code-dependencies-during-build) - + * [@andrewfarley](https://github.com/andrewfarley) - Implemented download caching and static caching diff --git a/index.js b/index.js index f64c0680..ff73b1f7 100644 --- a/index.js +++ b/index.js @@ -12,7 +12,7 @@ const { const { injectAllRequirements } = require('./lib/inject'); const { installAllRequirements } = require('./lib/pip'); const { pipfileToRequirements } = require('./lib/pipenv'); -const { cleanup } = require('./lib/clean'); +const { cleanup, cleanupCache } = require('./lib/clean'); BbPromise.promisifyAll(fse); @@ -39,6 +39,10 @@ class ServerlessPythonRequirements { dockerSsh: false, dockerImage: null, dockerFile: null, + useStaticCache: false, + useDownloadCache: false, + cacheLocation: false, + staticCacheMaxVersions: 0, pipCmdExtraArgs: [], noDeploy: [ 'boto3', @@ -115,6 +119,11 @@ class ServerlessPythonRequirements { install: { usage: 'install requirements manually', lifecycleEvents: ['install'] + }, + cleanCache: { + usage: + 'Removes all items in the pip download/static cache (if present)', + lifecycleEvents: ['cleanCache'] } } } @@ -128,6 +137,11 @@ class ServerlessPythonRequirements { return args[1].functionObj.runtime.startsWith('python'); }; + const clean = () => + BbPromise.bind(this) + .then(cleanup) + .then(removeVendorHelper); + const before = () => { if (!isFunctionRuntimePython(arguments)) { return; @@ -155,13 +169,13 @@ class ServerlessPythonRequirements { const invalidateCaches = () => { if (this.options.invalidateCaches) { - return BbPromise.bind(this) - .then(cleanup) - .then(removeVendorHelper); + return clean; } return BbPromise.resolve(); }; + const cleanCache = () => BbPromise.bind(this).then(cleanupCache); + this.hooks = { 'after:package:cleanup': invalidateCaches, 'before:package:createDeploymentArtifacts': before, @@ -172,16 +186,9 @@ class ServerlessPythonRequirements { this.serverless.cli.generateCommandsHelp(['requirements']); return BbPromise.resolve(); }, - 'requirements:install:install': () => - BbPromise.bind(this) - .then(pipfileToRequirements) - .then(addVendorHelper) - .then(installAllRequirements) - .then(packRequirements), - 'requirements:clean:clean': () => - BbPromise.bind(this) - .then(cleanup) - .then(removeVendorHelper) + 'requirements:install:install': before, + 'requirements:clean:clean': clean, + 'requirements:cleanCache:cleanCache': cleanCache }; } } diff --git a/lib/clean.js b/lib/clean.js index 332ceb37..119ab586 100644 --- a/lib/clean.js +++ b/lib/clean.js @@ -1,6 +1,8 @@ const BbPromise = require('bluebird'); const fse = require('fs-extra'); const path = require('path'); +const glob = require('glob-all'); +const { getUserCachePath } = require('./shared'); BbPromise.promisifyAll(fse); @@ -29,4 +31,32 @@ function cleanup() { ); } -module.exports = { cleanup }; +/** + * Clean up static cache, remove all items in there + * @return {Promise} + */ +function cleanupCache() { + const cacheLocation = getUserCachePath(this.options); + if (fse.existsSync(cacheLocation)) { + if (this.serverless) { + this.serverless.cli.log(`Removing static caches at: ${cacheLocation}`); + } + + // Only remove cache folders that we added, just incase someone accidentally puts a weird + // static cache location so we don't remove a bunch of personal stuff + const promises = []; + glob + .sync([path.join(cacheLocation, '*slspyc/')], { mark: true, dot: false }) + .forEach(file => { + promises.push(fse.removeAsync(file)); + }); + return BbPromise.all(promises); + } else { + if (this.serverless) { + this.serverless.cli.log(`No static cache found`); + } + return BbPromise.resolve(); + } +} + +module.exports = { cleanup, cleanupCache }; diff --git a/lib/docker.js b/lib/docker.js index 26cbf6de..db2e81b5 100644 --- a/lib/docker.js +++ b/lib/docker.js @@ -49,8 +49,11 @@ function findTestFile(servicePath) { if (fse.pathExistsSync(path.join(servicePath, 'serverless.json'))) { return 'serverless.json'; } + if (fse.pathExistsSync(path.join(servicePath, 'requirements.txt'))) { + return 'requirements.txt'; + } throw new Error( - 'Unable to find serverless.yml or serverless.yaml or serverless.json for getBindPath()' + 'Unable to find serverless.{yml|yaml|json} or requirements.txt for getBindPath()' ); } @@ -154,7 +157,7 @@ function getDockerUid(bindPath) { 'stat', '-c', '%u', - '/test/.serverless' + '/bin/sh' ]; const ps = dockerCommand(options); return ps.stdout.trim(); diff --git a/lib/pip.js b/lib/pip.js index 4e5e24e1..044f6c78 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -4,38 +4,61 @@ const path = require('path'); const get = require('lodash.get'); const set = require('lodash.set'); const { spawnSync } = require('child_process'); +const { quote } = require('shell-quote'); const { buildImage, getBindPath, getDockerUid } = require('./docker'); const { getStripCommand, deleteFiles } = require('./slim'); +const { + checkForAndDeleteMaxCacheVersions, + md5Path, + getRequirementsWorkingPath, + getUserCachePath +} = require('./shared'); /** - * Install requirements described in requirementsPath to targetFolder + * Just generate the requirements file in the .serverless folder * @param {string} requirementsPath - * @param {string} targetFolder + * @param {string} targetFile * @param {Object} serverless * @param {string} servicePath * @param {Object} options * @return {undefined} */ -function installRequirements( +function installRequirementsFile( requirementsPath, - targetFolder, + targetFile, serverless, servicePath, options ) { - // Create target folder if it does not exist - const targetRequirementsFolder = path.join(targetFolder, 'requirements'); - fse.ensureDirSync(targetRequirementsFolder); - - const dotSlsReqs = path.join(targetFolder, 'requirements.txt'); if (options.usePipenv && fse.existsSync(path.join(servicePath, 'Pipfile'))) { - generateRequirementsFile(dotSlsReqs, dotSlsReqs, options); + generateRequirementsFile( + path.join(servicePath, '.serverless/requirements.txt'), + targetFile, + options + ); + serverless.cli.log( + `Parsed requirements.txt from Pipfile in ${targetFile}...` + ); } else { - generateRequirementsFile(requirementsPath, dotSlsReqs, options); + generateRequirementsFile(requirementsPath, targetFile, options); + serverless.cli.log( + `Generated requirements from ${requirementsPath} in ${targetFile}...` + ); } +} + +/** + * Install requirements described from requirements in the targetFolder into that same targetFolder + * @param {string} targetFolder + * @param {Object} serverless + * @param {Object} options + * @return {undefined} + */ +function installRequirements(targetFolder, serverless, options) { + const targetRequirementsTxt = path.join(targetFolder, 'requirements.txt'); serverless.cli.log( - `Installing requirements of ${requirementsPath} in ${targetFolder}...` + `Installing requirements from ${targetRequirementsTxt} ...` ); let cmd; @@ -45,13 +68,41 @@ function installRequirements( '-m', 'pip', 'install', - '-t', - dockerPathForWin(options, targetRequirementsFolder), - '-r', - dockerPathForWin(options, dotSlsReqs), ...options.pipCmdExtraArgs ]; + // Check if we're using the legacy --cache-dir command... + if (options.pipCmdExtraArgs.indexOf('--cache-dir') > -1) { + if (options.dockerizePip) { + throw 'Error: You can not use --cache-dir with Docker any more, please\n' + + ' use the new option useDownloadCache instead. Please see:\n' + + ' https://github.com/UnitedIncome/serverless-python-requirements#caching'; + } else { + serverless.cli.log('=================================================='); + serverless.cli.log( + 'Warning: You are using a deprecated --cache-dir inside\n' + + ' your pipCmdExtraArgs which may not work properly, please use the\n' + + ' useDownloadCache option instead. Please see: \n' + + ' https://github.com/UnitedIncome/serverless-python-requirements#caching' + ); + serverless.cli.log('=================================================='); + } + } + if (!options.dockerizePip) { + // Push our local OS-specific paths for requirements and target directory + pipCmd.push('-t', dockerPathForWin(options, targetFolder)); + pipCmd.push('-r', dockerPathForWin(options, targetRequirementsTxt)); + // If we want a download cache... + if (options.useDownloadCache) { + const downloadCacheDir = path.join( + getUserCachePath(options), + 'downloadCacheslspyc' + ); + serverless.cli.log(`Using download cache directory ${downloadCacheDir}`); + fse.ensureDirSync(downloadCacheDir); + pipCmd.push('--cache-dir', downloadCacheDir); + } + // Check if pip has Debian's --system option and set it if so const pipTestRes = spawnSync(options.pythonBin, [ '-m', @@ -71,9 +122,14 @@ function installRequirements( pipCmd.push('--system'); } } + // If we are dockerizing pip if (options.dockerizePip) { cmd = 'docker'; + // Push docker-specific paths for requirements and target directory + pipCmd.push('-t', '/var/task/'); + pipCmd.push('-r', '/var/task/requirements.txt'); + // Build docker image if required let dockerImage; if (options.dockerFile) { @@ -87,25 +143,74 @@ function installRequirements( serverless.cli.log(`Docker Image: ${dockerImage}`); // Prepare bind path depending on os platform - const bindPath = getBindPath(serverless, servicePath); + const bindPath = getBindPath(serverless, targetFolder); cmdOptions = ['run', '--rm', '-v', `"${bindPath}:/var/task:z"`]; if (options.dockerSsh) { // Mount necessary ssh files to work with private repos cmdOptions.push( '-v', - `${process.env.HOME}/.ssh/id_rsa:/root/.ssh/id_rsa:z` + `"${process.env.HOME}/.ssh/id_rsa:/root/.ssh/id_rsa:z"` ); cmdOptions.push( '-v', - `${process.env.HOME}/.ssh/known_hosts:/root/.ssh/known_hosts:z` + `"${process.env.HOME}/.ssh/known_hosts:/root/.ssh/known_hosts:z"` ); - cmdOptions.push('-v', `${process.env.SSH_AUTH_SOCK}:/tmp/ssh_sock:z`); + cmdOptions.push('-v', `"${process.env.SSH_AUTH_SOCK}:/tmp/ssh_sock:z"`); cmdOptions.push('-e', 'SSH_AUTH_SOCK=/tmp/ssh_sock'); } + + // If we want a download cache... + const dockerDownloadCacheDir = '/var/useDownloadCache'; + if (options.useDownloadCache) { + const downloadCacheDir = path.join( + getUserCachePath(options), + 'downloadCacheslspyc' + ); + serverless.cli.log(`Using download cache directory ${downloadCacheDir}`); + fse.ensureDirSync(downloadCacheDir); + // This little hack is necessary because getBindPath requires something inside of it to test... + // Ugh, this is so ugly, but someone has to fix getBindPath in some other way (eg: make it use + // its own temp file) + fse.closeSync( + fse.openSync(path.join(downloadCacheDir, 'requirements.txt'), 'w') + ); + const windowsized = getBindPath(serverless, downloadCacheDir); + // And now push it to a volume mount and to pip... + cmdOptions.push('-v', `"${windowsized}:${dockerDownloadCacheDir}:z"`); + pipCmd.push('--cache-dir', dockerDownloadCacheDir); + } + if (process.platform === 'linux') { // Use same user so requirements folder is not root and so --cache-dir works - cmdOptions.push('-u', `${process.getuid()}`); + var commands = []; + if (options.useDownloadCache) { + // Set the ownership of the download cache dir to root + commands.push(quote(['chown', '-R', '0:0', dockerDownloadCacheDir])); + } + // Install requirements with pip + commands.push(quote(pipCmd)); + // Set the ownership of the current folder to user + commands.push( + quote([ + 'chown', + '-R', + `${process.getuid()}:${process.getgid()}`, + '/var/task' + ]) + ); + if (options.useDownloadCache) { + // Set the ownership of the download cache dir back to user + commands.push( + quote([ + 'chown', + '-R', + `${process.getuid()}:${process.getgid()}`, + dockerDownloadCacheDir + ]) + ); + } + pipCmd = ['/bin/bash', '-c', '"' + commands.join(' && ') + '"']; } else { // Use same user so --cache-dir works cmdOptions.push('-u', getDockerUid(bindPath)); @@ -119,10 +224,10 @@ function installRequirements( // If enabled slimming, strip so files if (options.slim === true || options.slim === 'true') { - const preparedPath = dockerPathForWin(options, targetRequirementsFolder); + const preparedPath = dockerPathForWin(options, targetFolder); cmdOptions.push(getStripCommand(options, preparedPath)); } - let spawnArgs = { cwd: servicePath, shell: true }; + let spawnArgs = { cwd: targetFolder, shell: true }; if (process.env.SLS_DEBUG) { spawnArgs.stdio = 'inherit'; } @@ -143,7 +248,7 @@ function installRequirements( } // If enabled slimming, delete files in slimPatterns if (options.slim === true || options.slim === 'true') { - deleteFiles(options, targetRequirementsFolder); + deleteFiles(options, targetFolder); } } @@ -161,6 +266,8 @@ function dockerPathForWin(options, path) { } /** create a filtered requirements.txt without anything from noDeploy + * then remove all comments and empty lines, and sort the list which + * assist with matching the static cache * @param {string} source requirements * @param {string} target requirements where results are written * @param {Object} options @@ -171,8 +278,13 @@ function generateRequirementsFile(source, target, options) { .readFileSync(source, { encoding: 'utf-8' }) .split(/\r?\n/); const filteredRequirements = requirements.filter(req => { + req = req.trim(); + if (req.length == 0 || req[0] == '#') { + return false; + } return !noDeploy.has(req.split(/[=<> \t]/)[0].trim()); }); + filteredRequirements.sort(); // Sort them alphabetically fse.writeFileSync(target, filteredRequirements.join('\n')); } @@ -185,15 +297,15 @@ function generateRequirementsFile(source, target, options) { */ function copyVendors(vendorFolder, targetFolder, serverless) { // Create target folder if it does not exist - const targetRequirementsFolder = path.join(targetFolder, 'requirements'); + fse.ensureDirSync(targetFolder); serverless.cli.log( - `Copying vendor libraries from ${vendorFolder} to ${targetRequirementsFolder}...` + `Copying vendor libraries from ${vendorFolder} to ${targetFolder}...` ); fse.readdirSync(vendorFolder).map(file => { let source = path.join(vendorFolder, file); - let dest = path.join(targetRequirementsFolder, file); + let dest = path.join(targetFolder, file); if (fse.existsSync(dest)) { rimraf.sync(dest); } @@ -202,11 +314,129 @@ function copyVendors(vendorFolder, targetFolder, serverless) { } /** - * pip install the requirements to the .serverless/requirements directory + * This evaluates if requirements are actually needed to be installed, but fails + * gracefully if no req file is found intentionally. It also assists with code + * re-use for this logic pertaining to individually packaged functions + * @param {string} servicePath + * @param {string} modulePath + * @param {Object} options + * @param {Object} funcOptions + * @param {Object} serverless + * @return {string} + */ +function installRequirementsIfNeeded( + servicePath, + modulePath, + options, + funcOptions, + serverless +) { + // Our source requirements, under our service path, and our module path (if specified) + const fileName = path.join(servicePath, modulePath, options.fileName); + + // First, generate the requirements file to our local .serverless folder + fse.ensureDirSync(path.join(servicePath, '.serverless')); + const slsReqsTxt = path.join(servicePath, '.serverless', 'requirements.txt'); + + installRequirementsFile( + fileName, + slsReqsTxt, + serverless, + servicePath, + options + ); + + // If no requirements file or an empty requirements file, then do nothing + if (!fse.existsSync(slsReqsTxt) || fse.statSync(slsReqsTxt).size == 0) { + serverless.cli.log( + `Skipping empty output requirements.txt file from ${slsReqsTxt}` + ); + return false; + } + + // Copy our requirements to another filename in .serverless (incase of individually packaged) + if (modulePath && modulePath != '.') { + fse.existsSync(path.join(servicePath, '.serverless', modulePath)); + const destinationFile = path.join( + servicePath, + '.serverless', + modulePath, + 'requirements.txt' + ); + serverless.cli.log( + `Copying from ${slsReqsTxt} into ${destinationFile} ...` + ); + fse.copySync(slsReqsTxt, destinationFile); + } + + // Then generate our MD5 Sum of this requirements file to determine where it should "go" to and/or pull cache from + const reqChecksum = md5Path(slsReqsTxt); + + // Then figure out where this cache should be, if we're caching, if we're in a module, etc + const workingReqsFolder = getRequirementsWorkingPath( + reqChecksum, + servicePath, + options + ); + + // Check if our static cache is present and is valid + if (fse.existsSync(workingReqsFolder)) { + if ( + fse.existsSync(path.join(workingReqsFolder, '.completed_requirements')) && + workingReqsFolder.endsWith('_slspyc') + ) { + serverless.cli.log( + `Using static cache of requirements found at ${workingReqsFolder} ...` + ); + // We'll "touch" the folder, as to bring it to the start of the FIFO cache + fse.utimesSync(workingReqsFolder, new Date(), new Date()); + return workingReqsFolder; + } + // Remove our old folder if it didn't complete properly, but _just incase_ only remove it if named properly... + if ( + workingReqsFolder.endsWith('_slspyc') || + workingReqsFolder.endsWith('.requirements') + ) { + rimraf.sync(workingReqsFolder); + } + } + + // Ensuring the working reqs folder exists + fse.ensureDirSync(workingReqsFolder); + + // Copy our requirements.txt into our working folder... + fse.copySync(slsReqsTxt, path.join(workingReqsFolder, 'requirements.txt')); + + // Then install our requirements from this folder + installRequirements(workingReqsFolder, serverless, options); + + // Copy vendor libraries to requirements folder + if (options.vendor) { + copyVendors(options.vendor, workingReqsFolder, serverless); + } + if (funcOptions.vendor) { + copyVendors(funcOptions.vendor, workingReqsFolder, serverless); + } + + // Then touch our ".completed_requirements" file so we know we can use this for static cache + if (options.useStaticCache) { + fse.closeSync( + fse.openSync(path.join(workingReqsFolder, '.completed_requirements'), 'w') + ); + } + return workingReqsFolder; +} + +/** + * pip install the requirements to the requirements directory * @return {undefined} */ function installAllRequirements() { - fse.ensureDirSync(path.join(this.servicePath, '.serverless')); + // fse.ensureDirSync(path.join(this.servicePath, '.serverless')); + // First, check and delete cache versions, if enabled + checkForAndDeleteMaxCacheVersions(this.options, this.serverless); + + // Then if we're going to package functions individually... if (this.serverless.service.package.individually) { let doneModules = []; this.targetFuncs @@ -219,36 +449,70 @@ function installAllRequirements() { if (!get(f, 'module')) { set(f, ['module'], '.'); } + // If we didn't already process a module (functions can re-use modules) if (!doneModules.includes(f.module)) { - installRequirements( - path.join(f.module, this.options.fileName), - path.join('.serverless', f.module), - this.serverless, + const reqsInstalledAt = installRequirementsIfNeeded( + this.servicePath, + f.module, + this.options, + f, + this.serverless + ); + // Add modulePath into .serverless for each module so it's easier for injecting and for users to see where reqs are + let modulePath = path.join( this.servicePath, - this.options + '.serverless', + `${f.module}`, + 'requirements' ); - if (f.vendor) { - // copy vendor libraries to requirements folder - copyVendors( - f.vendor, - path.join('.serverless', f.module), - this.serverless - ); + // Only do if we didn't already do it + if ( + reqsInstalledAt && + !fse.existsSync(modulePath) && + reqsInstalledAt != modulePath + ) { + if (this.options.useStaticCache) { + // Windows can't symlink so we have to copy on Windows, + // it's not as fast, but at least it works + if (process.platform == 'win32') { + fse.copySync(reqsInstalledAt, modulePath); + } else { + fse.symlink(reqsInstalledAt, modulePath); + } + } else { + fse.rename(reqsInstalledAt, modulePath); + } } doneModules.push(f.module); } }); } else { - installRequirements( - this.options.fileName, - '.serverless', - this.serverless, + const reqsInstalledAt = installRequirementsIfNeeded( this.servicePath, - this.options + '', + this.options, + {}, + this.serverless ); - if (this.options.vendor) { - // copy vendor libraries to requirements folder - copyVendors(this.options.vendor, '.serverless', this.serverless); + // Add symlinks into .serverless for so it's easier for injecting and for users to see where reqs are + let symlinkPath = path.join( + this.servicePath, + '.serverless', + `requirements` + ); + // Only do if we didn't already do it + if ( + reqsInstalledAt && + !fse.existsSync(symlinkPath) && + reqsInstalledAt != symlinkPath + ) { + // Windows can't symlink so we have to copy on Windows, + // it's not as fast, but at least it works + if (process.platform == 'win32') { + fse.copySync(reqsInstalledAt, symlinkPath); + } else { + fse.symlink(reqsInstalledAt, symlinkPath); + } } } } diff --git a/lib/shared.js b/lib/shared.js new file mode 100644 index 00000000..b3a1ffaa --- /dev/null +++ b/lib/shared.js @@ -0,0 +1,108 @@ +const Appdir = require('appdirectory'); +const rimraf = require('rimraf'); +const md5File = require('md5-file'); +const glob = require('glob-all'); +const path = require('path'); +const fse = require('fs-extra'); + +/** + * This helper will check if we're using static cache and have max + * versions enabled and will delete older versions in a fifo fashion + * @param {Object} options + * @param {Object} serverless + * @return {undefined} + */ +function checkForAndDeleteMaxCacheVersions(options, serverless) { + // If we're using the static cache, and we have static cache max versions enabled + if ( + options.useStaticCache && + options.staticCacheMaxVersions && + parseInt(options.staticCacheMaxVersions) > 0 + ) { + // Get the list of our cache files + const files = glob.sync( + [path.join(getUserCachePath(options), '*_slspyc/')], + { mark: true } + ); + // Check if we have too many + if (files.length >= options.staticCacheMaxVersions) { + // Sort by modified time + files.sort(function(a, b) { + return ( + fse.statSync(a).mtime.getTime() - fse.statSync(b).mtime.getTime() + ); + }); + // Remove the older files... + var items = 0; + for ( + var i = 0; + i < files.length - options.staticCacheMaxVersions + 1; + i++ + ) { + rimraf.sync(files[i]); + items++; + } + // Log the number of cache files flushed + serverless.cli.log( + `Removed ${items} items from cache because of staticCacheMaxVersions` + ); + } + } +} + +/** + * The working path that all requirements will be compiled into + * @param {string} subfolder + * @param {string} servicePath + * @param {Object} options + * @return {string} + */ +function getRequirementsWorkingPath(subfolder, servicePath, options) { + // If we want to use the static cache + if (options && options.useStaticCache) { + if (subfolder) { + subfolder = subfolder + '_slspyc'; + } + // If we have max number of cache items... + + return path.join(getUserCachePath(options), subfolder); + } + + // If we don't want to use the static cache, then fallback to the way things used to work + return path.join(servicePath, '.serverless', 'requirements'); +} + +/** + * The static cache path that will be used for this system + options, used if static cache is enabled + * @param {Object} options + * @return {string} + */ +function getUserCachePath(options) { + // If we've manually set the static cache location + if (options && options.cacheLocation) { + return path.resolve(options.cacheLocation); + } + + // Otherwise, find/use the python-ey appdirs cache location + const dirs = new Appdir({ + appName: 'serverless-python-requirements', + appAuthor: 'UnitedIncome' + }); + return dirs.userCache(); +} + +/** + * Helper to get the md5 a a file's contents to determine if a requirements has a static cache + * @param {string} fullpath + * @return {string} + */ +function md5Path(fullpath) { + return md5File.sync(fullpath); +} + +module.exports = { + checkForAndDeleteMaxCacheVersions, + getRequirementsWorkingPath, + getUserCachePath, + md5Path +}; diff --git a/package.json b/package.json index 5487c04c..67be6ac4 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "prettier": "*" }, "dependencies": { + "appdirectory": "^0.1.0", "bluebird": "^3.0.6", "fs-extra": "^7.0.0", "glob-all": "^3.1.0", @@ -54,9 +55,11 @@ "jszip": "^3.1.0", "lodash.get": "^4.4.2", "lodash.set": "^4.3.2", - "lodash.values": "^4.3.0", "lodash.uniqby": "^4.0.0", - "rimraf": "^2.6.2" + "lodash.values": "^4.3.0", + "md5-file": "^3.2.3", + "rimraf": "^2.6.2", + "shell-quote": "^1.6.1" }, "eslintConfig": { "extends": "eslint:recommended", diff --git a/test.bats b/test.bats index 4501ba52..07634168 100755 --- a/test.bats +++ b/test.bats @@ -7,12 +7,35 @@ setup() { export LC_ALL=C.UTF-8 export LANG=C.UTF-8 fi + export USR_CACHE_DIR=`node -e 'console.log(require("./lib/shared").getUserCachePath())'` + # Please note: If you update change the requirements.txt in test/base this value will + # change. Run a test which uses this variable manually step by step and list the cache + # folder to find the new hash if you do this + export CACHE_FOLDER_HASH="b8b9d2be59f6f2ea5778e8b2aa4d2ddc_slspyc" + if [ -d "${USR_CACHE_DIR}" ] ; then + rm -Rf "${USR_CACHE_DIR}" + fi } teardown() { rm -rf puck puck2 puck3 node_modules .serverless .requirements.zip .requirements-cache \ foobar package-lock.json serverless-python-requirements-*.tgz if [ -f serverless.yml.bak ]; then mv serverless.yml.bak serverless.yml; fi + if [ -f slimPatterns.yml ]; then rm -f slimPatterns.yml; fi + if [ -d "${USR_CACHE_DIR}" ] ; then + rm -Rf "${USR_CACHE_DIR}" + fi +} + +@test "py3.6 supports custom file name with fileName option" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n fileName: puck/' serverless.yml + echo "requests" > puck + sls package + ls .serverless/requirements/requests + ! ls .serverless/requirements/flask } @test "py3.6 can package flask with default options" { @@ -43,10 +66,9 @@ teardown() { @test "py3.6 can package flask with slim & slimPatterns options" { cd tests/base - mv _slimPatterns.yml slimPatterns.yml + cat _slimPatterns.yml > slimPatterns.yml npm i $(npm pack ../..) sls --slim=true package - mv slimPatterns.yml _slimPatterns.yml unzip .serverless/sls-py-req-test.zip -d puck ls puck/flask test $(find puck -name "*.pyc" | wc -l) -eq 0 @@ -110,36 +132,132 @@ teardown() { @test "py3.6 can package flask with slim & dockerizePip & slimPatterns options" { cd tests/base - mv _slimPatterns.yml slimPatterns.yml + cat _slimPatterns.yml > slimPatterns.yml npm i $(npm pack ../..) ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" sls --dockerizePip=true --slim=true package - mv slimPatterns.yml _slimPatterns.yml unzip .serverless/sls-py-req-test.zip -d puck ls puck/flask test $(find puck -name "*.pyc" | wc -l) -eq 0 test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0 } -@test "py3.6 uses cache with dockerizePip option" { +@test "py3.6 uses download cache with useDownloadCache option" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n useDownloadCache: true/' serverless.yml + sls package + USR_CACHE_DIR=`node -e 'console.log(require("../../lib/shared").getUserCachePath())'` + ls $USR_CACHE_DIR/downloadCacheslspyc/http +} + +@test "py3.6 uses download cache with cacheLocation option" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n useDownloadCache: true\n cacheLocation: .requirements-cache/' serverless.yml + sls package + ls .requirements-cache/downloadCacheslspyc/http +} + +@test "py3.6 uses download cache with dockerizePip option" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n useDownloadCache: true/' serverless.yml + sls --dockerizePip=true package + USR_CACHE_DIR=`node -e 'console.log(require("../../lib/shared").getUserCachePath())'` + ls $USR_CACHE_DIR/downloadCacheslspyc/http +} + +@test "py3.6 uses download cache with dockerizePip + cacheLocation option" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n useDownloadCache: true\n cacheLocation: .requirements-cache/' serverless.yml + sls --dockerizePip=true package + ls .requirements-cache/downloadCacheslspyc/http +} + +@test "py3.6 uses static and download cache" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n useDownloadCache: true\n useStaticCache: true/' serverless.yml + sls package + USR_CACHE_DIR=`node -e 'console.log(require("../../lib/shared").getUserCachePath())'` + ls $USR_CACHE_DIR/$CACHE_FOLDER_HASH/flask + ls $USR_CACHE_DIR/downloadCacheslspyc/http +} + +@test "py3.6 uses static and download cache with dockerizePip option" { cd tests/base npm i $(npm pack ../..) ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" - perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n pipCmdExtraArgs: ["--cache-dir", ".requirements-cache"]/' serverless.yml + perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n useDownloadCache: true\n useStaticCache: true/' serverless.yml sls --dockerizePip=true package - ls .requirements-cache/http + USR_CACHE_DIR=`node -e 'console.log(require("../../lib/shared").getUserCachePath())'` + ls $USR_CACHE_DIR/$CACHE_FOLDER_HASH/flask + ls $USR_CACHE_DIR/downloadCacheslspyc/http } -@test "py3.6 uses cache with dockerizePip & slim option" { +@test "py3.6 uses static cache" { cd tests/base npm i $(npm pack ../..) ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" - perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n pipCmdExtraArgs: ["--cache-dir", ".requirements-cache"]/' serverless.yml + perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n useStaticCache: true/' serverless.yml + sls package + USR_CACHE_DIR=`node -e 'console.log(require("../../lib/shared").getUserCachePath())'` + ls $USR_CACHE_DIR/$CACHE_FOLDER_HASH/flask + ls $USR_CACHE_DIR/$CACHE_FOLDER_HASH/.completed_requirements +} + +@test "py3.6 uses static cache with cacheLocation option" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n useStaticCache: true\n cacheLocation: .requirements-cache/' serverless.yml + sls package + USR_CACHE_DIR=`node -e 'console.log(require("../../lib/shared").getUserCachePath())'` + ls .requirements-cache/$CACHE_FOLDER_HASH/flask + ls .requirements-cache/$CACHE_FOLDER_HASH/.completed_requirements +} + +@test "py3.6 checking that static cache actually pulls from cache (by poisoning it)" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n useStaticCache: true/' serverless.yml + sls package + cp .serverless/sls-py-req-test.zip ./puck + USR_CACHE_DIR=`node -e 'console.log(require("../../lib/shared").getUserCachePath())'` + echo "injected new file into static cache folder" > $USR_CACHE_DIR/$CACHE_FOLDER_HASH/injected_file_is_bad_form + sls package + [ `wc -c ./.serverless/sls-py-req-test.zip | awk '{ print $1 }'` -gt `wc -c ./puck | awk '{ print $1 }'` ] +} + +@test "py3.6 uses static cache with dockerizePip & slim option" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n useStaticCache: true/' serverless.yml sls --dockerizePip=true --slim=true package - ls .requirements-cache/http + ls $USR_CACHE_DIR/$CACHE_FOLDER_HASH/flask + unzip .serverless/sls-py-req-test.zip -d puck test $(find puck -name "*.pyc" | wc -l) -eq 0 } +@test "py3.6 uses download cache with dockerizePip & slim option" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n useDownloadCache: true/' serverless.yml + sls --dockerizePip=true --slim=true package + ls $USR_CACHE_DIR/downloadCacheslspyc/http + unzip .serverless/sls-py-req-test.zip -d puck + test $(find puck -name "*.pyc" | wc -l) -eq 0 +} @test "py2.7 can package flask with default options" { cd tests/base @@ -168,10 +286,9 @@ teardown() { @test "py2.7 can package flask with slim & dockerizePip & slimPatterns options" { cd tests/base - mv _slimPatterns.yml slimPatterns.yml + cat _slimPatterns.yml > slimPatterns.yml npm i $(npm pack ../..) sls --runtime=python2.7 --slim=true packag - mv slimPatterns.yml _slimPatterns.yml unzip .serverless/sls-py-req-test.zip -d puck ls puck/flask test $(find puck -name "*.pyc" | wc -l) -eq 0 @@ -234,11 +351,10 @@ teardown() { @test "py2.7 can package flask with slim & dockerizePip & slimPatterns options" { cd tests/base - mv _slimPatterns.yml slimPatterns.yml + cat _slimPatterns.yml > slimPatterns.yml npm i $(npm pack ../..) ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" sls --dockerizePip=true --slim=true --runtime=python2.7 package - mv slimPatterns.yml _slimPatterns.yml unzip .serverless/sls-py-req-test.zip -d puck ls puck/flask test $(find puck -name "*.pyc" | wc -l) -eq 0 @@ -265,9 +381,8 @@ teardown() { @test "pipenv py3.6 can package flask with slim & slimPatterns option" { cd tests/pipenv npm i $(npm pack ../..) - mv _slimPatterns.yml slimPatterns.yml + cat _slimPatterns.yml > slimPatterns.yml sls --slim=true package - mv slimPatterns.yml _slimPatterns.yml unzip .serverless/sls-py-req-test.zip -d puck ls puck/flask test $(find puck -name "*.pyc" | wc -l) -eq 0 @@ -385,6 +500,21 @@ teardown() { ls puck/lambda_decorators.py } +@test "py3.6 can package lambda-decorators using vendor and invidiually option" { + cd tests/base + npm i $(npm pack ../..) + sls --individually=true --vendor=./vendor package + unzip .serverless/hello.zip -d puck + unzip .serverless/hello2.zip -d puck2 + unzip .serverless/hello3.zip -d puck3 + ls puck/flask + ls puck2/flask + ! ls puck3/flask + ls puck/lambda_decorators.py + ls puck2/lambda_decorators.py + ! ls puck3/lambda_decorators.py +} + @test "Don't nuke execute perms" { cd tests/base npm i $(npm pack ../..)