From 4fbe5b138cc8005e36f75ce4165274a3000a441a Mon Sep 17 00:00:00 2001 From: Rafal Boni Date: Tue, 12 Jan 2016 19:31:43 -0500 Subject: [PATCH 1/4] Add '-x' / '--excludeGlobs' args for general file excludion on deploy. This lets me e.g. exclude not only `.env` but by `.env.prod` / `.env.dev` and any other junky / sample files that happen to live in the method's directory. Or for that matter to move all those files into a sub- directory and exclude that. Less policy, more flexibility. --- bin/node-lambda | 3 ++ lib/main.js | 122 ++++++++++++++++++++++++++++++------------------ test/main.js | 66 ++++++++++++++++++++++---- 3 files changed, 136 insertions(+), 55 deletions(-) diff --git a/bin/node-lambda b/bin/node-lambda index ad755f49..b1b91eab 100755 --- a/bin/node-lambda +++ b/bin/node-lambda @@ -9,6 +9,7 @@ dotenv.load(); var AWS_ENVIRONMENT = process.env.AWS_ENVIRONMENT || ''; var CONFIG_FILE = process.env.CONFIG_FILE || ''; +var EXCLUDE_GLOBS = process.env.EXCLUDE_GLOBS || ''; var AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID || 'missing'; var AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY || 'missing'; var AWS_SESSION_TOKEN = process.env.AWS_SESSION_TOKEN || ''; @@ -45,6 +46,8 @@ program .option('-v, --version [' + AWS_FUNCTION_VERSION + ']', 'Lambda Function Version', AWS_FUNCTION_VERSION) .option('-f, --configFile [' + CONFIG_FILE + ']', 'Path to file holding secret environment variables (e.g. "deploy.env")', CONFIG_FILE) + .option('-x, --excludeGlobs [' + EXCLUDE_GLOBS + ']', + 'Space-separated glob pattern(s) for additional exclude files (e.g. ".env deploy.env")', EXCLUDE_GLOBS) .action(function (prg) { lambda.deploy(prg); }); diff --git a/lib/main.js b/lib/main.js index 8e147765..ee3f2e9c 100644 --- a/lib/main.js +++ b/lib/main.js @@ -97,7 +97,16 @@ Lambda.prototype._zipfileTmpPath = function (program) { }; Lambda.prototype._rsync = function (program, codeDirectory, callback) { - exec('rsync -r --exclude=.git --exclude=*.log --exclude=node_modules . ' + codeDirectory, function (err) { + var extraExcludes = ''; + + if (program.excludeGlobs) { + var excludes = program.excludeGlobs.split(' '); + excludes.forEach(function(exclude) { + extraExcludes += '--exclude=' + exclude + ' '; + }); + } + + exec('rsync -r --exclude=.git --exclude=.env --exclude=deploy.env --exclude=*.log --exclude=node_modules ' + extraExcludes + ' . ' + codeDirectory, function (err) { if (err) { throw err; } @@ -162,6 +171,22 @@ Lambda.prototype._codeDirectory = function (program) { return os.tmpDir() + '/' + program.functionName + '-' + epoch_time; }; +Lambda.prototype._cleanDirectory = function (codeDirectory, callback) { + exec('rm -rf ' + codeDirectory, function (err) { + if (err) { + throw err; + } + + fs.mkdir(codeDirectory, function(err) { + if (err) { + throw err; + } + + return callback(null, true); + }); + }); +}; + Lambda.prototype._setEnvironmentVars = function (program, codeDirectory) { console.log('=> Setting "environment variables" for Lambda from %s', program.configFile); // Which file is the handler? @@ -200,72 +225,77 @@ Lambda.prototype.deploy = function (program) { var regions = program.region.split(','); var codeDirectory = _this._codeDirectory(program); - console.log('=> Moving files to temporary directory'); - // Move all files to tmp folder (except .git, .log, event.json and node_modules) - - _this._rsync(program, codeDirectory, function (err) { + _this._cleanDirectory(codeDirectory, function (err) { if (err) { console.error(err); return; } - console.log('=> Running npm install --production'); - _this._npmInstall(program, codeDirectory, function (err) { + console.log('=> Moving files to temporary directory'); + // Move all files to tmp folder (except .git, .log, event.json and node_modules) + + _this._rsync(program, codeDirectory, function (err) { if (err) { console.error(err); return; } - - // Add custom environment variables if program.configFile is defined - if (program.configFile) { - _this._setEnvironmentVars(program, codeDirectory); - } - console.log('=> Zipping deployment package'); - - var archive = process.platform !== 'win32' ? _this._nativeZip : _this._zip; - archive = archive.bind(_this); - - archive(program, codeDirectory, function (err, buffer) { + console.log('=> Running npm install --production'); + _this._npmInstall(program, codeDirectory, function (err) { if (err) { console.error(err); return; } - console.log('=> Reading zip file to memory'); - var params = _this._params(program, buffer); + // Add custom environment variables if program.configFile is defined + if (program.configFile) { + _this._setEnvironmentVars(program, codeDirectory); + } + console.log('=> Zipping deployment package'); - async.map(regions, function (region, cb) { - console.log('=> Uploading zip file to AWS Lambda ' + region + ' with parameters:'); - console.log(params); + var archive = process.platform !== 'win32' ? _this._nativeZip : _this._zip; + archive = archive.bind(_this); - var aws_security = { - accessKeyId: program.accessKey, - secretAccessKey: program.secretKey, - region: region - }; + archive(program, codeDirectory, function (err, buffer) { + if (err) { + console.error(err); + return; + } - if (program.sessionToken){ - aws_security.sessionToken = program.sessionToken; - }; + console.log('=> Reading zip file to memory'); + var params = _this._params(program, buffer); - aws.config.update(aws_security); + async.map(regions, function (region, cb) { + console.log('=> Uploading zip file to AWS Lambda ' + region + ' with parameters:'); + console.log(params); - var lambda = new aws.Lambda({ - apiVersion: '2014-11-11' - }); + var aws_security = { + accessKeyId: program.accessKey, + secretAccessKey: program.secretKey, + region: region + }; - lambda.uploadFunction(params, function (err, data) { - cb(err, data); - }); + if (program.sessionToken){ + aws_security.sessionToken = program.sessionToken; + }; - }, function (err, results) { - if (err) { - console.error(err); - } else { - console.log('=> Zip file(s) done uploading. Results follow: '); - console.log(results); - } - }); + aws.config.update(aws_security); + var lambda = new aws.Lambda({ + apiVersion: '2014-11-11' + }); + + lambda.uploadFunction(params, function (err, data) { + cb(err, data); + }); + + }, function (err, results) { + if (err) { + console.error(err); + } else { + console.log('=> Zip file(s) done uploading. Results follow: '); + console.log(results); + } + }); + }); }); }); }); diff --git a/test/main.js b/test/main.js index c0db0185..d9539c8a 100644 --- a/test/main.js +++ b/test/main.js @@ -64,25 +64,67 @@ describe('node-lambda', function () { }); describe('_rsync', function () { + beforeEach(function (done) { + lambda._cleanDirectory(codeDirectory, done); + }); + it('rsync an index.js as well as other files', function (done) { lambda._rsync(program, codeDirectory, function (err, result) { var contents = fs.readdirSync(codeDirectory); - result = _.includes(contents, 'index.js'); + result = _.includes(contents, 'index.js') || + _.includes(contents, 'package.json'); assert.equal(result, true); done(); }); }); + + describe("when there are excluded files", function (done) { + beforeEach(function (done) { + program.excludeGlobs="*.png test" + done(); + }); + + it('rsync an index.js as well as other files', function (done) { + lambda._rsync(program, codeDirectory, function (err, result) { + var contents = fs.readdirSync(codeDirectory); + + result = _.includes(contents, 'index.js') || + _.includes(contents, 'package.json'); + assert.equal(result, true); + + done(); + }); + }); + + it('rsync excludes files matching excludeGlobs', function (done) { + lambda._rsync(program, codeDirectory, function (err, result) { + var contents = fs.readdirSync(codeDirectory); + + result = _.includes(contents, 'node-lambda.png') || + _.includes(contents, 'test'); + assert.equal(result, false); + + done(); + }); + }); + }); }); describe('_npmInstall', function () { beforeEach(function (done) { - lambda._rsync(program, codeDirectory, function (err) { + lambda._cleanDirectory(codeDirectory, function (err) { if (err) { - return done(err); + throw err; } - done(); + + lambda._rsync(program, codeDirectory, function (err) { + if (err) { + return done(err); + } + done(); + }); }); }); @@ -92,7 +134,7 @@ describe('node-lambda', function () { lambda._npmInstall(program, codeDirectory, function (err, result) { var contents = fs.readdirSync(codeDirectory); - result = _.includes(contents, 'index.js'); + result = _.includes(contents, 'node_modules'); assert.equal(result, true); done(); @@ -103,15 +145,21 @@ describe('node-lambda', function () { describe('_zip', function () { beforeEach(function (done) { this.timeout(30000); // give it time to build the node modules - lambda._rsync(program, codeDirectory, function (err) { + lambda._cleanDirectory(codeDirectory, function (err) { if (err) { - return done(err); + throw err; } - lambda._npmInstall(program, codeDirectory, function (err) { + + lambda._rsync(program, codeDirectory, function (err) { if (err) { return done(err); } - done(); + lambda._npmInstall(program, codeDirectory, function (err) { + if (err) { + return done(err); + } + done(); + }); }); }); }); From 927a462047fb10be2cdbb3fc0adaebe5b09e3075 Mon Sep 17 00:00:00 2001 From: Rafal Boni Date: Tue, 12 Jan 2016 19:51:07 -0500 Subject: [PATCH 2/4] Make standard excludes a little bit more generic, tweak usage example. --- bin/node-lambda | 2 +- lib/main.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/node-lambda b/bin/node-lambda index b1b91eab..3b212676 100755 --- a/bin/node-lambda +++ b/bin/node-lambda @@ -47,7 +47,7 @@ program .option('-f, --configFile [' + CONFIG_FILE + ']', 'Path to file holding secret environment variables (e.g. "deploy.env")', CONFIG_FILE) .option('-x, --excludeGlobs [' + EXCLUDE_GLOBS + ']', - 'Space-separated glob pattern(s) for additional exclude files (e.g. ".env deploy.env")', EXCLUDE_GLOBS) + 'Space-separated glob pattern(s) for additional exclude files (e.g. "event.json dotenv.sample")', EXCLUDE_GLOBS) .action(function (prg) { lambda.deploy(prg); }); diff --git a/lib/main.js b/lib/main.js index ee3f2e9c..bd6afa79 100644 --- a/lib/main.js +++ b/lib/main.js @@ -106,7 +106,7 @@ Lambda.prototype._rsync = function (program, codeDirectory, callback) { }); } - exec('rsync -r --exclude=.git --exclude=.env --exclude=deploy.env --exclude=*.log --exclude=node_modules ' + extraExcludes + ' . ' + codeDirectory, function (err) { + exec('rsync -r --exclude=.git --exclude=*.env --exclude=*.log --exclude=node_modules ' + extraExcludes + ' . ' + codeDirectory, function (err) { if (err) { throw err; } From c57c70a43088b98f6b0fd5a0fbf2909e6b96d3a9 Mon Sep 17 00:00:00 2001 From: Rafal Boni Date: Tue, 12 Jan 2016 22:28:37 -0500 Subject: [PATCH 3/4] Minor test cleanup. --- test/main.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/main.js b/test/main.js index d9539c8a..8392a92f 100644 --- a/test/main.js +++ b/test/main.js @@ -116,7 +116,7 @@ describe('node-lambda', function () { beforeEach(function (done) { lambda._cleanDirectory(codeDirectory, function (err) { if (err) { - throw err; + return done(err); } lambda._rsync(program, codeDirectory, function (err) { @@ -147,7 +147,7 @@ describe('node-lambda', function () { this.timeout(30000); // give it time to build the node modules lambda._cleanDirectory(codeDirectory, function (err) { if (err) { - throw err; + return done(err); } lambda._rsync(program, codeDirectory, function (err) { From e3f4e7e8bd744d74fad31346b98791fee465675f Mon Sep 17 00:00:00 2001 From: Rafal Boni Date: Tue, 12 Jan 2016 22:36:17 -0500 Subject: [PATCH 4/4] Add EXCLUDE_GLOBS example to .env This excludes some of the files created by `node-lambda setup`, which while handy for development, aren't at all needed as part of the data uploaded to AWS Lambda. --- lib/.env.example | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/.env.example b/lib/.env.example index 7280b4d1..afcab3d2 100644 --- a/lib/.env.example +++ b/lib/.env.example @@ -11,3 +11,4 @@ AWS_MEMORY_SIZE=128 AWS_TIMEOUT=3 AWS_DESCRIPTION= AWS_RUNTIME=nodejs +EXCLUDE_GLOBS="deploy.env event.json"