diff --git a/auto-update.js b/auto-update.js index 1bdf071ab..f29248fb9 100644 --- a/auto-update.js +++ b/auto-update.js @@ -1,23 +1,6 @@ -var Hipchat = require('node-hipchat'); - -var HC = new Hipchat(process.env.HIPCHAT); -var hipchat = { - message: function(color, message) { - if (process.env.HIPCHAT) { - var params = { - room: 165440, - from: 'Auto Update', - message: message, - color: color, - notify: 0 - }; - HC.postMessage(params, function(data) {}); - } else { - console.log('No Hipchat API Key'); - } - } -}; -var path = require("path"), +var Hipchat = require('node-hipchat'), + path = require("path"), + assert = require("assert"), fs = require("fs-extra"), glob = require("glob"), _ = require('lodash'), @@ -26,24 +9,27 @@ var path = require("path"), tarball = require('tarball-extract'), mkdirp = require('mkdirp'); -fs.mkdirParent = function(dirPath, mode, callback) { - //Call the standard fs.mkdir - fs.mkdir(dirPath, mode, function(error) { - //When it fail in this way, do the custom steps - if (error && error.errno === 34) { - //Create all the parents recursively - fs.mkdirParent(path.dirname(dirPath), mode, callback); - //And then the directory - fs.mkdirParent(dirPath, mode, callback); - } - //Manually run the callback since we used our own callback to do all these - callback && callback(error); - }); +var HC = new Hipchat(process.env.HIPCHAT); +var hipchat = { + message: function(color, message) { + if (process.env.HIPCHAT) { + var params = { + room: 165440, + from: 'Auto Update', + message: message, + color: color, + notify: 0 + }; + HC.postMessage(params, function(data) {}); + } else { + console.log('No Hipchat API Key'); + } + } }; hipchat.message('gray', 'Auto Update Started'); var newVersionCount = 0; -var parse = function(json_file, ignore_missing, ignore_parse_fail) { +var parse = function (json_file, ignore_missing, ignore_parse_fail) { var content; try { @@ -64,108 +50,216 @@ var parse = function(json_file, ignore_missing, ignore_parse_fail) { } } -var updateLibrary = function(pkg, callback) { - console.log('Checking versions for ' + pkg.npmName); +var reEscape = function(s){ + return s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&'); +} - var versionUpdates = []; +/** + * Check if an npmFileMap object contains any path which are not normalized, and thus could allow access to parent dirs + * @param pkg + * @returns {*} + */ +var isValidFileMap = function(pkg){ + var isValidPath = function(p){ + if(p !== null){ //don't allow parent dir access, or tricky paths + p = p.replace(/\/+/g, '/'); //dont penalize for consequtive path seperators + return p === path.normalize(p); + } + return false + }; + + if(pkg && pkg.npmFileMap){ + return _.every(pkg.npmFileMap, function(fileSpec){ + if(isValidPath(fileSpec.basePath || "/")){ + return _.every(fileSpec.files, isValidPath); + } + return false; + }); + } + return false +}; + +var error = function(msg, name){ + var err = new Error(msg); + err.name = name; + console.log(msg); + hipchat.message('red', msg); + return err; +} +error.PKG_NAME = 'BadPackageName' +error.FILE_PATH = 'BadFilePath' + +/** + * returns a fucntion that takes N args, where each arg is a path that must not outside of libPath. + * returns true if all paths are within libPath, else false + */ +var isAllowedPathFn = function(libPath){ //is path within the lib dir? if not, they shouldnt be writing/reading there + libPath = path.normalize(libPath || "/"); + return function(){ + var paths = 1 <= arguments.length ? [].slice.call(arguments, 0) : []; + var re = new RegExp("^"+reEscape(libPath)); + return _.every(paths, function(p) { + p = path.normalize(p); + return p.match(re); + }); + } +}; + +var invalidNpmName = function(name){ + return !!~name.indexOf(".."); //doesnt contain +} + +/** + * Attempt to update the npmFileMap from extracted package.json, then using npmFileMap move required files to libPath/../ + * If the npmFileMap tries to modify files outside of libPath, dont let it! + * @param pkg + * @param libPath = root folder for extracted lib + * @returns {Array} = array of security related errors triggered during operation. + */ +var processNewVersion = function(pkg, version){ + var extractLibPath = path.join(getPackageTempPath(pkg, version), 'package'); + var libPath = getPackagePath(pkg, version) + + var isAllowedPath = isAllowedPathFn(extractLibPath); + + var newPath = path.join(libPath, 'package.json') + if(false && fs.existsSync(newPath)){ //turn this off for now + var newPkg = parse(newPath); + if(isValidFileMap(newPkg)){ + pkg.npmFileMap = newPkg.npmFileMap; + } + } + var npmFileMap = pkg.npmFileMap; + var errors = []; + + _.each(npmFileMap, function(fileSpec) { + var basePath = fileSpec.basePath || ""; + + _.each(fileSpec.files, function(file) { + var libContentsPath = path.normalize(path.join(extractLibPath, basePath)); + if(!isAllowedPath(libContentsPath)){ + errors.push(error(pkg.npmName+" contains a malicious file path: "+libContentsPath, error.FILE_PATH)); + return + } + var files = glob.sync(path.join(libContentsPath, file)); + var copyPath = path.join(libPath, basePath) + + _.each(files, function(extractFilePath) { + if(extractFilePath.match(/(dependencies|\.zip\s*$)/i)) return; + + var copyPart = path.relative(libContentsPath, extractFilePath); + var copyPath = path.join(libPath, copyPart) + fs.mkdirsSync(path.dirname(copyPath)) + //TODO remove me: + console.log('rename:',extractFilePath, copyPath) + + fs.renameSync(extractFilePath, copyPath); + }); + }); + }); + return errors; +} - request.get('http://registry.npmjs.org/' + pkg.npmName, function(result) { - _.each(result.body.versions, function(data, version) { - var path = './ajax/libs/' + pkg.name + '/' + version; - if (!fs.existsSync(path)) { - console.log('Dont have this verison', path); +var getPackageTempPath = function(pkg, version){ + return path.normalize(path.join(__dirname, 'temp', pkg.name, version)) +} +var getPackagePath = function(pkg, version){ + return path.normalize(path.join(__dirname, 'ajax', 'libs', pkg.name, version)); +} +/** + * download and extract a tarball for a single npm version, get the files in npmFileMap and delete the rest + * @param pkg + * @param tarballUrl + * @param version + * @param cb + * @returns {*} + */ +var updateLibraryVersion = function(pkg, tarballUrl, version, cb) { + if(invalidNpmName(pkg.name)){ + return cb(error(pkg.npmName+" has a malicious package name:"+ pkg.name, error.PKG_NAME)); + } + var extractLibPath = getPackageTempPath(pkg, version); + var libPath = getPackagePath(pkg, version); - versionUpdates.push(function(callback) { - fs.mkdirSync(path); - var url = data.dist.tarball; - var download_file = path + '/dist.tar.gz'; - console.log('Downloading...'); - try { - tarball.extractTarballDownload(url, download_file, path, {}, function(err, result) { - console.log('Downloaded'); - fs.unlinkSync(download_file); - if (err) { - fs.removeSync(path + '/' + folderName); - console.log('critcal'); - callback(); - return false; - } - var folderName = fs.readdirSync(path)[0]; - var npmFileMap = pkg.npmFileMap; - - _.each(npmFileMap, function(fileSpec) { - var basePath = fileSpec.basePath || ""; - console.log('looping through files'); - _.each(fileSpec.files, function(file) { - var extractPath = basePath + "/" + file; - var files = glob.sync(path + "/" + folderName + "/" + basePath + "/" + file); - - _.each(files, function(extractFilePath) { - if (extractFilePath.slice(-4) == ".zip") return; - if (extractFilePath.indexOf("dependencies") !== -1) return; - var replacePath = folderName + "/" + basePath + "/"; - replacePath = replacePath.replace(/\/\//g, "/"); - replacePath = replacePath.replace(/\/\//g, "/"); - var actualPath = extractFilePath.replace(replacePath, ""); - console.log('checking extract', replacePath); - if (fs.existsSync(extractFilePath)) { - var bla = actualPath; - fs.mkdirParent(bla.substr(2, bla.lastIndexOf('/') - 1), function() { - console.log('made dir', bla.substr(2, bla.lastIndexOf('/') - 1)) - fs.renameSync(extractFilePath, actualPath); - }) - } else { - console.log('ERRRRRORRRRRR', extractFilePath, actualPath); - } - }); - }); - }); - console.log('fiinished'); - setTimeout(function() { - fs.removeSync(path + '/' + folderName); - callback(); - }, 200); - }); - } catch (e) { - console.log('so erro rprone', e) - } - - });; + if(!fs.existsSync(libPath)) { + fs.mkdirsSync(extractLibPath); + var url = tarballUrl; + var downloadFile = path.join(extractLibPath, 'dist.tar.gz'); + tarball.extractTarballDownload(url , downloadFile, extractLibPath, {}, function(err, result) { + if(fs.existsSync(downloadFile)){ + processNewVersion(pkg, version); newVersionCount++; console.log("Do not have version", version, "of", pkg.npmName); + } else { + console.log("error downloading "+ version+ "of "+pkg.npmName+" it didnt exist: ", result, err) } + cb() }); + } else { + cb() + } +}; - - async.series(versionUpdates, function(err, results) { +/** + * grab all versions of a lib that has an 'npmFileMap' and 'npmName' in its package.json + * @param pkg + * @param tarballUrl + * @param cb + */ +var updateLibrary = function (pkg, cb) { + if(!isValidFileMap(pkg)){ + console.log(pkg.npmName+" has a malicious npmFileMap"); + hipchat.message('red', pkg.npmName+" has a malicious npmFileMap: "+ JSON.stringify(pkg.npmFileMap)); + return cb(null); + } + console.log('Checking versions for ' + pkg.npmName); + request.get('http://registry.npmjs.org/' + pkg.npmName, function(result) { + async.eachLimit(_.pairs(result.body.versions), 5, function(p, cb){ //extract 5 at a time + var data = p[1]; + var version = p[0]; + updateLibraryVersion(pkg, data.dist.tarball, version, cb) + }, function(err){ var npmVersion = result.body['dist-tags'] && result.body['dist-tags'].latest || 0; pkg.version = npmVersion; fs.writeFileSync('ajax/libs/' + pkg.name + '/package.json', JSON.stringify(pkg, null, 2), 'utf8'); - - callback(null, pkg['npm-name']); + cb(null); }); }); } -console.log('Looking for npm enabled libraries...'); +exports.run = function(){ + fs.removeSync(path.join(__dirname, 'temp')) + console.log('Looking for npm enabled libraries...'); -// load up those files -var packages = glob.sync("./ajax/libs/**/package.json"); -packages = _(packages).map(function(pkg) { - var parsedPkg = parse(pkg); - return parsedPkg.npmName ? parsedPkg : null; -}).compact().value(); -hipchat.message('green', 'Found ' + packages.length + ' npm enabled libraries'); -console.log('Found ' + packages.length + ' npm enabled libraries'); -var libraryUpdates = []; -_.each(packages, function(pkg) { - libraryUpdates.push(function(callback) { - updateLibrary(pkg, callback); - });; -}); -async.series(libraryUpdates, function(err, results) { - console.log('Script completed'); - hipchat.message('green', 'Auto Update Completed - ' + newVersionCount + ' versions were updated'); -}); + // load up those files + var packages = glob.sync("./ajax/libs/*/package.json"); + packages = _(packages).map(function (pkg) { + var parsedPkg = parse(pkg); + return (parsedPkg.npmName && parsedPkg.npmFileMap) ? parsedPkg : null; + }).compact().value(); + hipchat.message('green', 'Found ' + packages.length + ' npm enabled libraries'); + console.log('Found ' + packages.length + ' npm enabled libraries'); + + async.eachSeries(packages, updateLibrary, function(err) { + console.log('Script completed'); + hipchat.message('green', 'Auto Update Completed - ' + newVersionCount + ' versions were updated'); + fs.removeSync(path.join(__dirname, 'temp')) + }); +} +exports.updateLibrary = updateLibrary; +exports.updateLibraryVersion = updateLibraryVersion; +exports.processNewVersion = processNewVersion; +exports.error = error; +exports.isAllowedPathFn = isAllowedPathFn; +exports.isValidFileMap = isValidFileMap; +exports.invalidNpmName = invalidNpmName; + + +var args = process.argv.slice(2); +if(args.length > 0 && args[0] == 'run'){ + exports.run() +} else { + console.log('to start, pass the "run" arg') +} diff --git a/auto-update.sh b/auto-update.sh index 24a28a512..84dc58f22 100755 --- a/auto-update.sh +++ b/auto-update.sh @@ -10,7 +10,7 @@ echo npm install for good measure /usr/local/bin/npm install echo Starting auto update script -/usr/local/bin/node auto-update.js >> node.log +/usr/local/bin/node auto-update.js run >> node.log echo Pushing new versions git add . diff --git a/test/auto-update-test.js b/test/auto-update-test.js new file mode 100644 index 000000000..bcb640084 --- /dev/null +++ b/test/auto-update-test.js @@ -0,0 +1,109 @@ +var assert = require("assert"), + path = require("path"), + fs = require("fs"), + vows = require("vows-si"), + _ = require('lodash'), + au = require('./../auto-update'); + + + + +var suite = vows.describe('NPM Auto Update - stand alone methods'); +suite.addBatch({ + 'npm name validation': { + topic: ["floatthead", "../evil"], + 'This is a valid npm name': function (arr) { + assert.equal(au.invalidNpmName(arr[0]), false); + }, + 'This is an invalid npm name': function (arr) { + assert.equal(au.invalidNpmName(arr[1]), true); + } + }, + 'npmFileMap validation - simple': { + topic: {"npmFileMap": [ + { + "basePath": "/dist/", + "files": [ + "*.js", + "blee/blah//script.js", + "blee/blah//script.min.js", + "styles.css", + "/test/**/*.*" + ] + } + ]}, + 'This is a valid npm file map': function (obj) { + assert.equal(au.isValidFileMap(obj), true); + }, + 'file paths are ok too': function(obj){ + var map = obj.npmFileMap[0]; + var testFn = au.isAllowedPathFn(path.join('someplace', map.basePath)); + assert.equal(testFn.apply(null, _.map(map.files, function(f){ return path.join("someplace", map.basePath, f)})), true) + } + }, + 'npmFileMap validation - arrays': { + topic: {"npmFileMap": [ + { + "basePath": "/dist/", + "files": [ + "*.js", + "blee/blah//script.js", + "blee/blah//script.min.js", + "styles.css", + "/test/**/*.*" + ] + }, + { + "basePath": "", + "files": [ + "test.css", + "/blee.js", + "this_is_ok_right_now.zip" + ] + }, + { + "basePath": "/", + "files": [ + "*" + ] + } + ]}, + 'valid array of file maps': function (obj) { + assert.equal(au.isValidFileMap(obj), true); + }, + 'these paths are ok too': function(obj){ + var map = obj.npmFileMap[0]; + var testFn = au.isAllowedPathFn(path.join('someplace', map.basePath)); + assert.equal(testFn.apply(null, _.map(map.files, function(f){ return path.join("someplace", map.basePath, f)})), true) + }, + 'these paths are also allowed': function(obj){ + var map = obj.npmFileMap[1]; + var testFn = au.isAllowedPathFn(path.join('someplace', map.basePath)); + assert.equal(testFn.apply(null, _.map(map.files, function(f){ return path.join("someplace", map.basePath, f)})), true) + } + + }, + 'npmFileMap validation - invalid 1': { + topic: {"npmFileMap": [ + { + "basePath": "/dist/", + "files": [ + "*.js", + "blee/blah/../../../script.js", + "/../../../../../../../../../../etc/hosts", + "styles.css", + "/test/**/*.*" + ] + } + ]}, + 'this npm filemap is doing evil things': function (obj) { + assert.equal(au.isValidFileMap(obj), false); + }, + 'these paths are bad': function(obj){ + var map = obj.npmFileMap[0]; + var testFn = au.isAllowedPathFn(path.join('someplace', map.basePath)); + assert.equal(testFn.apply(null, _.map(map.files, function(f){ return path.join("someplace", map.basePath, f)})), false) + } + } +}); +suite.export(module);