merged

auto-update.js

@@ -1,23 +1,6 @@
-var Hipchat = require('node-hipchat');
-
-var HC = new Hipchat(process.env.HIPCHAT);
-var hipchat = {
-    message: function(color, message) {
-        if (process.env.HIPCHAT) {
-            var params = {
-                room: 165440,
-                from: 'Auto Update',
-                message: message,
-                color: color,
-                notify: 0
-            };
-            HC.postMessage(params, function(data) {});
-        } else {
-            console.log('No Hipchat API Key');
-        }
-    }
-};
-var path = require("path"),
+var Hipchat = require('node-hipchat'),
+    path = require("path"),
+    assert = require("assert"),
     fs = require("fs-extra"),
     glob = require("glob"),
     _ = require('lodash'),
@@ -26,24 +9,27 @@ var path = require("path"),
     tarball = require('tarball-extract'),
     mkdirp = require('mkdirp');
 
-fs.mkdirParent = function(dirPath, mode, callback) {
-    //Call the standard fs.mkdir
-    fs.mkdir(dirPath, mode, function(error) {
-        //When it fail in this way, do the custom steps
-        if (error && error.errno === 34) {
-            //Create all the parents recursively
-            fs.mkdirParent(path.dirname(dirPath), mode, callback);
-            //And then the directory
-            fs.mkdirParent(dirPath, mode, callback);
-        }
-        //Manually run the callback since we used our own callback to do all these
-        callback && callback(error);
-    });
+var HC = new Hipchat(process.env.HIPCHAT);
+var hipchat = {
+  message: function(color, message) {
+    if (process.env.HIPCHAT) {
+      var params = {
+        room: 165440,
+        from: 'Auto Update',
+        message: message,
+        color: color,
+        notify: 0
+      };
+      HC.postMessage(params, function(data) {});
+    } else {
+      console.log('No Hipchat API Key');
+    }
+  }
 };
 
 hipchat.message('gray', 'Auto Update Started');
 var newVersionCount = 0;
-var parse = function(json_file, ignore_missing, ignore_parse_fail) {
+var parse = function (json_file, ignore_missing, ignore_parse_fail) {
     var content;
 
     try {
@@ -64,108 +50,216 @@ var parse = function(json_file, ignore_missing, ignore_parse_fail) {
     }
 }
 
-var updateLibrary = function(pkg, callback) {
-    console.log('Checking versions for ' + pkg.npmName);
+var reEscape = function(s){
+    return s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
+}
 
-    var versionUpdates = [];
+/**
+ * Check if an npmFileMap object contains any path which are not normalized, and thus could allow access to parent dirs
+ * @param pkg
+ * @returns {*}
+ */
+var isValidFileMap = function(pkg){
+    var isValidPath = function(p){
+        if(p !== null){ //don't allow parent dir access, or tricky paths
+            p = p.replace(/\/+/g, '/'); //dont penalize for consequtive path seperators
+            return p === path.normalize(p);
+        }
+        return false
+    };
+
+    if(pkg && pkg.npmFileMap){
+        return _.every(pkg.npmFileMap, function(fileSpec){
+           if(isValidPath(fileSpec.basePath || "/")){
+               return _.every(fileSpec.files, isValidPath);
+           }
+           return false;
+        });
+    }
+    return false
+};
+
+var error = function(msg, name){
+    var err = new Error(msg);
+    err.name = name;
+    console.log(msg);
+    hipchat.message('red', msg);
+    return err;
+}
+error.PKG_NAME = 'BadPackageName'
+error.FILE_PATH = 'BadFilePath'
+
+/**
+ * returns a fucntion that takes N args, where each arg is a path that must not outside of libPath.
+ * returns true if all paths are within libPath, else false
+ */
+var isAllowedPathFn = function(libPath){ //is path within the lib dir? if not, they shouldnt be writing/reading there
+    libPath = path.normalize(libPath || "/");
+    return function(){
+        var paths = 1 <= arguments.length ? [].slice.call(arguments, 0) : [];
+        var re = new RegExp("^"+reEscape(libPath));
+        return _.every(paths, function(p) {
+            p = path.normalize(p);
+            return p.match(re);
+        });
+    }
+};
+
+var invalidNpmName = function(name){
+    return !!~name.indexOf(".."); //doesnt contain
+}
+
+/**
+ * Attempt to update the npmFileMap from extracted package.json, then using npmFileMap move required files to libPath/../
+ * If the npmFileMap tries to modify files outside of libPath, dont let it!
+ * @param pkg
+ * @param libPath = root folder for extracted lib
+ * @returns {Array} = array of security related errors triggered during operation.
+ */
+var processNewVersion = function(pkg, version){
+    var extractLibPath = path.join(getPackageTempPath(pkg, version), 'package');
+    var libPath = getPackagePath(pkg, version)
+
+    var isAllowedPath = isAllowedPathFn(extractLibPath);
+
+    var newPath = path.join(libPath, 'package.json')
+    if(false && fs.existsSync(newPath)){ //turn this off for now
+        var newPkg = parse(newPath);
+        if(isValidFileMap(newPkg)){
+            pkg.npmFileMap = newPkg.npmFileMap;
+        }
+    }
+    var npmFileMap = pkg.npmFileMap;
+    var errors = [];
+
+    _.each(npmFileMap, function(fileSpec) {
+        var basePath = fileSpec.basePath || "";
+
+        _.each(fileSpec.files, function(file) {
+            var libContentsPath = path.normalize(path.join(extractLibPath, basePath));
+            if(!isAllowedPath(libContentsPath)){
+                errors.push(error(pkg.npmName+" contains a malicious file path: "+libContentsPath, error.FILE_PATH));
+                return
+            }
+            var files = glob.sync(path.join(libContentsPath, file));
+            var copyPath = path.join(libPath, basePath)
+
+            _.each(files, function(extractFilePath) {
+                if(extractFilePath.match(/(dependencies|\.zip\s*$)/i)) return;
+
+                var copyPart = path.relative(libContentsPath, extractFilePath);
+                var copyPath = path.join(libPath, copyPart)
+                fs.mkdirsSync(path.dirname(copyPath))
+                //TODO remove me:
+                console.log('rename:',extractFilePath, copyPath)
+
+                fs.renameSync(extractFilePath, copyPath);
+            });
+        });
+    });
+    return errors;
+}
 
 
-    request.get('http://registry.npmjs.org/' + pkg.npmName, function(result) {
-        _.each(result.body.versions, function(data, version) {
-            var path = './ajax/libs/' + pkg.name + '/' + version;
-            if (!fs.existsSync(path)) {
-                console.log('Dont have this verison', path);
+var getPackageTempPath = function(pkg, version){
+    return path.normalize(path.join(__dirname, 'temp', pkg.name, version))
+}
+var getPackagePath = function(pkg, version){
+    return path.normalize(path.join(__dirname, 'ajax', 'libs', pkg.name, version));
+}
+/**
+ * download and extract a tarball for a single npm version, get the files in npmFileMap and delete the rest
+ * @param pkg
+ * @param tarballUrl
+ * @param version
+ * @param cb
+ * @returns {*}
+ */
+var updateLibraryVersion = function(pkg, tarballUrl, version, cb) {
+    if(invalidNpmName(pkg.name)){
+        return cb(error(pkg.npmName+" has a malicious package name:"+ pkg.name, error.PKG_NAME));
+    }
+    var extractLibPath = getPackageTempPath(pkg, version);
+    var libPath = getPackagePath(pkg, version);
 
-                versionUpdates.push(function(callback) {
-                    fs.mkdirSync(path);
-                    var url = data.dist.tarball;
-                    var download_file = path + '/dist.tar.gz';
-                    console.log('Downloading...');
-                    try {
-                        tarball.extractTarballDownload(url, download_file, path, {}, function(err, result) {
-                            console.log('Downloaded');
-                            fs.unlinkSync(download_file);
-                            if (err) {
-                                fs.removeSync(path + '/' + folderName);
-                                console.log('critcal');
-                                callback();
-                                return false;
-                            }
-                            var folderName = fs.readdirSync(path)[0];
 
-                            var npmFileMap = pkg.npmFileMap;
-
-                            _.each(npmFileMap, function(fileSpec) {
-                                var basePath = fileSpec.basePath || "";
-                                console.log('looping through files');
-                                _.each(fileSpec.files, function(file) {
-                                    var extractPath = basePath + "/" + file;
-                                    var files = glob.sync(path + "/" + folderName + "/" + basePath + "/" + file);
-
-                                    _.each(files, function(extractFilePath) {
-                                        if (extractFilePath.slice(-4) == ".zip") return;
-                                        if (extractFilePath.indexOf("dependencies") !== -1) return;
-                                        var replacePath = folderName + "/" + basePath + "/";
-                                        replacePath = replacePath.replace(/\/\//g, "/");
-                                        replacePath = replacePath.replace(/\/\//g, "/");
-                                        var actualPath = extractFilePath.replace(replacePath, "");
-                                        console.log('checking extract', replacePath);
-                                        if (fs.existsSync(extractFilePath)) {
-                                            var bla = actualPath;
-                                            fs.mkdirParent(bla.substr(2, bla.lastIndexOf('/') - 1), function() {
-                                                console.log('made dir', bla.substr(2, bla.lastIndexOf('/') - 1))
-                                                fs.renameSync(extractFilePath, actualPath);
-                                            })
-                                        } else {
-                                            console.log('ERRRRRORRRRRR', extractFilePath, actualPath);
-                                        }
-                                    });
-                                });
-                            });
-                            console.log('fiinished');
-                            setTimeout(function() {
-                                fs.removeSync(path + '/' + folderName);
-                                callback();
-                            }, 200);
-                        });
-                    } catch (e) {
-                        console.log('so erro rprone', e)
-                    }
-
-                });;
+    if(!fs.existsSync(libPath)) {
+        fs.mkdirsSync(extractLibPath);
+        var url = tarballUrl;
+        var downloadFile = path.join(extractLibPath, 'dist.tar.gz');
+        tarball.extractTarballDownload(url , downloadFile, extractLibPath, {}, function(err, result) {
+            if(fs.existsSync(downloadFile)){
+                processNewVersion(pkg, version);
                 newVersionCount++;
                 console.log("Do not have version", version, "of", pkg.npmName);
+            } else {
+                console.log("error downloading "+ version+ "of "+pkg.npmName+" it didnt exist: ", result, err)
             }
+            cb()
         });
+    } else {
+        cb()
+    }
+};
 
-
-        async.series(versionUpdates, function(err, results) {
+/**
+ * grab all versions of a lib that has an 'npmFileMap' and 'npmName' in its package.json
+ * @param pkg
+ * @param tarballUrl
+ * @param cb
+ */
+var updateLibrary = function (pkg, cb) {
+    if(!isValidFileMap(pkg)){
+        console.log(pkg.npmName+" has a malicious npmFileMap");
+        hipchat.message('red', pkg.npmName+" has a malicious npmFileMap: "+ JSON.stringify(pkg.npmFileMap));
+        return cb(null);
+    }
+    console.log('Checking versions for ' + pkg.npmName);
+    request.get('http://registry.npmjs.org/' + pkg.npmName, function(result) {
+        async.eachLimit(_.pairs(result.body.versions), 5, function(p, cb){ //extract 5 at a time
+            var data = p[1];
+            var version = p[0];
+            updateLibraryVersion(pkg, data.dist.tarball, version, cb)
+        }, function(err){
             var npmVersion = result.body['dist-tags'] && result.body['dist-tags'].latest || 0;
             pkg.version = npmVersion;
             fs.writeFileSync('ajax/libs/' + pkg.name + '/package.json', JSON.stringify(pkg, null, 2), 'utf8');
-
-            callback(null, pkg['npm-name']);
+            cb(null);
         });
     });
 }
 
-console.log('Looking for npm enabled libraries...');
+exports.run = function(){
+    fs.removeSync(path.join(__dirname, 'temp'))
+    console.log('Looking for npm enabled libraries...');
 
-// load up those files
-var packages = glob.sync("./ajax/libs/**/package.json");
-packages = _(packages).map(function(pkg) {
-    var parsedPkg = parse(pkg);
-    return parsedPkg.npmName ? parsedPkg : null;
-}).compact().value();
-hipchat.message('green', 'Found ' + packages.length + ' npm enabled libraries');
-console.log('Found ' + packages.length + ' npm enabled libraries');
-var libraryUpdates = [];
-_.each(packages, function(pkg) {
-    libraryUpdates.push(function(callback) {
-        updateLibrary(pkg, callback);
-    });;
-});
-async.series(libraryUpdates, function(err, results) {
-    console.log('Script completed');
-    hipchat.message('green', 'Auto Update Completed - ' + newVersionCount + ' versions were updated');
-});
+    // load up those files
+    var packages = glob.sync("./ajax/libs/*/package.json");
+    packages = _(packages).map(function (pkg) {
+        var parsedPkg = parse(pkg);
+        return (parsedPkg.npmName && parsedPkg.npmFileMap) ? parsedPkg : null;
+    }).compact().value();
+    hipchat.message('green', 'Found ' + packages.length + ' npm enabled libraries');
+    console.log('Found ' + packages.length + ' npm enabled libraries');
+
+    async.eachSeries(packages, updateLibrary, function(err) {
+        console.log('Script completed');
+        hipchat.message('green', 'Auto Update Completed - ' + newVersionCount + ' versions were updated');
+        fs.removeSync(path.join(__dirname, 'temp'))
+    });
+}
+exports.updateLibrary = updateLibrary;
+exports.updateLibraryVersion = updateLibraryVersion;
+exports.processNewVersion = processNewVersion;
+exports.error = error;
+exports.isAllowedPathFn = isAllowedPathFn;
+exports.isValidFileMap = isValidFileMap;
+exports.invalidNpmName = invalidNpmName;
+
+
+var args = process.argv.slice(2);
+if(args.length > 0 && args[0] == 'run'){
+    exports.run()
+} else {
+    console.log('to start, pass the "run" arg')
+}

auto-update.sh

@@ -10,7 +10,7 @@ echo npm install for good measure
 /usr/local/bin/npm install
 
 echo Starting auto update script
-/usr/local/bin/node auto-update.js >> node.log
+/usr/local/bin/node auto-update.js run >> node.log
 
 echo Pushing new versions
 git add .

test/auto-update-test.js

@@ -0,0 +1,109 @@
+var assert = require("assert"),
+    path = require("path"),
+    fs = require("fs"),
+    vows = require("vows-si"),
+    _  = require('lodash'),
+    au = require('./../auto-update');
+
+
+
+
+var suite = vows.describe('NPM Auto Update - stand alone methods');
+suite.addBatch({
+    'npm name validation': {
+        topic: ["floatthead", "../evil"],
+        'This is a valid npm name': function (arr) {
+            assert.equal(au.invalidNpmName(arr[0]), false);
+        },
+        'This is an invalid npm name': function (arr) {
+            assert.equal(au.invalidNpmName(arr[1]), true);
+        }
+    },
+    'npmFileMap validation - simple': {
+        topic: {"npmFileMap": [
+            {
+                "basePath": "/dist/",
+                "files": [
+                    "*.js",
+                    "blee/blah//script.js",
+                    "blee/blah//script.min.js",
+                    "styles.css",
+                    "/test/**/*.*"
+                ]
+            }
+        ]},
+        'This is a valid npm file map': function (obj) {
+            assert.equal(au.isValidFileMap(obj), true);
+        },
+        'file paths are ok too': function(obj){
+            var map = obj.npmFileMap[0];
+            var testFn = au.isAllowedPathFn(path.join('someplace', map.basePath));
+            assert.equal(testFn.apply(null, _.map(map.files, function(f){ return path.join("someplace", map.basePath, f)})), true)
+        }
+    },
+    'npmFileMap validation - arrays': {
+        topic: {"npmFileMap": [
+            {
+                "basePath": "/dist/",
+                "files": [
+                    "*.js",
+                    "blee/blah//script.js",
+                    "blee/blah//script.min.js",
+                    "styles.css",
+                    "/test/**/*.*"
+                ]
+            },
+            {
+                "basePath": "",
+                "files": [
+                    "test.css",
+                    "/blee.js",
+                    "this_is_ok_right_now.zip"
+                ]
+            },
+            {
+                "basePath": "/",
+                "files": [
+                    "*"
+                ]
+            }
+        ]},
+        'valid array of file maps': function (obj) {
+            assert.equal(au.isValidFileMap(obj), true);
+        },
+        'these paths are ok too': function(obj){
+            var map = obj.npmFileMap[0];
+            var testFn = au.isAllowedPathFn(path.join('someplace', map.basePath));
+            assert.equal(testFn.apply(null, _.map(map.files, function(f){ return path.join("someplace", map.basePath, f)})), true)
+        },
+        'these paths are also allowed': function(obj){
+            var map = obj.npmFileMap[1];
+            var testFn = au.isAllowedPathFn(path.join('someplace', map.basePath));
+            assert.equal(testFn.apply(null, _.map(map.files, function(f){ return path.join("someplace", map.basePath, f)})), true)
+        }
+
+    },
+    'npmFileMap validation - invalid 1': {
+        topic: {"npmFileMap": [
+            {
+                "basePath": "/dist/",
+                "files": [
+                    "*.js",
+                    "blee/blah/../../../script.js",
+                    "/../../../../../../../../../../etc/hosts",
+                    "styles.css",
+                    "/test/**/*.*"
+                ]
+            }
+        ]},
+        'this npm filemap is doing evil things': function (obj) {
+            assert.equal(au.isValidFileMap(obj), false);
+        },
+        'these paths are bad': function(obj){
+            var map = obj.npmFileMap[0];
+            var testFn = au.isAllowedPathFn(path.join('someplace', map.basePath));
+            assert.equal(testFn.apply(null, _.map(map.files, function(f){ return path.join("someplace", map.basePath, f)})), false)
+        }
+    }
+});
+suite.export(module);