Downloading source for binary node.js packages

I’ve recently developed a website in node.js which relies on various third-party packages. The recommended way to redistribute these packages (especially if you don’t require your end user to have an internet connection) is to add your node_modules folder to source control and ship it with your site. See:

This is all well and good if the packages you use don’t require compiling, you can simply add them to source control and ship the lot.
But if the packages need compiling, two problems arise. Firstly no-one likes binaries in their source control system and secondly, the binaries created on one OS may not work on another. This means these binary packages need to be compiled as a post-install step.

The package manager for node, npm, has a great way of compiling or recompiling the packages once deployed. Simply execute the following in the root of your project:

$> npm rebuild

Unfortunately, npm doesn’t have a way to retrieve just the source for you. The ‘npm install’ command both downloads and compiles the packages. To this end, I have created a node.js script which takes advantage of the ‘npm cache’ method to recursively discover and download just the source for your node.js project. To execute it, save the following to a file (npm-cache.js for example) and execute it in the root of your node project. It parses the package.json file and grabs source into the node_modules directory in place. Enjoy!

var fs = require('fs'),
path = require('path'),
exec = require('child_process').exec,
util = require('util');

var packageFileName = 'package.json';
var modulesDirName = 'node_modules';
var cacheDirectory = process.cwd();
var npmCacheAddMask = 'npm cache add %s@%s; echo %s';
var sourceDirMask = '%s/%s/%s/package';
var targetDirMask = '%s/node_modules/%s';

function deleteFolder(folder) {
    if (fs.existsSync(folder)) {
        var files = fs.readdirSync(folder);
        files.forEach(function(file) {
            file = folder + "/" + file;
            if (fs.lstatSync(file).isDirectory()) {
            } else {

function downloadSource(folder) {
    var packageFile = path.join(folder, packageFileName);
    if (fs.existsSync(packageFile)) {
        var data = fs.readFileSync(packageFile);
        var package = JSON.parse(data);

        function getVersion(data) {
            var version = data.match(/-([^-]+)\.tgz/);
            return version[1];

        var callback = function(error, stdout, stderr) {
            var dependency = stdout.trim();
            var version = getVersion(stderr);
            var sourceDir = util.format(sourceDirMask, cacheDirectory, dependency, version);
            var targetDir = util.format(targetDirMask, folder, dependency);
            var modulesDir = folder + '/' + modulesDirName;

            if (!fs.existsSync(modulesDir)) {

            fs.renameSync(sourceDir, targetDir);
            deleteFolder(cacheDirectory + '/' + dependency);

        for (dependency in package.dependencies) {
            var version = package.dependencies[dependency];
            exec(util.format(npmCacheAddMask, dependency, version, dependency), callback);

if (!fs.existsSync(path.join(process.cwd(), packageFileName))) {
    console.log(util.format("Unable to find file '%s'.", packageFileName));

deleteFolder(path.join(process.cwd(), modulesDirName));
process.env.npm_config_cache = cacheDirectory;
Written by Rob in: Development | Tags: , ,

No Comments »

RSS feed for comments on this post. TrackBack URL

Leave a comment

Powered by WordPress | Design: WordPress Themes | Modifications: theGecko