diff -r 4fca650b9b8a -r fac22d8c2df8 common/corpus-common-addon/lib/commands/dl-fixtures.js --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/corpus-common-addon/lib/commands/dl-fixtures.js Thu Feb 25 10:20:01 2016 +0100 @@ -0,0 +1,112 @@ +'use strict'; + +var Command = require('ember-cli/lib/models/command'); +var chalk = require('chalk'); +var path = require('path'); +var Q = require("q"); +var request = require("request"); +var _ = require('lodash'); +var fs = require('fs'); + +// taken from http://stackoverflow.com/a/17238793 +// `condition` is a function that returns a boolean +// `body` is a function that returns a promise +// returns a promise for the completion of the loop +function promiseWhile(condition, body) { + + var done = Q.defer(); + + function loop() { + // When the result of calling `condition` is no longer true, we are + // done. + if (!condition()) return done.resolve(); + // Use `when`, in case `body` does not return a promise. + // When it completes loop again otherwise, if it fails, reject the + // done promise + Q.when(body(), loop, done.reject); + } + + // Start running the loop in the next tick so that this function is + // completely async. It would be unexpected if `body` was called + // synchronously the first time. + Q.nextTick(loop); + + // The promise + return done.promise; +} + + +module.exports = Command.extend({ + name: 'dl-fixtures', + description: "Download fixtures", + works: 'everywhere', + availableOptions: [ + { name: 'type', type: String, default: 'documents', aliases: ['t'], description: "type of obejcts to downloads" }, + { name: 'url', type: String, aliases: ['u'], description: "Source url" }, + { name: 'dest', type: String, aliases: ['d'], description: "File destination" }, + { name: 'page', type: Number, default: 1 , aliases: ['p'], description: "number of page to download"}, + ], + run: function(commandOptions, rawArgs) { + + //TODO: check that commandOptions.url is not empty + + var dest = commandOptions.dest || "." + path.sep + commandOptions.type + ".js" + + var ids = []; + var pageIndex = 1; + var nextPageUrl = commandOptions.url; + + return promiseWhile( + function() { return pageIndex <= commandOptions.page && nextPageUrl }, + function() { + var deferred = Q.defer(); + request.get({url: nextPageUrl, json: true}, function (err, res, body) { + if (err) { + return deferred.reject(err); + } else if (res.statusCode !== 200) { + err = new Error("Unexpected status code: " + res.statusCode); + err.res = res; + return deferred.reject(err); + } + nextPageUrl = body.next_page_url; + pageIndex++; + + ids = _.reduce( + body.documents, + function(res, doc) { + res.push(doc.id); + return res; + }, + ids + ); + deferred.resolve(ids); + }); + return deferred.promise; + } + ).then(function() { + return Q.all(_.map(ids, function(id) { + var deferred = Q.defer(); + request.get({url: commandOptions.url + id, json: true}, function (err, res, body) { + if (err) { + return deferred.reject(err); + } else if (res.statusCode !== 200) { + err = new Error("Unexpected status code: " + res.statusCode); + err.res = res; + return deferred.reject(err); + } + deferred.resolve(body.document); + }); + return deferred.promise; + })); + }).then(function(res) { + var deferred = Q.defer(); + fs.writeFile(dest, "module.exports = " + JSON.stringify(res,null,2) + ";", function(err) { + if(err) { + return deferred.reject(err); + } + deferred.resolve(); + }); + return deferred.promise; + }); + } +});