improve document model and propagate changes. This include the change of document fixtures to better reflect what the api is effectively returning
/*eslint-env node */
/*global require:true*/
'use strict';
var Command = require('ember-cli/lib/models/command');
var path = require('path');
var Q = require('q');
var request = require('request');
var _ = require('lodash');
var fs = require('fs');
var chalk = require('chalk');
// taken from http://stackoverflow.com/a/17238793
// `condition` is a function that returns a boolean
// `body` is a function that returns a promise
// returns a promise for the completion of the loop
function promiseWhile(condition, body) {
var done = Q.defer();
function loop(res) {
// When the result of calling `condition` is no longer true, we are
// done.
if (!condition()) return done.resolve(res);
// Use `when`, in case `body` does not return a promise.
// When it completes loop again otherwise, if it fails, reject the
// done promise
Q.when(body(), loop, done.reject);
}
// Start running the loop in the next tick so that this function is
// completely async. It would be unexpected if `body` was called
// synchronously the first time.
Q.nextTick(loop);
// The promise
return done.promise;
}
module.exports = Command.extend({
name: 'dl-fixtures',
description: 'Download fixtures',
works: 'everywhere',
availableOptions: [
{ name: 'type', type: String, default: 'documents', aliases: ['t'], description: 'type of obejcts to downloads' },
{ name: 'url', type: String, aliases: ['u'], description: 'Source url' },
{ name: 'dest', type: String, aliases: ['d'], description: 'File destination' },
{ name: 'page', type: Number, default: 1 , aliases: ['p'], description: 'number of page to download'},
{ name: 'format', type: String, default: 'es6' , aliases: ['f'], description: 'Format for module export, es6 or require'},
{ name: 'extra', type: String, default: '', aliases: ['e'], description: 'additionnal data to download (comma separated list of document ids)'}
],
dl_countmap: function(commandOptions, rootKey) {
var done = Q.defer();
var that = this;
request.get({url: commandOptions.url, json: true}, function (err, res, body) {
var objectList = _.reduce(
_.keys(body[rootKey]),
function(res, objKey) {
var objValue = body[rootKey][objKey];
res.push({id: objKey, count: objValue.count, label: objValue.label});
return res;
},
[]
);
var prefix = (that.format==='es6')?'export default ':'module.exports = ';
fs.writeFile(that.dest, prefix + JSON.stringify(objectList,null,2) + ';', function(err) {
if(err) {
return done.reject(err);
}
done.resolve();
});
});
return done.promise;
},
dl_discourses: function(commandOptions) {
return this.dl_countmap(commandOptions, 'discourses');
},
dl_themes: function(commandOptions) {
return this.dl_countmap(commandOptions, 'themes');
},
dl_documents_ids: function(commandOptions, rawArgs, ui) {
var nextPageUrl = commandOptions.url;
var pageIndex = 1;
var ids = commandOptions.extra?_.map(commandOptions.extra.split(','), function(id) { return { id: id};}):[];
return promiseWhile(
function() { return pageIndex <= commandOptions.page && nextPageUrl; },
function() {
var deferred = Q.defer();
ui.writeLine(chalk.yellow('Download Documents : getting page ' + pageIndex));
request.get({url: nextPageUrl, json: true}, function (err, res, body) {
if (err) {
return deferred.reject(err);
} else if (res.statusCode !== 200) {
err = new Error('Unexpected status code: ' + res.statusCode);
err.res = res;
return deferred.reject(err);
}
nextPageUrl = body.next_page_url;
pageIndex++;
ids = _.reduce(
body.documents,
function(res, doc) {
res.push(doc);
return res;
},
ids
);
deferred.resolve(ids);
});
return deferred.promise;
}
);
},
dl_transcripts: function(commandOptions, rawArgs, ui) { // eslint-disable-line no-unused-vars
return this.dl_documents_ids(commandOptions, rawArgs, ui).then(function(docs) {
return Q.all(_.map(docs, function(doc) {
var id = doc.id;
var deferred = Q.defer();
request.get({url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)) + '/transcript', json: true}, function (err, res, body) {
ui.writeLine(chalk.green('Download transcripts : getting transcript ' + id));
if (err) {
return deferred.reject(err);
} else if (res.statusCode === 404) {
deferred.resolve({'id': id, 'transcript': undefined});
} else if (res.statusCode !== 200) {
err = new Error('Unexpected status code: ' + res.statusCode);
err.res = res;
return deferred.reject(err);
} else {
deferred.resolve({'id': id, 'transcript': body});
}
});
return deferred.promise;
}));
}).then(function(res) {
var deferred = Q.defer();
var prefix = (this.format==='es6')?'export default ':'module.exports = ';
fs.writeFile(this.dest, prefix + JSON.stringify(res,null,2) + ';', function(err) {
if(err) {
return deferred.reject(err);
}
deferred.resolve();
});
return deferred.promise;
}.bind(this));
},
dl_documents: function(commandOptions, rawArgs, ui) { // eslint-disable-line no-unused-vars
var destFiles = {
docs: path.join(path.dirname(this.dest), 'details_'+path.basename(this.dest)),
list: this.dest
};
return this.dl_documents_ids(commandOptions, rawArgs, ui).then(function(docs) {
return Q.all(_.map(docs, function(doc) {
var deferred = Q.defer();
if(Object.keys(doc).length === 1) {
var id = doc.id;
request.get(
{
url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)),
qs: {short: true},
json: true
},
function (err, res, body) {
ui.writeLine(chalk.green('Download documents : completing doc ' + id));
if (err) {
return deferred.reject(err);
} else if (res.statusCode !== 200) {
err = new Error('Unexpected status code: ' + res.statusCode);
err.res = res;
return deferred.reject(err);
}
deferred.resolve(body.document);
}
);
}
else {
deferred.resolve(doc);
}
return deferred.promise;
}));
}).then(function(docs) {
var docsdeferred = Q.defer();
Q.all(_.map(docs, function(docu) {
var id = docu.id;
var deferred = Q.defer();
request.get({url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)), json: true}, function (err, res, body) {
ui.writeLine(chalk.green('Download documents : getting doc ' + id));
if (err) {
return deferred.reject(err);
} else if (res.statusCode !== 200) {
err = new Error('Unexpected status code: ' + res.statusCode);
err.res = res;
return deferred.reject(err);
}
deferred.resolve(body.document);
});
return deferred.promise;
})).then(
function(downloadedDocs) {
docsdeferred.resolve({list: docs, docs: downloadedDocs});
},
function(err) {
return docsdeferred.reject(err);
}
);
return docsdeferred.promise;
}).then(function(resList) {
var format = this.format;
return Q.all(_.map(resList, function(res, key) {
ui.writeLine(chalk.green('Writing ' + key + ' in file ' + destFiles[key]));
var deferred = Q.defer();
var prefix = (format==='es6')?'export default ':'module.exports = ';
fs.writeFile(destFiles[key], prefix + JSON.stringify(res,null,2) + ';', function(err) {
if(err) {
return deferred.reject(err);
}
deferred.resolve();
});
return deferred.promise;
}));
}.bind(this));
},
run: function(commandOptions, rawArgs) { // eslint-disable-line no-unused-vars
var type = commandOptions.type || 'documents';
this.dest = commandOptions.dest || '.' + path.sep + type + '.js';
this.format = commandOptions.format || 'es6';
return this['dl_' + type](commandOptions, rawArgs, this.ui);
}
});