diff -r 59c68fc4848e -r 5f1e1cc17e8a common/corpus-common-addon/lib/commands/dl-fixtures.js --- a/common/corpus-common-addon/lib/commands/dl-fixtures.js Sun May 29 16:50:17 2016 +0200 +++ b/common/corpus-common-addon/lib/commands/dl-fixtures.js Mon May 30 00:32:20 2016 +0200 @@ -76,11 +76,9 @@ dl_themes: function(commandOptions) { return this.dl_countmap(commandOptions, 'themes'); }, - dl_documents: function(commandOptions, rawArgs) { // eslint-disable-line no-unused-vars - - var ids = []; + dl_documents_ids: function(commandOptions, rawArgs, ids) { + var nextPageUrl = commandOptions.url; var pageIndex = 1; - var nextPageUrl = commandOptions.url; return promiseWhile( function() { return pageIndex <= commandOptions.page && nextPageUrl; }, @@ -109,10 +107,51 @@ }); return deferred.promise; } - ).then(function() { + ); + }, + dl_transcripts: function(commandOptions, rawArgs) { // eslint-disable-line no-unused-vars + + var ids = []; + + return this.dl_documents_ids(commandOptions, rawArgs, ids).then(function() { return Q.all(_.map(ids, function(id) { var deferred = Q.defer(); - request.get({url: commandOptions.url + id, json: true}, function (err, res, body) { + request.get({url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)) + '/transcript', json: true}, function (err, res, body) { + if (err) { + return deferred.reject(err); + } else if (res.statusCode === 404) { + deferred.resolve({'id': id, 'transcript': undefined}); + } else if (res.statusCode !== 200) { + err = new Error('Unexpected status code: ' + res.statusCode); + err.res = res; + return deferred.reject(err); + } else { + deferred.resolve({'id': id, 'transcript': body}); + } + }); + return deferred.promise; + })); + }).then(function(res) { + var deferred = Q.defer(); + var prefix = (this.format==='es6')?'export default ':'module.exports = '; + fs.writeFile(this.dest, prefix + JSON.stringify(res,null,2) + ';', function(err) { + if(err) { + return deferred.reject(err); + } + deferred.resolve(); + }); + return deferred.promise; + }.bind(this)); + + }, + dl_documents: function(commandOptions, rawArgs) { // eslint-disable-line no-unused-vars + + var ids = []; + + return this.dl_documents_ids(commandOptions, rawArgs, ids).then(function() { + return Q.all(_.map(ids, function(id) { + var deferred = Q.defer(); + request.get({url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)), json: true}, function (err, res, body) { if (err) { return deferred.reject(err); } else if (res.statusCode !== 200) {