--- a/common/corpus-common-addon/lib/commands/dl-fixtures.js Sun May 29 16:50:17 2016 +0200
+++ b/common/corpus-common-addon/lib/commands/dl-fixtures.js Mon May 30 00:32:20 2016 +0200
@@ -76,11 +76,9 @@
dl_themes: function(commandOptions) {
return this.dl_countmap(commandOptions, 'themes');
},
- dl_documents: function(commandOptions, rawArgs) { // eslint-disable-line no-unused-vars
-
- var ids = [];
+ dl_documents_ids: function(commandOptions, rawArgs, ids) {
+ var nextPageUrl = commandOptions.url;
var pageIndex = 1;
- var nextPageUrl = commandOptions.url;
return promiseWhile(
function() { return pageIndex <= commandOptions.page && nextPageUrl; },
@@ -109,10 +107,51 @@
});
return deferred.promise;
}
- ).then(function() {
+ );
+ },
+ dl_transcripts: function(commandOptions, rawArgs) { // eslint-disable-line no-unused-vars
+
+ var ids = [];
+
+ return this.dl_documents_ids(commandOptions, rawArgs, ids).then(function() {
return Q.all(_.map(ids, function(id) {
var deferred = Q.defer();
- request.get({url: commandOptions.url + id, json: true}, function (err, res, body) {
+ request.get({url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)) + '/transcript', json: true}, function (err, res, body) {
+ if (err) {
+ return deferred.reject(err);
+ } else if (res.statusCode === 404) {
+ deferred.resolve({'id': id, 'transcript': undefined});
+ } else if (res.statusCode !== 200) {
+ err = new Error('Unexpected status code: ' + res.statusCode);
+ err.res = res;
+ return deferred.reject(err);
+ } else {
+ deferred.resolve({'id': id, 'transcript': body});
+ }
+ });
+ return deferred.promise;
+ }));
+ }).then(function(res) {
+ var deferred = Q.defer();
+ var prefix = (this.format==='es6')?'export default ':'module.exports = ';
+ fs.writeFile(this.dest, prefix + JSON.stringify(res,null,2) + ';', function(err) {
+ if(err) {
+ return deferred.reject(err);
+ }
+ deferred.resolve();
+ });
+ return deferred.promise;
+ }.bind(this));
+
+ },
+ dl_documents: function(commandOptions, rawArgs) { // eslint-disable-line no-unused-vars
+
+ var ids = [];
+
+ return this.dl_documents_ids(commandOptions, rawArgs, ids).then(function() {
+ return Q.all(_.map(ids, function(id) {
+ var deferred = Q.defer();
+ request.get({url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)), json: true}, function (err, res, body) {
if (err) {
return deferred.reject(err);
} else if (res.statusCode !== 200) {