6 var path = require('path'); |
6 var path = require('path'); |
7 var Q = require('q'); |
7 var Q = require('q'); |
8 var request = require('request'); |
8 var request = require('request'); |
9 var _ = require('lodash'); |
9 var _ = require('lodash'); |
10 var fs = require('fs'); |
10 var fs = require('fs'); |
|
11 var chalk = require('chalk'); |
11 |
12 |
12 // taken from http://stackoverflow.com/a/17238793 |
13 // taken from http://stackoverflow.com/a/17238793 |
13 // `condition` is a function that returns a boolean |
14 // `condition` is a function that returns a boolean |
14 // `body` is a function that returns a promise |
15 // `body` is a function that returns a promise |
15 // returns a promise for the completion of the loop |
16 // returns a promise for the completion of the loop |
75 return this.dl_countmap(commandOptions, 'discourses'); |
76 return this.dl_countmap(commandOptions, 'discourses'); |
76 }, |
77 }, |
77 dl_themes: function(commandOptions) { |
78 dl_themes: function(commandOptions) { |
78 return this.dl_countmap(commandOptions, 'themes'); |
79 return this.dl_countmap(commandOptions, 'themes'); |
79 }, |
80 }, |
80 dl_documents_ids: function(commandOptions, rawArgs, ids) { |
81 dl_documents_ids: function(commandOptions, rawArgs, ids, ui) { |
81 var nextPageUrl = commandOptions.url; |
82 var nextPageUrl = commandOptions.url; |
82 var pageIndex = 1; |
83 var pageIndex = 1; |
83 |
84 |
84 ids.push.apply(ids,commandOptions.extra.split(',')); |
85 ids.push.apply(ids,commandOptions.extra.split(',')); |
85 |
86 |
86 return promiseWhile( |
87 return promiseWhile( |
87 function() { return pageIndex <= commandOptions.page && nextPageUrl; }, |
88 function() { return pageIndex <= commandOptions.page && nextPageUrl; }, |
88 function() { |
89 function() { |
89 var deferred = Q.defer(); |
90 var deferred = Q.defer(); |
|
91 ui.writeLine(chalk.yellow('Download Documents : getting page ' + pageIndex)); |
|
92 |
90 request.get({url: nextPageUrl, json: true}, function (err, res, body) { |
93 request.get({url: nextPageUrl, json: true}, function (err, res, body) { |
91 if (err) { |
94 if (err) { |
92 return deferred.reject(err); |
95 return deferred.reject(err); |
93 } else if (res.statusCode !== 200) { |
96 } else if (res.statusCode !== 200) { |
94 err = new Error('Unexpected status code: ' + res.statusCode); |
97 err = new Error('Unexpected status code: ' + res.statusCode); |
110 }); |
113 }); |
111 return deferred.promise; |
114 return deferred.promise; |
112 } |
115 } |
113 ); |
116 ); |
114 }, |
117 }, |
115 dl_transcripts: function(commandOptions, rawArgs) { // eslint-disable-line no-unused-vars |
118 dl_transcripts: function(commandOptions, rawArgs, ui) { // eslint-disable-line no-unused-vars |
116 |
119 |
117 var ids = []; |
120 var ids = []; |
118 |
121 |
119 return this.dl_documents_ids(commandOptions, rawArgs, ids).then(function() { |
122 return this.dl_documents_ids(commandOptions, rawArgs, ids, ui).then(function() { |
120 return Q.all(_.map(ids, function(id) { |
123 return Q.all(_.map(ids, function(id) { |
121 var deferred = Q.defer(); |
124 var deferred = Q.defer(); |
122 request.get({url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)) + '/transcript', json: true}, function (err, res, body) { |
125 request.get({url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)) + '/transcript', json: true}, function (err, res, body) { |
|
126 ui.writeLine(chalk.green('Download transcripts : getting transcript ' + id)); |
123 if (err) { |
127 if (err) { |
124 return deferred.reject(err); |
128 return deferred.reject(err); |
125 } else if (res.statusCode === 404) { |
129 } else if (res.statusCode === 404) { |
126 deferred.resolve({'id': id, 'transcript': undefined}); |
130 deferred.resolve({'id': id, 'transcript': undefined}); |
127 } else if (res.statusCode !== 200) { |
131 } else if (res.statusCode !== 200) { |
145 }); |
149 }); |
146 return deferred.promise; |
150 return deferred.promise; |
147 }.bind(this)); |
151 }.bind(this)); |
148 |
152 |
149 }, |
153 }, |
150 dl_documents: function(commandOptions, rawArgs) { // eslint-disable-line no-unused-vars |
154 dl_documents: function(commandOptions, rawArgs, ui) { // eslint-disable-line no-unused-vars |
151 |
155 |
152 var ids = []; |
156 var ids = []; |
153 |
157 |
154 return this.dl_documents_ids(commandOptions, rawArgs, ids).then(function() { |
158 return this.dl_documents_ids(commandOptions, rawArgs, ids, ui).then(function() { |
155 console.log(ids); |
|
156 return Q.all(_.map(ids, function(id) { |
159 return Q.all(_.map(ids, function(id) { |
157 var deferred = Q.defer(); |
160 var deferred = Q.defer(); |
158 request.get({url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)), json: true}, function (err, res, body) { |
161 request.get({url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)), json: true}, function (err, res, body) { |
|
162 ui.writeLine(chalk.green('Download transcripts : getting doc ' + id)); |
159 if (err) { |
163 if (err) { |
160 return deferred.reject(err); |
164 return deferred.reject(err); |
161 } else if (res.statusCode !== 200) { |
165 } else if (res.statusCode !== 200) { |
162 err = new Error('Unexpected status code: ' + res.statusCode); |
166 err = new Error('Unexpected status code: ' + res.statusCode); |
163 err.res = res; |
167 err.res = res; |
184 var type = commandOptions.type || 'documents'; |
188 var type = commandOptions.type || 'documents'; |
185 this.dest = commandOptions.dest || '.' + path.sep + type + '.js'; |
189 this.dest = commandOptions.dest || '.' + path.sep + type + '.js'; |
186 this.format = commandOptions.format || 'es6'; |
190 this.format = commandOptions.format || 'es6'; |
187 |
191 |
188 |
192 |
189 return this['dl_' + type](commandOptions, rawArgs); |
193 return this['dl_' + type](commandOptions, rawArgs, this.ui); |
190 } |
194 } |
191 }); |
195 }); |