common/corpus-common-addon/lib/commands/dl-fixtures.js
author ymh <ymh.work@gmail.com>
Thu, 02 Jun 2016 18:24:19 +0200
changeset 168 17f10b56c079
parent 166 ce580300e4b9
child 173 cf7b221238fd
permissions -rw-r--r--
improve document model and propagate changes. This include the change of document fixtures to better reflect what the api is effectively returning

/*eslint-env node */
/*global require:true*/
'use strict';

var Command = require('ember-cli/lib/models/command');
var path    = require('path');
var Q       = require('q');
var request = require('request');
var _       = require('lodash');
var fs      = require('fs');
var chalk   = require('chalk');

// taken from http://stackoverflow.com/a/17238793
// `condition` is a function that returns a boolean
// `body` is a function that returns a promise
// returns a promise for the completion of the loop
function promiseWhile(condition, body) {

    var done = Q.defer();

    function loop(res) {
        // When the result of calling `condition` is no longer true, we are
        // done.
        if (!condition()) return done.resolve(res);
        // Use `when`, in case `body` does not return a promise.
        // When it completes loop again otherwise, if it fails, reject the
        // done promise
        Q.when(body(), loop, done.reject);
    }

    // Start running the loop in the next tick so that this function is
    // completely async. It would be unexpected if `body` was called
    // synchronously the first time.
    Q.nextTick(loop);

    // The promise
    return done.promise;
}

module.exports = Command.extend({
    name: 'dl-fixtures',
    description: 'Download fixtures',
    works: 'everywhere',
    availableOptions: [
        { name: 'type', type: String, default: 'documents', aliases: ['t'], description: 'type of obejcts to downloads' },
        { name: 'url',  type: String, aliases: ['u'], description: 'Source url' },
        { name: 'dest', type: String, aliases: ['d'], description: 'File destination'  },
        { name: 'page', type: Number, default: 1 , aliases: ['p'], description: 'number of page to download'},
        { name: 'format', type: String, default: 'es6' , aliases: ['f'], description: 'Format for module export, es6 or require'},
        { name: 'extra', type: String, default: '', aliases: ['e'], description: 'additionnal data to download (comma separated list of document ids)'}
    ],
    dl_countmap: function(commandOptions, rootKey) {
        var done = Q.defer();
        var that = this;
        request.get({url: commandOptions.url, json: true}, function (err, res, body) {
            var objectList = _.reduce(
                _.keys(body[rootKey]),
                function(res, objKey) {
                    var objValue = body[rootKey][objKey];
                    res.push({id: objKey, count: objValue.count, label: objValue.label});
                    return res;
                },
                []
            );
            var prefix = (that.format==='es6')?'export default ':'module.exports = ';
            fs.writeFile(that.dest, prefix + JSON.stringify(objectList,null,2) + ';', function(err) {
                if(err) {
                    return done.reject(err);
                }
                done.resolve();
            });
        });
        return done.promise;
    },
    dl_discourses: function(commandOptions) {
        return this.dl_countmap(commandOptions, 'discourses');
    },
    dl_themes: function(commandOptions) {
        return this.dl_countmap(commandOptions, 'themes');
    },
    dl_documents_ids: function(commandOptions, rawArgs, ui) {

        var nextPageUrl = commandOptions.url;
        var pageIndex = 1;
        var ids = commandOptions.extra?_.map(commandOptions.extra.split(','), function(id) { return { id: id};}):[];

        return promiseWhile(
            function() { return pageIndex <= commandOptions.page && nextPageUrl; },
            function() {
                var deferred = Q.defer();
                ui.writeLine(chalk.yellow('Download Documents : getting page ' + pageIndex));

                request.get({url: nextPageUrl, json: true}, function (err, res, body) {
                    if (err) {
                        return deferred.reject(err);
                    } else if (res.statusCode !== 200) {
                        err = new Error('Unexpected status code: ' + res.statusCode);
                        err.res = res;
                        return deferred.reject(err);
                    }
                    nextPageUrl = body.next_page_url;
                    pageIndex++;

                    ids = _.reduce(
                        body.documents,
                        function(res, doc) {
                            res.push(doc);
                            return res;
                        },
                        ids
                    );
                    deferred.resolve(ids);
                });
                return deferred.promise;
            }
        );
    },
    dl_transcripts: function(commandOptions, rawArgs, ui) {  // eslint-disable-line no-unused-vars

        return this.dl_documents_ids(commandOptions, rawArgs, ui).then(function(docs) {
            return Q.all(_.map(docs, function(doc) {
                var id = doc.id;
                var deferred = Q.defer();
                request.get({url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)) + '/transcript', json: true}, function (err, res, body) {
                    ui.writeLine(chalk.green('Download transcripts : getting transcript ' + id));
                    if (err) {
                        return deferred.reject(err);
                    } else if (res.statusCode === 404) {
                        deferred.resolve({'id': id, 'transcript': undefined});
                    } else if (res.statusCode !== 200) {
                        err = new Error('Unexpected status code: ' + res.statusCode);
                        err.res = res;
                        return deferred.reject(err);
                    } else {
                        deferred.resolve({'id': id, 'transcript': body});
                    }
                });
                return deferred.promise;
            }));
        }).then(function(res) {
            var deferred = Q.defer();
            var prefix = (this.format==='es6')?'export default ':'module.exports = ';
            fs.writeFile(this.dest, prefix + JSON.stringify(res,null,2) + ';', function(err) {
                if(err) {
                    return deferred.reject(err);
                }
                deferred.resolve();
            });
            return deferred.promise;
        }.bind(this));

    },
    dl_documents: function(commandOptions, rawArgs, ui) { // eslint-disable-line no-unused-vars

        var destFiles = {
            docs: path.join(path.dirname(this.dest), 'details_'+path.basename(this.dest)),
            list: this.dest
        };

        return this.dl_documents_ids(commandOptions, rawArgs, ui).then(function(docs) {
            return Q.all(_.map(docs, function(doc) {
                var deferred = Q.defer();

                if(Object.keys(doc).length === 1) {
                    var id = doc.id;
                    request.get(
                        {
                            url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)),
                            qs: {short: true},
                            json: true
                        },
                        function (err, res, body) {
                            ui.writeLine(chalk.green('Download documents : completing doc ' + id));
                            if (err) {
                                return deferred.reject(err);
                            } else if (res.statusCode !== 200) {
                                err = new Error('Unexpected status code: ' + res.statusCode);
                                err.res = res;
                                return deferred.reject(err);
                            }
                            deferred.resolve(body.document);
                        }
                    );
                }
                else {
                    deferred.resolve(doc);
                }

                return deferred.promise;
            }));
        }).then(function(docs) {
            var docsdeferred = Q.defer();
            Q.all(_.map(docs, function(docu) {
                var id = docu.id;
                var deferred = Q.defer();
                request.get({url: commandOptions.url + encodeURIComponent(encodeURIComponent(id)), json: true}, function (err, res, body) {
                    ui.writeLine(chalk.green('Download documents : getting doc ' + id));
                    if (err) {
                        return deferred.reject(err);
                    } else if (res.statusCode !== 200) {
                        err = new Error('Unexpected status code: ' + res.statusCode);
                        err.res = res;
                        return deferred.reject(err);
                    }
                    deferred.resolve(body.document);
                });
                return deferred.promise;
            })).then(
                function(downloadedDocs) {
                    docsdeferred.resolve({list: docs, docs: downloadedDocs});
                },
                function(err) {
                    return docsdeferred.reject(err);
                }
            );

            return docsdeferred.promise;

        }).then(function(resList) {
            var format = this.format;
            return Q.all(_.map(resList, function(res, key) {
                ui.writeLine(chalk.green('Writing ' + key + ' in file ' + destFiles[key]));
                var deferred = Q.defer();
                var prefix = (format==='es6')?'export default ':'module.exports = ';
                fs.writeFile(destFiles[key], prefix + JSON.stringify(res,null,2) + ';', function(err) {
                    if(err) {
                        return deferred.reject(err);
                    }
                    deferred.resolve();
                });
                return deferred.promise;
            }));
        }.bind(this));
    },
    run: function(commandOptions, rawArgs) { // eslint-disable-line no-unused-vars
        var type = commandOptions.type || 'documents';
        this.dest = commandOptions.dest || '.' + path.sep + type + '.js';
        this.format = commandOptions.format || 'es6';


        return this['dl_' + type](commandOptions, rawArgs, this.ui);
    }
});