common/corpus-common-addon/lib/commands/dl-fixtures.js
author ymh <ymh.work@gmail.com>
Mon, 25 Apr 2016 13:21:42 +0200
changeset 156 7203efa0ddbf
parent 131 31a5c314d575
child 158 366509ae2f37
permissions -rw-r--r--
upgrade dependencies
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
130
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
'use strict';
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
     2
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
     3
var Command = require('ember-cli/lib/models/command');
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
     4
var chalk   = require('chalk');
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
var path    = require('path');
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
     6
var Q       = require("q");
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
var request = require("request");
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
     8
var _       = require('lodash');
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
     9
var fs      = require('fs');
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    10
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
// taken from http://stackoverflow.com/a/17238793
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    12
// `condition` is a function that returns a boolean
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    13
// `body` is a function that returns a promise
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    14
// returns a promise for the completion of the loop
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    15
function promiseWhile(condition, body) {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    16
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    17
    var done = Q.defer();
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    18
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    19
    function loop() {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    20
        // When the result of calling `condition` is no longer true, we are
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    21
        // done.
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    22
        if (!condition()) return done.resolve();
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    23
        // Use `when`, in case `body` does not return a promise.
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    24
        // When it completes loop again otherwise, if it fails, reject the
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    25
        // done promise
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    26
        Q.when(body(), loop, done.reject);
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    27
    }
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    28
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    29
    // Start running the loop in the next tick so that this function is
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    30
    // completely async. It would be unexpected if `body` was called
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    31
    // synchronously the first time.
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    32
    Q.nextTick(loop);
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    33
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    34
    // The promise
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    35
    return done.promise;
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    36
}
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    37
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    38
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    39
module.exports = Command.extend({
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    40
  name: 'dl-fixtures',
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    41
  description: "Download fixtures",
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    42
  works: 'everywhere',
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    43
  availableOptions: [
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    44
    { name: 'type', type: String, default: 'documents', aliases: ['t'], description: "type of obejcts to downloads" },
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    45
    { name: 'url',  type: String, aliases: ['u'], description: "Source url" },
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    46
    { name: 'dest', type: String, aliases: ['d'], description: "File destination"  },
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    47
    { name: 'page', type: Number, default: 1 , aliases: ['p'], description: "number of page to download"},
131
31a5c314d575 format is not the same for fixtures
ymh <ymh.work@gmail.com>
parents: 130
diff changeset
    48
    { name: 'format', type: String, default: 'es6' , aliases: ['f'], description: "Format for module export, es6 or require"},
130
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    49
  ],
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    50
  run: function(commandOptions, rawArgs) {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    51
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    52
    //TODO: check that commandOptions.url is not empty
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    53
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    54
    var dest = commandOptions.dest || "." + path.sep + commandOptions.type + ".js"
131
31a5c314d575 format is not the same for fixtures
ymh <ymh.work@gmail.com>
parents: 130
diff changeset
    55
    var format = commandOptions.format || 'es6';
130
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    56
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    57
    var ids = [];
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    58
    var pageIndex = 1;
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    59
    var nextPageUrl = commandOptions.url;
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    60
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    61
    return promiseWhile(
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    62
      function() { return pageIndex <= commandOptions.page && nextPageUrl },
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    63
      function() {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    64
        var deferred = Q.defer();
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    65
        request.get({url: nextPageUrl, json: true}, function (err, res, body) {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    66
          if (err) {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    67
              return deferred.reject(err);
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    68
          } else if (res.statusCode !== 200) {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    69
              err = new Error("Unexpected status code: " + res.statusCode);
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    70
              err.res = res;
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    71
              return deferred.reject(err);
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    72
          }
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    73
          nextPageUrl = body.next_page_url;
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    74
          pageIndex++;
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    75
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    76
          ids = _.reduce(
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    77
            body.documents,
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    78
            function(res, doc) {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    79
              res.push(doc.id);
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    80
              return res;
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    81
            },
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    82
            ids
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    83
          );
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    84
          deferred.resolve(ids);
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    85
        });
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    86
        return deferred.promise;
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    87
      }
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    88
    ).then(function() {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    89
      return Q.all(_.map(ids, function(id) {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    90
        var deferred = Q.defer();
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    91
        request.get({url: commandOptions.url + id, json: true}, function (err, res, body) {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    92
          if (err) {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    93
              return deferred.reject(err);
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    94
          } else if (res.statusCode !== 200) {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    95
              err = new Error("Unexpected status code: " + res.statusCode);
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    96
              err.res = res;
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    97
              return deferred.reject(err);
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    98
          }
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
    99
          deferred.resolve(body.document);
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   100
        });
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   101
        return deferred.promise;
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   102
      }));
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   103
    }).then(function(res) {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   104
      var deferred = Q.defer();
131
31a5c314d575 format is not the same for fixtures
ymh <ymh.work@gmail.com>
parents: 130
diff changeset
   105
      var prefix = (format==='es6')?'export default ':"module.exports = ";
31a5c314d575 format is not the same for fixtures
ymh <ymh.work@gmail.com>
parents: 130
diff changeset
   106
      fs.writeFile(dest, prefix + JSON.stringify(res,null,2) + ";", function(err) {
130
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   107
        if(err) {
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   108
          return deferred.reject(err);
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   109
        }
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   110
        deferred.resolve();
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   111
      });
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   112
      return deferred.promise;
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   113
    });
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   114
  }
fac22d8c2df8 add subjects to model + simple display on bo + add command to downloads documents to fixtures for test
ymh <ymh.work@gmail.com>
parents:
diff changeset
   115
});