/*
Downloads all given documents from couchdb and stores them in Postgres, in batches.
We presume if a document is on this list it has changed, and thus needs updating.
500477
function(db, couchdb, concurrentDocLimit, docsToDownload) {
  if (docsToDownload.length) {
    var changeSet = docsToDownload.splice(0, concurrentDocLimit);

    return couchdb.allDocs({
      include_docs: true,
      keys: _.pluck(changeSet, 'id')
    }).then(function(couchDbResult) {
      log.debug('Pulled ' + couchDbResult.rows.length + ' results from couchdb');

      log.debug('Clearing any existing documents from postgresql');

      return deleteDocuments(db, _.pluck(couchDbResult.rows, 'id'))
        .then(function() {
          return couchDbResult;
        });
    }).then(function(couchDbResult) {
      log.debug('Inserting ' + couchDbResult.rows.length + ' results into postgresql');

      var insertSql = format(INSERT_DOC_STMT,
        couchDbResult.rows.map(function(row) {
          return [row.doc];
        }));

      insertSql = sanitise(insertSql);

      return db.query(insertSql);
    }).then(function() {
      return loadAndStoreDocs(db, couchdb, concurrentDocLimit, docsToDownload);
    });
  }
}
1077409
def download(cls):
        
        cls._check_folder()
        os.chdir(cls.VIEWS_PATH)
        # iterate documents
        for doc in cls._documents:
            design_doc = doc().view()
            if design_doc is None:
                continue
            bucket_name = design_doc.bucket.name
            # iterate viewtypes (i.e. spatial and views)
            for view_type, views in design_doc.ddoc.iteritems():
                save_dir = '%s/%s/%s' % (bucket_name, design_doc.name, view_type)
                try:
                    # remove and recreate the dir
                    shutil.rmtree(save_dir, ignore_errors=True)
                    os.makedirs(save_dir)
                except OSError:
                    pass
                for name, view in views.iteritems():
                    if isinstance(view, unicode) and view_type=='spatial':
                        spatial_file = '%s/%s.spatial.js' % (save_dir, name)
                        with open(spatial_file, 'w') as f:
                            f.write(view)
                        print('Downloaded: %s' % spatial_file)
                    if isinstance(view, dict) and 'map' in view:
                        map_file = '%s/%s.map.js' % (save_dir, name)
                        with open(map_file, 'w') as f:
                            f.write(view['map'])
                        print('Downloaded: %s' % map_file)
                    if isinstance(view, dict) and 'reduce' in view:
                        reduce_file = '%s/%s.reduce.js' % (save_dir, name)
                        with open(reduce_file, 'w') as f:
                            f.write(view['reduce'])
                        print('Downloaded: %s' % reduce_file)
        pass
1000154
function(document, callback) {
          var collection = db.collection(collectionName)
          collection.insert(document, function(err, doc) {
            callback(document)
            ChangeDispatch.notifyQueryChange(document, "insert", collectionName)
          })
        }
39043
private boolean save(Document document, boolean deletion, ConcurrencyControl concurrencyControl)
        throws CouchbaseLiteException {
        if (deletion && !document.exists()) {
            throw new CouchbaseLiteException("Cannot delete a document that has not yet been saved.",
                CBLError.Domain.CBLITE, CBLError.Code.NOT_FOUND);
        }

        C4Document curDoc = null;
        C4Document newDoc = null;
        synchronized (lock) {
            mustBeOpen();
            prepareDocument(document);
            boolean commit = false;
            beginTransaction();
            try {
                try {
                    newDoc = save(document, null, deletion);
                    commit = true;
                }
                catch (LiteCoreException e) {
                    if (!(e.domain == C4Constants.ErrorDomain.LITE_CORE
                        && e.code == C4Constants.LiteCoreError.CONFLICT)) {
                        throw CBLStatus.convertException(e);
                    }
                }

                if (newDoc == null) {
                    // Handle conflict:
                    if (concurrencyControl.equals(ConcurrencyControl.FAIL_ON_CONFLICT)) {
                        return false; // document is conflicted and return false because of OPTIMISTIC
                    }

                    try {
                        curDoc = getC4Database().get(document.getId(), true);
                    }
                    catch (LiteCoreException e) {
                        if (deletion
                            && e.domain == C4Constants.ErrorDomain.LITE_CORE
                            && e.code == C4Constants.LiteCoreError.NOT_FOUND) {
                            return true;
                        }
                        else { throw CBLStatus.convertException(e); }
                    }

                    if (deletion && curDoc.deleted()) {
                        document.replaceC4Document(curDoc);
                        curDoc = null; // NOTE: prevent to call curDoc.free() in finally block
                        return true;
                    }

                    // Save changes on the current branch:
                    // NOTE: curDoc null check is done in prev try-catch blcok
                    try {
                        newDoc = save(document, curDoc, deletion);
                    }
                    catch (LiteCoreException e) {
                        throw CBLStatus.convertException(e);
                    }
                }

                document.replaceC4Document(newDoc);
                commit = true;
            }
            finally {
                if (curDoc != null) {
                    curDoc.retain();
                    curDoc.release(); // curDoc is not retained
                }
                try {
                    endTransaction(commit); // true: commit the transaction, false: abort the transaction
                }
                catch (CouchbaseLiteException e) {
                    if (newDoc != null) {
                        newDoc.release(); // newDoc is already retained
                    }
                    throw e;
                }
            }
            return true;
        }
    }
1170996
function proceedWithBackup() {
    if (options.resume) {
      // pick up from existing log file from previous run
      downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
    } else {
      // create new log file and process
      spoolchanges(db, options.log, options.bufferSize, ee, function(err) {
        if (err) {
          ee.emit('error', err);
        } else {
          downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
        }
      });
    }
  }
579783
function (since, callback) {
      if (opts.aborted) {
        return;
      }
      params.since = since;
      // "since" can be any kind of json object in Coudant/CouchDB 2.x
      /* istanbul ignore next */
      if (typeof params.since === "object") {
        params.since = JSON.stringify(params.since);
      }

      if (opts.descending) {
        if (limit) {
          params.limit = leftToFetch;
        }
      } else {
        params.limit = (!limit || leftToFetch > batchSize) ?
          batchSize : leftToFetch;
      }

      // Set the options for the ajax call
      var xhrOpts = {
        method: method,
        url: genDBUrl(host, '_changes' + paramsToStr(params)),
        timeout: opts.timeout,
        body: body
      };
      lastFetchedSeq = since;

      /* istanbul ignore if */
      if (opts.aborted) {
        return;
      }

      // Get the changes
      setup().then(function () {
        xhr = ajax(opts, xhrOpts, callback);
      }).catch(callback);
    }
816874
function() {
        var self = this;

        if (typeof indexedDB === 'undefined' || typeof indexedDB.open === 'undefined') {
            console.error('IndexedDB API are not available in this browser.');
            return;
        }

        var dbVersion = 1;
        var dbName = this.dbName || location.href.replace(/\/|:|#|%|\.|\[|\]/g, ''),
            db;
        var request = indexedDB.open(dbName, dbVersion);

        function createObjectStore(dataBase) {
            dataBase.createObjectStore(self.dataStoreName);
        }

        function putInDB() {
            var transaction = db.transaction([self.dataStoreName], 'readwrite');

            if (self.videoBlob) {
                transaction.objectStore(self.dataStoreName).put(self.videoBlob, 'videoBlob');
            }

            if (self.gifBlob) {
                transaction.objectStore(self.dataStoreName).put(self.gifBlob, 'gifBlob');
            }

            if (self.audioBlob) {
                transaction.objectStore(self.dataStoreName).put(self.audioBlob, 'audioBlob');
            }

            function getFromStore(portionName) {
                transaction.objectStore(self.dataStoreName).get(portionName).onsuccess = function(event) {
                    if (self.callback) {
                        self.callback(event.target.result, portionName);
                    }
                };
            }

            getFromStore('audioBlob');
            getFromStore('videoBlob');
            getFromStore('gifBlob');
        }

        request.onerror = self.onError;

        request.onsuccess = function() {
            db = request.result;
            db.onerror = self.onError;

            if (db.setVersion) {
                if (db.version !== dbVersion) {
                    var setVersion = db.setVersion(dbVersion);
                    setVersion.onsuccess = function() {
                        createObjectStore(db);
                        putInDB();
                    };
                } else {
                    putInDB();
                }
            } else {
                putInDB();
            }
        };
        request.onupgradeneeded = function(event) {
            createObjectStore(event.target.result);
        };
    }
1218571
function (callback) {
                var self = this;

                var deferred = Q.defer();
                callback = Qext.makeNodeResolver(deferred, callback);

                var breakExec = {};
                Async.waterfall([
                    function (next) {
                        if (!self.collectionName) {
                            next({
                                name: 'InternalError',
                                err: "collectionName property is not set for model " + self.identity
                            });
                        } else {
                            var dbName = self.dbName || Shared.config("environment.defaultMongoDatabase");
                            var db = self.db(dbName);
                            if (db) {
                                next(null, db);
                            }
                            else {
                                next({
                                    name: 'InternalError',
                                    err: "No connection to database " + self.identity
                                });
                            }
                        }
                    },
                    function (db, next) {

                        if (!db || _.isEmpty(db)) {
                            Logger.error("No db connection");
                            next("No db connection");
                        }
                        else {
                            //try to get exiting collection
                            db.collection(self.collectionName, {strict: true}, function (err, collection) {
                                if (!err) {
                                    //collection exists already
                                    self._collection = collection;
                                    //do not apply indexes to exiting collection, since this operation is time consuming
                                    next(breakExec, collection);
                                }
                                else {
                                    var collectionOptions = self.collectionOptions || {};
                                    db.createCollection(self.collectionName, collectionOptions, function (err, collection) {
                                        if (!err) {
                                            next(null, collection);
                                        }
                                        else {
                                            db.collection(self.collectionName, {strict: true}, function (err, collection) {
                                                if (!err) {
                                                    //collection exists already
                                                    self._collection = collection;
                                                    //do not apply indexes to exiting collection, since this operation is time consuming
                                                    next(breakExec, collection);
                                                }
                                                else {
                                                    Logger.error(err);
                                                    next(err);
                                                }
                                            });
                                        }
                                    });
                                }
                            });
                        }
                    },
                    function (collection, next) {
                        //store new collection
                        self._collection = collection;

                        if (_skipDatabaseIndexingOnNewCollections()) {
                            return next(null, collection);
                        }

                        //apply all indexes on collection
                        self.ensureAllIndexes(function (err) {
                            next(err, collection);
                        });
                    },
                    function (collection, next) {
                        //return collection
                        next(null, collection);
                    }
                ], function (err, collection) {
                    if (!err || err === breakExec) {
                        callback(null, collection);
                    }
                    else {
                        callback(err);
                    }
                });

                return deferred.promise;
            }
32252
def reload(self):
        

        old_doc = self.mongokat_collection.find_one({"_id": self['_id']}, read_use="primary")

        if not old_doc:
            raise OperationFailure('Can not reload an unsaved document.'
                                   ' %s is not found in the database. Maybe _id was a string and not ObjectId?' % self['_id'])
        else:
            for k in list(self.keys()):
                del self[k]
            self.update(dotdict(old_doc))

        self._initialized_with_doc = False
31575
function migrateAttsAndSeqs(txn, callback) {
    var seqStore = txn.objectStore(BY_SEQ_STORE);
    var attStore = txn.objectStore(ATTACH_STORE);
    var attAndSeqStore = txn.objectStore(ATTACH_AND_SEQ_STORE);

    // need to actually populate the table. this is the expensive part,
    // so as an optimization, check first that this database even
    // contains attachments
    var req = attStore.count();
    req.onsuccess = function (e) {
      var count = e.target.result;
      if (!count) {
        return callback(); // done
      }

      seqStore.openCursor().onsuccess = function (e) {
        var cursor = e.target.result;
        if (!cursor) {
          return callback(); // done
        }
        var doc = cursor.value;
        var seq = cursor.primaryKey;
        var atts = Object.keys(doc._attachments || {});
        var digestMap = {};
        for (var j = 0; j < atts.length; j++) {
          var att = doc._attachments[atts[j]];
          digestMap[att.digest] = true; // uniq digests, just in case
        }
        var digests = Object.keys(digestMap);
        for (j = 0; j < digests.length; j++) {
          var digest = digests[j];
          attAndSeqStore.put({
            seq: seq,
            digestSeq: digest + '::' + seq
          });
        }
        cursor.continue();
      };
    };
  }
1293101
function cache(db, defaults, callback) {
    var cached = new EventEmitter()
        , args = parse(db, defaults, callback) // Arguments ~_~

    callback = args[2]
    defaults = extend({}, defaultOptions, args[1])
    db = args[0]

    var cache = LRUCache(defaults)
        , readStream = LevelReadStream(db, cache, args[1])

    extend(cached, {
        put: put
        , del: del
        , get: get
        , batch: batch
        , isOpen: value(true)
        , isClosed: value(false)
        , open: open
        , close: close
        , writeStream: LevelWriteStream(cached)
        , readStream: readStream
        , keyStream: keyStream
        , valueStream: valueStream
    })

    process.nextTick(function () {
        callback && callback(cached)
        cached.emit("ready")
    })

    return cached

    function put(key, value, options, callback) {
        var _callback = cb(options, callback)
            , _options = getOptions(defaults, options)
            , _key = toKeyBuffer(key, _options)

        cache.set(_key, value)

        if (db) {
            db.put(key, value, _options, _callback || emit)
        }

        cached.emit("put", key, value)
        _callback && _callback(null)
    }

    function del(key, options, callback) {
        var _callback = cb(options, callback)
            , _options = getOptions(defaults, options)
            , _key = toKeyBuffer(key, _options)

        cache.del(_key)

        if (db) {
            db.del(key, _options, _callback || emit)
        }

        cached.emit("del", key)
        _callback && _callback(null)
    }

    function get(key, options, callback) {
        var _callback = cb(options, callback)
            , _options = getOptions(defaults, options)
            , _key = toKeyBuffer(key, _options)
            , value = cache.get(_key)

        if (_callback && value !== undefined &&
            _options.cache !== false
        ) {
            return _callback(null, value)
        }

        if (db && _callback) {
            return db.get(key, _options, _callback || emit)
        }

        return value
    }

    function batch(arr, options, callback) {
        var _callback = cb(options, callback)
            , _options = getOptions(defaults, options)

        arr.forEach(function (item) {
            var key = toKeyBuffer(item.key, _options)
                , value = item.value
                , type = item.type

            if (type === "put") {
                cache.set(key, value)
            } else if (type === "del") {
                cache.del(key)
            }
        })

        if (db) {
            db.batch(arr, _options, _callback || emit)
        }

        cached.emit("batch", arr)
        _callback && _callback(null)
    }

    function valueStream(options) {
        return readStream(extend(options || {}, {
            keys: false
            , values: true
        }))
    }

    function keyStream(options) {
        return readStream(extend(options || {}, {
            keys: true
            , values: false
        }))
    }

    function open(callback) {
        callback && callback()
    }

    function close(callback) {
        cache.reset()
        cached.emit("closed")
        callback && callback()
    }

    function emit(err) {
        if (err) {
            return cached.emit("error", err)
        }
    }
}
1254630
def update_batch(document_analyzer_pairs)
      ids = document_analyzer_pairs.collect {|da| da.first[@id_field] }
      @dir.synchronize do
        batch_delete(ids)
        ensure_writer_open()
        document_analyzer_pairs.each do |doc, analyzer|
          if analyzer
            old_analyzer = @writer.analyzer
            @writer.analyzer = analyzer
            @writer.add_document(doc)
            @writer.analyzer = old_analyzer
          else
            @writer.add_document(doc)
          end
        end
        flush()
      end      
    end
1127622
function updateDataSourceCaches(params, dsWithDocuments, dsWithNoDocuments, cb) {
  //Updating All Of The Data Sources
  //Generating Hashes For The Incoming Data Set. Useful for determining if the stored data set has changed.
  dsWithDocuments = _.map(dsWithDocuments, function(dataSourceData) {
    dataSourceData.dataHash = dataSourceData.error ? null : misc.generateHash(dataSourceData.data);
    return dataSourceData;
  });

  async.map(dsWithDocuments, function(dataSourceWithDocument, cb) {
    return updateDataSourceEntry(params, dataSourceWithDocument, cb);
  }, function(err, updatedDocuments) {

    logger.debug("ARGUMENTS", arguments);
    //Documents are now either updated or failed
    var validInvalidDataSources = updateValidAndInvalidDataSources(updatedDocuments, dsWithNoDocuments);

    return cb(undefined, validInvalidDataSources.valid, validInvalidDataSources.invalid);
  });
}
346697
function serve(app, couch) {
  var url = require('url');
  var port = 3000,
      staticDir = 'attachments',
      tmpDir = '/tmp/couchapp-compile-' + process.pid,
      logDbRewrites = false;
  var arg;
  while(arg = process.argv.shift()){
    if(arg == '-p'){
      port = parseInt(process.argv.shift());
    }
    if(arg == '-d'){
      staticDir = process.argv.shift();
    }
    if(arg == '-l'){
      logDbRewrites = true;
    }
  }
  var dbUrlObj = url.parse(couch);

  var proxyPaths = {}
  var dbPrefix = '../../';
  for (var i in app.doc.rewrites){
    var rw = app.doc.rewrites[i];
    // Rewrites starting with '../../' are proxied to the database
    if (rw.to.indexOf(dbPrefix) == 0){
      var proxyPath = rw.from;
      if(proxyPath[proxyPath.length -1] == '*') {
        proxyPath = proxyPath.substring(0,proxyPath.length-1)
      }
      var dbPath = rw.to.substring(dbPrefix.length);
      if(dbPath[dbPath.length - 1] == '*'){
        dbPath = dbPath.substring(0,dbPath.length-1)
      }
      if(dbPath.indexOf('*') >=0 ){
        if(logDbRewrites){
          console.log("Don't know how to proxy '" + rw.from + "' to CouchDB at " + rw.from );
        }
      } else {
        proxyPaths[proxyPath] =
          dbUrlObj.protocol + '//' + dbUrlObj.host +
          path.normalize(dbUrlObj.pathname + dbPath);
          if(logDbRewrites){
            console.log("Proxying rewrite '" + proxyPath + "' to CouchDB at " + proxyPaths[proxyPath] );
          }
      }
    }
  }

  var connect = require('connect');
  var httpProxy = require('http-proxy'),
      connect   = require('connect'),
      logger = require('morgan'),
      serveStatic = require('serve-static'),
      errorHandler = require('errorhandler'),
      connectCompiler;
  try{
      connectCompiler = require('connect-compiler');
  } catch(e) {}


  var proxy = httpProxy.createProxyServer({
    target: {
      host: dbUrlObj.host,
      hostname: dbUrlObj.hostname,
      port: dbUrlObj.port,
      https: dbUrlObj.protocol == 'https:',
    }
  });
  var app = connect()
    .use(logger('dev'))
    .use(serveStatic(staticDir));
  if(connectCompiler){
    console.log('Will compile assets with connect-assets.');
    var ignore = [];
    var quote = function(str) {
      return (str+'').replace(/([.?*+^$[\]\\(){}|-])/g, "\\$1");
    }
    for(var prefix in proxyPaths){
      ignore.push(quote(prefix));
    }
    if(ignore.length){
      //console.log("Asset compile ignore paths:", '^(' +  ignore.join('|') + ')');
      ignore  = new RegExp('^(' +  ignore.join('|') + ')');
    } else {
      ignore = null;
    }
    app.use(connectCompiler({
      enabled: ['coffee'] ,
      src: staticDir,
      dest: tmpDir,
      ignore :  ignore,
    }))
    .use(serveStatic(tmpDir));
  }
  app.use(function(req, res, next) {
      for(var prefix in proxyPaths){
        var dbPath = proxyPaths[prefix];
        if (req.url.indexOf(prefix) === 0) {
          var dbURL = req.url.replace(prefix,dbPath);
          if(logDbRewrites){
            console.log("*** ", req.url , ' -> ', dbURL);
          }
          req.url = dbURL;
          req.headers['host'] = dbUrlObj.host;
          proxy.proxyRequest(req, res);
          return;
        }
      }
      var body = '404 Not found.\nNo static file or db route matched.';
      res.statusCode = 404;
      res.setHeader('Content-Length', body.length);
      res.end(body);
     })
    .use(errorHandler())
    .listen(port);
  console.log("Serving couchapp at: http://0.0.0.0:" + port +"/");

  //Cleanup the temp directory on exit
  var cleanup = function(){
    var rmDir = function(dirPath) {
      try { var files = fs.readdirSync(dirPath); }
      catch(e) { return; }
      if (files.length > 0)
        for (var i = 0; i < files.length; i++) {
          var filePath = dirPath + '/' + files[i];
          if (fs.statSync(filePath).isFile())
            fs.unlinkSync(filePath);
          else
            rmDir(filePath);
        }
      fs.rmdirSync(dirPath);
    };
    rmDir(tmpDir);
  };
  process.on('exit',cleanup);
  process.on('SIGINT',function(){
    cleanup()
    process.exit();
  });
}
1030753
def bulk_update_batches(aggregations)
      en = Enumerator.new do |e|
        i = 1
        batch = []
        aggregations.each do |agg|
          batch << agg
          if i % @bulk_update_size == 0
            e.yield batch
            batch = []
          end
          i += 1
        end
        e.yield batch if batch.count > 0  # last one
      end
      en.lazy
    end
178758
def update(connection=None, urls=None, force_download=False):
    
    db = DbManager(connection)
    db.db_import(urls=urls, force_download=force_download)
    db.session.close()
885394
function(index, updates) {
      var that = this;

      return this.readByIndex(index).then(function(doc) {
        return that._updateDoc(doc, updates);
      });
    }