fine-tuned data engine. MongoDB legacy
341260
def _run_custom_data_engine(self, start_id_obj, end_id_obj, start_timeperiod, end_timeperiod):
        
        collection_name = context.process_context[self.process_name].source
        iteration = 0
        while True:
            cursor = self.ds.cursor_fine(collection_name,
                                         start_id_obj,
                                         end_id_obj,
                                         iteration,
                                         start_timeperiod,
                                         end_timeperiod)

            if iteration == 0 and cursor.count(with_limit_and_skip=True) == 0:
                msg = 'No entries in {0} at range [{1} : {2}]'.format(collection_name, start_id_obj, end_id_obj)
                self.logger.warning(msg)
                break

            start_id_obj = None
            for document in cursor:
                start_id_obj = document['_id']
                self._process_single_document(document)
                self.performance_tracker.increment_success()
            if start_id_obj is None:
                break
            iteration += 1

        self._cursor_exploited()
        msg = 'Cursor exploited after {0} iterations'.format(iteration)
        self.logger.info(msg)
197135
def make_db_data_fetcher(postgresql_conn_info, template_path, reload_templates,
                         query_cfg, io_pool):
    

    sources = parse_source_data(query_cfg)
    queries_generator = make_queries_generator(
        sources, template_path, reload_templates)
    return DataFetcher(
        postgresql_conn_info, queries_generator, io_pool)
432109
function(dbOrUri, stream, callback) {
    var db;
    if (arguments.length === 2) {
      callback = stream;
      stream = undefined;
    }
    if (!stream) {
      stream = process.stdin;
    }

    var state = 'start';
    var version;
    var collection;
    var endOfCollection;
    var insertQueue = [];
    var insertQueueSize = 0;

    // State machine

    var processors = {
      start: function(item, callback) {
        try {
          item = BSON.deserialize(item);
        } catch (e) {
          return callback(e);
        }
        if (typeof(item) !== 'object') {
          return callback(new Error('First BSON element is not an object'));
        }
        if (item.type !== 'mongo-dump-stream') {
          return callback(new Error('type property is not mongo-dump-stream'));
        }
        version = item.version;
        if (item.version > 2) {
          return callback(new Error('Incoming mongo-dump-stream is of a newer version, bailing out'));
        }
        endOfCollection = item.endOfCollection;
        state = 'collection';
        return callback(null);
      },
      collection: function(item, callback) {
        try {
          item = BSON.deserialize(item);
        } catch (e) {
          return callback(e);
        }
        if (item.type === 'endDatabase') {
          state = 'done';
          return callback(null);
        }
        if (item.type !== 'collection') {
          return callback(new Error('collection expected'));
        }
        var name = item.name;
        var indexes = item.indexes;
        async.series({
          create: function(callback) {
            return db.collection(name, function(err, _collection) {
              if (err) {
                return callback(err);
              }
              collection = _collection;
              return callback(null);
            });
          },
          indexes: function(callback) {
            return async.eachSeries(item.indexes, function(item, callback) {
              if (item.name === '_id_') {
                // automatic
                return setImmediate(callback);
              }
              return collection.ensureIndex(item.key, _.omit(item, 'ns', 'key'), callback);
            }, callback);
          }
        }, function(err) {
          if (err) {
            return callback(err);
          }
          state = 'documents';
          return callback(null);
        });
      },
      documents: function(item, callback) {
        if (version === '1') {
          item = BSON.deserialize(item);
          if (item.type === 'endCollection') {
            state = 'collection';
            return setImmediate(callback);
          }
          if (item.type !== 'document') {
            return callback(new Error('Document expected'));
          }
          return collection.insert(item.document, callback);
        }

        // Just scan for the unique random string that
        // appears in the end-of-collection object. No need
        // to waste time parsing BSON for this. Also
        // leverage the fact that it won't ever be
        // part of a large object to avoid scanning them all
        if (item.length < 100) {
          if (buffertools.indexOf(item, endOfCollection) !== -1) {
            return flush(function(err) {
              if (err) {
                return callback(err);
              }
              state = 'collection';
              return callback(null);
            });
          }
        }
        return async.series({
          flushIfNeeded: function(callback) {
            if (insertQueueSize + item.length <= 16777216) {
              return setImmediate(callback);
            }
            return flush(callback);
          },
          insertInQueue: function(callback) {
            insertQueueSize += item.length;
            insertQueue.push(item);
            return setImmediate(callback);
          }
        }, callback);

        function flush(callback) {
          if (!insertQueue.length) {
            // MongoDB considers it an error to be asked
            // to insert 0 documents ):
            return setImmediate(callback);
          }
          // watch out for race condition
          var insert = insertQueue;
          insertQueue = [];
          insertQueueSize = 0;
          return collection.insert(insert, function(err) {
            insertQueue = [];
            return callback(err);
          });
        }
      }
    };

    return async.series({
      connect: function(callback) {
        if (typeof(dbOrUri) !== 'string') {
          // Already a mongodb connection
          db = dbOrUri;
          return setImmediate(callback);
        }
        return mongodb.MongoClient.connect(dbOrUri, function(err, _db) {
          if (err) {
            return callback(err);
          }
          db = _db;
          return callback(null);
        });
      },
      dropCollections: function(callback) {
        return db.collections(function(err, collections) {
          if (err) {
            return callback(err);
          }
          return async.eachSeries(collections, function(item, callback) {
            if (item.collectionName.match(/^system\./)) {
              return setImmediate(callback);
            }
            return item.drop(callback);
          }, callback);
        });
      },
      loadCollections: function(finalCallback) {
        return reader({ raw: true, from: stream }, function(doc, callback) {
          return processors[state](doc, function(err) {
            if (err) {
              return callback(err);
            }
            if (state === 'done') {
              return finalCallback(null);
            }
            return callback(null);
          });
        }, function(err) {
          if (err) {
            return finalCallback(err);
          }
          return finalCallback(new Error('Premature end of stream'));
        });
      }
    }, callback);
  }
690860
function(db) {

		function get_documents(data, config, plv8, ERROR, WARNING) {

			function get_document(id, fields) {
				fields = (fields || [{'query':'*'}]);

				var map = {};

				var obj = plv8.execute("SELECT "+fields.map(function(f, i) {
					var k;
					if(f.key) {
						k = 'p__' + i;
						map[k] = f;
						return f.query + ' AS ' + k;
					} else {
						return f.query;
					}
				}).join(', ')+" FROM documents WHERE id = $1 LIMIT 1", [id])[0];

				if(!obj) {
					return obj;
				}

				Object.keys(obj).forEach(function(key) {
					if(!map.hasOwnProperty(key)) {
						return;
					}

					var spec = map[key];
					if(!is_object(spec)) {
						return error('mapped resource missing');
					}
					if(!is_string(spec.key)) {
						return error('key missing');
					}
					var value = obj[key];
					delete obj[key];

					if(is_string(spec.datakey)) {
						if(!obj[spec.datakey]) {
							obj[spec.datakey] = {};
						}
						obj[spec.datakey][spec.key] = value;
					} else {
						obj[spec.key] = value;
					}
				});

				return obj;
			}

			//function get_type(id) {
			//	return plv8.execute("SELECT * FROM types WHERE id = $1 LIMIT 1", [id])[0];
			//}

			function error(msg) {
				plv8.elog(ERROR, msg);
			}

			function warn(msg) {
				plv8.elog(WARNING, msg);
			}

			function is_object(a) {
				return a && (typeof a === 'object');
			}

			function is_array(a) {
				return a && (a instanceof Array);
			}

			function is_string(a) {
				return a && (typeof a === 'string');
			}

			function is_uuid(obj) {
				return /^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$/.test(obj);
			}

			function get_property_step(last, current) {
				return is_object(last) ? last[current] : undefined;
			}

			/** Get value of property from provided object using a path.
			 * @param obj {object} The object from where to find the value.
			 * @param path {string} The path to the value in object.
			 * @returns {mixed} The value from the object at the end of path or `undefined` if any part is missing.
			 */
			function get_property(obj, path) {
				if(!is_object(obj)) { return error('get_property(obj, ...) not object: '+ obj); }
				if(!is_string(path)) { return error('get_property(..., path) not string: '+ path); }
				return path.split('.').reduce(get_property_step, obj);
			}

			/** Get document by UUID */
			function fetch_object_by_uuid(data, prop, fields, uuid) {
				if(!is_object(data)) { return error('fetch_object_by_uuid(data, ..., ...) not object: '+ data); }
				if(!is_string(prop)) { return error('fetch_object_by_uuid(..., prop, ...) not string: '+ prop); }
				if(!is_uuid(uuid)) { return warn('Property ' + prop + ' was not uuid: ' + uuid); }
				if(data.documents[uuid] === undefined) {
					data.documents[uuid] = get_document(uuid, fields);
				} else {
					return warn('Document already fetched: ' + uuid);
				}
			}

			/* */
			function fetch_object(data, config) {
				var prop, fields;

				if(is_object(config)) {
					prop = config.prop;
					fields = config.fields;
				} else if(is_string(config)) {
					prop = config;
				}

				if(!is_string(prop)) {
					return error('property is not valid: ' + prop);
				}

				fields = fields ? fields : [{'query':'*'}];

				if(!is_array(fields)) {
					return error('fields are not valid: ' + fields);
				}

				var uuid = get_property(data, prop);

				if(is_array(uuid)) {
					// FIXME: This could be implemented with one query
					return uuid.forEach(fetch_object_by_uuid.bind(undefined, data, prop, fields));
				}
				return fetch_object_by_uuid(data, prop, fields, uuid);
			}

			// Check for bad input

			if(!is_object(data)) { return error("get_documents(data, ...) not object"); }
			if(!is_array(config)) { return error("get_documents(..., config) not array"); }

			//var types_id = data.types_id;
			//if(!is_uuid(types_id)) { return error("data.types_id not valid UUID: " + types_id); }

			// Get document type
			//var type = get_type(types_id);
			//if(!is_object(type)) { return error("type not valid object: " + type); }

			//var relations = type.relations || {};
			//if(!is_object(relations)) { return error("type.relations not valid object: " + relations); }

			// Populate documents from `config`
			data.documents = {};
			config.forEach( fetch_object.bind(undefined, data) );
			return data.documents;
		}

		return db.query('CREATE OR REPLACE FUNCTION get_documents(data json, config json) RETURNS json LANGUAGE plv8 VOLATILE AS ' + NoPg._escapeFunction(get_documents, ["data", "config", "plv8", "ERROR", "WARNING"]));
	}
1358835
def rebuild_collection_tree(old_collection_ident, new_document_id_map, plpy):
    
    get_tree = plpy.prepare('''
    WITH RECURSIVE t(node, parent, document, title, childorder, latest, path)
        AS (SELECT tr.nodeid, tr.parent_id, tr.documentid, tr.title,
                   tr.childorder, tr.latest, ARRAY[tr.nodeid]
            FROM trees tr
            WHERE tr.documentid = $1 AND tr.is_collated = 'False'
    UNION ALL
        SELECT c.nodeid, c.parent_id, c.documentid, c.title,
               c.childorder, c.latest, path || ARRAY[c.nodeid]
        FROM trees c JOIN t ON (c.parent_id = t.node)
        WHERE not c.nodeid = ANY(t.path)
    )
    SELECT * FROM t
    ''', ('integer',))

    def get_old_tree():
        return plpy.execute(get_tree, (old_collection_ident,))

    tree = {}  # { old_nodeid: {'data': ...}, ...}
    children = {}  # { nodeid: [child_nodeid, ...], child_nodeid: [...]}
    for i in get_old_tree():
        tree[i['node']] = {'data': i, 'new_nodeid': None}
        children.setdefault(i['parent'], [])
        children[i['parent']].append(i['node'])

    insert_tree = plpy.prepare('''
    INSERT INTO trees (nodeid, parent_id, documentid,
        title, childorder, latest)
    VALUES (DEFAULT, $1, $2, $3, $4, $5)
    RETURNING nodeid
    ''', ('integer', 'integer', 'text', 'integer', 'boolean'))

    def execute(fields):
        results = plpy.execute(insert_tree, fields, 1)
        return results[0]['nodeid']

    root_node = children[None][0]

    def build_tree(node, parent):
        data = tree[node]['data']
        new_node = execute([parent, new_document_id_map.get(data['document'],
                            data['document']), data['title'],
                            data['childorder'], data['latest']])
        for i in children.get(node, []):
            build_tree(i, new_node)
    build_tree(root_node, None)
719700
def _find_matching_collections_externally(collections, record):
    
    index, doc_type = RecordIndexer().record_to_index(record)
    body = {"doc": record.dumps()}
    results = current_search_client.percolate(
        index=index,
        doc_type=doc_type,
        allow_no_indices=True,
        ignore_unavailable=True,
        body=body
    )
    prefix_len = len('collection-')
    for match in results['matches']:
        collection_name = match['_id']
        if collection_name.startswith('collection-'):
            name = collection_name[prefix_len:]
            if name in collections:
                yield collections[name]['ancestors']
    raise StopIteration
642607
def execute_legacy(self, sock_info, generator, write_concern):
        
        coll = self.collection
        full_result = {
            "writeErrors": [],
            "writeConcernErrors": [],
            "nInserted": 0,
            "nUpserted": 0,
            "nMatched": 0,
            "nRemoved": 0,
            "upserted": [],
        }
        op_id = _randint()
        stop = False
        for run in generator:
            for idx, operation in enumerate(run.ops):
                try:
                    # To do per-operation reporting we have to do ops one
                    # at a time. That means the performance of bulk insert
                    # will be slower here than calling Collection.insert()
                    if run.op_type == _INSERT:
                        coll._insert(sock_info,
                                     operation,
                                     self.ordered,
                                     write_concern=write_concern,
                                     op_id=op_id)
                        result = {}
                    elif run.op_type == _UPDATE:
                        doc = operation['u']
                        check_keys = True
                        if doc and next(iter(doc)).startswith('$'):
                            check_keys = False
                        result = coll._update(sock_info,
                                              operation['q'],
                                              doc,
                                              operation['upsert'],
                                              check_keys,
                                              operation['multi'],
                                              write_concern=write_concern,
                                              op_id=op_id,
                                              ordered=self.ordered)
                    else:
                        result = coll._delete(sock_info,
                                              operation['q'],
                                              not operation['limit'],
                                              write_concern,
                                              op_id,
                                              self.ordered)
                    _merge_legacy(run, full_result, result, idx)
                except DocumentTooLarge as exc:
                    # MongoDB 2.6 uses error code 2 for "too large".
                    error = _make_error(
                        run.index(idx), _BAD_VALUE, str(exc), operation)
                    full_result['writeErrors'].append(error)
                    if self.ordered:
                        stop = True
                        break
                except OperationFailure as exc:
                    if not exc.details:
                        # Some error not related to the write operation
                        # (e.g. kerberos failure). Re-raise immediately.
                        raise
                    _merge_legacy(run, full_result, exc.details, idx)
                    # We're supposed to continue if errors are
                    # at the write concern level (e.g. wtimeout)
                    if self.ordered and full_result["writeErrors"]:
                        stop = True
                        break
            if stop:
                break

        if full_result["writeErrors"] or full_result['writeConcernErrors']:
            if full_result['writeErrors']:
                full_result['writeErrors'].sort(
                    key=lambda error: error['index'])
            raise BulkWriteError(full_result)
        return full_result
1236797
def create_index(self, collection, index_name, **kwargs):
        """"""
        try:
            self.connection[collection].create_index(index_name, **kwargs)
        except Exception as exc:
            LOG.warn("Error tuning mongodb database: %s", exc)
404487
function createDatabase() {
  var idx, a, b;
  
  db = new loki('sorting-bench.db');
   
  coll = db.addCollection('profile', { indices: ['a', 'b', 'c'] });
  
  for(idx=0;idx<INITIAL_DOCUMENT_COUNT;idx++) {
    coll.insert(genRandomObject());
  }
}
84123
def _canonicalize_query(self, query):

        

        def transform_query(q):

            if isinstance(q, dict):
                nq = {}
                for key,value in q.items():
                    nq[key] = transform_query(value)
                return nq
            elif isinstance(q, (list,QuerySet,tuple)):
                return [transform_query(x) for x in q]
            elif isinstance(q,Document):
                collection = self.get_collection_for_obj(q)
                ref = "%s:%s" % (collection,q.pk)
                return ref
            else:
                return q

        return transform_query(query)
367898
def _legacy_grains(grains):
    '''
    
    '''
    # parse legacy sdc grains
    if 'mdata' in grains and 'sdc' in grains['mdata']:
        if 'server_uuid' not in grains['mdata']['sdc'] or 'FAILURE' in grains['mdata']['sdc']['server_uuid']:
            grains['hypervisor_uuid'] = 'unknown'
        else:
            grains['hypervisor_uuid'] = grains['mdata']['sdc']['server_uuid']

        if 'datacenter_name' not in grains['mdata']['sdc'] or 'FAILURE' in grains['mdata']['sdc']['datacenter_name']:
            grains['datacenter'] = 'unknown'
        else:
            grains['datacenter'] = grains['mdata']['sdc']['datacenter_name']

    # parse rules grains
    if 'mdata' in grains and 'rules' in grains['mdata']:
        grains['roles'] = grains['mdata']['roles'].split(',')

    return grains
1167262
def dataset(self, ref, load_all=False, exception=True):
        """"""
        return self.database.dataset(ref, load_all=load_all, exception=exception)
1167839
def _generate_query_analysis(self, parsed_query, db_name, collection_name):
        """"""

        analyzed_fields = []
        field_count = 0
        supported = True
        sort_fields = []
        query_mask = None

        if 'command' in parsed_query and parsed_query['command'] not in SUPPORTED_COMMANDS:
            supported = False
        else:
            #if 'orderby' in parsed_query:
            sort_component = parsed_query['orderby'] if 'orderby' in parsed_query else []
            sort_seq = 0
            for key in sort_component:
                sort_field = {'fieldName': key,
                              'fieldType': SORT_TYPE,
                              'seq': sort_seq}
                sort_fields.append(key)
                analyzed_fields.append(sort_field)
                field_count += 1
                sort_seq += 1

            query_component = parsed_query['query'] if 'query' in parsed_query else {}
            for key in query_component:
                if key not in sort_fields:
                    field_type = UNSUPPORTED_TYPE
                    if ((key not in UNSUPPORTED_QUERY_OPERATORS) and
                            (key not in COMPOSITE_QUERY_OPERATORS)):
                        try:
                            if query_component[key] == {}:
                                raise
                            nested_field_list = query_component[key].keys()
                        except:
                            field_type = EQUIV_TYPE
                        else:
                            for nested_field in nested_field_list:
                                if ((nested_field in RANGE_QUERY_OPERATORS) and
                                    (nested_field not in UNSUPPORTED_QUERY_OPERATORS)):
                                    field_type = RANGE_TYPE
                                else:
                                    supported = False
                                    field_type = UNSUPPORTED_TYPE
                                    break

                    if field_type is UNSUPPORTED_TYPE:
                        supported = False

                    analyzed_field = {'fieldName': key,
                                      'fieldType': field_type}
                    analyzed_fields.append(analyzed_field)
                    field_count += 1

        query_mask = parsed_query['queryMask']

        # QUERY ANALYSIS
        return OrderedDict({
            'analyzedFields': analyzed_fields,
            'fieldCount': field_count,
            'supported': supported,
            'queryMask': query_mask
        })
420198
def get_data(
    dataset,
    query=None,
    crs="epsg:4326",
    bounds=None,
    sortby=None,
    pagesize=10000,
    max_workers=5,
):
    
    param_dicts = define_request(dataset, query, crs, bounds, sortby, pagesize)

    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        results = executor.map(make_request, param_dicts)

    outjson = dict(type="FeatureCollection", features=[])
    for result in results:
        outjson["features"] += result
    return outjson
630128
def extract_db_info(self, obj, keys):
        """"""

        objl = self.convert(obj)

        result = super(DataFrameType, self).extract_db_info(objl, keys)
        ext = self.datamodel.extractor_map['fits']
        if objl:
            with objl.open() as hdulist:
                for field in keys:
                    result[field] = ext.extract(field, hdulist)

                tags = result['tags']
                for field in self.tags_keys:
                    tags[field] = ext.extract(field, hdulist)

                return result
        else:
            return result
1353523
def _connect_database(config):
    
    settings = config.registry.settings

    mongo_uri = "mongodb://localhost:27017"
    mongodb_name = "test"

    if settings.get("mongo_url"):
        mongo_uri = settings["mongo_url"]

    if settings.get("mongodb_name"):
        mongodb_name = settings["mongodb_name"]

    return mongoengine.connect(mongodb_name, host=mongo_uri)
709028
def _transform_legacy_stats(self, stats):
        """"""
        # Fill pools for legacy driver reports
        if stats and 'pools' not in stats:
            pool = stats.copy()
            pool['pool_name'] = self.id
            for key in ('driver_version', 'shared_targets',
                        'sparse_copy_volume', 'storage_protocol',
                        'vendor_name', 'volume_backend_name'):
                pool.pop(key, None)
            stats['pools'] = [pool]
        return stats