Return data to a mongodb server
505305
def returner(ret):
    '''
    
    '''
    conn, mdb = _get_conn(ret)

    if isinstance(ret['return'], dict):
        back = _remove_dots(ret['return'])
    else:
        back = ret['return']

    if isinstance(ret, dict):
        full_ret = _remove_dots(ret)
    else:
        full_ret = ret

    log.debug(back)
    sdata = {'minion': ret['id'], 'jid': ret['jid'], 'return': back, 'fun': ret['fun'], 'full_ret': full_ret}
    if 'out' in ret:
        sdata['out'] = ret['out']

    # save returns in the saltReturns collection in the json format:
    # { 'minion': <minion_name>, 'jid': <job_id>, 'return': <return info with dots removed>,
    #   'fun': <function>, 'full_ret': <unformatted return with dots removed>}
    #
    # again we run into the issue with deprecated code from previous versions

    if PYMONGO_VERSION > _LooseVersion('2.3'):
        #using .copy() to ensure that the original data is not changed, raising issue with pymongo team
        mdb.saltReturns.insert_one(sdata.copy())
    else:
        mdb.saltReturns.insert(sdata.copy())
99715
def event_return(events):
    '''
    
    '''
    conn, mdb = _get_conn(ret=None)

    if isinstance(events, list):
        events = events[0]

    if isinstance(events, dict):
        log.debug(events)

        if PYMONGO_VERSION > _LooseVersion('2.3'):
            mdb.events.insert_one(events.copy())
        else:
            mdb.events.insert(events.copy())
1187270
function(data, params, callback) {
			if (!data.name) {
				return callback({error:'name is required'});
			}
		  this.db.createCollection(data.name, {}, function(err, collection){
		  	var response = {
		  		name:collection.collectionName,
		  		_id:collection.collectionName
		  	};
				return callback(null, response);
		  });
		}
1166996
function connect(uri, callback) {
  console.log('connecting to database (' + uri + ')');
  mongo.MongoClient.connect(uri, {safe: true}, function (err, client) {
    if (err) return callback(err);

    db = client;
    db.addListener("error", function (error) {
      console.log("mongo client error: " + error);
    });

    callback(null, db);
  });
}
789097
function(resp, options) {
    // Mongodb sometimes returns an array of one document
    if (_.isArray(resp)) {
      resp = resp[0];
    }
    resp = _.defaultsDeep({}, resp, this.defaults());
    return resp;
  }
876920
def collect(self):
        """"""

        if pymongo is None:
            self.log.error('Unable to import pymongo')
            return

        hosts = self.config.get('hosts')

        # Convert a string config value to be an array
        if isinstance(hosts, basestring):
            hosts = [hosts]

        # we need this for backwards compatibility
        if 'host' in self.config:
            hosts = [self.config['host']]

        # convert network_timeout to integer
        if self.config['network_timeout']:
            self.config['network_timeout'] = int(
                self.config['network_timeout'])

        # convert collection_sample_rate to float
        if self.config['collection_sample_rate']:
            self.config['collection_sample_rate'] = float(
                self.config['collection_sample_rate'])

        # use auth if given
        if 'user' in self.config:
            user = self.config['user']
        else:
            user = None

        if 'passwd' in self.config:
            passwd = self.config['passwd']
        else:
            passwd = None

        for host in hosts:
            matches = re.search('((.+)\@)?(.+)?', host)
            alias = matches.group(2)
            host = matches.group(3)

            if alias is None:
                if len(hosts) == 1:
                    # one host only, no need to have a prefix
                    base_prefix = []
                else:
                    base_prefix = [re.sub('[:\.]', '_', host)]
            else:
                base_prefix = [alias]

            try:
                # Ensure that the SSL option is a boolean.
                if type(self.config['ssl']) is str:
                    self.config['ssl'] = str_to_bool(self.config['ssl'])

                if ReadPreference is None:
                    conn = pymongo.MongoClient(
                        host,
                        socketTimeoutMS=self.config['network_timeout'],
                        ssl=self.config['ssl'],
                    )
                else:
                    conn = pymongo.MongoClient(
                        host,
                        socketTimeoutMS=self.config['network_timeout'],
                        ssl=self.config['ssl'],
                        read_preference=ReadPreference.SECONDARY,
                    )
            except Exception as e:
                self.log.error('Couldnt connect to mongodb: %s', e)
                continue

            # try auth
            if user:
                try:
                    conn.admin.authenticate(user, passwd)
                except Exception as e:
                    self.log.error(
                        'User auth given, but could not autheticate' +
                        ' with host: %s, err: %s' % (host, e))
                    return{}

            data = conn.db.command('serverStatus')
            self._publish_transformed(data, base_prefix)
            if str_to_bool(self.config['simple']):
                data = self._extract_simple_data(data)
            if str_to_bool(self.config['replica']):
                try:
                    replset_data = conn.admin.command('replSetGetStatus')
                    self._publish_replset(replset_data, base_prefix)
                except pymongo.errors.OperationFailure as e:
                    self.log.error('error getting replica set status', e)

            self._publish_dict_with_prefix(data, base_prefix)
            db_name_filter = re.compile(self.config['databases'])
            ignored_collections = re.compile(self.config['ignore_collections'])
            sample_threshold = self.MAX_CRC32 * self.config[
                'collection_sample_rate']
            for db_name in conn.database_names():
                if not db_name_filter.search(db_name):
                    continue
                db_stats = conn[db_name].command('dbStats')
                db_prefix = base_prefix + ['databases', db_name]
                self._publish_dict_with_prefix(db_stats, db_prefix)
                for collection_name in conn[db_name].collection_names():
                    if ignored_collections.search(collection_name):
                        continue
                    if (self.config['collection_sample_rate'] < 1 and (
                            zlib.crc32(collection_name) & 0xffffffff
                    ) > sample_threshold):
                        continue

                    collection_stats = conn[db_name].command('collstats',
                                                             collection_name)
                    if str_to_bool(self.config['translate_collections']):
                        collection_name = collection_name.replace('.', '_')
                    collection_prefix = db_prefix + [collection_name]
                    self._publish_dict_with_prefix(collection_stats,
                                                   collection_prefix)
153891
function retrieveEJSON() {
  let EJSON = null;
  try {
    EJSON = requireOptional('mongodb-extjson');
  } catch (error) {} // eslint-disable-line
  if (!EJSON) {
    EJSON = {
      parse: noEJSONError,
      deserialize: noEJSONError,
      serialize: noEJSONError,
      stringify: noEJSONError,
      setBSONModule: noEJSONError,
      BSON: noEJSONError
    };
  }

  return EJSON;
}
854741
def write_mongo(document, database_name,
                collection_name, update=False):
    """"""

    l = []
    response_dict = {}
    try:
        mongodb_client_url = getattr(settings, 'MONGODB_CLIENT',
                                 'mongodb://localhost:27017/')
        mc = MongoClient(mongodb_client_url,document_class=OrderedDict)
        db = mc[str(database_name)]
        collection = db[str(collection_name)]

        # Cast the query to integers
        # if settings.CAST_ININGS_TO_INTEGERS:
        #    query = cast_number_strings_to_integers(query)

        potential_key_found = False
        existing_transaction_id = None
        existing_mongo_id = None

        # enforce non-repudiation constraint on create
        # if document.has_key("transaction_id"):
        #    existing_transaction_id = collection.find_one({'transaction_id':document['transaction_id']})
        #    if existing_transaction_id:
        #        potential_key_found = True

        if "id" in document:
            document["_id"] = ObjectId(document["id"])
            del document["id"]

        if "_id" in document:
            existing_mongo_id = collection.find_one({'_id': document['_id']})
            if existing_mongo_id:
                potential_key_found = True

        if update == False and potential_key_found == True:
            """409 conflict"""
            response_dict['code'] = 409
            response_dict['type'] = "Error"
            response_dict['results'] = []
            response_dict[
                'message'] = "Perhaps you meant to perform an update instead?"
            response_dict['errors'] = [
                "Conflict. This transaction_id has already been created.", ]
            return response_dict

        elif update and potential_key_found:  # this is an update
            # set kwargs _id to the existing_id to force to overwrite existing
            # document

            # if existing_transaction_id:
            #
            #    document['_id'] = ObjectId(existing_transaction_id['_id'])
            #    document['history']=True
            #    history_collection_name = "%s_history" % str(collection_name)
            #    history_collection   = db[str(history_collection_name)]
            #
            #    history_object = existing_transaction_id
            #    history_object['historical_id'] = existing_transaction_id['_id']
            #    del history_object['_id']
            #    #now write the record to the historical collection
            #    written_object = history_collection.insert(history_object)

            if existing_mongo_id:
                document['_id'] = ObjectId(existing_mongo_id['_id'])
                document['history'] = True
                document['verified'] = False
                history_collection_name = "%s_history" % str(collection_name)
                history_collection = db[str(history_collection_name)]

                # print history_collection
                # print existing_mongo_id

                history_object = existing_mongo_id

                history_object['historical_id'] = existing_mongo_id['_id']
                del history_object['_id']
                # print history_object

                # now write the record to the historical collection
                written_object = history_collection.insert(history_object)

            # update the record
            myobjectid = collection.save(document)

        else:
            # this is new so perform an insert.
            myobjectid = collection.insert(document)

        # now fetch the record we just wrote so that we write it back to the
        # DB.
        myobject = collection.find_one({'_id': myobjectid})
        response_dict['code'] = 200
        response_dict['type'] = "write-results"
        myobject['id'] = myobject['_id'].__str__()
        del myobject['_id']
        l.append(myobject)
        response_dict['results'] = l

    except:
        # print "Error reading from Mongo"
        # print str(sys.exc_info())
        response_dict['code'] = 400
        response_dict['type'] = "Error"
        response_dict['results'] = []
        response_dict['message'] = str(sys.exc_info())
    return response_dict
655763
def query(options, collection_name,
          num_to_skip, num_to_return, query, field_selector=None):
    
    data = struct.pack("<I", options)
    data += bson._make_c_string(collection_name)
    data += struct.pack("<i", num_to_skip)
    data += struct.pack("<i", num_to_return)
    data += bson.BSON.encode(query)
    if field_selector is not None:
        data += bson.BSON.encode(field_selector)
    return __pack_message(2004, data)
765311
def _query(options, collection_name, num_to_skip,
           num_to_return, query, field_selector, opts, check_keys):
    """"""
    encoded = _dict_to_bson(query, check_keys, opts)
    if field_selector:
        efs = _dict_to_bson(field_selector, False, opts)
    else:
        efs = b""
    max_bson_size = max(len(encoded), len(efs))
    return b"".join([
        _pack_int(options),
        _make_c_string(collection_name),
        _pack_int(num_to_skip),
        _pack_int(num_to_return),
        encoded,
        efs]), max_bson_size
1372381
def _format_return_data(retcode, stdout=None, stderr=None):
    '''
    
    '''
    ret = {'retcode': retcode}
    if stdout is not None:
        ret['stdout'] = stdout
    if stderr is not None:
        ret['stderr'] = stderr
    return ret
738203
function getCollection(key, collectionName, database, options, callback) {
    getDatabaseCached(database, options, getOrCreateCollection)

    function getOrCreateCollection(err, db) {
        if (err) {
            return callback(err)
        }

        // get the collection
        db.collection(collectionName, extend({
            safe: true
        }, options.collectionOptions), createIfNotExist)

        function createIfNotExist(err, collection) {
            // if not exist create it
            if (err && err.message.indexOf(" does not exist") !== -1) {
                return db.createCollection(collectionName, extend({
                    safe: true
                }, options.createCollectionOptions), returnCollection)
            }

            if (err) {
                return callback(err)
            }

            callback(null, collection)
        }
    }

    function returnCollection(err, col) {
        if (err) {
            return callback(err)
        }

        callback(null, col)
    }
}
702429
def _get_db():
    
    from .settings import settings
    mongo = settings.MONGODB

    if 'URI' in mongo and mongo['URI']:
        uri = mongo['URI']
    else:
        uri = 'mongodb://'

        if all(mongo.get(key) for key in ('USERNAME', 'PASSWORD')):
            uri += '{0}:{1}@'.format(mongo['USERNAME'], mongo['PASSWORD'])

        if 'HOSTS' in mongo and mongo['HOSTS']:
            uri += ','.join(
                '{0}:{1}'.format(host, port)
                for (host, port) in zip(mongo['HOSTS'], mongo['PORTS']),
            )
        else:
            uri += '{0}:{1}'.format(mongo['HOST'], mongo.get('PORT', 27017))

        uri += '/' + mongo['DATABASE']

        if 'OPTIONS' in mongo and mongo['OPTIONS']:
            uri += '?{0}'.format('&'.join(mongo['OPTIONS']))

    client = ConnectionFailureProxy(MongoClient(uri, connect=False))
    database = client[parse_uri(uri)['database']]

    return database
1136364
def query(options, collection_name, num_to_skip,
          num_to_return, query, field_selector, opts, check_keys=False):
    
    data = struct.pack("<I", options)
    data += bson._make_c_string(collection_name)
    data += struct.pack("<i", num_to_skip)
    data += struct.pack("<i", num_to_return)
    encoded = bson.BSON.encode(query, check_keys, opts)
    data += encoded
    max_bson_size = len(encoded)
    if field_selector is not None:
        encoded = bson.BSON.encode(field_selector, False, opts)
        data += encoded
        max_bson_size = max(len(encoded), max_bson_size)
    (request_id, query_message) = __pack_message(2004, data)
    return (request_id, query_message, max_bson_size)
80974
def get(self, mongo_id):
        """"""
        try:
            return self.filter(self.type.mongo_id == mongo_id).first()
        except exceptions.BadValueException:
            return None
190645
def db(self, request):
        ''''''
        with self.mapper.begin() as session:
            world = session.query(World).get(randint(1, 10000))
        return Json(self.get_json(world)).http_response(request)
2320
def returner(ret):
    '''
    
    '''
    # if a minion is returning a standalone job, get a jobid
    if ret['jid'] == 'req':
        ret['jid'] = prep_jid(nocache=ret.get('nocache', False))
        save_load(ret['jid'], ret)

    try:
        with _get_serv(ret, commit=True) as cur:
            sql = '''INSERT INTO `salt_returns`
                     (`fun`, `jid`, `return`, `id`, `success`, `full_ret`)
                     VALUES (%s, %s, %s, %s, %s, %s)'''

            cur.execute(sql, (ret['fun'], ret['jid'],
                              salt.utils.json.dumps(ret['return']),
                              ret['id'],
                              ret.get('success', False),
                              salt.utils.json.dumps(ret)))
    except salt.exceptions.SaltMasterError as exc:
        log.critical(exc)
        log.critical('Could not store return with MySQL returner. MySQL server unavailable.')