store each record in MongoDB and output tab delimited lines
17232
def reducer_output(self, metro, values):
        
        records = []
        # build up list of data for each metro area and submit as one network
        # call instead of individually 
        for value in values:
            total, poi = value
            records.append(dict(
                metro_area=metro,
                poi=poi,
                count=total
            ))
            output = "{0}\t{1}\t{2}"
            output = output.format(metro.encode('utf-8'), total, poi.encode('utf-8'))
            yield None, output
        if self.mongo:
            self.mongo.insert_many(records)
604896
def records(self):
        
        output = tempfile.NamedTemporaryFile(suffix='.json')
        try:
            log.info("Loading table from (%s)...", self._obj)
            shutil.copyfileobj(self.fh(), output)
            output.seek(0)

            for line in output.file:
                yield json.loads(line, object_hook=json_hook)

        finally:
            try:
                output.close()
            except:
                pass
1093556
def dump_database_object(obj: T, fieldlist: Iterable[str]) -> None:
    """"""
    log.info(_LINE_EQUALS)
    log.info("DUMP OF: {}", obj)
    for f in fieldlist:
        log.info(u"{f}: {v}", f=f, v=getattr(obj, f))
    log.info(_LINE_EQUALS)
1300171
def _iter_table_records(self):
        
        count = self._table_count
        bufr = self._stream.read(offset=12, length=count*16)
        tmpl = '>4sLLL'
        for i in range(count):
            offset = i * 16
            tag, checksum, off, len_ = unpack_from(tmpl, bufr, offset)
            yield tag.decode('utf-8'), off, len_
1071645
def action_aggregate(reader, *args):
    """"""
    all_aggregated = aggregated_records(reader)
    first_row = next(all_aggregated)
    keys = sorted(first_row.keys())
    print(*keys, sep='\t')

    # Join the first row with the rest of the rows and print them
    iterable = chain([first_row], all_aggregated)
    for item in iterable:
        print(*[item[k] for k in keys], sep='\t')
608433
def process_data(lines)  
      data = {}
      lines.each do |line|
        line = line.split
        key = line.first.split('.value').first
        data[key] = line.last
      end
      data
    end
787415
def dump_collection(cfg, f, indent=0):
    ''''''

    for i, value in enumerate(cfg):
        dump_value(None, value, f, indent)
        if i < len(cfg) - 1:
            f.write(u',\n')
363959
function(callback){
        var cursor = db.db(dbName).collection(collectionName).find({}).toArray(function(err,result){
            if(result.length!=0){
                roadmapNum = roadMapIdTemp = parseInt(result[result.length-1]['roadMapId'])+1;
            }
            else{
                roadmapNum = roadMapIdTemp = 1;
            }
            db.db(dbName).collection(collectionName).insertOne({
                "roadMapId" : roadMapIdTemp.toString(),
                "clientId" : source['clientId'],
                "orderNode" : source['orderNode'],
                "initNode" : source['initNode'],
                "lastNode" : source['lastNode'],
                "incomingNode" : source['incomingNode'],
                "outingNode" : source['outingNode'],
                "isInput" : source['isInput'],
                "isOutput" : source['isOutput'],
                "nodeIds" : source['nodeIds']
            },function(err, result){
                response.send(roadmapNum.toString());
            });
        });

    }
572915
def insert_many(conn, tablename, column_names, records, chunksize=2500):
    

    groups = chunks(records, chunksize)
    column_str = ','.join(column_names)
    insert_template = 'INSERT INTO {table} ({columns}) VALUES {values}'.format(
        table=tablename, columns=column_str, values='{0}')

    with conn:
        with conn.cursor() as cursor:
            for recs in groups:
                record_group = list(recs)
                records_template_str = ','.join(['%s'] * len(record_group))
                insert_query = insert_template.format(records_template_str)
                cursor.execute(insert_query, record_group)
305619
def format_lines(var_name, seq, raw=False, indent_level=0):
    """"""
    lines = []
    base_indent = ' ' * indent_level * 4
    inner_indent = ' ' * (indent_level + 1) * 4
    lines.append(base_indent + var_name + ' = (')
    if raw:
        # These should be preformatted reprs of, say, tuples.
        for i in seq:
            lines.append(inner_indent + i + ',')
    else:
        for i in seq:
            # Force use of single quotes
            r = repr(i + '"')
            lines.append(inner_indent + r[:-2] + r[-1] + ',')
    lines.append(base_indent + ')')
    return '\n'.join(lines)
579890
private BasicDBList getSplitData(DBCollection collection) {

        final DBObject cmd = BasicDBObjectBuilder.start("splitVector", collection.getFullName())
                .add("keyPattern", new BasicDBObject(MONGO_DEFAULT_ID, 1))
                .add("force", false)
                .add("maxChunkSize", splitSize)
                .get();

        CommandResult splitVectorResult = collection.getDB().getSisterDB("admin").command(cmd);
        return (BasicDBList) splitVectorResult.get(SPLIT_KEYS);

    }
1130788
function(dbOrUri, stream, callback) {
    if (arguments.length === 2) {
      callback = stream;
      stream = undefined;
    }
    if (!stream) {
      stream = process.stdout;
    }
    var db;
    var out = stream;
    var endOfCollection = crypto.pseudoRandomBytes(8).toString('base64');
    write({
      type: 'mongo-dump-stream',
      version: '2',
      endOfCollection: endOfCollection
    });
    return async.series({
      connect: function(callback) {
        if (typeof(dbOrUri) !== 'string') {
          // Already a mongodb connection
          db = dbOrUri;
          return setImmediate(callback);
        }
        return mongodb.MongoClient.connect(dbOrUri, function(err, _db) {
          if (err) {
            return callback(err);
          }
          db = _db;
          return callback(null);
        });
      },
      getCollections: function(callback) {
        return db.collections(function(err, _collections) {
          if (err) {
            return callback(err);
          }
          collections = _collections;
          return callback(null);
        });
      },
      dumpCollections: function(callback) {
        return async.eachSeries(collections, function(collection, callback) {
          if (collection.collectionName.match(/^system\./)) {
            return setImmediate(callback);
          }
          return async.series({
            getIndexes: function(callback) {
              return collection.indexInformation({ full: true }, function(err, info) {
                if (err) {
                  return callback(err);
                }
                write({
                  type: 'collection',
                  name: collection.collectionName,
                  indexes: info
                });
                return callback(null);
              });
            },
            getDocuments: function(callback) {
              var cursor = collection.find({}, { raw: true });
              iterate();
              function iterate() {
                return cursor.nextObject(function(err, item) {
                  if (err) {
                    return callback(err);
                  }
                  if (!item) {
                    write({
                      // Ensures we don't confuse this with
                      // a legitimate database object
                      endOfCollection: endOfCollection
                    });
                    return callback(null);
                  }
                  // v2: just a series of actual data documents
                  out.write(item);

                  // If we didn't have the raw BSON document,
                  // we could do this instead, but it would be very slow
                  // write({
                  //   type: 'document',
                  //   document: item
                  // });

                  return setImmediate(iterate);
                });
              }
            },
          }, callback);
        }, callback);
      },
      endDatabase: function(callback) {
        write({
          type: 'endDatabase'
        }, callback);
      }
    }, function(err) {
      if (err) {
        return callback(err);
      }
      return callback(null);
    });
    function write(o, callback) {
      out.write(BSON.serialize(o, false, true, false), callback);
    }
  }
722344
def _concat_records(items_by_key, input_order):
    
    all_records = []
    for (k, t) in input_order.items():
        if t == "record":
            all_records.append(k)
    out_items_by_key = utils.deepish_copy(items_by_key)
    out_input_order = utils.deepish_copy(input_order)
    if len(all_records) > 1:
        final_k = all_records[0]
        final_v = items_by_key[final_k]
        for k in all_records[1:]:
            final_v += items_by_key[k]
            del out_items_by_key[k]
            del out_input_order[k]
        out_items_by_key[final_k] = final_v
    return out_items_by_key, out_input_order
13861
def _split_lines(self):
        ''''''
        parsed_lines = {}
        for rt in all_record_types:
            parsed_lines[rt] = []
        parsed_lines[0] = []

        for line in self.lines:
            linetype = line[0:6]
            if linetype in all_record_types:
                parsed_lines[linetype].append(line)
            else:
                parsed_lines[0].append(line)

        self.parsed_lines = parsed_lines
        self._update_structure_lines()
90824
def parse_results(fields, data):
    
    master = []

    for record in data['records']:  # for each 'record' in response
        row = [None] * len(fields)  # create null list the length of number of columns
        for obj, value in record.iteritems():  # for each obj in record
            if not isinstance(value, (dict, list, tuple)):  # if not data structure
                if obj in fields:
                    row[fields.index(obj)] = ensure_utf(value)

            elif isinstance(value, dict) and obj != 'attributes':  # traverse down into object
                path = obj
                _traverse_results(value, fields, row, path)

        master.append(row)
    return master
1295802
def dump_as_json(record, output_file):
    
    def default_func(value):
        """
        Default conversion for JSON value.
        """
        if isinstance(value, frozenset):
            return sorted(value)
        raise TypeError(repr(value) + " is not JSON serializable")

    converted_record = record._replace(
        ordered_statistics=[x._asdict() for x in record.ordered_statistics])
    json.dump(
        converted_record._asdict(), output_file,
        default=default_func, ensure_ascii=False)
    output_file.write(os.linesep)
550894
def tabulate(collection, headers, datetime_fmt='%Y-%m-%d %H:%M:%S', **kwargs):
    """"""
    if isinstance(headers, dict):
        attrs = headers.keys()
        # if mapping is not specified keep original
        names = [
            key if value is None else value for key, value in headers.items()
        ]
    else:
        attrs = names = headers
    table = [(
        format_cell(cell, datetime_fmt=datetime_fmt)
        for cell in attrgetter(*attrs)(c)
    ) for c in collection]
    return tblte(table, headers=[h.upper() for h in names], **kwargs)