Sink log entries to bigquery.
171801
def sink_bigquery(client, to_delete):
    """"""
    dataset = _sink_bigquery_setup(client)
    to_delete.append(dataset)
    SINK_NAME = "robots-bigquery-%d" % (_millis(),)
    FILTER = "textPayload:robot"

    # [START sink_bigquery_create]
    DESTINATION = "bigquery.googleapis.com%s" % (dataset.path,)
    sink = client.sink(SINK_NAME, filter_=FILTER, destination=DESTINATION)
    assert not sink.exists()  # API call
    sink.create()  # API call
    assert sink.exists()  # API call
    # [END sink_bigquery_create]
    to_delete.insert(0, sink)
247840
def _get_logs(job_id):
    """"""
    # Avoid SQLAlchemy "Unicode type received non-unicode bind param value"
    # warnings.
    job_id = unicode(job_id)

    results = ENGINE.execute(
        LOGS_TABLE.select().where(LOGS_TABLE.c.job_id == job_id)).fetchall()

    results = [dict(result) for result in results]

    for result in results:
        result.pop("job_id")

    return results
315115
def bigquery_schema(table):
    """"""
    fields = OrderedDict((el.name, dt.dtype(el)) for el in table.schema)
    partition_info = table._properties.get('timePartitioning', None)

    # We have a partitioned table
    if partition_info is not None:
        partition_field = partition_info.get('field', NATIVE_PARTITION_COL)

        # Only add a new column if it's not already a column in the schema
        fields.setdefault(partition_field, dt.timestamp)
    return sch.schema(fields)
429791
def entries(self):
        """"""
        def add(x, y):
            return x + y
        try:
            return reduce(add, list(self.cache.values()))
        except:
            return []
828645
def ReadHuntOutputPluginLogEntries(self,
                                     hunt_id,
                                     output_plugin_id,
                                     offset,
                                     count,
                                     with_type=None):
    """"""

    all_entries = []
    for flow_obj in self._GetHuntFlows(hunt_id):
      for entry in self.ReadFlowOutputPluginLogEntries(
          flow_obj.client_id,
          flow_obj.flow_id,
          output_plugin_id,
          0,
          sys.maxsize,
          with_type=with_type):
        all_entries.append(
            rdf_flow_objects.FlowOutputPluginLogEntry(
                hunt_id=hunt_id,
                client_id=flow_obj.client_id,
                flow_id=flow_obj.flow_id,
                output_plugin_id=output_plugin_id,
                log_entry_type=entry.log_entry_type,
                timestamp=entry.timestamp,
                message=entry.message))

    return sorted(all_entries, key=lambda x: x.timestamp)[offset:offset + count]
789758
private static Bigquery.Builder newBigQueryClient(BigQueryOptions options) {
    return new Bigquery.Builder(Transport.getTransport(), Transport.getJsonFactory(),
        chainHttpRequestInitializer(
            options.getGcpCredential(),
            // Do not log 404. It clutters the output and is possibly even required by the caller.
            new RetryHttpRequestInitializer(ImmutableList.of(404))))
        .setApplicationName(options.getAppName())
        .setGoogleClientRequestInitializer(options.getGoogleApiTrace());
  }
138327
def logs(stage, period, tail, config_file):
    """"""
    config = _load_config(config_file)
    if stage is None:
        stage = config['devstage']

    cfn = boto3.client('cloudformation')
    try:
        stack = cfn.describe_stacks(StackName=config['name'])['Stacks'][0]
    except botocore.exceptions.ClientError:
        print('{} has not been deployed yet.'.format(config['name']))
        return
    function = _get_from_stack(stack, 'Output', 'FunctionArn').split(':')[-1]
    version = _get_from_stack(stack, 'Parameter', stage.title() + 'Version')
    api_id = _get_from_stack(stack, 'Output', 'ApiId')

    try:
        start = float(period[:-1])
    except ValueError:
        raise ValueError('Invalid period ' + period)
    if period[-1] == 's':
        start = time.time() - start
    elif period[-1] == 'm':
        start = time.time() - start * 60
    elif period[-1] == 'h':
        start = time.time() - start * 60 * 60
    elif period[-1] == 'd':
        start = time.time() - start * 60 * 60 * 24
    elif period[-1] == 'w':
        start = time.time() - start * 60 * 60 * 24 * 7
    else:
        raise ValueError('Invalid period ' + period)
    start = int(start * 1000)

    logs = boto3.client('logs')
    lambda_log_group = '/aws/lambda/' + function
    log_groups = [lambda_log_group]
    if api_id:
        log_groups.append('API-Gateway-Execution-Logs_' + api_id + '/' + stage)
    log_version = '[' + version + ']'
    log_start = {g: start for g in log_groups}
    while True:
        kwargs = {}
        events = []
        for log_group in log_groups:
            while True:
                try:
                    l = logs.filter_log_events(logGroupName=log_group,
                                               startTime=log_start[log_group],
                                               interleaved=True, **kwargs)
                except botocore.exceptions.ClientError:
                    # the log group does not exist yet
                    l = {'events': []}
                if log_group == lambda_log_group:
                    events += [ev for ev in l['events']
                               if log_version in ev['logStreamName']]
                else:
                    events += l['events']
                if len(l['events']):
                    log_start[log_group] = l['events'][-1]['timestamp'] + 1
                if 'nextToken' not in l:
                    break
                kwargs['nextToken'] = l['nextToken']
        events.sort(key=lambda ev: ev['timestamp'])
        for ev in events:
            tm = datetime.fromtimestamp(ev['timestamp'] / 1000)
            print(tm.strftime('%b %d %X ') + ev['message'].strip())
        if not tail:
            break
        time.sleep(5)
1328637
def ReadFlowOutputPluginLogEntries(self,
                                     client_id,
                                     flow_id,
                                     output_plugin_id,
                                     offset,
                                     count,
                                     with_type=None,
                                     cursor=None):
    """"""
    query = ("SELECT log_entry_type, message, UNIX_TIMESTAMP(timestamp) "
             "FROM flow_output_plugin_log_entries "
             "FORCE INDEX (flow_output_plugin_log_entries_by_flow) "
             "WHERE client_id = %s AND flow_id = %s AND output_plugin_id = %s ")
    args = [
        db_utils.ClientIDToInt(client_id),
        db_utils.FlowIDToInt(flow_id),
        db_utils.OutputPluginIDToInt(output_plugin_id)
    ]

    if with_type is not None:
      query += "AND log_entry_type = %s "
      args.append(int(with_type))

    query += "ORDER BY log_id ASC LIMIT %s OFFSET %s"
    args.append(count)
    args.append(offset)

    cursor.execute(query, args)

    ret = []
    for log_entry_type, message, timestamp in cursor.fetchall():
      ret.append(
          rdf_flow_objects.FlowOutputPluginLogEntry(
              client_id=client_id,
              flow_id=flow_id,
              output_plugin_id=output_plugin_id,
              log_entry_type=log_entry_type,
              message=message,
              timestamp=mysql_utils.TimestampToRDFDatetime(timestamp)))

    return ret
484229
def db(context, data):
    """"""
    table = context.params.get("table", context.crawler.name)
    params = context.params
    params["table"] = table
    _recursive_upsert(context, params, data)
1026865
async function quickstart() {
  const bigqueryDataTransfer = require('@google-cloud/bigquery-data-transfer');
  const client = new bigqueryDataTransfer.v1.DataTransferServiceClient();
  const projectId = await client.getProjectId();

  // Iterate over all elements.
  const formattedParent = client.projectPath(projectId, 'us-central1');
  let nextRequest = {parent: formattedParent};
  const options = {autoPaginate: false};
  console.log('Data sources:');
  do {
    // Fetch the next page.
    const responses = await client.listDataSources(nextRequest, options);
    // The actual resources in a response.
    const resources = responses[0];
    // The next request if the response shows that there are more responses.
    nextRequest = responses[1];
    // The actual response object, if necessary.
    // const rawResponse = responses[2];
    resources.forEach(resource => {
      console.log(`  ${resource.name}`);
    });
  } while (nextRequest);

  console.log('\n\n');
  console.log('Sources via stream:');

  client
    .listDataSourcesStream({parent: formattedParent})
    .on('data', element => {
      console.log(`  ${element.name}`);
    });
}
782228
def ReadFlowOutputPluginLogEntries(self,
                                     client_id,
                                     flow_id,
                                     output_plugin_id,
                                     offset,
                                     count,
                                     with_type=None):
    """"""
    entries = sorted(
        self.flow_output_plugin_log_entries.get((client_id, flow_id), []),
        key=lambda e: e.timestamp)

    entries = [e for e in entries if e.output_plugin_id == output_plugin_id]

    if with_type is not None:
      entries = [e for e in entries if e.log_entry_type == with_type]

    return entries[offset:offset + count]
317508
def _datalog(self, parameter, run, maxrun, det_id):
        ""
        values = {
            'parameter_name': parameter,
            'minrun': run,
            'maxrun': maxrun,
            'detid': det_id,
        }
        data = urlencode(values)
        content = self._get_content('streamds/datalognumbers.txt?' + data)
        if content.startswith('ERROR'):
            log.error(content)
            return None
        try:
            dataframe = read_csv(content)
        except ValueError:
            log.warning(
                "Empty dataset"
            )    # ...probably. Waiting for more info
            return make_empty_dataset()
        else:
            add_datetime(dataframe)
            try:
                self._add_converted_units(dataframe, parameter)
            except KeyError:
                log.warning(
                    "Could not add converted units for {0}".format(parameter)
                )
            return dataframe
377808
def bulk_log_data(registers = register_list)
      registers.map do |reg|
        read_scaled_range(reg.start, reg.count)
      end.flatten
    end
1258000
def bigquery_data_type(self):
        """"""

        # BigQuery data type = {source_database: [data type, ...], ...}
        BQ_DATA_TYPE_DIC = OrderedDict()
        BQ_DATA_TYPE_DIC["STRING"] = {None: [re.compile(r"(CHAR|TEXT|CLOB|JSON|UUID)")]}
        BQ_DATA_TYPE_DIC["INTEGER"] = {None: [re.compile(r"INT|SERIAL|YEAR")]}
        BQ_DATA_TYPE_DIC["FLOAT"] = {None: [re.compile(r"(FLOAT|DOUBLE)"), "REAL", "MONEY"]}
        BQ_DATA_TYPE_DIC["DATETIME"] = {
            None: ["DATETIME", "TIMESTAMP", "TIMESTAMP WITHOUT TIME ZONE"],
            self.DATABASE.oracle: ["DATE"]
        }
        BQ_DATA_TYPE_DIC["TIMESTAMP"] = {None: ["TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE"]}
        BQ_DATA_TYPE_DIC["DATE"] = {None: ["DATE"]}
        BQ_DATA_TYPE_DIC["TIME"] = {None: ["TIME"]}
        BQ_DATA_TYPE_DIC["BOOLEAN"] = {None: [re.compile(r"BOOL")]}

        for bq_type, conditions in BQ_DATA_TYPE_DIC.items():
            for source_db, source_datatypes in conditions.items():
                for source_datatype in source_datatypes:

                    if isinstance(source_datatype, str):
                        if self._data_type == source_datatype \
                            and (  self._source_database == source_db
                                or (self._source_database is not None and source_db is None)):
                            return bq_type

                    elif re.search(source_datatype, self._data_type) \
                        and (  self._source_database == source_db
                            or (self._source_database is not None and source_db is None)):
                        return bq_type

        if self._data_type in ["NUMERIC", "NUMBER", "DECIMAL"]:
            if self._scale is not None:
                return "FLOAT"

            if self._data_type == "NUMBER" \
                and self._source_database == self.DATABASE.oracle \
                and self._length is None:
                return "FLOAT"

            return "INTEGER"

        raise ValueError("Unknown data type : '{}'".format(self._data_type))
669936
private void logHeaders(Set<Map.Entry<String, List<String>>> entries) {
        for (Map.Entry<String, List<String>> header : entries) {
            if (header.getKey() != null) {
                LOGGER.debug("{}: {}", header.getKey(), header.getValue());
            }
        }
    }
160653
def get_log_entries_by_search(self, log_entry_query, log_entry_search):
        """"""
        # Implemented from azosid template for -
        # osid.resource.ResourceSearchSession.get_resources_by_search_template
        if not self._can('search'):
            raise PermissionDenied()
        return self._provider_session.get_log_entries_by_search(log_entry_query, log_entry_search)
26843
private BigQueryLoadJobReference triggerBigQueryLoadJob() {
    Job job = createJob();
    // Set up Bigquery Insert
    try {
      Insert insert =
          BigQueryLoadGoogleCloudStorageFilesJob.getBigquery().jobs().insert(projectId, job);
      Job executedJob = insert.execute();
      log.info("Triggered the bigQuery load job for files " + fileSet + " . Job Id = "
          + executedJob.getId());
      return new BigQueryLoadJobReference(projectId, executedJob.getJobReference());
    } catch (IOException e) {
      throw new RuntimeException("Error in triggering BigQuery load job for files " + fileSet, e);
    }
  }