Pull out the application metadata for each flow in the input_stream
766726
def http_meta_data(self):
        """"""

        # For each flow process the contents
        for flow in self.input_stream:

            # Client to Server
            if flow['direction'] == 'CTS':
                try:
                    request = dpkt.http.Request(flow['payload'])
                    request_data = data_utils.make_dict(request)
                    request_data['uri'] = self._clean_uri(request['uri'])
                    flow['http'] = {'type':'HTTP_REQUEST', 'data':request_data}
                except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError):
                    flow['http'] = None

            # Server to Client
            else:
                try:
                    response = dpkt.http.Response(flow['payload'])
                    flow['http'] = {'type': 'HTTP_RESPONSE', 'data': data_utils.make_dict(response)}
                except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError):
                    flow['http'] = None

            # Mark non-TCP HTTP
            if flow['http'] and flow['protocol'] != 'TCP':
                flow['http'].update({'weird': 'UDP-HTTP'})

            # All done
            yield flow
364158
public Metadata[] readMetadata(StreamInfo streamInfo) throws IOException {
        if (streamInfo.isLast()) return new Metadata[0];
        ArrayList<Metadata> metadataList = new ArrayList<Metadata>();
        Metadata metadata;
        do {
            metadata = readNextMetadata();
            metadataList.add(metadata);
        } while (!metadata.isLast());
        return (Metadata[])metadataList.toArray(new Metadata[0]);
    }
539150
def ReadFlowRequestsReadyForProcessing(self,
                                         client_id,
                                         flow_id,
                                         next_needed_request=None):
    """"""
    request_dict = self.flow_requests.get((client_id, flow_id), {})
    response_dict = self.flow_responses.get((client_id, flow_id), {})

    res = {}
    for request_id in sorted(request_dict):
      # Ignore outdated requests.
      if request_id < next_needed_request:
        continue
      # The request we are currently looking for is not in yet, we are done.
      if request_id != next_needed_request:
        break
      request = request_dict[request_id]
      if not request.needs_processing:
        break

      responses = sorted(
          itervalues(response_dict.get(request_id, {})),
          key=lambda response: response.response_id)
      # Serialize/deserialize responses to better simulate the
      # real DB behavior (where serialization/deserialization is almost
      # guaranteed to be done).
      # TODO(user): change mem-db implementation to do
      # serialization/deserialization everywhere in a generic way.
      responses = [
          r.__class__.FromSerializedString(r.SerializeToString())
          for r in responses
      ]
      res[request_id] = (request, responses)
      next_needed_request += 1

    return res
1007489
function ApplicationData(buffer) {
  const len = numberToBuffer(buffer.length);
  return Buffer.concat([stb('170303'), len, buffer]);
}
152274
def _ExtractInterfaceMetadata(self, metadata):
    
    interfaces = []
    for network_interface in metadata:
      mac_address = network_interface.get('mac')
      interface = self.network_utils.GetNetworkInterface(mac_address)
      ip_addresses = []
      if interface:
        ip_addresses.extend(network_interface.get('forwardedIps', []))
        if self.ip_aliases:
          ip_addresses.extend(network_interface.get('ipAliases', []))
        if self.target_instance_ips:
          ip_addresses.extend(network_interface.get('targetInstanceIps', []))
        interfaces.append(NetworkDaemon.NetworkInterface(
            interface, ip_addresses, network_interface.get('ip', [])))
      else:
        message = 'Network interface not found for MAC address: %s.'
        self.logger.warning(message, mac_address)
    return interfaces
494314
def _get_stream_handle_ids(self, handles):
        
        stream_handle_ids = defaultdict(dict)
        for stream in self.streams:
            for h in self.models:
                if h in handles:
                    handle_id = handles[h].ref['id']
                    stream_handle_ids[stream][h] = handle_id
        return stream_handle_ids
573901
def _dump(self, stream, pattern=None, object_ids=False):
        """"""
        return self._lists[pattern]._dump(stream, '', object_ids=object_ids)
335457
def parse_streams(self):
        
        logger.debug("Parsing streams of {}".format(self.input_file))

        cmd = [
            self.ffmpeg_normalize.ffmpeg_exe, '-i', self.input_file,
            '-c', 'copy', '-t', '0', '-map', '0',
            '-f', 'null', NUL
        ]

        cmd_runner = CommandRunner(cmd)
        cmd_runner.run_command()
        output = cmd_runner.get_output()

        logger.debug("Stream parsing command output:")
        logger.debug(output)

        output_lines = [line.strip() for line in output.split('\n')]

        for line in output_lines:

            if not line.startswith('Stream'):
                continue

            stream_id_match = re.search(r'#0:([\d]+)', line)
            if stream_id_match:
                stream_id = int(stream_id_match.group(1))
                if stream_id in self._stream_ids():
                    continue
            else:
                continue

            if 'Audio' in line:
                logger.debug("Found audio stream at index {}".format(stream_id))
                sample_rate_match = re.search(r'(\d+) Hz', line)
                sample_rate = int(sample_rate_match.group(1)) if sample_rate_match else None
                bit_depth_match = re.search(r's(\d+)p?,', line)
                bit_depth = int(bit_depth_match.group(1)) if bit_depth_match else None
                self.streams['audio'][stream_id] = AudioStream(self, stream_id, sample_rate, bit_depth)

            elif 'Video' in line:
                logger.debug("Found video stream at index {}".format(stream_id))
                self.streams['video'][stream_id] = VideoStream(self, stream_id)

            elif 'Subtitle' in line:
                logger.debug("Found subtitle stream at index {}".format(stream_id))
                self.streams['subtitle'][stream_id] = SubtitleStream(self, stream_id)

        if not self.streams['audio']:
            raise FFmpegNormalizeError(
                "Input file {} does not contain any audio streams"
                .format(self.input_file))

        if os.path.splitext(self.output_file)[1].lower() in ['.wav', '.mp3', '.aac']:
            logger.warning(
                "Output file only supports one stream. "
                "Keeping only first audio stream."
            )
            first_stream = list(self.streams['audio'].values())[0]
            self.streams['audio'] = {first_stream.stream_id: first_stream}
            self.streams['video'] = {}
            self.streams['subtitle'] = {}
681997
def load_stream(cls, st):
        """"""
        y = yaml.load(st)
        return [ Automaton(k, v) for k, v in y.iteritems() ]
581591
def MultiReadClientMetadata(self, client_ids):
    """"""
    res = {}
    for client_id in client_ids:
      md = self.metadatas.get(client_id, None)
      if md is None:
        continue

      res[client_id] = rdf_objects.ClientMetadata(
          certificate=md.get("certificate"),
          fleetspeak_enabled=md.get("fleetspeak_enabled"),
          first_seen=md.get("first_seen"),
          ping=md.get("ping"),
          clock=md.get("clock"),
          ip=md.get("ip"),
          last_foreman_time=md.get("last_foreman_time"),
          last_crash_timestamp=md.get("last_crash_timestamp"),
          startup_info_timestamp=md.get("startup_info_timestamp"))

    return res
76766
def ReadAllFlowObjects(self,
                         client_id = None,
                         min_create_time = None,
                         max_create_time = None,
                         include_child_flows = True,
                         cursor=None):
    """"""
    conditions = []
    args = []

    if client_id is not None:
      conditions.append("client_id = %s")
      args.append(db_utils.ClientIDToInt(client_id))

    if min_create_time is not None:
      conditions.append("timestamp >= FROM_UNIXTIME(%s)")
      args.append(mysql_utils.RDFDatetimeToTimestamp(min_create_time))

    if max_create_time is not None:
      conditions.append("timestamp <= FROM_UNIXTIME(%s)")
      args.append(mysql_utils.RDFDatetimeToTimestamp(max_create_time))

    if not include_child_flows:
      conditions.append("parent_flow_id IS NULL")

    query = "SELECT {} FROM flows".format(self.FLOW_DB_FIELDS)
    if conditions:
      query += " WHERE " + " AND ".join(conditions)

    cursor.execute(query, args)
    return [self._FlowObjectFromRow(row) for row in cursor.fetchall()]
993054
def inputs(ctx, client, revision, paths):
    
    """
    from renku.models.provenance import ProcessRun

    graph = Graph(client)
    paths = set(paths)
    nodes = graph.build(revision=revision)

    commits = {node.commit for node in nodes}
    candidates = {(node.commit, node.path)
                  for node in nodes if not paths or node.path in paths}

    input_paths = set()

    for commit in commits:
        activity = graph.activities[commit]

        if isinstance(activity, ProcessRun):
            for usage in activity.qualified_usage:
                for entity in usage.entity.entities:
                    path = str((usage.client.path / entity.path).relative_to(
                        client.path
                    ))
                    usage_key = (entity.commit, entity.path)

                    if path not in input_paths and usage_key in candidates:
                        input_paths.add(path)

    click.echo('\n'.join(graph._format_path(path) for path in input_paths))
    ctx.exit(0 if not paths or len(input_paths) == len(paths) else 1)
364638
def _parse_metadata(self, meta):
        """"""
        formatted_fields = self.settings['FORMATTED_FIELDS']

        output = collections.OrderedDict()
        for name, value in meta.items():
            name = name.lower()
            if name in formatted_fields:
                rendered = self._render(value).strip()
                output[name] = self.process_metadata(name, rendered)
            else:
                output[name] = self.process_metadata(name, value)
        return output
783488
def load_from_stream(self, var):
        """"""
        dims = []
        for d in var.shape:
            dim = Dimension(None, d.name)
            dim.load_from_stream(d)
            dims.append(dim)

        self.dimensions = tuple(dim.name for dim in dims)
        self.shape = tuple(dim.size for dim in dims)
        self.ndim = len(var.shape)
        self._unpack_attrs(var.atts)

        data, dt, type_name = unpack_variable(var)
        if data is not None:
            data = data.reshape(self.shape)
        self._data = data
        self.dtype = dt
        self.datatype = type_name

        if hasattr(var, 'enumType') and var.enumType:
            self.datatype = var.enumType
            self._enum = True
587840
def _copy_streamflow_values(self):
        
        log('Creating streamflow variable', 'INFO')
        q_var = self.cf_nc.createVariable(
            self.output_flow_var_name, 'f4', (self.output_id_dim_name, 'time'))
        q_var.long_name = 'Discharge'
        q_var.units = 'm^3/s'
        q_var.coordinates = 'time lat lon z'
        q_var.grid_mapping = 'crs'
        q_var.source = ('Generated by the Routing Application for Parallel '
                        'computatIon of Discharge (RAPID) river routing '
                        'model.')
        q_var.references = 'http://rapid-hub.org/'
        q_var.comment = ('lat, lon, and z values taken at midpoint of river '
                         'reach feature')

        log('Copying streamflow values', 'INFO')
        master_begin_time_step_index = 1
        # to reduce RAM, copy by chunks
        max_2d_dimension = 1000000000  # ~8GB Max
        for raw_nc in self.raw_nc_list:
            max_time_step_size = min(raw_nc.size_time,
                                     max(1, int(float(max_2d_dimension) /
                                                float(raw_nc.size_river_id))))
            raw_nc_begin_time_step_index = 0
            for raw_nc_time_index in \
                    xrange(0, raw_nc.size_time, max_time_step_size):
                time_interval_size = \
                    max(1, min(raw_nc.size_time - raw_nc_time_index,
                               max_time_step_size))

                raw_nc_end_time_step_index = \
                    raw_nc_begin_time_step_index + time_interval_size
                master_end_time_step_index = \
                    master_begin_time_step_index + time_interval_size

                q_var[:,
                      master_begin_time_step_index:master_end_time_step_index]\
                    = raw_nc.get_qout(
                        time_index_start=raw_nc_begin_time_step_index,
                        time_index_end=raw_nc_end_time_step_index)

                master_begin_time_step_index = master_end_time_step_index
                raw_nc_begin_time_step_index = raw_nc_end_time_step_index

        log('Adding initial streamflow values', 'INFO')
        # add initial flow to RAPID output file
        if self.qinit_file and self.rapid_connect_file:
            lookup_table = csv_to_list(self.rapid_connect_file)
            lookup_comids = np.array([int(float(row[0])) for row
                                      in lookup_table])

            init_flow_table = csv_to_list(self.qinit_file)

            for index, comid in enumerate(
                    self.cf_nc.variables[self.output_id_dim_name][:]):
                try:
                    lookup_index = np.where(lookup_comids == comid)[0][0]
                except IndexError:
                    log('COMID %s misssing in rapid_connect file' % comid,
                        'ERROR')
                q_var[index, 0] = float(init_flow_table[lookup_index][0])
        else:
            for index, comid in enumerate(
                    self.cf_nc.variables[self.output_id_dim_name][:]):
                q_var[index, 0] = 0
289221
def _GetMetadataForClients(self, client_urns):
    """"""

    result = {}
    metadata_to_fetch = set()

    for urn in client_urns:
      try:
        result[urn] = self._cached_metadata[urn]
      except KeyError:
        metadata_to_fetch.add(urn)

    if metadata_to_fetch:
      if data_store.RelationalDBEnabled():
        client_ids = set(urn.Basename() for urn in metadata_to_fetch)
        infos = data_store.REL_DB.MultiReadClientFullInfo(client_ids)

        fetched_metadata = [
            export.GetMetadata(client_id, info)
            for client_id, info in infos.items()
        ]
      else:
        client_fds = aff4.FACTORY.MultiOpen(
            metadata_to_fetch, mode="r", token=self.token)

        fetched_metadata = [
            export.GetMetadataLegacy(client_fd, token=self.token)
            for client_fd in client_fds
        ]

      for metadata in fetched_metadata:
        metadata.source_urn = self.source_urn

        self._cached_metadata[metadata.client_urn] = metadata
        result[metadata.client_urn] = metadata
        metadata_to_fetch.remove(metadata.client_urn)

      for urn in metadata_to_fetch:
        default_mdata = export.ExportedMetadata(source_urn=self.source_urn)
        result[urn] = default_mdata
        self._cached_metadata[urn] = default_mdata

    return [result[urn] for urn in client_urns]
1169850
def BatchConvert(self, metadata_value_pairs, token=None):
    """"""

    urns = [urn for metadata, urn in metadata_value_pairs]
    urns_dict = dict((urn, metadata) for metadata, urn in metadata_value_pairs)

    results = []
    for hash_urn, client_files in filestore.HashFileStore.GetClientsForHashes(
        urns, token=token):
      for hit in client_files:
        metadata = ExportedMetadata(urns_dict[hash_urn])
        metadata.client_urn = rdfvalue.RDFURN(hit).Split(2)[0]

        result = ExportedFileStoreHash(
            metadata=metadata,
            hash=hash_urn.hash_value,
            fingerprint_type=hash_urn.fingerprint_type,
            hash_type=hash_urn.hash_type,
            target_urn=hit)
        results.append(result)

    return results