This client generates a similarity graph from features in PE Files.
4789
def run():
    """"""

    # Grab server args
    args = client_helper.grab_server_args()

    # Start up workbench connection
    workbench = zerorpc.Client(timeout=300, heartbeat=60)
    workbench.connect('tcp://'+args['server']+':'+args['port'])

    # Test out PEFile -> pe_deep_sim -> pe_jaccard_sim -> graph
    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../data/pe/bad')
    bad_files = [os.path.join(data_path, child) for child in os.listdir(data_path)][:5]
    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../data/pe/good')
    good_files = [os.path.join(data_path, child) for child in os.listdir(data_path)][:5]

    # Clear any graph in the Neo4j database
    workbench.clear_graph_db()

    # First throw them into workbench and add them as nodes into the graph
    all_md5s = add_it(workbench, bad_files, ['exe', 'bad']) + add_it(workbench, good_files, ['exe', 'good'])

    # Make a sample set
    sample_set = workbench.store_sample_set(all_md5s)

    # Compute pe_features on all files of type pe, just pull back the sparse features
    import_gen = workbench.set_work_request('pe_features', sample_set, ['md5', 'sparse_features.imported_symbols'])
    imports = [{'md5': r['md5'], 'features': r['imported_symbols']} for r in import_gen]

    # Compute pe_features on all files of type pe, just pull back the sparse features
    warning_gen = workbench.set_work_request('pe_features', sample_set, ['md5', 'sparse_features.pe_warning_strings'])
    warnings = [{'md5': r['md5'], 'features': r['pe_warning_strings']} for r in warning_gen]

    # Compute strings on all files of type pe, just pull back the string_list
    string_gen = workbench.set_work_request('strings', sample_set, ['md5', 'string_list'])
    strings = [{'md5': r['md5'], 'features': r['string_list']} for r in string_gen]

    # Compute pe_peid on all files of type pe, just pull back the match_list
    # Fixme: commenting this out until we figure out why peid is SO slow
    '''
    peid_gen = workbench.set_work_request('pe_peid', sample_set, ['md5', 'match_list']})
    peids = [{'md5': r['md5'], 'features': r['match_list']} for r in peid_gen]
    '''

    # Compute the Jaccard Index between imported systems and store as relationships
    sims = jaccard_sims(imports)
    for sim_info in sims:
        workbench.add_rel(sim_info['source'], sim_info['target'], 'imports')

    # Compute the Jaccard Index between warnings and store as relationships
    sims = jaccard_sims(warnings)
    for sim_info in sims:
        workbench.add_rel(sim_info['source'], sim_info['target'], 'warnings')

    # Compute the Jaccard Index between strings and store as relationships
    sims = jaccard_sims(strings)
    for sim_info in sims:
        workbench.add_rel(sim_info['source'], sim_info['target'], 'strings')

    # Compute the Jaccard Index between peids and store as relationships
    # Fixme: commenting this out until we figure out why peid is SO slow
    '''
    sims = jaccard_sims(peids)
    for sim_info in sims:
        workbench.add_rel(sim_info['source'], sim_info['target'], 'peids')
    '''

    # Compute pe_deep_sim on all files of type pe
    results = workbench.set_work_request('pe_deep_sim', sample_set)

    # Store the ssdeep sims as relationships
    for result in list(results):
        for sim_info in result['sim_list']:
            workbench.add_rel(result['md5'], sim_info['md5'], 'ssdeep')

    # Let them know where they can get there graph
    print 'All done: go to http://localhost:7474/browser and execute this query: "%s"' % \
        ('match (n)-[r]-() return n,r')
388086
function loadGraphSONSync(graph, filename) {
        var FileInputStream = Tinkerpop.autoImport('FileInputStream');
        var stream = new FileInputStream(filename);
        var builder = Tinkerpop.GraphSONReader.build();
        var mapper = _newGraphSONMapperSync();
        builder.mapper(mapper);
        var reader = builder.create();
        reader.readGraph(stream, graph);
        return graph;
    }
1308268
def _shared_features(adense, bdense):
    
    a_indices = set(nonzero(adense))
    b_indices = set(nonzero(bdense))

    shared = list(a_indices & b_indices)
    diff = list(a_indices - b_indices)
    Ndiff = len(diff)

    return Ndiff
758942
def graph_from_dot_file(path):
    

    fd = open(path, 'rb')
    data = fd.read()
    fd.close()

    return graph_from_dot_data(data)
422294
def equiv_graph(self):
        
        eg = nx.Graph()
        for (u,v,d) in self.get_graph().edges(data=True):
            if d['pred'] == 'equivalentTo':
                eg.add_edge(u,v)
        return eg
492066
def create_graph():
  """"""
  # Creates graph from saved graph_def.pb.
  with tf.gfile.FastGFile(os.path.join(
      FLAGS.model_dir, 'classify_image_graph_def.pb'), 'rb') as f:
    graph_def = tf.GraphDef()
    graph_def.ParseFromString(f.read())
    _ = tf.import_graph_def(graph_def, name='')
968030
def generate_graphs(data, name, results_dir):
    
    graphs.resp_graph_raw(data['raw'], name + '_response_times.svg', results_dir)
    graphs.resp_graph(data['compiled'], name + '_response_times_intervals.svg', results_dir)
    graphs.tp_graph(data['compiled'], name + '_throughput.svg', results_dir)
943088
def from_perseus(network_table, networks):
    
    graphs = []
    for guid, graph_attr in zip(network_table['GUID'], network_table.values):
        network = networks[guid]
        edge_table = network['edge_table']
        if edge_table[['Source', 'Target']].duplicated().any():
            warnings.warn('Duplicate edges were found and ignored in network {}'.format(network['name']))
        G = nx.from_pandas_edgelist(edge_table, 'Source', 'Target', True, create_using=nx.DiGraph())
        for attr, value in zip(network_table.columns, graph_attr):
            G.graph[attr] = value
        node_table = network['node_table']
        if node_table['Node'].duplicated().any():
            warnings.warn('Duplicate nodes were found and ignored in network {}'.format(network['name']))
        node_column = node_table['Node']
        for name, attributes in zip(node_column, node_table.values):
            if name not in G:
                G.add_node(name)
            for attr, value in zip(node_table.columns, attributes):
                G.node[name][attr] = value
        graphs.append(G)
    return graphs
372846
public static ParametricFactorGraph buildSequenceModel(Iterable<String> emissionFeatureLines,
      String featureDelimiter) {
    // Read in the possible values of each variable.
    List<String> words = StringUtils.readColumnFromDelimitedLines(emissionFeatureLines, 0, featureDelimiter);
    List<String> labels = StringUtils.readColumnFromDelimitedLines(emissionFeatureLines, 1, featureDelimiter);
    List<String> emissionFeatures = StringUtils.readColumnFromDelimitedLines(emissionFeatureLines, 2, featureDelimiter);
    // Create dictionaries for each variable's values.
    DiscreteVariable wordType = new DiscreteVariable("word", words);
    DiscreteVariable labelType = new DiscreteVariable("label", labels);
    DiscreteVariable emissionFeatureType = new DiscreteVariable("emissionFeature", emissionFeatures);

    // Create a dynamic factor graph with a single plate replicating
    // the input/output variables.
    ParametricFactorGraphBuilder builder = new ParametricFactorGraphBuilder();
    builder.addPlate(PLATE_NAME, new VariableNumMap(Ints.asList(1, 2),
        Arrays.asList(INPUT_NAME, OUTPUT_NAME), Arrays.asList(wordType, labelType)), 10000);
    String inputPattern = PLATE_NAME + "/?(0)/" + INPUT_NAME;
    String outputPattern = PLATE_NAME + "/?(0)/" + OUTPUT_NAME;
    String nextOutputPattern = PLATE_NAME + "/?(1)/" + OUTPUT_NAME;
    VariableNumMap plateVars = new VariableNumMap(Ints.asList(1, 2),
        Arrays.asList(inputPattern, outputPattern), Arrays.asList(wordType, labelType));

    // Read in the emission features (for the word/label weights).
    VariableNumMap x = plateVars.getVariablesByName(inputPattern);
    VariableNumMap y = plateVars.getVariablesByName(outputPattern);
    VariableNumMap emissionFeatureVar = VariableNumMap.singleton(0, "emissionFeature", emissionFeatureType);
    TableFactor emissionFeatureFactor = TableFactor.fromDelimitedFile(
        Arrays.asList(x, y, emissionFeatureVar), emissionFeatureLines,
        featureDelimiter, false, SparseTensorBuilder.getFactory())
        .cacheWeightPermutations();

    System.out.println(emissionFeatureFactor.getVars());

    // Add a parametric factor for the word/label weights
    DiscreteLogLinearFactor emissionFactor = new DiscreteLogLinearFactor(x.union(y), emissionFeatureVar,
        emissionFeatureFactor);
    builder.addFactor(WORD_LABEL_FACTOR, emissionFactor,
        VariableNamePattern.fromTemplateVariables(plateVars, VariableNumMap.EMPTY));

    // Create a factor connecting adjacent labels
    VariableNumMap adjacentVars = new VariableNumMap(Ints.asList(0, 1),
        Arrays.asList(outputPattern, nextOutputPattern), Arrays.asList(labelType, labelType));
    builder.addFactor(TRANSITION_FACTOR, DiscreteLogLinearFactor.createIndicatorFactor(adjacentVars),
        VariableNamePattern.fromTemplateVariables(adjacentVars, VariableNumMap.EMPTY));

    return builder.build();
  }
549166
def makefile(graph):
    """"""
    from renku.models.provenance.activities import ProcessRun, WorkflowRun

    for activity in graph.activities.values():
        if not isinstance(activity, ProcessRun):
            continue
        elif isinstance(activity, WorkflowRun):
            steps = activity.subprocesses.values()
        else:
            steps = [activity]

        for step in steps:
            click.echo(' '.join(step.outputs) + ': ' + ' '.join(step.inputs))
            tool = step.process
            click.echo(
                '\t@' + ' '.join(tool.to_argv()) + ' ' + ' '.join(
                    tool.STD_STREAMS_REPR[key] + ' ' + str(path)
                    for key, path in tool._std_streams().items()
                )
            )
1013670
def _generate_examples(self, filepath):
    """"""
    rows_per_pair_id = collections.defaultdict(list)

    with tf.io.gfile.GFile(filepath) as f:
      reader = csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
      for row in reader:
        rows_per_pair_id[row['pairID']].append(row)

    for rows in six.itervalues(rows_per_pair_id):
      premise = {row['language']: row['sentence1'] for row in rows}
      hypothesis = {row['language']: row['sentence2'] for row in rows}
      yield {
          'premise': premise,
          'hypothesis': hypothesis,
          'label': rows[0]['gold_label'],
      }
1172241
def files_log_graph(self, stream):
        '''  '''
        file_log = list(stream)
        print 'Entering file_log_graph...(%d rows)' % len(file_log)
        for row in file_log:

            # If the mime-type is interesting add the uri and the host->uri->host relationships
            if row['mime_type'] not in self.exclude_mime_types:

                # Check for weird conditions
                if (row['total_bytes'] == '-'):
                    continue
                if ('-' in row['md5']):
                    continue

                # Check for missing bytes
                if row['missing_bytes']:
                    labels = ['file','missing']
                else:
                    labels = ['file']

                # Make the file node name kewl
                name = '%6s  %s  %.0f-KB' % (row['md5'][:6], row['mime_type'], row['total_bytes']/1024.0)
                if row['missing_bytes']:
                    name += '*'
                name = name.replace('application/','')

                # Add the file node
                self.add_node(row['md5'], name, labels)

                # Add the tx_host
                self.add_node(row['tx_hosts'], row['tx_hosts'], ['host'])

                # Add the file->tx_host relationship
                self.add_rel(row['tx_hosts'], row['md5'], 'file')
1096219
def sequence_to_graph(G, seq, color='black'):
    
    for x in seq:
        if x.endswith("_1"):  # Mutation
            G.node(x, color=color, width="0.1", shape="circle", label="")
        else:
            G.node(x, color=color)
    for a, b in pairwise(seq):
        G.edge(a, b, color=color)
536049
def _gen_machine_graph(start, end, force_overwrite=False):
    
    filename = graphs.get_machine_graph_filename(start, end)
    csv_filename = os.path.join(GRAPH_ROOT, filename + '.csv')
    png_filename = os.path.join(GRAPH_ROOT, filename + '.png')

    _check_directory_exists(csv_filename)
    _check_directory_exists(png_filename)

    if not settings.GRAPH_DEBUG or force_overwrite:
        if os.path.exists(csv_filename):
            if os.path.exists(png_filename):
                return

    machine_list = Machine.objects.all()

    plt.subplots(figsize=(4, 4))

    data = []
    labels = []

    with open(csv_filename, 'w') as csv_file:
        csv_writer = csv.writer(csv_file)
        for machine in machine_list.iterator():
            hours, jobs = usage.get_machine_usage(machine, start, end)
            if hours > 0:
                csv_writer.writerow([machine.name, hours, jobs])
                data.append(hours)
                labels.append(machine.name)

    plt.pie(data, labels=labels, autopct='%1.1f%%', shadow=True)

    del data
    del labels

    plt.tight_layout()
    plt.savefig(png_filename)
    plt.close()
106856
def to_graph(self):
        
        edges = set()
        for clause, target in self.edges_iter():
            for source, signature in clause:
                edges.add((source, target, signature))

        return Graph.from_tuples(edges)
49972
def _generate_examples(self, file_id, extracted_dirs):
    """"""
    filedir = os.path.join(extracted_dirs["img_align_celeba"],
                           "img_align_celeba")
    img_list_path = extracted_dirs["list_eval_partition"]
    landmarks_path = extracted_dirs["landmarks_celeba"]
    attr_path = extracted_dirs["list_attr_celeba"]

    with tf.io.gfile.GFile(img_list_path) as f:
      files = [
          line.split()[0]
          for line in f.readlines()
          if int(line.split()[1]) == file_id
      ]

    attributes = self._process_celeba_config_file(attr_path)
    landmarks = self._process_celeba_config_file(landmarks_path)

    for file_name in sorted(files):
      path = os.path.join(filedir, file_name)

      yield {
          "image": path,
          "landmarks": {
              k: v for k, v in zip(landmarks[0], landmarks[1][file_name])
          },
          "attributes": {
              # atributes value are either 1 or -1, so convert to bool
              k: v > 0 for k, v in zip(attributes[0], attributes[1][file_name])
          },
      }
717118
def _get_graph(graph, filename):
    """"""
    try:
        rendered = graph.rendered_file
    except AttributeError:
        try:
            graph.render(os.path.join(server.tmpdir, filename), format='png')
            rendered = filename
        except OSError:
            rendered = None
    graph.rendered_file = rendered
    return rendered