This worker puts the output of pe_features into a dictionary of dataframes
510279
def execute(self, input_data):
        """"""
        if 'sample' in input_data:
            print 'Warning: PEFeaturesDF is supposed to be called on a sample_set'
            self.samples.append(input_data['sample']['md5'])
        else:
            self.samples = input_data['sample_set']['md5_list']

        # Make a sample set
        sample_set = self.workbench.store_sample_set(self.samples)

        # Dense Features               
        dense_features = self.workbench.set_work_request('pe_features', sample_set, ['md5', 'tags', 'dense_features'])

        # Fixme: There's probably a nicer/better way to do this
        flat_features = []
        for feat in dense_features:
            feat['dense_features'].update({'md5': feat['md5'], 'tags': feat['tags']})
            flat_features.append(feat['dense_features'])
        dense_df = pd.DataFrame(flat_features)
        df_packed = dense_df.to_msgpack()
        dense_df_md5 = self.workbench.store_sample(df_packed, 'pe_features_dense_df', 'dataframe')
        
        # Sparse Features
        sparse_features = self.workbench.set_work_request('pe_features', sample_set, ['md5', 'tags', 'sparse_features'])

        # Fixme: There's probably a nicer/better way to do this
        flat_features = []
        for feat in sparse_features:
            feat['sparse_features'].update({'md5': feat['md5'], 'tags': feat['tags']})
            flat_features.append(feat['sparse_features'])
        sparse_df = pd.DataFrame(flat_features)
        df_packed = sparse_df.to_msgpack()
        sparse_df_md5 = self.workbench.store_sample(df_packed, 'pe_features_sparse_df', 'dataframe')

        # Return the dataframes
        return {'dense_features': dense_df_md5, 'sparse_features': sparse_df_md5}
870876
def _post_process_output(res):
    
    if isinstance(res, list):
        if len(res) == 0:
            return res
        elif len(res) == 1:
            return _post_process_output(res[0])
        elif isinstance(res[0], numpy.ndarray):
            return numpy.array(res)
        elif isinstance(res[0], dict):
            import pandas
            return pandas.DataFrame(res).values
        else:
            ls = [len(r) for r in res]
            mi = min(ls)
            if mi != max(ls):
                raise NotImplementedError("Unable to postprocess various number of outputs in [{0}, {1}]".format(min(ls), max(ls)))
            if mi > 1:
                output = []
                for i in range(mi):
                    output.append(_post_process_output([r[i] for r in res]))
                return output
            elif isinstance(res[0], list):
                # list of lists
                if isinstance(res[0][0], list):
                    return numpy.array(res)
                elif len(res[0]) == 1 and isinstance(res[0][0], dict):
                    return _post_process_output([r[0] for r in res])
                elif len(res) == 1:
                    return res
                else:
                    if len(res[0]) != 1:
                        raise NotImplementedError("Not conversion implemented for {0}".format(res))
                    st = [r[0] for r in res]
                    return numpy.vstack(st)
            else:
                return res
    else:
        return res
435672
def help_dataframe_pe(self):
        
        help =  '%sMaking a DataFrame: %s how to make a dataframe from pe files' % (color.Yellow, color.Green)
        help += '\n\n%sPE Files Example (loading a directory):'  % (color.Green)
        help += '\n\t%s> load_sample /path/to/pe/bad [\'bad\', \'case_69\']' % (color.LightBlue)
        help += '\n\n\t%sSearch for all samples in the database that are pe files,'  % (color.Green)
        help += '\n\t%sthis command returns the sample_set containing the matching items'% (color.Green)
        help += '\n\t%s> my_exes = search([\'exe\'])' % (color.LightBlue)
        help += '\n\n\t%sRun workers on this sample_set:'  % (color.Green)
        help += '\n\t%s> pe_outputs = set_work_request(\'pe_features\', my_exes, [\'md5\', \'dense_features.*\', \'tags\'])' % (color.LightBlue)
        help += '\n\n\t%sMake a DataFrame:'  % (color.Green)
        help += '\n\t%s> pe_df = pd.DataFrame(pe_outputs) %s' % (color.LightBlue, color.Normal)
        help += '\n\t%s> pe_df.head() %s' % (color.LightBlue, color.Normal)
        help += '\n\t%s> pe_df = flatten_tags(pe_df) %s' % (color.LightBlue, color.Normal)
        help += '\n\t%s> pe_df.hist(\'check_sum\',\'tags\') %s' % (color.LightBlue, color.Normal)
        help += '\n\t%s> pe_df.bloxplot(\'check_sum\',\'tags\') %s' % (color.LightBlue, color.Normal)
        return help
557716
def _decode_input_tensor_to_features_dict(feature_map, hparams):
  
  inputs = tf.convert_to_tensor(feature_map["inputs"])
  input_is_image = False

  x = inputs
  p_hparams = hparams.problem_hparams
  # Add a third empty dimension
  x = tf.expand_dims(x, axis=[2])
  x = tf.to_int32(x)
  input_space_id = tf.constant(p_hparams.input_space_id)
  target_space_id = tf.constant(p_hparams.target_space_id)

  features = {}
  features["input_space_id"] = input_space_id
  features["target_space_id"] = target_space_id
  features["decode_length"] = (
      IMAGE_DECODE_LENGTH if input_is_image else tf.shape(x)[1] + 50)
  features["inputs"] = x
  return features
189268
def pad_batch(features, batch_multiple):
  """"""
  feature = list(features.items())[0][1]
  batch_size = tf.shape(feature)[0]
  mod = batch_size % batch_multiple
  has_mod = tf.cast(tf.cast(mod, tf.bool), tf.int32)
  batch_padding = batch_multiple * has_mod - mod

  padded_features = {}
  for k, feature in features.items():
    rank = len(feature.shape)
    paddings = [[0, 0] for _ in range(rank)]
    paddings[0][1] = batch_padding
    padded_feature = tf.pad(feature, paddings)
    padded_features[k] = padded_feature
  return padded_features
967608
private void saveFeatureIndexes(FeatureIndex[] index, String outputDir) throws IOException
	{
		logger.info("");

		// save the indexes
		for (int i=0;i<index.length;i++)
		{
			logger.debug("dic" + i + " size " + index[i].size());

			File tmp = new File(outputDir + "dic" + i);
			//File tmp = File.createTempFile("dic" + i, null);
			//tmp.deleteOnExit();

			BufferedWriter bwd = new BufferedWriter(new FileWriter(tmp));
			index[i].write(bwd);
			bwd.close();

			// add the dictionary to the model
			//model.add(tmp, "dic" + i);
		} // end for
	}
651907
def to_dict(self):
        
        final_res = {
            "param": self._param,
            "unused_param": self.unused_param,
            "execution_time": self._exec_time,
            "output": {"accuracy": self.get_accuracy(),
                       "weights": self.get_weights(),
                       "splines": self._splines
                       }
        }
        return final_res
572938
def make_dataframe(result):
    
    import pandas as pd
    ret = {}
    if isinstance(result,dict):
        if 'timeseries' in result:
            result = result['timeseries']
    for uuid, data in result.items():
        df = pd.DataFrame(data)
        if len(df.columns) == 5: # statistical data
            df.columns = ['time','min','mean','max','count']
        else:
            df.columns = ['time','value']
        df['time'] = pd.to_datetime(df['time'],unit='ns')
        df = df.set_index(df.pop('time'))
        ret[uuid] = df
    return ret
1163275
def _extract_features(self, preprocessed_data, verbose=True):
        
        last_progress_update = _time.time()
        progress_header_printed = False

        deep_features = _tc.SArrayBuilder(_np.ndarray)
        from mxnet.gluon import utils

        if _mac_ver() < (10, 14):
            # Use MXNet
            preprocessed_data = mx.nd.array(preprocessed_data)

            ctx_list = self.ctx
            if len(preprocessed_data) < len(ctx_list):
                ctx_list = ctx_list[:len(preprocessed_data)]
            batches = utils.split_and_load(preprocessed_data, ctx_list=ctx_list, even_split=False)

            for i, cur_batch in enumerate(batches):
                y = self.vggish_model.forward(cur_batch).asnumpy()
                for j in y:
                    deep_features.append(j)

                # If `verbose` is set, print an progress update about every 20s
                if verbose and _time.time() - last_progress_update >= 20:
                    if not progress_header_printed:
                        print("Extracting deep features -")
                        progress_header_printed = True
                    print("Extracted {} of {} batches".format(i, len(batches)))
                    last_progress_update = _time.time()
            if progress_header_printed:
                print("Extracted {} of {} batches\n".format(len(batches), len(batches)))

        else:
            # Use Core ML
            for i, cur_example in enumerate(preprocessed_data):
                for cur_frame in cur_example:
                    x = {'input1': [cur_frame]}
                    y = self.vggish_model.predict(x)
                    deep_features.append(y['output1'])

                # If `verbose` is set, print an progress update about every 20s
                if verbose and _time.time() - last_progress_update >= 20:
                    if not progress_header_printed:
                        print("Extracting deep features -")
                        progress_header_printed = True
                    print("Extracted {} of {}".format(i, len(preprocessed_data)))
                    last_progress_update = _time.time()
            if progress_header_printed:
                print("Extracted {} of {}\n".format(len(preprocessed_data), len(preprocessed_data)))

        return deep_features.close()
495189
def wireframe(dataset):
        
        alg = vtk.vtkExtractEdges()
        alg.SetInputDataObject(dataset)
        alg.Update()
        return _get_output(alg)
1314866
def write_to_files_np(features, tokenizer, max_seq_length,
                      max_predictions_per_seq, output_files):
    # pylint: disable=unused-argument
    """"""
    next_sentence_labels = []
    valid_lengths = []

    assert len(output_files) == 1, 'numpy format only support single output file'
    output_file = output_files[0]
    (input_ids, segment_ids, masked_lm_positions, masked_lm_ids,
     masked_lm_weights, next_sentence_labels, valid_lengths) = features
    total_written = len(next_sentence_labels)

    # store variable length numpy array object directly.
    outputs = collections.OrderedDict()
    outputs['input_ids'] = np.array(input_ids, dtype=object)
    outputs['segment_ids'] = np.array(segment_ids, dtype=object)
    outputs['masked_lm_positions'] = np.array(masked_lm_positions, dtype=object)
    outputs['masked_lm_ids'] = np.array(masked_lm_ids, dtype=object)
    outputs['masked_lm_weights'] = np.array(masked_lm_weights, dtype=object)
    outputs['next_sentence_labels'] = np.array(next_sentence_labels, dtype='int32')
    outputs['valid_lengths'] = np.array(valid_lengths, dtype='int32')

    np.savez_compressed(output_file, **outputs)
    logging.info('Wrote %d total instances', total_written)
633835
def shp_dict(shp_fn, fields=None, geom=True):
    
    from pygeotools.lib import timelib
    ds = ogr.Open(shp_fn)
    lyr = ds.GetLayer()
    nfeat = lyr.GetFeatureCount()
    print('%i input features\n' % nfeat)
    if fields is None:
        fields = shp_fieldnames(lyr)
    d_list = []
    for n,feat in enumerate(lyr):
        d = {}
        if geom:
            geom = feat.GetGeometryRef()
            d['geom'] = geom
        for f_name in fields:
            i = str(feat.GetField(f_name))
            if 'date' in f_name:
                # date_f = f_name
                #If d is float, clear off decimal
                i = i.rsplit('.')[0]
                i = timelib.strptime_fuzzy(str(i))
            d[f_name] = i
        d_list.append(d)
    #d_list_sort = sorted(d_list, key=lambda k: k[date_f])
    return d_list
248213
def to_meta_df(trained_regressor,
               target_gene_name):
    
    n_estimators = len(trained_regressor.estimators_)

    return pd.DataFrame({'target': [target_gene_name], 'n_estimators': [n_estimators]})
1042692
def to_example_dict(encoder, inputs, mask, outputs):
  """"""
  # Inputs
  bases = []
  input_ids = []
  last_idx = -1
  for row in np.argwhere(inputs):
    idx, base_id = row
    idx, base_id = int(idx), int(base_id)
    assert idx > last_idx  # if not, means 2 True values in 1 row
    # Some rows are all False. Those rows are mapped to UNK_ID.
    while idx != last_idx + 1:
      bases.append(encoder.UNK)
      last_idx += 1
    bases.append(encoder.BASES[base_id])
    last_idx = idx
  assert len(inputs) == len(bases)

  input_ids = encoder.encode(bases)
  input_ids.append(text_encoder.EOS_ID)

  # Targets: mask and output
  targets_mask = [float(v) for v in mask]
  # The output is (n, m); store targets_shape so that it can be reshaped
  # properly on the other end.
  targets = [float(v) for v in outputs.flatten()]
  targets_shape = [int(dim) for dim in outputs.shape]
  assert mask.shape[0] == outputs.shape[0]

  example_keys = ["inputs", "targets_mask", "targets", "targets_shape"]
  ex_dict = dict(
      zip(example_keys, [input_ids, targets_mask, targets, targets_shape]))
  return ex_dict
881315
def _dict_to_tf_features(example_dict):
  """"""
  features = {k: _item_to_tf_feature(v, k) for k, v
              in six.iteritems(example_dict)}
  return tf.train.Features(feature=features)
446661
def _maybe_pandas_data(data, feature_names, feature_types):
    

    if not isinstance(data, DataFrame):
        return data, feature_names, feature_types

    data_dtypes = data.dtypes
    if not all(dtype.name in PANDAS_DTYPE_MAPPER for dtype in data_dtypes):
        bad_fields = [data.columns[i] for i, dtype in
                      enumerate(data_dtypes) if dtype.name not in PANDAS_DTYPE_MAPPER]

        msg = """DataFrame.dtypes for data must be int, float or bool.
                Did not expect the data types in fields """
        raise ValueError(msg + ', '.join(bad_fields))

    if feature_names is None:
        if isinstance(data.columns, MultiIndex):
            feature_names = [
                ' '.join([str(x) for x in i])
                for i in data.columns
            ]
        else:
            feature_names = data.columns.format()

    if feature_types is None:
        feature_types = [PANDAS_DTYPE_MAPPER[dtype.name] for dtype in data_dtypes]

    data = data.values.astype('float')

    return data, feature_names, feature_types
225545
def as_data_frame(self) -> pandas.DataFrame:
        
        header_gene = {}
        header_multiplex = {}
        headr_transitions = {}
        for gene in self.influence_graph.genes:
            header_gene[gene] = repr(gene)
            header_multiplex[gene] = f"active multiplex on {gene!r}"
            headr_transitions[gene] = f"K_{gene!r}"
        
        columns = defaultdict(list)
        for state in self.table.keys():
            for gene in self.influence_graph.genes:
                columns[header_gene[gene]].append(state[gene])
                columns[header_multiplex[gene]].append(self._repr_multiplexes(gene, state))
                columns[headr_transitions[gene]].append(self._repr_transition(gene, state))

        header = list(header_gene.values()) + list(header_multiplex.values()) + list(headr_transitions.values())
        return pandas.DataFrame(columns, columns=header)