Rank features using Random Forest classifier and the recursive feature elimination
12446
def rank_features(self, inputs: pd.DataFrame, targets: pd.DataFrame, problem_type='classification') -> pd.DataFrame:
        
        try:

            X = normalize( \
                                inputs.apply(pd.to_numeric, errors='coerce') \
                                .applymap(lambda x: 0.0 if np.isnan(x) else x) \
                                .values, axis=0)
            y = targets.iloc[:,0].values
            original_columns = list(inputs)
            print(original_columns)

            if problem_type == 'classification':
                predictor = RandomForestClassifier()
            else:
                predictor = RandomForestRegressor()

            rfe = RFE(predictor, n_features_to_select=1)
            rfe = rfe.fit(X, y)
            
            best_features = [original_columns[i-1] for i in rfe.ranking_]

        except:
            logging.exception("Failed")
            # If any error occurs, just return indices in original order
            # In the future we should consider a better error handling strategy
            best_features = [i for i in range(inputs.shape[1])]

        return best_features
399555
def get_row_missing(xc, xd, cdiffs, index, cindices, dindices):
    
    row = np.empty(0, dtype=np.double)  # initialize empty row
    cinst1 = xc[index]  # continuous-valued features for index instance
    dinst1 = xd[index]  # discrete-valued features for index instance
    # Boolean mask locating missing values for continuous features for index instance
    can = cindices[index]
    # Boolean mask locating missing values for discrete features for index instance
    dan = dindices[index]
    tf = len(cinst1) + len(dinst1)  # total number of features.

    # Progressively compare current instance to all others. Excludes comparison with self indexed instance. (Building the distance matrix triangle).
    for j in range(index):
        dist = 0
        dinst2 = xd[j]  # discrete-valued features for compared instance
        cinst2 = xc[j]  # continuous-valued features for compared instance

        # Manage missing values in discrete features
        # Boolean mask locating missing values for discrete features for compared instance
        dbn = dindices[j]
        # indexes where there is at least one missing value in the feature between an instance pair.
        idx = np.unique(np.append(dan, dbn))
        # Number of features excluded from distance calculation due to one or two missing values within instance pair. Used to normalize distance values for comparison.
        dmc = len(idx)
        d1 = np.delete(dinst1, idx)  # delete unique missing features from index instance
        d2 = np.delete(dinst2, idx)  # delete unique missing features from compared instance

        # Manage missing values in continuous features
        # Boolean mask locating missing values for continuous features for compared instance
        cbn = cindices[j]
        # indexes where there is at least one missing value in the feature between an instance pair.
        idx = np.unique(np.append(can, cbn))
        # Number of features excluded from distance calculation due to one or two missing values within instance pair. Used to normalize distance values for comparison.
        cmc = len(idx)
        c1 = np.delete(cinst1, idx)  # delete unique missing features from index instance
        c2 = np.delete(cinst2, idx)  # delete unique missing features from compared instance
        # delete unique missing features from continuous value difference scores
        cdf = np.delete(cdiffs, idx)

        # Add discrete feature distance contributions (missing values excluded) - Hamming distance
        dist += len(d1[d1 != d2])

        # Add continuous feature distance contributions (missing values excluded) - Manhattan distance (Note that 0-1 continuous value normalization is included ~ subtraction of minimums cancel out)
        dist += np.sum(np.absolute(np.subtract(c1, c2)) / cdf)

        # Normalize distance calculation based on total number of missing values bypassed in either discrete or continuous features.
        tnmc = tf - dmc - cmc  # Total number of unique missing counted
        # Distance normalized by number of features included in distance sum (this seeks to handle missing values neutrally in distance calculation)
        dist = dist/float(tnmc)

        row = np.append(row, dist)

    return row
640944
def feature_importances(self):
        
        feature_names = [feature for feature in self.features.train]
        return list(zip(feature_names, self.feature_importances_))
1017585
public function getFeatureImportances(): array
    {
        // Traverse each tree and sum importance of the columns
        $sum = [];
        foreach ($this->classifiers as $tree) {
            /** @var DecisionTree $tree */
            $importances = $tree->getFeatureImportances();

            foreach ($importances as $column => $importance) {
                if (array_key_exists($column, $sum)) {
                    $sum[$column] += $importance;
                } else {
                    $sum[$column] = $importance;
                }
            }
        }

        // Normalize & sort the importance values
        $total = array_sum($sum);
        array_walk($sum, function (&$importance) use ($total): void {
            $importance /= $total;
        });
        arsort($sum);

        return $sum;
    }
757194
static void detectAdjacentAndPositionPatterns(int N, QrCodeCodeWordLocations matrix, FoundFeatures features) {

		for (int foo = 0; foo < 2; foo++) {
			for (int row = 0; row < N; row++) {
				int index = row * N;
				for (int col = 1; col < N; col++, index++) {
					if (matrix.data[index] == matrix.data[index + 1])
						features.adjacent++;
				}
				index = row * N;
				for (int col = 6; col < N; col++, index++) {
					if (matrix.data[index] && !matrix.data[index + 1] &&
							matrix.data[index + 2] && matrix.data[index + 3] && matrix.data[index + 4] &&
							!matrix.data[index + 5] && matrix.data[index + 6])
						features.position++;
				}
			}
			CommonOps_BDRM.transposeSquare(matrix);
		}

		// subtract out false positives by the mask applied to position patterns, timing, mode
		features.adjacent -= 8*9 + 2*9*8 + (N-18)*2;
	}
992043
def train_classifier(download=True, parameters=None, ngram_range=(1, 1)):
    """"""
    if download:
        download_wiki()

    path = os.path.join(l.TOPDIR, 'train.json')
    training_set = json.load(open(path))
    path = os.path.join(l.TOPDIR, 'wiki.json')
    wiki_set = json.load(open(path))

    target_names = list(set([i['unit'] for i in training_set + wiki_set]))
    train_data, train_target = [], []
    for example in training_set + wiki_set:
        train_data.append(clean_text(example['text']))
        train_target.append(target_names.index(example['unit']))

    tfidf_model = TfidfVectorizer(sublinear_tf=True,
                                  ngram_range=ngram_range,
                                  stop_words='english')

    matrix = tfidf_model.fit_transform(train_data)

    if parameters is None:
        parameters = {'loss': 'log', 'penalty': 'l2', 'n_iter': 50,
                      'alpha': 0.00001, 'fit_intercept': True}

    clf = SGDClassifier(**parameters).fit(matrix, train_target)
    obj = {'tfidf_model': tfidf_model,
           'clf': clf,
           'target_names': target_names}
    path = os.path.join(l.TOPDIR, 'clf.pickle')
    pickle.dump(obj, open(path, 'w'))
121488
def p_feature_simple(self, t):
        """"""

        if len(t) == 6:
            t[0] = Feature(t[1], t[2], t[4], line=t.lineno(1))
        elif len(t) == 5:
            t[0] = Feature(t[1], t[2], t[3], unit=t[4], line=t.lineno(1))
        elif len(t) == 4:
            t[0] = Feature(t[1], t[2], t[3], line=t.lineno(1))
70218
def _classify(self, gadget, classifier, gadget_type, iters):
        
        # Collect REIL instructions of the gadgets.
        instrs = [ir_instr for asm_instr in gadget.instrs for ir_instr in asm_instr.ir_instrs]

        # Repeat classification.
        results = []

        for _ in range(iters):
            # Reset emulator.
            self._ir_emulator.reset()

            # Generate random values for registers.
            regs_initial = self._init_regs_random()

            # Emulate gadgets.
            try:
                regs_final, mem_final = self._ir_emulator.execute_lite(
                    instrs,
                    regs_initial
                )
            except:
                # Catch emulator exceptions like ZeroDivisionError, etc.
                results += [([], [])]

                continue

            # Compute values for all registers. For example, in x86, it
            # computes 'al' from 'eax'.
            regs_initial_full = self._compute_full_context(regs_initial)
            regs_final_full = self._compute_full_context(regs_final)

            # Get written and read registers.
            regs_written = self._ir_emulator.written_registers
            regs_read = self._ir_emulator.read_registers

            # Compute modified registers.
            mod_regs = self._compute_mod_regs(
                regs_initial_full,
                regs_final_full
            )

            # Classified gadgets based on initial and final context.
            matches = classifier(
                regs_initial_full,
                regs_final_full,
                mem_final,
                regs_written,
                regs_read
            )

            # Save results.
            results += [(matches, mod_regs)]

        # Analyze results and compute candidate gadgets.
        candidates, mod_regs = self._analyze_execution_results(results)

        # Create classified gadgets.
        classified = self._create_typed_gadgets(
            gadget,
            candidates,
            mod_regs,
            gadget_type
        )

        return classified
97756
def fastrcnn_predictions(boxes, scores):
    
    assert boxes.shape[1] == cfg.DATA.NUM_CLASS
    assert scores.shape[1] == cfg.DATA.NUM_CLASS
    boxes = tf.transpose(boxes, [1, 0, 2])[1:, :, :]  # #catxnx4
    scores = tf.transpose(scores[:, 1:], [1, 0])  # #catxn

    def f(X):
        """
        prob: n probabilities
        box: nx4 boxes

        Returns: n boolean, the selection
        """
        prob, box = X
        output_shape = tf.shape(prob, out_type=tf.int64)
        # filter by score threshold
        ids = tf.reshape(tf.where(prob > cfg.TEST.RESULT_SCORE_THRESH), [-1])
        prob = tf.gather(prob, ids)
        box = tf.gather(box, ids)
        # NMS within each class
        selection = tf.image.non_max_suppression(
            box, prob, cfg.TEST.RESULTS_PER_IM, cfg.TEST.FRCNN_NMS_THRESH)
        selection = tf.gather(ids, selection)

        if get_tf_version_tuple() >= (1, 13):
            sorted_selection = tf.sort(selection, direction='ASCENDING')
            mask = tf.sparse.SparseTensor(indices=tf.expand_dims(sorted_selection, 1),
                                          values=tf.ones_like(sorted_selection, dtype=tf.bool),
                                          dense_shape=output_shape)
            mask = tf.sparse.to_dense(mask, default_value=False)
        else:
            # this function is deprecated by TF
            sorted_selection = -tf.nn.top_k(-selection, k=tf.size(selection))[0]
            mask = tf.sparse_to_dense(
                sparse_indices=sorted_selection,
                output_shape=output_shape,
                sparse_values=True,
                default_value=False)
        return mask

    # TF bug in version 1.11, 1.12: https://github.com/tensorflow/tensorflow/issues/22750
    buggy_tf = get_tf_version_tuple() in [(1, 11), (1, 12)]
    masks = tf.map_fn(f, (scores, boxes), dtype=tf.bool,
                      parallel_iterations=1 if buggy_tf else 10)     # #cat x N
    selected_indices = tf.where(masks)  # #selection x 2, each is (cat_id, box_id)
    scores = tf.boolean_mask(scores, masks)

    # filter again by sorting scores
    topk_scores, topk_indices = tf.nn.top_k(
        scores,
        tf.minimum(cfg.TEST.RESULTS_PER_IM, tf.size(scores)),
        sorted=False)
    filtered_selection = tf.gather(selected_indices, topk_indices)
    cat_ids, box_ids = tf.unstack(filtered_selection, axis=1)

    final_scores = tf.identity(topk_scores, name='scores')
    final_labels = tf.add(cat_ids, 1, name='labels')
    final_ids = tf.stack([cat_ids, box_ids], axis=1, name='all_ids')
    final_boxes = tf.gather_nd(boxes, final_ids, name='boxes')
    return final_boxes, final_scores, final_labels
1087155
def predict(self, L, P, R):
        
        phi = self.extract_one(L, P, R)
        return self.classifier.predict(phi)
1363236
def normalize_features(features):
    """"""
    return (features - N.min(features)) / (N.max(features) - N.min(features))
1150531
def parents(self, term, recursive=False, **kwargs):
        
        def dfs(term, recursive=False, visited={}, **kwargs):
            if term in visited: # Break on cyclic relations.
                return []
            visited[term], a = True, []
            if dict.__contains__(self, term):
                a = self[term][0].keys()
            for classifier in self.classifiers:
                a.extend(classifier.parents(term, **kwargs) or [])
            if recursive:
                for w in a: a += dfs(w, recursive, visited, **kwargs)
            return a
        return unique(dfs(self._normalize(term), recursive, {}, **kwargs))
707080
public function featureImportances() : array
    {
        if (!$this->forest or !$this->featureCount) {
            return [];
        }

        $importances = array_fill(0, $this->featureCount, 0.);

        foreach ($this->forest as $tree) {
            foreach ($tree->featureImportances() as $column => $value) {
                $importances[$column] += $value;
            }
        }

        $k = count($this->forest);

        foreach ($importances as &$importance) {
            $importance /= $k;
        }

        return $importances;
    }
1306743
private void calculateFeatureWeights(Dataframe trainingData) {
        ModelParameters modelParameters = knowledgeBase.getModelParameters();
        TrainingParameters trainingParameters = knowledgeBase.getTrainingParameters();
        Map<Object, TypeInference.DataType> columnTypes = trainingData.getXDataTypes();
        
        Map<Object, Double> featureWeights = modelParameters.getFeatureWeights();
        
        if(trainingParameters.isWeighted()==false) {
            //the unweighted version of the algorithm
            double gammaWeight = trainingParameters.getCategoricalGamaMultiplier();
            streamExecutor.forEach(StreamMethods.stream(columnTypes.entrySet().stream(), isParallelized()), e -> {
                //standard kmeans has equal weights in all numeric features
                //for categorical, dummy or ordinal feature, use the Gamma Multiplier of Kprototypes
                double weight = (e.getValue()!=TypeInference.DataType.NUMERICAL)?gammaWeight:1.0;
                featureWeights.put(e.getKey(), weight);
            });
        }
        else {
            //calculate weights for the features
            
            int n = trainingData.size();
            
            StorageEngine storageEngine = knowledgeBase.getStorageEngine();
            
            Map<Object, Double> tmp_categoricalFrequencies = storageEngine.getBigMap("tmp_categoricalFrequencies", Object.class, Double.class, MapType.HASHMAP, StorageHint.IN_MEMORY, true, true);
            Map<Object, Double> tmp_varianceSumX = storageEngine.getBigMap("tmp_varianceSumX", Object.class, Double.class, MapType.HASHMAP, StorageHint.IN_MEMORY, true, true);
            Map<Object, Double> tmp_varianceSumXsquare = storageEngine.getBigMap("tmp_varianceSumXsquare", Object.class, Double.class, MapType.HASHMAP, StorageHint.IN_MEMORY, true, true);
        
            //calculate variance and frequencies
            for(Record r : trainingData) { 
                for(Map.Entry<Object, Object> e : r.getX().entrySet()) {
                    Double value = TypeInference.toDouble(e.getValue());
                    if (value!=null && value!=0.0) {
                        Object feature = e.getKey();
                        
                        if(columnTypes.get(feature)!=TypeInference.DataType.NUMERICAL) {
                            Double previousValue = tmp_categoricalFrequencies.getOrDefault(feature, 0.0);
                            tmp_categoricalFrequencies.put(feature, previousValue+1.0);
                        }
                        else {
                            Double previousValueSumX = tmp_varianceSumX.getOrDefault(feature, 0.0);
                            Double previousValueSumXsquare = tmp_varianceSumXsquare.getOrDefault(feature, 0.0);
                            tmp_varianceSumX.put(feature, previousValueSumX+value);
                            tmp_varianceSumXsquare.put(feature, previousValueSumXsquare+value*value);
                        }
                    }
                }
            }
            
            double gammaWeight = trainingParameters.getCategoricalGamaMultiplier();
            
            //estimate weights
            streamExecutor.forEach(StreamMethods.stream(columnTypes.entrySet().stream(), isParallelized()), e -> {
                Object feature = e.getKey();
                TypeInference.DataType type = e.getValue();
                
                double weight;
                if(type!=TypeInference.DataType.NUMERICAL) {
                    double percentage = tmp_categoricalFrequencies.get(feature)/n;
                    weight = 1.0 -percentage*percentage;
                }
                else {
                    double mean = tmp_varianceSumX.get(feature)/n;
                    weight = 2.0*((tmp_varianceSumXsquare.get(feature)/n)-mean*mean);
                }
                
                
                if(weight>0) {
                    weight = 1/weight;
                }
                
                //apply muliplier if categorical
                if(type!=TypeInference.DataType.NUMERICAL) {
                    weight *= gammaWeight;
                }
                
                featureWeights.put(feature, weight);
            });
            
            //Drop the temporary Collection
            storageEngine.dropBigMap("tmp_categoricalFrequencies", tmp_categoricalFrequencies);
            storageEngine.dropBigMap("tmp_varianceSumX", tmp_categoricalFrequencies);
            storageEngine.dropBigMap("tmp_varianceSumXsquare", tmp_categoricalFrequencies);
        }
    }
786792
function improveRankOrdering(g, fixed, movable) {
    var weights = rankWeights(g, fixed, movable);

    var result = [];

    var layer = movable.slice(0);
    
    // Move fixed nodes into the result array first
    for (var i = 0; i < layer.length; ++i) {
      var u = layer[i];
      if (weights[u] === -1) {
        result[i] = u;
        layer[i] = null;
      }
    }

    layer.sort(function(x, y) { return (x ? weights[x] : -1) - (y ? weights[y] : -1); });

    var nextIdx = 0;
    for (var i = 0; i < layer.length; ++i) {
      if (layer[i] !== null) {
        while (result[nextIdx] !== undefined) {
          ++nextIdx;
        }
        result[nextIdx] = layer[i];
      }
    }

    return result;
  }
1173865
private void buildForest(Instance inst)
	{
		this.dimensions = inst.numAttributes();
		double[]max = new double[dimensions];
		double[]min = new double[dimensions];
		double sq;
		
		for (int i = 0 ; i < this.numTrees ; i++)
		{
			for(int j = 0 ; j < this.dimensions ; j++)
			{
				sq = this.classifierRandom.nextDouble();
				min[j] = sq - (2.0*Math.max(sq, 1.0-sq));
				max[j] = sq + (2.0*Math.max(sq, 1.0-sq));
			}

			forest[i] = new HSTreeNode(min, max, 1, maxDepth);
		}
	}
364655
def classify (data, features = nil)
      raise "Untrained model" unless (@fv_test)
  
      data_point = Eluka::DataPoint.new(data, @analyzer)
      @fv_test.add(data_point.vector)
  
      File.open(@directory + "/classify", "w") do |f| f.puts @fv_test.to_libSVM(features) end
      output = `#{@svm_predict_path} #{@directory}/classify #{@directory}/model #{@directory}/result`
      
      puts output if (@verbose)
  
      return @labels.lookup( File.open( @directory + "/result", "r" ).read.to_i )
    end