Calculates the p-value of null Hypothesis.

@param transposeDataCollection
@return
355506
public static double getPvalue(TransposeDataCollection transposeDataCollection) {
        if(transposeDataCollection.size()!=2) {
            throw new IllegalArgumentException("The collection must contain observations from 2 groups.");
        }
        
        Object[] keys = transposeDataCollection.keySet().toArray();
        
        //counter of uncencored internalData in each group
        Map<Object, Integer> n = new HashMap<>();
        n.put(keys[0], 0);
        n.put(keys[1], 0);
        
        Queue<Double> censoredData = new PriorityQueue<>();
        Queue<Double> uncensoredData = new PriorityQueue<>();
        for(Map.Entry<Object, FlatDataCollection> entry : transposeDataCollection.entrySet()) {
            Object j = entry.getKey();
            FlatDataCollection flatDataCollection = entry.getValue();
            
            for(Object value : flatDataCollection) {
                String str = value.toString();
                if(str.endsWith(CensoredDescriptives.CENSORED_NUMBER_POSTFIX)) {
                    //censored internalData encoded as 4.3+ or -4.3+
                    censoredData.add(Double.valueOf(str.substring(0,str.length()-CensoredDescriptives.CENSORED_NUMBER_POSTFIX.length()))); //remove the trailing char and convert it to double
                }
                else {
                    //uncensored internalData
                    uncensoredData.add(TypeInference.toDouble(value)); //convert it to double
                }
                n.put(j, n.get(j)+1);
            }
        }

        
        Double currentCensored = null;
        Double currentUncensored = null;
        AssociativeArray2D testTable = new AssociativeArray2D();
        
        do {
            if(currentCensored==null) {
                currentCensored=censoredData.poll();
            }
            if(currentUncensored==null) {
                currentUncensored=uncensoredData.poll();
            }
            
            Double ti;
            String key;
            if(currentUncensored == null) {
                key=currentCensored.toString().concat((CensoredDescriptives.CENSORED_NUMBER_POSTFIX));
                ti = currentCensored;
                currentCensored = null;
            }
            else if(currentCensored == null) {
                key=currentUncensored.toString();
                ti = currentUncensored;
                currentUncensored = null;
            }
            else if(currentCensored<currentUncensored) { //NOT EQUAL! Uncensored internalData of the same value are always larger
                key=currentCensored.toString().concat(CensoredDescriptives.CENSORED_NUMBER_POSTFIX);
                ti = currentCensored;
                currentCensored = null;
            }
            else {
                key=currentUncensored.toString();
                ti = currentUncensored;
                currentUncensored = null;
            }
            
            Object value = testTable.get2d(key, "mi");
            if(value==null) {
                testTable.put2d(key, "mi", 1);
                testTable.put2d(key, "rti", 0);
            }
            else {
                testTable.put2d(key, "mi", ((Integer)value) +1); 
                continue; //continue in order not to count twice the r*ti below
            }
            
            for(Map.Entry<Object, FlatDataCollection> entry : transposeDataCollection.entrySet()) {
                FlatDataCollection flatDataCollection = entry.getValue();
                
                for(Object value2 : flatDataCollection) {
                    double v;
                    String str = value2.toString();
                    if(str.endsWith(CensoredDescriptives.CENSORED_NUMBER_POSTFIX)) {
                        //censored internalData encoded as 4.3+ or -4.3+
                        v = Double.valueOf(str.substring(0,str.length()-CensoredDescriptives.CENSORED_NUMBER_POSTFIX.length())); //remove the trailing char and convert it to double
                    }
                    else {
                        //uncensored internalData
                        v = TypeInference.toDouble(value2); //convert it to double
                    }
                    
                    if(v>=ti) {
                        testTable.put2d(key, "rti", (Integer)testTable.get2d(key, "rti") +1);
                    }
                }
            }
                
        } 
        while(currentCensored!=null || currentUncensored!=null || 
              !censoredData.isEmpty() || !uncensoredData.isEmpty());
        
        //censoredData=null;
        //uncensoredData=null;
        

        double VarS=0.0;

        Object previousUncencoredKey = null;
        for(Map.Entry<Object, AssociativeArray> entry : testTable.entrySet()) {
            Object ti = entry.getKey();
            AssociativeArray testRow = entry.getValue();

            double previousUncencoredValue=0;
            
            Object tmp = testTable.get2d(previousUncencoredKey, "eti");
            if(tmp!=null) {
                previousUncencoredValue = TypeInference.toDouble(tmp);
            }
            
            if(!ti.toString().endsWith(CensoredDescriptives.CENSORED_NUMBER_POSTFIX)) { //uncensored
                double mi = testRow.getDouble("mi");
                double rti = testRow.getDouble("rti");
                double eti = previousUncencoredValue+mi/rti;
                
                testRow.put("eti", eti);
                testRow.put("wi", 1-eti);
                previousUncencoredKey=ti;
            }
            else { //censored
                testRow.put("wi", -previousUncencoredValue);
            }

            double wi = testRow.getDouble("wi");
            VarS+= testRow.getDouble("mi")*wi*wi;
        }


        double S=0.0;
        for(Object value : transposeDataCollection.get(keys[0])) { //if ti belongs to the first group
            Object key; //we must first convert the number into to double and then append the + if necessary. This is why it's converted like this.
            String str = value.toString();
            if(str.endsWith(CensoredDescriptives.CENSORED_NUMBER_POSTFIX)) {
                //censored internalData encoded as 4.3+ or -4.3+
                Double v = Double.valueOf(str.substring(0,str.length()-CensoredDescriptives.CENSORED_NUMBER_POSTFIX.length())); //remove the trailing char and convert it to double
                key = v.toString()+CensoredDescriptives.CENSORED_NUMBER_POSTFIX;
            }
            else {
                //uncensored internalData
                Double v = TypeInference.toDouble(value); //convert it to double
                key = v.toString();
            }
            double wi = TypeInference.toDouble(testTable.get2d(key, "wi"));
            S+= wi;
        }
        //testTable = null;
        
        double n0 = n.get(keys[0]).doubleValue();
        double n1 = n.get(keys[1]).doubleValue();
        
        VarS*=n0*n1/((n0+n1)*(n0+n1-1.0));

        double Z=S/Math.sqrt(VarS);
        
        double pvalue = scoreToPvalue(Z);

        return pvalue;
    }
725103
public static MultiDataSet toMultiDataSet(DataSet dataSet) {
        INDArray f = dataSet.getFeatures();
        INDArray l = dataSet.getLabels();
        INDArray fMask = dataSet.getFeaturesMaskArray();
        INDArray lMask = dataSet.getLabelsMaskArray();

        INDArray[] fNew = f == null ? null : new INDArray[] {f};
        INDArray[] lNew = l == null ? null : new INDArray[] {l};
        INDArray[] fMaskNew = (fMask != null ? new INDArray[] {fMask} : null);
        INDArray[] lMaskNew = (lMask != null ? new INDArray[] {lMask} : null);

        return new org.nd4j.linalg.dataset.MultiDataSet(fNew, lNew, fMaskNew, lMaskNew);
    }
193077
public static double min(FlatDataCollection flatDataCollection) {
        double min=Double.POSITIVE_INFINITY;
        
        Iterator<Double> it = flatDataCollection.iteratorDouble();
        while(it.hasNext()) {
            Double v = it.next();
            if(v != null && min > v) {
                min=v;
            }
        } 
        
        return min;
    }
1097335
public static <T, P, R extends Collection<T>> R rejectWith(
            Iterable<T> iterable,
            Predicate2<? super T, ? super P> predicate,
            P parameter,
            R targetCollection)
    {
        if (iterable instanceof RichIterable)
        {
            return ((RichIterable<T>) iterable).rejectWith(predicate, parameter, targetCollection);
        }
        if (iterable instanceof ArrayList)
        {
            return ArrayListIterate.rejectWith(
                    (ArrayList<T>) iterable,
                    predicate,
                    parameter,
                    targetCollection);
        }
        if (iterable instanceof RandomAccess)
        {
            return RandomAccessListIterate.rejectWith((List<T>) iterable, predicate, parameter, targetCollection);
        }
        if (iterable != null)
        {
            return IterableIterate.rejectWith(iterable, predicate, parameter, targetCollection);
        }
        throw new IllegalArgumentException("Cannot perform a rejectWith on null");
    }
90139
public static <T> T[] toArray(final Collection<T> collection) {

        T next = getFirstNotNullValue(collection);
        if (next != null) {
            Object[] objects = collection.toArray();
            @SuppressWarnings("unchecked")
            T[] convertedObjects = (T[]) Array.newInstance(next.getClass(), objects.length);
            System.arraycopy(objects, 0, convertedObjects, 0, objects.length);
            return convertedObjects;
        } else {
            return null;
        }
    }
959563
public static double getPvalue(TransposeDataCollection transposeDataCollection) {
        //flatten the original internalData table
        AssociativeArray associativeArray = new AssociativeArray();
        
        for(Map.Entry<Object, FlatDataCollection> entry : transposeDataCollection.entrySet()) {
            Object i = entry.getKey();
            FlatDataCollection row = entry.getValue();
            
            Integer j=0;
            for (Object value : row) {
                Map.Entry<Object, Object> i_j = new AbstractMap.SimpleEntry<>(i, (Object)j);
                associativeArray.put(i_j, value);
                ++j;
            }
        }
        
        //converts the values of the flatDataCollection with their Ranks
        AssociativeArray tiesCounter = Ranks.getRanksFromValues(associativeArray);
        int n = associativeArray.size();
        
        double C=0;
        //Correct for ties
        if(!tiesCounter.isEmpty()) {
            for(Object value : tiesCounter.values()) {
                double Ti = TypeInference.toDouble(value);
                C+=((Ti*Ti-1)*Ti); //faster than using pow()
            }
            C/=((n*n-1.0)*n); //faster than using pow()
        }
        //tiesCounter = null;

        //Important note! Remember that the "i" value is the number of the group while the j is the number of observation within the group and NOT the other way around.
        Map<Object, Integer> ni = new HashMap<>(); //stores the total number of observations in each group
        Map<Object, Double> Ridot = new HashMap<>(); //stores the sum of Ranks for each group
        
        for(Map.Entry<Object, Object> entry : associativeArray.entrySet()) {
            @SuppressWarnings("unchecked")
            Map.Entry<Object, Object> i_j = (Map.Entry<Object, Object>)entry.getKey();
            
            Object i = i_j.getKey(); //get i and j values
            Double rank = TypeInference.toDouble(entry.getValue());
            
            if(Ridot.containsKey(i)==false) { //if this "i" value is found for first time then define 
                Ridot.put(i, rank);
                ni.put(i, 1);
            }
            else {
                Ridot.put(i, Ridot.get(i)+rank);
                ni.put(i, ni.get(i)+1);
            }
        }
        //associativeArray=null;

        int k=ni.size();

        //Calculate Kruskal Wallis scrore based on the above
        double KWscore=0.0;
        
        for(Map.Entry<Object, Double> entry : Ridot.entrySet()) {
            Object i = entry.getKey();
            Double value = entry.getValue();
            
            KWscore+=value*value/ni.get(i);
        }
        //Ridot=null;
        //ni=null;


        KWscore=(12.0/(n*(n+1.0)))*KWscore - 3.0*(n+1.0);

        KWscore/=(1.0-C); //correction for ties

        double pvalue= scoreToPvalue(KWscore, k);

        return pvalue;
    }
904182
public static double mean(TransposeDataCollection sampleDataCollection, AssociativeArray populationNh) {
        double populationN = Descriptives.sum(populationNh.toFlatDataCollection());
        
        if(populationN<=0) {
            throw new IllegalArgumentException("The populationN parameter must be positive.");
        }
        
        double mean = 0.0;
        
        for(Map.Entry<Object, FlatDataCollection> entry : sampleDataCollection.entrySet()) {
            Object strata = entry.getKey();
            Integer strataPopulation = ((Number)populationNh.get(strata)).intValue();
        
            if(strataPopulation==null) {
                throw new IllegalArgumentException("Invalid strata population size.");
            }
            
            mean += strataPopulation*SimpleRandomSampling.mean(entry.getValue())/populationN;
        }
        
        return mean;
    }
694253
public static double getPvalue(DataTable2D dataTable) {

        AssociativeArray result = getScore(dataTable);
        double score = result.getDouble("score");
        int n = result.getDouble("n").intValue();
        int k = result.getDouble("k").intValue();
        
        double pvalue = scoreToPvalue(score, n, k);

        return pvalue;
    }
1337206
public static double kurtosisSE(FlatDataCollection flatDataCollection) {
	    int n = count(flatDataCollection);
        if(n<=3) {
            throw new IllegalArgumentException("The provided collection must have more than 3 elements.");
        }
        
        double kurtosisSE = Math.sqrt(24.0/n);
        
        return kurtosisSE;
    }
881660
public static double calculateCorrelation(TransposeDataList transposeDataList) { //unsafe internalData pointer. modifying the variable internalData
        Object[] keys = transposeDataList.keySet().toArray();
        if(keys.length!=2) {
            throw new IllegalArgumentException("The collection must contain observations from 2 groups.");
        }
        
        Object keyX = keys[0];
        Object keyY = keys[1];
        
        FlatDataList flatDataListX = transposeDataList.get(keyX);
        FlatDataList flatDataListY = transposeDataList.get(keyY);

        int n = flatDataListX.size();
        if(n<=0 || n!=flatDataListY.size()) {
            throw new IllegalArgumentException("The number of observations in each group must be equal and larger than 0.");
        }

        //The following loops calculate twice the scores. That the check both if A>B and B<A and count it twice
        int numberOfConcordant=0;
        int numberOfDisconcordant=0;
        
        
        for(int i=0;i<n;++i) {
            for(int j=0;j<n;++j) {
                double xi = flatDataListX.getDouble(i);
                double xj = flatDataListX.getDouble(j);
                double yi = flatDataListY.getDouble(i);
                double yj = flatDataListY.getDouble(j);
                
                double sign=(xi-xj)*(yi-yj); //if this has a positive sign then they are concordant
                if(sign>0) {
                    ++numberOfConcordant;
                }
                else if(sign<0) {
                    ++numberOfDisconcordant;
                }
            }
        }
        /*
        foreach(transposeDataList[0] as i=>&xi) {
            foreach(transposeDataList[0] as j=>&xj) {
            }
        }
        */
        numberOfConcordant/=2; //divide by 2 because we counted them twice
        numberOfDisconcordant/=2; //divide by 2 because we counted them twice

        double R=(numberOfConcordant-numberOfDisconcordant)/(n*(n-1.0)/2.0);

        return R;
    }
540229
public static double xbarVariance(FlatDataCollection flatDataCollection) {
        double n = flatDataCollection.size();
        if(n<=1) {
            throw new IllegalArgumentException("The provided collection must have more than 1 elements.");
        } 
        
        //As explained at http://www.fao.org/docrep/003/x6831e/x6831e12.htm
        //we take the first differences
        Double previousValue = null;
        double xbarVariance = 0.0;
        Iterator<Double> it = flatDataCollection.iteratorDouble();
        while(it.hasNext()) {
            Double yi = it.next();
            if(previousValue!=null) {
                double delta = yi - previousValue;
                xbarVariance+=delta*delta;
            }
            previousValue = yi;
        }
        xbarVariance/=(2*n*(n-1));

        return xbarVariance;
    }
1350607
@SuppressWarnings("unchecked")
    static <T> Collection<T> asCollection(Object value, PDescriptor itemType) throws ProvidenceConfigException {
        if (value instanceof Collection) {
            Collection<T> out = new ArrayList<>();
            for (Object item : (Collection) value) {
                out.add((T) asType(itemType, item));
            }
            return out;
        }
        throw new ProvidenceConfigException(
                "Unable to convert " + value.getClass().getSimpleName() + " to a collection");
    }
1072621
public static <T, J extends Convertable<T>> ArrayList<T> convert(Collection<J> collection) {
        if (collection == null) {
            return null;
        }

        ArrayList<T> list = new ArrayList<>(collection.size());

        for (J json : collection) {
            try {
                T item = json == null ? null : json.convert();
                if (item != null) {
                    list.add(item);
                }
            } catch (ParseException e) {
                Log.e("ConvertUtils", "Error converting item (" + json.getClass().getSimpleName()
                        + ") : " + e.getMessage());
            }
        }

        return list;
    }
860181
@Override
    public ZMatrixRMaj getQ(ZMatrixRMaj Q , boolean transposed ) {
        Q = UtilDecompositons_ZDRM.checkIdentity(Q,N,N);

        Arrays.fill(w,0,N*2,0);

        if( transposed ) {
            for( int j = N-2; j >= 0; j-- ) {
                QrHelperFunctions_ZDRM.extractHouseholderRow(QT,j,j+1,N,w,0);
                QrHelperFunctions_ZDRM.rank1UpdateMultL(Q, w, 0, gammas[j], j+1, j+1 , N);
            }
        } else {
            for( int j = N-2; j >= 0; j-- ) {
                QrHelperFunctions_ZDRM.extractHouseholderRow(QT,j,j+1,N,w,0);
                QrHelperFunctions_ZDRM.rank1UpdateMultR(Q, w, 0, gammas[j], j+1, j+1 , N, b);
            }
        }

        return Q;
    }
306931
public static double getPvalue(TransposeDataList transposeDataList) {
        Object[] keys = transposeDataList.keySet().toArray();
        if(keys.length!=2) {
            throw new IllegalArgumentException("The collection must contain observations from 2 groups.");
        }
        
        Object keyX = keys[0];
        Object keyY = keys[1];
        
        FlatDataList flatDataListX = transposeDataList.get(keyX);
        FlatDataList flatDataListY = transposeDataList.get(keyY);

        int n = flatDataListX.size();
        if(n<=0 || n!=flatDataListY.size()) {
            throw new IllegalArgumentException("The number of observations in each group must be equal and larger than 0.");
        }

        int Tplus=0;
        
        for(int j=0;j<n;++j) {
            double delta= flatDataListX.getDouble(j) - flatDataListY.getDouble(j);
            if(delta==0) {
                continue; //don't count it at all
            }
            if(delta>0) {
                ++Tplus;
            }
        }

        double pvalue = scoreToPvalue(Tplus, n);

        return pvalue;
    }
309108
public static int arrayBindValueCount(Map<String, ParameterBindingDTO> bindValues)
  {
    if (!isArrayBind(bindValues))
    {
      return 0;
    }
    else
    {
      ParameterBindingDTO bindSample = bindValues.values().iterator().next();
      List<String> bindSampleValues = (List<String>) bindSample.getValue();
      return bindValues.size() * bindSampleValues.size();
    }
  }
225817
@Override
    public double inverseCumulativeProbability(final double p) throws OutOfRangeException {
        /*
         * IMPLEMENTATION NOTES
         * --------------------
         * Where applicable, use is made of the one-sided Chebyshev inequality
         * to bracket the root. This inequality states that
         * P(X - mu >= k * sig) <= 1 / (1 + k^2),
         * mu: mean, sig: standard deviation. Equivalently
         * 1 - P(X < mu + k * sig) <= 1 / (1 + k^2),
         * F(mu + k * sig) >= k^2 / (1 + k^2).
         *
         * For k = sqrt(p / (1 - p)), we find
         * F(mu + k * sig) >= p,
         * and (mu + k * sig) is an upper-bound for the root.
         *
         * Then, introducing Y = -X, mean(Y) = -mu, sd(Y) = sig, and
         * P(Y >= -mu + k * sig) <= 1 / (1 + k^2),
         * P(-X >= -mu + k * sig) <= 1 / (1 + k^2),
         * P(X <= mu - k * sig) <= 1 / (1 + k^2),
         * F(mu - k * sig) <= 1 / (1 + k^2).
         *
         * For k = sqrt((1 - p) / p), we find
         * F(mu - k * sig) <= p,
         * and (mu - k * sig) is a lower-bound for the root.
         *
         * In cases where the Chebyshev inequality does not apply, geometric
         * progressions 1, 2, 4, ... and -1, -2, -4, ... are used to bracket
         * the root.
         */
        if (p < 0.0 || p > 1.0) {
            throw new OutOfRangeException(p, 0, 1);
        }

        double lowerBound = getSupportLowerBound();
        if (p == 0.0) {
            return lowerBound;
        }

        double upperBound = getSupportUpperBound();
        if (p == 1.0) {
            return upperBound;
        }

        final double mu = getNumericalMean();
        final double sig = FastMath.sqrt(getNumericalVariance());
        final boolean chebyshevApplies;
        chebyshevApplies = !(Double.isInfinite(mu) || Double.isNaN(mu) || Double.isInfinite(sig) || Double.isNaN(sig));

        if (lowerBound == Double.NEGATIVE_INFINITY) {
            if (chebyshevApplies) {
                lowerBound = mu - sig * FastMath.sqrt((1. - p) / p);
            } else {
                lowerBound = -1.0;
                while (cumulativeProbability(lowerBound) >= p) {
                    lowerBound *= 2.0;
                }
            }
        }

        if (upperBound == Double.POSITIVE_INFINITY) {
            if (chebyshevApplies) {
                upperBound = mu + sig * FastMath.sqrt(p / (1. - p));
            } else {
                upperBound = 1.0;
                while (cumulativeProbability(upperBound) < p) {
                    upperBound *= 2.0;
                }
            }
        }

        final UnivariateFunction toSolve = new UnivariateFunction() {

            public double value(final double x) {
                return cumulativeProbability(x) - p;
            }
        };

        double x = UnivariateSolverUtils.solve(toSolve, lowerBound, upperBound, getSolverAbsoluteAccuracy());

        if (!isSupportConnected()) {
            /* Test for plateau. */
            final double dx = getSolverAbsoluteAccuracy();
            if (x - dx >= getSupportLowerBound()) {
                double px = cumulativeProbability(x);
                if (cumulativeProbability(x - dx) == px) {
                    upperBound = x;
                    while (upperBound - lowerBound > dx) {
                        final double midPoint = 0.5 * (lowerBound + upperBound);
                        if (cumulativeProbability(midPoint) < px) {
                            lowerBound = midPoint;
                        } else {
                            upperBound = midPoint;
                        }
                    }
                    return upperBound;
                }
            }
        }
        return x;
    }