TODO move to ixa-pipe-ml
5849
private static List<String> buildSegmentedSentences(final BufferedReader breader) {
    String line;
    List<String> sentences = new ArrayList<>();
    try {
      while ((line = breader.readLine()) != null) {
        sentences.add(line);
      }
    } catch (final IOException e) {
      LOG.error("IOException", e);
    }
    return sentences;
  }
207467
function (fnCallbackAsyncL1) {
            /*
             order of artifacts
             1) DELETE files
             2) DELETE folders (starting with upper levels)
             3) CREATE folders (starting with lower levels)
             4) UPDATE folders -> not supported by ADT; but added to flow for completeness
             5) CREATE files
             6) UPDATE files
             */

            // level counter
            aArtifactsSync = aArtifactsSync.map(function (oItem) {
                oItem.levelCount = oItem.id.split('/').length - 1;
                return oItem;
            });

            me._oLogger.logVerbose('Artifacts to Sync: ', aArtifactsSync);

            // sort
            var aDeleteFiles = aArtifactsSync.filter(function (oItem) {
                return (oItem.type === util.OBJECT_TYPE.file && oItem.modif === util.MODIDF.delete);
            });
            var aDeleteFolders = aArtifactsSync.filter(function (oItem) {
                return (oItem.type === util.OBJECT_TYPE.folder && oItem.modif === util.MODIDF.delete);
            }).sort(function (oItem1, oItem2) {
                if (oItem1.levelCount > oItem2.levelCount) {
                    return -1;
                }
                if (oItem1.levelCount < oItem2.levelCount) {
                    return 1;
                }
                return 0;
            });

            var aCreateFolders = aArtifactsSync.filter(function (oItem) {
                return (oItem.type === util.OBJECT_TYPE.folder && oItem.modif === util.MODIDF.create);
            }).sort(function (oItem1, oItem2) {
                if (oItem1.levelCount < oItem2.levelCount) {
                    return -1;
                }
                if (oItem1.levelCount > oItem2.levelCount) {
                    return 1;
                }
                return 0;
            });
            var aUpdateFolders = aArtifactsSync.filter(function (oItem) {
                return (oItem.type === util.OBJECT_TYPE.folder && oItem.modif === util.MODIDF.update);
            });
            var aCreateFiles = aArtifactsSync.filter(function (oItem) {
                return (oItem.type === util.OBJECT_TYPE.file && oItem.modif === util.MODIDF.create);
            });
            var aUpdateFiles = aArtifactsSync.filter(function (oItem) {
                return (oItem.type === util.OBJECT_TYPE.file && oItem.modif === util.MODIDF.update);
            });

            aArtifactsSync = aDeleteFiles.concat(aDeleteFolders, aCreateFolders, aUpdateFolders, aCreateFiles, aUpdateFiles);
            aArtifactsSyncWork = aArtifactsSync.slice(0);

            fnCallbackAsyncL1(null, null);
        }
477056
@Override
    @Deprecated
    public void iterateSample(T w1, T w2, AtomicLong nextRandom, double alpha) {
        if (w2 == null || w2.getIndex() < 0 || w1.getIndex() == w2.getIndex() || w1.getLabel().equals("STOP")
                        || w2.getLabel().equals("STOP") || w1.getLabel().equals("UNK") || w2.getLabel().equals("UNK"))
            return;
        //current word vector
        INDArray l1 = this.syn0.slice(w2.getIndex());


        //error for current word and context
        INDArray neu1e = Nd4j.create(vectorLength);


        for (int i = 0; i < w1.getCodeLength(); i++) {
            int code = w1.getCodes().get(i);
            int point = w1.getPoints().get(i);
            if (point >= syn0.rows() || point < 0)
                throw new IllegalStateException("Illegal point " + point);
            //other word vector

            INDArray syn1 = this.syn1.slice(point);


            double dot = Nd4j.getBlasWrapper().dot(l1, syn1);

            if (dot < -MAX_EXP || dot >= MAX_EXP)
                continue;


            int idx = (int) ((dot + MAX_EXP) * ((double) expTable.length / MAX_EXP / 2.0));
            if (idx >= expTable.length)
                continue;

            //score
            double f = expTable[idx];
            //gradient
            double g = useAdaGrad ? w1.getGradient(i, (1 - code - f), lr.get()) : (1 - code - f) * alpha;

            Nd4j.getBlasWrapper().level1().axpy(syn1.length(), g, syn1, neu1e);
            Nd4j.getBlasWrapper().level1().axpy(syn1.length(), g, l1, syn1);

        }


        int target = w1.getIndex();
        int label;
        //negative sampling
        if (negative > 0)
            for (int d = 0; d < negative + 1; d++) {
                if (d == 0)
                    label = 1;
                else {
                    nextRandom.set(nextRandom.get() * 25214903917L + 11);

                    // FIXME: int cast
                    int idx = (int) Math.abs((int) (nextRandom.get() >> 16) % table.length());

                    target = table.getInt(idx);
                    if (target <= 0)
                        target = (int) nextRandom.get() % (vocab.numWords() - 1) + 1;

                    if (target == w1.getIndex())
                        continue;
                    label = 0;
                }


                if (target >= syn1Neg.rows() || target < 0)
                    continue;

                double f = Nd4j.getBlasWrapper().dot(l1, syn1Neg.slice(target));
                double g;
                if (f > MAX_EXP)
                    g = useAdaGrad ? w1.getGradient(target, (label - 1), alpha) : (label - 1) * alpha;
                else if (f < -MAX_EXP)
                    g = label * (useAdaGrad ? w1.getGradient(target, alpha, alpha) : alpha);
                else
                    g = useAdaGrad ? w1
                                    .getGradient(target,
                                                    label - expTable[(int) ((f + MAX_EXP)
                                                                    * (expTable.length / MAX_EXP / 2))],
                                                    alpha)
                                    : (label - expTable[(int) ((f + MAX_EXP) * (expTable.length / MAX_EXP / 2))])
                                                    * alpha;
                if (syn0.data().dataType() == DataType.DOUBLE)
                    Nd4j.getBlasWrapper().axpy(g, syn1Neg.slice(target), neu1e);
                else
                    Nd4j.getBlasWrapper().axpy((float) g, syn1Neg.slice(target), neu1e);

                if (syn0.data().dataType() == DataType.DOUBLE)
                    Nd4j.getBlasWrapper().axpy(g, l1, syn1Neg.slice(target));
                else
                    Nd4j.getBlasWrapper().axpy((float) g, l1, syn1Neg.slice(target));
            }

        if (syn0.data().dataType() == DataType.DOUBLE)
            Nd4j.getBlasWrapper().axpy(1.0, neu1e, l1);

        else
            Nd4j.getBlasWrapper().axpy(1.0f, neu1e, l1);

    }
643149
def downstream_index(dir_value, i, j, alg='taudem'):
        """"""
        assert alg.lower() in FlowModelConst.d8_deltas
        delta = FlowModelConst.d8_deltas.get(alg.lower())
        drow, dcol = delta[int(dir_value)]
        return i + drow, j + dcol
659800
private void slinkstep4(DBIDRef id, DBIDArrayIter it, int n, WritableDBIDDataStore pi, WritableDoubleDataStore lambda) {
    DBIDVar p_i = DBIDUtil.newVar();
    // for i = 1..n
    for(it.seek(0); it.getOffset() < n; it.advance()) {
      double l_i = lambda.doubleValue(it);
      p_i.from(pi, it); // p_i = pi(it)
      double lp_i = lambda.doubleValue(p_i);

      // if L(i) >= L(P(i))
      if(l_i >= lp_i) {
        // P(i) = n+1
        pi.put(it, id);
      }
    }
  }
1307275
def pipe():
    
    r, w = os.pipe()
    return File.fromfd(r, 'rb'), File.fromfd(w, 'wb')
127087
private void clinkstep8(DBIDRef id, DBIDArrayIter it, int n, WritableDBIDDataStore pi, WritableDoubleDataStore lambda, WritableDoubleDataStore m) {
    DBIDVar p_i = DBIDUtil.newVar(), pp_i = DBIDUtil.newVar();
    for(it.seek(0); it.getOffset() < n; it.advance()) {
      p_i.from(pi, it); // p_i = pi[i]
      pp_i.from(pi, p_i); // pp_i = pi[pi[i]]
      if(DBIDUtil.equal(pp_i, id) && lambda.doubleValue(it) >= lambda.doubleValue(p_i)) {
        pi.putDBID(it, id);
      }
    }
  }
1085074
function plumb() {
  var fns = slice.call(arguments)
  return function (val) {
    return pipe.apply(this, [val].concat(fns))
  }
}
1316839
public static <E> void pipe(E[] array, OutputIterator<E> outputIterator) {
        new ConsumeIntoOutputIterator<>(outputIterator).apply(new ArrayIterator<>(array));
    }
737677
public static int cusparseSsctr(
        cusparseHandle handle, 
        int nnz, 
        Pointer xVal, 
        Pointer xInd, 
        Pointer y, 
        int idxBase)
    {
        return checkResult(cusparseSsctrNative(handle, nnz, xVal, xInd, y, idxBase));
    }
766300
def re_pipe(FlowRate, Diam, Nu):
    """"""
    #Checking input validity
    ut.check_range([FlowRate, ">0", "Flow rate"], [Diam, ">0", "Diameter"],
                   [Nu, ">0", "Nu"])
    return (4 * FlowRate) / (np.pi * Diam * Nu)
557251
def d8flowdir(np, filleddem, flowdir, slope, workingdir=None, mpiexedir=None, exedir=None,
                  log_file=None, runtime_file=None, hostfile=None):
        """"""
        fname = TauDEM.func_name('d8flowdir')
        return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
                          {'-fel': filleddem}, workingdir,
                          None,
                          {'-p': flowdir, '-sd8': slope},
                          {'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
                          {'logfile': log_file, 'runtimefile': runtime_file})
1149879
def par_xstep(i):
    

    """
    global mp_X
    global mp_DX
    YU0f = sl.rfftn(mp_Y0[[i]] - mp_U0[[i]], mp_Nv, mp_axisN)
    YU1f = sl.rfftn(mp_Y1[mp_grp[i]:mp_grp[i+1]] -
                    1/mp_alpha*mp_U1[mp_grp[i]:mp_grp[i+1]], mp_Nv, mp_axisN)
    if mp_Cd == 1:
        b = np.conj(mp_Df[mp_grp[i]:mp_grp[i+1]]) * YU0f + mp_alpha**2*YU1f
        Xf = sl.solvedbi_sm(mp_Df[mp_grp[i]:mp_grp[i+1]], mp_alpha**2, b,
                            mp_cache[i], axis=mp_axisM)
    else:
        b = sl.inner(np.conj(mp_Df[mp_grp[i]:mp_grp[i+1]]), YU0f,
                     axis=mp_C) + mp_alpha**2*YU1f
        Xf = sl.solvemdbi_ism(mp_Df[mp_grp[i]:mp_grp[i+1]], mp_alpha**2, b,
                              mp_axisM, mp_axisC)
    mp_X[mp_grp[i]:mp_grp[i+1]] = sl.irfftn(Xf, mp_Nv,
                                            mp_axisN)
    mp_DX[i] = sl.irfftn(sl.inner(mp_Df[mp_grp[i]:mp_grp[i+1]], Xf,
                                  mp_axisM), mp_Nv, mp_axisN)
70170
function pipe(commands, input) {
  if (!commands) {
    return input;
  }
  return commands.reduce(function(input, cmd) {
    return npmRun.sync(cmd, {
      input: input
    });
  }, input);
}
1182075
def bfill(arr, dim=None, limit=None):
    ''''''
    axis = arr.get_axis_num(dim)

    # work around for bottleneck 178
    _limit = limit if limit is not None else arr.shape[axis]

    return apply_ufunc(_bfill, arr,
                       dask='parallelized',
                       keep_attrs=True,
                       output_dtypes=[arr.dtype],
                       kwargs=dict(n=_limit, axis=axis)).transpose(*arr.dims)
352832
@Override
    public void scal(long N, double alpha, INDArray X) {
        if (Nd4j.getExecutioner().getProfilingMode() == OpExecutioner.ProfilingMode.ALL)
            OpProfiler.getInstance().processBlasCall(false, X);

        if (X.isSparse()) {
            Nd4j.getSparseBlasWrapper().level1().scal(N, alpha, X);
        } else if (X.data().dataType() == DataType.DOUBLE)
            dscal(N, alpha, X, BlasBufferUtil.getBlasStride(X));
        else if (X.data().dataType() == DataType.FLOAT)
            sscal(N, (float) alpha, X, BlasBufferUtil.getBlasStride(X));
        else if (X.data().dataType() == DataType.HALF)
            Nd4j.getExecutioner().exec(new ScalarMultiplication(X, alpha));
    }
389170
private int emitNodes(FSA fsa, OutputStream os, IntArrayList linearized) throws IOException {
    int offset = 0;

    // Add epsilon state.
    offset += emitNodeData(os, 0);
    if (fsa.getRootNode() != 0)
      offset += emitArc(os, BIT_LAST_ARC, (byte) '^', offsets.get(fsa.getRootNode()));
    else
      offset += emitArc(os, BIT_LAST_ARC, (byte) '^', 0);

    boolean offsetsChanged = false;
    final int max = linearized.size();
    for (IntCursor c : linearized) {
      final int state = c.value;
      final int nextState = c.index + 1 < max ? linearized.get(c.index + 1) : NO_STATE;

      if (os == null) {
        offsetsChanged |= (offsets.get(state) != offset);
        offsets.put(state, offset);
      } else {
        assert offsets.get(state) == offset : state + " " + offsets.get(state) + " " + offset;
      }

      offset += emitNodeData(os, withNumbers ? numbers.get(state) : 0);
      offset += emitNodeArcs(fsa, os, state, nextState);
    }

    return offsetsChanged ? offset : 0;
  }