As per {@link #scoreExamples(DataSet, boolean)} - the outputs (example scores) for all DataSets in the iterator are concatenated
204
public INDArray scoreExamples(DataSetIterator iter, boolean addRegularizationTerms) {
        List<INDArray> out = new ArrayList<>();

        while (iter.hasNext()) {
            out.add(scoreExamples(iter.next(), addRegularizationTerms));
        }
        return Nd4j.toFlattened('f', out);
    }
87430
@Override
    public DataSet next() {
        DataSet next = iter.next();
        next.setLabels(next.getFeatures());
        return next;
    }
502729
public static List<List<List<Writable>>> sampleSequence(int count, JavaRDD<List<List<Writable>>> data) {
        return data.takeSample(false, count);
    }
183426
@Override
    public double f1Score(INDArray input, INDArray labels) {
        feedForward(input);
        setLabels(labels);
        Evaluation eval = new Evaluation();
        eval.eval(labels, output(input));
        return eval.f1();
    }
317302
public void displayResults(boolean showMismatches,PrintStream out) throws IOException 
    {
        PrintfFormat fmt = new PrintfFormat("%s %3d %7.2f | %30s | %30s\n");
        for (int i=0; i<pairs.length; i++) {
            if (pairs[i]!=null) {
                String label = pairs[i].isCorrect() ? "+" : "-";
                String aText = (pairs[i].getA()==null) ? "***" : pairs[i].getA().unwrap();
                String bText = (pairs[i].getB()==null) ? "***" : pairs[i].getB().unwrap();
                if (showMismatches || "+".equals(label)) {
                    out.print( fmt.sprintf( new Object[] { 
                                label,
                                new Integer(i+1),
                                new Double(pairs[i].getDistance()),
                                aText,
                                bText
                            }));
                }
            }
        }
    }
1096797
protected static void printDataSet(DataSet dataSet) {
        final String SPACER = "| \t |";

        if (dataSet == null) System.out.println("DataSet is null");
        if (dataSet.getRowCount() == 0) System.out.println("DataSet is empty");

        List<DataColumn> dataSetColumns = dataSet.getColumns();
        int colColunt = dataSetColumns.size();
        int rowCount = dataSet.getRowCount();

        System.out.println("********************************************************************************************************************************************************");
        for (int row = 0; row < rowCount; row++) {
            System.out.print(SPACER);
            for (int col= 0; col< colColunt; col++) {
                Object value = dataSet.getValueAt(row, col);
                String colId = dataSet.getColumnByIndex(col).getId();
                System.out.print(colId + ": " +  value);
                System.out.print(SPACER);
            }
            System.out.println("");
        }
        System.out.println("********************************************************************************************************************************************************");
    }
184570
@Override
	public DataSet<Tuple2<VertexKey, VertexValue>> createResult() {
		if (this.initialVertices == null) {
			throw new IllegalStateException("The input data set has not been set.");
		}
		
		// prepare some type information
		TypeInformation<Tuple2<VertexKey, VertexValue>> vertexTypes = initialVertices.getType();
		TypeInformation<VertexKey> keyType = ((TupleTypeInfo<?>) initialVertices.getType()).getTypeAt(0);
		TypeInformation<Tuple2<VertexKey, Message>> messageTypeInfo = new TupleTypeInfo<Tuple2<VertexKey,Message>>(keyType, messageType);		
		
		// set up the iteration operator
		final String name = (this.name != null) ? this.name :
			"Vertex-centric iteration (" + updateFunction + " | " + messagingFunction + ")";
		final int[] zeroKeyPos = new int[] {0};
	
		final DeltaIteration<Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, VertexValue>> iteration =
			this.initialVertices.iterateDelta(this.initialVertices, this.maximumNumberOfIterations, zeroKeyPos);
		iteration.name(name);
		iteration.parallelism(parallelism);
		
		// register all aggregators
		for (Map.Entry<String, Class<? extends Aggregator<?>>> entry : this.aggregators.entrySet()) {
			iteration.registerAggregator(entry.getKey(), entry.getValue());
		}
		
		// build the messaging function (co group)
		CoGroupOperator<?, ?, Tuple2<VertexKey, Message>> messages;
		if (edgesWithoutValue != null) {
			MessagingUdfNoEdgeValues<VertexKey, VertexValue, Message> messenger = new MessagingUdfNoEdgeValues<VertexKey, VertexValue, Message>(messagingFunction, messageTypeInfo);
			messages = this.edgesWithoutValue.coGroup(iteration.getWorkset()).where(0).equalTo(0).with(messenger);
		}
		else {
			MessagingUdfWithEdgeValues<VertexKey, VertexValue, Message, EdgeValue> messenger = new MessagingUdfWithEdgeValues<VertexKey, VertexValue, Message, EdgeValue>(messagingFunction, messageTypeInfo);
			messages = this.edgesWithValue.coGroup(iteration.getWorkset()).where(0).equalTo(0).with(messenger);
		}
		
		// configure coGroup message function with name and broadcast variables
		messages = messages.name("Messaging");
		for (Tuple2<String, DataSet<?>> e : this.bcVarsMessaging) {
			messages = messages.withBroadcastSet(e.f1, e.f0);
		}
		
		VertexUpdateUdf<VertexKey, VertexValue, Message> updateUdf = new VertexUpdateUdf<VertexKey, VertexValue, Message>(updateFunction, vertexTypes);
		
		// build the update function (co group)
		CoGroupOperator<?, ?, Tuple2<VertexKey, VertexValue>> updates =
				messages.coGroup(iteration.getSolutionSet()).where(0).equalTo(0).with(updateUdf);
		
		// configure coGroup update function with name and broadcast variables
		updates = updates.name("Vertex State Updates");
		for (Tuple2<String, DataSet<?>> e : this.bcVarsUpdate) {
			updates = updates.withBroadcastSet(e.f1, e.f0);
		}

		// let the operator know that we preserve the key field
		updates.withConstantSetFirst("0").withConstantSetSecond("0");
		
		return iteration.closeWith(updates, updates);
		
	}
1295508
public void setSamples(java.util.Collection<Sample> samples) {
        if (samples == null) {
            this.samples = null;
            return;
        }

        this.samples = new java.util.ArrayList<Sample>(samples);
    }
220008
public void evaluate(final ObjectStream<SequenceLabelSample> samples,
      final int nFolds) throws IOException {

    // Note: The name samples need to be grouped on a document basis.

    final CrossValidationPartitioner<DocumentSample> partitioner = new CrossValidationPartitioner<DocumentSample>(
        new NameToDocumentSampleStream(samples), nFolds);

    while (partitioner.hasNext()) {

      final CrossValidationPartitioner.TrainingSampleStream<DocumentSample> trainingSampleStream = partitioner
          .next();

      SequenceLabelerModel model = null;
      if (this.factory != null) {
        model = SequenceLabelerME.train(this.languageCode,
            new DocumentToNameSampleStream(trainingSampleStream), this.params,
            this.factory);
      } else {
        System.err.println("You need to implement a SequenceLabelerFactory!");
        System.exit(1);
      }
      
      String corpusFormat = Flags.getCorpusFormat(this.params);
      // do testing
      final SequenceLabelerEvaluator evaluator = new SequenceLabelerEvaluator(corpusFormat,
          new SequenceLabelerME(model), this.listeners);

      evaluator.evaluate(new DocumentToNameSampleStream(
          trainingSampleStream.getTestSampleStream()));

      this.fmeasure.mergeInto(evaluator.getFMeasure());
    }
  }
828066
public List<Object> getAll(int dataSet) throws SerializationException {
		List<Object> result = new ArrayList<Object>();
		for (Iterator<DataSet> i = dataSets.iterator(); i.hasNext();) {
			DataSet ds = i.next();
			DataSetInfo info = ds.getInfo();
			if (info.getDataSetNumber() == dataSet) {
				result.add(getData(ds));
			}
		}
		return result;
	}
412450
def isscalar(cls, dataset, dim):
        
        if not dataset.data:
            return True
        ds = cls._inner_dataset_template(dataset)
        isscalar = []
        for d in dataset.data:
            ds.data = d
            isscalar.append(ds.interface.isscalar(ds, dim))
        return all(isscalar)
965021
@Deprecated
  public Counter<L> scoresOf(RVFDatum<L, F> example) {
    Counter<L> scores = new ClassicCounter<L>();
    for (L l : labels()) {
      scores.setCount(l, scoreOf(example, l));
    }
    //System.out.println("Scores are: " + scores + "   (gold: " + example.label() + ")");
    return scores;
  }
725103
public static MultiDataSet toMultiDataSet(DataSet dataSet) {
        INDArray f = dataSet.getFeatures();
        INDArray l = dataSet.getLabels();
        INDArray fMask = dataSet.getFeaturesMaskArray();
        INDArray lMask = dataSet.getLabelsMaskArray();

        INDArray[] fNew = f == null ? null : new INDArray[] {f};
        INDArray[] lNew = l == null ? null : new INDArray[] {l};
        INDArray[] fMaskNew = (fMask != null ? new INDArray[] {fMask} : null);
        INDArray[] lMaskNew = (lMask != null ? new INDArray[] {lMask} : null);

        return new org.nd4j.linalg.dataset.MultiDataSet(fNew, lNew, fMaskNew, lMaskNew);
    }
1018158
def values(cls, dataset, dimension, expanded, flat):
        
        if not dataset.data:
            return np.array([])
        values = []
        ds = cls._inner_dataset_template(dataset)
        for d in dataset.data:
            ds.data = d
            dvals = ds.interface.values(ds, dimension, expanded, flat)
            if not len(dvals):
                continue
            elif expanded:
                values.append(dvals)
                values.append([np.NaN])
            else:
                values.append(dvals)
        if not values:
            return np.array([])
        elif expanded:
            return np.concatenate(values[:-1])
        else:
            return np.concatenate(values)
65815
@Override
	public DataSet<Vertex<K, VV>> createResult() {
		if (this.initialVertices == null) {
			throw new IllegalStateException("The input data set has not been set.");
		}

		// prepare some type information
		TypeInformation<K> keyType = ((TupleTypeInfo<?>) initialVertices.getType()).getTypeAt(0);
		TypeInformation<Tuple2<K, Message>> messageTypeInfo = new TupleTypeInfo<>(keyType, messageType);

		// create a graph
		Graph<K, VV, EV> graph =
				Graph.fromDataSet(initialVertices, edgesWithValue, initialVertices.getExecutionEnvironment());

		// check whether the numVertices option is set and, if so, compute the total number of vertices
		// and set it within the scatter and gather functions

		DataSet<LongValue> numberOfVertices = null;
		if (this.configuration != null && this.configuration.isOptNumVertices()) {
			try {
				numberOfVertices = GraphUtils.count(this.initialVertices);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}

		if (this.configuration != null) {
			scatterFunction.setDirection(this.configuration.getDirection());
		} else {
			scatterFunction.setDirection(EdgeDirection.OUT);
		}

		// retrieve the direction in which the updates are made and in which the messages are sent
		EdgeDirection messagingDirection = scatterFunction.getDirection();

		// check whether the degrees option is set and, if so, compute the in and the out degrees and
		// add them to the vertex value
		if (this.configuration != null && this.configuration.isOptDegrees()) {
			return createResultVerticesWithDegrees(graph, messagingDirection, messageTypeInfo, numberOfVertices);
		} else {
			return createResultSimpleVertex(messagingDirection, messageTypeInfo, numberOfVertices);
		}
	}
1333990
public static char[] getLabelsArray(Set dataSet) {
        switch (dataSet) {
            case COMPLETE:
                return LABELS_COMPLETE;
            case MERGE:
                return LABELS_MERGE;
            case BALANCED:
                return LABELS_BALANCED;
            case LETTERS:
                return LABELS_LETTERS;
            case DIGITS:
            case MNIST:
                return LABELS_DIGITS;
            default:
                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
        }
    }
52280
public static SequenceBatchCSVRecord fromDataSet(MultiDataSet dataSet) {
        SequenceBatchCSVRecord batchCSVRecord = new SequenceBatchCSVRecord();
        for (int i = 0; i < dataSet.numFeatureArrays(); i++) {
            batchCSVRecord.add(Arrays.asList(BatchCSVRecord.fromDataSet(new DataSet(dataSet.getFeatures(i),dataSet.getLabels(i)))));
        }

        return batchCSVRecord;
    }