LOF算出は行わず、学習データモデルへのデータ追加のみを行う。

@param recievedPoint 受信データ
@param dataSet 学習モデルデータ
7600
protected void addDataWithoutCalculate(LofPoint recievedPoint, LofDataSet dataSet)
    {
        dataSet.addData(recievedPoint);

        // 中間データを保持する場合はデータ追加時に中間データの生成を行う。
        if (this.hasIntermediate)
        {
            LofCalculator.initDataSet(this.kn, dataSet);
        }
    }
1053212
protected static void updateLrd(LofPoint targetPoint, LofDataSet dataSet)
    {
        // 対象点の局所到達可能密度を更新する。
        double lrd = calculateLrd(targetPoint, dataSet);
        targetPoint.setLrd(lrd);
    }
1001276
public void lookupDataSet(final DataSetDef def,
                              final DataSetLookup request,
                              final DataSetReadyCallback listener) throws Exception {

        if (dataSetLookupServices != null) {
            try {
                dataSetLookupServices.call(
                        new RemoteCallback<DataSet>() {
                            public void callback(DataSet result) {
                                if (result == null) {
                                    listener.notFound();
                                } else {
                                    listener.callback(result);
                                }
                            }
                        },
                        new ErrorCallback<Message>() {
                            @Override
                            public boolean error(Message message,
                                                 Throwable throwable) {
                                return listener.onError(new ClientRuntimeError(throwable));
                            }
                        })
                        .lookupDataSet(def,
                                       request);
            } catch (Exception e) {
                listener.onError(new ClientRuntimeError(e));
            }
        }
        // Data set not found on client.
        else {
            listener.notFound();
        }
    }
903803
private void publishFileSet(CopyEntity.DatasetAndPartition datasetAndPartition,
      Collection<WorkUnitState> datasetWorkUnitStates) throws IOException {
    Map<String, String> additionalMetadata = Maps.newHashMap();

    Preconditions.checkArgument(!datasetWorkUnitStates.isEmpty(),
        "publishFileSet received an empty collection work units. This is an error in code.");

    CopyableDatasetMetadata metadata = CopyableDatasetMetadata
        .deserialize(datasetWorkUnitStates.iterator().next().getProp(CopySource.SERIALIZED_COPYABLE_DATASET));
    Path datasetWriterOutputPath = new Path(this.writerOutputDir, datasetAndPartition.identifier());

    log.info("Merging all split work units.");
    DistcpFileSplitter.mergeAllSplitWorkUnits(this.fs, datasetWorkUnitStates);

    log.info(String.format("[%s] Publishing fileSet from %s for dataset %s", datasetAndPartition.identifier(),
        datasetWriterOutputPath, metadata.getDatasetURN()));

    List<CommitStep> prePublish = getCommitSequence(datasetWorkUnitStates, PrePublishStep.class);
    List<CommitStep> postPublish = getCommitSequence(datasetWorkUnitStates, PostPublishStep.class);
    log.info(String.format("[%s] Found %d prePublish steps and %d postPublish steps.", datasetAndPartition.identifier(),
        prePublish.size(), postPublish.size()));

    executeCommitSequence(prePublish);
    if (hasCopyableFiles(datasetWorkUnitStates)) {
      // Targets are always absolute, so we start moving from root (will skip any existing directories).
      HadoopUtils.renameRecursively(this.fs, datasetWriterOutputPath, new Path("/"));
    } else {
      log.info(String.format("[%s] No copyable files in dataset. Proceeding to postpublish steps.", datasetAndPartition.identifier()));
    }
    executeCommitSequence(postPublish);

    this.fs.delete(datasetWriterOutputPath, true);

    long datasetOriginTimestamp = Long.MAX_VALUE;
    long datasetUpstreamTimestamp = Long.MAX_VALUE;
    Optional<String> fileSetRoot = Optional.<String>absent();

    for (WorkUnitState wus : datasetWorkUnitStates) {
      if (wus.getWorkingState() == WorkingState.SUCCESSFUL) {
        wus.setWorkingState(WorkUnitState.WorkingState.COMMITTED);
      }
      CopyEntity copyEntity = CopySource.deserializeCopyEntity(wus);
      if (copyEntity instanceof CopyableFile) {
        CopyableFile copyableFile = (CopyableFile) copyEntity;
        DataFileVersionStrategy srcVS = this.srcDataFileVersionStrategy;
        DataFileVersionStrategy dstVS = this.dstDataFileVersionStrategy;

        // Prefer to use copyableFile's specific version strategy
        if (copyableFile.getDataFileVersionStrategy() != null) {
          Config versionStrategyConfig = ConfigFactory.parseMap(ImmutableMap.of(
              DataFileVersionStrategy.DATA_FILE_VERSION_STRATEGY_KEY, copyableFile.getDataFileVersionStrategy()));
          srcVS = DataFileVersionStrategy.instantiateDataFileVersionStrategy(this.srcFs, versionStrategyConfig);
          dstVS = DataFileVersionStrategy.instantiateDataFileVersionStrategy(this.fs, versionStrategyConfig);
        }

        if (copyableFile.getPreserve().preserve(PreserveAttributes.Option.VERSION)
            && dstVS.hasCharacteristic(DataFileVersionStrategy.Characteristic.SETTABLE)) {
          dstVS.setVersion(copyableFile.getDestination(),
              srcVS.getVersion(copyableFile.getOrigin().getPath()));
        }

        if (wus.getWorkingState() == WorkingState.COMMITTED) {
          CopyEventSubmitterHelper.submitSuccessfulFilePublish(this.eventSubmitter, copyableFile, wus);
          // Dataset Output path is injected in each copyableFile.
          // This can be optimized by having a dataset level equivalent class for copyable entities
          // and storing dataset related information, e.g. dataset output path, there.

          // Currently datasetOutputPath is only present for hive datasets.
          if (!fileSetRoot.isPresent() && copyableFile.getDatasetOutputPath() != null) {
            fileSetRoot = Optional.of(copyableFile.getDatasetOutputPath());
          }
          if (lineageInfo.isPresent()) {
            lineageInfo.get().putDestination(copyableFile.getDestinationData(), 0, wus);
          }
        }
        if (datasetOriginTimestamp > copyableFile.getOriginTimestamp()) {
          datasetOriginTimestamp = copyableFile.getOriginTimestamp();
        }
        if (datasetUpstreamTimestamp > copyableFile.getUpstreamTimestamp()) {
          datasetUpstreamTimestamp = copyableFile.getUpstreamTimestamp();
        }
      }
    }

    // if there are no valid values for datasetOriginTimestamp and datasetUpstreamTimestamp, use
    // something more readable
    if (Long.MAX_VALUE == datasetOriginTimestamp) {
      datasetOriginTimestamp = 0;
    }
    if (Long.MAX_VALUE == datasetUpstreamTimestamp) {
      datasetUpstreamTimestamp = 0;
    }

    additionalMetadata.put(SlaEventKeys.SOURCE_URI, this.state.getProp(SlaEventKeys.SOURCE_URI));
    additionalMetadata.put(SlaEventKeys.DESTINATION_URI, this.state.getProp(SlaEventKeys.DESTINATION_URI));
    additionalMetadata.put(SlaEventKeys.DATASET_OUTPUT_PATH, fileSetRoot.or("Unknown"));
    CopyEventSubmitterHelper.submitSuccessfulDatasetPublish(this.eventSubmitter, datasetAndPartition,
        Long.toString(datasetOriginTimestamp), Long.toString(datasetUpstreamTimestamp), additionalMetadata);
    }
456993
public void addData(LofPoint addedPoint)
    {
        this.dataIdList.add(addedPoint.getDataId());
        this.dataMap.put(addedPoint.getDataId(), addedPoint);
    }
1097609
function(dataset){
    arrayRemove(this.sources, dataset);

    // remove event listeners from dataset
    if (this.listen.source)
      dataset.removeHandler(this.listen.source, this);

    // process removing dataset objects and update member map
    var memberMap = this.members_;
    for (var objectId in dataset.items_)
      memberMap[objectId].count--;
  }
679107
public void addDataPointSet(String series, Set<DataPoint> dataPoints) {
        DataSet set = store.get(series);
        if (set == null) {
            set = new DataSet(dataPoints);
            store.put(series, set);
        } else {
            set.addAll(dataPoints);
        }
    }
250598
private void putDataPoint(final int i, final DataPoint dp) {
    timestamps[i] = dp.timestamp();
    if (dp.isInteger()) {
      //LOG.debug("Putting #" + i + " (long) " + dp.longValue()
      //          + " @ time " + dp.timestamp());
      values[i] = dp.longValue();
    } else {
      //LOG.debug("Putting #" + i + " (double) " + dp.doubleValue()
      //          + " @ time " + dp.timestamp());
      values[i] = Double.doubleToRawLongBits(dp.doubleValue());
      timestamps[i] |= FLAG_FLOAT;
    }
  }
73186
@SuppressWarnings({"ForLoopReplaceableByForEach"})
	protected void selectMatchSet(List<Point> dataSet, double threshold, Model param) {
		candidatePoints.clear();
		modelDistance.setModel(param);

		for (int i = 0; i < dataSet.size(); i++) {
			Point point = dataSet.get(i);

			double distance = modelDistance.computeDistance(point);
			if (distance < threshold) {
				matchToInput[candidatePoints.size()] = i;
				candidatePoints.add(point);
			}
		}
	}
817246
protected static double calculateLrd(LofPoint basePoint, LofDataSet dataSet)
    {
        int countedData = 0;
        double totalAmount = 0.0d;

        for (String targetDataId : basePoint.getkDistanceNeighbor())
        {
            LofPoint targetPoint = dataSet.getDataMap().get(targetDataId);

            if (targetPoint == null)
            {
                continue;
            }

            double reachDist = calculateReachDistance(basePoint, targetPoint);
            totalAmount = totalAmount + reachDist;
            countedData++;
        }

        if (totalAmount == 0.0d)
        {
            return totalAmount;
        }

        return (countedData) / totalAmount;
    }
701503
static void dataSetAssertion(CallInfo callInfo, DataSet expected, DataSet actual) {
    validateDataSetAssertion(expected, actual);
    DataSource source = expected.getSource();
    Delta delta = new Delta(expected, actual); 
    DataSetAssertion assertion = new DataSetAssertion(expected, delta);
    source.getDB().log(callInfo, assertion);
    
    if (! assertion.passed()) {
      throw new DBAssertionError(callInfo.getMessage());
    }
  }
102855
@SafeVarargs
  public static void
  takeSnapshot(DataSource... dataSources)  {
    ensureDistinctDataSources(dataSources, Function.identity());
    foreach(dataSources, 
           (callInfo, dataSource) -> dataSource.executeQuery(callInfo, true), 
           CallInfo.create());
  }
521789
public static List<List<Writable>> toRecords(DataSet dataSet) {
        if (isClassificationDataSet(dataSet)) {
            return getClassificationWritableMatrix(dataSet);
        } else {
            return getRegressionWritableMatrix(dataSet);
        }
    }
743
public static double calculateLofWithoutUpdate(int kn, LofPoint targetPoint, LofDataSet dataSet)
    {
        // 対象点のK距離、K距離近傍を算出する。
        KDistanceResult kResult = calculateKDistance(kn, targetPoint, dataSet);

        LofPoint tmpPoint = targetPoint.deepCopy();
        tmpPoint.setkDistance(kResult.getkDistance());
        tmpPoint.setkDistanceNeighbor(kResult.getkDistanceNeighbor());

        updateLrd(tmpPoint, dataSet);

        // 対象の局所外れ係数を算出する。
        double lof = calculateLof(tmpPoint, dataSet);
        return lof;
    }
654702
public void connecDataSet(IntermediateDataSet dataSet) {
		if (dataSet == null) {
			throw new NullPointerException();
		}
		if (this.source != null) {
			throw new IllegalStateException("The edge is already connected.");
		}
		if (!dataSet.getId().equals(sourceId)) {
			throw new IllegalArgumentException("The data set to connect does not match the sourceId.");
		}
		
		this.source = dataSet;
	}
856504
private void onDataSetStaleEvent(@Observes DataSetStaleEvent event) {
        checkNotNull("event",
                     event);
        String uuid = event.getDataSetDef().getUUID();

        // Remove any stale data existing on the client.
        // This will force next lookup requests to push a refreshed data set.
        clientDataSetManager.removeDataSet(uuid);
        remoteMetadataMap.remove(uuid);

        // If a data set has been updated on the sever then fire an event.
        // In this case the notification is always send, no matter whether the data set is pushed to the client or not.
        dataSetModifiedEvent.fire(new DataSetModifiedEvent(event.getDataSetDef()));
    }
1295073
public static void write(IDataSet dataSet, OutputStream out) throws IOException, DataSetException {
		logger.debug("write(dataSet={}, out={}) - start", dataSet, out);

		FlatXmlWriter datasetWriter = new FlatXmlWriter(out);
		datasetWriter.setIncludeEmptyTable(true);
		datasetWriter.write(dataSet);
	}