指定したK値、対象点、データセットを基に局所外れ係数スコアを算出する。<br>
本メソッド呼び出しによってデータセットの更新は行われない。<br>
学習データの更新を伴わないため、高速に処理が可能となっている。

@param kn K値
@param targetPoint 対象点
@param dataSet 学習データセット
@return LOFスコア
743
public static double calculateLofWithoutUpdate(int kn, LofPoint targetPoint, LofDataSet dataSet)
    {
        // 対象点のK距離、K距離近傍を算出する。
        KDistanceResult kResult = calculateKDistance(kn, targetPoint, dataSet);

        LofPoint tmpPoint = targetPoint.deepCopy();
        tmpPoint.setkDistance(kResult.getkDistance());
        tmpPoint.setkDistanceNeighbor(kResult.getkDistanceNeighbor());

        updateLrd(tmpPoint, dataSet);

        // 対象の局所外れ係数を算出する。
        double lof = calculateLof(tmpPoint, dataSet);
        return lof;
    }
1339818
public void setTargetConfigurationValueSet(java.util.Collection<TargetReservationValue> targetConfigurationValueSet) {
        if (targetConfigurationValueSet == null) {
            this.targetConfigurationValueSet = null;
            return;
        }

        this.targetConfigurationValueSet = new com.amazonaws.internal.SdkInternalList<TargetReservationValue>(targetConfigurationValueSet);
    }
1373777
public static <K, V> Set<Map<K, V>> setify(ObjectMapper mapper, Object source, Class<K> targetKeyType, Class<V> targetValueType) {
        return (Set<Map<K, V>>) collectify(mapper, source, Set.class, targetKeyType, targetValueType);
    }
377164
@Override
    protected LofDataSet mergeState(LofDataSet baseDataSet, LofDataSet targetDataSet,
            Map<String, Object> mergeConfig)
    {
        int kn = Integer.parseInt(mergeConfig.get(LofConfKey.KN).toString());
        int maxDataCount = Integer.parseInt(mergeConfig.get(LofConfKey.MAX_DATA_COUNT).toString());

        // 学習データの保持する対象点を新しい方から最大保持数個だけ取得
        LofDataSet mergedDataSet = LofCalculator.mergeDataSet(baseDataSet, targetDataSet,
                maxDataCount);

        // 中間データ保持設定が存在しない場合は中間データの生成は行わない。
        if (mergeConfig.containsKey(LofConfKey.HAS_INTERMEDIATE) == false)
        {
            return mergedDataSet;
        }

        // 中間データ保持設定が存在し、かつ「true」の場合のみ中間データの生成を行う。
        if (Boolean.getBoolean(mergeConfig.get(LofConfKey.HAS_INTERMEDIATE).toString()) == true)
        {
            LofCalculator.initDataSet(kn, mergedDataSet);
        }

        return mergedDataSet;
    }
300055
private int collapsedGibbsSampling(Dataframe dataset) {
        AbstractModelParameters modelParameters = knowledgeBase.getModelParameters();
        
        Map<Integer, CL> clusterMap = modelParameters.getClusterMap();
        AbstractTrainingParameters trainingParameters = knowledgeBase.getTrainingParameters();
        
        double alpha = trainingParameters.getAlpha();
        
        //Initialize clusters, create a cluster for every xi
        Integer newClusterId = clusterMap.size(); //start counting the Ids based on clusters in the list

        if(trainingParameters.getInitializationMethod()==AbstractTrainingParameters.Initialization.ONE_CLUSTER_PER_RECORD) {
            for(Map.Entry<Integer, Record> e : dataset.entries()) {
                Integer rId = e.getKey();
                Record r = e.getValue();
                
                //generate a new cluster
                CL cluster = createNewCluster(newClusterId);
                cluster.add(r);
                clusterMap.put(newClusterId, cluster);

                //add the record in the new cluster
                r = new Record(r.getX(), r.getY(), newClusterId, r.getYPredictedProbabilities());
                dataset._unsafe_set(rId, r);

                ++newClusterId;
            }            
        }
        else {
            int numberOfNewClusters = (int)(Math.max(alpha, 1)*Math.log(dataset.size())); //a*log(n) clusters on average
            if(numberOfNewClusters<=0) {
                numberOfNewClusters=1;
            }
            
            //generate new clusters
            for(int i=0;i<numberOfNewClusters;++i) {
                //generate a new cluster
                CL cluster = createNewCluster(newClusterId);
                clusterMap.put(newClusterId, cluster);

                ++newClusterId;
            }
            
            int clusterMapSize = newClusterId;
            for(Map.Entry<Integer, Record> e : dataset.entries()) {
                Integer rId = e.getKey();
                Record r = e.getValue();
                
                Integer assignedClusterId = PHPMethods.mt_rand(0, clusterMapSize-1);
                
                r = new Record(r.getX(), r.getY(), assignedClusterId, r.getYPredictedProbabilities());
                dataset._unsafe_set(rId, r);
                
                CL c = getFromClusterMap(assignedClusterId, clusterMap);
                c.add(r);
                clusterMap.put(assignedClusterId, c);
            }
        }

                
        int n = clusterMap.size();
        
        int maxIterations = trainingParameters.getMaxIterations();
        
        boolean noChangeMade=false;
        int iteration=0;
        while(iteration<maxIterations && noChangeMade==false) {
            
            logger.debug("Iteration {}", iteration);
            
            noChangeMade=true;
            for(Map.Entry<Integer, Record> e : dataset.entries()) {
                Integer rId = e.getKey();
                Record r = e.getValue();
                
                Integer pointClusterId = (Integer) r.getYPredicted();
                CL ci = getFromClusterMap(pointClusterId, clusterMap);
                
                //remove the point from the cluster
                ci.remove(r);
                
                //if empty cluster remove it
                if(ci.size()==0) {
                    clusterMap.remove(pointClusterId);
                }
                else {
                    clusterMap.put(pointClusterId, ci);
                }
                
                AssociativeArray condProbCiGivenXiAndOtherCi = clusterProbabilities(r, n, clusterMap);
                                
                //Calculate the probabilities of assigning the point to a new cluster
                //compute P*(X[i]) = P(X[i]|λ)
                
                CL cNew = createNewCluster(newClusterId);
                
                double priorLogPredictive = cNew.posteriorLogPdf(r);

                //compute P(z[i] = * | z[-i], Data) = α/(α+N-1)
                double probNewCluster = alpha/(alpha+n-1.0);

                condProbCiGivenXiAndOtherCi.put(newClusterId, priorLogPredictive+Math.log(probNewCluster));
                
                //normalize probabilities P(z[i])
                Descriptives.normalizeExp(condProbCiGivenXiAndOtherCi);
                
                Integer sampledClusterId = (Integer) SimpleRandomSampling.weightedSampling(condProbCiGivenXiAndOtherCi, 1, true).iterator().next();
                //condProbCiGivenXiAndOtherCi=null;
                
                //Add Xi back to the sampled AbstractCluster
                if(Objects.equals(sampledClusterId, newClusterId)) { //if new cluster
                    //add the record in the new cluster
                    r = new Record(r.getX(), r.getY(), newClusterId, r.getYPredictedProbabilities());
                    dataset._unsafe_set(rId, r);
                    
                    cNew.add(r);
                    
                    clusterMap.put(newClusterId, cNew);
                    
                    noChangeMade=false;
                    
                    ++newClusterId;
                }
                else {
                    if(!Objects.equals(pointClusterId, sampledClusterId)) { //if it assigned in a different cluster update the record
                        r = new Record(r.getX(), r.getY(), sampledClusterId, r.getYPredictedProbabilities());
                        dataset._unsafe_set(rId, r);
                        noChangeMade=false;
                    }
                    
                    CL c = getFromClusterMap(sampledClusterId, clusterMap);
                    c.add(r); //add it to the cluster (or just add it back)
                    clusterMap.put(sampledClusterId, c);
                }
                
            }
            
            ++iteration;
        }
        
        return iteration;
    }
419345
private void createDataset(String datasetId, @Nullable String location)
      throws IOException, InterruptedException {
    Dataset dataset = new Dataset();
    DatasetReference reference = new DatasetReference();
    reference.setProjectId(projectId);
    reference.setDatasetId(datasetId);
    dataset.setDatasetReference(reference);
    if (location != null) {
      dataset.setLocation(location);
    }

    executeWithBackOff(
        client.datasets().insert(projectId, dataset),
        String.format(
            "Error when trying to create the temporary dataset %s in project %s.",
            datasetId, projectId));
  }
1354827
public void lookupDataSet(final DataSetLookup request,
                              final DataSetReadyCallback listener) throws Exception {

        // Look always into the client data set manager.
        if (clientDataSetManager.getDataSet(request.getDataSetUUID()) != null) {
            DataSet dataSet = clientDataSetManager.lookupDataSet(request);
            listener.callback(dataSet);
        }
        // If the data set is not in client, then look up remotely (only if the remote access is available).
        else if (dataSetLookupServices != null) {

            // First of all, get the target data set estimated size.
            fetchMetadata(request.getDataSetUUID(),
                          new DataSetMetadataCallback() {
                              public void callback(DataSetMetadata metatada) {

                                  // Push the data set to client if and only if the push feature is enabled, the data set is
                                  // pushable & the data set is smaller than the max push size defined.
                                  DataSetDef dsetDef = metatada.getDefinition();
                                  int estimatedSize = metatada.getEstimatedSize() / 1000;
                                  boolean isPushable = dsetDef != null && dsetDef.isPushEnabled() && estimatedSize < dsetDef.getPushMaxSize();
                                  if (pushRemoteDataSetEnabled && isPushable) {

                                      // Check if a push is already in progress.
                                      // (This is necessary in order to avoid repeating multiple push requests over the same data set).
                                      DataSetPushHandler pushHandler = pushRequestMap.get(request.getDataSetUUID());
                                      if (pushHandler == null) {
                                          // Create a push handler.
                                          pushHandler = new DataSetPushHandler(metatada);

                                          // Send the lookup request to the server...
                                          DataSetLookup lookupSourceDataSet = new DataSetLookup(request.getDataSetUUID());
                                          _lookupDataSet(lookupSourceDataSet,
                                                         pushHandler);
                                      }
                                      // Register the lookup request into the current handler.
                                      pushHandler.registerLookup(request,
                                                                 listener);
                                  }
                                  // Lookup the remote data set otherwise.
                                  else {
                                      _lookupDataSet(request,
                                                     listener);
                                  }
                              }

                              // Data set metadata not found
                              public void notFound() {
                                  listener.notFound();
                              }

                              @Override
                              public boolean onError(final ClientRuntimeError error) {
                                  return listener.onError(error);
                              }
                          });
        }
        // Data set not found on client.
        else {
            listener.notFound();
        }
    }
34981
public void addDataPoint(String series, DataPoint dataPoint) {
        DataSet set = store.get(series);
        if (set == null) {
            set = new DataSet();
            store.put(series, set);
        }
        set.add(dataPoint);
    }
394713
@Override
    public void fit(DataSet dataSet) {
        featureStats = (S) newBuilder().addFeatures(dataSet).build();
        if (isFitLabel()) {
            labelStats = (S) newBuilder().addLabels(dataSet).build();
        }
    }
936921
static DataSetDto toDataSetDto(final IDataSet dataSet) {
        List<List<Object>> table = new ArrayList<>(dataSet.rowCount());
        
        for (IDsRow r : dataSet) {
            List<Object> row = new ArrayList<>(r.cellCount());
            
            for (IDsCell c : r) {
                row.add(c.getValue().get());
            }
            
            table.add(row);
        }
        
        DataSetDto dto = new DataSetDto();
        
        dto.table = table;
        
        dto.dataModelId = dataSet.getDataModelId().toString();
        dto.name = dataSet.getName();
        
        return dto;
    }
489411
@Override
	public DataSet<Vertex<K, VV>> createResult() {
		if (vertexDataSet == null) {
			throw new IllegalStateException("The input data set has not been set.");
		}

		// Prepare type information
		TypeInformation<K> keyType = ((TupleTypeInfo<?>) vertexDataSet.getType()).getTypeAt(0);
		TypeInformation<M> messageType = TypeExtractor.createTypeInfo(gather, GatherFunction.class, gather.getClass(), 2);
		TypeInformation<Tuple2<K, M>> innerType = new TupleTypeInfo<>(keyType, messageType);
		TypeInformation<Vertex<K, VV>> outputType = vertexDataSet.getType();

		// check whether the numVertices option is set and, if so, compute the total number of vertices
		// and set it within the gather, sum and apply functions

		DataSet<LongValue> numberOfVertices = null;
		if (this.configuration != null && this.configuration.isOptNumVertices()) {
			try {
				numberOfVertices = GraphUtils.count(this.vertexDataSet);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}

		// Prepare UDFs
		GatherUdf<K, VV, EV, M> gatherUdf = new GatherUdf<>(gather, innerType);
		SumUdf<K, VV, EV, M> sumUdf = new SumUdf<>(sum, innerType);
		ApplyUdf<K, VV, EV, M> applyUdf = new ApplyUdf<>(apply, outputType);

		final int[] zeroKeyPos = new int[] {0};
		final DeltaIteration<Vertex<K, VV>, Vertex<K, VV>> iteration =
				vertexDataSet.iterateDelta(vertexDataSet, maximumNumberOfIterations, zeroKeyPos);

		// set up the iteration operator
		if (this.configuration != null) {

			iteration.name(this.configuration.getName(
					"Gather-sum-apply iteration (" + gather + " | " + sum + " | " + apply + ")"));
			iteration.parallelism(this.configuration.getParallelism());
			iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory());

			// register all aggregators
			for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) {
				iteration.registerAggregator(entry.getKey(), entry.getValue());
			}
		}
		else {
			// no configuration provided; set default name
			iteration.name("Gather-sum-apply iteration (" + gather + " | " + sum + " | " + apply + ")");
		}

		// Prepare the neighbors
		if (this.configuration != null) {
			direction = this.configuration.getDirection();
		}
		DataSet<Tuple2<K, Neighbor<VV, EV>>> neighbors;
		switch(direction) {
			case OUT:
				neighbors = iteration
				.getWorkset().join(edgeDataSet)
				.where(0).equalTo(0).with(new ProjectKeyWithNeighborOUT<>());
				break;
			case IN:
				neighbors = iteration
				.getWorkset().join(edgeDataSet)
				.where(0).equalTo(1).with(new ProjectKeyWithNeighborIN<>());
				break;
			case ALL:
				neighbors =  iteration
						.getWorkset().join(edgeDataSet)
						.where(0).equalTo(0).with(new ProjectKeyWithNeighborOUT<>()).union(iteration
								.getWorkset().join(edgeDataSet)
								.where(0).equalTo(1).with(new ProjectKeyWithNeighborIN<>()));
				break;
			default:
				neighbors = iteration
						.getWorkset().join(edgeDataSet)
						.where(0).equalTo(0).with(new ProjectKeyWithNeighborOUT<>());
				break;
		}

		// Gather, sum and apply
		MapOperator<Tuple2<K, Neighbor<VV, EV>>, Tuple2<K, M>> gatherMapOperator = neighbors.map(gatherUdf);

		// configure map gather function with name and broadcast variables
		gatherMapOperator = gatherMapOperator.name("Gather");

		if (this.configuration != null) {
			for (Tuple2<String, DataSet<?>> e : this.configuration.getGatherBcastVars()) {
				gatherMapOperator = gatherMapOperator.withBroadcastSet(e.f1, e.f0);
			}
			if (this.configuration.isOptNumVertices()) {
				gatherMapOperator = gatherMapOperator.withBroadcastSet(numberOfVertices, "number of vertices");
			}
		}
		DataSet<Tuple2<K, M>> gatheredSet = gatherMapOperator;

		ReduceOperator<Tuple2<K, M>> sumReduceOperator = gatheredSet.groupBy(0).reduce(sumUdf);

		// configure reduce sum function with name and broadcast variables
		sumReduceOperator = sumReduceOperator.name("Sum");

		if (this.configuration != null) {
			for (Tuple2<String, DataSet<?>> e : this.configuration.getSumBcastVars()) {
				sumReduceOperator = sumReduceOperator.withBroadcastSet(e.f1, e.f0);
			}
			if (this.configuration.isOptNumVertices()) {
				sumReduceOperator = sumReduceOperator.withBroadcastSet(numberOfVertices, "number of vertices");
			}
		}
		DataSet<Tuple2<K, M>> summedSet = sumReduceOperator;

		JoinOperator<?, ?, Vertex<K, VV>> appliedSet = summedSet
				.join(iteration.getSolutionSet())
				.where(0)
				.equalTo(0)
				.with(applyUdf);

		// configure join apply function with name and broadcast variables
		appliedSet = appliedSet.name("Apply");

		if (this.configuration != null) {
			for (Tuple2<String, DataSet<?>> e : this.configuration.getApplyBcastVars()) {
				appliedSet = appliedSet.withBroadcastSet(e.f1, e.f0);
			}
			if (this.configuration.isOptNumVertices()) {
				appliedSet = appliedSet.withBroadcastSet(numberOfVertices, "number of vertices");
			}
		}

		// let the operator know that we preserve the key field
		appliedSet.withForwardedFieldsFirst("0").withForwardedFieldsSecond("0");

		return iteration.closeWith(appliedSet, appliedSet);
	}
592550
protected static KDistanceResult calculateKDistance(int kn, List<DistanceResult> distances)
    {
        // 算出した距離を用いてK距離とK距離近傍データのIDを算出
        int countedDataNum = 0;
        List<String> idList = new ArrayList<>();
        double nowDistance = 0.0d;

        // K値の値までIDのカウントを行い、距離を設定する。
        for (DistanceResult distanceResult : distances)
        {
            nowDistance = distanceResult.getDistance();
            idList.add(distanceResult.getDataId());
            countedDataNum++;

            if (kn <= countedDataNum)
            {
                break;
            }
        }

        KDistanceResult kResult = new KDistanceResult();
        kResult.setkDistance(nowDistance);
        kResult.setkDistanceNeighbor(idList);
        return kResult;
    }
783635
protected BaseFeatureData selectFeatures(IDataSet dataSet)
    {
        ChiSquareFeatureExtractor chiSquareFeatureExtractor = new ChiSquareFeatureExtractor();

        logger.start("使用卡方检测选择特征中...");
        //FeatureStats对象包含文档中所有特征及其统计信息
        BaseFeatureData featureData = chiSquareFeatureExtractor.extractBasicFeatureData(dataSet); //执行统计

        //我们传入这些统计信息到特征选择算法中，得到特征与其分值
        Map<Integer, Double> selectedFeatures = chiSquareFeatureExtractor.chi_square(featureData);

        //从统计数据中删掉无用的特征并重建特征映射表
        int[][] featureCategoryJointCount = new int[selectedFeatures.size()][];
        featureData.wordIdTrie = new BinTrie<Integer>();
        String[] wordIdArray = dataSet.getLexicon().getWordIdArray();
        int p = -1;
        for (Integer feature : selectedFeatures.keySet())
        {
            featureCategoryJointCount[++p] = featureData.featureCategoryJointCount[feature];
            featureData.wordIdTrie.put(wordIdArray[feature], p);
        }
        logger.finish(",选中特征数:%d / %d = %.2f%%\n", featureCategoryJointCount.length,
                      featureData.featureCategoryJointCount.length,
                      featureCategoryJointCount.length / (double)featureData.featureCategoryJointCount.length * 100.);
        featureData.featureCategoryJointCount = featureCategoryJointCount;

        return featureData;
    }
134033
protected final void setUp(DataSet dataSet, Set<Integer> categoricalToRemove, Set<Integer> numericalToRemove)
    {
        for(int i : categoricalToRemove)
            if (i >= dataSet.getNumCategoricalVars())
                throw new RuntimeException("The data set does not have a categorical value " + i + " to remove");
        for(int i : numericalToRemove)
            if (i >= dataSet.getNumNumericalVars())
                throw new RuntimeException("The data set does not have a numercal value " + i + " to remove");
        
        catIndexMap = new int[dataSet.getNumCategoricalVars()-categoricalToRemove.size()];
        newCatHeader = new CategoricalData[catIndexMap.length];
        numIndexMap = new int[dataSet.getNumNumericalVars()-numericalToRemove.size()];
        int k = 0;
        for(int i = 0; i < dataSet.getNumCategoricalVars(); i++)
        {
            if(categoricalToRemove.contains(i))
                continue;
            newCatHeader[k] = dataSet.getCategories()[i].clone();
            catIndexMap[k++] = i;
        }
        k = 0;
        for(int i = 0; i < dataSet.getNumNumericalVars(); i++)
        {
            if(numericalToRemove.contains(i))
                continue;
            numIndexMap[k++] = i;
        }
    }
986725
private boolean kNNABOD(Database db, Relation<V> relation, DBIDs ids, WritableDoubleDataStore abodvalues, DoubleMinMax minmaxabod) {
    DistanceQuery<V> dq = db.getDistanceQuery(relation, SquaredEuclideanDistanceFunction.STATIC);
    KNNQuery<V> knnq = db.getKNNQuery(dq, DatabaseQuery.HINT_OPTIMIZED_ONLY);
    boolean squared = true;
    if(knnq == null) {
      dq = db.getDistanceQuery(relation, EuclideanDistanceFunction.STATIC);
      knnq = db.getKNNQuery(dq, DatabaseQuery.HINT_OPTIMIZED_ONLY);
      if(knnq == null) {
        return false;
      }
      squared = false;
    }
    SimilarityQuery<V> lk = db.getSimilarityQuery(relation, LinearKernelFunction.STATIC);
    int k1 = k + 1; // We will get the query point back by the knnq.

    MeanVariance s = new MeanVariance();
    for(DBIDIter pA = ids.iter(); pA.valid(); pA.advance()) {
      KNNList nl = knnq.getKNNForDBID(pA, k1);
      double simAA = lk.similarity(pA, pA);

      s.reset();
      DoubleDBIDListIter iB = nl.iter(), iC = nl.iter();
      for(; iB.valid(); iB.advance()) {
        double dAB = iB.doubleValue();
        double simAB = lk.similarity(pA, iB);
        if(!(dAB > 0.)) {
          continue;
        }
        for(iC.seek(iB.getOffset() + 1); iC.valid(); iC.advance()) {
          double dAC = iC.doubleValue();
          double simAC = lk.similarity(pA, iC);
          if(!(dAC > 0.)) {
            continue;
          }
          // Exploit bilinearity of scalar product:
          // <B-A, C-A> = <B, C-A> - <A,C-A>
          // = <B,C> - <B,A> - <A,C> + <A,A>
          double simBC = lk.similarity(iB, iC);
          double numerator = simBC - simAB - simAC + simAA;
          if(squared) {
            double div = 1. / (dAB * dAC);
            s.put(numerator * div, FastMath.sqrt(div));
          }
          else {
            double sqrtdiv = 1. / (dAB * dAC);
            s.put(numerator * sqrtdiv * sqrtdiv, sqrtdiv);
          }
        }
      }
      final double abof = s.getNaiveVariance();
      minmaxabod.put(abof);
      abodvalues.putDouble(pA, abof);
    }
    return true;
  }
1083960
public void setK(final int K)
    {
        if(K < 2)
            throw new IllegalArgumentException("At least 2 topics must be learned");
        this.K = K;
        gammaLocal = new ThreadLocal<Vec>()
        {
            @Override
            protected Vec initialValue()
            {
                return new DenseVector(K);
            }
        };
        logThetaLocal  = new ThreadLocal<Vec>()
        {
            @Override
            protected Vec initialValue()
            {
                return new DenseVector(K);
            }
        };
        expLogThetaLocal = new ThreadLocal<Vec>()
        {
            @Override
            protected Vec initialValue()
            {
                return new DenseVector(K);
            }
        };
        
        lambda = null;
    }
654702
public void connecDataSet(IntermediateDataSet dataSet) {
		if (dataSet == null) {
			throw new NullPointerException();
		}
		if (this.source != null) {
			throw new IllegalStateException("The edge is already connected.");
		}
		if (!dataSet.getId().equals(sourceId)) {
			throw new IllegalArgumentException("The data set to connect does not match the sourceId.");
		}
		
		this.source = dataSet;
	}