Return a JavaRDD of Object by unpickling

    It will convert each Python object into Java object by Pyrolite, whenever the
    RDD is serialized in batch or not.
3081
def _to_java_object_rdd(rdd):
    
    rdd = rdd._reserialize(AutoBatchedSerializer(PickleSerializer()))
    return rdd.ctx._jvm.org.apache.spark.mllib.api.python.SerDe.pythonToJava(rdd._jrdd, True)
502120
public Pair<VocabCache<VocabWord>, GloveWeightLookupTable> train(JavaRDD<String> rdd) throws Exception {
        // Each `train()` can use different parameters
        final JavaSparkContext sc = new JavaSparkContext(rdd.context());
        final SparkConf conf = sc.getConf();
        final int vectorLength = assignVar(VECTOR_LENGTH, conf, Integer.class);
        final boolean useAdaGrad = assignVar(ADAGRAD, conf, Boolean.class);
        final double negative = assignVar(NEGATIVE, conf, Double.class);
        final int numWords = assignVar(NUM_WORDS, conf, Integer.class);
        final int window = assignVar(WINDOW, conf, Integer.class);
        final double alpha = assignVar(ALPHA, conf, Double.class);
        final double minAlpha = assignVar(MIN_ALPHA, conf, Double.class);
        final int iterations = assignVar(ITERATIONS, conf, Integer.class);
        final int nGrams = assignVar(N_GRAMS, conf, Integer.class);
        final String tokenizer = assignVar(TOKENIZER, conf, String.class);
        final String tokenPreprocessor = assignVar(TOKEN_PREPROCESSOR, conf, String.class);
        final boolean removeStop = assignVar(REMOVE_STOPWORDS, conf, Boolean.class);

        Map<String, Object> tokenizerVarMap = new HashMap<String, Object>() {
            {
                put("numWords", numWords);
                put("nGrams", nGrams);
                put("tokenizer", tokenizer);
                put("tokenPreprocessor", tokenPreprocessor);
                put("removeStop", removeStop);
            }
        };
        Broadcast<Map<String, Object>> broadcastTokenizerVarMap = sc.broadcast(tokenizerVarMap);


        TextPipeline pipeline = new TextPipeline(rdd, broadcastTokenizerVarMap);
        pipeline.buildVocabCache();
        pipeline.buildVocabWordListRDD();


        // Get total word count
        Long totalWordCount = pipeline.getTotalWordCount();
        VocabCache<VocabWord> vocabCache = pipeline.getVocabCache();
        JavaRDD<Pair<List<String>, AtomicLong>> sentenceWordsCountRDD = pipeline.getSentenceWordsCountRDD();
        final Pair<VocabCache<VocabWord>, Long> vocabAndNumWords = new Pair<>(vocabCache, totalWordCount);

        vocabCacheBroadcast = sc.broadcast(vocabAndNumWords.getFirst());

        final GloveWeightLookupTable gloveWeightLookupTable = new GloveWeightLookupTable.Builder()
                        .cache(vocabAndNumWords.getFirst()).lr(conf.getDouble(GlovePerformer.ALPHA, 0.01))
                        .maxCount(conf.getDouble(GlovePerformer.MAX_COUNT, 100))
                        .vectorLength(conf.getInt(GlovePerformer.VECTOR_LENGTH, 300))
                        .xMax(conf.getDouble(GlovePerformer.X_MAX, 0.75)).build();
        gloveWeightLookupTable.resetWeights();

        gloveWeightLookupTable.getBiasAdaGrad().historicalGradient = Nd4j.ones(gloveWeightLookupTable.getSyn0().rows());
        gloveWeightLookupTable.getWeightAdaGrad().historicalGradient =
                        Nd4j.ones(gloveWeightLookupTable.getSyn0().shape());


        log.info("Created lookup table of size " + Arrays.toString(gloveWeightLookupTable.getSyn0().shape()));
        CounterMap<String, String> coOccurrenceCounts = sentenceWordsCountRDD
                        .map(new CoOccurrenceCalculator(symmetric, vocabCacheBroadcast, windowSize))
                        .fold(new CounterMap<String, String>(), new CoOccurrenceCounts());
        Iterator<Pair<String, String>> pair2 = coOccurrenceCounts.getIterator();
        List<Triple<String, String, Float>> counts = new ArrayList<>();

        while (pair2.hasNext()) {
            Pair<String, String> next = pair2.next();
            if (coOccurrenceCounts.getCount(next.getFirst(), next.getSecond()) > gloveWeightLookupTable.getMaxCount()) {
                coOccurrenceCounts.setCount(next.getFirst(), next.getSecond(),
                                (float) gloveWeightLookupTable.getMaxCount());
            }
            counts.add(new Triple<>(next.getFirst(), next.getSecond(),
                    (float) coOccurrenceCounts.getCount(next.getFirst(), next.getSecond())));

        }

        log.info("Calculated co occurrences");

        JavaRDD<Triple<String, String, Float>> parallel = sc.parallelize(counts);
        JavaPairRDD<String, Tuple2<String, Float>> pairs = parallel
                        .mapToPair(new PairFunction<Triple<String, String, Float>, String, Tuple2<String, Float>>() {
                            @Override
                            public Tuple2<String, Tuple2<String, Float>> call(
                                            Triple<String, String, Float> stringStringDoubleTriple) throws Exception {
                                return new Tuple2<>(stringStringDoubleTriple.getFirst(),
                                                new Tuple2<>(stringStringDoubleTriple.getSecond(),
                                                                stringStringDoubleTriple.getThird()));
                            }
                        });

        JavaPairRDD<VocabWord, Tuple2<VocabWord, Float>> pairsVocab = pairs.mapToPair(
                        new PairFunction<Tuple2<String, Tuple2<String, Float>>, VocabWord, Tuple2<VocabWord, Float>>() {
                            @Override
                            public Tuple2<VocabWord, Tuple2<VocabWord, Float>> call(
                                            Tuple2<String, Tuple2<String, Float>> stringTuple2Tuple2) throws Exception {
                                VocabWord w1 = vocabCacheBroadcast.getValue().wordFor(stringTuple2Tuple2._1());
                                VocabWord w2 = vocabCacheBroadcast.getValue().wordFor(stringTuple2Tuple2._2()._1());
                                return new Tuple2<>(w1, new Tuple2<>(w2, stringTuple2Tuple2._2()._2()));
                            }
                        });


        for (int i = 0; i < iterations; i++) {
            JavaRDD<GloveChange> change =
                            pairsVocab.map(new Function<Tuple2<VocabWord, Tuple2<VocabWord, Float>>, GloveChange>() {
                                @Override
                                public GloveChange call(
                                                Tuple2<VocabWord, Tuple2<VocabWord, Float>> vocabWordTuple2Tuple2)
                                                throws Exception {
                                    VocabWord w1 = vocabWordTuple2Tuple2._1();
                                    VocabWord w2 = vocabWordTuple2Tuple2._2()._1();
                                    INDArray w1Vector = gloveWeightLookupTable.getSyn0().slice(w1.getIndex());
                                    INDArray w2Vector = gloveWeightLookupTable.getSyn0().slice(w2.getIndex());
                                    INDArray bias = gloveWeightLookupTable.getBias();
                                    double score = vocabWordTuple2Tuple2._2()._2();
                                    double xMax = gloveWeightLookupTable.getxMax();
                                    double maxCount = gloveWeightLookupTable.getMaxCount();
                                    //w1 * w2 + bias
                                    double prediction = Nd4j.getBlasWrapper().dot(w1Vector, w2Vector);
                                    prediction += bias.getDouble(w1.getIndex()) + bias.getDouble(w2.getIndex());

                                    double weight = FastMath.pow(Math.min(1.0, (score / maxCount)), xMax);

                                    double fDiff = score > xMax ? prediction : weight * (prediction - Math.log(score));
                                    if (Double.isNaN(fDiff))
                                        fDiff = Nd4j.EPS_THRESHOLD;
                                    //amount of change
                                    double gradient = fDiff;

                                    Pair<INDArray, Float> w1Update = update(gloveWeightLookupTable.getWeightAdaGrad(),
                                                    gloveWeightLookupTable.getBiasAdaGrad(),
                                                    gloveWeightLookupTable.getSyn0(), gloveWeightLookupTable.getBias(),
                                                    w1, w1Vector, w2Vector, gradient);
                                    Pair<INDArray, Float> w2Update = update(gloveWeightLookupTable.getWeightAdaGrad(),
                                                    gloveWeightLookupTable.getBiasAdaGrad(),
                                                    gloveWeightLookupTable.getSyn0(), gloveWeightLookupTable.getBias(),
                                                    w2, w2Vector, w1Vector, gradient);
                                    return new GloveChange(w1, w2, w1Update.getFirst(), w2Update.getFirst(),
                                                    w1Update.getSecond(), w2Update.getSecond(), fDiff,
                                                    gloveWeightLookupTable.getWeightAdaGrad().getHistoricalGradient()
                                                                    .slice(w1.getIndex()),
                                                    gloveWeightLookupTable.getWeightAdaGrad().getHistoricalGradient()
                                                                    .slice(w2.getIndex()),
                                                    gloveWeightLookupTable.getBiasAdaGrad().getHistoricalGradient()
                                                                    .getDouble(w2.getIndex()),
                                                    gloveWeightLookupTable.getBiasAdaGrad().getHistoricalGradient()
                                                                    .getDouble(w1.getIndex()));

                                }
                            });



            List<GloveChange> gloveChanges = change.collect();
            double error = 0.0;
            for (GloveChange change2 : gloveChanges) {
                change2.apply(gloveWeightLookupTable);
                error += change2.getError();
            }


            List l = pairsVocab.collect();
            Collections.shuffle(l);
            pairsVocab = sc.parallelizePairs(l);

            log.info("Error at iteration " + i + " was " + error);



        }

        return new Pair<>(vocabAndNumWords.getFirst(), gloveWeightLookupTable);
    }
828121
def serialize_function_from_method(meth)
          if pry?
            meth = Pry::Method.new(meth)
          end

          {type: 'method', name: meth.name, content: meth.source}
        rescue
          raise Spark::SerializeError, 'Method can not be serialized. Use full path or Proc.'
        end
1171534
def collect(self):
        
        with SCCallSiteSync(self.context) as css:
            sock_info = self.ctx._jvm.PythonRDD.collectAndServe(self._jrdd.rdd())
        return list(_load_from_socket(sock_info, self._jrdd_deserializer))
807093
def callMLlibFunc(name, *args):
    
    sc = SparkContext.getOrCreate()
    api = getattr(sc._jvm.PythonMLLibAPI(), name)
    return callJavaFunc(sc, api, *args)
1158949
def from_jd(jd: float, fmt: str = 'jd') -> datetime:
    
    jd, jdf = __from_format(jd, fmt)

    l = jd+68569
    n = 4*l//146097
    l = l-(146097*n+3)//4
    i = 4000*(l+1)//1461001
    l = l-1461*i//4+31
    j = 80*l//2447
    k = l-2447*j//80
    l = j//11
    j = j+2-12*l
    i = 100*(n-49)+i+l

    year = int(i)
    month = int(j)
    day = int(k)

    # in microseconds
    frac_component = int(jdf * (1e6*24*3600))

    hours = int(frac_component // (1e6*3600))
    frac_component -= hours * 1e6*3600

    minutes = int(frac_component // (1e6*60))
    frac_component -= minutes * 1e6*60

    seconds = int(frac_component // 1e6)
    frac_component -= seconds*1e6

    frac_component = int(frac_component)

    dt = datetime(year=year, month=month, day=day,
                  hour=hours, minute=minutes, second=seconds, microsecond=frac_component)
    return dt
139142
def union(self, other):
        
        if self._jrdd_deserializer == other._jrdd_deserializer:
            rdd = RDD(self._jrdd.union(other._jrdd), self.ctx,
                      self._jrdd_deserializer)
        else:
            # These RDDs contain data in different serialized formats, so we
            # must normalize them to the default serializer.
            self_copy = self._reserialize()
            other_copy = other._reserialize()
            rdd = RDD(self_copy._jrdd.union(other_copy._jrdd), self.ctx,
                      self.ctx.serializer)
        if (self.partitioner == other.partitioner and
                self.getNumPartitions() == rdd.getNumPartitions()):
            rdd.partitioner = self.partitioner
        return rdd
139459
def to_python(self, value):
        """"""
        cache = None
        if value is None:
            return None

        if isinstance(value, DataDescriptor):
            return value
        elif isinstance(value, dict):
            # Allow pre-hydrated data objects.
            cache = value
            try:
                value = cache['__id']
            except KeyError:
                raise ValidationError("dictionary must contain an '__id' element")
        elif not isinstance(value, int):
            raise ValidationError("field must be a DataDescriptor, data object primary key or dict")

        return DataDescriptor(value, self, cache)
907123
def toJSON(self, use_unicode=True):
        
        rdd = self._jdf.toJSON()
        return RDD(rdd.toJavaRDD(), self._sc, UTF8Deserializer(use_unicode))
1355888
def _get_method(self, rdata):
        
        if 'method' in rdata:
            if not isinstance(rdata['method'], basestring):
                raise InvalidRequestError
        else:
            raise InvalidRequestError

        if rdata['method'] not in self.method_data.keys():
            raise MethodNotFoundError

        return rdata['method']
1274074
public <T> JavaRDD<T> createJavaRDD(
            ExtractorConfig<T> config) {
        return new DeepJavaRDD<>((DeepRDD<T, ExtractorConfig<T>>) createRDD(config));
    }
1363234
public void train(JavaRDD<String> corpusRDD) throws Exception {
        log.info("Start training ...");

        if (workers > 0)
            corpusRDD.repartition(workers);

        // SparkContext
        final JavaSparkContext sc = new JavaSparkContext(corpusRDD.context());

        // Pre-defined variables
        Map<String, Object> tokenizerVarMap = getTokenizerVarMap();
        Map<String, Object> word2vecVarMap = getWord2vecVarMap();

        // Variables to fill in train
        final JavaRDD<AtomicLong> sentenceWordsCountRDD;
        final JavaRDD<List<VocabWord>> vocabWordListRDD;
        final JavaPairRDD<List<VocabWord>, Long> vocabWordListSentenceCumSumRDD;
        final VocabCache<VocabWord> vocabCache;
        final JavaRDD<Long> sentenceCumSumCountRDD;
        int maxRep = 1;

        // Start Training //
        //////////////////////////////////////
        log.info("Tokenization and building VocabCache ...");
        // Processing every sentence and make a VocabCache which gets fed into a LookupCache
        Broadcast<Map<String, Object>> broadcastTokenizerVarMap = sc.broadcast(tokenizerVarMap);
        TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap);
        pipeline.buildVocabCache();
        pipeline.buildVocabWordListRDD();

        // Get total word count and put into word2vec variable map
        word2vecVarMap.put("totalWordCount", pipeline.getTotalWordCount());

        // 2 RDDs: (vocab words list) and (sentence Count).Already cached
        sentenceWordsCountRDD = pipeline.getSentenceCountRDD();
        vocabWordListRDD = pipeline.getVocabWordListRDD();

        // Get vocabCache and broad-casted vocabCache
        Broadcast<VocabCache<VocabWord>> vocabCacheBroadcast = pipeline.getBroadCastVocabCache();
        vocabCache = vocabCacheBroadcast.getValue();

        log.info("Vocab size: {}", vocabCache.numWords());

        //////////////////////////////////////
        log.info("Building Huffman Tree ...");
        // Building Huffman Tree would update the code and point in each of the vocabWord in vocabCache
        /*
        We don't need to build tree here, since it was built earlier, at TextPipeline.buildVocabCache() call.
        
        Huffman huffman = new Huffman(vocabCache.vocabWords());
        huffman.build();
        huffman.applyIndexes(vocabCache);
        */
        //////////////////////////////////////
        log.info("Calculating cumulative sum of sentence counts ...");
        sentenceCumSumCountRDD = new CountCumSum(sentenceWordsCountRDD).buildCumSum();

        //////////////////////////////////////
        log.info("Mapping to RDD(vocabWordList, cumulative sentence count) ...");
        vocabWordListSentenceCumSumRDD =
                        vocabWordListRDD.zip(sentenceCumSumCountRDD).setName("vocabWordListSentenceCumSumRDD");

        /////////////////////////////////////
        log.info("Broadcasting word2vec variables to workers ...");
        Broadcast<Map<String, Object>> word2vecVarMapBroadcast = sc.broadcast(word2vecVarMap);
        Broadcast<double[]> expTableBroadcast = sc.broadcast(expTable);



        /////////////////////////////////////
        log.info("Training word2vec sentences ...");
        FlatMapFunction firstIterFunc =
                        new FirstIterationFunction(word2vecVarMapBroadcast, expTableBroadcast, vocabCacheBroadcast);
        @SuppressWarnings("unchecked")
        JavaRDD<Pair<VocabWord, INDArray>> indexSyn0UpdateEntryRDD =
                        vocabWordListSentenceCumSumRDD.mapPartitions(firstIterFunc).map(new MapToPairFunction());

        // Get all the syn0 updates into a list in driver
        List<Pair<VocabWord, INDArray>> syn0UpdateEntries = indexSyn0UpdateEntryRDD.collect();

        // Instantiate syn0
        INDArray syn0 = Nd4j.zeros(vocabCache.numWords(), layerSize);

        // Updating syn0 first pass: just add vectors obtained from different nodes
        log.info("Averaging results...");
        Map<VocabWord, AtomicInteger> updates = new HashMap<>();
        Map<Long, Long> updaters = new HashMap<>();
        for (Pair<VocabWord, INDArray> syn0UpdateEntry : syn0UpdateEntries) {
            syn0.getRow(syn0UpdateEntry.getFirst().getIndex()).addi(syn0UpdateEntry.getSecond());

            // for proper averaging we need to divide resulting sums later, by the number of additions
            if (updates.containsKey(syn0UpdateEntry.getFirst())) {
                updates.get(syn0UpdateEntry.getFirst()).incrementAndGet();
            } else
                updates.put(syn0UpdateEntry.getFirst(), new AtomicInteger(1));

            if (!updaters.containsKey(syn0UpdateEntry.getFirst().getVocabId())) {
                updaters.put(syn0UpdateEntry.getFirst().getVocabId(), syn0UpdateEntry.getFirst().getAffinityId());
            }
        }

        // Updating syn0 second pass: average obtained vectors
        for (Map.Entry<VocabWord, AtomicInteger> entry : updates.entrySet()) {
            if (entry.getValue().get() > 1) {
                if (entry.getValue().get() > maxRep)
                    maxRep = entry.getValue().get();
                syn0.getRow(entry.getKey().getIndex()).divi(entry.getValue().get());
            }
        }

        long totals = 0;

        log.info("Finished calculations...");


        vocab = vocabCache;
        InMemoryLookupTable<VocabWord> inMemoryLookupTable = new InMemoryLookupTable<VocabWord>();
        Environment env = EnvironmentUtils.buildEnvironment();
        env.setNumCores(maxRep);
        env.setAvailableMemory(totals);
        update(env, Event.SPARK);
        inMemoryLookupTable.setVocab(vocabCache);
        inMemoryLookupTable.setVectorLength(layerSize);
        inMemoryLookupTable.setSyn0(syn0);
        lookupTable = inMemoryLookupTable;
        modelUtils.init(lookupTable);
    }
896697
public K getSavedId() {
        if (dbObjects.length == 0) {
            throw new MongoException("No objects to return");
        }
        if (dbObjects[0] instanceof JacksonDBObject) {
            throw new UnsupportedOperationException("Generated _id retrieval not supported when using stream serialization");
        }
        return jacksonDBCollection.convertFromDbId(dbObjects[0].get("_id"));
    }
1268013
def rdd(self):
        
        if self._lazy_rdd is None:
            jrdd = self._jdf.javaToPython()
            self._lazy_rdd = RDD(jrdd, self.sql_ctx._sc, BatchedSerializer(PickleSerializer()))
        return self._lazy_rdd
383672
def _to_java(self):
        
        sc = SparkContext._active_spark_context
        java_models = [model._to_java() for model in self.models]
        java_models_array = JavaWrapper._new_java_array(
            java_models, sc._gateway.jvm.org.apache.spark.ml.classification.ClassificationModel)
        metadata = JavaParams._new_java_obj("org.apache.spark.sql.types.Metadata")
        _java_obj = JavaParams._new_java_obj("org.apache.spark.ml.classification.OneVsRestModel",
                                             self.uid, metadata.empty(), java_models_array)
        _java_obj.set("classifier", self.getClassifier()._to_java())
        _java_obj.set("featuresCol", self.getFeaturesCol())
        _java_obj.set("labelCol", self.getLabelCol())
        _java_obj.set("predictionCol", self.getPredictionCol())
        return _java_obj
260309
def pyrdf2(value, class_type=None, datatype=None, lang=None, **kwargs):
    
    try:

        if isinstance(value, dict):
            # test to see if the type is a literal, a literal will have a another
            # dictionary key call datatype. Feed the datatype to the lookup to
            # return the value else convert it to a XsdString
            if value.get('type') == "literal":
                if not value.get("datatype"):
                    return XsdString(value['value'])
                else:
                    try:
                        if value.get("lang"):
                            # The lang keyword only applies to XsdString types
                            return DT_LOOKUP[value['datatype']](value['value'],
                                    lang=value.get("lang"))
                        else:
                            return DT_LOOKUP[value['datatype']](value['value'])
                    except:
                        rtn_val = BaseRdfDataType(value['value'])
                        rtn_val.datatype = Uri(value['datatype'])
                        return rtn_val
            else:
                return DT_LOOKUP[value['type']](value['value'])
        elif isinstance(value, BaseRdfDataType):
            return value
        else:
            return DT_LOOKUP[type(value)](value)
    except:
        pdb.set_trace()
        pass
951582
def from_jd(jd, method=None):
    ''''''
    method = method or 'equinox'

    if method == 'equinox':
        return _from_jd_equinox(jd)

    else:
        return _from_jd_schematic(jd, method)