Loads a word vectors file
507270
def load(fname, kind="auto", *args, **kwargs):
    
    if kind == "auto":
        if fname.endswith(".bin"):
            kind = "bin"
        elif fname.endswith(".txt"):
            kind = "txt"
        else:
            raise Exception("Could not identify kind")
    if kind == "bin":
        return word2vec.WordVectors.from_binary(fname, *args, **kwargs)
    elif kind == "txt":
        return word2vec.WordVectors.from_text(fname, *args, **kwargs)
    elif kind == "mmap":
        return word2vec.WordVectors.from_mmap(fname, *args, **kwargs)
    else:
        raise Exception("Unknown kind")
8510
@Deprecated
    public static void writeWordVectors(@NonNull ParagraphVectors vectors, @NonNull File path) {
        try (FileOutputStream fos = new FileOutputStream(path)) {
            writeWordVectors(vectors, fos);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
1301204
def get_anki_vocab(lang=['eng'], limit=None, filename='anki_en_vocabulary.csv'):
    
    texts = get_anki_phrases(lang=lang, limit=limit)
    docs = nlp(texts, lang=lang)
    vocab = get_vocab(docs)
    vocab['vector'] = get_word_vectors(vocab)  # TODO: turn this into a KeyedVectors object
    if filename:
        vocab.to_csv(os.path.join(BIGDATA_PATH, filename))
    return vocab
1306701
def load(filename, loader=None, implicit_tuple=True, env={}, schema=None):
  """"""
  with open(filename, 'r') as f:
    return loads(f.read(),
                 filename=filename,
                 loader=loader,
                 implicit_tuple=implicit_tuple,
                 env=env,
                 schema=schema)
1265634
public static Pair<InMemoryLookupTable, VocabCache> loadTxt(File vectorsFile)
            throws IOException, UnsupportedEncodingException {

        AbstractCache cache = new AbstractCache<>();
        BufferedReader reader = createReader(vectorsFile);
        LineIterator iter = IOUtils.lineIterator(reader);
        String line = null;
        boolean hasHeader = false;
        if (iter.hasNext()) {
            line = iter.nextLine(); // skip header line
            //look for spaces
            if (!line.contains(" ")) {
                log.debug("Skipping first line");
                hasHeader = true;
            } else {
                // we should check for something that looks like proper word vectors here. i.e: 1 word at the 0 position, and bunch of floats further
                String[] split = line.split(" ");
                try {
                    long[] header = new long[split.length];
                    for (int x = 0; x < split.length; x++) {
                        header[x] = Long.parseLong(split[x]);
                    }
                    if (split.length < 4)
                        hasHeader = true;
                    // now we know, if that's all ints - it's just a header
                    // [0] - number of words
                    // [1] - vectorSize
                    // [2] - number of documents <-- DL4j-only value
                    if (split.length == 3)
                        cache.incrementTotalDocCount(header[2]);

                    printOutProjectedMemoryUse(header[0], (int) header[1], 1);

                    hasHeader = true;

                    try {
                        reader.close();
                    } catch (Exception ex) {
                    }
                } catch (Exception e) {
                    // if any conversion exception hits - that'll be considered header
                    hasHeader = false;

                }
            }

        }

        //reposition buffer to be one line ahead
        if (hasHeader) {
            line = "";
            iter.close();
            //reader = new BufferedReader(new FileReader(vectorsFile));
            reader = createReader(vectorsFile);
            iter = IOUtils.lineIterator(reader);
            iter.nextLine();
        }

        List<INDArray> arrays = new ArrayList<>();
        long[] vShape = new long[]{1, -1};
        while (iter.hasNext()) {
            if (line.isEmpty())
                line = iter.nextLine();
            String[] split = line.split(" ");
            String word = decodeB64(split[0]); //split[0].replaceAll(whitespaceReplacement, " ");
            VocabWord word1 = new VocabWord(1.0, word);

            word1.setIndex(cache.numWords());

            cache.addToken(word1);

            cache.addWordToIndex(word1.getIndex(), word);

            cache.putVocabWord(word);

            float[] vector = new float[split.length - 1];

            for (int i = 1; i < split.length; i++) {
                vector[i - 1] = Float.parseFloat(split[i]);
            }

            vShape[1] = vector.length;
            INDArray row = Nd4j.create(vector, vShape);

            arrays.add(row);

            // workaround for skipped first row
            line = "";
        }

        INDArray syn = Nd4j.vstack(arrays);

        InMemoryLookupTable lookupTable =
                (InMemoryLookupTable) new InMemoryLookupTable.Builder().vectorLength(arrays.get(0).columns())
                        .useAdaGrad(false).cache(cache).useHierarchicSoftmax(false).build();

        lookupTable.setSyn0(syn);

        iter.close();

        try {
            reader.close();
        } catch (Exception e) {
        }

        return new Pair<>(lookupTable, (VocabCache) cache);
    }
12948
private Map<String,SemanticVector> loadSparseText(File sspaceFile) 
        throws IOException {

        LOGGER.info("loading sparse text TSS from " + sspaceFile);

        BufferedReader br = new BufferedReader(new FileReader(sspaceFile));
        String[] header = br.readLine().split("\\s+");
        int words = Integer.parseInt(header[0]);
        dimensions = Integer.parseInt(header[1]);
        
        Map<String,SemanticVector> wordToSemantics = 
            new HashMap<String,SemanticVector>(words, 2f);
            
        for (int wordIndex = 0; wordIndex < words; ++wordIndex) {
            String[] wordAndSemantics = br.readLine().split("\\|");
            String word = wordAndSemantics[0];
            SemanticVector semantics = new SemanticVector(dimensions);
            wordToSemantics.put(word, semantics);

            LOGGER.info("loading " + wordAndSemantics.length + 
                " timesteps for word " + word); 

            // read in each of the timesteps
            for (int tsIndx = 1; tsIndx < wordAndSemantics.length; ++tsIndx) {
                String[] tsAndVec = wordAndSemantics[tsIndx].split("%");
                String[] tsAndNonZero = tsAndVec[0].split(" ");
                long timeStep = Long.parseLong(tsAndNonZero[0]);
                updateTimeRange(timeStep);
                int nonZero = Integer.parseInt(tsAndNonZero[1]);
                String[] vecElements = tsAndVec[1].split(",");
                
                Map<Integer,Double> sparseArr = new IntegerMap<Double>();
                // elements are ordered as pairs of index,value,index,value,...
                for (int i = 0; i < vecElements.length; i += 2) {
                    Integer index = Integer.valueOf(vecElements[i]);
                    Double value = Double.valueOf(vecElements[i+1]);
                }
                semantics.setSemantics(timeStep, sparseArr);
            }
        }

        return wordToSemantics;
    }
378136
def load_common(self, wlist):
        """"""
        self.com_dict = os.path.join(self.base_dir, 'dicts', 'EN_vocab.txt')
        with open(self.com_dict) as words_file:
            data = words_file.read()
        self.common_words = set(data.splitlines())
        if wlist:
            new_words = ''
            for wl in wlist:
                with open(wl) as f:
                    new_words += f.read()
            if new_words:
                new_dict = set(data.splitlines())
                self.common_words.update(new_dict)
112779
def load_nouns(self, file):
        
        with open(os.path.join(main_dir, file + '.dat'), 'r') as f:
            self.nouns = json.load(f)
629253
def _read_words(filename):
  """"""
  with tf.gfile.GFile(filename, "r") as f:
    if sys.version_info[0] >= 3:
      return f.read().replace("\n", " %s " % EOS).split()
    else:
      return f.read().decode("utf-8").replace("\n", " %s " % EOS).split()
330872
public static void generateVectorFile(String raw, String lexiconF,
            File propertiesFile) throws FileNotFoundException, IOException {
        Properties props = new Properties();
        props.load(new FileInputStream(propertiesFile));

        String vector_location = props.getProperty("vector_location");
        String newLexicon = props.getProperty("new_lexicon_file");

        boolean compact = "true".equals(props
                .getProperty("compact.attribute.nums"));

        String format = props.getProperty("format");

        // load the lexicon and the raw file
        Lexicon lexicon = new Lexicon(lexiconF);

        String weightingScheme = props.getProperty(
                "classification_weight_scheme", "tfidf");
        WeightingMethod method = WeightingMethod
                .methodFromString(weightingScheme);
        lexicon.setMethod(method);

        // get the raw file
        FileTrainingCorpus ftc = new FileTrainingCorpus(new File(raw));

        int keepNBestAttributes = Integer.parseInt(props.getProperty(
                "keepNBestAttributes", "-1"));

        if (keepNBestAttributes != -1) {
            // double scores[] = logLikelihoodAttributeFilter.getScores(ftc,
            // lexicon);
            // lexicon.setLogLikelihoodRatio(scores);
            // lexicon.keepTopNAttributesLLR(keepNBestAttributes);
            AttributeScorer scorer = logLikelihoodAttributeScorer.getScorer(
                    ftc, lexicon);
            lexicon.setAttributeScorer(scorer);
            lexicon.applyAttributeFilter(scorer, keepNBestAttributes);
        } else {
            // apply the filters on the Lexicon
            int minFreq = Integer.parseInt(props
                    .getProperty("classification_minFreq"));
            int maxFreq = Integer.MAX_VALUE;

            lexicon.pruneTermsDocFreq(minFreq, maxFreq);
        }

        // change the indices of the attributes to remove
        // gaps between them
        Map<Integer, Integer> equiv = null;
        if (compact) {
            // create a new Lexicon object
            equiv = lexicon.compact();
        }

        // save the modified lexicon file
        if (newLexicon != null)
            lexicon.saveToFile(newLexicon);

        // dump a new vector file
        Utils.writeExamples(ftc, lexicon, true, vector_location, equiv, format);
    }
1092211
def _load_from_file_object(self, f):
    
    subtoken_strings = []
    for line in f:
      s = line.strip()
      # Some vocab files wrap words in single quotes, but others don't
      if ((s.startswith("'") and s.endswith("'")) or
          (s.startswith("\"") and s.endswith("\""))):
        s = s[1:-1]
      subtoken_strings.append(native_to_unicode(s))
    self._init_subtokens_from_list(subtoken_strings)
    self._init_alphabet_from_tokens(subtoken_strings)
594294
def load_stanford(cls, filename):
        

        dct = {}
        vectors = array.array('d')

        # Read in the data.
        with io.open(filename, 'r', encoding='utf-8') as savefile:
            for i, line in enumerate(savefile):
                tokens = line.split(' ')

                word = tokens[0]
                entries = tokens[1:]

                dct[word] = i
                vectors.extend(float(x) for x in entries)

        # Infer word vectors dimensions.
        no_components = len(entries)
        no_vectors = len(dct)

        # Set up the model instance.
        instance = Glove()
        instance.no_components = no_components
        instance.word_vectors = (np.array(vectors)
                                 .reshape(no_vectors,
                                          no_components))
        instance.word_biases = np.zeros(no_vectors)
        instance.add_dictionary(dct)

        return instance
485939
def load(self, text, fieldnames=None):
    """"""
    lines = text.split('\n')
    fieldnames = load_line(lines[0])
    values = load_line(lines[1])
    self.__dict__ = dict(zip(fieldnames, values))
941252
def load_file(self, filename):
        """"""

        self.counter.clear()
        # python2 issues
        try:
            etree = ET.parse(filename)
        except ET.ParseError:
            parser = ET.XMLParser(encoding='UTF-8')
            etree = ET.parse(filename, parser)
        eroot = etree.getroot()

        self.remove_all()
        self.previewer.remove_all()
        self.widget_editor.hide_all()

        self.previewer.resource_paths.append(os.path.dirname(filename))
        for element in eroot:
            self.populate_tree('', eroot, element,from_file=True)
        children = self.treeview.get_children('')
        for child in children:
            self.draw_widget(child)
        self.previewer.show_selected(None, None)
311593
private int loadSparseTextOffsets(RandomAccessBufferedReader textSSpace) 
            throws IOException {
        String line = textSSpace.readLine();
        if (line == null)
            throw new IOError(new Throwable(
                        "An empty file has been passed in"));

        // Strip off the 4-byte (2 char) header
        if (containsHeader) {
            line = line.substring(4);
            System.out.println(line);
        }        

        String[] dimensions = line.split("\\s");
        int columns = Integer.parseInt(dimensions[1]);
        int rows = Integer.parseInt(dimensions[0]);
        int row = 1;
        
        while ((line = textSSpace.readLine()) != null) {
            String[] termVectorPair = line.split("\\|");
            termToOffset.put(termVectorPair[0], Long.valueOf(row));
            row++;
        }
        if ((row - 1) != rows)
            throw new IOException(String.format(
                "Different number of rows than specified (%d): %d", rows, row));
        return columns;    
    }
1044368
def _load_text(handle, split=False, encoding="utf-8"):
    """"""
    string = handle.read().decode(encoding)
    return string.splitlines() if split else string
647935
def raw_vector_with(word_list)
      vec = if $GSL
              GSL::Vector.alloc(word_list.size)
            else
              Array.new(word_list.size, 0)
            end

      @word_hash.each_key do |word|
        vec[word_list[word]] = @word_hash[word] if word_list[word]
      end

      # Perform the scaling transform and force floating point arithmetic
      if $GSL
        sum = 0.0
        vec.each { |v| sum += v }
        total_words = sum
      else
        total_words = vec.reduce(0, :+).to_f
      end

      total_unique_words = 0

      if $GSL
        vec.each { |word| total_unique_words += 1 if word != 0.0 }
      else
        total_unique_words = vec.count { |word| word != 0 }
      end

      # Perform first-order association transform if this vector has more
      # then one word in it.
      if total_words > 1.0 && total_unique_words > 1
        weighted_total = 0.0
        # Cache calculations, this takes too long on large indexes
        cached_calcs = Hash.new do |hash, term|
          hash[term] = ((term / total_words) * Math.log(term / total_words))
        end

        vec.each do |term|
          weighted_total += cached_calcs[term] if term > 0.0
        end

        # Cache calculations, this takes too long on large indexes
        cached_calcs = Hash.new do |hash, val|
          hash[val] = Math.log(val + 1) / -weighted_total
        end

        vec.collect! do |val|
          cached_calcs[val]
        end
      end

      if $GSL
        @raw_norm   = vec.normalize
        @raw_vector = vec
      else
        @raw_norm   = Vector[*vec].normalize
        @raw_vector = Vector[*vec]
      end
    end