Start the lexical analysis.
340610
def analyse(string = nil)
      raise(ArgumentError, 'Lexer: failed to start analysis: no source given!') unless
        string || @scanner

      self.data = string || @scanner.string

      until @scanner.eos?
        send("parse_#{MODE[@mode]}")
      end

      push([false, '$end'])
    end
343520
def paragraph_starts(self):
        """"""
        if not self.is_tagged(PARAGRAPHS):
            self.tokenize_paragraphs()
        return self.starts(PARAGRAPHS)
286259
def analyze(self, text, index=None, analyzer=None, tokenizer=None, filters=None, field=None):
        
        if filters is None:
            filters = []
        argsets = 0
        args = {}

        if analyzer:
            args['analyzer'] = analyzer
            argsets += 1
        if tokenizer or filters:
            if tokenizer:
                args['tokenizer'] = tokenizer
            if filters:
                args['filters'] = ','.join(filters)
            argsets += 1
        if field:
            args['field'] = field
            argsets += 1

        if argsets > 1:
            raise ValueError('Argument conflict: Specify either analyzer, tokenizer/filters or field')

        if field and index is None:
            raise ValueError('field can only be specified with an index')

        path = make_path(index, '_analyze')
        return self.conn._send_request('POST', path, text, args)
135467
public function Start()
    {
        $this->definitions_path = rtrim($this->definitions_path, "/\\");

        $this->document = new DOMDocument();
        $this->document->load($this->definitions_path . "/index.xml");

        $this->ExtractConstants($this->document);
        $this->ExtractEnumerations($this->document);
        $this->ExtractVariables($this->document);
        $this->ExtractTypeDefinitions($this->document);
        $this->ExtractFunctions($this->document);
        $this->ExtractClasses($this->document);
    }
70521
def _pre_analysis(self):
        

        # Fill up self._starts
        for item in self._starts:
            callstack = None
            if isinstance(item, tuple):
                # (addr, jumpkind)
                ip = item[0]
                state = self._create_initial_state(item[0], item[1])

            elif isinstance(item, SimState):
                # SimState
                state = item.copy()  # pylint: disable=no-member
                ip = state.solver.eval_one(state.ip)
                self._reset_state_mode(state, 'fastpath')

            else:
                raise AngrCFGError('Unsupported CFG start type: %s.' % str(type(item)))

            self._symbolic_function_initial_state[ip] = state
            path_wrapper = CFGJob(ip, state, self._context_sensitivity_level, None, None, call_stack=callstack)
            key = path_wrapper.block_id
            if key not in self._start_keys:
                self._start_keys.append(key)

            self._insert_job(path_wrapper)
            self._register_analysis_job(path_wrapper.func_addr, path_wrapper)
371135
private function loadFullDictionary(): void
    {
        $file = __DIR__ . '/../Dictionary/' . $this->language;
        if (!file_exists($file)) {
            $file = __DIR__ . '/../Dictionary/en_US';
        }

        self::$dictionary = array_filter(explode(PHP_EOL, file_get_contents($file)));
    }
782225
def run_analysis(args):
  """"""

  # Read the schema and input feature types
  schema_list = json.loads(
      file_io.read_file_to_string(args.schema_file))

  run_numerical_categorical_analysis(args, schema_list)

  # Also save a copy of the schema in the output folder.
  file_io.copy(args.schema_file,
               os.path.join(args.output_dir, SCHEMA_FILE),
               overwrite=True)
912212
def main(self):
        
        if not os.path.isfile(self.gdcs_report):
            logging.info('Starting {} analysis pipeline'.format(self.analysistype))
            # Run the analyses
            ShortKSippingMethods(self, self.cutoff)
            # Create the reports
            self.reporter()
        else:
            self.report_parse()
244951
private void read() throws IOException, GrammarException {
	try {
	    logger.debug("Read grammar file...");
	    logger.debug("Starting lexer...");
	    Lexer lexer = new RegExpLexer(uhuraGrammar);
	    TokenStream tokenStream = lexer.lex(SourceCode.read(reader,
		    new UnspecifiedSourceCodeLocation()));
	    logger.debug("Starting parser...");
	    parse(tokenStream);
	    logger.debug("done reading grammar file.");
	} catch (LexerException e) {
	    logger.error(e.getMessage(), e);
	    throw new IOException(e.getMessage(), e);
	} catch (ParserException e) {
	    logger.error(e.getMessage(), e);
	    throw new IOException(e.getMessage(), e);
	}
    }
492611
def start(self):
        """"""
        error, output = None, None
        try:
            time.sleep(0.1)
            output = self.method(*self.args, **self.kwargs)
        except Exception as err:
            logger.debug(str((self.method.__module__, self.method.__name__,
                              err)))
            error = str(err)
            error = error.replace('(', '')
            error = error.replace(')', '')
#            try:
#                error = err[0]
#            except Exception:
#                try:
#                    error = err.message
#                except Exception as err2:
#                    error = ''

        self.sig_finished.emit(self, output, str(error))
        self._is_finished = True
426064
def analyze(self, text, tokenizer=str.split):
        
        if not self.tagger:
            self.tagger = Tagger(self.model,
                                 preprocessor=self.p,
                                 tokenizer=tokenizer)

        return self.tagger.analyze(text)
168703
def setup_data_split(X, y, tokenizer, proc_data_dir, **kwargs):
    
    X_train, X_val, X_test, y_train, y_val, y_test = split_data(X, y)

    # only build vocabulary on training data
    tokenizer.build_vocab(X_train)

    process_save(X_train, y_train, tokenizer, path.join(
        proc_data_dir, 'train.bin'), train=True, **kwargs)
    process_save(X_val, y_val, tokenizer, path.join(
        proc_data_dir, 'val.bin'), **kwargs)
    process_save(X_test, y_test, tokenizer, path.join(
        proc_data_dir, 'test.bin'), **kwargs)
483625
def _start(self):
        """"""
        error = None
        output = None

        try:
            output = self.func(*self.args, **self.kwargs)
        except Exception as err:
            error = err

        if not self._is_finished:
            self.sig_finished.emit(self, output, error)
        self._is_finished = True
782796
def Main(url):
    '''
    
    '''
    # The object of Web-Scraping.
    web_scrape = WebScraping()
    # Execute Web-Scraping.
    document = web_scrape.scrape(url)
    # The object of NLP.
    nlp_base = NlpBase()
    # Set tokenizer. This is japanese tokenizer with MeCab.
    nlp_base.tokenizable_doc = MeCabTokenizer()

    sentence_list = nlp_base.listup_sentence(document)

    batch_size = 10
    if len(sentence_list) < batch_size:
        raise ValueError("The number of extracted sentences is insufficient.")

    all_token_list = []
    for i in range(len(sentence_list)):
        nlp_base.tokenize(sentence_list[i])
        all_token_list.extend(nlp_base.token)
        sentence_list[i] = nlp_base.token
    
    vectorlizable_sentence = LSTMRTRBM()
    vectorlizable_sentence.learn(
        sentence_list=sentence_list, 
        token_master_list=list(set(all_token_list)),
        hidden_neuron_count=1000,
        batch_size=batch_size,
        learning_rate=1e-03,
        seq_len=5
    )
    test_list = sentence_list[:batch_size]
    feature_points_arr = vectorlizable_sentence.vectorize(test_list)

    print("Feature points (Top 5 sentences):")
    print(feature_points_arr)
1182731
def cli():
    """"""
    args = docopt.docopt(__doc__, version=__VERSION__)
    secure = args['--secure']
    numberofwords = int(args['<numberofwords>'])

    dictpath = args['--dict']
    if dictpath is not None:
        dictfile = open(dictpath)
    else:
        dictfile = load_stream('words.txt')
    with dictfile:
        wordlist = read_wordlist(dictfile)

    words = generate_words(numberofwords, wordlist, secure=secure)
    print(' '.join(words))
86942
function start(stats, {f, format = 'human'} = {}) {
	try {
		const result = chunkalyse(stats);
		let output;

		switch (f || format) {
			case 'json':
				output = JSON.stringify(result, null, 2);
				break;
			case 'human':
			default:
				output = require('../lib/humanise')(result);
		}

		console.log(output);
	} catch (error) {
		console.log('I\'ve had trouble finding the chunks\n');
		throw error;
	}
}
1053314
def analyze(self, filename):
        """"""
        if self.dockwidget and not self.ismaximized:
            self.dockwidget.setVisible(True)
            self.dockwidget.setFocus()
            self.dockwidget.raise_()
        pythonpath = self.main.get_spyder_pythonpath()
        runconf = get_run_configuration(filename)
        wdir, args = None, []
        if runconf is not None:
            if runconf.wdir_enabled:
                wdir = runconf.wdir
            if runconf.args_enabled:
                args = runconf.args
        self.profiler.analyze(filename, wdir=wdir, args=args,
                              pythonpath=pythonpath)