Create tokens for setting is_noy_spec on describes
763714
def make_describe_attrs(self):
        """"""
        lst = []
        if self.all_groups:
            lst.append((NEWLINE, '\n'))
            lst.append((INDENT, ''))

            for group in self.all_groups:
                if group.name:
                    lst.extend(self.tokens.make_describe_attr(group.kls_name))

        return lst
71142
function remarkSwagger (spec, opts) {
  opts = opts || {}

  assert.equal(typeof opts, 'object', 'opts is an object')

  const title = opts.title || 'API'
  const position = opts.position || 4

  if (opts.yaml) spec = yaml.parse(spec)
  if (opts.json) spec = JSON.parse(spec)

  assert.equal(typeof spec, 'object', 'spec is object')

  return function attacher () {
    return function transformer (ast, file) {
      const table = createTable(spec)
      const heading = u('heading', { depth: 2 }, [
        u('text', { value: title })
      ])

      const head = ast.children[position]
      if (head.type === 'header' && head.children[0].value === title) {
        // inject code into existing node
        ast.children[position + 1] = table
      } else {
        // inject code into new node
        ast.children.splice(position, 0, table)
        ast.children.splice(position, 0, heading)
      }
    }
  }
}
851866
def _proc_token(spec, mlines):
    """"""
    spec = spec.strip().replace(" ", "")
    regexp = re.compile(r".*[^0123456789\-,]+.*")
    tokens = spec.split(",")
    cond = any([not item for item in tokens])
    if ("--" in spec) or ("-," in spec) or (",-" in spec) or cond or regexp.match(spec):
        raise RuntimeError("Argument `lrange` is not valid")
    lines = []
    for token in tokens:
        if token.count("-") > 1:
            raise RuntimeError("Argument `lrange` is not valid")
        if "-" in token:
            subtokens = token.split("-")
            lmin, lmax = (
                int(subtokens[0]),
                int(subtokens[1]) if subtokens[1] else mlines,
            )
            for num in range(lmin, lmax + 1):
                lines.append(num)
        else:
            lines.append(int(token))
    if lines != sorted(lines):
        raise RuntimeError("Argument `lrange` is not valid")
    return lines
621045
def template2features(sent, i, token_syntax, debug=True):
    
    columns = []
    for j in range(len(sent[0])):
        columns.append([t[j] for t in sent])
    matched = re.match(
        "T\[(?P<index1>\-?\d+)(\,(?P<index2>\-?\d+))?\](\[(?P<column>.*)\])?(\.(?P<function>.*))?",
        token_syntax)
    column = matched.group("column")
    column = int(column) if column else 0
    index1 = int(matched.group("index1"))
    index2 = matched.group("index2")
    index2 = int(index2) if index2 else None
    func = matched.group("function")
    if debug:
        prefix = "%s=" % token_syntax
    else:
        prefix = ""
    if i + index1 < 0:
        return ["%sBOS" % prefix]
    if i + index1 >= len(sent):
        return ["%sEOS" % prefix]
    if index2 is not None:
        if i + index2 >= len(sent):
            return ["%sEOS" % prefix]
        word = " ".join(columns[column][i + index1: i + index2 + 1])
    else:
        word = sent[i + index1][column]
    if func is not None:
        result = apply_function(func, word)
    else:
        result = word
    return ["%s%s" % (prefix, result)]
595373
function prepareInternalJSCode(source, options) {
	// Parse
	let tokens = parse(source)

	// Transform
	if (options.transformer) {
		tokens = options.transformer(tokens) || tokens
	}

	let reducedTokens = reduce(tokens, options)

	return createCode(reducedTokens, options, false)
}
696268
def markup_code(tokens)
            src = ""
            tokens.each do |t|
                next unless t
                #    p t.class
                #        style = STYLE_MAP[t.class]
                style = case t
                when RubyToken::TkCONSTANT then "ruby-constant"
                when RubyToken::TkKW       then "ruby-keyword kw"
                when RubyToken::TkIVAR     then "ruby-ivar"
                when RubyToken::TkOp       then "ruby-operator"
                when RubyToken::TkId       then "ruby-identifier"
                when RubyToken::TkNode     then "ruby-node"
                when RubyToken::TkCOMMENT  then "ruby-comment cmt"
                when RubyToken::TkREGEXP   then "ruby-regexp re"
                when RubyToken::TkSTRING   then "ruby-value str"
                when RubyToken::TkVal      then "ruby-value"
                else
                    nil
                end

                text = CGI.escapeHTML(t.text)

                if style
                    src << "<span class=\"#{style}\">#{text}</span>"
                else
                    src << text
                end
            end

            add_line_numbers(src)
            src
        end
1056923
def process_token(self, tok):
        """"""
        if(tok[0].__str__() in ('Token.Comment.Multiline', 'Token.Comment',
                'Token.Literal.String.Doc')):
            self.comments += tok[1].count('\n')+1
        elif(tok[0].__str__() in ('Token.Comment.Single')):
            self.comments += 1
        elif(self.contains_code and tok[0].__str__().startswith('Token.Text')
                and tok[1].count(u'\n')):
            # start new line
            self.contains_code = False
            self.sloc += 1
        # for c style includes
        elif(tok[0].__str__() == 'Token.Comment.Preproc' and
                tok[1].count(u'\n')):
            # start new line
            self.contains_code = False
            self.sloc += 1
        elif(tok[0][0] in token_types):
            self.contains_code = True
533623
def build_specfile_sections(spec):
    
    str = ""

    mandatory_sections = {
        'DESCRIPTION'  : '\n%%description\n%s\n\n', }

    str = str + SimpleTagCompiler(mandatory_sections).compile( spec )

    optional_sections = {
        'DESCRIPTION_'        : '%%description -l %s\n%s\n\n',
        'CHANGELOG'           : '%%changelog\n%s\n\n',
        'X_RPM_PREINSTALL'    : '%%pre\n%s\n\n',
        'X_RPM_POSTINSTALL'   : '%%post\n%s\n\n',
        'X_RPM_PREUNINSTALL'  : '%%preun\n%s\n\n',
        'X_RPM_POSTUNINSTALL' : '%%postun\n%s\n\n',
        'X_RPM_VERIFY'        : '%%verify\n%s\n\n',

        # These are for internal use but could possibly be overridden
        'X_RPM_PREP'          : '%%prep\n%s\n\n',
        'X_RPM_BUILD'         : '%%build\n%s\n\n',
        'X_RPM_INSTALL'       : '%%install\n%s\n\n',
        'X_RPM_CLEAN'         : '%%clean\n%s\n\n',
        }

    # Default prep, build, install and clean rules
    # TODO: optimize those build steps, to not compile the project a second time
    if 'X_RPM_PREP' not in spec:
        spec['X_RPM_PREP'] = '[ -n "$RPM_BUILD_ROOT" -a "$RPM_BUILD_ROOT" != / ] && rm -rf "$RPM_BUILD_ROOT"' + '\n%setup -q'

    if 'X_RPM_BUILD' not in spec:
        spec['X_RPM_BUILD'] = '[ ! -e "$RPM_BUILD_ROOT" -a "$RPM_BUILD_ROOT" != / ] && mkdir "$RPM_BUILD_ROOT"'

    if 'X_RPM_INSTALL' not in spec:
        spec['X_RPM_INSTALL'] = 'scons --install-sandbox="$RPM_BUILD_ROOT" "$RPM_BUILD_ROOT"'

    if 'X_RPM_CLEAN' not in spec:
        spec['X_RPM_CLEAN'] = '[ -n "$RPM_BUILD_ROOT" -a "$RPM_BUILD_ROOT" != / ] && rm -rf "$RPM_BUILD_ROOT"'

    str = str + SimpleTagCompiler(optional_sections, mandatory=0).compile( spec )

    return str
207893
def build_spec(user, repo, sha=None, prov=None, extraMetadata=[]):
    """"""
    loader = grlc.utils.getLoader(user, repo, sha=sha, prov=prov)

    files = loader.fetchFiles()
    raw_repo_uri = loader.getRawRepoUri()

    # Fetch all .rq files
    items = []

    allowed_ext = ["rq", "sparql", "json", "tpf"]
    for c in files:
        glogger.debug('>>>>>>>>>>>>>>>>>>>>>>>>>c_name: {}'.format(c['name']))
        extension = c['name'].split('.')[-1]
        if extension in allowed_ext:
            call_name = c['name'].split('.')[0]

            # Retrieve extra metadata from the query decorators
            query_text = loader.getTextFor(c)

            item = None
            if extension == "json":
                query_text = json.loads(query_text)

            if extension in ["rq", "sparql", "json"]:
                glogger.debug("===================================================================")
                glogger.debug("Processing SPARQL query: {}".format(c['name']))
                glogger.debug("===================================================================")
                item = process_sparql_query_text(query_text, loader, call_name, extraMetadata)
            elif "tpf" == extension:
                glogger.debug("===================================================================")
                glogger.debug("Processing TPF query: {}".format(c['name']))
                glogger.debug("===================================================================")
                item = process_tpf_query_text(query_text, raw_repo_uri, call_name, extraMetadata)
            else:
                glogger.info("Ignoring unsupported source call name: {}".format(c['name']))

            if item:
                items.append(item)
    return items
206764
def do_set(parser, token):
  ''''''
  code = token.contents
  firstspace = code.find(' ')
  if firstspace >= 0:
    code = code[firstspace+1:]
  return Setter(code)
1307669
def _expect_token(lexer, types):
    """"""
    tok = next(lexer)
    if any(isinstance(tok, t) for t in types):
        return tok
    else:
        raise Error("unexpected token: " + str(tok))
1109629
private boolean _validate(Token token, Term term) {

		if (token == null) {
			return true;
		}

		Set<String> terms = token.getTerms();
		if ((!terms.contains(Token.ALL)) && !terms.contains(term.getName()) && !(terms.contains(":" + term.getNatureStr()))) {
			return true;
		}

		boolean flag = token.getRegexs().size() != 0;

		for (String regex : token.getRegexs()) {
			if (term.getName().matches(regex)) {
				flag = false;
				break;
			}
		}

		return flag;
	}
170129
def custom_token(spacy_token) -> Token:
        

        """Add custom attributesAdd full_shape attribute. Eg. 21.33 => dd.dd, esadDeweD23 => xxxxXxxxXdd"""

        def get_shape(token):
            full_shape = ""
            for i in token.text:
                if i.isdigit():
                    full_shape += "d"
                elif i.islower():
                    full_shape += "x"
                elif i.isupper():
                    full_shape += "X"
                else:
                    full_shape += i
            return full_shape

        spacy_token.set_extension("full_shape", getter=get_shape, force=True)

        def is_integer(token):
            pattern = re.compile('^[-+]?[0-9]+$')
            return bool(pattern.match(token.text))

        spacy_token.set_extension("is_integer", getter=is_integer, force=True)

        def is_decimal(token):
            pattern = re.compile('^[-+]?[0-9]+\.[0-9]+$')
            return bool(pattern.match(token.text))

        spacy_token.set_extension("is_decimal", getter=is_decimal, force=True)

        def is_ordinal(token):
            return token.orth_[-2:] in ['rd', 'st', 'th', 'nd']

        spacy_token.set_extension("is_ordinal", getter=is_ordinal, force=True)

        def is_mixed(token):
            if not token.is_title and not token.is_lower and not token.is_upper:
                return True
            else:
                return False

        spacy_token.set_extension("is_mixed", getter=is_mixed, force=True)

        """Add custom methodsAdd get_prefix method. RETURN length N prefix"""

        def n_prefix(token, n):
            return token.text[:n]

        spacy_token.set_extension("n_prefix", method=n_prefix, force=True)

        """Add get_suffix method. RETURN length N suffix"""

        def n_suffix(token, n):
            return token.text[-n:]

        spacy_token.set_extension("n_suffix", method=n_suffix, force=True)

        return spacy_token
135790
def generate_tokens(self):
        """"""
        if self._io_error:
            self.report_error(1, 0, 'E902 %s' % self._io_error, readlines)
        tokengen = tokenize.generate_tokens(self.readline)
        try:
            for token in tokengen:
                if token[2][0] > self.total_lines:
                    return
                self.noqa = token[4] and noqa(token[4])
                self.maybe_check_physical(token)
                yield token
        except (SyntaxError, tokenize.TokenError):
            self.report_invalid_syntax()
203138
def _make_name(current, new):
    '''
    
    '''

    current_tokens = current.split()
    new_tokens = new.split()

    new = ' '.join(
        new_token for new_token in new_tokens
        if new_token not in current_tokens
    )

    return '/'.join((current, new))
499623
def preformat_plain_description(testcase):
    """"""
    description = testcase.get("description")

    if not description:
        return

    # naive approach to removing indent from pytest docstrings
    nodeid = testcase.get("nodeid") or ""
    indent = None
    if "::Test" in nodeid:
        indent = 8 * " "
    elif "::test_" in nodeid:
        indent = 4 * " "

    if indent:
        orig_lines = description.split("\n")
        new_lines = []
        for line in orig_lines:
            if line.startswith(indent):
                line = line.replace(indent, "", 1)
            new_lines.append(line)
        description = "\n".join(new_lines)

    testcase["description"] = "<pre>\n{}\n</pre>".format(description)
701177
function makeSignature(params) {
	var r = ['('], desc;
	if ( params ) {
		for (var i = 0, p; (p = params[i]); i++) {
			// ignore @param tags for 'virtual' params that are used to document members of config-like params
			// (e.g. like "@param param1.key ...")
			if (p.name && p.name.indexOf('.') == -1) {
				if (i > 0) {
					r.push(', ');
				}

				r.push('<span');

				var types = listTypes(p.type, true);
				if ( (desc = asPlainSummary(p.description)) || types ) {
					r.push(' title="');
					if (types) {
						r.push('(');
						r.push(types);
						r.push(') ');
					}
					r.push(desc);
					r.push('"');
				}

				r.push('>');
				r.push(p.name);
				r.push('</span>');
				if ( p.optional ) {
					r.push('<i class="help" title="Optional parameter">?</i>');
				}
			}
		}
	}
	r.push(')');
	return r.join('');
}