Add tokens for hanging singature if any
506649
def finish_hanging(self):
        """"""
        if self.groups.starting_signature:
            if self.groups.starting_group:
                self.add_tokens_for_group(with_pass=True)

            elif self.groups.starting_single:
                self.add_tokens_for_single(ignore=True)
802498
def StringFinish(self, **_):
    """"""
    if self.state == "ARG":
      return self.InsertArg(string=self.string)
476641
def __add_shared(self, original_token):
        """"""
        sid = self.__new_sid()
        token = SymbolToken(original_token.text, sid, self.__import_location(sid))
        self.__add(token)
        return token
1290863
protected void doSplitTokenImpl(Token token, ITokenAcceptor result) {
		String text = token.getText();
		int indentation = computeIndentation(text);
		if (indentation == -1 || indentation == currentIndentation) {
			// no change of indentation level detected simply process the token
			result.accept(token);
		} else if (indentation > currentIndentation) {
			// indentation level increased
			splitIntoBeginToken(token, indentation, result);
		} else if (indentation < currentIndentation) {
			// indentation level decreased
			int charCount = computeIndentationRelevantCharCount(text);
			if (charCount > 0) {
				// emit whitespace including newline
				splitWithText(token, text.substring(0, charCount), result);	
			}
			// emit end tokens at the beginning of the line
			decreaseIndentation(indentation, result);
			if (charCount != text.length()) {
				handleRemainingText(token, text.substring(charCount), indentation, result);
			}
		} else {
			throw new IllegalStateException(String.valueOf(indentation));
		}
	}
743511
def _process_token(cls, token):
        """"""
        assert type(token) is _TokenType or callable(token), \
            'token type must be simple type or callable, not %r' % (token,)
        return token
1184397
public ArgumentListBuilder addTokenized(String s) {
        if(s==null) return this;
        add(Util.tokenize(s));
        return this;
    }
96962
function isLastTokenInCurrentLine(token) {
            const tokenAfter = sourceCode.getTokenAfter(token);

            return !(tokenAfter && astUtils.isTokenOnSameLine(token, tokenAfter));
        }
828351
function setOffsetToToken(token, offset, baseline) {
            if (Array.isArray(token)) {
                for (const t of token) {
                    setOffsetToToken(t, offset, baseline)
                }
                return
            }
            setOffsetToLoc(token.loc.start, offset, baseline)
        }
527949
def _add_offsets_to_token_nodes(self):
        
        for edge_index in self._textual_relation_ids:
            token_node_index = self.edges[edge_index].source
            self.nodes[token_node_index].onset = self.edges[edge_index].onset
            self.nodes[token_node_index].offset = self.edges[edge_index].offset
165660
public static void putToken(Bundle bundle, String value) {
        Validate.notNull(bundle, "bundle");
        Validate.notNull(value, "value");
        bundle.putString(TOKEN_KEY, value);
    }
165948
function checkEscapedString(i) {
  const start = i;
  let l;

  if (i >= tokensLength) return 0;

  if (tokens[i].type === TokenType.Tilde && (l = checkString(i + 1)))
    return i + l - start;
  else return 0;
}
1159963
function toOptionToken(token) {
    if (!/^-/.test(token)) {
        token = token.split(' ')[0];
        if (!/^(\[|<)/.test(token))
            token = "[" + token + "]";
    }
    var pre = token;
    var suffix = '';
    var reqIdx = token.indexOf('<');
    var optIdx = token.indexOf('[');
    var idx = !!~reqIdx ? reqIdx : !!~optIdx ? optIdx : null;
    if (idx) {
        pre = token.slice(0, idx);
        suffix = token.slice(idx);
    }
    pre = chek_1.split(pre.trim(), constants_2.SPLIT_CHARS).join('.').replace(/\s/g, '');
    return pre + ' ' + suffix;
}
917252
function addParensIndent(tokens) {
            const parenStack = [];
            const parenPairs = [];

            tokens.forEach(nextToken => {

                // Accumulate a list of parenthesis pairs
                if (astUtils.isOpeningParenToken(nextToken)) {
                    parenStack.push(nextToken);
                } else if (astUtils.isClosingParenToken(nextToken)) {
                    parenPairs.unshift({ left: parenStack.pop(), right: nextToken });
                }
            });

            parenPairs.forEach(pair => {
                const leftParen = pair.left;
                const rightParen = pair.right;

                // We only want to handle parens around expressions, so exclude parentheses that are in function parameters and function call arguments.
                if (!parameterParens.has(leftParen) && !parameterParens.has(rightParen)) {
                    const parenthesizedTokens = new Set(sourceCode.getTokensBetween(leftParen, rightParen));

                    parenthesizedTokens.forEach(token => {
                        if (!parenthesizedTokens.has(offsets.getFirstDependency(token))) {
                            offsets.setDesiredOffset(token, leftParen, 1);
                        }
                    });
                }

                offsets.setDesiredOffset(rightParen, leftParen, 0);
            });
        }
465519
def push_token(self, tok):
        ""
        if self.debug >= 1:
            print("shlex: pushing token " + repr(tok))
        self.pushback.appendleft(tok)
834753
boolean addOutputToken(Token idToken, Token paraToken) {
		tokens.add(idToken);
		tokens.add(paraToken);
		previousTextToken = null;
		return prepareNextScan(0);
	}
688320
function(midiToks) {
          if (midiToks.length > 0) {
            var t = midiToks.shift();
            var p = t.token;
            if (t.type === "number")
              p = t.intt;
            return p;
          }
          else
            return null;
        }
213069
def _store_token(self, token, remember=False):
        """"""
        if token and remember:
            try:
                keyring.set_password('github', 'token', token)
            except Exception:
                if self._show_msgbox:
                    QMessageBox.warning(self.parent_widget,
                                        _('Failed to store token'),
                                        _('It was not possible to securely '
                                          'save your token. You will be '
                                          'prompted for your Github token '
                                          'next time you want to report '
                                          'an issue.'))
                remember = False
        CONF.set('main', 'report_error/remember_token', remember)