Traverse the given tokens.
@param {Token|Token[]} tokens tokens.
@param {object} options The option object.
@returns {void}
766766
function traverse(tokens, options) {
    const traverser = new Traverser()
    const nodes = Array.isArray(tokens) ? tokens : [tokens]
    for (const node of nodes) {
        traverser.traverse(node, options)
        if (traverser.isBroken()) {
            return
        }
    }
}
817734
function traverseOptions(options, currentIndex, distance) {
    let goToIndex = currentIndex;

    if (distance < 0 && currentIndex !== 0) {
        goToIndex = (currentIndex + options.length + distance) % options.length;
    } else if (distance > 0 && currentIndex !== (options.length - 1)) {
        goToIndex = (currentIndex + distance) % options.length;
    }

    return goToIndex;
}
780854
function Tokens (options) {
  if (!(this instanceof Tokens)) {
    return new Tokens(options)
  }

  var opts = options || {}

  var saltLength = opts.saltLength !== undefined
    ? opts.saltLength
    : 8

  if (typeof saltLength !== 'number' || !isFinite(saltLength) || saltLength < 1) {
    throw new TypeError('option saltLength must be finite number > 1')
  }

  var secretLength = opts.secretLength !== undefined
    ? opts.secretLength
    : 18

  if (typeof secretLength !== 'number' || !isFinite(secretLength) || secretLength < 1) {
    throw new TypeError('option secretLength must be finite number > 1')
  }

  this.saltLength = saltLength
  this.secretLength = secretLength
}
575219
function tokensToRegExp (tokens, options) {
	  options = options || {}
	
	  var strict = options.strict
	  var end = options.end !== false
	  var route = ''
	  var lastToken = tokens[tokens.length - 1]
	  var endsWithSlash = typeof lastToken === 'string' && /\/$/.test(lastToken)
	
	  // Iterate over the tokens and create our regexp string.
	  for (var i = 0; i < tokens.length; i++) {
	    var token = tokens[i]
	
	    if (typeof token === 'string') {
	      route += escapeString(token)
	    } else {
	      var prefix = escapeString(token.prefix)
	      var capture = '(?:' + token.pattern + ')'
	
	      if (token.repeat) {
	        capture += '(?:' + prefix + capture + ')*'
	      }
	
	      if (token.optional) {
	        if (!token.partial) {
	          capture = '(?:' + prefix + '(' + capture + '))?'
	        } else {
	          capture = prefix + '(' + capture + ')?'
	        }
	      } else {
	        capture = prefix + '(' + capture + ')'
	      }
	
	      route += capture
	    }
	  }
	
	  // In non-strict mode we allow a slash at the end of match. If the path to
	  // match already ends with a slash, we remove it for consistency. The slash
	  // is valid at the end of a path match, not in the middle. This is important
	  // in non-ending mode, where "/test/" shouldn't match "/test//route".
	  if (!strict) {
	    route = (endsWithSlash ? route.slice(0, -2) : route) + '(?:\\/(?=$))?'
	  }
	
	  if (end) {
	    route += '$'
	  } else {
	    // In non-ending mode, we need the capturing groups to match as much as
	    // possible by using a positive lookahead to the end or next path segment.
	    route += strict && endsWithSlash ? '' : '(?=\\/|$)'
	  }
	
	  return new RegExp('^' + route, flags(options))
	}
777523
function traverseRecursive(rootNode, options) {
  options = extend({
    subnodesAccessor: function(node) { return node.subnodes; },
    onNode: function(node, callback, userdata) {},
    onComplete: function(rootNode) {},
    userdata: null
  }, options);

  (function visitNode(node, callback) {
    var subnodes = options.subnodesAccessor(node);
    async.eachSeries(subnodes, function(subnode, next) {
      visitNode(subnode, function() {
        async.setImmediate(next);
      });
    },
    function() {
      options.onNode(node, function() {
        async.setImmediate(callback);
      }, options.userdata);
    });
  })(rootNode, function() {
    options.onComplete(rootNode);
  });
}
331385
function init (optionTokens = {}) {
  Object.keys(optionTokens).forEach((key) => assign(key, optionTokens[key]))
}
501564
function nestTokens(tokens) {
    var tree = [];
    var collector = tree;
    var sections = [];
    var token, section;

    for (var i = 0; i < tokens.length; ++i) {
      token = tokens[i];

      switch (token[0]) {
      case "#":
      case "^":
        token[4] = [];
        sections.push(token);
        collector.push(token);
        collector = token[4];
        break;
      case "/":
        if (sections.length === 0) {
          throw new Error("Unopened section: " + token[1]);
        }

        section = sections.pop();

        if (section[1] !== token[1]) {
          throw new Error("Unclosed section: " + section[1]);
        }

        if (sections.length > 0) {
          collector = sections[sections.length - 1][4];
        } else {
          collector = tree;
        }
        break;
      default:
        collector.push(token);
      }
    }

    // Make sure there were no open sections when we're done.
    section = sections.pop();

    if (section) {
      throw new Error("Unclosed section: " + section[1]);
    }

    return tree;
  }
606871
def call(self, tokens, *args, **kwargs):
        
        tokens.append([evaluate, [args, kwargs], {}])
        return tokens
494798
function(token) {
      if (Array.isArray(token)) {
        var tokens = /**@type {!Array.<number>}*/(token);
        while (tokens.length)
          this.tokens.unshift(tokens.shift());
      } else {
        this.tokens.unshift(token);
      }
    }
1364673
def parse_tokens(self, tokens):
        
        if len(tokens) == 1:
            return list(), tokens, {"kubectl": self.ast.help}
        else:
            tokens.reverse()
        parsed, unparsed, suggestions = self.treewalk(self.ast, parsed=list(), unparsed=tokens)
        if not suggestions and unparsed:
            # TODO: @vogxn: This is hack until we include expected value types for each option and argument.
            # Whenver we recieve no suggestions but are left with unparsed tokens we pop the value and walk the
            # tree again without values
            logger.debug("unparsed tokens remain, possible value encountered")
            unparsed.pop()
            parsed.reverse()
            unparsed.extend(parsed)
            logger.debug("resuming treewalk with tokens: %s", unparsed)
            return self.treewalk(self.ast, parsed=list(), unparsed=unparsed)
        else:
            return parsed, unparsed, suggestions
707353
function tokensToRegExp (tokens, keys, options) {
  options = options || {}

  var strict = options.strict
  var start = options.start !== false
  var end = options.end !== false
  var delimiter = options.delimiter || DEFAULT_DELIMITER
  var endsWith = [].concat(options.endsWith || []).map(escapeString).concat('$').join('|')
  var route = start ? '^' : ''

  // Iterate over the tokens and create our regexp string.
  for (var i = 0; i < tokens.length; i++) {
    var token = tokens[i]

    if (typeof token === 'string') {
      route += escapeString(token)
    } else {
      var capture = token.repeat
        ? '(?:' + token.pattern + ')(?:' + escapeString(token.delimiter) + '(?:' + token.pattern + '))*'
        : token.pattern

      if (keys) keys.push(token)

      if (token.optional) {
        if (!token.prefix) {
          route += '(' + capture + ')?'
        } else {
          route += '(?:' + escapeString(token.prefix) + '(' + capture + '))?'
        }
      } else {
        route += escapeString(token.prefix) + '(' + capture + ')'
      }
    }
  }

  if (end) {
    if (!strict) route += '(?:' + escapeString(delimiter) + ')?'

    route += endsWith === '$' ? '$' : '(?=' + endsWith + ')'
  } else {
    var endToken = tokens[tokens.length - 1]
    var isEndDelimited = typeof endToken === 'string'
      ? endToken[endToken.length - 1] === delimiter
      : endToken === undefined

    if (!strict) route += '(?:' + escapeString(delimiter) + '(?=' + endsWith + '))?'
    if (!isEndDelimited) route += '(?=' + escapeString(delimiter) + '|' + endsWith + ')'
  }

  return new RegExp(route, flags(options))
}
406609
def tokens
      options.collection.each do |option|
        if @findings.nil?
          @findings = option.tokenize(self)
        else
          @findings &= option.tokenize(self)
        end
      end

      Tokenizer.new(self).tokens
    end
1075280
function postProcessing(tokens, options, source){

  function tokenName(token){
    return (token.prefix ? token.prefix + ':' : '') + token.name;
  }

  function getTokenAttrs(token){
    return token.attrs.reduce(function(res, attr){
      res[tokenName(attr)] = attr.value;
      return res;
    }, {});
  }

  function buildLocationIndex(){
    var line = 1;
    var column = 0;

    lineIdx = new Array(source.length);
    columnIdx = new Array(source.length);

    for (var i = 0; i < source.length + 1; i++)
    {
      lineIdx[i] = line;
      columnIdx[i] = column;

      if (source[i] === '\n')
      {
        line++;
        column = 0;
      }
      else
        column++;
    }
  }


  function findLocationByOffset(offset){
    return {
      line: lineIdx[offset],
      column: columnIdx[offset]
    };
  }

  function getLocationFromRange(range){
    return {
      start: findLocationByOffset(range.start_),
      end: findLocationByOffset(range.end_)
    };
  }

  function convertRange(token){
    if (options.loc)
    {
      token.loc = getLocationFromRange(token.range);
      if (token.valueRange)
        token.valueLoc = getLocationFromRange(token.valueRange);
    }

    if (options.range)
    {
      token.range = [token.range.start_, token.range.end_];
      if (token.valueRange)
        token.valueRange = [token.valueRange.start_, token.valueRange.end_];
    }
    else
    {
      delete token.range;
      delete token.valueRange;
    }
  }

  function walk(tokens){
    var token;
    var prev;

    for (var i = 0; token = tokens[i++]; prev = token)
    {
      // process element
      if (token.type == TYPE_ELEMENT)
      {
        // process attribute content
        var attrs = getTokenAttrs(token);
        for (var j = 0, attr; attr = token.attrs[j++];)
        {
          var mode = attr.name;

          // special case modification instruction in <b:include>
          if (token.prefix == 'b' && attr.name == 'value')
          {
            // parse value attribute in
            //   <b:class>/<b:append-class>/<b:set-class>/<b:remove-class>
            //   <b:attr name="name">/<b:append-attr name="name">/<b:set-attr name="name">/<b:remove-attr name="name">
            var m = token.name.match(ATTRIBUTE_MODE);
            if (m)
              mode = m[1] == 'class' ? 'class' : attrs.name;
          }

          // process bindings and decode HTML tokens
          processAttr(attr, mode, convertRange);

          convertRange(attr);
        }

        // walk recursive
        walk(token.children);
      }

      // process text
      if (token.type == TYPE_TEXT)
      {
        // decode HTML tokens in value
        token.value = decodeHTMLTokens(token.value);

        // try join text tokens
        if (!token.refs && prev && prev.type == TYPE_TEXT && !prev.refs)
        {
          prev.value += token.value;
          prev.end_ = token.end_;
          tokens.splice(--i, 1);
        }
      }

      // process comment
      if (token.type == TYPE_COMMENT)
      {
        // decode HTML tokens in value
        token.value = decodeHTMLTokens(token.value);
      }

      convertRange(token);
    }
  }

  var lineIdx;
  var columnIdx;

  if (options.loc)
    buildLocationIndex();

  walk(tokens);
}
936567
public function tokens($text, $options = [])
    {
        self::getClient()->indices()->analyze(array_merge(['index' => $this->getName(), 'text' => $text], $options));
    }
603419
function Lexer (options) {
  this.tokens = []
  this.options = options || defaults
  this.rules = block
  if (this.options.gfm) {
    Object.assign(this.rules, block.opts.gfm)
  }
  if (this.options.include) {
    Object.assign(this.rules, block.opts.include)
  }
  if (this.options.numberedheadings) {
    Object.assign(this.rules, block.opts.numberedheadings)
  }
  if (this.options.toc) {
    Object.assign(this.rules, block.opts.toc)
  }
  if (this.options.ref) {
    Object.assign(this.rules, block.opts.ref)
  }
}
258935
function (options) {
    this.queue = [];
    this.ownerTokenId = null;
    this.options = _.extend({}, AsyncLock.defaultOptions, options);
}
1240562
function Templater(options) {
  if (options == null) {
    options = { allowEval: true };
  }
  this.options = options;
  if (options.stack && Array.isArray(options.stack)) {
    this.stack = options.stack;
  }
  else if (typeof options.stack === 'string') {
    this.stack = [ options.stack ];
  }
  else {
    this.stack = [];
  }
  this.nodes = [];
}