Forms the given array of `tokens` into a nested tree structure where
tokens that represent a section have a fifth item: an array that contains
all tokens in that section.
501564
function nestTokens(tokens) {
    var tree = [];
    var collector = tree;
    var sections = [];
    var token, section;

    for (var i = 0; i < tokens.length; ++i) {
      token = tokens[i];

      switch (token[0]) {
      case "#":
      case "^":
        token[4] = [];
        sections.push(token);
        collector.push(token);
        collector = token[4];
        break;
      case "/":
        if (sections.length === 0) {
          throw new Error("Unopened section: " + token[1]);
        }

        section = sections.pop();

        if (section[1] !== token[1]) {
          throw new Error("Unclosed section: " + section[1]);
        }

        if (sections.length > 0) {
          collector = sections[sections.length - 1][4];
        } else {
          collector = tree;
        }
        break;
      default:
        collector.push(token);
      }
    }

    // Make sure there were no open sections when we're done.
    section = sections.pop();

    if (section) {
      throw new Error("Unclosed section: " + section[1]);
    }

    return tree;
  }
1008753
function addNode (node, nest) {
    var obj = getHeadingObject(node)
    var level = getHeadingLevel(node)
    var array = nest
    var lastItem = getLastItem(array)
    var lastItemLevel = lastItem
      ? lastItem.headingLevel
      : 0
    var counter = level - lastItemLevel

    while (counter > 0) {
      lastItem = getLastItem(array)
      if (lastItem && lastItem.children !== undefined) {
        array = lastItem.children
      }
      counter--
    }

    if (level >= options.collapseDepth) {
      obj.isCollapsed = true
    }

    array.push(obj)
    return array
  }
643662
public static String[] convertListTokenToArrayStrings(
      final List<Token> tokenizedSentence) {
    final List<String> tokensList = new ArrayList<>();
    for (final Token token : tokenizedSentence) {
      tokensList.add(token.getTokenValue());
    }
    return tokensList.toArray(new String[tokensList.size()]);
  }
746980
public function process($tokens, $keys = array()) {

        $base_expr = "";
        $expr = array();
        $currCategory = "";

        if ($this->isStatement($keys)) {
            foreach ($tokens as $token) {

                $trim = trim($token);
                $base_expr .= $token;

                if ($trim === '') {
                    continue;
                }

                $upper = strtoupper($trim);

                switch ($upper) {

                case 'EXTENDED':
                case 'PARTITIONS':
                    return array('expr_type' => ExpressionType::RESERVED, 'base_expr' => $token);
                    break;

                case 'FORMAT':
                    if ($currCategory === '') {
                        $currCategory = $upper;
                        $expr[] = array('expr_type' => ExpressionType::RESERVED, 'base_expr' => $trim);
                    }
                    // else?
                    break;

                case '=':
                    if ($currCategory === 'FORMAT') {
                        $expr[] = array('expr_type' => ExpressionType::OPERATOR, 'base_expr' => $trim);
                    }
                    // else?
                    break;

                case 'TRADITIONAL':
                case 'JSON':
                    if ($currCategory === 'FORMAT') {
                        $expr[] = array('expr_type' => ExpressionType::RESERVED, 'base_expr' => $trim);
                        return array('expr_type' => ExpressionType::EXPRESSION, 'base_expr' => trim($base_expr),
                                     'sub_tree' => $expr);
                    }
                    // else?
                    break;

                default:
                // ignore the other stuff
                    break;
                }
            }
            return empty($expr) ? null : $expr;
        }

        foreach ($tokens as $token) {

            $trim = trim($token);

            if ($trim === '') {
                continue;
            }

            switch ($currCategory) {

            case 'TABLENAME':
                $currCategory = 'WILD';
                $expr[] = array('expr_type' => ExpressionType::COLREF, 'base_expr' => $trim,
                                'no_quotes' => $this->revokeQuotation($trim));
                break;

            case '':
                $currCategory = 'TABLENAME';
                $expr[] = array('expr_type' => ExpressionType::TABLE, 'table' => $trim,
                                'no_quotes' => $this->revokeQuotation($trim), 'alias' => false, 'base_expr' => $trim);
                break;

            default:
                break;
            }
        }
        return empty($expr) ? null : $expr;
    }
487702
function tokensId(tokens, prepend) {
    if (!prepend) return null;

    return _.reduce(tokens, function(prev, token) {
        var tokId = [
            token.index,
            token.offset,
            token.value
        ].join('-');

        return [prev, tokId].join(':');
    }, prepend);
}
23954
function prepareContents(tokens) {
	let contents = new Map

	for (let i = 0, len = tokens.length; i < len; i++) {
		let token = tokens[i]

		if (token.type === 'element' && token.name === 'eh-content') {
			// Find attribute 'name'
			let name = getNameAttributeValue(token),
				arr = getArr(name)

			for (let j = 0, len2 = token.children.length; j < len2; j++) {
				arr.push(token.children[j])
			}
		} else {
			getArr('').push(token)
		}
	}

	/**
	 * @param {string} name
	 * @returns {Array<Token>}
	 */
	function getArr(name) {
		if (!contents.has(name)) {
			let arr = []
			contents.set(name, arr)
			return arr
		}
		return contents.get(name)
	}

	return contents
}
1212675
protected function createSegments(array $tokens) : array
    {
        // Stack of opened tags.
        $active = new TagStack();

        return array_reduce($tokens, function (array $segments, $token) use ($active) {
            if ($token instanceof TagToken) {
                /** @var TagToken $token */
                if ($token->isOpening()) {
                    // If tag token is opening, add tag to the stack.
                    $active->push(Tag::createFromToken($token));
                } else {
                    // If tag token is closing, find matching tag in the stack.
                    $tagName = $token->getTagName();
                    $isFound = $active->removeLast($tagName);
                    if (!$isFound) {
                        // If not found, return tag token as a text.
                        $text = $token->getRawText();
                        $tags = $active->asArray();
                        $segments[] = new TextSegment($text, $tags);
                    }
                }
            } else {
                // Return text segment with currently active tags.
                /** @var TextToken $token */
                $text = $token->getRawText();
                $tags = $active->asArray();
                $segments[] = new TextSegment($text, $tags);
            }

            return $segments;
        }, []);
    }
947853
protected function recurseArrayForTokens($arr, $tokens)
    {
        $out = [];
        foreach ($arr as $name => $value) {
            if (is_array($value)) {
                $out[$name] = $this->recurseArrayForTokens($value, $tokens);
                continue;
            }
            if (is_string($value)) {
                foreach ($tokens as $token => $tval) {
                    $value = str_replace($token, $tval, $value);
                }
                $out[$name] = $value;
                continue;
            }
            $out[$name] = $value;
        }

        return $out;
    }
483678
function getAllChildSpans(node) {
  return (
    Array.isArray(node.children)
      ? node.children
          .map(n => (n.spans || []).concat(getAllChildSpans(n)))
          .reduce((all, arr) => all.concat(arr))
      : []
  );
}
595001
@Override
  public MtasTokenCollection createTokenCollection(Reader reader)
      throws MtasParserException, MtasConfigException {
    tokenCollection = new MtasTokenCollection();
    MtasTokenIdFactory mtasTokenIdFactory = new MtasTokenIdFactory();
    List<Level> levels = new ArrayList<>();
//    Map<String,MtasToken> referencesNode = new HashMap<>();
//    Map<String,List<MtasToken>> referencesNullElement = new HashMap<>();
    try {
      MtasPennTreebankReader treebankReader = new MtasPennTreebankReader(
          reader);
      // variables main administration
      int event = treebankReader.getEventType();
      int position = 0;
      boolean ignore = false;
      Level level = null;
      // variables for code
      List<Integer> codePositions = new ArrayList<>();
      Integer codeOffsetStart = null;
      Integer codeOffsetEnd = null;
      // variables for string
      String stringValue = null;
      int stringOffsetStart;
      int stringOffsetEnd;
      // loop
      while (true) {
        switch (event) {
        case MtasPennTreebankReader.EVENT_STARTBRACKET:
          if (level != null && level.code) {
            throw new MtasParserException(
                "unexpected start bracket for " + NODE_CODE);
          } else {
            level = new Level();
            level.ignore = ignore;
            level.realOffsetStart = treebankReader.getPosition() + 1;
            levels.add(level);
          }
          break;
        case MtasPennTreebankReader.EVENT_ENDBRACKET:
          Objects.requireNonNull(level, "no level while ending bracket");
          level.realOffsetEnd = treebankReader.getPosition() - 1;
          Level parentLevel = levels.size() > 1 ? levels.get(levels.size() - 2)
              : null;
          createNodeMappings(mtasTokenIdFactory, level, parentLevel);
          // remove level
          if (parentLevel != null) {
            if (level.positionStart != null && level.positionEnd != null) {
              parentLevel.addPositionRange(level.positionStart,
                  level.positionEnd);
            }
            parentLevel.offsetStart = parentLevel.offsetStart == null
                ? level.offsetStart : parentLevel.offsetStart;
            parentLevel.offsetEnd = level.offsetEnd == null
                ? parentLevel.offsetEnd : level.offsetEnd;
            levels.remove(levels.size() - 1);
            level = parentLevel;
            ignore = level.ignore;
          } else {
            levels.clear();
            level = null;
            ignore = false;
//            referencesNode.clear();
//            referencesNullElement.clear();
          }
          break;
        case MtasPennTreebankReader.EVENT_NODE:
          Objects.requireNonNull(level, "no level while handling node");
          // register node with level
          level.node = treebankReader.getString();
          if (ignoreNodes.contains(level.node)) {
            ignore = true;
            level.ignore = true;
          }
          if (level.node.equals(NODE_CODE)) {
            level.code = true;
            if (!treebankReader.next() || (event = treebankReader
                .getEventType()) != MtasPennTreebankReader.EVENT_STRING) {
              throw new MtasParserException("expected string for " + NODE_CODE);
            } else if (!level.ignore) {
              stringValue = treebankReader.getString();
              stringOffsetStart = treebankReader.getPosition();
              stringOffsetEnd = stringOffsetStart + stringValue.length();              
              if (!codePositions.isEmpty()) {
                createCodeMappings(mtasTokenIdFactory, level, stringValue,
                    codeOffsetStart, codeOffsetEnd, stringOffsetStart,
                    stringOffsetEnd, codePositions);
              } else {
                log.error("CODE without codePositions for "+stringValue);                
              }
              codePositions.clear();
              codeOffsetStart = null;
              codeOffsetEnd = null;
            }
          }
          break;
        case MtasPennTreebankReader.EVENT_STRING:
          Objects.requireNonNull(level, "no level while handling string");
          if (level.code) {
            throw new MtasParserException("unexpected string for " + NODE_CODE);
          } else if (!level.ignore) {
            stringValue = treebankReader.getString();
            stringOffsetStart = treebankReader.getPosition();
            stringOffsetEnd = stringOffsetStart + stringValue.length();
            if(level.offsetStart == null) {
              level.offsetStart = stringOffsetStart;
            }
            level.offsetEnd = stringOffsetEnd;
            if (stringValue.startsWith(NODE_CODE_PREFIX)) {
              codePositions.add(position);
              stringValue = stringValue.substring(NODE_CODE_PREFIX.length(),
                  stringValue.length());
              if(codeOffsetStart == null) {
                codeOffsetStart = stringOffsetStart;
              }
              codeOffsetEnd = stringOffsetEnd;
            }
            // register position
            level.addPosition(position);
            // create mappings
            createStringMappings(mtasTokenIdFactory, level, stringValue,
                stringOffsetStart, stringOffsetEnd, position);
            // increase position
            position++;
          }
          break;
        default:
          break;
        }
        if (!treebankReader.next()) {
          break;
        } else {
          event = treebankReader.getEventType();
        }
      }
    } catch (IOException e) {
      log.debug(e);
      throw new MtasParserException(
          "No valid Penn Treebank syntax: " + e.getMessage());
    }
    // final check
    tokenCollection.check(autorepair, makeunique);
    return tokenCollection;

  }
170995
public int[] buildSuffixArray(T[] tokens) {
        final int length = tokens.length;
        /*
         * Allocate slightly more space, some suffix construction strategies need it and
         * we don't want to waste space for multiple symbol mappings.
         */
        input = new int[length + SuffixArrays.MAX_EXTRA_TRAILING_SPACE];

        //System.out.println("Assigning token ids ...");

        /*
         * We associate every token to an id, all `equal´ tokens to the same id.
         * The suffix array is built using only the the ids.
         */
        tokIDs = new TreeMap<>(comparator);

        for (int i = 0; i < length; i++) {
            tokIDs.putIfAbsent(tokens[i], i);
            input[i] = tokIDs.get(tokens[i]);
        }

        //System.out.println("Token ids assigned.");

        return delegate.buildSuffixArray(input, 0, length);
    }
79654
function parse(tokens) {
    var ast = [];

    var current = ast;

    var _line = 0;
    var _char = 0;

    var openCount = 0;
    var closeCount = 0;

    for (var i = 0; i < tokens.length; i++) {
        var token = tokens[i];

        switch (token.type) {
            case TYPES.open:
                // Every time we open a list, we drop into a sub-array
                var child = [];
                child.parent = current;
                current.push(child);
                current = child;
                openCount += 1;
                break;

            case TYPES.open_array:
            case TYPES.quote:
                var child = [];
                var quote = {"'": child };
                child.parent = current;
                current.push(quote);
                current = child;
                openCount += 1;
                break;

            case TYPES.close_array:
            case TYPES.close:
                // If no current, we probably have too many closing parens
                if (!current) _tooManyClosingParensErr(closeCount, openCount);
                // If we close a list, jump back up to the parent list
                current = current.parent;
                closeCount += 1;
                break;

            case TYPES.identifier:
            case TYPES.suffixed_number:
                current.push(token.string);
                break;

            case TYPES.number:
                current.push(Number(token.string));
                break;

            case TYPES.string:
                // We need to strip the quotes off the string entity
                var dequoted = token.string.slice(1).slice(0, token.string.length - 2);
                current.push(dequoted);
                break;

            case TYPES.json:
                try {
                    var deticked = token.string.slice(1).slice(0, token.string.length - 2);
                    current.push(JSON.parse(deticked));
                }
                catch (e) {
                    throw new Error([
                        'Runiq: Couldn\'t parse inlined JSON!',
                        '--- Error occurred at line ' + _line + ', char ' + _char
                    ].join('\n'));
                }
                break;
        }

        // Capture line numbers and columns for future use
        var lines = token.string.split(NEWLINE);
        _line += lines.length - 1;
        if (lines.length > 1) _char = lines[lines.length - 1].length;
        else _char += token.string.length;
    }

    // Raise error if we have a parentheses mismatch
    if (openCount > closeCount) _tooManyOpenParensErr(closeCount, openCount);
    if (openCount < closeCount) _tooManyClosingParensErr(closeCount, openCount);

    // For both safety and to ensure we actually have a JSON-able AST
    return JSON.parse(JSONStableStringify(ast));
}
1175153
function groupNestedPredicates(_nestedPredicates) {
    let maxDepth = 0;
    const nestedPredicateObj = {};
    const nestedPredicateParts = _.map(_nestedPredicates, (predicateArr) => {
      const predicate = predicateArr[0];
      const retVal = predicate.split('.');
      nestedPredicateObj[predicate] = predicateArr[1];
      retVal.length > maxDepth && (maxDepth = retVal.length);
      return retVal;
    });
    const groups = {};

    for (let i = 0; i < maxDepth; i++) {
      groups[i] = _.groupBy(nestedPredicateParts, (predicateParts) => {
        let retval = '';
        for (let j = 0; j < i + 1; j++) {
          retval += (predicateParts[j] ? `${predicateParts[j]}.` : '');
        }
        return retval.substr(0, retval.length - 1);
      });
    }

    const completed = {};
    const levels = {};
    const paths = {};

    // Simplifies the grouping
    for (let i = maxDepth - 1; i >= 0; i--) {
      _.each(groups[i], (values, key) => {
        _.each(values, (value) => {
          const strKey = value.join('.');
          if (!completed[strKey] && values.length > 1) {
            (!levels[i] && (levels[i] = []));
            levels[i].push(strKey);
            (completed[strKey] = true);
            paths[i] = key;
          }
          if (!completed[strKey] && i < 1) {
            (!levels[i] && (levels[i] = []));
            levels[i].push(strKey);
            (completed[strKey] = true);
            paths[i] = key;
          }
        });
      });
    }

    return { groups: levels, paths, nestedPredicateObj };
  }
232861
def _MakeGroupFromRootSection(root_section, undefined_str):
    
    group = {}
    for statement in root_section.Statements():
        if isinstance(statement, six.string_types):
            continue
        func, args = statement
        # here the function acts as ID for the block type
        if func is _DoDef and isinstance(args, _Section):
            section = args
            # Construct a Template instance from a this _Section subtree
            t = Template._FromSection(section, group, undefined_str)
            group[section.section_name] = t
    return group
766766
function traverse(tokens, options) {
    const traverser = new Traverser()
    const nodes = Array.isArray(tokens) ? tokens : [tokens]
    for (const node of nodes) {
        traverser.traverse(node, options)
        if (traverser.isBroken()) {
            return
        }
    }
}
599571
function isTaskListItemToken(tokens, index) {
    return tokens[index].type === 'inline'
        && tokens[index - 1].type === 'paragraph_open'
        && tokens[index - 2].type === 'list_item_open'
        && (tokens[index].content.indexOf('[ ]') === 0
            || tokens[index].content.indexOf('[x]') === 0
            || tokens[index].content.indexOf('[X]') === 0);
}
23053
private function buildTree(string $originalValue, array $tokens): array
    {
        $tree = [];
        $functions = [];

        foreach ($tokens as $key => $value) {
            if ($this->tokenizer->isOpeningToken($value)) {
                $functions[$key] = null;    // The value here is ignored

                continue;
            }

            if ($this->tokenizer->isClosingToken($value)) {
                if (false === $this->tokenizer->isTheLastFunction($functions)) {
                    end($functions);
                    $lastFunctionKey = key($functions);
                    if (null === $lastFunctionKey) {
                        throw ExpressionLanguageExceptionFactory::createForMalformedFunction($originalValue);
                    }

                    unset($functions[$lastFunctionKey]);

                    continue;
                }

                end($functions);
                $lastFunctionKey = key($functions);
                $this->append($tree, $tokens, $lastFunctionKey, $key);
                unset($functions[$lastFunctionKey]);

                continue;
            }

            if ($this->tokenizer->functionIsNotClosed($functions)) {
                continue;
            }

            $tree[] = $value;
        }

        if ([] !== $functions) {
            throw ExpressionLanguageExceptionFactory::createForMalformedFunction($originalValue);
        }

        return $tree;
    }