Parse a single field from {@code tokenizer} and merge it into
{@code builder}.
3216
private static void mergeField(final Tokenizer tokenizer,
                                 final ExtensionRegistry extensionRegistry,
                                 final Message.Builder builder)
                                 throws ParseException {
    FieldDescriptor field;
    final Descriptor type = builder.getDescriptorForType();
    ExtensionRegistry.ExtensionInfo extension = null;

    if (tokenizer.tryConsume("[")) {
      // An extension.
      final StringBuilder name =
          new StringBuilder(tokenizer.consumeIdentifier());
      while (tokenizer.tryConsume(".")) {
        name.append('.');
        name.append(tokenizer.consumeIdentifier());
      }

      extension = extensionRegistry.findExtensionByName(name.toString());

      if (extension == null) {
        throw tokenizer.parseExceptionPreviousToken(
          "Extension \"" + name + "\" not found in the ExtensionRegistry.");
      } else if (extension.descriptor.getContainingType() != type) {
        throw tokenizer.parseExceptionPreviousToken(
          "Extension \"" + name + "\" does not extend message type \"" +
          type.getFullName() + "\".");
      }

      tokenizer.consume("]");

      field = extension.descriptor;
    } else {
      final String name = tokenizer.consumeIdentifier();
      field = type.findFieldByName(name);

      // Group names are expected to be capitalized as they appear in the
      // .proto file, which actually matches their type names, not their field
      // names.
      if (field == null) {
        // Explicitly specify US locale so that this code does not break when
        // executing in Turkey.
        final String lowerName = name.toLowerCase(Locale.US);
        field = type.findFieldByName(lowerName);
        // If the case-insensitive match worked but the field is NOT a group,
        if (field != null && field.getType() != FieldDescriptor.Type.GROUP) {
          field = null;
        }
      }
      // Again, special-case group names as described above.
      if (field != null && field.getType() == FieldDescriptor.Type.GROUP &&
          !field.getMessageType().getName().equals(name)) {
        field = null;
      }

      if (field == null) {
        throw tokenizer.parseExceptionPreviousToken(
          "Message type \"" + type.getFullName() +
          "\" has no field named \"" + name + "\".");
      }
    }

    Object value = null;

    if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
      tokenizer.tryConsume(":");  // optional

      final String endToken;
      if (tokenizer.tryConsume("<")) {
        endToken = ">";
      } else {
        tokenizer.consume("{");
        endToken = "}";
      }

      final Message.Builder subBuilder;
      if (extension == null) {
        subBuilder = builder.newBuilderForField(field);
      } else {
        subBuilder = extension.defaultInstance.newBuilderForType();
      }

      while (!tokenizer.tryConsume(endToken)) {
        if (tokenizer.atEnd()) {
          throw tokenizer.parseException(
            "Expected \"" + endToken + "\".");
        }
        mergeField(tokenizer, extensionRegistry, subBuilder);
      }

      value = subBuilder.buildPartial();

    } else {
      tokenizer.consume(":");

      switch (field.getType()) {
        case INT32:
        case SINT32:
        case SFIXED32:
          value = tokenizer.consumeInt32();
          break;

        case INT64:
        case SINT64:
        case SFIXED64:
          value = tokenizer.consumeInt64();
          break;

        case UINT32:
        case FIXED32:
          value = tokenizer.consumeUInt32();
          break;

        case UINT64:
        case FIXED64:
          value = tokenizer.consumeUInt64();
          break;

        case FLOAT:
          value = tokenizer.consumeFloat();
          break;

        case DOUBLE:
          value = tokenizer.consumeDouble();
          break;

        case BOOL:
          value = tokenizer.consumeBoolean();
          break;

        case STRING:
          value = tokenizer.consumeString();
          break;

        case BYTES:
          value = tokenizer.consumeByteString();
          break;

        case ENUM:
          final EnumDescriptor enumType = field.getEnumType();

          if (tokenizer.lookingAtInteger()) {
            final int number = tokenizer.consumeInt32();
            value = enumType.findValueByNumber(number);
            if (value == null) {
              throw tokenizer.parseExceptionPreviousToken(
                "Enum type \"" + enumType.getFullName() +
                "\" has no value with number " + number + '.');
            }
          } else {
            final String id = tokenizer.consumeIdentifier();
            value = enumType.findValueByName(id);
            if (value == null) {
              throw tokenizer.parseExceptionPreviousToken(
                "Enum type \"" + enumType.getFullName() +
                "\" has no value named \"" + id + "\".");
            }
          }

          break;

        case MESSAGE:
        case GROUP:
          throw new RuntimeException("Can't get here.");
      }
    }

    if (field.isRepeated()) {
      builder.addRepeatedField(field, value);
    } else {
      builder.setField(field, value);
    }
  }
1178867
public static Builder push(String field, Object value) {
        return new Builder().push(field, value);
    }
559611
def _SkipFieldValue(tokenizer):
  
  # String/bytes tokens can come in multiple adjacent string literals.
  # If we can consume one, consume as many as we can.
  if tokenizer.TryConsumeByteString():
    while tokenizer.TryConsumeByteString():
      pass
    return

  if (not tokenizer.TryConsumeIdentifier() and
      not _TryConsumeInt64(tokenizer) and not _TryConsumeUint64(tokenizer) and
      not tokenizer.TryConsumeFloat()):
    raise ParseError('Invalid field value: ' + tokenizer.token)
131117
public static <T> void mergeFrom(JsonParser parser, T message, Schema<T> schema,
            boolean numeric) throws IOException
    {
        if (parser.nextToken() != JsonToken.START_OBJECT)
        {
            throw new JsonInputException("Expected token: { but was " +
                    parser.getCurrentToken() + " on message " +
                    schema.messageFullName());
        }

        schema.mergeFrom(new JsonInput(parser, numeric), message);

        if (parser.getCurrentToken() != JsonToken.END_OBJECT)
        {
            throw new JsonInputException("Expected token: } but was " +
                    parser.getCurrentToken() + " on message " +
                    schema.messageFullName());
        }
    }
795197
static public function parse($text, $builder = null) {
        $tokenizer = new HTML5_Tokenizer($text, $builder);
        $tokenizer->parse();
        return $tokenizer->save();
    }
544876
private SelectorBuilderImpl singleValuePredicate(String field, String propertyValue,
      PredicateOperator operator) {
    Predicate predicate = new Predicate();
    predicate.setField(field);
    predicate.setOperator(operator);

    String[] values = new String[1];
    values[0] = propertyValue;
    predicate.setValues(values);

    this.predicates.add(predicate);

    return this;
  }
244856
@Override
    public void parse(final ClassNameAndSuperclassInfo.Builder builder) throws InvalidClassFileFormatException {

        builder.setCodeBaseEntry(codeBaseEntry);
        builder.setAccessFlags(javaClass.getAccessFlags());
        ClassDescriptor classDescriptor = DescriptorFactory.createClassDescriptorFromDottedClassName(javaClass.getClassName());
        if (expectedClassDescriptor != null && expectedClassDescriptor.equals(classDescriptor)) {
            throw new InvalidClassFileFormatException("Expected " + expectedClassDescriptor, classDescriptor, codeBaseEntry);
        }
        builder.setClassDescriptor(classDescriptor);

        builder.setSuperclassDescriptor(DescriptorFactory.createClassDescriptorFromDottedClassName(javaClass.getSuperclassName()));
        String[] allInterfaces = javaClass.getInterfaceNames();
        ClassDescriptor[] allInterfaceDescriptiors;
        if (allInterfaces.length == 0) {
            allInterfaceDescriptiors = ClassDescriptor.EMPTY_ARRAY;
        } else {
            allInterfaceDescriptiors = new ClassDescriptor[allInterfaces.length];
            for (int i = 0; i < allInterfaces.length; i++) {
                allInterfaceDescriptiors[i] = DescriptorFactory.createClassDescriptorFromDottedClassName(allInterfaces[i]);
            }
        }
        builder.setInterfaceDescriptorList(allInterfaceDescriptiors);
    }
934629
@Override
    public void parse(ClassInfo.Builder builder) throws InvalidClassFileFormatException {
        parse((ClassNameAndSuperclassInfo.Builder) builder);

        final List<FieldDescriptor> fieldDescriptorList = new LinkedList<>();
        final List<MethodDescriptor> methodDescriptorList = new LinkedList<>();
        final TreeSet<ClassDescriptor> referencedClassSet = new TreeSet<>();
        javaClass.accept(new AnnotationVisitor() {
            @Override
            public void visit(Method obj) {
                methodDescriptorList.add(parseMethod(obj));
            }

            @Override
            public void visit(Field obj) {
                fieldDescriptorList.add(parseField(obj));
            }

            @Override
            public void visit(ConstantClass obj) {
                @SlashedClassName
                String className = obj.getBytes(javaClass.getConstantPool());
                if (className.indexOf('[') >= 0) {
                    ClassParser.extractReferencedClassesFromSignature(referencedClassSet, className);
                } else if (ClassName.isValidClassName(className)) {
                    referencedClassSet.add(DescriptorFactory.instance().getClassDescriptor(className));
                }
            }

            @Override
            public void visit(ConstantNameAndType obj) {
                String signature = obj.getSignature(javaClass.getConstantPool());
                ClassParser.extractReferencedClassesFromSignature(referencedClassSet, signature);
            }
        });

    }
685681
public static void doParse(Element element, BeanDefinitionBuilder builder) {
        Element messageSelectorElement = DomUtils.getChildElementByTagName(element, "selector");
        if (messageSelectorElement != null) {
            Element selectorStringElement = DomUtils.getChildElementByTagName(messageSelectorElement, "value");
            if (selectorStringElement != null) {
                builder.addPropertyValue("messageSelector", DomUtils.getTextValue(selectorStringElement));
            }

            Map<String, String> messageSelectorMap = new HashMap<>();
            List<?> messageSelectorElements = DomUtils.getChildElementsByTagName(messageSelectorElement, "element");
            for (Iterator<?> iter = messageSelectorElements.iterator(); iter.hasNext();) {
                Element selectorElement = (Element) iter.next();
                messageSelectorMap.put(selectorElement.getAttribute("name"), selectorElement.getAttribute("value"));
            }
            builder.addPropertyValue("messageSelectorMap", messageSelectorMap);
        }
    }
951501
public static void merge(CharSequence input,
                             ExtensionRegistry extensionRegistry,
                             Message.Builder builder) throws ParseException {
        Tokenizer tokenizer = new Tokenizer(input);

        // Based on the state machine @ http://json.org/

        tokenizer.consume("{"); // Needs to happen when the object starts.
        while (!tokenizer.tryConsume("}")) { // Continue till the object is done
            mergeField(tokenizer, extensionRegistry, builder);
        }
    }
1239415
def parse_type(field):
  
  if field.type_id == 'string':
    if 'size' in field.options:
      return "parser.getString(%d)" % field.options['size'].value
    else:
      return "parser.getString()"
  elif field.type_id in JAVA_TYPE_MAP:
    # Primitive java types have extractor methods in SBPMessage.Parser
    return "parser.get" + field.type_id.capitalize() + "()"
  if field.type_id == 'array':
    # Call function to build array
    t = field.options['fill'].value
    if t in JAVA_TYPE_MAP:
      if 'size' in field.options:
        return "parser.getArrayof%s(%d)" % (t.capitalize(), field.options['size'].value)
      else:
        return "parser.getArrayof%s()" % t.capitalize()
    else:
      if 'size' in field.options:
        return "parser.getArray(%s.class, %d)" % (t, field.options['size'].value)
      else:
        return "parser.getArray(%s.class)" % t
  else:
    # This is an inner class, call default constructor
    return "new %s().parse(parser)" % field.type_id
905073
void annotation( AnnotatedDescrBuilder<?> adb ) {
        AnnotationDescrBuilder<?> annotation = null;

        try {
            if (speculateFullAnnotation()) {
                boolean buildState = exprParser.isBuildDescr();
                exprParser.setBuildDescr(true);
                exprParser.fullAnnotation(adb);
                exprParser.setBuildDescr(buildState);
            } else {

                // '@'
                Token at = match(input,
                        DRL6Lexer.AT,
                        null,
                        null,
                        DroolsEditorType.SYMBOL);
                if (state.failed)
                    return;

                // identifier
                String fqn = qualifiedIdentifier();
                if (state.failed)
                    return;

                if (state.backtracking == 0) {
                    annotation = adb.newAnnotation(fqn);
                    helper.setStart(annotation,
                            at);
                }

                try {
                    if (input.LA(1) == DRL6Lexer.LEFT_PAREN) {
                        String value = chunk(DRL6Lexer.LEFT_PAREN,
                                DRL6Lexer.RIGHT_PAREN,
                                -1).trim();
                        if (state.failed)
                            return;
                        if (state.backtracking == 0) {
                            annotation.value(value);
                        }
                    }
                } finally {
                    if (state.backtracking == 0) {
                        helper.setEnd(annotation);
                    }
                }
            }

        } catch (RecognitionException re) {
            reportError(re);
        }
    }
1098654
public AnnotatedTypeBuilder<X> addToField(AnnotatedField<? super X> field, Annotation annotation) {
       return addToField(field.getJavaMember(), annotation);
   }
1078509
@Override
    protected TokenStreamComponents createComponents(String fieldName) {
        //final Tokenizer source = new AlphaNumericTokenizer();
        final Tokenizer source = new WhitespaceTokenizer();
        TokenStream stream = source;

        stream = new UrlTokenizingFilter(stream);
        stream = new AlphaNumericFilter(stream);
        stream = new WordDelimiterGraphFilter(stream,
                WordDelimiterGraphFilter.GENERATE_WORD_PARTS
                //| WordDelimiterGraphFilter.GENERATE_NUMBER_PARTS
                | WordDelimiterGraphFilter.PRESERVE_ORIGINAL
                | WordDelimiterGraphFilter.SPLIT_ON_CASE_CHANGE
                | WordDelimiterGraphFilter.SPLIT_ON_NUMERICS
                | WordDelimiterGraphFilter.STEM_ENGLISH_POSSESSIVE, null);

        stream = new LowerCaseFilter(stream);

        stream = new StopFilter(stream, stopWords);
        concatenatingFilter = new TokenPairConcatenatingFilter(stream);

        return new TokenStreamComponents(source, concatenatingFilter);
    }
1041279
public static void parseJsonToSingleRowBlock(
            JsonParser parser,
            SingleRowBlockWriter singleRowBlockWriter,
            BlockBuilderAppender[] fieldAppenders,
            Optional<Map<String, Integer>> fieldNameToIndex)
            throws IOException
    {
        if (parser.getCurrentToken() == START_ARRAY) {
            for (int i = 0; i < fieldAppenders.length; i++) {
                parser.nextToken();
                fieldAppenders[i].append(parser, singleRowBlockWriter);
            }
            if (parser.nextToken() != JsonToken.END_ARRAY) {
                throw new JsonCastException(format("Expected json array ending, but got %s", parser.getText()));
            }
        }
        else {
            verify(parser.getCurrentToken() == START_OBJECT);
            if (!fieldNameToIndex.isPresent()) {
                throw new JsonCastException("Cannot cast a JSON object to anonymous row type. Input must be a JSON array.");
            }
            boolean[] fieldWritten = new boolean[fieldAppenders.length];
            int numFieldsWritten = 0;

            while (parser.nextToken() != JsonToken.END_OBJECT) {
                if (parser.currentToken() != FIELD_NAME) {
                    throw new JsonCastException(format("Expected a json field name, but got %s", parser.getText()));
                }
                String fieldName = parser.getText().toLowerCase(Locale.ENGLISH);
                Integer fieldIndex = fieldNameToIndex.get().get(fieldName);
                parser.nextToken();
                if (fieldIndex != null) {
                    if (fieldWritten[fieldIndex]) {
                        throw new JsonCastException("Duplicate field: " + fieldName);
                    }
                    fieldWritten[fieldIndex] = true;
                    numFieldsWritten++;
                    fieldAppenders[fieldIndex].append(parser, singleRowBlockWriter.getFieldBlockBuilder(fieldIndex));
                }
                else {
                    parser.skipChildren();
                }
            }

            if (numFieldsWritten != fieldAppenders.length) {
                for (int i = 0; i < fieldWritten.length; i++) {
                    if (!fieldWritten[i]) {
                        singleRowBlockWriter.getFieldBlockBuilder(i).appendNull();
                    }
                }
            }
        }
    }
894079
def parse_fields_definition(lexer: Lexer) -> List[FieldDefinitionNode]:
    """"""
    return (
        cast(
            List[FieldDefinitionNode],
            many_nodes(
                lexer, TokenKind.BRACE_L, parse_field_definition, TokenKind.BRACE_R
            ),
        )
        if peek(lexer, TokenKind.BRACE_L)
        else []
    )
135824
public static Element parse(InputStream xmlStream, DocumentBuilder builder) throws IOException
   {
      try
      {
         Document doc;
         synchronized (builder) //synchronize to prevent concurrent parsing on the same DocumentBuilder
         {
            doc = builder.parse(xmlStream);
         }
         return doc.getDocumentElement();
      }
      catch (SAXException se)
      {
         throw new IOException(se.toString());
      }
      finally
      {
         xmlStream.close();
      }
   }