Transform Avro schema into H2O type.

@param s  avro schema
@return  a byte representing H2O column type
@throws IllegalArgumentException  if schema is not supported
14200
public static byte schemaToColumnType(Schema s) {
    Schema.Type typ =  s.getType();
    switch (typ) {
      case BOOLEAN:
      case INT:
      case LONG:
      case FLOAT:
      case DOUBLE:
        return Vec.T_NUM;
      case ENUM:
        return Vec.T_CAT;
      case STRING:
        return Vec.T_STR;
      case NULL:
        return Vec.T_BAD;
      case BYTES:
        return Vec.T_STR;
      case UNION: // Flattenize the union
        List<Schema> unionSchemas = s.getTypes();
        if (unionSchemas.size() == 1) {
          return schemaToColumnType(unionSchemas.get(0));
        } else if (unionSchemas.size() == 2) {
          Schema s1 = unionSchemas.get(0);
          Schema s2 = unionSchemas.get(1);
          if (s1.getType().equals(Schema.Type.NULL)) return schemaToColumnType(s2);
          else if (s2.getType().equals(Schema.Type.NULL)) return schemaToColumnType(s1);
        }
      default:
        throw new IllegalArgumentException("Unsupported Avro schema type: " + s);
    }
  }
629953
@Override
    public Schema transform(Schema schema) {
        int nCols = schema.numColumns();
        List<String> colNames = schema.getColumnNames();
        List<ColumnMetaData> meta = schema.getColumnMetaData();
        List<ColumnMetaData> newMeta = new ArrayList<>(nCols);

        for (int i = 0; i < nCols; i++) {
            String name = colNames.get(i);
            ColumnMetaData inMeta = meta.get(i);

            if (keyColumnsSet != null && keyColumnsSet.contains(name)) {
                //No change to key columns
                newMeta.add(inMeta);
                continue;
            }

            //First: check for a custom reduction on this column
            if (customReductions != null && customReductions.containsKey(name)) {
                AggregableColumnReduction reduction = customReductions.get(name);

                List<String> outName = reduction.getColumnsOutputName(name);
                List<ColumnMetaData> outMeta = reduction.getColumnOutputMetaData(outName, inMeta);
                newMeta.addAll(outMeta);
                continue;
            }

            //Second: check for conditional reductions on this column:
            if (conditionalReductions != null && conditionalReductions.containsKey(name)) {
                ConditionalReduction reduction = conditionalReductions.get(name);

                List<String> outNames = reduction.getOutputNames();
                List<ReduceOp> reductions = reduction.getReductions();
                for (int j = 0; j < reduction.getReductions().size(); j++) {
                    ReduceOp red = reductions.get(j);
                    String outName = outNames.get(j);
                    ColumnMetaData m = getMetaForColumn(red, name, inMeta);
                    m.setName(outName);
                    newMeta.add(m);
                }
                continue;
            }


            //Otherwise: get the specified (built-in) reduction op
            //If no reduction op is specified for that column: use the default
            List<ReduceOp> lop = opMap.containsKey(name) ? opMap.get(name) : Collections.singletonList(defaultOp);
            if (lop != null)
                for (ReduceOp op : lop) {
                    newMeta.add(getMetaForColumn(op, name, inMeta));
                }
        }

        return schema.newSchema(newMeta);
    }
26523
public static ByteBuffer toByteBuffer(String s) {
        ByteBuffer buffer = ByteBuffer.allocateDirect(s.length());
        buffer.put(s.getBytes(UTF_8));
        buffer.flip();
        return buffer;
    }
145208
private static byte[] typeConvert(MappingConfig.ColumnItem columnItem, MappingConfig.HbaseMapping hbaseMapping,
                                      Object value) {
        if (value == null) {
            return null;
        }
        byte[] bytes = null;
        if (columnItem == null || columnItem.getType() == null || "".equals(columnItem.getType())) {
            if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
                bytes = Bytes.toBytes(value.toString());
            } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
                bytes = TypeUtil.toBytes(value);
            } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
                PhType phType = PhType.getType(value.getClass());
                bytes = PhTypeUtil.toBytes(value, phType);
            }
        } else {
            if (hbaseMapping.getMode() == MappingConfig.Mode.STRING) {
                bytes = Bytes.toBytes(value.toString());
            } else if (hbaseMapping.getMode() == MappingConfig.Mode.NATIVE) {
                Type type = Type.getType(columnItem.getType());
                bytes = TypeUtil.toBytes(value, type);
            } else if (hbaseMapping.getMode() == MappingConfig.Mode.PHOENIX) {
                PhType phType = PhType.getType(columnItem.getType());
                bytes = PhTypeUtil.toBytes(value, phType);
            }
        }
        return bytes;
    }
493584
public Schema loadSchema(String schema) {
    try {
      JSONObject rawSchema = new JSONObject(new JSONTokener(schema));
      return SchemaLoader.load(rawSchema);
    } catch (JSONException | SchemaException e) {
      throw new InvalidJsonSchemaException(e);
    }
  }
1336934
@Override
  public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException {
    if (this.fieldRemover.isPresent()) {
      return this.fieldRemover.get().removeFields(inputSchema);
    }
    return inputSchema;
  }
743158
public static <SI, SO> GlobalMetadataBuilder<SO> builderWithInput(GlobalMetadata<SI> inputMetadata, Optional<SO> outputSchema) {
    GlobalMetadataBuilder<SO> builder = (GlobalMetadataBuilder<SO>) builder();

    if (outputSchema.isPresent()) {
      builder.schema(outputSchema.get());
    }

    return builder;
  }
596080
public static org.apache.arrow.vector.types.pojo.Schema toArrowSchema(Schema schema) {
        List<Field> fields = new ArrayList<>(schema.numColumns());
        for(int i = 0; i < schema.numColumns(); i++) {
            fields.add(getFieldForColumn(schema.getName(i),schema.getType(i)));
        }

        return new org.apache.arrow.vector.types.pojo.Schema(fields);
    }
431774
private Path getOrGenerateSchemaFile(Schema schema) throws IOException {

    Preconditions.checkNotNull(schema, "Avro Schema should not be null");

    String hashedSchema = Hashing.sha256().hashString(schema.toString(), StandardCharsets.UTF_8).toString();

    if (!this.schemaPaths.containsKey(hashedSchema)) {

      Path schemaFilePath = new Path(this.schemaDir, String.valueOf(System.currentTimeMillis() + ".avsc"));
      AvroUtils.writeSchemaToFile(schema, schemaFilePath, fs, true);

      this.schemaPaths.put(hashedSchema, schemaFilePath);
    }

    return this.schemaPaths.get(hashedSchema);
  }
1092183
public static <T> byte[] toByteArray(T message, Schema<T> schema)
    {
        return toByteArray(message, schema, DEFAULT_OUTPUT_FACTORY);
    }
208210
@Override
  public Converter<Schema, JdbcEntrySchema, GenericRecord, JdbcEntryData> init(WorkUnitState workUnit) {
    String avroToJdbcFieldsPairJsonStr = workUnit.getProp(ConfigurationKeys.CONVERTER_AVRO_JDBC_ENTRY_FIELDS_PAIRS);
    if (!StringUtils.isEmpty(avroToJdbcFieldsPairJsonStr)) {
      if (!this.avroToJdbcColPairs.isPresent()) {
        ImmutableMap.Builder<String, String> avroToJdbcBuilder = ImmutableMap.builder();
        ImmutableMap.Builder<String, String> jdbcToAvroBuilder = ImmutableMap.builder();

        JsonObject json = new JsonParser().parse(avroToJdbcFieldsPairJsonStr).getAsJsonObject();
        for (Map.Entry<String, JsonElement> entry : json.entrySet()) {
          if (!entry.getValue().isJsonPrimitive()) {
            throw new IllegalArgumentException("Json value should be a primitive String. "
                + ConfigurationKeys.CONVERTER_AVRO_JDBC_ENTRY_FIELDS_PAIRS + " : " + avroToJdbcFieldsPairJsonStr);
          }
          avroToJdbcBuilder.put(entry.getKey(), entry.getValue().getAsString());
          jdbcToAvroBuilder.put(entry.getValue().getAsString(), entry.getKey());
        }
        this.avroToJdbcColPairs = Optional.of((Map<String, String>) avroToJdbcBuilder.build());
      }
    }
    return this;
  }
1112642
public static StreamSchema deserialize(String schemaString) {
    Iterable<String> fieldArray = Splitter.on(';').trimResults().omitEmptyStrings().split(schemaString);
    StreamSchema.Builder builder = new StreamSchema.Builder();
    for (String field : fieldArray) {
      Iterable<String> defnArray = Splitter.onPattern("\\s+").trimResults().omitEmptyStrings().split(field);
      List<String> defnList = Lists.newArrayList(defnArray);
      //We expect FieldType, FieldName (optional : FieldAttribute)
      Preconditions.checkArgument(defnList.size() >= 2);
      GDATFieldType type = GDATFieldType.getGDATFieldType(defnList.get(0).toUpperCase());
      String fieldName = defnList.get(1);
      builder.addField(fieldName, type);
    }
    return builder.build();
  }
1352859
@Override
  public Schema getSchema() {
    if (this.workUnit.contains(ConfigurationKeys.SOURCE_SCHEMA)) {
      return new Schema.Parser().parse(this.workUnit.getProp(ConfigurationKeys.SOURCE_SCHEMA));
    }

    AvroFsHelper hfsHelper = (AvroFsHelper) this.fsHelper;
    if (this.filesToPull.isEmpty()) {
      return null;
    }
    try {
      return hfsHelper.getAvroSchema(this.filesToPull.get(0));
    } catch (FileBasedHelperException e) {
      Throwables.propagate(e);
      return null;
    }
  }
760469
public final static byte[] readByteArray(final ObjectInputStream s) throws IOException {
		final byte[] a = new byte[readVByte(s)];
		s.readFully(a);
		return a;
	}
293046
public Schema compileSchema( InputStream stream )
		throws VerifierConfigurationException, SAXException, IOException {
		
		return compileSchema(stream,null);
	}
520579
protected Field convertFieldSchema(Schema inputSchema, Field field, WorkUnitState workUnit)
      throws SchemaConversionException {
    if (field.name().equals(payloadField)) {
      // Create a payload field with latest schema
      return createLatestPayloadField(field);
    }
    // Make a copy of the field to the output schema
    return new Field(field.name(), field.schema(), field.doc(), field.defaultValue(), field.order());
  }
1115282
public static <T extends Schema> T newInstance(Class<T> clz) {
    try { return clz.newInstance(); }
    catch (Exception e) { throw H2O.fail("Failed to instantiate schema of class: " + clz.getCanonicalName(),e); }
  }