Create the voldemort key and value from the input Avro record by
extracting the key and value and map it out for each of the responsible
voldemort nodes


The output value is the node_id & partition_id of the responsible node
followed by serialized value
349971
@Override
    public void map(GenericData.Record record,
                    AvroCollector<Pair<ByteBuffer, ByteBuffer>> collector,
                    Reporter reporter) throws IOException {

        byte[] keyBytes = null;
        byte[] valBytes = null;
        Object keyRecord = null;
        Object valRecord = null;
        try {
            keyRecord = record.get(keyField);
            valRecord = record.get(valField);
            keyBytes = keySerializer.toBytes(keyRecord);
            valBytes = valueSerializer.toBytes(valRecord);

            this.collectorWrapper.setCollector(collector);
            this.mapper.map(keyBytes, valBytes, this.collectorWrapper);

            recordCounter++;
        } catch (OutOfMemoryError oom) {
            logger.error(oomErrorMessage(reporter));
            if (keyBytes == null) {
                logger.error("keyRecord caused OOM!");
            } else {
                logger.error("keyRecord: " + keyRecord);
                logger.error("valRecord: " + (valBytes == null ? "caused OOM" : valRecord));
            }
            throw new VoldemortException(oomErrorMessage(reporter), oom);
        }
    }
1312423
private JsonObject parse(JsonObject record, JsonSchema schema)
      throws DataConversionException {
      JsonObject output = new JsonObject();
      for (int i = 0; i < schema.fieldsCount(); i++) {
        JsonSchema schemaElement = schema.getFieldSchemaAt(i);
        String columnKey = schemaElement.getColumnName();
        JsonElement parsed;
        if (!record.has(columnKey)) {
          output.add(columnKey, JsonNull.INSTANCE);
          continue;
        }

        JsonElement columnValue = record.get(columnKey);
        switch (schemaElement.getType()) {
          case UNION:
            parsed = parseUnionType(schemaElement, columnValue);
            break;
          case ENUM:
            parsed = parseEnumType(schemaElement, columnValue);
            break;
          default:
            if (columnValue.isJsonArray()) {
              parsed = parseJsonArrayType(schemaElement, columnValue);
            } else if (columnValue.isJsonObject()) {
              parsed = parseJsonObjectType(schemaElement, columnValue);
            } else {
              parsed = parsePrimitiveType(schemaElement, columnValue);
            }
        }
        output.add(columnKey, parsed);
      }
      return output;
  }
226471
def create(cls, interface_id, address, network_value,
               nodeid=1, **kwargs):
        
        data = {'address': address,
                'network_value': network_value,
                'nicid': str(interface_id),
                'auth_request': False,
                'backup_heartbeat': False,
                'nodeid': nodeid,
                'outgoing': False,
                'primary_mgt': False,
                'primary_heartbeat': False}

        for k, v in kwargs.items():
            data.update({k: v})

        return cls(data)
68109
def _record_extension(self, key, value):
        
        record_bean = {
            'value': value,
            'displayName': self._text_bean(key),
            'description': self._text_bean(key),
            'displayLabel': self._text_bean(key),
            'associatedId': str(self.ident)
        }
        return record_bean
690923
def make_avsc_object(json_data, names=None):
    # type: (Union[Dict[Text, Text], List[Any], Text], Optional[Names]) -> Schema
    
    if names is None:
        names = Names()
    assert isinstance(names, Names)

    # JSON object (non-union)
    if hasattr(json_data, 'get') and callable(json_data.get):  # type: ignore
        assert isinstance(json_data, Dict)
        atype = cast(Text, json_data.get('type'))
        other_props = get_other_props(json_data, SCHEMA_RESERVED_PROPS)
        if atype in PRIMITIVE_TYPES:
            return PrimitiveSchema(atype, other_props)
        if atype in NAMED_TYPES:
            name = cast(Text, json_data.get('name'))
            namespace = cast(Text, json_data.get('namespace',
                                                 names.default_namespace))
            if atype == 'enum':
                symbols = cast(List[Text], json_data.get('symbols'))
                doc = json_data.get('doc')
                return EnumSchema(name, namespace, symbols, names, doc, other_props)
            if atype in ['record', 'error']:
                fields = cast(List, json_data.get('fields'))
                doc = json_data.get('doc')
                return RecordSchema(name, namespace, fields, names, atype, doc, other_props)
            raise SchemaParseException('Unknown Named Type: %s' % atype)
        if atype in VALID_TYPES:
            if atype == 'array':
                items = cast(List, json_data.get('items'))
                return ArraySchema(items, names, other_props)
        if atype is None:
            raise SchemaParseException('No "type" property: %s' % json_data)
        raise SchemaParseException('Undefined type: %s' % atype)
    # JSON array (union)
    if isinstance(json_data, list):
        return UnionSchema(json_data, names)
    # JSON string (primitive)
    if json_data in PRIMITIVE_TYPES:
        return PrimitiveSchema(cast(Text, json_data))
    # not for us!
    fail_msg = "Could not make an Avro Schema object from %s." % json_data
    raise SchemaParseException(fail_msg)
360090
def _make_key_from_dsd(self, flow_id, key):
        '''
        
        '''
        # get the dataflow and the DSD ID
        dataflow = self.get('dataflow', flow_id,
                            memcache='dataflow' + flow_id)
        dsd_id = dataflow.msg.dataflow[flow_id].structure.id
        dsd_resp = self.get('datastructure', dsd_id,
                            memcache='datastructure' + dsd_id)
        dsd = dsd_resp.msg.datastructure[dsd_id]
        # Extract dimensions excluding the dimension at observation (time, time-period)
        # as we are only interested in dimensions for columns, not rows.
        dimensions = [d for d in dsd.dimensions.aslist() if d.id not in
                      ['TIME', 'TIME_PERIOD']]
        dim_names = [d.id for d in dimensions]
        # Retrieve any ContentConstraint
        try:
            constraint_l = [c for c in dataflow.constraint.aslist()
                            if c.constraint_attachment.id == flow_id]
            if constraint_l:
                constraint = constraint_l[0]
        except:
            constraint = None
        # Validate the key dict
        # First, check correctness of dimension names
        invalid = [d for d in key.keys()
                   if d not in dim_names]
        if invalid:
            raise ValueError(
                'Invalid dimension name {0}, allowed are: {1}'.format(invalid, dim_names))
        # Check for each dimension name if values are correct and construct
        # string of the form 'value1.value2.value3+value4' etc.
        parts = []
        # Iterate over the dimensions. If the key dict
        # contains a value for the dimension, append it to the 'parts' list. Otherwise
        # append ''. Then join the parts to form the dotted str.
        for d in dimensions:
            try:
                values = key[d.id]
                values_l = values.split('+')
                codelist = d.local_repr.enum
                codes = codelist.keys()
                invalid = [v for v in values_l if v not in codes]
                if invalid:
                    # ToDo: attach codelist to exception.
                    raise ValueError("'{0}' is not in codelist for dimension '{1}: {2}'".
                                     format(invalid, d.id, codes))
                # Check if values are in Contentconstraint if present
                if constraint:
                    try:
                        invalid = [
                            v for v in values_l if (d.id, v) not in constraint]
                        if invalid:
                            raise ValueError("'{0}' out of content_constraint for '{1}'.".
                                             format(invalid, d.id))
                    except NotImplementedError:
                        pass
                part = values
            except KeyError:
                part = ''
            parts.append(part)
        return '.'.join(parts)
1019440
@Override
    public AbstractHttpEntity harvest(long generation) {
        DecoderHelper decoder = null;
        DataFileWriter<GenericRecord> fileWriter = null;
        ByteBufferEntity entity = null;
        //Now iterate over records and write
        Iterator<GenericRecord> itr = m_records.iterator();
        try {
            while (itr.hasNext()) {
                GenericRecord record = itr.next();
                if (decoder == null) {
                    decoder = decoder(generation, record.getSchema());
                }
                fileWriter = decoder.m_fileWriter;
                fileWriter.append(record);
                itr.remove();
            }

            if (fileWriter != null) {
                fileWriter.flush();
            }
            if (decoder != null) {
                entity = new ByteBufferEntity(decoder.m_bbos.toByteBuffer(), AvroContentType);
            }
        }
        catch (IOException e) {
            throw new BulkException("failed to append to the avro data container", e);
        }
        finally {
            if (decoder != null) {
                decoder.m_bbos.reset();
            }
            m_records.clear();
        }

        return entity;
    }
431774
private Path getOrGenerateSchemaFile(Schema schema) throws IOException {

    Preconditions.checkNotNull(schema, "Avro Schema should not be null");

    String hashedSchema = Hashing.sha256().hashString(schema.toString(), StandardCharsets.UTF_8).toString();

    if (!this.schemaPaths.containsKey(hashedSchema)) {

      Path schemaFilePath = new Path(this.schemaDir, String.valueOf(System.currentTimeMillis() + ".avsc"));
      AvroUtils.writeSchemaToFile(schema, schemaFilePath, fs, true);

      this.schemaPaths.put(hashedSchema, schemaFilePath);
    }

    return this.schemaPaths.get(hashedSchema);
  }
71711
def from_response(cls, response, attrs):
        
        proj = response['Projection']
        hash_key = attrs.get(response['KeySchema'][0]['AttributeName'])
        range_key = None
        if len(response['KeySchema']) > 1:
            range_key = attrs[response['KeySchema'][1]['AttributeName']]
        throughput = Throughput.from_response(
            response['ProvisionedThroughput'])
        index = cls(proj['ProjectionType'], response['IndexName'], hash_key,
                    range_key, proj.get('NonKeyAttributes'), throughput)
        index.response = response
        return index
134568
private Record newRecordFromCql(Key key, Iterable<Row> rows, Placement placement, String rowKey) {
        Session session = placement.getKeyspace().getCqlSession();
        ProtocolVersion protocolVersion = session.getCluster().getConfiguration().getProtocolOptions().getProtocolVersion();
        CodecRegistry codecRegistry = session.getCluster().getConfiguration().getCodecRegistry();

        Iterator<Map.Entry<DeltaClusteringKey, Change>> changeIter = decodeChangesFromCql(new CqlDeltaIterator(rows.iterator(), BLOCK_RESULT_SET_COLUMN, CHANGE_ID_RESULT_SET_COLUMN, VALUE_RESULT_SET_COLUMN, false, _deltaPrefixLength, protocolVersion, codecRegistry, rowKey));
        Iterator<Map.Entry<DeltaClusteringKey, Compaction>> compactionIter = decodeCompactionsFromCql(new CqlDeltaIterator(rows.iterator(), BLOCK_RESULT_SET_COLUMN, CHANGE_ID_RESULT_SET_COLUMN, VALUE_RESULT_SET_COLUMN, false, _deltaPrefixLength, protocolVersion, codecRegistry, rowKey));
        Iterator<RecordEntryRawMetadata> rawMetadataIter = rawMetadataFromCql(new CqlDeltaIterator(rows.iterator(), BLOCK_RESULT_SET_COLUMN, CHANGE_ID_RESULT_SET_COLUMN, VALUE_RESULT_SET_COLUMN, false, _deltaPrefixLength, protocolVersion, codecRegistry, rowKey));

        return new RecordImpl(key, compactionIter, changeIter, rawMetadataIter);
    }
437315
function hash_serializer(result, key, value) {
  var matches = key.match(brackets);

  // Has brackets? Use the recursive assignment function to walk the keys,
  // construct any missing objects in the result tree and make the assignment
  // at the end of the chain.
  if (matches) {
    var keys = parse_keys(key);
    hash_assign(result, keys, value);
  }
  else {
    // Non bracket notation can make assignments directly.
    var existing = result[key];

    // If the value has been assigned already (for instance when a radio and
    // a checkbox have the same name attribute) convert the previous value
    // into an array before pushing into it.
    //
    // NOTE: If this requirement were removed all hash creation and
    // assignment could go through `hash_assign`.
    if (existing) {
      if (!Array.isArray(existing)) {
        result[key] = [ existing ];
      }

      result[key].push(value);
    }
    else {
      result[key] = value;
    }
  }

  return result;
}
155853
@Override
  public Iterable<QueryBasedHiveConversionEntity> convertRecord(Schema outputAvroSchema, QueryBasedHiveConversionEntity conversionEntity, WorkUnitState workUnit)
      throws DataConversionException {

    Preconditions.checkNotNull(outputAvroSchema, "Avro schema must not be null");
    Preconditions.checkNotNull(conversionEntity, "Conversion entity must not be null");
    Preconditions.checkNotNull(workUnit, "Workunit state must not be null");
    Preconditions.checkNotNull(conversionEntity.getTable(), "Hive table within conversion entity must not be null");

    EventWorkunitUtils.setBeginDDLBuildTimeMetadata(workUnit, System.currentTimeMillis());

    this.hiveDataset = conversionEntity.getConvertibleHiveDataset();

    if (!hasConversionConfig()) {
      return new SingleRecordIterable<>(conversionEntity);
    }

    // Avro table name and location
    String avroTableName = conversionEntity.getTable().getTableName();

    // ORC table name and location
    String orcTableName = getConversionConfig().getDestinationTableName();
    String orcStagingTableName = getOrcStagingTableName(getConversionConfig().getDestinationStagingTableName());
    String orcTableDatabase = getConversionConfig().getDestinationDbName();
    String orcDataLocation = getOrcDataLocation();
    String orcStagingDataLocation = getOrcStagingDataLocation(orcStagingTableName);
    boolean isEvolutionEnabled = getConversionConfig().isEvolutionEnabled();
    Pair<Optional<Table>, Optional<List<Partition>>> destinationMeta = HiveConverterUtils.getDestinationTableMeta(orcTableDatabase,
        orcTableName, workUnit.getProperties());
    Optional<Table> destinationTableMeta = destinationMeta.getLeft();

    // Optional
    // View registration blacklist / whitelist
    Optional<WhitelistBlacklist> optionalViewRegistrationWhiteBlacklist = getViewWhiteBackListFromWorkUnit(workUnit);

    // wrapperViewName          : If specified view with 'wrapperViewName' is created if not already exists
    //                            over destination table
    // isUpdateViewAlwaysEnabled: If false 'wrapperViewName' is only updated when schema evolves; if true
    //                            'wrapperViewName' is always updated (everytime publish happens)
    Optional<String> wrapperViewName = Optional.<String>absent();
    if (optionalViewRegistrationWhiteBlacklist.isPresent()) {
      wrapperViewName = optionalViewRegistrationWhiteBlacklist.get().acceptTable(orcTableDatabase, orcTableName)
          ? getConversionConfig().getDestinationViewName() : wrapperViewName;
    } else {
      wrapperViewName = getConversionConfig().getDestinationViewName();
    }
    boolean shouldUpdateView = getConversionConfig().isUpdateViewAlwaysEnabled();

    // Other properties
    Optional<List<String>> clusterBy =
        getConversionConfig().getClusterBy().isEmpty()
            ? Optional.<List<String>> absent()
            : Optional.of(getConversionConfig().getClusterBy());
    Optional<Integer> numBuckets = getConversionConfig().getNumBuckets();
    Optional<Integer> rowLimit = getConversionConfig().getRowLimit();
    Properties tableProperties = getConversionConfig().getDestinationTableProperties();

    // Partition dir hint helps create different directory for hourly and daily partition with same timestamp, such as:
    // .. daily_2016-01-01-00 and hourly_2016-01-01-00
    // This helps existing hourly data from not being deleted at the time of roll up, and so Hive queries in flight
    // .. do not fail
    List<String> sourceDataPathIdentifier = getConversionConfig().getSourceDataPathIdentifier();

    // Populate optional partition info
    Map<String, String> partitionsDDLInfo = Maps.newHashMap();
    Map<String, String> partitionsDMLInfo = Maps.newHashMap();
    HiveConverterUtils.populatePartitionInfo(conversionEntity, partitionsDDLInfo, partitionsDMLInfo);

    /*
     * Create ORC data location with the same permissions as Avro data
     *
     * Note that hive can also automatically create the non-existing directories but it does not
     * seem to create it with the desired permissions.
     * According to hive docs permissions for newly created directories/files can be controlled using uMask like,
     *
     * SET hive.warehouse.subdir.inherit.perms=false;
     * SET fs.permissions.umask-mode=022;
     * Upon testing, this did not work
     */
    try {
      FileStatus sourceDataFileStatus = this.fs.getFileStatus(conversionEntity.getTable().getDataLocation());
      FsPermission sourceDataPermission = sourceDataFileStatus.getPermission();
      if (!this.fs.mkdirs(new Path(getConversionConfig().getDestinationDataPath()), sourceDataPermission)) {
        throw new RuntimeException(String.format("Failed to create path %s with permissions %s", new Path(
            getConversionConfig().getDestinationDataPath()), sourceDataPermission));
      } else {
        this.fs.setPermission(new Path(getConversionConfig().getDestinationDataPath()), sourceDataPermission);

        // Explicitly set group name for destination location if specified otherwise preserve source group name
        String destinationGroupName;
        if (workUnit.contains(HIVE_DATASET_DESTINATION_GROUP_NAME)) {
          destinationGroupName = workUnit.getProp(HIVE_DATASET_DESTINATION_GROUP_NAME);
        } else {
          destinationGroupName = sourceDataFileStatus.getGroup();
        }
        if (!workUnit.getPropAsBoolean(HIVE_DATASET_DESTINATION_SKIP_SETGROUP,
            DEFAULT_HIVE_DATASET_DESTINATION_SKIP_SETGROUP)) {
          this.fs.setOwner(new Path(getConversionConfig().getDestinationDataPath()), null, destinationGroupName);
        }
        log.info(String.format("Created %s with permissions %s and group %s", new Path(getConversionConfig()
            .getDestinationDataPath()), sourceDataPermission, sourceDataFileStatus.getGroup()));

        // Explicitly set group name for staging directory if specified
        if (workUnit.contains(HIVE_DATASET_STAGING_GROUP_NAME)) {
          String stagingGroupName = workUnit.getProp(HIVE_DATASET_STAGING_GROUP_NAME);
          log.info("Setting staging directory group name as " + stagingGroupName);
          this.fs.mkdirs(new Path(getOrcStagingDataLocation(orcStagingTableName)));
          this.fs.setOwner(new Path(getOrcStagingDataLocation(orcStagingTableName)), null, stagingGroupName);

          // Staging directory will be renamed to getOrcDataLocation() and hence it's group name should match
          // with the group name of the staging directory
          this.fs.mkdirs(new Path(getOrcDataLocation()));
          this.fs.setOwner(new Path(getOrcDataLocation()), null, stagingGroupName);
        }
      }
    } catch (IOException e) {
      Throwables.propagate(e);
    }

    // Set hive runtime properties from conversion config
    for (Map.Entry<Object, Object> entry : getConversionConfig().getHiveRuntimeProperties().entrySet()) {
      conversionEntity.getQueries().add(String.format("SET %s=%s", entry.getKey(), entry.getValue()));
    }
    // Set hive runtime properties for tracking
    conversionEntity.getQueries().add(String.format("SET %s=%s", GOBBLIN_DATASET_URN_KEY,
        conversionEntity.getTable().getCompleteName()));
    if (conversionEntity.getPartition().isPresent()) {
      conversionEntity.getQueries().add(String.format("SET %s=%s", GOBBLIN_PARTITION_NAME_KEY,
          conversionEntity.getPartition().get().getCompleteName()));
    }
    conversionEntity.getQueries().add(String
        .format("SET %s=%s", GOBBLIN_WORKUNIT_CREATE_TIME_KEY,
            workUnit.getWorkunit().getProp(SlaEventKeys.ORIGIN_TS_IN_MILLI_SECS_KEY)));

    // Create DDL statement for table
    Map<String, String> hiveColumns = new LinkedHashMap<>();
    String createStagingTableDDL =
        HiveAvroORCQueryGenerator.generateCreateTableDDL(outputAvroSchema,
            orcStagingTableName,
            orcStagingDataLocation,
            Optional.of(orcTableDatabase),
            Optional.of(partitionsDDLInfo),
            clusterBy,
            Optional.<Map<String, HiveAvroORCQueryGenerator.COLUMN_SORT_ORDER>>absent(),
            numBuckets,
            Optional.<String>absent(),
            Optional.<String>absent(),
            Optional.<String>absent(),
            tableProperties,
            isEvolutionEnabled,
            destinationTableMeta,
            hiveColumns);
    conversionEntity.getQueries().add(createStagingTableDDL);
    log.debug("Create staging table DDL: " + createStagingTableDDL);

    // Create DDL statement for partition
    String orcStagingDataPartitionDirName = HiveConverterUtils.getStagingDataPartitionDirName(conversionEntity, sourceDataPathIdentifier);
    String orcStagingDataPartitionLocation = orcStagingDataLocation + Path.SEPARATOR + orcStagingDataPartitionDirName;
    if (partitionsDMLInfo.size() > 0) {
      List<String> createStagingPartitionDDL =
          HiveAvroORCQueryGenerator.generateCreatePartitionDDL(orcTableDatabase,
              orcStagingTableName,
              orcStagingDataPartitionLocation,
              partitionsDMLInfo);

      conversionEntity.getQueries().addAll(createStagingPartitionDDL);
      log.debug("Create staging partition DDL: " + createStagingPartitionDDL);
    }

    // Create DML statement
    String insertInORCStagingTableDML =
        HiveAvroORCQueryGenerator
            .generateTableMappingDML(conversionEntity.getHiveTable().getAvroSchema(),
                outputAvroSchema,
                avroTableName,
                orcStagingTableName,
                Optional.of(conversionEntity.getTable().getDbName()),
                Optional.of(orcTableDatabase),
                Optional.of(partitionsDMLInfo),
                Optional.<Boolean>absent(),
                Optional.<Boolean>absent(),
                isEvolutionEnabled,
                destinationTableMeta,
                rowLimit);
    conversionEntity.getQueries().add(insertInORCStagingTableDML);
    log.debug("Conversion staging DML: " + insertInORCStagingTableDML);

    // TODO: Split this method into two (conversion and publish)
    // Addition to WUS for Staging publish:
    // A. Evolution turned on:
    //    1. If table does not exists: simply create it (now it should exist)
    //    2. If table exists:
    //      2.1 Evolve table (alter table)
    //      2.2 If snapshot table:
    //          2.2.1 Delete data in final table directory
    //          2.2.2 Move data from staging to final table directory
    //          2.2.3 Drop this staging table and delete directories
    //      2.3 If partitioned table, move partitions from staging to final table; for all partitions:
    //          2.3.1 Drop if exists partition in final table
    //          2.3.2 Move partition directory
    //          2.3.3 Create partition with location
    //          2.3.4 Drop this staging table and delete directories
    // B. Evolution turned off:
    //    1. If table does not exists: simply create it (now it should exist)
    //    2. If table exists:
    //      2.1 Do not evolve table
    //      2.2 If snapshot table:
    //          2.2.1 Delete data in final table directory
    //          2.2.2 Move data from staging to final table directory
    //          2.2.3 Drop this staging table and delete directories
    //      2.3 If partitioned table, move partitions from staging to final table; for all partitions:
    //          2.3.1 Drop if exists partition in final table
    //          2.3.2 Move partition directory
    //          2.3.3 Create partition with location
    //          2.3.4 Drop this staging table and delete directories
    // Note: The queries below also serve as compatibility check module before conversion, an incompatible
    //      .. schema throws a Runtime exeption, hence preventing further execution
    QueryBasedHivePublishEntity publishEntity = new QueryBasedHivePublishEntity();
    List<String> publishQueries = publishEntity.getPublishQueries();
    Map<String, String> publishDirectories = publishEntity.getPublishDirectories();
    List<String> cleanupQueries = publishEntity.getCleanupQueries();
    List<String> cleanupDirectories = publishEntity.getCleanupDirectories();

    // Step:
    // A.1, B.1: If table does not exists, simply create it
    if (!destinationTableMeta.isPresent()) {
      String createTargetTableDDL =
          HiveAvroORCQueryGenerator.generateCreateTableDDL(outputAvroSchema,
              orcTableName,
              orcDataLocation,
              Optional.of(orcTableDatabase),
              Optional.of(partitionsDDLInfo),
              clusterBy,
              Optional.<Map<String, HiveAvroORCQueryGenerator.COLUMN_SORT_ORDER>>absent(),
              numBuckets,
              Optional.<String>absent(),
              Optional.<String>absent(),
              Optional.<String>absent(),
              tableProperties,
              isEvolutionEnabled,
              destinationTableMeta,
              new HashMap<String, String>());
      publishQueries.add(createTargetTableDDL);
      log.debug("Create final table DDL: " + createTargetTableDDL);
    }

    // Step:
    // A.2.1: If table pre-exists (destinationTableMeta would be present), evolve table
    // B.2.1: No-op
    List<String> evolutionDDLs = HiveAvroORCQueryGenerator.generateEvolutionDDL(orcStagingTableName,
        orcTableName,
        Optional.of(orcTableDatabase),
        Optional.of(orcTableDatabase),
        outputAvroSchema,
        isEvolutionEnabled,
        hiveColumns,
        destinationTableMeta);
    log.debug("Evolve final table DDLs: " + evolutionDDLs);
    EventWorkunitUtils.setEvolutionMetadata(workUnit, evolutionDDLs);

    // View (if present) must be updated if evolution happens
    shouldUpdateView |= evolutionDDLs.size() > 0;

    publishQueries.addAll(evolutionDDLs);


    if (partitionsDDLInfo.size() == 0) {
      // Step:
      // A.2.2, B.2.2: Snapshot table

      // Step:
      // A.2.2.1, B.2.2.1: Delete data in final table directory
      // A.2.2.2, B.2.2.2: Move data from staging to final table directory
      log.info("Snapshot directory to move: " + orcStagingDataLocation + " to: " + orcDataLocation);
      publishDirectories.put(orcStagingDataLocation, orcDataLocation);

      // Step:
      // A.2.2.3, B.2.2.3: Drop this staging table and delete directories
      String dropStagingTableDDL = HiveAvroORCQueryGenerator.generateDropTableDDL(orcTableDatabase, orcStagingTableName);

      log.debug("Drop staging table DDL: " + dropStagingTableDDL);
      cleanupQueries.add(dropStagingTableDDL);

      // Delete: orcStagingDataLocation
      log.info("Staging table directory to delete: " + orcStagingDataLocation);
      cleanupDirectories.add(orcStagingDataLocation);

    } else {
      // Step:
      // A.2.3, B.2.3: If partitioned table, move partitions from staging to final table; for all partitions:

      // Step:
      // A.2.3.2, B.2.3.2: Move partition directory
      // Move: orcStagingDataPartitionLocation to: orcFinalDataPartitionLocation
      String orcFinalDataPartitionLocation = orcDataLocation + Path.SEPARATOR + orcStagingDataPartitionDirName;
      Optional<Path> destPartitionLocation = getDestinationPartitionLocation(destinationTableMeta, workUnit,
          conversionEntity.getPartition().get().getName());
      orcFinalDataPartitionLocation =
          HiveConverterUtils.updatePartitionLocation(orcFinalDataPartitionLocation, workUnit, destPartitionLocation);
      log.info(
          "Partition directory to move: " + orcStagingDataPartitionLocation + " to: " + orcFinalDataPartitionLocation);
      publishDirectories.put(orcStagingDataPartitionLocation, orcFinalDataPartitionLocation);
      // Step:
      // A.2.3.1, B.2.3.1: Drop if exists partition in final table

      // Step:
      // If destination partition already exists, alter the partition location
      // A.2.3.3, B.2.3.3: Create partition with location (and update storage format if not in ORC already)
      List<String> dropPartitionsDDL =
          HiveAvroORCQueryGenerator.generateDropPartitionsDDL(orcTableDatabase,
              orcTableName,
              partitionsDMLInfo);
      log.debug("Drop partitions if exist in final table: " + dropPartitionsDDL);
      publishQueries.addAll(dropPartitionsDDL);
      if (workUnit.getPropAsBoolean(HIVE_CONVERSION_SETSERDETOAVROEXPLICITELY,
          DEFAULT_HIVE_CONVERSION_SETSERDETOAVROEXPLICITELY)) {
        List<String> createFinalPartitionDDL =
            HiveAvroORCQueryGenerator.generateCreatePartitionDDL(orcTableDatabase,
                orcTableName,
                orcFinalDataPartitionLocation,
                partitionsDMLInfo,
                Optional.<String>absent());

        log.debug("Create final partition DDL: " + createFinalPartitionDDL);
        publishQueries.addAll(createFinalPartitionDDL);

        // Updating storage format non-transactionally is a stop gap measure until Hive supports transactionally update
        // .. storage format in ADD PARITTION command (today it only supports specifying location)
        List<String> updatePartitionStorageFormatDDL =
            HiveAvroORCQueryGenerator.generateAlterTableOrPartitionStorageFormatDDL(orcTableDatabase,
                orcTableName,
                Optional.of(partitionsDMLInfo),
                ORC_FORMAT);
        log.debug("Update final partition storage format to ORC (if not already in ORC)");
        publishQueries.addAll(updatePartitionStorageFormatDDL);
      } else {
        List<String> createFinalPartitionDDL =
            HiveAvroORCQueryGenerator.generateCreatePartitionDDL(orcTableDatabase,
                orcTableName,
                orcFinalDataPartitionLocation,
                partitionsDMLInfo,
                Optional.fromNullable(ORC_FORMAT));

        log.debug("Create final partition DDL: " + createFinalPartitionDDL);
        publishQueries.addAll(createFinalPartitionDDL);
      }

      // Step:
      // A.2.3.4, B.2.3.4: Drop this staging table and delete directories
      String dropStagingTableDDL = HiveAvroORCQueryGenerator.generateDropTableDDL(orcTableDatabase, orcStagingTableName);

      log.debug("Drop staging table DDL: " + dropStagingTableDDL);
      cleanupQueries.add(dropStagingTableDDL);

      // Delete: orcStagingDataLocation
      log.info("Staging table directory to delete: " + orcStagingDataLocation);
      cleanupDirectories.add(orcStagingDataLocation);
    }

    /*
     * Drop the replaced partitions if any. This is required in case the partition being converted is derived from
     * several other partitions. E.g. Daily partition is a replacement of hourly partitions of the same day. When daily
     * partition is converted to ORC all it's hourly ORC partitions need to be dropped.
     */
    publishQueries.addAll(HiveAvroORCQueryGenerator.generateDropPartitionsDDL(orcTableDatabase,
        orcTableName,
        getDropPartitionsDDLInfo(conversionEntity)));

    /*
     * Create or update view over the ORC table if specified in the config (ie. wrapper view name is present in config)
     */
    if (wrapperViewName.isPresent()) {
      String viewName = wrapperViewName.get();
      List<String> createOrUpdateViewDDLs = HiveAvroORCQueryGenerator.generateCreateOrUpdateViewDDL(orcTableDatabase,
          orcTableName, orcTableDatabase, viewName, shouldUpdateView);
      log.debug("Create or update View DDLs: " + createOrUpdateViewDDLs);
      publishQueries.addAll(createOrUpdateViewDDLs);

    }

    HiveAvroORCQueryGenerator.serializePublishCommands(workUnit, publishEntity);
    log.debug("Publish partition entity: " + publishEntity);


    log.debug("Conversion Query " + conversionEntity.getQueries());

    EventWorkunitUtils.setEndDDLBuildTimeMetadata(workUnit, System.currentTimeMillis());

    return new SingleRecordIterable<>(conversionEntity);
  }
192891
public void map(DocumentID key, LineDocTextAndOp value,
      OutputCollector<DocumentID, DocumentAndOp> output, Reporter reporter)
      throws IOException {

    DocumentAndOp.Op op = value.getOp();
    Document doc = null;
    Term term = null;

    if (op == DocumentAndOp.Op.INSERT || op == DocumentAndOp.Op.UPDATE) {
      doc = new Document();
      doc.add(new Field(docidFieldName, key.getText().toString(),
          Field.Store.YES, Field.Index.UN_TOKENIZED));
      doc.add(new Field(contentFieldName, value.getText().toString(),
          Field.Store.NO, Field.Index.TOKENIZED));
    }

    if (op == DocumentAndOp.Op.DELETE || op == DocumentAndOp.Op.UPDATE) {
      term = new Term(docidFieldName, key.getText().toString());
    }

    output.collect(key, new DocumentAndOp(op, doc, term));
  }
372523
private Record newRecordFromCql(Key key, Iterable<Row> rows) {
        Iterator<Map.Entry<DeltaClusteringKey, Change>> changeIter = decodeChangesFromCql(rows.iterator());
        Iterator<Map.Entry<DeltaClusteringKey, Compaction>> compactionIter = decodeCompactionsFromCql(rows.iterator());
        Iterator<RecordEntryRawMetadata> rawMetadataIter = rawMetadataFromCql(rows.iterator());

        return new RecordImpl(key, compactionIter, changeIter, rawMetadataIter);
    }
1321152
def create_raw(self, key, value):
        
        data = None
        if key is not None and value is not None:
            data = self.db.create(key.strip(), value)
        else:
            self.tcex.log.warning(u'The key or value field was None.')
        return data
444362
def _record_from_json(value, field):
    """"""
    if _not_null(value, field):
        record = {}
        record_iter = zip(field.fields, value["f"])
        for subfield, cell in record_iter:
            converter = _CELLDATA_FROM_JSON[subfield.field_type]
            if subfield.mode == "REPEATED":
                value = [converter(item["v"], subfield) for item in cell["v"]]
            else:
                value = converter(cell["v"], subfield)
            record[subfield.name] = value
        return record
45568
function(records) {
    var self = this.chain()
    var relation = self.getInternal('relation')
    var parentRecord = self.getInternal('relation_to')

    if (!records) return self
    if (!Array.isArray(records)) records = [records]

    // build a map to check if existing records were added - via primary key
    const primaryKeys = self.definition.primaryKeys
    const map = {}
    self.forEach(function(record) {
      var mapKey = []
      primaryKeys.forEach(function(key) {
        if (record[key]) mapKey.push(record[key])
      })
      if (mapKey.length > 0) map[mapKey.join('|')] = record
    })

    records.forEach(function(record) {
      var fromKeys = false
      var mapKey = []

      if (record) {
        // check if `records` is a primitive, or an array of primitives with the same amount of elements as our primary keys
        if (
          typeof record !== 'object' ||
          (Array.isArray(record) &&
            record.length === primaryKeys.length &&
            typeof record[0] !== 'object')
        ) {
          // not a record/object... so it must be a primary key!
          if (self.options.polymorph)
            throw new Error('Polymorphic relations need to add a record!')

          var keys = record
          const tmpRecord = {}

          if (!Array.isArray(keys)) keys = [keys]

          primaryKeys.forEach(function(key, index) {
            if (keys[index]) {
              tmpRecord[key] = keys[index]
              mapKey.push(keys[index])
            }
          })

          if (mapKey.length > 0 && map[mapKey.join('|')]) {
            // found a record no update needed, because we only got the id...
            return
          }

          record = tmpRecord
          fromKeys = true
        }

        primaryKeys.forEach(function(key) {
          if (record[key]) mapKey.push(record[key])
        })

        if (mapKey.length > 0 && map[mapKey.join('|')]) {
          // found a record for update!
          map[mapKey.join('|')].set(record)
          return
        }

        const originalInput = record
        // convert object to a record
        if (self.options.polymorph) {
          if (record.model && !(record instanceof record.model))
            throw new Error(
              'Record/Model instance expected in polymorphic relation!'
            )
        } else {
          if (!(record instanceof self.model)) record = self.model.new(record)
        }

        record.__chainedModel = self

        if (fromKeys) {
          record._exists()
        } else {
          if (mapKey.length > 0 && mapKey.length === primaryKeys.length) {
            record._exists()
            if (!(originalInput instanceof self.model)) {
              // set changes. if for example the record is not loaded, but we now have the primary keys. Only the given fields are changes.
              // all fields that are not present in the original input, must not be changed, because we dont now their value.
              Object.keys(originalInput).forEach(function(key) {
                // ignore primary keys
                if (primaryKeys.indexOf(key) === -1) {
                  record.changes[key] = [undefined, originalInput[key]]
                }
              })
            }
          }
        }

        // a new one
        self.push(record)

        if (relation && parentRecord) {
          if (record && typeof relation.add === 'function') {
            relation.add.call(self, parentRecord, record)
          }
        }
      }
    })

    return self
  }