/* (non-Javadoc)
@see FileOutputFormat#getRecordWriter(FileSystem, JobConf, String, Progressable)
14004
public RecordWriter<Shard, Text> getRecordWriter(final FileSystem fs,
      JobConf job, String name, final Progressable progress)
      throws IOException {

    final Path perm = new Path(getWorkOutputPath(job), name);

    return new RecordWriter<Shard, Text>() {
      public void write(Shard key, Text value) throws IOException {
        assert (IndexUpdateReducer.DONE.equals(value));

        String shardName = key.getDirectory();
        shardName = shardName.replace("/", "_");

        Path doneFile =
            new Path(perm, IndexUpdateReducer.DONE + "_" + shardName);
        if (!fs.exists(doneFile)) {
          fs.createNewFile(doneFile);
        }
      }

      public void close(final Reporter reporter) throws IOException {
      }
    };
  }
349460
@Override
    public PartitionMetaData write(List<Writable> record) throws IOException {
        if (!record.isEmpty()) {
            List<Writable> recordList = record instanceof List ? (List<Writable>) record : new ArrayList<>(record);

            /* Infer label columns, if necessary. The default is
             * to assume that last column is a label and that the
             * first label column immediately follows the
             * last feature column.
             */
            if (hasLabel) {
                if (labelLastColumn < 0)
                    labelLastColumn = record.size() - 1;
                if (labelFirstColumn < 0) {
                    if (featureLastColumn > 0)
                        labelFirstColumn = featureLastColumn + 1;
                    else
                        labelFirstColumn = record.size() - 1;
                }
            }

            /* Infer feature columns, if necessary. The default is
             * to assume that the first column is a feature and that
             * the last feature column immediately precedes the first
             * label column, if there are any.
             */
            if (featureLastColumn < 0) {
                if (labelFirstColumn > 0)
                    featureLastColumn = labelFirstColumn - 1;
                else
                    featureLastColumn = recordList.size() - 1;
            }

            StringBuilder result = new StringBuilder();
            // Process labels
            if (hasLabel) {
                // Track label indeces
                int labelIndex = zeroBasedLabelIndexing ? 0 : 1;
                for (int i = labelFirstColumn; i <= labelLastColumn; i++) {
                    Writable w = record.get(i);
                    // Handle array-structured Writables, which themselves have multiple columns
                    if (w instanceof ArrayWritable) {
                        ArrayWritable arr = (ArrayWritable) w;
                        for (int j = 0; j < arr.length(); j++) {
                            double val = arr.getDouble(j);
                            // If multilabel, only store indeces of non-zero labels
                            if (multilabel) {
                                if (val == 1.0) {
                                    result.append(SVMLightRecordReader.LABEL_DELIMITER + labelIndex);
                                } else if (val != 0.0 && val != -1.0)
                                    throw new NumberFormatException("Expect value -1, 0, or 1 for multilabel targets (found " + val + ")");
                            } else { // Store value of standard label
                                result.append(SVMLightRecordReader.LABEL_DELIMITER + val);
                            }
                            labelIndex++; // Increment label index for each entry in array
                        }
                    } else { // Handle scalar Writables
                        // If multilabel, only store indeces of non-zero labels
                        if (multilabel) {
                            double val = Double.valueOf(w.toString());
                            if (val == 1.0) {
                                result.append(SVMLightRecordReader.LABEL_DELIMITER + labelIndex);
                            } else if (val != 0.0 && val != -1.0)
                                throw new NumberFormatException("Expect value -1, 0, or 1 for multilabel targets (found " + val + ")");
                        } else { // Store value of standard label
                            try { // Encode label as integer, if possible
                                int val = Integer.valueOf(w.toString());
                                result.append(SVMLightRecordReader.LABEL_DELIMITER + val);
                            } catch (Exception e) {
                                double val = Double.valueOf(w.toString());
                                result.append(SVMLightRecordReader.LABEL_DELIMITER + val);
                            }
                        }
                        labelIndex++; // Increment label index once per scalar Writable
                    }
                }
            }
            if (result.toString().equals("")) { // Add "unlabeled" label if no labels found
                result.append(SVMLightRecordReader.LABEL_DELIMITER + UNLABELED);
            }

            // Track feature indeces
            int featureIndex = zeroBasedIndexing ? 0 : 1;
            for (int i = featureFirstColumn; i <= featureLastColumn; i++) {
                Writable w = record.get(i);
                // Handle array-structured Writables, which themselves have multiple columns
                if (w instanceof ArrayWritable) {
                    ArrayWritable arr = (ArrayWritable) w;
                    for (int j = 0; j < arr.length(); j++) {
                        double val = arr.getDouble(j);
                        if (val != 0) {
                            result.append(SVMLightRecordReader.PREFERRED_DELIMITER + featureIndex);
                            result.append(SVMLightRecordReader.FEATURE_DELIMITER + val);
                        }
                        featureIndex++; // Increment feature index for each entry in array
                    }
                } else {
                    double val = w.toDouble();
                    if (val != 0) {
                        result.append(SVMLightRecordReader.PREFERRED_DELIMITER + featureIndex);
                        result.append(SVMLightRecordReader.FEATURE_DELIMITER + val);
                    }
                    featureIndex++; // Increment feature index once per scalar Writable
                }
            }

            // Remove extra label delimiter at beginning
            String line = result.substring(1).toString();
            out.write(line.getBytes());
            out.write(NEW_LINE.getBytes());

        }

        return PartitionMetaData.builder().numRecordsUpdated(1).build();
    }
1049263
private Iterator<D> getIterator() throws DataRecordException, IOException {
    if (Boolean.valueOf(this.workUnitState.getProp(ConfigurationKeys.SOURCE_QUERYBASED_IS_SPECIFIC_API_ACTIVE))) {
      return this.getRecordSetFromSourceApi(this.schema, this.entity, this.workUnit, this.predicateList);
    }
    return this.getRecordSet(this.schema, this.entity, this.workUnit, this.predicateList);
  }
309618
@SuppressWarnings("unchecked")
	public List<RecordWriter<SerializationDelegate<T>>> getWriters() {
		return Collections.unmodifiableList(Arrays.asList(this.writers));
	}
1075729
public RecordWriter<K,SAMRecordWritable> getRecordWriter(
            TaskAttemptContext ctx, Path out)
            throws IOException
    {
        if (this.header == null)
            throw new IOException(
                    "Can't create a RecordWriter without the SAM header");

        return new KeyIgnoringCRAMRecordWriter<K>(out, header, writeHeader, ctx);
    }
1335760
public TextWriterWriterInterface<?> getWriterFor(Object o) {
    if(o == null) {
      return null;
    }
    TextWriterWriterInterface<?> writer = writers.getHandler(o);
    if(writer != null) {
      return writer;
    }
    try {
      final Class<?> decl = o.getClass().getMethod("toString").getDeclaringClass();
      if(decl == Object.class) {
        return null; // TODO: cache this, too
      }
      writers.insertHandler(decl, fallbackwriter);
      return fallbackwriter;
    }
    catch(NoSuchMethodException | SecurityException e) {
      return null;
    }
  }
711826
@Override
    public RecordWriter<NullWritable, WARCWritable> getRecordWriter(TaskAttemptContext context)
            throws IOException, InterruptedException {
        return new WARCWriter(context);
    }
156577
public ObjectOutputStream getWriter()
    {
        if (m_writer == null)
        {
            try {
                String filename = this.getFilename();
                OutputStream os = null;
                if (filename.indexOf(':') == -1)
                    os = new FileOutputStream(filename);
                else
                {
                    URL url = new URL(filename);    // "ftp://user01:pass1234@ftp.foo.com/README.txt;type=i"
                    URLConnection urlc = url.openConnection();
                    os = urlc.getOutputStream(); // To upload                    
                }
                m_writer = new ObjectOutputStream(os);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return m_writer;
    }
1126065
public void write(ProgressVisitor progress, ResultSet rs, File fileName, String encoding) throws SQLException, IOException {
        if (FileUtil.isExtensionWellFormated(fileName, "json")) {
            JsonEncoding jsonEncoding = JsonEncoding.UTF8;
            if (encoding != null) {
                try {
                    jsonEncoding = JsonEncoding.valueOf(encoding);
                } catch (IllegalArgumentException ex) {
                    throw new SQLException("Only UTF-8, UTF-16BE, UTF-16LE, UTF-32BE, UTF-32LE encoding is supported");
                }
            }
            FileOutputStream fos = null;
            try {
            fos = new FileOutputStream(fileName);
            int rowCount = 0;
                int type = rs.getType();
                if (type == ResultSet.TYPE_SCROLL_INSENSITIVE || type == ResultSet.TYPE_SCROLL_SENSITIVE) {
                    rs.last();
                    rowCount = rs.getRow();
                    rs.beforeFirst();
                }
                ProgressVisitor copyProgress = progress.subProcess(rowCount);   
                try ( // Read table content
                    Statement st = connection.createStatement()) {
                    JsonFactory jsonFactory = new JsonFactory();
                    JsonGenerator jsonGenerator = jsonFactory.createGenerator(new BufferedOutputStream(fos), jsonEncoding);                    
                    try {
                        ResultSetMetaData rsmd = rs.getMetaData();
                        int numColumns = rsmd.getColumnCount();
                        while (rs.next()) {
                            jsonGenerator.writeStartObject();
                            for (int i = 1; i < numColumns + 1; i++) {
                                String column_name = rsmd.getColumnName(i);
                                switch (rsmd.getColumnType(i)) {
                                    case java.sql.Types.ARRAY:
                                        Object[] values = (Object[]) rs.getArray(i).getArray();
                                        if(values !=null){
                                            jsonGenerator.writeArrayFieldStart(column_name);
                                            for (Object value : values) {
                                                jsonGenerator.writeObject(value);
                                            }
                                            jsonGenerator.writeEndArray();
                                        }
                                        break;
                                    case java.sql.Types.BIGINT:
                                        jsonGenerator.writeObjectField(column_name, rs.getLong(i));
                                        break;
                                    case java.sql.Types.REAL:
                                        jsonGenerator.writeObjectField(column_name, rs.getFloat(i));
                                        break;
                                    case java.sql.Types.BOOLEAN:
                                        jsonGenerator.writeObjectField(column_name, rs.getBoolean(i));
                                        break;
                                    case java.sql.Types.BLOB:
                                        jsonGenerator.writeObjectField(column_name, rs.getBlob(i));
                                        break;
                                    case java.sql.Types.DOUBLE:
                                        jsonGenerator.writeObjectField(column_name, rs.getDouble(i));
                                        break;
                                    case java.sql.Types.FLOAT:
                                        jsonGenerator.writeObjectField(column_name, rs.getDouble(i));
                                        break;
                                    case java.sql.Types.INTEGER:
                                        jsonGenerator.writeObjectField(column_name, rs.getInt(i));
                                        break;
                                    case java.sql.Types.NVARCHAR:
                                        jsonGenerator.writeObjectField(column_name, rs.getNString(i));
                                        break;
                                    case java.sql.Types.VARCHAR:
                                        jsonGenerator.writeObjectField(column_name, rs.getString(i));
                                        break;
                                    case java.sql.Types.CHAR:
                                        jsonGenerator.writeObjectField(column_name, rs.getString(i));
                                        break;
                                    case java.sql.Types.NCHAR:
                                        jsonGenerator.writeObjectField(column_name, rs.getNString(i));
                                        break;
                                    case java.sql.Types.LONGNVARCHAR:
                                        jsonGenerator.writeObjectField(column_name, rs.getNString(i));
                                        break;
                                    case java.sql.Types.LONGVARCHAR:
                                        jsonGenerator.writeObjectField(column_name, rs.getString(i));
                                        break;
                                    case java.sql.Types.TINYINT:
                                        jsonGenerator.writeObjectField(column_name, rs.getByte(i));
                                        break;
                                    case java.sql.Types.SMALLINT:
                                        jsonGenerator.writeObjectField(column_name, rs.getShort(i));
                                        break;
                                    case java.sql.Types.DATE:
                                        jsonGenerator.writeObjectField(column_name, rs.getDate(i));
                                        break;
                                    case java.sql.Types.TIME:
                                        jsonGenerator.writeObjectField(column_name, rs.getTime(i));
                                        break;
                                    case java.sql.Types.TIMESTAMP:
                                        jsonGenerator.writeObjectField(column_name, rs.getTimestamp(i));
                                        break;
                                    case java.sql.Types.BINARY:
                                        jsonGenerator.writeObjectField(column_name, rs.getBytes(i));
                                        break;
                                    case java.sql.Types.VARBINARY:
                                        jsonGenerator.writeObjectField(column_name, rs.getBytes(i));
                                        break;
                                    case java.sql.Types.LONGVARBINARY:
                                        jsonGenerator.writeObjectField(column_name, rs.getBinaryStream(i));
                                        break;
                                    case java.sql.Types.BIT:
                                        jsonGenerator.writeObjectField(column_name, rs.getBoolean(i));
                                        break;
                                    case java.sql.Types.CLOB:
                                        jsonGenerator.writeObjectField(column_name, rs.getClob(i));
                                        break;
                                    case java.sql.Types.NUMERIC:
                                        jsonGenerator.writeObjectField(column_name, rs.getBigDecimal(i));
                                        break;
                                    case java.sql.Types.DECIMAL:
                                        jsonGenerator.writeObjectField(column_name, rs.getBigDecimal(i));
                                        break;
                                    case java.sql.Types.DATALINK:
                                        jsonGenerator.writeObjectField(column_name, rs.getURL(i));
                                        break;
                                    case java.sql.Types.REF:
                                        jsonGenerator.writeObjectField(column_name, rs.getRef(i));
                                        break;
                                    case java.sql.Types.STRUCT:
                                        jsonGenerator.writeObjectField(column_name, rs.getObject(i));
                                        break;
                                    case java.sql.Types.DISTINCT:
                                        jsonGenerator.writeObjectField(column_name, rs.getObject(i));
                                        break;
                                    case java.sql.Types.JAVA_OBJECT:
                                        jsonGenerator.writeObjectField(column_name, rs.getObject(i));
                                        break;
                                    default:
                                        jsonGenerator.writeObjectField(column_name, rs.getString(i));
                                        break;
                                }
                            }
                            jsonGenerator.writeEndObject();

                            copyProgress.endStep();
                        }
                        copyProgress.endOfProgress();
                        jsonGenerator.flush();
                        jsonGenerator.close();
                    } finally {
                        rs.close();
                    }
                }
            
        } catch (FileNotFoundException ex) {
            throw new SQLException(ex);

        } finally {
            try {
                if (fos != null) {
                    fos.close();
                }
                } catch (IOException ex) {
                    throw new SQLException(ex);
                }
            }
        } else {
            throw new SQLException("Only .json extension is supported");
        }
    }
108568
private void processRecords(Hadoop2RecordType recType, JSONObject eventDetails)
      throws JSONException {

    switch (recType) {
    case JobFinished:
      // this setting is needed since the job history file is missing
      // the jobStatus field in the JOB_FINISHED event
      this.jobStatus = JOB_STATUS_SUCCEEDED;
    case JobInfoChange:
    case JobInited:
    case JobPriorityChange:
    case JobStatusChanged:
    case JobQueueChange:
    case JobSubmitted:
    case JobUnsuccessfulCompletion:
      Put pJob = new Put(this.jobKeyBytes);
      iterateAndPreparePuts(eventDetails, pJob, recType);
      this.jobPuts.add(pJob);
      break;

    case AMStarted:
      byte[] amAttemptIdKeyBytes =
          getAMKey(AM_ATTEMPT_PREFIX, eventDetails.getString(APPLICATION_ATTEMPTID));
      // generate a new put per AM Attempt
      Put pAM = new Put(amAttemptIdKeyBytes);
      pAM.addColumn(Constants.INFO_FAM_BYTES, Constants.RECORD_TYPE_COL_BYTES,
        Bytes.toBytes(RecordTypes.Task.toString()));
      iterateAndPreparePuts(eventDetails, pAM, recType);
      taskPuts.add(pAM);
      break;

    case MapAttemptFinished:
      byte[] taskMAttemptIdKeyBytes =
          getTaskKey(TASK_ATTEMPT_PREFIX, this.jobNumber, eventDetails.getString(ATTEMPTID));
      Put pMTaskAttempt = new Put(taskMAttemptIdKeyBytes);
      pMTaskAttempt.addColumn(Constants.INFO_FAM_BYTES, Constants.RECORD_TYPE_COL_BYTES,
        Bytes.toBytes(RecordTypes.MapAttempt.toString()));
      iterateAndPreparePuts(eventDetails, pMTaskAttempt, recType);
      this.taskPuts.add(pMTaskAttempt);
      break;

    case ReduceAttemptFinished:
      byte[] taskRAttemptIdKeyBytes =
          getTaskKey(TASK_ATTEMPT_PREFIX, this.jobNumber, eventDetails.getString(ATTEMPTID));
      Put pRTaskAttempt = new Put(taskRAttemptIdKeyBytes);
      pRTaskAttempt.addColumn(Constants.INFO_FAM_BYTES, Constants.RECORD_TYPE_COL_BYTES,
        Bytes.toBytes(RecordTypes.ReduceAttempt.toString()));
      iterateAndPreparePuts(eventDetails, pRTaskAttempt, recType);
      this.taskPuts.add(pRTaskAttempt);
      break;

    case TaskAttemptFinished:
    case TaskAttemptStarted:
    case TaskAttemptUnsuccessfulCompletion:
      byte[] taskAttemptIdKeyBytes =
          getTaskKey(TASK_ATTEMPT_PREFIX, this.jobNumber, eventDetails.getString(ATTEMPTID));
      Put pTaskAttempt = new Put(taskAttemptIdKeyBytes);
      pTaskAttempt.addColumn(Constants.INFO_FAM_BYTES, Constants.RECORD_TYPE_COL_BYTES,
        Bytes.toBytes(RecordTypes.Task.toString()));
      iterateAndPreparePuts(eventDetails, pTaskAttempt, recType);
      taskPuts.add(pTaskAttempt);
      break;

    case TaskFailed:
    case TaskStarted:
    case TaskUpdated:
    case TaskFinished:
      byte[] taskIdKeyBytes =
          getTaskKey(TASK_PREFIX, this.jobNumber, eventDetails.getString(TASKID));
      Put pTask = new Put(taskIdKeyBytes);
      pTask.addColumn(Constants.INFO_FAM_BYTES, Constants.RECORD_TYPE_COL_BYTES,
        Bytes.toBytes(RecordTypes.Task.toString()));
      iterateAndPreparePuts(eventDetails, pTask, recType);
      taskPuts.add(pTask);
      break;
    default:
      LOG.error("Check if recType was modified and has new members?");
      throw new ProcessingException("Check if recType was modified and has new members? " + recType);
    }
  }
536501
@Override
    public void write(final Writer writer) throws ConfigurationException, IOException {
        Preconditions.checkNotNull(writer);

        @SuppressWarnings("unchecked") final
        HashMap<String, Object> settings = (HashMap<String, Object>) fromNode(this.getModel().getInMemoryRepresentation());
        this.mapper.writerWithDefaultPrettyPrinter().writeValue(writer, settings);
    }
1094110
public static S3RecoverableWriter writer(
			final FileSystem fs,
			final FunctionWithException<File, RefCountedFile, IOException> tempFileCreator,
			final S3AccessHelper s3AccessHelper,
			final Executor uploadThreadPool,
			final long userDefinedMinPartSize,
			final int maxConcurrentUploadsPerStream) {

		checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE);

		final S3RecoverableMultipartUploadFactory uploadFactory =
				new S3RecoverableMultipartUploadFactory(
						fs,
						s3AccessHelper,
						maxConcurrentUploadsPerStream,
						uploadThreadPool,
						tempFileCreator);

		return new S3RecoverableWriter(s3AccessHelper, uploadFactory, tempFileCreator, userDefinedMinPartSize);
	}
72848
static public Class<? extends Writable> getSequenceFileOutputValueClass(JobConf conf) { 
    return conf.getClass("mapred.seqbinary.output.value.class", 
                         conf.getOutputValueClass().asSubclass(Writable.class),
                         Writable.class);
  }
777217
@Override
    public synchronized void writeRecord(String record) {
        long length = record.length() + LoggingConstants.nlen;
        PrintStream ps = getPrintStream(length);
        ps.println(record);
        if (ps.checkError()) {
            setStreamStatus(StreamStatus.CLOSED, null, null, DummyOutputStream.psInstance);
            // to avoid junit test to print an error message
            if (System.getProperty("test.classesDir") == null && System.getProperty("test.buildDir") == null) {
                System.err.println(Tr.formatMessage(getTc(), "FAILED_TO_WRITE_LOG", new Object[] { getPrimaryFile().getAbsolutePath() }));
            }
        }
    }
1249563
def _get_writer(self, i):
        
        if self._recording_dir is None:
            return None
        if self._log_n is None:
            self._log_n = [None] * self.n
        if self._log_n[i] is None:
            self._log_n[i] = RecordingWriter(self._recording_dir, self._instance_id, i, async_write=self._async_write)
        return self._log_n[i]
360636
private static JobConf createJobConf(Configuration conf, boolean useFastCopy) {
    Class<? extends InputFormat> inputFormat =
      (useFastCopy) ? FastCopyInputFormat.class : CopyInputFormat.class;
    JobConf jobconf = new JobConf(conf, DistCp.class);
    jobconf.setJobName(NAME);

    // turn off speculative execution, because DFS doesn't handle
    // multiple writers to the same file.
    jobconf.setReduceSpeculativeExecution(false);
    jobconf.setMapOutputKeyClass(FilePairComparable.class);
    jobconf.setMapOutputValueClass(Text.class);
    jobconf.setOutputKeyClass(FilePairComparable.class);
    jobconf.setOutputValueClass(Text.class);

    jobconf.setInputFormat(inputFormat);
    jobconf.setMapperClass(CopyFilesTask.class);
    jobconf.setReducerClass(CopyFilesTask.class);
      
    // Prevent the reducer from starting until all maps are done.
    jobconf.setInt("mapred.job.rushreduce.reduce.threshold", 0);
    jobconf.setFloat("mapred.reduce.slowstart.completed.maps", 1.0f);
    
    return jobconf;
  }
1342476
public Record getRecord(String strFileName)
    {
        Record record = super.getRecord(strFileName);
        if (record == null)
            if (Script.SCRIPT_FILE.equalsIgnoreCase(strFileName))
                record = new Script(this);
        return record;
    }