Implements Spark with Alluxio integration checker.

@param sc current JavaSparkContext
@param reportWriter save user-facing messages to a generated file
@return performIntegrationChecks results
14413
private Status run(JavaSparkContext sc, PrintWriter reportWriter, AlluxioConfiguration conf) {
    // Check whether Spark driver can recognize Alluxio classes and filesystem
    Status driverStatus = CheckerUtils.performIntegrationChecks();
    String driverAddress = sc.getConf().get("spark.driver.host");
    switch (driverStatus) {
      case FAIL_TO_FIND_CLASS:
        reportWriter.printf("Spark driver: %s failed to recognize Alluxio classes.%n%n",
            driverAddress);
        return driverStatus;
      case FAIL_TO_FIND_FS:
        reportWriter.printf("Spark driver: %s failed to recognize Alluxio filesystem.%n%n",
            driverAddress);
        return driverStatus;
      default:
        reportWriter.printf("Spark driver: %s can recognize Alluxio filesystem.%n%n",
            driverAddress);
        break;
    }

    if (!CheckerUtils.supportAlluxioHA(reportWriter, conf)) {
      return Status.FAIL_TO_SUPPORT_HA;
    }

    return runSparkJob(sc, reportWriter);
  }
1214717
@Override
    public void writeFragmentTo(MwsWriter w) {
        w.write("ListOrdersResult", listOrdersResult);
        w.write("ResponseMetadata", responseMetadata);
    }
1306246
@SuppressWarnings("rawtypes")
    public static long importCSV(final File file, final Connection conn, final String insertSQL, final Map<String, ? extends Type> columnTypeMap)
            throws UncheckedSQLException, UncheckedIOException {
        return importCSV(file, 0, Long.MAX_VALUE, conn, insertSQL, 200, 0, columnTypeMap);
    }
312106
@Override
    public WRITE_ACTION writeOrAddToQ(H2WriteQEntry entry) throws FlowControlException {
        synchronized (qSync) {

            if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                Tr.debug(tc, "process write entry with qStatus: " + qStatus + " entry: " + entry.hashCode());
            }
            if (entry.getFrameType() == FrameTypes.DATA) {
                decreaseConnectionWindowUpdateWriteLimit(entry.getPayloadLength());
            }

            if (qStatus == Q_STATUS.NOT_IN_USE) {
                // perform the write on this thread, outside the sync block
                qStatus = Q_STATUS.BYPASSED;
                entry.setServicedOnQ(false);

            } else if ((qStatus == Q_STATUS.ACTIVE) || (qStatus == Q_STATUS.STAND_BY)) {
                // queue is in use, so add this write entry to the queue/tree and leave
                entry.setServicedOnQ(true);
                addEntryToQ(entry);
                return WRITE_ACTION.QUEUED;

            } else if (qStatus == Q_STATUS.BYPASSED) {
                // queue is not in use, but needs to be now, since we are want to write with a write outstanding.
                entry.setServicedOnQ(true);
                qStatus = Q_STATUS.STAND_BY;
                addEntryToQ(entry);
                // wait for outstanding request to complete before allowing the queue thread to make more write requests
                try {
                    if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                        Tr.debug(tc, "waiting to start write Q thread");
                    }
                    qSync.wait();
                    qStatus = Q_STATUS.ACTIVE;
                    if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                        Tr.debug(tc, "start write Q thread");
                    }
                    startQThread();
                    return WRITE_ACTION.QUEUED;
                } catch (InterruptedException e) {
                    // TODO: handle this somehow
                }

            } else if ((qStatus == Q_STATUS.QUIT) || (qStatus == Q_STATUS.FINISHED)) {
                if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                    Tr.debug(tc, "do not process write - Q told to quit");
                }
                return WRITE_ACTION.CONNECTION_QUIT;
            }
        } // end synchronized

        // Determined from above that we are to handle this write on the callers thread
        if (!(entry.getServicedOnQ())) {
            if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                Tr.debug(tc, "perform write on caller's thread");
            }
            return writeEntry(entry);
        }

        // should never get here
        if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
            Tr.debug(tc, "uh-oh we're confused: WRITE_ACTION.CONFUSED");
        }
        return WRITE_ACTION.CONFUSED;
    }
1333455
public final ControlAreYouFlushed createNewControlAreYouFlushed() throws MessageCreateFailedException {
    if (TraceComponent.isAnyTracingEnabled() && tc.isEntryEnabled()) SibTr.entry(tc, "createNewControlAreYouFlushed");
    ControlAreYouFlushed msg = null;
    try {
      msg = new ControlAreYouFlushedImpl(MfpConstants.CONSTRUCTOR_NO_OP);
    }
    catch (MessageDecodeFailedException e) {
      /* No need to FFDC this as JsMsgObject will already have done so */
      // No FFDC code needed
      throw new MessageCreateFailedException(e);
    }
    if (TraceComponent.isAnyTracingEnabled() && tc.isEntryEnabled()) SibTr.exit(tc, "createNewControlAreYouFlushed");
    return msg;
  }
737420
public static int cusparseXcoo2csr(
        cusparseHandle handle, 
        Pointer cooRowInd, 
        int nnz, 
        int m, 
        Pointer csrSortedRowPtr, 
        int idxBase)
    {
        return checkResult(cusparseXcoo2csrNative(handle, cooRowInd, nnz, m, csrSortedRowPtr, idxBase));
    }
106989
@Override
    public void complete(com.ibm.wsspi.channelfw.VirtualConnection vc, com.ibm.wsspi.tcpchannel.TCPWriteRequestContext wsc) {
        if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled())  
            Tr.debug(tc, "complete called : " + vc + "WriteListener enabled: " + this._wl);

        synchronized(_upgradeOut) {

            if (null == vc) {
                return;
            }

            if (_upgradeOut.getBufferHelper().isOutputStream_closed()) {

                if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled())
                    Tr.debug(tc, "complete , outputStream closed ignoring complete : " + vc);

                return;
            }

            // clean up any request state we had on this thread.
            WebContainerRequestState reqState = WebContainerRequestState.getInstance(true);
            reqState.init();
            try{    
                _upgradeOut.getBufferHelper().writeRemainingToBuffers();                 

            } catch (Exception e) {
                if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()){  
                    Tr.debug(tc, "exception during writeBuffers" + e.toString());
                }
                this.error(vc, e);
                return;
            }


            // rest must be written now ,
            // this case will only be true if write which went aysnc was
            // from println, now we have to write crlf
            if ( _upgradeOut.getBufferHelper().isInternalReady() && _upgradeOut.getBufferHelper().write_crlf_pending) {
                if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled())
                    Tr.debug(tc, "write CRLF bytes  , WriteListener enabled: " + this._wl);
                try {
                    reqState.setAttribute("com.ibm.ws.webcontainer.upgrade.WriteAllowedonThisThread", true);
                    reqState.setAttribute("com.ibm.ws.webcontainer.upgrade.CRLFWriteinPorgress", true);
                    _upgradeOut.getBufferHelper().writeCRLFIfNeeded();
                } catch (IOException e) {
                    if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) 
                        Tr.debug(tc, " Exception during write CRLF bytes: " + e);

                    _upgradeOut.getBufferHelper().write_crlf_pending = false;
                    this.error(vc, e);
                    return;
                }
            }

            if( _upgradeOut.getBufferHelper().isInternalReady()){
                WebContainerRequestState.getInstance(true).setAttribute("com.ibm.ws.webcontainer.upgrade.WriteAllowedonThisThread", true);

                _upgradeOut.getBufferHelper().setReadyForApp(true);

                // if close was in progress
                if(_upgradeOut.getBufferHelper().isOutputStream_close_initiated_but_not_Flush_ready()){
                    if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()){
                        Tr.debug(tc, "complete , In process of closing outputStream, no more data is left to write, call close");
                    }
                    try {
                        if(_upCon.isOutputStream_CloseStartedFromWC()){
                            //at this point inputstream and handler are already closed
                            _upCon.closeOutputandConnection();
                        }
                        else{
                            _upgradeOut.close();
                        }
                    } catch (Exception e) {
                        if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled())
                            Tr.debug(tc, "complete , stack exception" + e.toString());

                        this.error(vc, e);
                    }

                }// no close in progress now check if onWP needs to be called.
                else{
                    if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()){  
                        Tr.debug(tc, "WriteListener enabled: " + this._wl + " ,status_not_ready_checked -->" + _upgradeOut.getBufferHelper().status_not_ready_checked);
                    }
                    if (_upgradeOut.getBufferHelper().status_not_ready_checked) {
                        try{
                            _upgradeOut.getBufferHelper().status_not_ready_checked = false;

                            // Push the original thread's context onto the current thread,
                            // also save off the current thread's context
                            _contextManager.pushContextData();
                            if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()){  
                                Tr.debug(tc, "Calling user's WriteListener onWritePossible");
                            }
                            // Call into the user's WriteListener to indicate more data cane
                            // be written
                            _wl.onWritePossible();
                        } catch (Exception e) {
                            if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()){  
                                Tr.debug(tc, "stack exception" + e.toString());
                            }
                            this.error(vc, e);
                        } finally {
                            // Revert back to the thread's current context
                            _contextManager.popContextData();
                        }
                    }
                    else{
                        if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()){  
                            Tr.debug(tc, "WriteListener enabled: " 
                                            + this._wl + " .onWritePossible will be skipped as isReady has not been checked since write went async.");
                        }
                    }
                }
            }
        }
    }
89416
@SuppressWarnings("deprecation")
  public static void writeFlowDetails(JsonGenerator jsonGenerator, Flow aFlow,
      DetailLevel selectedSerialization, Predicate<String> includeFilter)
      throws JsonGenerationException, IOException {
    jsonGenerator.writeStartObject();
    // serialize the FlowKey object
    filteredWrite("flowKey", includeFilter, aFlow.getFlowKey(), jsonGenerator);

    // serialize individual members of this class
    filteredWrite("flowName", includeFilter, aFlow.getFlowName(), jsonGenerator);
    filteredWrite("userName", includeFilter, aFlow.getUserName(), jsonGenerator);
    filteredWrite("jobCount", includeFilter, aFlow.getJobCount(), jsonGenerator);
    filteredWrite("totalMaps", includeFilter, aFlow.getTotalMaps(), jsonGenerator);
    filteredWrite("totalReduces", includeFilter, aFlow.getTotalReduces(), jsonGenerator);
    filteredWrite("mapFileBytesRead", includeFilter, aFlow.getMapFileBytesRead(), jsonGenerator);
    filteredWrite("mapFileBytesWritten", includeFilter, aFlow.getMapFileBytesWritten(), jsonGenerator);
    filteredWrite("reduceFileBytesRead", includeFilter, aFlow.getReduceFileBytesRead(), jsonGenerator);
    filteredWrite("hdfsBytesRead", includeFilter, aFlow.getHdfsBytesRead(), jsonGenerator);
    filteredWrite("hdfsBytesWritten", includeFilter, aFlow.getHdfsBytesWritten(), jsonGenerator);
    filteredWrite("mapSlotMillis", includeFilter, aFlow.getMapSlotMillis(), jsonGenerator);
    filteredWrite("reduceSlotMillis", includeFilter, aFlow.getReduceSlotMillis(), jsonGenerator);
    filteredWrite("megabyteMillis", includeFilter, aFlow.getMegabyteMillis(), jsonGenerator);
    filteredWrite("cost", includeFilter, aFlow.getCost(), jsonGenerator);
    filteredWrite("reduceShuffleBytes", includeFilter, aFlow.getReduceShuffleBytes(), jsonGenerator);
    filteredWrite("duration", includeFilter, aFlow.getDuration(), jsonGenerator);
    filteredWrite("wallClockTime", includeFilter, aFlow.getWallClockTime(), jsonGenerator);
    filteredWrite("cluster", includeFilter, aFlow.getCluster(), jsonGenerator);
    filteredWrite("appId", includeFilter, aFlow.getAppId(), jsonGenerator);
    filteredWrite("runId", includeFilter, aFlow.getRunId(), jsonGenerator);
    filteredWrite("version", includeFilter, aFlow.getVersion(), jsonGenerator);
    filteredWrite("hadoopVersion", includeFilter, aFlow.getHadoopVersion(), jsonGenerator);
    if (selectedSerialization == SerializationContext.DetailLevel.EVERYTHING) {
      filteredWrite("submitTime", includeFilter, aFlow.getSubmitTime(), jsonGenerator);
      filteredWrite("launchTime", includeFilter, aFlow.getLaunchTime(), jsonGenerator);
      filteredWrite("finishTime", includeFilter, aFlow.getFinishTime(), jsonGenerator);
    }
    filteredWrite(Constants.HRAVEN_QUEUE, includeFilter, aFlow.getQueue(), jsonGenerator);
    filteredWrite("counters", includeFilter, aFlow.getCounters(), jsonGenerator);
    filteredWrite("mapCounters", includeFilter, aFlow.getMapCounters(), jsonGenerator);
    filteredWrite("reduceCounters", includeFilter, aFlow.getReduceCounters(), jsonGenerator);

    // if flag, include job details
    if ((selectedSerialization == SerializationContext.DetailLevel.FLOW_SUMMARY_STATS_WITH_JOB_STATS)
        || (selectedSerialization == SerializationContext.DetailLevel.EVERYTHING)) {
      jsonGenerator.writeFieldName("jobs");
      jsonGenerator.writeObject(aFlow.getJobs());
    }
    jsonGenerator.writeEndObject();

  }
1151419
private void cmdExportResultActionPerformed(java.awt.event.ActionEvent evt) {// GEN-FIRST:event_buttonSaveResultsActionPerformed
		JFileChooser fileChooser = new JFileChooser(System.getProperty("user.dir"));
		fileChooser.setDialogTitle("Export to...");
		int response = fileChooser.showSaveDialog(this);
		if (response == JFileChooser.APPROVE_OPTION) {
			File targetFile = fileChooser.getSelectedFile();
			final String fileLocation = targetFile.getPath();
			if (canWrite(targetFile)) {
				Thread thread = new Thread() {
					public void run() {						
						getOBDASaveQueryToFileAction().run(fileLocation);
					}
				};
				thread.start();
			}
		}
	}
97011
public static void writeSchema(String outputPath, Schema schema, JavaSparkContext sc) throws IOException {
        writeStringToFile(outputPath, schema.toString(), sc);
    }
112380
public static long exportCSV(final Writer out, final ResultSet rs, final long offset, final long count, final boolean writeTitle, final boolean quoted)
            throws UncheckedSQLException, UncheckedIOException {
        return exportCSV(out, rs, null, offset, count, writeTitle, quoted);
    }
1187327
public static void write(final PrintWriter writer, final UicStats stats) {
		writer.println("<dl>");

		writer.print("<dt>Total root wcomponents found in UIC</dt>");
		writer.println("<dd>" + stats.getRootWCs().size() + "</dd>");

		writer.print("<dt>Size of UIC (by serialization)</dt>");
		writer.println("<dd>" + stats.getOverallSerializedSize() + "</dd>");

		writer.print("<dt>UI</dt>");
		writer.println("<dd>" + stats.getUI().getClass().getName() + "</dd>");

		writer.println("</dl>");

		for (Iterator<WComponent> it = stats.getWCsAnalysed(); it.hasNext();) {
			WComponent comp = it.next();
			Map<WComponent, UicStats.Stat> treeStats = stats.getWCTreeStats(comp);

			writer.println("<br /><strong>Analysed component:</strong> " + comp);
			writer.println("<br /><strong>Number of components in tree:</strong> " + treeStats.size());

			writeHeader(writer);
			writeProfileForTree(writer, treeStats);
			writeFooter(writer);
		}
	}
337717
public void sendError(int sc) throws IOException
    {
        String message;
        RequestProcessor ref = getCurrentServletReference();
        if (ref != null)
        {
            sendError(sc, MessageFormat.format(nls.getString("[{0}].reported.an.error", "[{0}] reported an error"), new Object[] { ref.getName()}));
        }
        else
        {
            sendError(
                      sc,
                      MessageFormat.format(nls.getString("[{0}].reported.an.error", "[{0}] reported an error"), new Object[] { getWebApp().getConfiguration().getDisplayName()}));
        }
    }
532928
public void write(PrintWriter out, boolean includeDoc) {
        // getValue() will update the timestamp (if relevant)
        T localValue = getValue();
        if (includeDoc) {
            out.println();
            Long lastUpdate = getLastUpdated();
            if (hasDoc() || lastUpdate != null) {
                out.print("# ");
                if (hasDoc()) {
                    out.print(makePropertiesSafe(getDoc(), null));
                }
                if (lastUpdate != null) {
                    out.print(" (last update: " + lastUpdate + ")");
                }
                out.println();
            }
        }
        out.println(toString());
    }
1260736
void writeReport() throws IOException {
		StringBuilder builder = new StringBuilder();
		for (DumpProcessingAction action : this.clientConfiguration
				.getActions()) {
			if (this.clientConfiguration.getReportFileName() != null) {
				builder.append(action.getActionName());
				builder.append(": ");
				if (action.isReady()) {
					builder.append(action.getReport());
				} else {
					builder.append("Action was not executed.");
				}
				builder.append(System.getProperty("line.separator"));
			} else {
				logger.info(action.getActionName() + ": " + action.getReport());
			}
		}
		if (this.clientConfiguration.getReportFileName() != null) {
			Path outputDirectory = Paths.get(
					this.clientConfiguration.getReportFileName()).getParent();
			if (outputDirectory == null) {
				outputDirectory = Paths.get(".");
			}
			DirectoryManager dm = DirectoryManagerFactory
					.createDirectoryManager(outputDirectory, false);
			OutputStream out = dm.getOutputStreamForFile(Paths
					.get(this.clientConfiguration.getReportFileName())
					.getFileName().toString());
			out.write(builder.toString().getBytes(StandardCharsets.UTF_8));
			out.close();
		}
	}
1193830
public alluxio.grpc.ConfigCheckReportOrBuilder getReportOrBuilder() {
    return report_ == null ? alluxio.grpc.ConfigCheckReport.getDefaultInstance() : report_;
  }
1249065
public void exportObjects2CSV(List<?> data, Class clazz, boolean isWriteHeader, OutputStream os)
            throws Excel4JException {

        try {
            Writer writer = new OutputStreamWriter(os);
            exportCSVByMapHandler(data, clazz, isWriteHeader, writer);
        } catch (Excel4JException | IOException e) {
            throw new Excel4JException(e);
        }
    }