checks and increments the number of runs for this app aggregation. no need
to retry, since another map task may have updated it in the mean time
@throws IOException
766993
boolean incrNumberRuns(List<Cell> column, Table aggTable,
      AppAggregationKey appAggKey) throws IOException {

    /*
     * check if this is the very first insert for numbers for this app in that
     * case, there will no existing column/value for number of run in info col
     * family null signifies non existence of the column for {@link
     * Table.checkAndPut}
     */
    long expectedValueBeforePut = 0L;
    if (column.size() > 0) {
      try {
        expectedValueBeforePut = Bytes.toLong(column.get(0).getValue());
      } catch (NumberFormatException e) {
        LOG.error(
            "Could not read existing value for number of runs during aggregation"
                + appAggKey.toString());
        return false;
      }
    }

    byte[] rowKey = aggConv.toBytes(appAggKey);

    long insertValue = 1L;
    byte[] expectedValueBeforePutBytes = null;
    if (expectedValueBeforePut != 0L) {
      insertValue = 1 + expectedValueBeforePut;
      expectedValueBeforePutBytes = Bytes.toBytes(expectedValueBeforePut);
    }
    byte[] insertValueBytes = Bytes.toBytes(insertValue);

    if (LOG.isTraceEnabled()) {
      LOG.trace(" before statusCheckAndPut " + insertValue + " "
          + expectedValueBeforePut);
    }
    return executeCheckAndPut(aggTable, rowKey, expectedValueBeforePutBytes,
        insertValueBytes, AggregationConstants.INFO_FAM_BYTES,
        AggregationConstants.NUMBER_RUNS_BYTES);
  }
842744
public void collect(Object writableComparable, Object writable) throws IOException {
    if (hadoopFlowProcess instanceof HadoopFlowProcess)
      ((HadoopFlowProcess) hadoopFlowProcess).getReporter().progress();

    writer.write(writableComparable, writable);
  }
107498
protected void incrementBatchCount() throws SQLException {
        batchCount++;
        if (batchCount == batchSize /* never true for batchSize of 0 */) {
            int[] result = delegate.executeBatch();
            processResult(result);
            batchCount = 0;
        }
    }
280643
public void commit()
      throws IOException {
    if (this.tasks == null || this.tasks.isEmpty()) {
      log.warn("No tasks to be committed in container " + containerIdOptional.or(""));
      return;
    }
    Iterator<Callable<Void>> callableIterator =
        Iterators.transform(this.tasks.iterator(), new Function<Task, Callable<Void>>() {
          @Override
          public Callable<Void> apply(final Task task) {
            return new Callable<Void>() {
              @Nullable
              @Override
              public Void call()
                  throws Exception {
                task.commit();
                return null;
              }
            };
          }
        });

    try {
      List<Either<Void, ExecutionException>> executionResults =
          new IteratorExecutor<>(callableIterator, this.getTaskCommitThreadPoolSize(),
              ExecutorsUtils.newDaemonThreadFactory(Optional.of(log), Optional.of("Task-committing-pool-%d")))
              .executeAndGetResults();
      IteratorExecutor.logFailures(executionResults, log, 10);
    } catch (InterruptedException ie) {
      log.error("Committing of tasks interrupted. Aborting.");
      throw new RuntimeException(ie);
    } finally {
      persistTaskStateStore();
      if (this.cleanupCommitSteps != null) {
        for (CommitStep cleanupCommitStep : this.cleanupCommitSteps) {
          log.info("Executing additional commit step.");
          cleanupCommitStep.execute();
        }
      }
    }
  }
572130
public void doUpdates(MetricsContext unused) {
    // In case of running in LocalMode tracker == null
    if (tracker != null) {
      synchronized (tracker) {
        synchronized (this) {
          numRunningMaps = 0;
          numRunningReduces = 0;

          numWaitingMaps = 0;
          numWaitingReduces = 0;
          numTasksInMemory = 0;

          List<JobInProgress> jobs = tracker.getRunningJobs();
          for (JobInProgress jip : jobs) {
            for (TaskInProgress tip : jip.maps) {
              if (tip.isRunning()) {
                numRunningMaps++;
              } else if (tip.isRunnable()) {
                numWaitingMaps++;
              }
            }
            for (TaskInProgress tip : jip.reduces) {
              if (tip.isRunning()) {
                numRunningReduces++;
              } else if (tip.isRunnable()) {
                numWaitingReduces++;
              }

            }
            numTasksInMemory += jip.getTasks(TaskType.MAP).length;
            numTasksInMemory += jip.getTasks(TaskType.REDUCE).length;
          }

          // Get tracker metrics
          numTrackersDead = tracker.getDeadNodes().size();
          ClusterStatus cs = tracker.getClusterStatus(false);
          numTrackersExcluded = cs.getNumExcludedNodes();
        }
      }
    }
    synchronized (this) {
      metricsRecord.setMetric("map_slots", numMapSlots);
      metricsRecord.setMetric("reduce_slots", numReduceSlots);
      metricsRecord.incrMetric("blacklisted_maps", numBlackListedMapSlots);
      metricsRecord.incrMetric("blacklisted_reduces",
          numBlackListedReduceSlots);
      metricsRecord.incrMetric("jobs_submitted", numJobsSubmitted);
      metricsRecord.incrMetric("jobs_completed", numJobsCompleted);
      metricsRecord.setMetric("waiting_maps", numWaitingMaps);
      metricsRecord.setMetric("waiting_reduces", numWaitingReduces);

      metricsRecord.incrMetric("reserved_map_slots", numReservedMapSlots);
      metricsRecord.incrMetric("reserved_reduce_slots", numReservedReduceSlots);
      metricsRecord.incrMetric("occupied_map_slots", numOccupiedMapSlots);
      metricsRecord.incrMetric("occupied_reduce_slots", numOccupiedReduceSlots);

      metricsRecord.incrMetric("jobs_failed", numJobsFailed);
      metricsRecord.incrMetric("jobs_killed", numJobsKilled);

      metricsRecord.incrMetric("jobs_preparing", numJobsPreparing);
      metricsRecord.incrMetric("jobs_running", numJobsRunning);

      metricsRecord.setMetric("running_maps", numRunningMaps);
      metricsRecord.setMetric("running_reduces", numRunningReduces);

      metricsRecord.setMetric("num_tasks_in_memory", numTasksInMemory);

      metricsRecord.setMetric("trackers", numTrackers);
      metricsRecord.setMetric("trackers_blacklisted", numTrackersBlackListed);
      metricsRecord.setMetric("trackers_decommissioned",
          numTrackersDecommissioned);
      metricsRecord.setMetric("trackers_excluded", numTrackersExcluded);
      metricsRecord.setMetric("trackers_dead", numTrackersDead);

      metricsRecord.incrMetric("num_launched_jobs", numJobsLaunched);
      metricsRecord.incrMetric("total_submit_time", totalSubmitTime);

      aggregateJobStats.incrementMetricsAndReset(metricsRecord);

      // Update additional speculation stats for measuring the performance
      // of different kinds of speculation
      for (SpecStats.TaskType taskType : SpecStats.TaskType.values()) {
        for (SpecStats.SpecType specType : SpecStats.SpecType.values()) {
          for(SpecStats.StatType statType : SpecStats.StatType.values()) {
            String key = "speculation_by_" + specType.toString().toLowerCase()  
                + "_" + "rate_" + taskType.toString().toLowerCase() + "_" + 
                statType.toString().toLowerCase();
            long value = specStats.getStat(taskType, specType, statType);
            metricsRecord.setMetric(key, value);
          }
        }
      }

      
      for (Group group: countersToMetrics) {
        String groupName = group.getName();
        for (Counter counter : group) {
          String name = groupName + "_" + counter.getName();
          name = name.replaceAll("[^a-zA-Z_]", "_").toLowerCase();
          metricsRecord.incrMetric(name, counter.getValue());
        }
      }
      clearCounters();

      numJobsSubmitted = 0;
      numJobsCompleted = 0;
      numWaitingMaps = 0;
      numWaitingReduces = 0;
      numBlackListedMapSlots = 0;
      numBlackListedReduceSlots = 0;

      numReservedMapSlots = 0;
      numReservedReduceSlots = 0;
      numOccupiedMapSlots = 0;
      numOccupiedReduceSlots = 0;

      numJobsFailed = 0;
      numJobsKilled = 0;

      numJobsPreparing = 0;
      numJobsRunning = 0;

      numRunningMaps = 0;
      numRunningReduces = 0;

      numTrackers = 0;
      numTrackersBlackListed = 0;

      totalSubmitTime = 0;
      numJobsLaunched = 0;
    }
    metricsRecord.update();
  }
1208169
@Override
    public void run() {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {

        }
        //TODO replace me with a timer task
        while (running) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            if (needsDataRefresh) {
                if (System.currentTimeMillis() - lastMovement > 500) {
                    generateMap();
                    needsDataRefresh = false;
                }
            }
        }
    }
1192661
public void incrementAll(CounterMap<F, S> other) {
        for (Map.Entry<F, Counter<S>> entry : other.maps.entrySet()) {
            F key = entry.getKey();
            Counter<S> innerCounter = entry.getValue();
            for (Map.Entry<S, AtomicDouble> innerEntry : innerCounter.entrySet()) {
                S value = innerEntry.getKey();
                incrementCount(key, value, innerEntry.getValue().get());
            }
        }
    }
426800
@Override
    public final void run() {
        try {
            registry_.updateCollection();

            final long t0 = System.nanoTime();

            processors_.forEach(p -> {
                try {
                    run_implementation_(p);
                } catch (Exception ex) {
                    logger.log(Level.SEVERE, p.getClass().getName() + " failed to run properly, some or all metrics may be missed this cycle", ex);
                }
            });

            final long t_processor = System.nanoTime();
            registry_.updateProcessorDuration(Duration.millis(TimeUnit.NANOSECONDS.toMillis(t_processor - t0)));
        } catch (Throwable t) {
            /*
             * We catch any and all throwables.
             * If we don't and let an exception or error escape,
             * the scheduled executor service will _silently_ drop our task.
             */
            logger.log(Level.SEVERE, "failed to perform collection", t);
        }
    }
1362076
@Override
  public void run() {
    MultipleObjectsBundle data = generator.loadData();
    if(LOG.isVerbose()) {
      LOG.verbose("Writing output ...");
    }
    try {
      if(outputFile.exists() && LOG.isVerbose()) {
        LOG.verbose("The file " + outputFile + " already exists, " + "the generator result will be APPENDED.");
      }

      try (OutputStreamWriter outStream = new FileWriter(outputFile, true)) {
        writeClusters(outStream, data);
      }
    }
    catch(IOException e) {
      throw new AbortException("IO Error in data generator.", e);
    }
    if(LOG.isVerbose()) {
      LOG.verbose("Done.");
    }
  }
703272
void increment() {
        long id = rand.nextInt(config.tuples);
        long toIncrement = rand.nextInt(5); // 0 - 4

        try {
            client.callProcedure(new CMCallback(), "Increment", toIncrement, id);
        }
        catch (IOException e) {
            // This is not ideal error handling for production, but should be
            // harmless in a benchmark like this
            try { Thread.sleep(50); } catch (Exception e2) {}
        }
    }
628738
public int countReduceTasks() {
    int reduceCount = 0;
    for (Iterator<TaskStatus> it = taskReports.iterator(); it.hasNext();) {
      TaskStatus ts = (TaskStatus) it.next();
      if ((!ts.getIsMap()) && isTaskRunning(ts)) {
        reduceCount++;
      }
    }
    return reduceCount;
  }
228096
public void loop(int count) throws ExecutionException, InterruptedException {
        if (count == -1) {
            while (true) {
                print(context.nextPacket());
            }
        } else {
            for (int i = 0; i < count; i++) {
                print(context.nextPacket());
            }
            executorService.shutdownNow();
        }
    }
722771
@Override
    public void run() {
        if (!context.getTopologyMetricContexts().containsKey(topologyId)) {
            LOG.warn("topology {} has been killed or has not started, skip update.", topologyId);
            return;
        }

        // double check and reset stream metrics if disabled
        if (!JStormMetrics.enableStreamMetrics) {
            topologyMetrics.set_streamMetric(new MetricInfo());
        }

        if (!JStormMetrics.CLUSTER_METRIC_KEY.equals(topologyId)) {
            updateClusterMetrics(topologyId, topologyMetrics);
        }

        // overwrite nimbus-local metrics data
        context.getMetricCache().putMetricData(topologyId, topologyMetrics);

        // below process is kind of a transaction, first we lock an empty slot, mark it as PRE_SET
        // by this time the slot is not yet ready for uploading as the upload thread looks for SET slots only
        // after all metrics data has been saved, we mark it as SET, then it's ready for uploading.
        int idx = context.getAndPresetFirstEmptyIndex();
        if (idx < 0) {
            LOG.error("Exceeding maxPendingUploadMetrics(too much metrics in local rocksdb), " +
                    "skip caching metrics data for topology:{}", topologyId);
            return;
        }

        try {
            TopologyMetricDataInfo summary = new TopologyMetricDataInfo();
            int total = 0;
            summary.topologyId = topologyId;
            summary.timestamp = timestamp;
            if (topologyId.equals(JStormMetrics.NIMBUS_METRIC_KEY) || topologyId.equals(JStormMetrics.CLUSTER_METRIC_KEY)) {
                summary.type = MetricUploader.METRIC_TYPE_TOPLOGY;
            } else {
                total += topologyMetrics.get_topologyMetric().get_metrics_size()
                        + topologyMetrics.get_componentMetric().get_metrics_size();

                int compStream = 0;
                if (topologyMetrics.is_set_compStreamMetric()) {
                    compStream = topologyMetrics.get_compStreamMetric().get_metrics_size();
                    total += compStream;
                }
                if (total > 0) {
                    int sub = topologyMetrics.get_taskMetric().get_metrics_size()
                            + topologyMetrics.get_workerMetric().get_metrics_size()
                            + topologyMetrics.get_nettyMetric().get_metrics_size()
                            + topologyMetrics.get_streamMetric().get_metrics_size();
                    if (sub > 0) {
                        total += sub;
                        summary.type = MetricUploader.METRIC_TYPE_ALL;
                    } else {
                        summary.type = MetricUploader.METRIC_TYPE_TOPLOGY;
                    }

                    LOG.debug("tp:{}, comp:{}, comp_stream:{}, task:{}, worker:{}, netty:{}, stream:{}",
                            topologyMetrics.get_topologyMetric().get_metrics_size(),
                            topologyMetrics.get_componentMetric().get_metrics_size(),
                            compStream,
                            topologyMetrics.get_taskMetric().get_metrics_size(),
                            topologyMetrics.get_workerMetric().get_metrics_size(),
                            topologyMetrics.get_nettyMetric().get_metrics_size(),
                            topologyMetrics.get_streamMetric().get_metrics_size());
                } else {
                    summary.type = MetricUploader.METRIC_TYPE_TASK;
                    total += topologyMetrics.get_taskMetric().get_metrics_size();
                }
            }

            context.getMetricCache().put(ClusterMetricsContext.PENDING_UPLOAD_METRIC_DATA_INFO + idx, summary);
            context.getMetricCache().put(ClusterMetricsContext.PENDING_UPLOAD_METRIC_DATA + idx, topologyMetrics);
            context.markSet(idx);
            LOG.debug("Put metric data to local cache, topology:{}, idx:{}, total:{}", topologyId, idx, total);
        } catch (Exception ex) {
            LOG.error("Error", ex);
            context.forceMarkUploaded(idx);
        }
    }
698566
private void populateTasks(JobDetails job) throws IOException {
    // TODO: see if we can merge common logic here with
    // populateTasks(List<Flow>)
    Table taskTable = hbaseConnection
        .getTable(TableName.valueOf(Constants.HISTORY_TASK_TABLE));
    Scan scan = getTaskScan(job.getJobKey());
    ResultScanner scanner = taskTable.getScanner(scan);
    try {
      // advance through the scanner til we pass keys matching the job
      for (Result currentResult : scanner) {
        if (currentResult == null || currentResult.isEmpty()) {
          break;
        }

        TaskKey taskKey = taskKeyConv.fromBytes(currentResult.getRow());
        TaskDetails task = new TaskDetails(taskKey);
        task.populate(currentResult.getFamilyMap(Constants.INFO_FAM_BYTES));
        job.addTask(task);
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug("Added " + job.getTasks().size() + " tasks to job "
            + job.getJobKey().toString());
      }
    } finally {
      scanner.close();
      taskTable.close();
    }
  }
242496
private void run() throws IOException, RepositoryException
   {
      // UUIDs of multiple nodes in the index
      Set<String> multipleEntries = new HashSet<String>();
      // collect all documents UUIDs
      documentUUIDs = new HashSet<String>();
      CachingMultiIndexReader reader = index.getIndexReader();
      try
      {
         for (int i = 0; i < reader.maxDoc(); i++)
         {
            if (i > 10 && i % (reader.maxDoc() / 5) == 0)
            {
               long progress = Math.round((100.0 * i) / (reader.maxDoc() * 2f));
               log.info("progress: " + progress + "%");
            }
            if (reader.isDeleted(i))
            {
               continue;
            }
            final int currentIndex = i;
            Document d = reader.document(currentIndex, FieldSelectors.UUID);
            String uuid = d.get(FieldNames.UUID);
            if (stateMgr.getItemData(uuid) != null)
            {
               if (!documentUUIDs.add(uuid))
               {
                  multipleEntries.add(uuid);
               }
            }
            else
            {
               errors.add(new NodeDeleted(uuid));
            }
         }
      }
      finally
      {
         reader.release();
      }

      // create multiple entries errors
      for (Iterator<String> it = multipleEntries.iterator(); it.hasNext();)
      {
         errors.add(new MultipleEntries(it.next()));
      }

      reader = index.getIndexReader();
      try
      {
         // run through documents again and check parent
         for (int i = 0; i < reader.maxDoc(); i++)
         {
            if (i > 10 && i % (reader.maxDoc() / 5) == 0)
            {
               long progress = Math.round((100.0 * i) / (reader.maxDoc() * 2f));
               log.info("progress: " + (progress + 50) + "%");
            }
            if (reader.isDeleted(i))
            {
               continue;
            }
            final int currentIndex = i;
            Document d = reader.document(currentIndex, FieldSelectors.UUID_AND_PARENT);
            String uuid = d.get(FieldNames.UUID);
            String parentUUIDString = d.get(FieldNames.PARENT);

            if (parentUUIDString == null || documentUUIDs.contains(parentUUIDString))
            {
               continue;
            }

            // parent is missing
            //NodeId parentId = new NodeId(parentUUID);
            if (stateMgr.getItemData(parentUUIDString) != null)
            {
               errors.add(new MissingAncestor(uuid, parentUUIDString));
            }
            else
            {
               errors.add(new UnknownParent(uuid, parentUUIDString));
            }
         }
      }
      finally
      {
         reader.release();
      }
   }
1029466
public void add(Counter<? extends T> counter) {
        for (Map.Entry<? extends T,Integer> e : counter)
            count(e.getKey(), e.getValue());
    }
    
    /**
     * Counts the object, increasing its total count by 1.
     */
    public int count(T obj) {
        int objIndex = (allowNewIndices) 
            ? objectIndices.index(obj)
            : objectIndices.find(obj);
        if (objIndex < 0)
            return 0;
        int curCount = indexToCount.get(objIndex);
        indexToCount.put(objIndex, curCount + 1);
        sum++;
        return curCount + 1;
    }
528790
@Override
    public void run() {

        if (this.state != null && this.state.cancel) {
            // we were already cancelled, don't run, but still post completion
            this.state.documentCounter = 0;
            this.state.batchCounter = 0;
            runComplete(null);
            return;
        }
        // reset internal state
        this.state = new State();

        Throwable errorInfo = null;

        try {
            this.useBulkGet = sourceDb.isBulkSupported();
            replicate();

        } catch (ExecutionException ex) {
            logger.log(Level.SEVERE, String.format("Batch %s ended with error:", this.state
                    .batchCounter), ex);
            errorInfo = ex.getCause();
        } catch (Throwable e) {
            logger.log(Level.SEVERE, String.format("Batch %s ended with error:", this.state
                    .batchCounter), e);
            errorInfo = e;
        }

        runComplete(errorInfo);
    }