Report the placement histogram to {@link RaidNodeMetrics}. This should only
be called right after a complete parity file traversal is done.
764953
public void clearAndReport() {
    synchronized (metrics) {
      for (Codec codec : Codec.getCodecs()) {
        String id = codec.id;
        int extra = 0;
        Map<Integer, MetricsLongValue> codecStatsMap =
          metrics.codecToMisplacedBlocks.get(id);
        // Reset the values.
        for (Entry<Integer, MetricsLongValue> e: codecStatsMap.entrySet()) {
          e.getValue().set(0);
        }
        for (Entry<Integer, Long> e : blockHistograms.get(id).entrySet()) {
          if (e.getKey() < RaidNodeMetrics.MAX_MONITORED_MISPLACED_BLOCKS - 1) {
            MetricsLongValue v = codecStatsMap.get(e.getKey());
            v.set(e.getValue());
          } else {
            extra += e.getValue();
          }
        }
        MetricsLongValue v = codecStatsMap.get(
            RaidNodeMetrics.MAX_MONITORED_MISPLACED_BLOCKS - 1);
        v.set(extra);
      }
    }
    lastBlockHistograms = blockHistograms;
    lastBlockHistogramsPerRack = blockHistogramsPerRack;
    lastUpdateFinishTime = RaidNode.now();
    lastUpdateUsedTime = lastUpdateFinishTime - lastUpdateStartTime;
    LOG.info("Reporting metrices:\n" + toString());
    createEmptyHistograms();
  }
1256284
public static void saveHistogram(Histogram histogram, String filename) throws IOException {
    File file;
    PrintStream log = null;
    try {
      file = new File(filename);
      if (file.exists() && !file.delete()) {
        System.err.println("Failed deleting previous histogram file: " + file.getAbsolutePath());
      }
      log = new PrintStream(new FileOutputStream(file), false);
      histogram.outputPercentileDistribution(log, 1.0);
    } finally {
      if (log != null) {
        log.close();
      }
    }
  }
411292
synchronized void reportErrorsOnDirectory(StorageDirectory sd, 
      FSImage image) {
    String lsd = listStorageDirectories();
    LOG.info("reportErrorsOnDirectory: Current list of storage dirs:" + lsd);

    LOG.error("reportErrorsOnDirectory: Error reported on storage directory "
        + sd.getRoot());

    if (this.storageDirs.remove(sd)) {
      try {
        sd.unlock();
      } catch (Exception e) {
        LOG.warn(
            "reportErrorsOnDirectory: Unable to unlock bad storage directory: "
                + sd.getRoot().getPath(), e);
      }
      this.removedStorageDirs.add(sd);
    }
    if (image != null) {
      image.reportErrorsOnImageManager(sd);
    }
    
    lsd = listStorageDirectories();
    LOG.info("reportErrorsOnDirectory: Current list of storage dirs:" + lsd);
  }
436216
protected synchronized void incrFilesCopied(long incr) {
    if (incr < 0) {
      throw new IllegalArgumentException("Cannot increment by negative value " +
                                         incr);
    }
    
    RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID).filesCopied.inc(incr);
    numFilesCopied += incr;
  }
817639
private static HdfsFileStatus createHdfsFileStatus(byte[] path, INode node) {
     long size = 0;     // length is zero for directories
     short replication = 0;
     long blocksize = 0;
     if (node instanceof INodeFile) {
       INodeFile fileNode = (INodeFile)node;
       size = fileNode.getFileSize();
       replication = fileNode.getReplication();
       blocksize = fileNode.getPreferredBlockSize();
     }
     else
     if (node.isDirectory()) {
       INodeDirectory dirNode = (INodeDirectory)node;
       //length is used to represent the number of children for directories.
       size = dirNode.getChildren().size();
     }
     return new HdfsFileStatus(
         size,
         node.isDirectory(),
         replication,
         blocksize,
         node.getModificationTime(),
         node.getAccessTime(),
         node.getFsPermission(),
         node.getUserName(),
         node.getGroupName(),
         path);
   }
198994
protected synchronized void incrFilesFixed(long incr) {
    if (incr < 0) {
      throw new IllegalArgumentException("Cannot increment by negative value " +
                                         incr);
    }
    
    RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID).filesFixed.inc(incr);
    numFilesFixed += incr;
  }
1123937
static ParityFilePair verifyParity(FileStatus src, FileStatus parity,
      Codec codec, Configuration conf, FileSystem destFs) throws IOException {
    if (parity.getModificationTime() != src.getModificationTime()) {
      return null;
    }
    int stripeLength = codec.stripeLength;
    int parityLegnth = codec.parityLength;
    long expectedSize = 0;
    List<FileStatus> lfs = null;
    if (codec.isDirRaid) {
      FileSystem srcFs = src.getPath().getFileSystem(conf);
      lfs = RaidNode.listDirectoryRaidFileStatus(conf, srcFs, 
          src.getPath());
      if (lfs == null) {
        return null;
      }

      long blockNum = DirectoryStripeReader.getBlockNum(lfs);
      long parityBlockSize = DirectoryStripeReader.getParityBlockSize(conf,
          lfs); 
      int parityBlocks = (int)Math.ceil(
          ((double)blockNum) / stripeLength) * parityLegnth;
      expectedSize = parityBlocks * parityBlockSize;
    } else {
      double sourceBlocks = Math.ceil(
          ((double)src.getLen()) / src.getBlockSize());
      int parityBlocks = (int)Math.ceil(
          sourceBlocks / stripeLength) * parityLegnth;
      expectedSize = parityBlocks * src.getBlockSize();
    }
    if (parity.getLen() != expectedSize) {
      RaidNode.LOG.error("Bad parity file:" + parity.getPath() +
          " File size doen't match. parity:" + parity.getLen() +
          " expected:" + expectedSize);
      return null;
    }
    return new ParityFilePair(parity.getPath(), parity, destFs, lfs);
  }
944115
public java.util.Map captureStatistics()
                    throws ObjectManagerException
    {
        if (Tracing.isAnyTracingEnabled() && trace.isEntryEnabled())
            trace.entry(this,
                        cclass,
                        "captureStatistics");

        java.util.Map statistics = super.captureStatistics();

        statistics.put("managedobjectsToWrite.size()",
                       Integer.toString(managedObjectsToWrite.size()));
        statistics.put("reservedSize",
                       reservedSize.toString());
        statistics.put("directoryReservedSize",
                       Long.toString(directoryReservedSize));
        statistics.put("tokensToDelete.size()",
                       Integer.toString(tokensToDelete.size()));
        statistics.put("directory.size()",
                       Long.toString(directory.size()));
        statistics.put("storeFileSizeUsed",
                       Long.toString(storeFileSizeUsed));
        statistics.put("storeFileSizeAllocated",
                       Long.toString(storeFileSizeAllocated));
        statistics.put("freeSpaceByLength.size()",
                       Integer.toString(freeSpaceByLength.size()));
        statistics.put("maxFreeSpaceCount",
                       Integer.toString(maxFreeSpaceCount));
        statistics.put("numberOfGetRequests",
                       Integer.toString(numberOfGetRequests));
        numberOfGetRequests = 0;
        statistics.put("storeFileExtendedDuringAllocation",
                       Long.toString(storeFileExtendedDuringAllocation));
        storeFileExtendedDuringAllocation = 0;
        statistics.put("numberOfDirectoryNodesRead",
                       Integer.toString(numberOfDirectoryNodesRead));
        numberOfDirectoryNodesRead = 0;
        statistics.put("numberOfStoreFullCheckpointsTriggered",
                       Integer.toString(numberOfStoreFullCheckpointsTriggered));
        numberOfStoreFullCheckpointsTriggered = 0;
        statistics.put("numberOfReservationCheckpointsTriggered",
                       Integer.toString(numberOfReservationCheckpointsTriggered));
        numberOfReservationCheckpointsTriggered = 0;
        statistics.put("pacedReservationsReleased",
                       Integer.toString(pacedReservationsReleased));
        pacedReservationsReleased = 0;
        statistics.put("numberOfManagedObjectsWritten",
                       Integer.toString(numberOfManagedObjectsWritten));
        numberOfManagedObjectsWritten = 0;
        statistics.put("writingMilliseconds",
                       Long.toString(writingMilliseconds));
        writingMilliseconds = 0;
        statistics.put("removingEntriesMilliseconds",
                       Long.toString(removingEntriesMilliseconds));
        removingEntriesMilliseconds = 0;
        statistics.put("allocatingEntrySpaceMilliseconds",
                       Long.toString(allocatingEntrySpaceMilliseconds));
        allocatingEntrySpaceMilliseconds = 0;
        statistics.put("releasingEntrySpaceMilliseconds",
                       Long.toString(releasingEntrySpaceMilliseconds));
        releasingEntrySpaceMilliseconds = 0;
        statistics.put("directoryWriteMilliseconds",
                       Long.toString(directoryWriteMilliseconds));
        directoryWriteMilliseconds = 0;

        if (Tracing.isAnyTracingEnabled() && trace.isEntryEnabled())
            trace.exit(this,
                       cclass,
                       "captureStatistics",
                       "return=" + statistics);
        return statistics;
    }
1341265
synchronized void reportErrorsOnDirectories(List<StorageDirectory> sds,
      FSImage image) throws IOException {
    for (StorageDirectory sd : sds) {
      reportErrorsOnDirectory(sd, image);
    }
    
    // check image managers (this will update image metrics)
    if (image != null) {
      image.checkImageManagers();
    }
    
    // only check if something was wrong
    if(!sds.isEmpty()) {
      if (this.getNumStorageDirs() == 0)  
        throw new IOException("No more storage directories left");
      
      // check image directories, edits are checked withing FSEditLog.checkJournals
      if (getNumStorageDirs(NameNodeDirType.IMAGE) == 0)
        throw new IOException("No more image storage directories left");
    }
  }
306447
@Override
	public void report() {
		// instead of locking here, we tolerate exceptions
		// we do this to prevent holding the lock for very long and blocking
		// operator creation and shutdown
		try {
			for (Map.Entry<Gauge<?>, String> entry : gauges.entrySet()) {
				if (closed) {
					return;
				}
				reportGauge(entry.getValue(), entry.getKey());
			}

			for (Map.Entry<Counter, String> entry : counters.entrySet()) {
				if (closed) {
					return;
				}
				reportCounter(entry.getValue(), entry.getKey());
			}

			for (Map.Entry<Histogram, String> entry : histograms.entrySet()) {
				reportHistogram(entry.getValue(), entry.getKey());
			}

			for (Map.Entry<Meter, String> entry : meters.entrySet()) {
				reportMeter(entry.getValue(), entry.getKey());
			}
		}
		catch (ConcurrentModificationException | NoSuchElementException e) {
			// ignore - may happen when metrics are concurrently added or removed
			// report next time
		}
	}
703744
@SuppressWarnings("unused")
	public synchronized void dumpStats(final PrintStream out) {
		if (!validState) {
			throw new InvalidStateException();
		}
		out.println("=== Stats ===");
		out.println("maxAllocatedInternalNodes=" + maxInternalNodes);
		out.println("maxAllocatedLeafNodes=" + maxLeafNodes);
		out.println("readCacheSizeInternalNodes=" + readCacheInternal);
		out.println("readCacheSizeLeafNodes=" + readCacheLeaf);
		out.println("leafNodeSize=" + leafNodeFactory.getStructMaxSize());
		out.println("internalNodeSize=" + internalNodeFactory.getStructMaxSize());
		out.println("blockSize=" + blockSize);
		out.println("currentCacheSize=" + (maxCacheSizeInBytes / 1024 / 1024) + "MB");
		out.println("minRecomendedCacheSize="
				+ ((blockSize * (maxInternalNodes + maxLeafNodes)) / 1024 / 1024) + "MB");

		if (!enableIOStats) {
			out.println("=== IOStats ===");
			out.println("not enabled");
			return;
		}
		//
		final IOStat[] ios = iostats.getValues();
		final Comparator<IOStat> ioComparator = new Comparator<IOStat>() {
			@Override
			public int compare(final IOStat o1, final IOStat o2) {
				if (o1 == null) {
					if (o2 == null) {
						return 0; // o1 == null & o2 == null
					}
					return 1; // o1 == null & o2 != null
				}
				if (o2 == null) {
					return -1; // o1 != null & o2 == null
				}
				final long thisVal = ((o1.id < 0 ? 0 : 1) << 63) + (o1.physRead << 16) + o1.physWrite; // o1.id;
				final long anotherVal = ((o2.id < 0 ? 0 : 1) << 63) + (o2.physRead << 16) + o2.physWrite; // o2.id;
				return ((thisVal < anotherVal) ? -1 : ((thisVal == anotherVal) ? 0 : 1));
			}
		};
		Arrays.sort(ios, ioComparator);
		for (final IOStat io : ios) {
			if (io == null) {
				break;
			}
			out.println(io.toString());
		}
	}
1372825
synchronized boolean add(
                           BlockInfo blockInfo,
                           int curReplicas, 
                           int decomissionedReplicas,
                           int expectedReplicas) {
    if(curReplicas<0 || expectedReplicas <= curReplicas) {
      return false;
    }
    int priLevel = getPriority(blockInfo, curReplicas, decomissionedReplicas,
                               expectedReplicas);
    INodeFile fileINode = blockInfo.getINode();
    if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS && fileINode != null &&
        fileINode.getStorageType().equals(StorageType.RAID_STORAGE)) {
      RaidCodec codec = ((INodeRaidStorage) fileINode.getStorage()).getCodec();
      return raidQueue.add(blockInfo, codec);
    }
    
    if(priLevel != LEVEL && priorityQueues.get(priLevel).add(blockInfo)) {
      if (NameNode.stateChangeLog.isDebugEnabled()) {
        NameNode.stateChangeLog.debug(
                                      "BLOCK* NameSystem.UnderReplicationBlock.add:"
                                      + blockInfo
                                      + " has only "+curReplicas
                                      + " replicas and need " + expectedReplicas
                                      + " replicas so is added to neededReplications"
                                      + " at priority level " + priLevel);
      }
      return true;
    }
    return false;
  }
954843
private void collectPoolInfoMetrics() {
    Map<PoolInfo, PoolInfoMetrics> newPoolNameToMetrics = new HashMap<PoolInfo,
      PoolInfoMetrics>();
    long now = ClusterManager.clock.getTime();

    Map<PoolInfo, Long> poolInfoAverageFirstWaitMs =
        sessionManager.getTypePoolInfoAveFirstWaitMs(type);

    // The gets + puts below are OK because only one thread is doing it.
    for (PoolGroupSchedulable poolGroup : poolGroupManager.getPoolGroups()) {
      int poolGroupSessions = 0;
      for (PoolSchedulable pool : poolGroup.getPools()) {
        MetricsRecord poolRecord =
            poolInfoToMetricsRecord.get(pool.getPoolInfo());
        if (poolRecord == null) {
          poolRecord = metrics.getContext().createRecord(
              "pool-" + pool.getName());
          poolInfoToMetricsRecord.put(pool.getPoolInfo(), poolRecord);
        }

        PoolInfoMetrics poolMetrics = new PoolInfoMetrics(pool.getPoolInfo(),
            type, poolRecord);
        poolMetrics.setCounter(
            MetricName.GRANTED, pool.getGranted());
        poolMetrics.setCounter(
            MetricName.REQUESTED, pool.getRequested());
        poolMetrics.setCounter(
            MetricName.SHARE, (long) pool.getShare());
        poolMetrics.setCounter(
            MetricName.MIN, pool.getMinimum());
        poolMetrics.setCounter(
            MetricName.MAX, pool.getMaximum());
        poolMetrics.setCounter(
            MetricName.WEIGHT, (long) pool.getWeight());
        poolMetrics.setCounter(
            MetricName.SESSIONS, pool.getScheduleQueue().size());
        poolMetrics.setCounter(
            MetricName.STARVING, pool.getStarvingTime(now) / 1000);
        Long averageFirstTypeMs =
            poolInfoAverageFirstWaitMs.get(pool.getPoolInfo());
        poolMetrics.setCounter(MetricName.AVE_FIRST_WAIT_MS,
            (averageFirstTypeMs == null) ?
                0 : averageFirstTypeMs.longValue());

        newPoolNameToMetrics.put(pool.getPoolInfo(), poolMetrics);
        poolGroupSessions += pool.getScheduleQueue().size();
      }

      MetricsRecord poolGroupRecord =
          poolInfoToMetricsRecord.get(poolGroup.getName());
      if (poolGroupRecord == null) {
        poolGroupRecord = metrics.getContext().createRecord(
            "poolgroup-" + poolGroup.getName());
        poolInfoToMetricsRecord.put(poolGroup.getPoolInfo(), poolGroupRecord);
      }

      PoolInfoMetrics poolGroupMetrics =
          new PoolInfoMetrics(poolGroup.getPoolInfo(), type, poolGroupRecord);
      poolGroupMetrics.setCounter(
          MetricName.GRANTED, poolGroup.getGranted());
      poolGroupMetrics.setCounter(
          MetricName.REQUESTED, poolGroup.getRequested());
      poolGroupMetrics.setCounter(
          MetricName.SHARE, (long) poolGroup.getShare());
      poolGroupMetrics.setCounter(
          MetricName.MIN, poolGroup.getMinimum());
      poolGroupMetrics.setCounter(
          MetricName.MAX, poolGroup.getMaximum());
      poolGroupMetrics.setCounter(
          MetricName.SESSIONS, poolGroupSessions);
      newPoolNameToMetrics.put(poolGroup.getPoolInfo(), poolGroupMetrics);
    }

    poolInfoToMetrics = newPoolNameToMetrics;
  }
217344
private void initOrderedIterator()
   {
      if (orderedNodes != null)
      {
         return;
      }
      long time = 0;
      if (LOG.isDebugEnabled())
      {
         time = System.currentTimeMillis();
      }
      ScoreNode[][] nodes = (ScoreNode[][])scoreNodes.toArray(new ScoreNode[scoreNodes.size()][]);

      final Set<String> invalidIDs = new HashSet<String>(2);

      /** Cache for Nodes obtainer during the order (comparator work) */
      final Map<String, NodeData> lcache = new HashMap<String, NodeData>();
      do
      {
         if (invalidIDs.size() > 0)
         {
            // previous sort run was not successful -> remove failed uuids
            List<ScoreNode[]> tmp = new ArrayList<ScoreNode[]>();
            for (int i = 0; i < nodes.length; i++)
            {
               if (!invalidIDs.contains(nodes[i][selectorIndex].getNodeId()))
               {
                  tmp.add(nodes[i]);
               }
            }
            nodes = (ScoreNode[][])tmp.toArray(new ScoreNode[tmp.size()][]);
            invalidIDs.clear();
         }

         try
         {
            // sort the uuids
            Arrays.sort(nodes, new ScoreNodeComparator(lcache, invalidIDs));
         }
         catch (SortFailedException e)
         {
            if (LOG.isTraceEnabled())
            {
               LOG.trace("An exception occurred: " + e.getMessage());
            }
         }

      }
      while (invalidIDs.size() > 0);

      if (LOG.isDebugEnabled())
      {
         LOG.debug("" + nodes.length + " node(s) ordered in " + (System.currentTimeMillis() - time) + " ms");
      }

      orderedNodes = new ScoreNodeIteratorImpl(nodes);
   }
949116
@Override
  public void reportMetric(final IMetric<?> metric) throws MetricException {
    final String metricName = metric.getName();
    if (!this.historyListMapping.containsKey(metricName)) {
      logger.info("First time capturing metric: " + metricName);
      this.historyListMapping.put(metricName, new LinkedBlockingDeque<>());
    }
    synchronized (this.historyListMapping.get(metricName)) {
      logger.debug("Ingesting metric: " + metricName);
      this.historyListMapping.get(metricName).add(new InMemoryHistoryNode(metric.getValue()));
      cleanUsingTime(metricName, this.historyListMapping.get(metricName).peekLast().getTimestamp());
    }
  }
504140
@SuppressWarnings("FutureReturnValueIgnored")
  public void reportMetric(final IMetric<?> metric) {
    if (metric != null && isAvailable()) {
      try {
        final IMetric<?> metricSnapshot;
        // take snapshot
        synchronized (metric) {
          metricSnapshot = metric.getSnapshot();
        }
        logger.debug(String
            .format("Submitting %s metric for metric emission pool", metricSnapshot.getName()));
        // report to all emitters
        for (final IMetricEmitter metricEmitter : this.metricEmitters) {
          this.executorService.submit(() -> {
            try {
              metricEmitter.reportMetric(metricSnapshot);
            } catch (final Exception ex) {
              logger.error(
                  String.format("Failed to report %s metric due to ", metricSnapshot.getName()),
                  ex);
            }
          });
        }
      } catch (final CloneNotSupportedException ex) {
        logger.error(
            String.format("Failed to take snapshot for %s metric", metric.getClass().getName()),
            ex);
      }
    }
  }
983882
private void mainQSort3(final BZip2CompressorOutputStream.Data dataShadow,
                            final int loSt, final int hiSt, final int dSt,
                            final int last) {
        final int[] stack_ll = this.stack_ll;
        final int[] stack_hh = this.stack_hh;
        final int[] stack_dd = this.stack_dd;
        final int[] fmap = dataShadow.fmap;
        final byte[] block = dataShadow.block;

        stack_ll[0] = loSt;
        stack_hh[0] = hiSt;
        stack_dd[0] = dSt;

        for (int sp = 1; --sp >= 0;) {
            final int lo = stack_ll[sp];
            final int hi = stack_hh[sp];
            final int d = stack_dd[sp];

            if ((hi - lo < SMALL_THRESH) || (d > DEPTH_THRESH)) {
                if (mainSimpleSort(dataShadow, lo, hi, d, last)) {
                    return;
                }
            } else {
                final int d1 = d + 1;
                final int med = med3(block[fmap[lo] + d1],
                                     block[fmap[hi] + d1], block[fmap[(lo + hi) >>> 1] + d1]) & 0xff;

                int unLo = lo;
                int unHi = hi;
                int ltLo = lo;
                int gtHi = hi;

                while (true) {
                    while (unLo <= unHi) {
                        final int n = (block[fmap[unLo] + d1] & 0xff)
                            - med;
                        if (n == 0) {
                            final int temp = fmap[unLo];
                            fmap[unLo++] = fmap[ltLo];
                            fmap[ltLo++] = temp;
                        } else if (n < 0) {
                            unLo++;
                        } else {
                            break;
                        }
                    }

                    while (unLo <= unHi) {
                        final int n = (block[fmap[unHi] + d1] & 0xff)
                            - med;
                        if (n == 0) {
                            final int temp = fmap[unHi];
                            fmap[unHi--] = fmap[gtHi];
                            fmap[gtHi--] = temp;
                        } else if (n > 0) {
                            unHi--;
                        } else {
                            break;
                        }
                    }

                    if (unLo <= unHi) {
                        final int temp = fmap[unLo];
                        fmap[unLo++] = fmap[unHi];
                        fmap[unHi--] = temp;
                    } else {
                        break;
                    }
                }

                if (gtHi < ltLo) {
                    stack_ll[sp] = lo;
                    stack_hh[sp] = hi;
                    stack_dd[sp] = d1;
                    sp++;
                } else {
                    int n = ((ltLo - lo) < (unLo - ltLo)) ? (ltLo - lo)
                        : (unLo - ltLo);
                    vswap(fmap, lo, unLo - n, n);
                    int m = ((hi - gtHi) < (gtHi - unHi)) ? (hi - gtHi)
                        : (gtHi - unHi);
                    vswap(fmap, unLo, hi - m + 1, m);

                    n = lo + unLo - ltLo - 1;
                    m = hi - (gtHi - unHi) + 1;

                    stack_ll[sp] = lo;
                    stack_hh[sp] = n;
                    stack_dd[sp] = d;
                    sp++;

                    stack_ll[sp] = n + 1;
                    stack_hh[sp] = m - 1;
                    stack_dd[sp] = d1;
                    sp++;

                    stack_ll[sp] = m;
                    stack_hh[sp] = hi;
                    stack_dd[sp] = d;
                    sp++;
                }
            }
        }
    }