Process the INodes under construction section of the fsimage.

@param in DataInputStream to process
@param v Visitor to walk over inodes
@param skipBlocks Walk over each block?
502665
private void processINodesUC(DataInputStream in, ImageVisitor v,
      boolean skipBlocks) throws IOException {
    int numINUC = in.readInt();

    v.visitEnclosingElement(ImageElement.INODES_UNDER_CONSTRUCTION,
                           ImageElement.NUM_INODES_UNDER_CONSTRUCTION, numINUC);

    for(int i = 0; i < numINUC; i++) {
      checkInterruption();
      v.visitEnclosingElement(ImageElement.INODE_UNDER_CONSTRUCTION);
      
      byte [] name = FSImageSerialization.readBytes(in);
      String n = new String(name, "UTF8");
      v.visit(ImageElement.INODE_PATH, n);
      
      if (LayoutVersion.supports(Feature.ADD_INODE_ID, imageVersion)) {
        v.visit(ImageElement.INODE_ID, in.readLong());
      }
      
      v.visit(ImageElement.REPLICATION, in.readShort());
      v.visit(ImageElement.MODIFICATION_TIME, formatDate(in.readLong()));

      v.visit(ImageElement.PREFERRED_BLOCK_SIZE, in.readLong());
      int numBlocks = in.readInt();
      processBlocks(in, v, numBlocks, skipBlocks);

      processPermission(in, v);
      v.visit(ImageElement.CLIENT_NAME, FSImageSerialization.readString(in));
      v.visit(ImageElement.CLIENT_MACHINE, FSImageSerialization.readString(in));

      // Skip over the datanode descriptors, which are still stored in the
      // file but are not used by the datanode or loaded into memory
      int numLocs = in.readInt();
      for(int j = 0; j < numLocs; j++) {
        in.readShort();
        in.readLong();
        in.readLong();
        in.readLong();
        in.readInt();
        FSImageSerialization.readString(in);
        FSImageSerialization.readString(in);
        WritableUtils.readEnum(in, AdminStates.class);
      }

      v.leaveEnclosingElement(); // INodeUnderConstruction
    }

    v.leaveEnclosingElement(); // INodesUnderConstruction
  }
108669
private void processBeginBlockCmd(String parms, int cmdTPosBegin, int cmdTPosEnd)
            throws MiniTemplator.TemplateSyntaxException {
        if (conditionalExclude(cmdTPosBegin, cmdTPosEnd)) {
            return;
        }
        int p0 = skipBlanks(parms, 0);
        if (p0 >= parms.length()) {
            throw new MiniTemplator.TemplateSyntaxException(
                    "Missing block name in $BeginBlock command in template at offset " + cmdTPosBegin + ".");
        }
        int p = skipNonBlanks(parms, p0);
        String blockName = parms.substring(p0, p);
        if (!isRestOfStringBlank(parms, p)) {
            throw new MiniTemplator.TemplateSyntaxException(
                    "Extra parameter in $BeginBlock command in template at offset " + cmdTPosBegin + ".");
        }
        int blockNo = registerBlock(blockName);
        BlockTabRec btr = blockTab[blockNo];
        btr.tPosBegin = cmdTPosBegin;
        btr.tPosContentsBegin = cmdTPosEnd;
        openBlocksTab[currentNestingLevel] = blockNo;
        currentNestingLevel++;
        if (currentNestingLevel > maxNestingLevel) {
            throw new MiniTemplator.TemplateSyntaxException("Block nesting overflow for block \"" + blockName
                    + "\" in template at offset " + cmdTPosBegin + ".");
        }
    }
1310987
@Override
    public void visit(NodeVisitor v) {
        if (v.visit(this)) {
            for (XmlFragment frag : fragments) {
                frag.visit(v);
            }
        }
    }
540766
boolean removeBlock(String path, INodeFileUnderConstruction fileNode,
                      Block block) throws IOException {
    waitForReady();

    writeLock();
    try {
      // modify file-> block and blocksMap
      fileNode.removeBlock(block);
      getFSNamesystem().blocksMap.removeBlock(block);
      // Remove the block from corruptReplicasMap
      getFSNamesystem().corruptReplicas.removeFromCorruptReplicasMap(block);

      // write modified block locations to log
      fsImage.getEditLog().logOpenFile(path, fileNode);
      if (NameNode.stateChangeLog.isDebugEnabled()) {
        NameNode.stateChangeLog.debug("DIR* FSDirectory.removeBlock: "
                                      +path+" with "+block
                                      +" block is removed from the file system");
      }
    } finally {
      writeUnlock();
    }
    return true;
  }
1284959
void process() {

        int len = vList.size();

        for (int i = 0; i < len; i++) {
            System.out.print(".");

            String file = (String) vList.elementAt(i);

            if (!processFile(file)) {
                System.out.println("in file " + file + " !");
            }
        }

        System.out.println("");
    }
1270664
public static void skip(final DataInput in) throws IOException {
		final int length = in.readInt();
		skipFully(in, length);
	}
147763
protected void processMisReplicatedBlocks() {
    writeLock();
    try {
      if (this.initializedReplQueues) {
        return;
      }
      String logPrefix = "Processing mis-replicated blocks: ";
      long nrInvalid = 0;
      
      // clear queues
      neededReplications.clear();
      overReplicatedBlocks.clear();
      raidEncodingTasks.clear();
      
      int totalBlocks = blocksMap.size();
      
      // shutdown initial block report executor
      setupInitialBlockReportExecutor(true);
      
      // simply return if there are no blocks
      if (totalBlocks == 0) {
        // indicate that we are populating replication queues
        setInitializedReplicationQueues(true);
        LOG.info(logPrefix + "Blocks map empty. Nothing to do.");
        return;
      }
      
      AtomicLong count = new AtomicLong(0);
      
      // shutdown initial block report thread executor
      setupInitialBlockReportExecutor(true);
      
      FLOG.info(logPrefix + "Starting, total number of blocks: " + totalBlocks);
      long start = now();
      
      // have one thread process at least 1M blocks
      // no more than configured number of threads
      // at least one thread if the number of blocks < 1M
      int numShards = Math.min(parallelProcessingThreads,
          ((totalBlocks + parallelRQblocksPerShard - 1) / parallelRQblocksPerShard));
      
      // get shard iterators
      List<Iterator<BlockInfo>> iterators = blocksMap
          .getBlocksIterarors(numShards);

      // sanity, as the number can be lower in corner cases
      numShards = iterators.size();
      
      FLOG.info(logPrefix + "Using " + numShards + " threads");
      
      List<MisReplicatedBlocksWorker> misReplicationWorkers 
        = new ArrayList<MisReplicatedBlocksWorker>();
      List<Thread> misReplicationThreads = new ArrayList<Thread>();
      
      // workers will synchronize on this object to insert into 
      // replication queues
      Object lock = new Object();
      
      // initialize mis-replication worker threads
      for (int i = 0; i < numShards; i++) {
        MisReplicatedBlocksWorker w = new MisReplicatedBlocksWorker(lock, i, count,
            iterators.get(i), totalBlocks);
        Thread t = new Thread(w);
        t.start();
        misReplicationWorkers.add(w);
        misReplicationThreads.add(t);
      }
      
      // join all threads
      for (Thread t : misReplicationThreads) {
        try {
          t.join();
        } catch (InterruptedException e) { 
          Thread.currentThread().interrupt();
        }
      }
      
      // get number of invalidated blocks
      for (MisReplicatedBlocksWorker w : misReplicationWorkers) {
        nrInvalid += w.nrInvalid;
      }
      
      // update safeblock count (finalized blocks)
      this.blocksSafe = blocksMap.size() - getMissingBlocksCount();
      // update safeblock count (under construction blocks)
      for (MisReplicatedBlocksWorker w : misReplicationWorkers) {
        this.blocksSafe -= w.missingUnderConstructionBlocks;
      }
      
      // indicate that we are populating replication queues
      setInitializedReplicationQueues(true);

      long stop = now();
      
      FLOG.info(logPrefix + "Total number of blocks = " + blocksMap.size());
      FLOG.info(logPrefix + "Number of invalid blocks = " + nrInvalid);
      FLOG.info(logPrefix + "Number of under-replicated blocks = " +
          neededReplications.size());
      FLOG.info(logPrefix + "Number of  over-replicated blocks = " +
          overReplicatedBlocks.size());
      FLOG.info(logPrefix + "Number of safe blocks = " + this.blocksSafe);
      FLOG.info(logPrefix + "Finished in " + (stop - start) + " ms");
    } finally {
      writeUnlock();
    }
  }
533101
def blockvisit(self, nodes, frame):
        
        if frame.buffer is None:
            self.writeline('if 0: yield None')
        else:
            self.writeline('pass')
        try:
            for node in nodes:
                self.visit(node, frame)
        except CompilerExit:
            pass
1100594
@Override
	protected void computeStatistics(T input, int innerWidth, int innerHeight) {
		final int statPixelStride = stats.getImageType().getNumBands();
		final int statStride = stats.stride;

		int vblocks = innerHeight/blockHeight;
		if( vblocks*blockHeight < innerHeight )
			vblocks++;

		//		for (int y = 0; y < innerHeight; y += blockHeight) {
		BoofConcurrency.loopFor(0, vblocks, vblock -> {
			BlockProcessor<T,S> processor = processors.pop();
			processor.init(blockWidth,blockHeight,thresholdFromLocalBlocks);
			int y = vblock*blockHeight;

			int indexStats = (y/blockHeight)*statStride;
			for (int x = 0; x < innerWidth; x += blockWidth, indexStats += statPixelStride) {
				processor.computeBlockStatistics(x,y,blockWidth,blockHeight,indexStats,input,stats);
			}
			// handle the case where the image's width isn't evenly divisible by the block's width
			if( innerWidth != input.width ) {
				processor.computeBlockStatistics(innerWidth,y,input.width-innerWidth,blockHeight,indexStats,input,stats);
			}
			processors.recycle(processor);
		});

		// NOTE: below could be thrown into its own thread before the code above. Not easy with current thread design
		// handle the case where the image's height isn't evenly divisible by the block's height
		if( innerHeight != input.height ) {
			BlockProcessor<T,S> processor = processors.pop();
			processor.init(blockWidth,blockHeight,thresholdFromLocalBlocks);

			int indexStats = (innerHeight/blockHeight)*statStride;
			int y = innerHeight;
			int blockHeight = input.height-innerHeight;
			for (int x = 0; x < innerWidth; x += blockWidth, indexStats += statPixelStride) {
				processor.computeBlockStatistics(x,y,blockWidth,blockHeight,indexStats,input,stats);
			}
			if( innerWidth != input.width ) {
				processor.computeBlockStatistics(innerWidth,y,input.width-innerWidth,blockHeight,indexStats,input,stats);
			}
		}
	}
435331
public void prepareProbePhase(IOManager ioAccess, Channel.Enumerator probeChannelEnumerator,
			LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException {
		if (isInMemory()) {
			return;
		}
		// ATTENTION: The following lines are duplicated code from finalizeBuildPhase
		this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
		this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
	}
876956
private void processINode(DataInputStream in, ImageVisitor v,
      boolean skipBlocks, String parentName) throws IOException {
    checkInterruption();
    v.visitEnclosingElement(ImageElement.INODE);

    String pathName = FSImageSerialization.readString(in);
    if (parentName != null) {  // local name
      pathName = "/" + pathName;
      if (!"/".equals(parentName)) { // children of non-root directory
        pathName = parentName + pathName;
      }
    }

    v.visit(ImageElement.INODE_PATH, pathName);
    
    
    if (LayoutVersion.supports(Feature.ADD_INODE_ID, imageVersion)) {
      v.visit(ImageElement.INODE_ID, in.readLong());
    }
    
    if (LayoutVersion.supports(Feature.HARDLINK, imageVersion)) {
      byte inodeType = in.readByte();
      if (inodeType == INode.INodeType.HARDLINKED_INODE.type) {
        v.visit(ImageElement.INODE_TYPE, INode.INodeType.HARDLINKED_INODE.toString());
        long hardlinkID =  WritableUtils.readVLong(in);
        v.visit(ImageElement.INODE_HARDLINK_ID, hardlinkID);
      } else if (inodeType == INode.INodeType.RAIDED_INODE.type) {
        v.visit(ImageElement.INODE_TYPE, INode.INodeType.RAIDED_INODE.toString());
        String codecId = WritableUtils.readString(in);
        v.visit(ImageElement.RAID_CODEC_ID, codecId);
      } else {
        v.visit(ImageElement.INODE_TYPE, INode.INodeType.REGULAR_INODE.toString());
      }
    }
    
    v.visit(ImageElement.REPLICATION, in.readShort());
    v.visit(ImageElement.MODIFICATION_TIME, in.readLong());
    if(LayoutVersion.supports(Feature.FILE_ACCESS_TIME, imageVersion))
      v.visit(ImageElement.ACCESS_TIME, in.readLong());
    v.visit(ImageElement.BLOCK_SIZE, in.readLong());
    int numBlocks = in.readInt();

    processBlocks(in, v, numBlocks, skipBlocks);

    // File or directory
    if (numBlocks > 0 || numBlocks == -1) {
      v.visit(ImageElement.NS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
      if (LayoutVersion.supports(Feature.DISKSPACE_QUOTA, imageVersion))
        v.visit(ImageElement.DS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
    }
    if (numBlocks == -2) {
      v.visit(ImageElement.SYMLINK, Text.readString(in));
    }

    processPermission(in, v);
    v.leaveEnclosingElement(); // INode
  }
225806
int getBlockIndex(INodeFile inode, Block blk, String file) throws IOException {
    if (inode == null) {
      throw new IOException("inode for file " + file + " is null");
    }
    readLock();
    try {
      return inode.getBlockIndex(blk, file);
    } finally {
      readUnlock();
    }
  }
478288
public static SkippableIterator and(final SkippableIterator... bitmap) {
		if (bitmap.length == 0)
			throw new RuntimeException("nothing to process");

		return new SkippableIterator() {

			int maxval;
			int wordval;
			boolean hasvalue = false;

			@Override
			public boolean hasValue() {
				return this.hasvalue;
			}

			public SkippableIterator init() {
				this.hasvalue = false;
				for (SkippableIterator i : bitmap)
					if (!i.hasValue())
						return this;
				this.maxval = bitmap[0].getCurrentWordOffset();
				for (int k = 1; k < bitmap.length; ++k)
					if (this.maxval < bitmap[k].getCurrentWordOffset())
						this.maxval = bitmap[k].getCurrentWordOffset();
				movetonext();
				return this;
			}

			public void movetonext() {
				this.hasvalue = false;
				boolean stable;
				do {
					stable = true;
					for (int k = 0; k < bitmap.length; ++k) {
						if (bitmap[k].getCurrentWordOffset() < this.maxval) {
							bitmap[k].advanceUntil(this.maxval);
							if (!bitmap[k].hasValue()) {
								return;
							}
							this.maxval = bitmap[k].getCurrentWordOffset();
							stable = false;
						}
					}
				} while (!stable);
				this.wordval = bitmap[0].getCurrentWord();
				for (int k = 1; k < bitmap.length; ++k) {
					this.wordval &= bitmap[k].getCurrentWord();
				}
				if (this.wordval == 0)
					advance();
				else {
					this.hasvalue = true;
				}
			}

			@Override
			public void advance() {
				for (SkippableIterator b : bitmap) {
					b.advanceUntil(this.maxval);
					if (!b.hasValue()) {
						this.hasvalue = false;
						return;
					}
					this.maxval = b.getCurrentWordOffset();
				}
				movetonext();
			}

			@Override
			public void advanceUntil(int min) {
				bitmap[bitmap.length - 1].advanceUntil(min);
				if (!bitmap[bitmap.length - 1].hasValue()) {
					this.hasvalue = false;
					return;
				}
				this.maxval = bitmap[bitmap.length - 1].getCurrentWordOffset();
				movetonext();
			}

			@Override
			public int getCurrentWord() {
				return this.wordval;
			}

			@Override
			public int getCurrentWordOffset() {
				return this.maxval;
			}

		}.init();
	}
1166718
public void seek(long block, long recordInBlock) throws IOException {

		List<BlockMetaData> blockMetaData = reader.getRowGroups();

		if (block == -1L && recordInBlock == -1L) {
			// the split was fully consumed
			currentBlock = blockMetaData.size() - 1;
			numReadRecords = numTotalRecords;
			numRecordsUpToCurrentBlock = numTotalRecords;
			return;
		}

		// init all counters for the start of the first block
		currentBlock = 0;
		numRecordsUpToPreviousBlock = 0;
		numRecordsUpToCurrentBlock = blockMetaData.get(0).getRowCount();
		numReadRecords = 0;

		// seek to the given block
		while (currentBlock < block) {
			currentBlock++;
			reader.skipNextRowGroup();
			numRecordsUpToPreviousBlock = numRecordsUpToCurrentBlock;
			numRecordsUpToCurrentBlock += blockMetaData.get(currentBlock).getRowCount();
			numReadRecords = numRecordsUpToPreviousBlock;
		}

		// seek to and read the given record
		PageReadStore pages = reader.readNextRowGroup();
		recordReader = createRecordReader(pages);
		for (int i = 0; i <= recordInBlock; i++) {
			readNextRecord();
		}
	}
723493
@Override
    public void visit(NodeVisitor v) {
        if (v.visit(this)) {
            target.visit(v);
            if (initializer != null) {
                initializer.visit(v);
            }
        }
    }
878947
public static void skipFully(DataInput in, int len) throws IOException {
        int total = 0;
        int cur = 0;

        while ((total < len) && ((cur = in.skipBytes(len - total)) > 0)) {
            total += cur;
        }

        if (total < len) {
            throw new IOException("Not able to skip " + len + " bytes, possibly " + "due to end of input.");
        }
    }
870165
private void skip() {
        try {
            int blockSize;
            do {
                blockSize = read();
                rawData.position(rawData.position() + blockSize);
            } while (blockSize > 0);
        } catch (IllegalArgumentException ex) {
        }
    }