This reconstructs a single part file block by recovering in sequence each
parity block in the part file block.
349849
private void processParityHarPartBlock(FileSystem dfs, Path partFile,
      long blockOffset,
      FileStatus partFileStat,
      HarIndex harIndex,
      File localBlockFile,
      Progressable progress)
  throws IOException {
    String partName = partFile.toUri().getPath(); // Temporarily.
    partName = partName.substring(1 + partName.lastIndexOf(Path.SEPARATOR));

    OutputStream out = new FileOutputStream(localBlockFile);

    try {
      // A HAR part file block could map to several parity files. We need to
      // use all of them to recover this block.
      final long blockEnd = Math.min(blockOffset + 
          partFileStat.getBlockSize(),
          partFileStat.getLen());
      for (long offset = blockOffset; offset < blockEnd; ) {
        HarIndex.IndexEntry entry = harIndex.findEntry(partName, offset);
        if (entry == null) {
          String msg = "Lost index file has no matching index entry for " +
          partName + ":" + offset;
          LOG.warn(msg);
          throw new IOException(msg);
        }
        Path parityFile = new Path(entry.fileName);
        Encoder encoder = null;
        for (Codec codec : Codec.getCodecs()) {
          if (isParityFile(parityFile, codec)) {
            encoder = new Encoder(getConf(), codec);
          }
        }
        if (encoder == null) {
          String msg = "Could not figure out codec correctly for " + parityFile;
          LOG.warn(msg);
          throw new IOException(msg);
        }
        Path srcFile = RaidUtils.sourcePathFromParityPath(parityFile, dfs);
        if (null == srcFile) {
          String msg = "Can not find the source path for parity file: " +
                          parityFile;
          LOG.warn(msg);
          throw new IOException(msg);
        }
        FileStatus srcStat = dfs.getFileStatus(srcFile);
        if (srcStat.getModificationTime() != entry.mtime) {
          String msg = "Modification times of " + parityFile + " and " +
          srcFile + " do not match.";
          LOG.warn(msg);
          throw new IOException(msg);
        }
        long lostOffsetInParity = offset - entry.startOffset;
        LOG.info(partFile + ":" + offset + " maps to " +
            parityFile + ":" + lostOffsetInParity +
            " and will be recovered from " + srcFile);
        encoder.recoverParityBlockToStream(dfs, srcStat,
            srcStat.getBlockSize(), parityFile,
            lostOffsetInParity, out, progress);
        // Finished recovery of one parity block. Since a parity block has the
        // same size as a source block, we can move offset by source block 
        // size.
        offset += srcStat.getBlockSize();
        LOG.info("Recovered " + srcStat.getBlockSize() + " part file bytes ");
        if (offset > blockEnd) {
          String msg =
            "Recovered block spills across part file blocks. Cannot continue";
          throw new IOException(msg);
        }
        progress.progress();
      }
    } finally {
      out.close();
    }
  }
1255458
def reassembly(self, info):
        
        BUFID = info.bufid  # Buffer Identifier
        DSN = info.dsn      # Data Sequence Number
        ACK = info.ack      # Acknowledgement Number
        FIN = info.fin      # Finish Flag (Termination)
        RST = info.rst      # Reset Connection Flag (Termination)
        SYN = info.syn      # Synchronise Flag (Establishment)

        # when SYN is set, reset buffer of this session
        if SYN and BUFID in self._buffer:
            self._dtgram += self.submit(self._buffer[BUFID], bufid=BUFID)
            del self._buffer[BUFID]

        # initialise buffer with BUFID & ACK
        if BUFID not in self._buffer:
            self._buffer[BUFID] = {
                'hdl': [Info(first=info.len, last=sys.maxsize)],
                ACK: dict(
                    ind=[info.num],
                    isn=info.dsn,
                    len=info.len,
                    raw=info.payload,
                ),
            }
        else:
            # initialise buffer with ACK
            if ACK not in self._buffer[BUFID]:
                self._buffer[BUFID][ACK] = dict(
                    ind=[info.num],
                    isn=info.dsn,
                    len=info.len,
                    raw=info.payload,
                )
            else:
                # append packet index
                self._buffer[BUFID][ACK]['ind'].append(info.num)

                # record fragment payload
                ISN = self._buffer[BUFID][ACK]['isn']   # Initial Sequence Number
                RAW = self._buffer[BUFID][ACK]['raw']   # Raw Payload Data
                if DSN >= ISN:  # if fragment goes after existing payload
                    LEN = self._buffer[BUFID][ACK]['len']
                    GAP = DSN - (ISN + LEN)     # gap length between payloads
                    if GAP >= 0:    # if fragment goes after existing payload
                        RAW += bytearray(GAP) + info.payload
                    else:           # if fragment partially overlaps existing payload
                        RAW[DSN-ISN:] = info.payload
                else:           # if fragment exceeds existing payload
                    LEN = info.len
                    GAP = ISN - (DSN + LEN)     # gap length between payloads
                    self._buffer[BUFID][ACK]['isn'] = DSN
                    if GAP >= 0:    # if fragment exceeds existing payload
                        RAW = info.payload + bytearray(GAP) + RAW
                    else:           # if fragment partially overlaps existing payload
                        RAW = info.payload + RAW[ISN-GAP:]
                self._buffer[BUFID][ACK]['raw'] = RAW       # update payload datagram
                self._buffer[BUFID][ACK]['len'] = len(RAW)  # update payload length

            # update hole descriptor list
            HDL = self._buffer[BUFID]['hdl']
            for (index, hole) in enumerate(HDL):                            # step one
                if info.first > hole.last:                                  # step two
                    continue
                if info.last < hole.first:                                  # step three
                    continue
                del HDL[index]                                              # step four
                if info.first > hole.first:                                 # step five
                    new_hole = Info(
                        first=hole.first,
                        last=info.first - 1,
                    )
                    HDL.insert(index, new_hole)
                    index += 1
                if info.last < hole.last and not FIN and not RST:           # step six
                    new_hole = Info(
                        first=info.last + 1,
                        last=hole.last
                    )
                    HDL.insert(index, new_hole)
                break                                                       # step seven
            self._buffer[BUFID]['hdl'] = HDL                                # update HDL

        # when FIN/RST is set, submit buffer of this session
        if FIN or RST:
            self._dtgram += self.submit(self._buffer[BUFID], bufid=BUFID)
            del self._buffer[BUFID]
259705
def reconstruct(self, D=None):
        """"""

        if D is None:
            Df = self.Xf
        else:
            Df = sl.rfftn(D, None, self.cri.axisN)

        Sf = np.sum(self.Zf * Df, axis=self.cri.axisM)
        return sl.irfftn(Sf, self.cri.Nv, self.cri.axisN)
542180
def parse_parts(self, file, boundary, content_length):
        
        in_memory = 0

        for ellt, ell in self.parse_lines(file, boundary, content_length):
            if ellt == _begin_file:
                headers, name, filename = ell
                is_file = True
                guard_memory = False
                filename, container = self.start_file_streaming(
                    filename, headers, content_length)
                _write = container.write

            elif ellt == _begin_form:
                headers, name = ell
                is_file = False
                container = []
                _write = container.append
                guard_memory = self.max_form_memory_size is not None

            elif ellt == _cont:
                _write(ell)
                # if we write into memory and there is a memory size limit we
                # count the number of bytes in memory and raise an exception if
                # there is too much data in memory.
                if guard_memory:
                    in_memory += len(ell)
                    if in_memory > self.max_form_memory_size:
                        self.in_memory_threshold_reached(in_memory)

            elif ellt == _end:
                if is_file:
                    container.seek(0)
                    yield ('file',
                           (name, FileStorage(container, filename, name,
                                              headers=headers)))
                else:
                    part_charset = self.get_part_charset(headers)
                    yield ('form',
                           (name, b''.join(container).decode(
                                part_charset, self.errors)))
73223
private Block[] allocateParityBlocks(int numParityBlocks) throws IOException {
    Block[] blocks = new Block[numParityBlocks];
    for (int i = 0; i < numParityBlocks; i++) {
      Block b = new Block(generateBlockId(), 0, 0);
      while (isValidBlock(b)) {
        b.setBlockId(generateBlockId());
      }
      b.setGenerationStamp(getGenerationStamp());
      blocks[i] = b;
    }
    return blocks;
  }
621211
def block_partition(block, i):
    
    i += 1
    new_block = BasicBlock(block.asm[i:])
    block.mem = block.mem[:i]
    block.asm = block.asm[:i]
    block.update_labels()
    new_block.update_labels()

    new_block.goes_to = block.goes_to
    block.goes_to = IdentitySet()

    new_block.label_goes = block.label_goes
    block.label_goes = []

    new_block.next = new_block.original_next = block.original_next
    new_block.prev = block
    new_block.add_comes_from(block)

    if new_block.next is not None:
        new_block.next.prev = new_block
        new_block.next.add_comes_from(new_block)
        new_block.next.delete_from(block)

    block.next = block.original_next = new_block
    block.update_next_block()
    block.add_goes_to(new_block)

    return block, new_block
1354739
private void makeBlock(String upHost, long offset, int blockSize, int chunkSize, ProgressHandler progress,
                           CompletionHandler _completionHandler, UpCancellationSignal c) {
        String path = format(Locale.ENGLISH, "/mkblk/%d", blockSize);
        try {
            file.seek(offset);
            file.read(chunkBuffer, 0, chunkSize);
        } catch (IOException e) {
            completionHandler.complete(key, ResponseInfo.fileError(e, token), null);
            return;
        }
        this.crc32 = Crc32.bytes(chunkBuffer, 0, chunkSize);
        String postUrl = String.format("%s%s", upHost, path);
        post(postUrl, chunkBuffer, 0, chunkSize, progress, _completionHandler, c);
    }
1129751
def _iter_straight_packed(self, byte_blocks):
        

        # length of row, in bytes
        rb = self.row_bytes
        a = bytearray()
        # The previous (reconstructed) scanline.
        # None indicates first line of image.
        recon = None
        for some_bytes in byte_blocks:
            a.extend(some_bytes)
            while len(a) >= rb + 1:
                filter_type = a[0]
                scanline = a[1: rb + 1]
                del a[: rb + 1]
                recon = self.undo_filter(filter_type, scanline, recon)
                yield recon
        if len(a) != 0:
            # :file:format We get here with a file format error:
            # when the available bytes (after decompressing) do not
            # pack into exact rows.
            raise FormatError('Wrong size for decompressed IDAT chunk.')
        assert len(a) == 0
150100
private void processBlocks(DataInputStream in, ImageVisitor v,
      int numBlocks, boolean skipBlocks) throws IOException {
    v.visitEnclosingElement(ImageElement.BLOCKS,
                            ImageElement.NUM_BLOCKS, numBlocks);
    
    // directory or symlink, no blocks to process    
    if(numBlocks == -1 || numBlocks == -2) { 
      v.leaveEnclosingElement(); // Blocks
      return;
    }
    
    if(skipBlocks) {
      int fieldsBytes = Long.SIZE * 3;
      if (LayoutVersion.supports(Feature.BLOCK_CHECKSUM, imageVersion)) {
        // For block checksum
        fieldsBytes += Integer.SIZE;
      }
      
      int bytesToSkip = ((fieldsBytes /* fields */) / 8 /*bits*/) * numBlocks;
      if(in.skipBytes(bytesToSkip) != bytesToSkip)
        throw new IOException("Error skipping over blocks");
      
    } else {
      for(int j = 0; j < numBlocks; j++) {
        v.visitEnclosingElement(ImageElement.BLOCK);
        v.visit(ImageElement.BLOCK_ID, in.readLong());
        v.visit(ImageElement.NUM_BYTES, in.readLong());
        v.visit(ImageElement.GENERATION_STAMP, in.readLong());
        if (LayoutVersion.supports(Feature.BLOCK_CHECKSUM, imageVersion)) {
          v.visit(ImageElement.BLOCK_CHECKSUM, in.readInt());
        }
        v.leaveEnclosingElement(); // Block
      }
    }
    v.leaveEnclosingElement(); // Blocks
  }
978815
public void mergeInternal(INode parityINodes[], INode sourceINodes[],
      String parity, String source, RaidCodec codec, int[] checksums)
          throws IOException {
    waitForReady();
    long now = FSNamesystem.now();
    unprotectedMerge(parityINodes, sourceINodes, parity, source, codec,
        checksums, now);
    fsImage.getEditLog().logMerge(parity, source, codec.id, checksums, now);
  }
882400
public boolean processFile(Path srcPath, ParityFilePair parityPair,
      Decoder decoder, Boolean fromStripeStore, Context context) 
          throws IOException,
      InterruptedException {
    LOG.info("Processing file " + srcPath);
    Progressable progress = context;
    if (progress == null) {
      progress = RaidUtils.NULL_PROGRESSABLE;
    }
    
    DistributedFileSystem srcFs = getDFS(srcPath);
    FileStatus srcStat = srcFs.getFileStatus(srcPath);
    long blockSize = srcStat.getBlockSize();
    long srcFileSize = srcStat.getLen();
    String uriPath = srcPath.toUri().getPath();

    int numBlocksReconstructed = 0;
    List<LocatedBlockWithMetaInfo> lostBlocks = lostBlocksInFile(srcFs,
        uriPath, srcStat);
    if (lostBlocks.size() == 0) {
      LOG.warn("Couldn't find any lost blocks in file " + srcPath + 
          ", ignoring...");
      return false;
    }
    List<Block> blocksLostChecksum = new ArrayList<Block>();
    List<Block> blocksLostStripe = new ArrayList<Block>();
    
    for (LocatedBlockWithMetaInfo lb: lostBlocks) {
      Block lostBlock = lb.getBlock();
      long lostBlockOffset = lb.getStartOffset();

      LOG.info("Found lost block " + lostBlock +
          ", offset " + lostBlockOffset);
      Long oldCRC = decoder.retrieveChecksum(lostBlock, 
          srcPath, lostBlockOffset, srcFs, context);
      if (abortReconstruction(oldCRC, decoder)) {
        blocksLostChecksum.add(lostBlock);
        continue;
      }
      StripeInfo si = decoder.retrieveStripe(lostBlock, 
          srcPath, lostBlockOffset, srcFs, context, false);
      if (si == null && decoder.stripeStore != null) {
        blocksLostStripe.add(lostBlock);
        continue;
      }

      final long blockContentsSize =
        Math.min(blockSize, srcFileSize - lostBlockOffset);
      File localBlockFile =
        File.createTempFile(lostBlock.getBlockName(), ".tmp");
      localBlockFile.deleteOnExit();

      try {
        CRC32 crc = null;
        
        if (fromStripeStore) {
          crc = decoder.recoverBlockToFileFromStripeInfo(srcFs, srcPath,
              lostBlock, localBlockFile, blockSize,
              lostBlockOffset, blockContentsSize, si, context);
        } else {
          crc = decoder.recoverBlockToFile(srcFs, srcStat,
                        parityPair.getFileSystem(),
                        parityPair.getPath(), blockSize,
                        lostBlockOffset, localBlockFile,
                        blockContentsSize, si, context);
        }
        LOG.info("Recovered crc: " + ((crc == null)?null: crc.getValue()) +
            " expected crc:" + oldCRC);
        if (crc != null && oldCRC != null &&
            crc.getValue() != oldCRC) {
          // checksum doesn't match, it's dangerous to send it
          IOException ioe = new IOException("Block " + lostBlock.toString() +
              " new checksum " + crc.getValue() +
              " doesn't match the old one " + oldCRC);
          LogUtils.logRaidReconstructionMetrics(LOGRESULTS.FAILURE, 0,
              decoder.codec, srcPath, lostBlockOffset,
              LOGTYPES.OFFLINE_RECONSTRUCTION_CHECKSUM_VERIFICATION,
              srcFs, ioe, context);
          throw ioe;
        }
        // Now that we have recovered the file block locally, send it.
        computeMetadataAndSendReconstructedBlock(localBlockFile,
            lostBlock, blockContentsSize,
            lb.getLocations(),
            lb.getDataProtocolVersion(), 
            lb.getNamespaceID(), 
            progress);
        
        numBlocksReconstructed++;

      } finally {
        localBlockFile.delete();
      }
      progress.progress();
    }
    
    LOG.info("Reconstructed " + numBlocksReconstructed + " blocks in " + srcPath);
    checkLostBlocks(blocksLostChecksum, blocksLostStripe, srcPath, decoder.codec);
    return true;
  }
140625
private FSDataOutputStream createNewPartStream(Path dst, int partId) throws IOException {
    String partName = PART_PREFIX + partId;
    Path output = new Path(dst, partName);
    FileSystem destFs = output.getFileSystem(conf);
    FSDataOutputStream partStream = destFs.create(output, false,
        conf.getInt("io.file.buffer.size", 4096), destFs.getDefaultReplication(),
        conf.getLong(HAR_BLOCKSIZE_LABEL, HAR_BLOCKSIZE_DEFAULT));
    return partStream;
  }
588260
public LocatedBlock recoverBlock(int namespaceId, Block block,
      boolean keepLength, DatanodeID[] datanodeids, boolean closeFile,
      long deadline) throws IOException {
    int errorCount = 0;

    // Number of "replicasBeingWritten" in 0.21 parlance - these are replicas
    // on DNs that are still alive from when the write was happening
    int rbwCount = 0;
    // Number of "replicasWaitingRecovery" in 0.21 parlance - these replicas
    // have survived a DN restart, and thus might be truncated (eg if the
    // DN died because of a machine power failure, and when the ext3 journal
    // replayed, it truncated the file
    int rwrCount = 0;

    List<BlockRecord> blockRecords = new ArrayList<BlockRecord>();
    List<InterDatanodeProtocol> datanodeProxies = new ArrayList<InterDatanodeProtocol>();
    // check generation stamps
    for (DatanodeID id : datanodeids) {
      try {
        InterDatanodeProtocol datanode;
        if (dnr!= null && dnr.equals(id)) {
          LOG.info("Skipping IDNPP creation for local id " + id
              + " when recovering " + block);
          datanode = dn;
        } else {
          LOG.info("Creating IDNPP for non-local id " + id + " (dnReg="
              + dnr + ") when recovering "
              + block);
          datanode = DataNode.createInterDataNodeProtocolProxy(id, conf,
              socketTimeout);
          datanodeProxies.add(datanode);
        }
        BlockSyncer.throwIfAfterTime(deadline);
        BlockRecoveryInfo info = datanode
            .startBlockRecovery(namespaceId, block);
        if (info == null) {
          LOG.info("No block metadata found for block " + block
              + " on datanode " + id);
          continue;
        }
        if (info.getBlock().getGenerationStamp() < block.getGenerationStamp()) {
          LOG.info("Only old generation stamp "
              + info.getBlock().getGenerationStamp() + " found on datanode "
              + id + " (needed block=" + block + ")");
          continue;
        }
        blockRecords.add(new BlockRecord(id, datanode, info));

        if (info.wasRecoveredOnStartup()) {
          rwrCount++;
        } else {
          rbwCount++;
        }
      } catch (BlockRecoveryTimeoutException e) {
        throw e;
      } catch (IOException e) {
        ++errorCount;
        InterDatanodeProtocol.LOG.warn(
            "Failed to getBlockMetaDataInfo for block (=" + block
                + ") from datanode (=" + id + ")", e);
      }
    }

    // If we *only* have replicas from post-DN-restart, then we should
    // include them in determining length. Otherwise they might cause us
    // to truncate too short.
    boolean shouldRecoverRwrs = (rbwCount == 0);

    List<BlockRecord> syncList = new ArrayList<BlockRecord>();
    long minlength = Long.MAX_VALUE;

    for (BlockRecord record : blockRecords) {
      BlockRecoveryInfo info = record.info;
      assert (info != null && info.getBlock().getGenerationStamp() >= block
          .getGenerationStamp());
      if (!shouldRecoverRwrs && info.wasRecoveredOnStartup()) {
        LOG.info("Not recovering replica " + record
            + " since it was recovered on "
            + "startup and we have better replicas");
        continue;
      }
      if (keepLength) {
        if (info.getBlock().getNumBytes() == block.getNumBytes()) {
          syncList.add(record);
        }
      } else {
        syncList.add(record);
        if (info.getBlock().getNumBytes() < minlength) {
          minlength = info.getBlock().getNumBytes();
        }
      }
    }

    if (syncList.isEmpty() && errorCount > 0) {
      DataNode.stopAllProxies(datanodeProxies);
      throw new IOException("All datanodes failed: block=" + block
          + ", datanodeids=" + Arrays.asList(datanodeids));
    }
    if (!keepLength) {
      block.setNumBytes(minlength);
    }
    return blockSyncer.syncBlock(block, syncList, closeFile, datanodeProxies,
        deadline);
  }
493864
def file_parts(self):
        
        file_parts = []
        for part in self.parts:
            try:
                for sub_part in part:
                    if isinstance(sub_part, FileToken):
                        file_parts.append(sub_part)
            except TypeError:
                if isinstance(part, FileToken):
                    file_parts.append(part)
        return file_parts
982172
def block_sep1(self, Y):
        """"""

        Y1 = Y[..., self.cri.M:]

        # If cri.Cd > 1 (multi-channel dictionary), we need to undo the
        # reshape performed in block_cat
        if self.cri.Cd > 1:
            shp = list(Y1.shape)
            shp[self.cri.axisM] = self.cri.dimN
            shp[self.cri.axisC] = self.cri.Cd
            Y1 = Y1.reshape(shp)

        # Axes are swapped here for similar reasons to those
        # motivating swapping in cbpdn.ConvTwoBlockCnstrnt.block_sep0
        Y1 = np.swapaxes(Y1[..., np.newaxis], self.cri.axisM, -1)

        return Y1
1223582
private LocatedBlock recoverBlock(int namespaceId, Block block, boolean keepLength,
      DatanodeID[] datanodeids, boolean closeFile, long deadline) throws IOException {
    InjectionHandler.processEvent(InjectionEvent.DATANODE_BEFORE_RECOVERBLOCK,
        this);

    // If the block is already being recovered, then skip recovering it.
    // This can happen if the namenode and client start recovering the same
    // file at the same time.
    synchronized (ongoingRecovery) {
      Block tmp = new Block();
      tmp.set(block.getBlockId(), block.getNumBytes(), GenerationStamp.WILDCARD_STAMP);
      if (ongoingRecovery.get(tmp) != null) {
        String msg = "Block " + block + " is already being recovered, " +
                     " ignoring this request to recover it.";
        LOG.info(msg);
        throw new IOException(msg);
      }
      ongoingRecovery.put(block, block);
    }
    try {
      BlockRecoveryCoordinator brc = new BlockRecoveryCoordinator(LOG,
          getConf(), socketTimeout, this, namespaceManager.get(namespaceId),
          getDNRegistrationForNS(namespaceId));
      
      return brc.recoverBlock(namespaceId, block, keepLength, datanodeids, closeFile,
          deadline);
    } finally {
      synchronized (ongoingRecovery) {
        ongoingRecovery.remove(block);
      }
    }
  }
165810
void recoverSegments(SyncTask task) throws IOException {
    // obtain the list of segments that are valid
    if (!prepareRecovery(task)) {
      return;
    }

    // iterate through all nodes
    for (InetSocketAddress jn : journalNodes) {
      if (isLocalIpAddress(jn.getAddress())
          && jn.getPort() == journalNode.getPort()) {
        // we do not need to talk to ourselves
        continue;
      }

      try {
        // get manifest for log that we care about
        List<EditLogFile> remoteLogFiles = getManifest(jn, task.journal,
            task.recoveryStartTxid);

        // go through all remote segments
        for (EditLogFile relf : remoteLogFiles) {
          recoverSegment(jn, relf, task);
        }

        // if we are done, there is no need to iterate more
        if (!task.hasMissingValidSegments()) {
          LOG.info(logMsg + "recovery finished.");
          break;
        }
      } catch (Exception e) {
        LOG.error(logMsg + "error", e);
        continue;
      } 
    }
  }