Synchronize a log segment from another JournalNode. The log is
downloaded from the provided URL into a temporary location on disk
162546
File syncLog(long stamp, final long startTxId, final URL url, String name, File tmpFile)
      throws IOException {
    final File[] localPaths = new File[] { tmpFile };

    // TODO add security if needed.
    LOG.info("Synchronizing log " + name + " from " + url);
    boolean success = false;
    try {
      TransferFsImage.doGetUrl(
          url,
          ImageSet.convertFilesToStreams(localPaths, journalStorage,
              url.toString()), journalStorage, true);
      assert tmpFile.exists();
      success = true;
    } finally {
      if (!success) {
        if (!tmpFile.delete()) {
          LOG.warn("Failed to delete temporary file " + tmpFile);
        }
      }
    }
    return tmpFile;
  }
804231
function downloadNode(next) {
        _downloadNode(
          version,
          options.nodeTempDir,
          options.nodeConfigureArgs,
          options.nodeMakeArgs,
          options.nodeVCBuildArgs,
          next,
        );
      }
927716
def sync():
    
    for blog in Blog.query.all():
        # parse the feed. feedparser.parse will never given an exception
        # but the bozo bit might be defined.
        feed = feedparser.parse(blog.feed_url)

        for entry in feed.entries:
            # get the guid. either the id if specified, otherwise the link.
            # if none is available we skip the entry.
            guid = entry.get("id") or entry.get("link")
            if not guid:
                continue

            # get an old entry for the guid to check if we need to update
            # or recreate the item
            old_entry = Entry.query.filter_by(guid=guid).first()

            # get title, url and text. skip if no title or no text is
            # given. if the link is missing we use the blog link.
            if "title_detail" in entry:
                title = entry.title_detail.get("value") or ""
                if entry.title_detail.get("type") in HTML_MIMETYPES:
                    title = strip_tags(title)
                else:
                    title = escape(title)
            else:
                title = entry.get("title")
            url = entry.get("link") or blog.blog_url
            text = (
                "content" in entry and entry.content[0] or entry.get("summary_detail")
            )

            if not title or not text:
                continue

            # if we have an html text we use that, otherwise we HTML
            # escape the text and use that one. We also handle XHTML
            # with our tag soup parser for the moment.
            if text.get("type") not in HTML_MIMETYPES:
                text = escape(nl2p(text.get("value") or ""))
            else:
                text = text.get("value") or ""

            # no text? continue
            if not text.strip():
                continue

            # get the pub date and updated date. This is rather complex
            # because different feeds do different stuff
            pub_date = (
                entry.get("published_parsed")
                or entry.get("created_parsed")
                or entry.get("date_parsed")
            )
            updated = entry.get("updated_parsed") or pub_date
            pub_date = pub_date or updated

            # if we don't have a pub_date we skip.
            if not pub_date:
                continue

            # convert the time tuples to datetime objects.
            pub_date = datetime(*pub_date[:6])
            updated = datetime(*updated[:6])
            if old_entry and updated <= old_entry.last_update:
                continue

            # create a new entry object based on the data collected or
            # update the old one.
            entry = old_entry or Entry()
            entry.blog = blog
            entry.guid = guid
            entry.title = title
            entry.url = url
            entry.text = text
            entry.pub_date = pub_date
            entry.last_update = updated
            session.add(entry)

    session.commit()
854896
function (log, seq, cb) {
    dag.logs.get(log, seq, function (err, entry) {
      if (err && err.notFound) return cb()
      if (err) return cb(err)
      if (entry.change > changes) return cb() // ensure snapshot

      entry.log = log
      entry.seq = seq

      var i = 0
      var loop = function () {
        if (i < entry.links.length) return sendHave(entry.links[i++], loop)
        entry.links = noarr // premature opt: less mem yo
        outgoing.push(entry, cb)
      }

      loop()
    })
  }
777251
def _merge_logfiles(self):
        """"""
        # open files, read first lines, extract first dates
        lines = [next(iter(logfile), None) for logfile in self.args['logfile']]

        # adjust lines by timezone
        for i in range(len(lines)):
            if lines[i] and lines[i].datetime:
                lines[i]._datetime = (lines[i].datetime +
                                      timedelta(hours=self.args['timezone']
                                                [i]))

        while any(lines):
            min_line = min(lines, key=self._datetime_key_for_merge)
            min_idx = lines.index(min_line)

            if self.args['markers'][min_idx]:
                min_line.merge_marker_str = self.args['markers'][min_idx]

            yield min_line

            # update lines array with a new line from the min_idx'th logfile
            lines[min_idx] = next(iter(self.args['logfile'][min_idx]), None)
            if lines[min_idx] and lines[min_idx].datetime:
                lines[min_idx]._datetime = (
                    lines[min_idx].datetime +
                    timedelta(hours=self.args['timezone'][min_idx]))
303848
private IndexNode copyWith(byte[] key, int file, int offset, int size, OverwriteHook overwriteHook, RecordChange recordChange) throws IOException {
      if (leafNodes == null) throw new IllegalArgumentException();
      byte[] newPrefix;
      byte[][] newKeyParts;
      LeafNode[] newLeafNodes;
      if (leafNodes.length == 0) {
         overwriteHook.setOverwritten(false, -1, -1);
         if (overwriteHook.check(-1, -1)) {
            return new IndexNode(segment, prefix, keyParts, new LeafNode[]{ new LeafNode(file, offset, (short) 1)});
         } else {
            segment.getCompactor().free(file, size);
            return this;
         }
      }

      int insertPart = getInsertionPoint(key);
      LeafNode oldLeafNode = leafNodes[insertPart];

      short numRecords = oldLeafNode.numRecords;
      switch (recordChange) {
         case INCREASE:
         case INCREASE_FOR_OLD:
            if (numRecords == Short.MAX_VALUE) {
               throw new IllegalStateException("Too many records for this key (short overflow)");
            }
            numRecords++;
            break;
         case MOVE:
            break;
         case DECREASE:
            numRecords--;
            break;
      }

      EntryRecord hak;
      try {
         hak = oldLeafNode.loadHeaderAndKey(segment.getFileProvider());
      } catch (IndexNodeOutdatedException e) {
         throw new IllegalStateException("Index cannot be outdated for segment updater thread", e);
      }
      int keyComp = compare(hak.getKey(), key);
      if (keyComp == 0) {
         if (numRecords > 0) {
            if (overwriteHook.check(oldLeafNode.file, oldLeafNode.offset)) {
               newPrefix = prefix;
               newKeyParts = keyParts;
               newLeafNodes = new LeafNode[leafNodes.length];
               System.arraycopy(leafNodes, 0, newLeafNodes, 0, leafNodes.length);
               // Do not update the file and offset for DROPPED IndexRequests
               if (recordChange == RecordChange.INCREASE || recordChange == RecordChange.MOVE) {
                  if (trace) {
                     log.trace(String.format("Overwriting %d:%d with %d:%d (%d)",
                           oldLeafNode.file, oldLeafNode.offset, file, offset, numRecords));
                  }
                  newLeafNodes[insertPart] = new LeafNode(file, offset, numRecords);
                  segment.getCompactor().free(oldLeafNode.file, hak.getHeader().totalLength());
               } else {
                  if (trace) {
                     log.trace(String.format("Updating num records for %d:%d to %d", oldLeafNode.file, oldLeafNode.offset, numRecords));
                  }
                  newLeafNodes[insertPart] = new LeafNode(oldLeafNode.file, oldLeafNode.offset, numRecords);
               }
               overwriteHook.setOverwritten(true, oldLeafNode.file, oldLeafNode.offset);
            } else {
               overwriteHook.setOverwritten(false, -1, -1);
               segment.getCompactor().free(file, size);
               return this;
            }
         } else {
            overwriteHook.setOverwritten(true, oldLeafNode.file, oldLeafNode.offset);
            if (keyParts.length <= 1) {
               newPrefix = Util.EMPTY_BYTE_ARRAY;
               newKeyParts = Util.EMPTY_BYTE_ARRAY_ARRAY;
            } else {
               newPrefix = prefix;
               newKeyParts = new byte[keyParts.length - 1][];
               if (insertPart == keyParts.length) {
                  System.arraycopy(keyParts, 0, newKeyParts, 0, newKeyParts.length);
               } else {
                  System.arraycopy(keyParts, 0, newKeyParts, 0, insertPart);
                  System.arraycopy(keyParts, insertPart + 1, newKeyParts, insertPart, newKeyParts.length - insertPart);
               }
            }
            if (leafNodes.length > 0) {
               newLeafNodes = new LeafNode[leafNodes.length - 1];
               System.arraycopy(leafNodes, 0, newLeafNodes, 0, insertPart);
               System.arraycopy(leafNodes, insertPart + 1, newLeafNodes, insertPart, newLeafNodes.length - insertPart);
            } else {
               newLeafNodes = leafNodes;
            }
            segment.getCompactor().free(oldLeafNode.file, hak.getHeader().totalLength());
         }
      } else {
         // IndexRequest cannot be MOVED or DROPPED when the key is not in the index
         assert recordChange == RecordChange.INCREASE;
         overwriteHook.setOverwritten(false, -1, -1);
         // We have to insert the record even if this is a delete request and the key was not found
         // because otherwise we would have incorrect numRecord count. Eventually, Compactor will
         // drop the tombstone and update index, removing this node
         if (keyParts.length == 0) {
            // TODO: we may use unnecessarily long keys here and the key is never shortened
            newPrefix = keyComp > 0 ? key : hak.getKey();
         } else {
            newPrefix = commonPrefix(prefix, key);
         }
         newKeyParts = new byte[keyParts.length + 1][];
         newLeafNodes = new LeafNode[leafNodes.length + 1];
         copyKeyParts(keyParts, 0, newKeyParts, 0, insertPart, prefix, newPrefix);
         copyKeyParts(keyParts, insertPart, newKeyParts, insertPart + 1, keyParts.length - insertPart, prefix, newPrefix);
         if (keyComp > 0) {
            newKeyParts[insertPart] = substring(key, newPrefix.length, keyComp);
            System.arraycopy(leafNodes, 0, newLeafNodes, 0, insertPart + 1);
            System.arraycopy(leafNodes, insertPart + 1, newLeafNodes, insertPart + 2, leafNodes.length - insertPart - 1);
            newLeafNodes[insertPart + 1] = new LeafNode(file, offset, (short) 1);
         } else {
            newKeyParts[insertPart] = substring(hak.getKey(), newPrefix.length, -keyComp);
            System.arraycopy(leafNodes, 0, newLeafNodes, 0, insertPart);
            System.arraycopy(leafNodes, insertPart, newLeafNodes, insertPart + 1, leafNodes.length - insertPart);
            newLeafNodes[insertPart] = new LeafNode(file, offset, (short) 1);
         }
      }
      return new IndexNode(segment, newPrefix, newKeyParts, newLeafNodes);
   }
908925
void recycleSegment(final CommitLogSegment segment)
    {
        boolean archiveSuccess = CommitLog.instance.archiver.maybeWaitForArchiving(segment.getName());
        activeSegments.remove(segment);
        if (!archiveSuccess)
        {
            // if archiving (command) was not successful then leave the file alone. don't delete or recycle.
            discardSegment(segment, false);
            return;
        }
        if (isCapExceeded())
        {
            discardSegment(segment, true);
            return;
        }

        logger.debug("Recycling {}", segment);
        segmentManagementTasks.add(new Callable<CommitLogSegment>()
        {
            public CommitLogSegment call()
            {
                return segment.recycle();
            }
        });
    }
574926
@Override
  public LocatedBlock syncBlock(
      Block block,
      List<BlockRecord> syncList,
    boolean closeFile, List<InterDatanodeProtocol> datanodeProxies,
    long deadline
  ) 
    throws IOException {
    if (LOG.isDebugEnabled()) {
      LOG.debug("block=" + block + ", (length=" + block.getNumBytes()
          + "), syncList=" + syncList + ", closeFile=" + closeFile);
    }

    //syncList.isEmpty() that all datanodes do not have the block
    //so the block can be deleted.
    if (syncList.isEmpty()) {
      bsp.commitBlockSynchronization(block, 0, 0, closeFile, true,
          DatanodeID.EMPTY_ARRAY);
      return null;
    }

    List<DatanodeID> successList = new ArrayList<DatanodeID>();

    throwIfAfterTime(deadline);
    long generationstamp = -1;
    try {
      generationstamp = bsp.nextGenerationStamp(block, closeFile);
    } catch (RemoteException e) {
      if (e.unwrapRemoteException() instanceof BlockAlreadyCommittedException) {
        throw new BlockAlreadyCommittedException(e);
      } else {
        throw e;
      }
    }
    Block newblock = new Block(block.getBlockId(), block.getNumBytes(), generationstamp);

    for(BlockRecord r : syncList) {
      try {
        throwIfAfterTime(deadline);
        LOG.info("Updating block " + r + " to " + newblock);
        r.datanode.updateBlock(namespaceId, r.info.getBlock(), newblock, closeFile);
        successList.add(r.id);
      } catch (BlockRecoveryTimeoutException e) {
        throw e;
      } catch (IOException e) {
        InterDatanodeProtocol.LOG.warn("Failed to updateBlock (newblock="
            + newblock + ", datanode=" + r.id + ")", e);
      }
    }

    LOG.info("Updated blocks on syncList for block " + block + " to " + newblock);

    DataNode.stopAllProxies(datanodeProxies);

    if (!successList.isEmpty()) {
      DatanodeID[] nlist = successList.toArray(new DatanodeID[successList.size()]);

      throwIfAfterTime(deadline);
      bsp.commitBlockSynchronization(block,
          newblock.getGenerationStamp(), newblock.getNumBytes(), closeFile, false,
          nlist);
      DatanodeInfo[] info = new DatanodeInfo[nlist.length];
      for (int i = 0; i < nlist.length; i++) {
        info[i] = new DatanodeInfo(nlist[i]);
      }
      return new LocatedBlock(newblock, info); // success
    }

    //failed
    StringBuilder b = new StringBuilder();
    for(BlockRecord r : syncList) {
      b.append("\n  " + r.id);
    }
    throw new IOException("Cannot recover " + block + ", none of these "
        + syncList.size() + " datanodes success {" + b + "\n}");
  }
1273751
public void commit( SynchronizationBarrier barrier ) throws JournalException
    {
    	// Refuse any further operation in case of previous failure
    	if (failing)
    		throw new JournalException("Store journal is failing");
    	
    	// Register as a barrier participant
    	if (barrier != null)
    		barrier.addParty();
        
        boolean newFlushRequired;
        synchronized (journalWriteQueue)
        {
        	writeCommit(barrier);
        	
        	newFlushRequired = !flushingJournal;
        	if (newFlushRequired)
        		flushingJournal = true;
        	unflushedJournalSize = 0;
        }
        
        if (newFlushRequired)
        {
	        // Flush journal to store (asynchronous)
	        try
	        {
	        	asyncTaskManager.execute(flushJournalAsyncTask);
	        }
	    	catch (JMSException e)
	    	{
	    		throw new JournalException("Cannot flush journal asynchronously : "+e);
	    	}
        }
    }
15112
private void maybeRotateLog() throws IOException {
    if (!mRotateLogForNextWrite) {
      return;
    }
    if (mJournalOutputStream != null) {
      mJournalOutputStream.close();
      mJournalOutputStream = null;
    }

    createNewLogFile(mNextSequenceNumber);
    mRotateLogForNextWrite = false;
  }
1052420
private void setupCurrentEditStream(long txid) throws IOException {
    // get new stream
    currentEditLogInputStream = JournalSet.getInputStream(remoteJournalManager,
        txid);
    // we just started a new log segment
    currentSegmentTxId = txid;
    // indicate that we successfully reopened the stream
    mostRecentlyReadTransactionTime = now();
  }
974034
@Nullable
  static UfsJournalFile decodeCheckpointFile(UfsJournal journal, String filename) {
    URI location = URIUtils.appendPathOrDie(journal.getCheckpointDir(), filename);
    try {
      String[] parts = filename.split("-");

      // There can be temporary files in logs directory. Skip them.
      if (parts.length != 2) {
        return null;
      }
      long start = Long.decode(parts[0]);
      long end = Long.decode(parts[1]);

      Preconditions.checkState(start == 0);
      return UfsJournalFile.createCheckpointFile(location, end);
    } catch (IllegalStateException e) {
      LOG.error("Illegal journal file {}.", location);
      throw e;
    } catch (NumberFormatException e) {
      // There can be temporary files (e.g. created for rename).
      return null;
    }
  }
1344535
public void replayTo(Log destination) {
		synchronized (this.lines) {
			for (Line line : this.lines) {
				logTo(destination, line.getLevel(), line.getMessage(),
						line.getThrowable());
			}
			this.lines.clear();
		}
	}
1020722
def _sync_extract(self, from_path, method, to_path):
    """"""
    to_path_tmp = '%s%s_%s' % (to_path, constants.INCOMPLETE_SUFFIX,
                               uuid.uuid4().hex)
    try:
      for path, handle in iter_archive(from_path, method):
        _copy(handle, path and os.path.join(to_path_tmp, path) or to_path_tmp)
    except BaseException as err:
      msg = 'Error while extracting %s to %s : %s' % (from_path, to_path, err)
      raise ExtractError(msg)
    # `tf.io.gfile.Rename(overwrite=True)` doesn't work for non empty
    # directories, so delete destination first, if it already exists.
    if tf.io.gfile.exists(to_path):
      tf.io.gfile.rmtree(to_path)
    tf.io.gfile.rename(to_path_tmp, to_path)
    self._pbar_path.update(1)
    return to_path
524272
private void emitRecordWithTimestampAndPeriodicWatermark(
			T record, KafkaTopicPartitionState<KPH> partitionState, long offset, long kafkaEventTimestamp) {
		@SuppressWarnings("unchecked")
		final KafkaTopicPartitionStateWithPeriodicWatermarks<T, KPH> withWatermarksState =
				(KafkaTopicPartitionStateWithPeriodicWatermarks<T, KPH>) partitionState;

		// extract timestamp - this accesses/modifies the per-partition state inside the
		// watermark generator instance, so we need to lock the access on the
		// partition state. concurrent access can happen from the periodic emitter
		final long timestamp;
		//noinspection SynchronizationOnLocalVariableOrMethodParameter
		synchronized (withWatermarksState) {
			timestamp = withWatermarksState.getTimestampForRecord(record, kafkaEventTimestamp);
		}

		// emit the record with timestamp, using the usual checkpoint lock to guarantee
		// atomicity of record emission and offset state update
		synchronized (checkpointLock) {
			sourceContext.collectWithTimestamp(record, timestamp);
			partitionState.setOffset(offset);
		}
	}
1312140
def _read_journal(self):
        """"""
        root = self._filesystem.inspect_get_roots()[0]
        inode = self._filesystem.stat('C:\\$Extend\\$UsnJrnl')['ino']

        with NamedTemporaryFile(buffering=0) as tempfile:
            self._filesystem.download_inode(root, inode, tempfile.name)

            journal = usn_journal(tempfile.name)

            return parse_journal(journal)
718964
private void joinSynchronization()
      throws IOException, TimeoutException, InterruptedException {
    Message sync = MessageBuilder.buildSyncHistory(persistence.getLatestZxid());
    sendMessage(this.electedLeader, sync);
    MessageTuple tuple =
      filter.getExpectedMessage(MessageType.SYNC_HISTORY_REPLY, electedLeader,
                                config.getTimeoutMs());
    Message msg = tuple.getMessage();
    // Updates the sync timeout based on leader's suggestion.
    setSyncTimeoutMs(msg.getSyncHistoryReply().getSyncTimeout());
    // Waits for the first synchronization completes.
    waitForSync(this.electedLeader);

    // Gets the last zxid in disk after first synchronization.
    Zxid lastZxid = persistence.getLatestZxid();
    LOG.debug("After first synchronization, the last zxid is {}", lastZxid);
    // Then issues the JOIN message.
    Message join = MessageBuilder.buildJoin(lastZxid);
    sendMessage(this.electedLeader, join);

    /* -- Synchronizing phase -- */
    changePhase(Phase.SYNCHRONIZING);
    waitForSync(this.electedLeader);
    waitForNewLeaderMessage();
    waitForCommitMessage();
    persistence.setProposedEpoch(persistence.getAckEpoch());
  }