All shards provided to this method will be considered for creating a compaction set.
774257
public Set<OrganizationSet> createCompactionSets(Table tableInfo, Collection<ShardIndexInfo> shards)
    {
        Collection<Collection<ShardIndexInfo>> shardsByDaysBuckets = getShardsByDaysBuckets(tableInfo, shards, temporalFunction);

        ImmutableSet.Builder<OrganizationSet> compactionSets = ImmutableSet.builder();
        for (Collection<ShardIndexInfo> shardInfos : shardsByDaysBuckets) {
            compactionSets.addAll(buildCompactionSets(tableInfo, ImmutableSet.copyOf(shardInfos)));
        }
        return compactionSets.build();
    }
852773
def push_all(self, record_shard_pairs):
        
        # Faster than inserting one at a time; the heap is sorted once after all inserts.
        for record, shard in record_shard_pairs:
            item = heap_item(self.clock, record, shard)
            self.heap.append(item)
        heapq.heapify(self.heap)
980335
private MRCompactorJobRunner getMRCompactorJobRunner(Dataset dataset) {
    try {
      @SuppressWarnings("unchecked")
      Class<? extends MRCompactorJobRunner> cls = (Class<? extends MRCompactorJobRunner>) Class
          .forName(this.state.getProp(COMPACTION_JOB_RUNNER_CLASS, DEFAULT_COMPACTION_JOB_RUNNER_CLASS));
      return cls.getDeclaredConstructor(Dataset.class, FileSystem.class).newInstance(dataset, this.fs);
    } catch (Exception e) {
      throw new RuntimeException("Cannot instantiate MRCompactorJobRunner", e);
    }
  }
1002267
public static <E> Set<E> createHashSet(int expectedMapSize) {
        final int initialCapacity = (int) (expectedMapSize / HASHSET_DEFAULT_LOAD_FACTOR) + 1;
        return new HashSet<E>(initialCapacity, HASHSET_DEFAULT_LOAD_FACTOR);
    }
893473
private void scheduleAllDeliveries()
    {
        logger.debug("Started scheduleAllDeliveries");

        // Force a major compaction to get rid of the tombstones and expired hints. Do it once, before we schedule any
        // individual replay, to avoid N - 1 redundant individual compactions (when N is the number of nodes with hints
        // to deliver to).
        compact();

        IPartitioner p = StorageService.getPartitioner();
        RowPosition minPos = p.getMinimumToken().minKeyBound();
        Range<RowPosition> range = new Range<>(minPos, minPos, p);
        IDiskAtomFilter filter = new NamesQueryFilter(ImmutableSortedSet.<CellName>of());
        List<Row> rows = hintStore.getRangeSlice(range, null, filter, Integer.MAX_VALUE, System.currentTimeMillis());
        for (Row row : rows)
        {
            UUID hostId = UUIDGen.getUUID(row.key.getKey());
            InetAddress target = StorageService.instance.getTokenMetadata().getEndpointForHostId(hostId);
            // token may have since been removed (in which case we have just read back a tombstone)
            if (target != null)
                scheduleHintDelivery(target, false);
        }

        logger.debug("Finished scheduleAllDeliveries");
    }
115094
private static int[] allocateShardsToNodes(int numShards, int numNodes, List<Set<Integer>> weightPerServer) {
      int[] shardsPerServer = new int[numNodes];
      Iterator<Integer> cyclicNodeIterator = Stream.iterate(0, i -> (i + 1) % numNodes).iterator();
      while (numShards > 0) {
         int slot = cyclicNodeIterator.next();
         if (!weightPerServer.get(slot).isEmpty()) {
            shardsPerServer[slot]++;
            numShards--;
         }
      }
      return shardsPerServer;
   }
399329
private static void initialize(Storage storage, String counterId) {
    LOG.debug("Initializing counter shards for resource {}", counterId);
    for (int startIndex = 0; startIndex < NUM_SHARDS; startIndex += TRANSACTION_GROUP_SIZE) {
      initShardRange(storage, counterId, startIndex,
                     Math.min(NUM_SHARDS, startIndex + TRANSACTION_GROUP_SIZE));
    }
  }
65845
public void save (FileSystemDataset dataset, State state) {
    state.setProp(SERIALIZE_COMPACTION_FILE_PATH_NAME, dataset.datasetURN());
  }
301896
private void createPartitions() {

        for (Map.Entry<String, String> partitionEntry : this.partitions.entrySet()) {
            try {
                this.addPartitionInternal(partitionEntry.getKey(), partitionEntry.getValue());
            } catch (Exception e) { //NOSONAR
                throw new AssertionError("Could not create partitions " + this.partitions, e);
            }
        }
    }
1261891
private void submitSlaEvent(Job job) {
    try {
      CompactionSlaEventHelper
          .getEventSubmitterBuilder(this.dataset, Optional.of(job), this.fs)
          .eventSubmitter(this.eventSubmitter)
          .eventName(CompactionSlaEventHelper.COMPACTION_COMPLETED_EVENT_NAME)
          .additionalMetadata(
              CompactionSlaEventHelper.LATE_RECORD_COUNT,
              Long.toString(this.lateOutputRecordCountProvider.getRecordCount(this.getApplicableFilePaths(this.dataset
                  .outputLatePath(), this.fs))))
          .additionalMetadata(
              CompactionSlaEventHelper.REGULAR_RECORD_COUNT,
              Long.toString(this.outputRecordCountProvider.getRecordCount(this.getApplicableFilePaths(this.dataset
                  .outputPath(), this.fs))))
          .additionalMetadata(CompactionSlaEventHelper.RECOMPATED_METADATA_NAME,
              Boolean.toString(this.dataset.needToRecompact())).build().submit();
    } catch (Throwable e) {
      LOG.warn("Failed to submit compaction completed event:" + e, e);
    }
  }
580028
public void compact() {
    if (prev == this) throw new IllegalStateException();
    if (!prev.owner) return; // Cannot compact: prev isn't writable.
    int byteCount = limit - pos;
    int availableByteCount = SIZE - prev.limit + (prev.shared ? 0 : prev.pos);
    if (byteCount > availableByteCount) return; // Cannot compact: not enough writable space.
    writeTo(prev, byteCount);
    pop();
    SegmentPool.recycle(this);
  }
268438
def finalize(self, ctx, shard_state):
    """"""
    filenames = []
    for filehandle in self._filehandles:
      filenames.append(filehandle.name)
      filehandle.close()
    shard_state.writer_state = {"shard_filenames": filenames}
1270955
private void createPersistentSubsets(TransactionCommon tran) throws SIResourceException
  {
    if (TraceComponent.isAnyTracingEnabled() && tc.isEntryEnabled())
      SibTr.entry(this, tc, "createPersistentSubsets", tran);
          
    for(int i=0; i<subsets.length; i++)
    {
      createPersistentSubset(getReliability(i),tran);
    }
    
    if (TraceComponent.isAnyTracingEnabled() && tc.isEntryEnabled())
      SibTr.exit(tc, "createPersistentSubsets");
  }
843911
def _combine_sets(self, sets, final_set):
        
        if self._has_sortedsets:
            self.cls.get_connection().zinterstore(final_set, list(sets))
        else:
            final_set = super(ExtendedCollectionManager, self)._combine_sets(sets, final_set)
        return final_set
1298954
protected boolean consumeCompactionBatch() {
        /*
         * Consume one compaction update batch generated by the compactor.
         */
        CompactionUpdateBatch updateBatch = _compactor.pollCompactionBatch();
        if(updateBatch != null) {
            try {
                consumeCompaction(updateBatch);
            } catch (Exception e) {
                _log.error("failed to consume compaction batch " + updateBatch.getDescriptiveId(), e);
            } finally {
                _compactor.recycleCompactionBatch(updateBatch);
            }
            return true;
        }
        
        return false;
    }
590324
public void setShards(java.util.Collection<Shard> shards) {
        if (shards == null) {
            this.shards = null;
            return;
        }

        this.shards = new com.amazonaws.internal.SdkInternalList<Shard>(shards);
    }
332170
private void scheduleCompaction(long lastApplied) {
    // Schedule compaction after a randomized delay to discourage snapshots on multiple nodes at the same time.
    logger.trace("Scheduling compaction in {}", COMPACT_DELAY);
    stateContext.schedule(COMPACT_DELAY, () -> compactLogs(lastApplied));
  }