Helper method to create marker documents for each partition.

@param core the core reference.
@param bucket the name of the bucket.
@return a list of created flush marker IDs once they are completely upserted.
356420
private static Observable<List<String>> createMarkerDocuments(final ClusterFacade core, final String bucket) {
        return Observable
            .from(FLUSH_MARKERS)
            .flatMap(new Func1<String, Observable<UpsertResponse>>() {
                @Override
                public Observable<UpsertResponse> call(final String id) {
                    return deferAndWatch(new Func1<Subscriber, Observable<? extends UpsertResponse>>() {
                        @Override
                        public Observable<? extends UpsertResponse> call(final Subscriber subscriber) {
                            UpsertRequest request = new UpsertRequest(id, Unpooled.copiedBuffer(id, CharsetUtil.UTF_8), bucket);
                            request.subscriber(subscriber);
                            return core.send(request);
                        }
                    });
                }
            })
            .doOnNext(new Action1<UpsertResponse>() {
                @Override
                public void call(UpsertResponse response) {
                    if (response.content() != null && response.content().refCnt() > 0) {
                        response.content().release();
                    }
                }
            })
            .last()
            .map(new Func1<UpsertResponse, List<String>>() {
                @Override
                public List<String> call(UpsertResponse response) {
                    return FLUSH_MARKERS;
                }
            });
    }
1136192
static <T extends Comparable<T>> void bucketSort(final List<T> c, final int fromIndex, final int toIndex) {
        bucketSort(c, fromIndex, toIndex, null);
    }
154143
static <T> void bucketSort(final List<? extends T> c, final int fromIndex, final int toIndex, final Comparator<? super T> cmp) {
        N.checkFromToIndex(fromIndex, toIndex, c == null ? 0 : c.size());

        if ((N.isNullOrEmpty(c) && fromIndex == 0 && toIndex == 0) || fromIndex == toIndex) {
            return;
        }

        if (toIndex - fromIndex < 32) {
            sort(c, fromIndex, toIndex, cmp);
            return;
        }

        final Comparator<? super T> comparator = cmp == null ? Comparators.NATURAL_ORDER : cmp;
        final Multiset<T> multiset = new Multiset<>();
        ListIterator<T> itr = (ListIterator<T>) c.listIterator(fromIndex);
        int i = fromIndex;

        while (itr.hasNext()) {
            if (i++ >= toIndex) {
                break;
            }

            multiset.add(itr.next());
        }

        final Map<T, Integer> m = multiset.toMapSortedBy(new Comparator<Map.Entry<T, MutableInt>>() {
            @Override
            public int compare(Entry<T, MutableInt> a, Entry<T, MutableInt> b) {
                return comparator.compare(a.getKey(), a.getKey());
            }
        });

        itr = (ListIterator<T>) c.listIterator(fromIndex);

        for (Map.Entry<T, Integer> entry : m.entrySet()) {
            final T key = entry.getKey();
            for (int j = 0; j < entry.getValue(); j++) {
                itr.next();
                itr.set(key);
            }
        }
    }
630366
@Override
	protected TopicAndPartition createKafkaPartitionHandle(KafkaTopicPartition partition) {
		return new TopicAndPartition(partition.getTopic(), partition.getPartition());
	}
915607
@VisibleForTesting
  static BucketOptions createBucketOptions(
      @javax.annotation.Nullable
          io.opencensus.metrics.export.Distribution.BucketOptions bucketOptions) {
    final BucketOptions.Builder builder = BucketOptions.newBuilder();
    if (bucketOptions == null) {
      return builder.build();
    }

    return bucketOptions.match(
        bucketOptionsExplicitFunction, Functions.<BucketOptions>throwIllegalArgumentException());
  }
1349663
private void createIdIndex(String bucketName)
    {
        Bucket bucket = null;
        try
        {
            bucket = CouchbaseBucketUtils.openBucket(cluster, bucketName, csmd.getBucketProperty("bucket.password"));

            /*
             * Ignoring if indexes pre-exist
             */

            bucket.bucketManager().createN1qlPrimaryIndex(buildIndexName(bucketName), true, false);

            LOGGER.debug("Niql primary Index are created for bucket [" + bucketName + "].");
        }
        catch (CouchbaseException cex)
        {
            LOGGER.error("Not able to create Niql primary index for bucket [" + bucketName + "].", cex);
            throw new KunderaException("Not able to create Niql primary index for bucket [" + bucketName + "].", cex);
        }

        finally
        {
            CouchbaseBucketUtils.closeBucket(bucket);
        }

    }
899001
void insertPoint(Point p){
		
		//check if there is enough space in the first bucket
		int cursize = this.buckets[0].cursize;	
		if(cursize >= this.maxBucketsize) {
			//printf("Bucket 0 full \n");
			//start spillover process
			int curbucket  = 0;
			int nextbucket = 1;

			//check if the next bucket is empty
			if(this.buckets[nextbucket].cursize == 0){
				//copy the bucket	
				int i;
				for(i=0; i<this.maxBucketsize; i++){
					this.buckets[nextbucket].points[i] = this.buckets[curbucket].points[i].clone();
					//copyPointWithoutInit: we should not copy coordinates? 
				}
				//bucket is now full
				this.buckets[nextbucket].cursize = this.maxBucketsize;
				//first bucket is now empty
				this.buckets[curbucket].cursize = 0;
				cursize = 0;
			} else {
				//printf("Bucket %d full \n",nextbucket);
				//copy bucket to spillover and continue
				int i;
				for(i=0;i<this.maxBucketsize;i++){
					this.buckets[nextbucket].spillover[i] = this.buckets[curbucket].points[i].clone();
					//copyPointWithoutInit: we should not copy coordinates? 
				}
				this.buckets[0].cursize=0;
				cursize = 0;
				curbucket++;
				nextbucket++;
				/*
				as long as the next bucket is full output the coreset to the spillover of the next bucket
				*/
				while(this.buckets[nextbucket].cursize == this.maxBucketsize){
					//printf("Bucket %d full \n",nextbucket);
					this.treeCoreset.unionTreeCoreset(this.maxBucketsize,this.maxBucketsize,
						this.maxBucketsize,p.dimension, 
						this.buckets[curbucket].points,this.buckets[curbucket].spillover,
						this.buckets[nextbucket].spillover, this.clustererRandom);
					//bucket now empty
					this.buckets[curbucket].cursize = 0;
					curbucket++;
					nextbucket++;
				}
				this.treeCoreset.unionTreeCoreset(this.maxBucketsize,this.maxBucketsize,
						this.maxBucketsize,p.dimension, 
						this.buckets[curbucket].points,this.buckets[curbucket].spillover,
						this.buckets[nextbucket].points, this.clustererRandom);
				this.buckets[curbucket].cursize = 0;
				this.buckets[nextbucket].cursize = this.maxBucketsize;
			}
		}
		//insert point into the first bucket
		this.buckets[0].points[cursize] = p.clone();
		//copyPointWithoutInit: we should not copy coordinates? 
		this.buckets[0].cursize++;
	}
577889
@Override
    void createEntry(int hash, K key, V value, int bucketIndex) {
	HashMapPro.Entry<K, V> old = table[bucketIndex];
	Entry<K, V> e = new Entry<K, V>(hash, key, value, old);
	table[bucketIndex] = e;
	e.addBefore(header);
	size++;
    }
716936
private boolean resizeHashTable() throws IOException {
		final int newNumBuckets = 2*this.numBuckets;
		final int bucketsPerSegment = this.bucketsPerSegmentMask + 1;
		final int newNumSegments = (newNumBuckets + (bucketsPerSegment-1)) / bucketsPerSegment;
		final int additionalSegments = newNumSegments-this.buckets.length;
		final int numPartitions = this.partitions.size();
		
		if (this.availableMemory.size() < additionalSegments) {
			for (int i = 0; i < numPartitions; i++) {
				compactPartition(i);
				if(this.availableMemory.size() >= additionalSegments) {
					break;
				}
			}
		}
		
		if (this.availableMemory.size() < additionalSegments || this.closed) {
			return false;
		}
		else {
			this.isResizing = true;
			// allocate new buckets
			final int startOffset = (this.numBuckets * HASH_BUCKET_SIZE) % this.segmentSize;
			final int oldNumBuckets = this.numBuckets;
			final int oldNumSegments = this.buckets.length;
			MemorySegment[] mergedBuckets = new MemorySegment[newNumSegments];
			System.arraycopy(this.buckets, 0, mergedBuckets, 0, this.buckets.length);
			this.buckets = mergedBuckets;
			this.numBuckets = newNumBuckets;
			// initialize all new buckets
			boolean oldSegment = (startOffset != 0);
			final int startSegment = oldSegment ? (oldNumSegments-1) : oldNumSegments;
			for (int i = startSegment, bucket = oldNumBuckets; i < newNumSegments && bucket < this.numBuckets; i++) {
				MemorySegment seg;
				int bucketOffset;
				if(oldSegment) { // the first couple of new buckets may be located on an old segment
					seg = this.buckets[i];
					for (int k = (oldNumBuckets % bucketsPerSegment) ; k < bucketsPerSegment && bucket < this.numBuckets; k++, bucket++) {
						bucketOffset = k * HASH_BUCKET_SIZE;	
						// initialize the header fields
						seg.put(bucketOffset + HEADER_PARTITION_OFFSET, assignPartition(bucket, (byte)numPartitions));
						seg.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0);
						seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET);
					}
				} else {
					seg = getNextBuffer();
					// go over all buckets in the segment
					for (int k = 0; k < bucketsPerSegment && bucket < this.numBuckets; k++, bucket++) {
						bucketOffset = k * HASH_BUCKET_SIZE;
						// initialize the header fields
						seg.put(bucketOffset + HEADER_PARTITION_OFFSET, assignPartition(bucket, (byte)numPartitions));
						seg.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0);
						seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET);
					}
				}				
				this.buckets[i] = seg;
				oldSegment = false; // we write on at most one old segment
			}
			int hashOffset;
			int hash;
			int pointerOffset;
			long pointer;
			IntArrayList hashList = new IntArrayList(NUM_ENTRIES_PER_BUCKET);
			LongArrayList pointerList = new LongArrayList(NUM_ENTRIES_PER_BUCKET);
			IntArrayList overflowHashes = new IntArrayList(64);
			LongArrayList overflowPointers = new LongArrayList(64);
			
			// go over all buckets and split them between old and new buckets
			for (int i = 0; i < numPartitions; i++) {
				InMemoryPartition<T> partition = this.partitions.get(i);
				final MemorySegment[] overflowSegments = partition.overflowSegments;
				
				int posHashCode;
				for (int j = 0, bucket = i; j < this.buckets.length && bucket < oldNumBuckets; j++) {
					MemorySegment segment = this.buckets[j];
					// go over all buckets in the segment belonging to the partition
					for (int k = bucket % bucketsPerSegment; k < bucketsPerSegment && bucket < oldNumBuckets; k += numPartitions, bucket += numPartitions) {
						int bucketOffset = k * HASH_BUCKET_SIZE;
						if((int)segment.get(bucketOffset + HEADER_PARTITION_OFFSET) != i) {
							throw new IOException("Accessed wrong bucket! wanted: " + i + " got: " + segment.get(bucketOffset + HEADER_PARTITION_OFFSET));
						}
						// loop over all segments that are involved in the bucket (original bucket plus overflow buckets)
						int countInSegment = segment.getInt(bucketOffset + HEADER_COUNT_OFFSET);
						int numInSegment = 0;
						pointerOffset = bucketOffset + BUCKET_POINTER_START_OFFSET;
						hashOffset = bucketOffset + BUCKET_HEADER_LENGTH;
						while (true) {
							while (numInSegment < countInSegment) {
								hash = segment.getInt(hashOffset);
								if((hash % this.numBuckets) != bucket && (hash % this.numBuckets) != (bucket+oldNumBuckets)) {
									throw new IOException("wanted: " + bucket + " or " + (bucket + oldNumBuckets) + " got: " + hash%this.numBuckets);
								}
								pointer = segment.getLong(pointerOffset);
								hashList.add(hash);
								pointerList.add(pointer);
								pointerOffset += POINTER_LEN;
								hashOffset += HASH_CODE_LEN;
								numInSegment++;
							}
							// this segment is done. check if there is another chained bucket
							final long forwardPointer = segment.getLong(bucketOffset + HEADER_FORWARD_OFFSET);
							if (forwardPointer == BUCKET_FORWARD_POINTER_NOT_SET) {
								break;
							}
							final int overflowSegNum = (int) (forwardPointer >>> 32);
							segment = overflowSegments[overflowSegNum];
							bucketOffset = (int) forwardPointer;
							countInSegment = segment.getInt(bucketOffset + HEADER_COUNT_OFFSET);
							pointerOffset = bucketOffset + BUCKET_POINTER_START_OFFSET;
							hashOffset = bucketOffset + BUCKET_HEADER_LENGTH;
							numInSegment = 0;
						}
						segment = this.buckets[j];
						bucketOffset = k * HASH_BUCKET_SIZE;
						// reset bucket for re-insertion
						segment.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0);
						segment.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET);
						// refill table
						if(hashList.size() != pointerList.size()) {
							throw new IOException("Pointer and hash counts do not match. hashes: " + hashList.size() + " pointer: " + pointerList.size());
						}
						int newSegmentIndex = (bucket + oldNumBuckets) / bucketsPerSegment;
						MemorySegment newSegment = this.buckets[newSegmentIndex];
						
						// we need to avoid overflows in the first run
						int oldBucketCount = 0;
						int newBucketCount = 0;
						while (!hashList.isEmpty()) {
							hash = hashList.removeLast();
							pointer = pointerList.removeLong(pointerList.size()-1);
							posHashCode = hash % this.numBuckets;
							if (posHashCode == bucket && oldBucketCount < NUM_ENTRIES_PER_BUCKET) {
								bucketOffset = (bucket % bucketsPerSegment) * HASH_BUCKET_SIZE;
								insertBucketEntryFromStart(segment, bucketOffset, hash, pointer, partition.getPartitionNumber());
								oldBucketCount++;
							}
							else if (posHashCode == (bucket + oldNumBuckets) && newBucketCount < NUM_ENTRIES_PER_BUCKET) {
								bucketOffset = ((bucket + oldNumBuckets) % bucketsPerSegment) * HASH_BUCKET_SIZE;
								insertBucketEntryFromStart(newSegment, bucketOffset, hash, pointer, partition.getPartitionNumber());
								newBucketCount++;
							}
							else if (posHashCode == (bucket + oldNumBuckets) || posHashCode == bucket) {
								overflowHashes.add(hash);
								overflowPointers.add(pointer);
							}
							else {
								throw new IOException("Accessed wrong bucket. Target: " + bucket + " or " + (bucket + oldNumBuckets) + " Hit: " + posHashCode);
							}
						}
						hashList.clear();
						pointerList.clear();
					}
				}
				// reset partition's overflow buckets and reclaim their memory
				this.availableMemory.addAll(partition.resetOverflowBuckets());
				// clear overflow lists
				int bucketArrayPos;
				int bucketInSegmentPos;
				MemorySegment bucket;
				while(!overflowHashes.isEmpty()) {
					hash = overflowHashes.removeLast();
					pointer = overflowPointers.removeLong(overflowPointers.size()-1);
					posHashCode = hash % this.numBuckets; 
					bucketArrayPos = posHashCode >>> this.bucketsPerSegmentBits;
					bucketInSegmentPos = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS;
					bucket = this.buckets[bucketArrayPos];
					insertBucketEntryFromStart(bucket, bucketInSegmentPos, hash, pointer, partition.getPartitionNumber());
				}
				overflowHashes.clear();
				overflowPointers.clear();
			}
			this.isResizing = false;
			return true;
		}
	}
317208
static List<PartitionDefinition> findSlicePartitions(RestClient client, Settings settings, MappingSet mappingSet,
                                                         Map<String, NodeInfo> nodes, List<List<Map<String, Object>>> shards, Log log) {
        QueryBuilder query = QueryUtils.parseQueryAndFilters(settings);
        Integer maxDocsPerPartition = settings.getMaxDocsPerPartition();
        Assert.notNull(maxDocsPerPartition, "Attempting to find slice partitions but maximum documents per partition is not set.");
        Resource readResource = new Resource(settings, true);
        Mapping resolvedMapping = mappingSet == null ? null : mappingSet.getResolvedView();

        List<PartitionDefinition> partitions = new ArrayList<PartitionDefinition>(shards.size());
        for (List<Map<String, Object>> group : shards) {
            String index = null;
            int shardId = -1;
            List<String> locationList = new ArrayList<String> ();
            for (Map<String, Object> replica : group) {
                ShardInfo shard = new ShardInfo(replica);
                index = shard.getIndex();
                shardId = shard.getName();
                if (nodes.containsKey(shard.getNode())) {
                    locationList.add(nodes.get(shard.getNode()).getPublishAddress());
                }
            }
            String[] locations = locationList.toArray(new String[0]);
            if (index == null) {
                // Could not find shards for this partition. Continue anyway?
                if (settings.getIndexReadAllowRedStatus()) {
                    log.warn("Shard information is missing from an index and will not be reached during job execution. " +
                            "Assuming shard is unavailable and cluster is red! Continuing with read operation by " +
                            "skipping this shard! This may result in incomplete data retrieval!");
                } else {
                    throw new IllegalStateException("Could not locate shard information for one of the read indices. " +
                            "Check your cluster status to see if it is unstable!");
                }
            } else {
                // TODO applyAliasMetaData should be called in order to ensure that the count are exact (alias filters and routing may change the number of documents)
                long numDocs;
                if (readResource.isTyped()) {
                    numDocs = client.count(index, readResource.type(), Integer.toString(shardId), query);
                } else {
                    numDocs = client.countIndexShard(index, Integer.toString(shardId), query);
                }
                int numPartitions = (int) Math.max(1, numDocs / maxDocsPerPartition);
                for (int i = 0; i < numPartitions; i++) {
                    PartitionDefinition.Slice slice = new PartitionDefinition.Slice(i, numPartitions);
                    partitions.add(new PartitionDefinition(settings, resolvedMapping, index, shardId, slice, locations));
                }
            }
        }
        return partitions;
    }
10051
function (prefix, callback, marker, contents) {

			var search = {
				Prefix: prefix, 
				Bucket: options.bucket
			};

			if (marker) {
				search.Marker = marker;
			}

			s3.listObjects(search, function (err, list) { 

				if (!err) {

					var objects = (contents) ? contents.concat(list.Contents) : list.Contents;

					if (list.IsTruncated) {
						var new_marker = _.last(list.Contents).Key;
						listObjects(prefix, callback, new_marker, objects);
					}
					else {
						callback(_.uniq(objects, function (o) { return o.Key; }));
					}
				}
				else {
					grunt.fatal('Failed to list content of bucket ' + options.bucket + '\n' + err);
				}
			});
		}
1152036
private ListMultipartUploadsResult listIncompleteUploads(String bucketName, String keyMarker, String uploadIdMarker,
                                                           String prefix, String delimiter, int maxUploads)
    throws InvalidBucketNameException, NoSuchAlgorithmException, InsufficientDataException, IOException,
           InvalidKeyException, NoResponseException, XmlPullParserException, ErrorResponseException,
           InternalException {
    if (maxUploads < 0 || maxUploads > 1000) {
      maxUploads = 1000;
    }

    Map<String,String> queryParamMap = new HashMap<>();
    queryParamMap.put("uploads", "");
    queryParamMap.put("max-uploads", Integer.toString(maxUploads));

    if (prefix != null) {
      queryParamMap.put("prefix", prefix);
    } else {
      queryParamMap.put("prefix", "");
    }

    if (delimiter != null) {
      queryParamMap.put("delimiter", delimiter);
    } else {
      queryParamMap.put("delimiter", "");
    }

    if (keyMarker != null) {
      queryParamMap.put("key-marker", keyMarker);
    }

    if (uploadIdMarker != null) {
      queryParamMap.put("upload-id-marker", uploadIdMarker);
    }

    HttpResponse response = executeGet(bucketName, null, null, queryParamMap);

    ListMultipartUploadsResult result = new ListMultipartUploadsResult();
    result.parseXml(response.body().charStream());
    response.body().close();
    return result;
  }
361926
private String[] collectPartitionKeyAnnotations(Method method) {
        Annotation[][] annotations = method.getParameterAnnotations();
        String[] keyMappings = new String[annotations.length];
        boolean keyMappingFound = false;
        Map<String, Integer> unique = Maps.newHashMap();
        for (int i = 0; i < annotations.length; i++) {
            PartitionKey annotation = findPartitionKeyAnnotation(annotations[i]);
            if (annotation == null) {
                continue;
            }
            String key = checkNotNull(annotation.value());
            Integer prev = unique.put(key, i);
            checkState(prev == null, "Method '%s' has multiple arguments annotated with the same @PartitionKey " +
                    "value '%s': arguments %s and %s", method, key, prev, i);
            keyMappings[i] = key;
            keyMappingFound = true;
        }
        return keyMappingFound ? keyMappings : null;
    }
965602
protected final Buckets createBuckets(Stopwatch stopwatch) {
		if (stopwatch.getCounter() > warmupCounter) {
			Buckets buckets = createBucketsAfterWarmup(stopwatch);
			// Add retained splits to buckets
			buckets.addValues(getBucketsValues(stopwatch));
			removeBucketsValues(stopwatch);
			return buckets;
		} else {
			return null;
		}
	}
785609
@Override
	public void insert(T record) throws IOException {
		if (closed) {
			return;
		}

		final int hashCode = MathUtils.jenkinsHash(buildSideComparator.hash(record));
		final int bucket = hashCode & numBucketsMask;
		final int bucketSegmentIndex = bucket >>> numBucketsPerSegmentBits; // which segment contains the bucket
		final MemorySegment bucketSegment = bucketSegments[bucketSegmentIndex];
		final int bucketOffset = (bucket & numBucketsPerSegmentMask) << bucketSizeBits; // offset of the bucket in the segment
		final long firstPointer = bucketSegment.getLong(bucketOffset);

		try {
			final long newFirstPointer = recordArea.appendPointerAndRecord(firstPointer, record);
			bucketSegment.putLong(bucketOffset, newFirstPointer);
		} catch (EOFException ex) {
			compactOrThrow();
			insert(record);
			return;
		}

		numElements++;
		resizeTableIfNecessary();
	}
217267
private static RTreeIndexCoreExtension getRTreeIndexExtension(
			GeoPackageCore geoPackage) {
		return new RTreeIndexCoreExtension(geoPackage) {
			@Override
			public void createMinYFunction() {
			}

			@Override
			public void createMinXFunction() {
			}

			@Override
			public void createMaxYFunction() {
			}

			@Override
			public void createMaxXFunction() {
			}

			@Override
			public void createIsEmptyFunction() {
			}
		};
	}
764641
@Override
  public List<GoogleCloudStorageItemInfo> listBucketInfo()
      throws IOException {
    logger.atFine().log("listBucketInfo()");
    List<Bucket> allBuckets = listBucketsInternal();
    List<GoogleCloudStorageItemInfo> bucketInfos = new ArrayList<>(allBuckets.size());
    for (Bucket bucket : allBuckets) {
      bucketInfos.add(new GoogleCloudStorageItemInfo(
          new StorageResourceId(bucket.getName()), bucket.getTimeCreated().getValue(), 0,
          bucket.getLocation(), bucket.getStorageClass()));
    }
    return bucketInfos;
  }