Display topic broker imbalance statistics.

    :param cluster_topologies: A dictionary mapping a string name to a
        ClusterTopology object.
508866
def display_topic_broker_imbalance(cluster_topologies):
    
    broker_ids = list(next(six.itervalues(cluster_topologies)).brokers.keys())
    assert all(
        set(broker_ids) == set(cluster_topology.brokers.keys())
        for cluster_topology in six.itervalues(cluster_topologies)
    )
    topic_names = list(next(six.itervalues(cluster_topologies)).topics.keys())
    assert all(
        set(topic_names) == set(cluster_topology.topics.keys())
        for cluster_topology in six.itervalues(cluster_topologies)
    )

    imbalances = [
        stats.get_topic_imbalance_stats(
            [cluster_topology.brokers[broker_id] for broker_id in broker_ids],
            [cluster_topology.topics[tname] for tname in topic_names],
        )
        for cluster_topology in six.itervalues(cluster_topologies)
    ]
    weighted_imbalances = [
        stats.get_weighted_topic_imbalance_stats(
            [cluster_topology.brokers[broker_id] for broker_id in broker_ids],
            [cluster_topology.topics[tname] for tname in topic_names],
        )
        for cluster_topology in six.itervalues(cluster_topologies)
    ]

    _display_table_title_multicolumn(
        'Extra-Topic-Partition Count',
        'Brokers',
        broker_ids,
        list(cluster_topologies.keys()),
        [
            [i[1][broker_id] for broker_id in broker_ids]
            for i in imbalances
        ]
    )

    print('')

    _display_table_title_multicolumn(
        'Weighted Topic Imbalance',
        'Brokers',
        broker_ids,
        list(cluster_topologies.keys()),
        [
            [wi[1][broker_id] for broker_id in broker_ids]
            for wi in weighted_imbalances
        ]
    )

    for name, topic_imbalance, weighted_topic_imbalance in zip(
            six.iterkeys(cluster_topologies),
            (i[0] for i in imbalances),
            (wi[0] for wi in weighted_imbalances),
    ):
        print(
            '\n'
            '{name}'
            'Topic partition imbalance count: {topic_imbalance}\n'
            'Weighted topic partition imbalance: {weighted_topic_imbalance}'
            .format(
                name='' if len(cluster_topologies) == 1 else name + '\n',
                topic_imbalance=topic_imbalance,
                weighted_topic_imbalance=weighted_topic_imbalance,
            )
        )
975839
def initialise_clusters(self, input_, clustercount):
        
        # initialise the clusters with empty lists
        self.__clusters = []
        for _ in range(clustercount):
            self.__clusters.append([])

        # distribute the items into the clusters
        count = 0
        for item in input_:
            self.__clusters[count % clustercount].append(item)
            count += 1
259210
def _elect_dest_broker(self, victim_partition):
        
        under_loaded_brokers = sorted(
            [
                broker
                for broker in self._brokers
                if (victim_partition not in broker.partitions and
                    not broker.inactive and
                    not broker.decommissioned)
            ],
            key=lambda b: len(b.partitions)
        )
        if not under_loaded_brokers:
            return None

        broker_topic_partition_cnt = [
            (broker, broker.count_partitions(victim_partition.topic))
            for broker in under_loaded_brokers
            if victim_partition not in broker.partitions
        ]
        min_count_pair = min(
            broker_topic_partition_cnt,
            key=lambda ele: ele[1],
        )
        return min_count_pair[0]
126024
def get_brokers(self, names_only=False):
        
        try:
            broker_ids = self.get_children("/brokers/ids")
        except NoNodeError:
            _log.info(
                "cluster is empty."
            )
            return {}
        # Return broker-ids only
        if names_only:
            return {int(b_id): None for b_id in broker_ids}
        return {int(b_id): self.get_broker_metadata(b_id) for b_id in broker_ids}
353842
def bounce_cluster(name):
    '''
    
    '''
    ret = {'name': name,
           'changes': {},
           'result': None,
           'comment': ''}

    if __opts__['test']:
        ret['comment'] = 'Bouncing cluster'
        return ret

    __salt__['trafficserver.bounce_cluster']()

    ret['result'] = True
    ret['comment'] = 'Bounced cluster'
    return ret
1230806
private AbstractTopology getTopology(StartAction startAction, Map<Integer, HostInfo> hostInfos,
            JoinCoordinator joinCoordinator)
    {
        AbstractTopology topology = null;
        if (startAction == StartAction.JOIN) {
            assert(joinCoordinator != null);
            topology = joinCoordinator.getTopology();
        } else if (startAction.doesRejoin()) {
            topology = TopologyZKUtils.readTopologyFromZK(m_messenger.getZK());
        } else {
            try {
                return TopologyZKUtils.readTopologyFromZK(m_messenger.getZK(), null);
            } catch (KeeperException.NoNodeException e) {
                hostLog.debug("Topology doesn't exist yet try to create it");
            } catch (KeeperException | InterruptedException | JSONException e) {
                VoltDB.crashLocalVoltDB("Unable to read topology from ZK, dying", true, e);
            }

            // initial start or recover
            int hostcount = m_clusterSettings.get().hostcount();
            if (hostInfos.size() != (hostcount - m_config.m_missingHostCount)) {
                VoltDB.crashLocalVoltDB("The total number of live and missing hosts must be the same as the cluster host count", false, null);
            }
            int kfactor = getKFactor();
            if (kfactor == 0 && m_config.m_missingHostCount > 0) {
                VoltDB.crashLocalVoltDB("A cluster with 0 kfactor can not be started with missing nodes ", false, null);
            }
            if (hostcount <= kfactor) {
                VoltDB.crashLocalVoltDB("Not enough nodes to ensure K-Safety.", false, null);
            }
            // Missing hosts can't be more than number of partition groups times k-factor
            int partitionGroupCount = m_clusterSettings.get().hostcount() / (kfactor + 1);
            if (m_config.m_missingHostCount > (partitionGroupCount * kfactor)) {
                VoltDB.crashLocalVoltDB("Too many nodes are missing at startup. This cluster only allow up to "
                        + (partitionGroupCount * kfactor) + " missing hosts.");
            }

            //startup or recover a cluster with missing nodes. make up the missing hosts to fool the topology
            //The topology will contain hosts which are marked as missing.The missing hosts will not host any master partitions.
            //At least one partition replica must be on the live hosts (not missing). Otherwise, the cluster will not be started up.
            //LeaderAppointer will ignore these hosts during startup.
            int sph = hostInfos.values().iterator().next().m_localSitesCount;
            int missingHostId = Integer.MAX_VALUE;
            Set<Integer> missingHosts = Sets.newHashSet();
            for (int i = 0; i < m_config.m_missingHostCount; i++) {
                hostInfos.put(missingHostId, new HostInfo("", AbstractTopology.PLACEMENT_GROUP_DEFAULT, sph, ""));
                missingHosts.add(missingHostId--);
            }
            int totalSites = sph * hostcount;
            if (totalSites % (kfactor + 1) != 0) {
                VoltDB.crashLocalVoltDB("Total number of sites is not divisible by the number of partitions.", false, null);
            }
            topology = AbstractTopology.getTopology(hostInfos, missingHosts, kfactor,
                    (m_config.m_restorePlacement && m_config.m_startAction.doesRecover()));
            String err;
            if ((err = topology.validateLayout(m_messenger.getLiveHostIds())) != null) {
                hostLog.warn("Unable to find optimal placement layout. " + err);
                hostLog.warn("When using placement groups, follow two rules to get better cluster availability:\n" +
                             "   1. Each placement group must have the same number of nodes, and\n" +
                             "   2. The number of partition replicas (kfactor + 1) must be a multiple of the number of placement groups.");
            }
            if (topology.hasMissingPartitions()) {
                VoltDB.crashLocalVoltDB("Some partitions are missing in the topology", false, null);
            }
            if (m_config.m_restorePlacement && m_config.m_startAction.doesRecover() && topology.version > 1) {
                consoleLog.info("Partition placement has been restored.");
            }
            topology = TopologyZKUtils.registerTopologyToZK(m_messenger.getZK(), topology);
        }
        return topology;
    }
316471
def print_brokers(cluster_config, brokers):
    
    print("Will restart the following brokers in {0}:".format(cluster_config.name))
    for id, host in brokers:
        print("  {0}: {1}".format(id, host))
775513
public void setClusters(java.util.Collection<ClusterSummary> clusters) {
        if (clusters == null) {
            this.clusters = null;
            return;
        }

        this.clusters = new com.amazonaws.internal.SdkInternalList<ClusterSummary>(clusters);
    }
818505
def cluster_centers_(self):
        
        # TODO: Handle agglomerative clustering and LDA
        for attr in ('cluster_centers_',):
            try:
                return getattr(self.estimator, attr)
            except AttributeError:
                continue

        raise AttributeError(
            "could not find or make cluster_centers_ for {}".format(
            self.estimator.__class__.__name__
        ))
611904
def clusterstatus(self):
        

        res = self.cluster_status_raw()

        cluster = res['cluster']['collections']
        out = {}
        try:
            for collection in cluster:
                out[collection] = {}
                for shard in cluster[collection]['shards']:
                    out[collection][shard] = {}
                    for replica in cluster[collection]['shards'][shard]['replicas']:
                        out[collection][shard][replica] = cluster[collection]['shards'][shard]['replicas'][replica]
                        if out[collection][shard][replica]['state'] != 'active':
                            out[collection][shard][replica]['doc_count'] = False
                        else:
                            out[collection][shard][replica]['doc_count'] = self._get_collection_counts(
                                out[collection][shard][replica])
        except Exception as e:
            self.logger.error("Couldn't parse response from clusterstatus API call")
            self.logger.exception(e)

        return out
1102871
private void updateClusterMetrics(String topologyId, TopologyMetric tpMetric) {
        if (tpMetric.get_topologyMetric().get_metrics_size() == 0) {
            return;
        }

        MetricInfo topologyMetrics = tpMetric.get_topologyMetric();
        // make a new MetricInfo to save the topologyId's metric
        MetricInfo clusterMetrics = MetricUtils.mkMetricInfo();
        Set<String> metricNames = new HashSet<>();

        for (Map.Entry<String, Map<Integer, MetricSnapshot>> entry : topologyMetrics.get_metrics().entrySet()) {
            String metricName = MetricUtils.topo2clusterName(entry.getKey());
            MetricType metricType = MetricUtils.metricType(metricName);
            Map<Integer, MetricSnapshot> winData = new HashMap<>();

            for (Map.Entry<Integer, MetricSnapshot> entryData : entry.getValue().entrySet()) {
                MetricSnapshot snapshot = entryData.getValue().deepCopy();
                winData.put(entryData.getKey(), snapshot);
                if (metricType == MetricType.HISTOGRAM) {
                    // reset topology metric points
                    entryData.getValue().set_points(new byte[0]);
                    entryData.getValue().set_pointSize(0);
                }
            }
            clusterMetrics.put_to_metrics(metricName, winData);
            metricNames.add(metricName);
        }

        // save to local cache, waiting for merging
        TopologyMetricContext clusterTpMetricContext = context.getClusterTopologyMetricContext();
        clusterTpMetricContext.addToMemCache(topologyId, clusterMetrics);
        context.registerMetrics(JStormMetrics.CLUSTER_METRIC_KEY, metricNames);

    }
826494
def get_topic_partition_metadata(hosts):
    
    kafka_client = KafkaToolClient(hosts, timeout=10)
    kafka_client.load_metadata_for_topics()
    topic_partitions = kafka_client.topic_partitions
    resp = kafka_client.send_metadata_request()

    for _, topic, partitions in resp.topics:
        for partition_error, partition, leader, replicas, isr in partitions:
            if topic_partitions.get(topic, {}).get(partition) is not None:
                topic_partitions[topic][partition] = PartitionMetadata(topic, partition, leader,
                                                                       replicas, isr, partition_error)
    return topic_partitions
689949
def _build_partitions(self, assignment):
        
        self.partitions = {}
        for partition_name, replica_ids in six.iteritems(assignment):
            # Get topic
            topic_id = partition_name[0]
            partition_id = partition_name[1]
            topic = self.topics.setdefault(
                topic_id,
                Topic(topic_id, replication_factor=len(replica_ids))
            )

            # Creating partition object
            partition = Partition(
                topic,
                partition_id,
                weight=self.partition_measurer.get_weight(partition_name),
                size=self.partition_measurer.get_size(partition_name),
            )
            self.partitions[partition_name] = partition
            topic.add_partition(partition)

            # Updating corresponding broker objects
            for broker_id in replica_ids:
                # Check if broker-id is present in current active brokers
                if broker_id not in list(self.brokers.keys()):
                    self.log.warning(
                        "Broker %s containing partition %s is not in "
                        "active brokers.",
                        broker_id,
                        partition,
                    )
                    self.brokers[broker_id] = self._create_broker(broker_id)

                self.brokers[broker_id].add_partition(partition)
344949
public void setTopologyStatus(TopologyAssignEvent event) throws Exception {
        StormClusterState stormClusterState = nimbusData.getStormClusterState();

        String topologyId = event.getTopologyId();
        String topologyName = event.getTopologyName();
        String group = event.getGroup();

        StormStatus status = new StormStatus(StatusType.active);
        if (event.getOldStatus() != null) {
            status = event.getOldStatus();
        }

        boolean isEnable = ConfigExtension.isEnablePerformanceMetrics(nimbusData.getConf());
        StormBase stormBase = stormClusterState.storm_base(topologyId, null);
        if (stormBase == null) {
            stormBase = new StormBase(topologyName, TimeUtils.current_time_secs(), status, group);
            stormBase.setEnableMonitor(isEnable);
            stormClusterState.activate_storm(topologyId, stormBase);
        } else {
            stormClusterState.update_storm(topologyId, status);
            stormClusterState.set_storm_monitor(topologyId, isEnable);

            // here exist one hack operation
            // when monitor/rebalance/startup topologyName is null
            if (topologyName == null) {
                event.setTopologyName(stormBase.getStormName());
            }
        }

        LOG.info("Update " + topologyId + " " + status);
    }
736125
def clusterStatus(self):
        
        servers = yield self.getClusterServers()

        d = {
            'workers': {},
            'crons': {},
            'queues': {}
        }

        now = time.time()

        reverse_map = {}

        for sname in servers:
            last = yield self._get_key('/server/%s/heartbeat' % sname)
            status = yield self._get_key('/server/%s/status' % sname)
            uuid = yield self._get_key('/server/%s/uuid' % sname)

            reverse_map[uuid] = sname

            if not last:
                last = 0

            last = float(last)

            if (status == 'ready') and (now - last > 5):
                status = 'offline'

            if not sname in d['workers']:
                d['workers'][sname] = []

            d['workers'][sname].append({
                'lastseen': last,
                'status': status,
                'id': uuid
            })

        # Crons
        crons = yield self.keys('/crons')

        for queue in crons:
            if queue not in d['crons']:
                d['crons'][queue] = {'methods': {}}

            methods = yield self.keys('/crons/%s' % queue)

            for method in methods:
                last = yield self._get_key('/crons/%s/%s' % (queue, method))
                if last:
                    d['crons'][queue]['methods'][method] = float(last)
            
            uid = yield self._get_key('/croner/%s' % queue)
            if uid:
                d['crons'][queue]['master'] = '%s:%s' % (uid, reverse_map[uid])

        # Queues
        queue_keys = yield self.keys('/qstats')

        for qname in queue_keys:
            if qname not in d['queues']:
                qlen = yield self.queueSize(qname)

                stats = yield self.getQueueMessageStats(qname)

                d['queues'][qname] = {
                    'waiting': qlen,
                    'messages': stats
                }

        defer.returnValue(d)
156074
def _readClusterSettings(self):
        
        instanceMetaData = get_instance_metadata()
        region = zoneToRegion(self._zone)
        conn = boto.ec2.connect_to_region(region)
        instance = conn.get_all_instances(instance_ids=[instanceMetaData["instance-id"]])[0].instances[0]
        self.clusterName = str(instance.tags["Name"])
        self._buildContext()
        self._subnetID = instance.subnet_id
        self._leaderPrivateIP = instanceMetaData['local-ipv4']  # this is PRIVATE IP
        self._keyName = list(instanceMetaData['public-keys'].keys())[0]
        self._tags = self.getLeader().tags
        self._masterPublicKey = self._setSSH()
489516
def _fill_topology_cfg(self, topo_dict):
        
        cfg_dict = {}
        if topo_dict.bond_member_ports is not None:
            cfg_dict.update({'bond_member_ports':
                             topo_dict.bond_member_ports})
        if topo_dict.bond_interface is not None:
            cfg_dict.update({'bond_interface':
                             topo_dict.bond_interface})
        return cfg_dict