Read the cluster settings from the instance, which should be the leader.
        See https://cloud.google.com/compute/docs/storing-retrieving-metadata for details about
        reading the metadata.
178716
def _readClusterSettings(self):
        
        metadata_server = "http://metadata/computeMetadata/v1/instance/"
        metadata_flavor = {'Metadata-Flavor': 'Google'}
        zone = requests.get(metadata_server + 'zone', headers = metadata_flavor).text
        self._zone = zone.split('/')[-1]

        project_metadata_server = "http://metadata/computeMetadata/v1/project/"
        self._projectId = requests.get(project_metadata_server + 'project-id', headers = metadata_flavor).text

        # From a GCE instance, these values can be blank. Only the projectId is needed
        self._googleJson = ''
        self._clientEmail = ''

        self._tags = requests.get(metadata_server + 'description', headers = metadata_flavor).text
        tags = json.loads(self._tags)
        self.clusterName = tags['clusterName']
        self._gceDriver = self._getDriver()
        self._instanceGroup = self._gceDriver.ex_get_instancegroup(self.clusterName, zone=self._zone)

        leader = self.getLeader()
        self._leaderPrivateIP = leader.privateIP

        # generate a public key for the leader, which is used to talk to workers
        self._masterPublicKey = self._setSSH()

        # The location of the Google credentials file on instances.
        self._credentialsPath = GoogleJobStore.nodeServiceAccountJson
        self._keyName = 'core' # key name leader users to communicate with works
        self._botoPath = self.NODE_BOTO_PATH
1250310
private void takeLeadership() {
    context.setLeader(context.getCluster().member().id());
    context.getClusterState().getRemoteMemberStates().forEach(m -> m.resetState(context.getLog()));
  }
968506
async def get(self):
        
        shepherd = self.request.app.vmshepherd
        data = {'presets': {}, 'config': shepherd.config}
        presets = await shepherd.preset_manager.list_presets()
        runtime = shepherd.runtime_manager
        for name in presets:
            preset = shepherd.preset_manager.get_preset(name)
            data['presets'][name] = {
                'preset': preset,
                'vms': preset.vms,
                'runtime': await runtime.get_preset_data(name),
                'vmshepherd_id': shepherd.instance_id,
                'now': time.time()
            }
        return data
1179227
def _initialize_metadata_service(cls):
        
        if cls.inited:
            return

        instance_id = cls.get_attribute('instance/id')

        if instance_id is not None:
            cls.is_running = True

            _GCP_METADATA_MAP['instance_id'] = instance_id

            # fetch attributes from metadata request
            for attribute_key, attribute_uri in _GCE_ATTRIBUTES.items():
                if attribute_key not in _GCP_METADATA_MAP:
                    attribute_value = cls.get_attribute(attribute_uri)
                    if attribute_value is not None:  # pragma: NO COVER
                        _GCP_METADATA_MAP[attribute_key] = attribute_value

        cls.inited = True
1235117
def read_config(ini_group=None):
    
    env_conf = dict(DEFAULT_CONFIG)
    for key in REQUIRED_CONFIG_KEYS.union(ALLOWED_CONFIG_KEYS):
        env_key = "CLOUDSTACK_{0}".format(key.upper())
        value = os.getenv(env_key)
        if value:
            env_conf[key] = value

    # overrides means we have a .ini to read
    overrides = os.getenv('CLOUDSTACK_OVERRIDES', '').strip()

    if not overrides and set(env_conf).issuperset(REQUIRED_CONFIG_KEYS):
        return env_conf

    ini_conf = read_config_from_ini(ini_group)

    overrides = {s.lower() for s in re.split(r'\W+', overrides)}
    config = dict(dict(env_conf, **ini_conf),
                  **{k: v for k, v in env_conf.items() if k in overrides})

    missings = REQUIRED_CONFIG_KEYS.difference(config)
    if missings:
        raise ValueError("the configuration is missing the following keys: " +
                         ", ".join(missings))

    # convert booleans values.
    bool_keys = ('dangerous_no_tls_verify',)
    for bool_key in bool_keys:
        if isinstance(config[bool_key], string_type):
            try:
                config[bool_key] = strtobool(config[bool_key])
            except ValueError:
                pass

    return config
311916
def describe_replication_group(name, region=None, key=None, keyid=None,
                               profile=None, parameter=None):
    '''
    
    '''
    conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)

    if not conn:
        return None
    try:
        cc = conn.describe_replication_groups(name)
    except boto.exception.BotoServerError as e:
        msg = 'Failed to get config for cache cluster {0}.'.format(name)
        log.error(msg)
        log.debug(e)
        return {}
    ret = odict.OrderedDict()
    cc = cc['DescribeReplicationGroupsResponse']['DescribeReplicationGroupsResult']
    cc = cc['ReplicationGroups'][0]

    attrs = ['status', 'description', 'primary_endpoint',
             'member_clusters', 'replication_group_id',
             'pending_modified_values', 'primary_cluster_id',
             'node_groups']
    for key, val in six.iteritems(cc):
        _key = boto.utils.pythonize_name(key)
        if _key == 'status':
            if val:
                ret[_key] = val
            else:
                ret[_key] = None
        if _key == 'description':
            if val:
                ret[_key] = val
            else:
                ret[_key] = None
        if _key == 'replication_group_id':
            if val:
                ret[_key] = val
            else:
                ret[_key] = None
        if _key == 'member_clusters':
            if val:
                ret[_key] = val
            else:
                ret[_key] = None
        if _key == 'node_groups':
            if val:
                ret[_key] = val
            else:
                ret[_key] = None
        if _key == 'pending_modified_values':
            if val:
                ret[_key] = val
            else:
                ret[_key] = None
    return ret
1293517
def read_settings(instance_id, remote_ip = nil)
      check_instance_ips(remote_ip, instance_id) if remote_ip

      get_instance(instance_id).settings
    end
268740
private static void retrieveServerMetadata() throws Exception {
        if (get() == null) {
            throw new IllegalStateException("Cannot retrieve the server metadata - no reference to Chameleon stored " +
                    "in the holder");
        }

        int factor = Integer.getInteger("time.factor", 1);
        if (factor != 1) {
            TimeUtils.TIME_FACTOR = factor;
        }

        // Before checking, ensure stability.
        ServiceReference[] references = get().waitForStability().context().getAllServiceReferences(WisdomEngine.class
                .getName(), null);

        if (references == null || references.length == 0) {
            references = get().waitForStability().context().getAllServiceReferences(WisdomEngine.class.getName(), null);
        }

        if (references == null || references.length == 0) {
            throw new IllegalStateException("Cannot retrieve the Wisdom Engine service");
        }

        Object engine = get().context().getService(references[0]);
        HOST_NAME = (String) engine.getClass().getMethod("hostname").invoke(engine);
        HTTP_PORT = (int) engine.getClass().getMethod("httpPort").invoke(engine);
        HTTPS_PORT = (int) engine.getClass().getMethod("httpsPort").invoke(engine);
    }
26266
def _get_coordinator_for_group(self, consumer_group):
        
        if self.consumer_group_to_brokers.get(consumer_group) is None:
            yield self.load_consumer_metadata_for_group(consumer_group)

        returnValue(self.consumer_group_to_brokers.get(consumer_group))
1214145
def discover(self):
        
        # need self.args['command'] so fail if it's not available
        if (not self.args or 'command' not in self.args or not
                self.args['command']):
            return

        # load .mlaunch_startup file for start, stop, list, use current
        # parameters for init
        if self.args['command'] == 'init':
            self.loaded_args = self.args
            self.loaded_unknown_args = self.unknown_args
        else:
            if not self._load_parameters():
                startup_file = os.path.join(self.dir, ".mlaunch_startup")
                raise SystemExit("Can't read %s, use 'mlaunch init ...' first."
                                 % startup_file)

        self.ssl_pymongo_options = self._get_ssl_pymongo_options(
            self.loaded_args)

        # reset cluster_* variables
        self.cluster_tree = {}
        self.cluster_tags = defaultdict(list)
        self.cluster_running = {}

        # get shard names
        shard_names = self._get_shard_names(self.loaded_args)

        # some shortcut variables
        is_sharded = ('sharded' in self.loaded_args and
                      self.loaded_args['sharded'] is not None)
        is_replicaset = ('replicaset' in self.loaded_args and
                         self.loaded_args['replicaset'])
        is_single = 'single' in self.loaded_args and self.loaded_args['single']
        has_arbiter = ('arbiter' in self.loaded_args and
                       self.loaded_args['arbiter'])

        # determine number of nodes to inspect
        if is_sharded:
            num_config = self.loaded_args['config']
            # at least one temp. mongos for adding shards, will be
            # killed later on
            num_mongos = max(1, self.loaded_args['mongos'])
            num_shards = len(shard_names)
        else:
            num_shards = 1
            num_config = 0
            num_mongos = 0

        num_nodes_per_shard = self.loaded_args['nodes'] if is_replicaset else 1
        if has_arbiter:
            num_nodes_per_shard += 1

        num_nodes = num_shards * num_nodes_per_shard + num_config + num_mongos

        current_port = self.loaded_args['port']

        # tag all nodes with 'all'
        self.cluster_tags['all'].extend(list(range(current_port,
                                                   current_port + num_nodes)))

        # tag all nodes with their port number (as string) and whether
        # they are running
        for port in range(current_port, current_port + num_nodes):
            self.cluster_tags[str(port)].append(port)

            running = self.is_running(port)
            self.cluster_running[port] = running
            self.cluster_tags['running' if running else 'down'].append(port)

        # find all mongos
        for i in range(num_mongos):
            port = i + current_port

            # add mongos to cluster tree
            self.cluster_tree.setdefault('mongos', []).append(port)
            # add mongos to tags
            self.cluster_tags['mongos'].append(port)

        current_port += num_mongos

        # find all mongods (sharded, replicaset or single)
        if shard_names is None:
            shard_names = [None]

        for shard in shard_names:
            port_range = list(range(current_port,
                                    current_port + num_nodes_per_shard))

            # all of these are mongod nodes
            self.cluster_tags['mongod'].extend(port_range)

            if shard:
                # if this is a shard, store in cluster_tree and tag shard name
                self.cluster_tree.setdefault('shard', []).append(port_range)
                self.cluster_tags[shard].extend(port_range)

            if is_replicaset:
                # get replica set states
                rs_name = shard if shard else self.loaded_args['name']

                try:
                    mrsc = self.client(
                        ','.join('localhost:%i' % i for i in port_range),
                        replicaSet=rs_name)

                    # primary, secondaries, arbiters
                    # @todo: this is no longer working because MongoClient
                    # is now non-blocking
                    if mrsc.primary:
                        self.cluster_tags['primary'].append(mrsc.primary[1])
                    self.cluster_tags['secondary'].extend(list(map
                                                          (itemgetter(1),
                                                           mrsc.secondaries)))
                    self.cluster_tags['arbiter'].extend(list(map(itemgetter(1),
                                                             mrsc.arbiters)))

                    # secondaries in cluster_tree (order is now important)
                    self.cluster_tree.setdefault('secondary', [])
                    for i, secondary in enumerate(sorted(map
                                                         (itemgetter(1),
                                                          mrsc.secondaries))):
                        if len(self.cluster_tree['secondary']) <= i:
                            self.cluster_tree['secondary'].append([])
                        self.cluster_tree['secondary'][i].append(secondary)

                except (ConnectionFailure, ConfigurationError):
                    pass

            elif is_single:
                self.cluster_tags['single'].append(current_port)

            # increase current_port
            current_port += num_nodes_per_shard

        # add config server to cluster tree
        self.cluster_tree.setdefault('config', []).append(port)

        # If not CSRS, set the number of config servers to be 1 or 3
        # This is needed, otherwise `mlaunch init --sharded 2 --replicaset
        # --config 2` on <3.3.0 will crash
        if not self.args.get('csrs') and self.args['command'] == 'init':
            if num_config >= 3:
                num_config = 3
            else:
                num_config = 1

        for i in range(num_config):
            port = i + current_port

            try:
                mc = self.client('localhost:%i' % port)
                mc.admin.command('ping')
                running = True

            except ConnectionFailure:
                # node not reachable
                running = False

            # add config server to cluster tree
            self.cluster_tree.setdefault('config', []).append(port)
            # add config server to tags
            self.cluster_tags['config'].append(port)
            self.cluster_tags['mongod'].append(port)

        current_port += num_mongos
482242
def metadata(self):
    
    if self._info is None:
      try:
        self._info = self._api.buckets_get(self._name)
      except Exception as e:
        raise e

    return BucketMetadata(self._info) if self._info else None
611904
def clusterstatus(self):
        

        res = self.cluster_status_raw()

        cluster = res['cluster']['collections']
        out = {}
        try:
            for collection in cluster:
                out[collection] = {}
                for shard in cluster[collection]['shards']:
                    out[collection][shard] = {}
                    for replica in cluster[collection]['shards'][shard]['replicas']:
                        out[collection][shard][replica] = cluster[collection]['shards'][shard]['replicas'][replica]
                        if out[collection][shard][replica]['state'] != 'active':
                            out[collection][shard][replica]['doc_count'] = False
                        else:
                            out[collection][shard][replica]['doc_count'] = self._get_collection_counts(
                                out[collection][shard][replica])
        except Exception as e:
            self.logger.error("Couldn't parse response from clusterstatus API call")
            self.logger.exception(e)

        return out
474291
public static Builder getNodeConfig(String clusterName, String nodeName,
            String networkHost, String homePath, boolean nodeIngest) {
        return Settings.builder()
                // .put("path.metric", dataPath).put("http.port", httpRange)
                // .put("transport.tcp.port", transportRange)
                .put("node.name", nodeName).put("cluster.name", clusterName)
                .put("network.host", networkHost).put("path.home", homePath)
                .put("transport.type", "netty4")
                // .put("discovery.type", "none")
                .put("http.type", "netty4").put("node.ingest", nodeIngest);
    }
1185304
def initialize():
    
    global runtime_context

    try:
        runtime_context = {}

        r = urlopen('http://169.254.169.254/latest/meta-data/instance-id', timeout=1)
        runtime_context['instance_id'] = r.read().decode('utf-8')

        r = urlopen('http://169.254.169.254/latest/meta-data/placement/availability-zone',
                    timeout=1)
        runtime_context['availability_zone'] = r.read().decode('utf-8')

    except Exception:
        runtime_context = None
        log.warning("failed to get ec2 instance metadata.")
1372144
def _get_cluster_dict(cluster_name, cluster_ref):
    '''
    
    '''

    log.trace('Building a dictionary representation of cluster \'%s\'', cluster_name)
    props = salt.utils.vmware.get_properties_of_managed_object(
        cluster_ref,
        properties=['configurationEx'])
    res = {'ha': {'enabled': props['configurationEx'].dasConfig.enabled},
           'drs': {'enabled': props['configurationEx'].drsConfig.enabled}}
    # Convert HA properties of interest
    ha_conf = props['configurationEx'].dasConfig
    log.trace('ha_conf = %s', ha_conf)
    res['ha']['admission_control_enabled'] = ha_conf.admissionControlEnabled
    if ha_conf.admissionControlPolicy and \
       isinstance(ha_conf.admissionControlPolicy,
                  vim.ClusterFailoverResourcesAdmissionControlPolicy):
        pol = ha_conf.admissionControlPolicy
        res['ha']['admission_control_policy'] = \
                {'cpu_failover_percent': pol.cpuFailoverResourcesPercent,
                 'memory_failover_percent': pol.memoryFailoverResourcesPercent}
    if ha_conf.defaultVmSettings:
        def_vm_set = ha_conf.defaultVmSettings
        res['ha']['default_vm_settings'] = \
                {'isolation_response': def_vm_set.isolationResponse,
                 'restart_priority': def_vm_set.restartPriority}
    res['ha']['hb_ds_candidate_policy'] = \
            ha_conf.hBDatastoreCandidatePolicy
    if ha_conf.hostMonitoring:
        res['ha']['host_monitoring'] = ha_conf.hostMonitoring
    if ha_conf.option:
        res['ha']['options'] = [{'key': o.key, 'value': o.value}
                                for o in ha_conf.option]
    res['ha']['vm_monitoring'] = ha_conf.vmMonitoring
    # Convert DRS properties
    drs_conf = props['configurationEx'].drsConfig
    log.trace('drs_conf = %s', drs_conf)
    res['drs']['vmotion_rate'] = 6 - drs_conf.vmotionRate
    res['drs']['default_vm_behavior'] = drs_conf.defaultVmBehavior
    # vm_swap_placement
    res['vm_swap_placement'] = props['configurationEx'].vmSwapPlacement
    # Convert VSAN properties
    si = salt.utils.vmware.get_service_instance_from_managed_object(
        cluster_ref)

    if salt.utils.vsan.vsan_supported(si):
        # XXX The correct way of retrieving the VSAN data (on the if branch)
        #  is not supported before 60u2 vcenter
        vcenter_info = salt.utils.vmware.get_service_info(si)
        if int(vcenter_info.build) >= 3634794:  # 60u2
            # VSAN API is fully supported by the VC starting with 60u2
            vsan_conf = salt.utils.vsan.get_cluster_vsan_info(cluster_ref)
            log.trace('vsan_conf = %s', vsan_conf)
            res['vsan'] = {'enabled': vsan_conf.enabled,
                           'auto_claim_storage':
                           vsan_conf.defaultConfig.autoClaimStorage}
            if vsan_conf.dataEfficiencyConfig:
                data_eff = vsan_conf.dataEfficiencyConfig
                res['vsan'].update({
                    # We force compression_enabled to be True/False
                    'compression_enabled':
                    data_eff.compressionEnabled or False,
                    'dedup_enabled': data_eff.dedupEnabled})
        else:  # before 60u2 (no advanced vsan info)
            if props['configurationEx'].vsanConfigInfo:
                default_config = \
                        props['configurationEx'].vsanConfigInfo.defaultConfig
                res['vsan'] = {
                    'enabled': props['configurationEx'].vsanConfigInfo.enabled,
                    'auto_claim_storage': default_config.autoClaimStorage}
    return res
1297625
def toProtocolElement(self):
        
        readGroupSet = protocol.ReadGroupSet()
        readGroupSet.id = self.getId()
        readGroupSet.read_groups.extend(
            [readGroup.toProtocolElement()
             for readGroup in self.getReadGroups()]
        )
        readGroupSet.name = self.getLocalId()
        readGroupSet.dataset_id = self.getParentContainer().getId()
        readGroupSet.stats.CopyFrom(self.getStats())
        self.serializeAttributes(readGroupSet)
        return readGroupSet
959023
def read_dataset_metadata(self):
    """"""
    if self.dataset_meta:
      return
    shell_call(['gsutil', 'cp',
                'gs://' + self.storage_client.bucket_name + '/'
                + 'dataset/' + self.dataset_name + '_dataset.csv',
                LOCAL_DATASET_METADATA_FILE])
    with open(LOCAL_DATASET_METADATA_FILE, 'r') as f:
      self.dataset_meta = eval_lib.DatasetMetadata(f)