Get lists of blocks on the disk sorted by blockId, per namespace
516942
private Map<Integer, DiskScanInfo[]> getDiskReportPerNamespace() {
    if (dataset.volumes == null) {
      LOG.warn("Dataset volumes are not initialized yet");
      return new HashMap<Integer, DiskScanInfo[]>();
    }
    // First get list of data directories
    FSVolume[] volumes = dataset.volumes.getVolumes();
    ScanInfoListPerNamespace[] volumeReports = 
        new ScanInfoListPerNamespace[volumes.length];
    
    Map<Integer, Future<ScanInfoListPerNamespace>> volumeCompilers =
      new HashMap<Integer, Future<ScanInfoListPerNamespace>>();
    for (int i = 0; i < volumes.length; i++) {
      // check volume is valid
      if (dataset.volumes.isValid(volumes[i])) {
        // and run compiler for it
        ReportCompiler reportCompiler =
          new ReportCompiler(volumes[i], datanode);
        Future<ScanInfoListPerNamespace> result = 
          reportCompileThreadPool.submit(reportCompiler);
        volumeCompilers.put(i, result);
      }
    }
    
    for(Entry<Integer, Future<ScanInfoListPerNamespace>> e : volumeCompilers.entrySet()) {
      try {
        int volume = e.getKey();
        volumeReports[volume] = e.getValue().get();
      } catch (Exception ex) {
        LOG.error("Error compiling report", ex);
        // Propagate ex to DataBlockScanner to deal with
        throw new RuntimeException(ex);
      }
    }

    // Compile consolidated report for all the volumes
    ScanInfoListPerNamespace list = new ScanInfoListPerNamespace();
    for (int i = 0; i < volumes.length; i++) {
      if (dataset.volumes.isValid(volumes[i])) { // volume is still valid
        list.addAll(volumeReports[i]);
      }
    }

    return list.toSortedArrays();
  }
587391
def get_all_importing_namespace_hashes( self ):
        
        cur = self.db.cursor()
        namespace_hashes = namedb_get_all_importing_namespace_hashes( cur, self.lastblock )
        return namespace_hashes
1005207
def all_blocks(self):

        status = OrderedDict.fromkeys(parameters.BLOCKS.keys())
        status['13AE'] = ['discovery complete', '50', '24.05']
        status['13AO'] = ['discovery complete', '36', '24.40']
        status['13BL'] = ['discovery complete', '79', '24.48']
        status['14BH'] = ['discovery running', '-', '-']
        status['15AP'] = ['discovery running', '-', '-']
        status['15AM'] = ['discovery running', '-', '-']

        '''
        '''
        bks = []
        for block in status.iterkeys():
            bk = [block, self.num_block_images(block)]  # if set in the .fromkeys(), doesn't give a unique list
            if status[block] is not None:
                bk = bk + status[block]
            else:
                bk = bk + ['awaiting triplets', '-', '-']
            bks.append(bk)

        retval = {'blocks': bks, 'status': status}

        return retval
757535
def cli(env, sortby, datacenter):
    """"""
    block_manager = SoftLayer.BlockStorageManager(env.client)
    mask = "mask[serviceResource[datacenter[name]],"\
           "replicationPartners[serviceResource[datacenter[name]]]]"
    block_volumes = block_manager.list_block_volumes(datacenter=datacenter,
                                                     mask=mask)

    # cycle through all block volumes and count datacenter occurences.
    datacenters = dict()
    for volume in block_volumes:
        service_resource = volume['serviceResource']
        if 'datacenter' in service_resource:
            datacenter_name = service_resource['datacenter']['name']
            if datacenter_name not in datacenters.keys():
                datacenters[datacenter_name] = 1
            else:
                datacenters[datacenter_name] += 1

    table = formatting.KeyValueTable(DEFAULT_COLUMNS)
    table.sortby = sortby
    for datacenter_name in datacenters:
        table.add_row([datacenter_name, datacenters[datacenter_name]])
    env.fout(table)
516973
def get(block_id):
    """"""
    try:
        sbi_details = DB.get_block_details([block_id]).__next__()
        sbi_details['processing_blocks'] = []
        pb_ids = sbi_details['processing_block_ids']
        for pb_details in DB.get_block_details(pb_ids):
            sbi_details['processing_blocks'].append(pb_details)
        del sbi_details['processing_block_ids']
        response = sbi_details
        _url = get_root_url()
        response['links'] = {
            'scheduling-blocks': '{}/scheduling-blocks'.format(_url),
            'home': '{}'.format(_url)
        }
        return sbi_details, HTTPStatus.OK

    except IndexError:
        return {'error': 'specified block id not found {}'.format(block_id)}, \
               HTTPStatus.NOT_FOUND
926433
def fetch_namespace():
    
    resp = blockstack.lib.client.get_all_names(hostport=blockstack_indexer_url)
    subdomain_names = blockstack.lib.subdomains.get_all_subdomains()

    all_names = list(resp) + list(subdomain_names)

    with open(SEARCH_BLOCKCHAIN_DATA_FILE, 'w') as fout:
        fout.write(json.dumps(all_names))
361913
def get_namespace_keys(app, limit):
  """"""
  ns_query = datastore.Query('__namespace__', keys_only=True, _app=app)
  return list(ns_query.Run(limit=limit, batch_size=limit))
744712
function getBlockNodes(nodes) {
  // TODO(perf): just check if all items in `nodes` are siblings and if they are return the original
  //             collection, otherwise update the original collection.
  var node = nodes[0];
  var endNode = nodes[nodes.length - 1];
  var blockNodes = [node];

  do {
    node = node.nextSibling;
    if (!node) break;
    blockNodes.push(node);
  } while (node !== endNode);

  return jqLite(blockNodes);
}
975854
private List<Thread> tryUpdateBlock(int namespaceId, 
      Block oldblock, Block newblock) throws IOException {
    lock.writeLock().lock();
    try {
      //check ongoing create threads
      ArrayList<Thread> activeThreads = getActiveThreads(namespaceId, oldblock);
      if (activeThreads != null) {
        return activeThreads;
      }

      DatanodeBlockInfo binfo = volumeMap.get(namespaceId, oldblock);
      if (binfo == null) {
        throw new IOException("Block " + oldblock
            + " doesn't exist or has been recovered to a new generation ");
      }

      File blockFile = binfo.getBlockDataFile().getFile();
  
      long oldgs;
      File oldMetaFile = null;
      if (binfo.isInlineChecksum()) {
        oldgs = BlockInlineChecksumReader
            .getGenerationStampFromInlineChecksumFile(blockFile.getName());
      } else {
        oldMetaFile = BlockWithChecksumFileWriter.findMetaFile(blockFile);
        oldgs = BlockWithChecksumFileReader.parseGenerationStampInMetaFile(
            blockFile, oldMetaFile);
      }
      
    // First validate the update

      //update generation stamp
      if (oldgs > newblock.getGenerationStamp()) {
        throw new IOException("Cannot update block (id=" + newblock.getBlockId()
            + ") generation stamp from " + oldgs
            + " to " + newblock.getGenerationStamp());
      }
      
      //update length
      if (newblock.getNumBytes() > oldblock.getNumBytes()) {
        throw new IOException("Cannot update block file (=" + blockFile
            + ") length from " + oldblock.getNumBytes() + " to " + newblock.getNumBytes());
      }

      // Although we've waited for the active threads all dead before updating
      // the map so there should be no data race there, we still create new
      // ActiveFile object to make sure in case another thread holds it,
      // it won't cause any problem for us.
      //
      try {
        volumeMap.copyOngoingCreates(namespaceId, oldblock);
      } catch (CloneNotSupportedException e) {
        // It should never happen.
        throw new IOException("Cannot clone ActiveFile object", e);
      }

      // Now perform the update
      File tmpMetaFile = null;
      if (!binfo.isInlineChecksum()) {
        // rename meta file to a tmp file
        tmpMetaFile = new File(oldMetaFile.getParent(),
            oldMetaFile.getName() + "_tmp" + newblock.getGenerationStamp());
        if (!oldMetaFile.renameTo(tmpMetaFile)) {
          throw new IOException("Cannot rename block meta file to " + tmpMetaFile);
        }
      }

      long oldBlockLength;
      if (!binfo.isInlineChecksum()) {
        oldBlockLength = blockFile.length();
      } else {
        oldBlockLength = BlockInlineChecksumReader.getBlockSizeFromFileLength(
            blockFile.length(), binfo.getChecksumType(),
            binfo.getBytesPerChecksum());
      }
      ActiveFile file = null;
      if (newblock.getNumBytes() < oldBlockLength) {
        if (!binfo.isInlineChecksum()) {
          new BlockWithChecksumFileWriter(binfo.getBlockDataFile(), tmpMetaFile)
            .truncateBlock(oldBlockLength, newblock.getNumBytes());
        } else {
          new BlockInlineChecksumWriter(binfo.getBlockDataFile(), binfo.getChecksumType(),
              binfo.getBytesPerChecksum(), datanode.writePacketSize)
            .truncateBlock(newblock.getNumBytes());
        }
        file = volumeMap.getOngoingCreates(namespaceId, oldblock);
        if (file != null) {
          file.setBytesAcked(newblock.getNumBytes());
          file.setBytesOnDisk(newblock.getNumBytes());
          file.setBytesReceived(newblock.getNumBytes());
        } else {
          // This should never happen unless called from unit tests.
          binfo.syncInMemorySize();
        }
      }

      String newDataFileName;
      if (!binfo.isInlineChecksum()) {
        //rename the tmp file to the new meta file (with new generation stamp)
        File newMetaFile = BlockWithChecksumFileWriter.getMetaFile(blockFile, newblock);
        if (!tmpMetaFile.renameTo(newMetaFile)) {
          throw new IOException("Cannot rename tmp meta file to " + newMetaFile);
        }
      } else {
        newDataFileName = BlockInlineChecksumWriter.getInlineChecksumFileName(
            newblock, binfo.getChecksumType(), binfo.getBytesPerChecksum());
        File newDataFile = new File(blockFile.getParent(), newDataFileName);
        if (!blockFile.renameTo(newDataFile)) {
          throw new IOException("Cannot rename data file to " + newDataFileName);
        }
        // fsyncIfPossible parent directory to persist rename.
        if (datanode.syncOnClose) {
          NativeIO.fsyncIfPossible(newDataFile.getParent());
        }
        setDataFileForBlock(namespaceId, oldblock, newDataFile);
      }
  
      if(volumeMap.getOngoingCreates(namespaceId, oldblock) != null){
        ActiveFile af = volumeMap.removeOngoingCreates(namespaceId, oldblock);
        volumeMap.addOngoingCreates(namespaceId, newblock, af);
      }
      volumeMap.update(namespaceId, oldblock, newblock);
  
      // paranoia! verify that the contents of the stored block 
      // matches the block file on disk.
      validateBlockMetadata(namespaceId, newblock);
      return null;
    } finally {
      lock.writeLock().unlock();
    }
  }
1367322
def get_processing_block_ids(self):
        """"""

        # Initialise empty list
        _processing_block_ids = []

        # Pattern used to search processing block ids
        pattern = '*:processing_block:*'
        block_ids = self._db.get_ids(pattern)

        for block_id in block_ids:
            id_split = block_id.split(':')[-1]
            _processing_block_ids.append(id_split)
        return sorted(_processing_block_ids)
1164834
def _get_memory_banks_listed_in_dir(path):
    """"""
    # Such directories contain entries named "node<id>" for each memory bank
    return [int(entry[4:]) for entry in os.listdir(path) if entry.startswith('node')]
519787
def get(self, block_alias, context):
        """"""
        contents = []

        dynamic_block_contents = self.get_contents_dynamic(block_alias, context)
        if dynamic_block_contents:
            contents.append(dynamic_block_contents)

        static_block_contents = self.get_contents_static(block_alias, context)
        if static_block_contents:
            contents.append(static_block_contents)

        if not contents:
            return ''

        return choice(contents)
245252
private ArrayList<Thread> getActiveThreads(int namespaceId, Block block) {
    lock.writeLock().lock();
    try {
      //check ongoing create threads
      final ActiveFile activefile = volumeMap.getOngoingCreates(namespaceId, block);
      if (activefile != null && !activefile.threads.isEmpty()) {
        //remove dead threads
        for(Iterator<Thread> i = activefile.threads.iterator(); i.hasNext(); ) {
          final Thread t = i.next();
          if (!t.isAlive()) {
            i.remove();
          }
        }
  
        //return living threads
        if (!activefile.threads.isEmpty()) {
          return new ArrayList<Thread>(activefile.threads);
        }
      }
    } finally {
      lock.writeLock().unlock();
    }
    return null;
  }
860658
def processing_blocks(self):
        """"""
        pb_list = ProcessingBlockList()
        # TODO(BMo) realtime, offline etc.
        return json.dumps(dict(active=pb_list.active,
                               completed=pb_list.completed,
                               aborted=pb_list.aborted))
1063148
def get_index_by_name(self, name):
        """"""
        for i in range(self.n_blocks):
            if self.get_block_name(i) == name:
                return i
        raise KeyError('Block name ({}) not found'.format(name))
424300
def namedb_get_all_revealed_namespace_ids( self, current_block ):
    
    
    query = "SELECT namespace_id FROM namespaces WHERE op = ? AND reveal_block < ?;"
    args = (NAMESPACE_REVEAL, current_block + NAMESPACE_REVEAL_EXPIRE )

    namespace_rows = namedb_query_execute( cur, query, args )
    ret = []
    for namespace_row in namespace_rows:
        ret.append( namespace_row['namespace_id'] )

    return ret
867956
def get_sub_array_ids(self):
        """"""

        # Initialise empty list
        _scheduling_block_ids = []
        _sub_array_ids = []

        for blocks_id in self.get_sched_block_instance_ids():
            _scheduling_block_ids.append(blocks_id)
        block_details = self.get_block_details(_scheduling_block_ids)
        for details in block_details:
            _sub_array_ids.append(details['sub_array_id'])
        _sub_array_ids = sorted(list(set(_sub_array_ids)))
        return _sub_array_ids