dataset: /a/b/c
        block: /a/b/c#d
501411
def execute(self, conn, dataset="", block_name="", data_tier_name="", origin_site_name="", logical_file_name="",
                run_num=-1, min_cdate=0, max_cdate=0, min_ldate=0, max_ldate=0, cdate=0,
                ldate=0, open_for_writing=-1, transaction = False):
        
        binds = {}

        basesql = self.sql
        joinsql = ""
        wheresql = ""
        generatedsql = ""

        if logical_file_name and logical_file_name != "%":
            joinsql +=  " JOIN %sFILES FL ON FL.BLOCK_ID = B.BLOCK_ID " %(self.owner)
            op =  ("=", "like")["%" in logical_file_name]
            wheresql +=  " WHERE LOGICAL_FILE_NAME %s :logical_file_name " % op
            binds.update( logical_file_name = logical_file_name )

        if  block_name and  block_name !="%":
            andorwhere = ("WHERE", "AND")[bool(wheresql)]
            op =  ("=", "like")["%" in block_name]
            wheresql +=  " %s B.BLOCK_NAME %s :block_name " % ((andorwhere, op))
            binds.update( block_name = block_name )

        if data_tier_name or (dataset and dataset!="%"):
            joinsql += "JOIN %sDATASETS DS ON DS.DATASET_ID = B.DATASET_ID "  % (self.owner)
            andorwhere = ("WHERE", "AND")[bool(wheresql)]
            if dataset:
                op = ("=", "like")["%" in dataset]
                wheresql += " %s DS.DATASET %s :dataset " % ((andorwhere, op))
                binds.update(dataset=dataset)
            if data_tier_name:
                joinsql += "JOIN {owner}DATA_TIERS DT ON DS.DATA_TIER_ID=DT.DATA_TIER_ID ".format(owner=self.owner)
                wheresql += " %s DT.DATA_TIER_NAME=:data_tier_name " % (andorwhere)
                binds.update(data_tier_name=data_tier_name)

        if origin_site_name and  origin_site_name != "%":
            op = ("=", "like")["%" in origin_site_name]
            wheresql += " AND B.ORIGIN_SITE_NAME %s :origin_site_name " % op
            binds.update(origin_site_name = origin_site_name)

        if open_for_writing == 0 or open_for_writing == 1:
            wheresql += " AND B.OPEN_FOR_WRITTING = :open_for_writing "

        if cdate != 0:
            wheresql += "AND B.CREATION_DATE = :cdate "
            binds.update(cdate = cdate)
        elif min_cdate != 0 and max_cdate != 0:
            wheresql += "AND B.CREATION_DATE BETWEEN :min_cdate and :max_cdate "
            binds.update(min_cdate = min_cdate)
            binds.update(max_cdate = max_cdate)
        elif min_cdate != 0 and max_cdate == 0:
            wheresql += "AND B.CREATION_DATE > :min_cdate "
            binds.update(min_cdate = min_cdate)
        elif min_cdate ==0 and max_cdate != 0:
            wheresql += "AND B.CREATION_DATE < :max_cdate "
            binds.update(max_cdate = max_cdate)
        else:
            pass
        if ldate != 0:
            wheresql += "AND B.LAST_MODIFICATION_DATE = :ldate "
            binds.update(ldate = ldate)
        elif min_ldate != 0 and max_ldate != 0:
            wheresql += "AND B.LAST_MODIFICATION_DATE BETWEEN :min_ldate and :max_ldate "
            binds.update(min_ldate = min_ldate)
            binds.update(max_ldate = max_ldate)
        elif min_ldate != 0 and max_ldate == 0:
            wheresql += "AND B.LAST_MODIFICATION_DATE > :min_ldate "
            binds.update(min_ldate = min_ldate)
        elif min_cdate ==0 and max_cdate != 0:
            wheresql += "AND B.LAST_MODIFICATION_DATE < :max_ldate "
            binds.update(max_ldate = max_ldate)
        else:
            pass

        #one may provide a list of runs , so it has to be the last one in building the bind.
        if run_num !=-1 :
            basesql = basesql.replace("SELECT", "SELECT DISTINCT") + " , FLM.RUN_NUM  "
            if not logical_file_name:
                joinsql +=  " JOIN %sFILES FL ON FL.BLOCK_ID = B.BLOCK_ID " %(self.owner)
            joinsql += " JOIN %sFILE_LUMIS FLM on FLM.FILE_ID = FL.FILE_ID " %(self.owner)
            run_list=[]
            wheresql_run_list=''
            wheresql_run_range=''
            #
            for r in parseRunRange(run_num):
                if isinstance(r, basestring) or isinstance(r, int) or isinstance(r, long):
                    run_list.append(str(r))
                if isinstance(r, run_tuple):
                    if r[0] == r[1]:
                        dbsExceptionHandler('dbsException-invalid-input', "DBS run_num range must be apart at least by 1.", self.logger.exception)
                    wheresql_run_range = " FLM.RUN_NUM between :minrun and :maxrun "
                    binds.update({"minrun":r[0]})
                    binds.update({"maxrun":r[1]})
            #
            if run_list:
                wheresql_run_list = " FLM.RUN_NUM in (SELECT TOKEN FROM TOKEN_GENERATOR) "
                generatedsql, run_binds = create_token_generator(run_list)
                binds.update(run_binds)
            #  
            if wheresql_run_range and wheresql_run_list:
                wheresql += " and (" + wheresql_run_range + " or " +  wheresql_run_list + " )"
            elif wheresql_run_range and not wheresql_run_list:
                wheresql +=  " and " + wheresql_run_range
            elif not wheresql_run_range and wheresql_run_list:
                wheresql += " and "  + wheresql_run_list
        #
        sql = " ".join((generatedsql, basesql, self.fromsql, joinsql, wheresql))

        cursors = self.dbi.processData(sql, binds, conn, transaction, returnCursor=True)
        for i in cursors:
            d = self.formatCursor(i, size=100)
            if isinstance(d, list) or isinstance(d, GeneratorType):
                for elem in d:
                    yield elem
            elif d:
                yield d
178823
def dataset(directory, images_file, labels_file):
  """"""

  images_file = download(directory, images_file)
  labels_file = download(directory, labels_file)

  check_image_file_header(images_file)
  check_labels_file_header(labels_file)

  def decode_image(image):
    # Normalize from [0, 255] to [0.0, 1.0]
    image = tf.decode_raw(image, tf.uint8)
    image = tf.cast(image, tf.float32)
    image = tf.reshape(image, [784])
    return image / 255.0

  def decode_label(label):
    label = tf.decode_raw(label, tf.uint8)  # tf.string -> [tf.uint8]
    label = tf.reshape(label, [])  # label is a scalar
    return tf.to_int32(label)

  images = tf.data.FixedLengthRecordDataset(
      images_file, 28 * 28, header_bytes=16).map(decode_image)
  labels = tf.data.FixedLengthRecordDataset(
      labels_file, 1, header_bytes=8).map(decode_label)
  return tf.data.Dataset.zip((images, labels))
621211
def block_partition(block, i):
    
    i += 1
    new_block = BasicBlock(block.asm[i:])
    block.mem = block.mem[:i]
    block.asm = block.asm[:i]
    block.update_labels()
    new_block.update_labels()

    new_block.goes_to = block.goes_to
    block.goes_to = IdentitySet()

    new_block.label_goes = block.label_goes
    block.label_goes = []

    new_block.next = new_block.original_next = block.original_next
    new_block.prev = block
    new_block.add_comes_from(block)

    if new_block.next is not None:
        new_block.next.prev = new_block
        new_block.next.add_comes_from(new_block)
        new_block.next.delete_from(block)

    block.next = block.original_next = new_block
    block.update_next_block()
    block.add_goes_to(new_block)

    return block, new_block
888176
def load_rocstories_dataset(dataset_path):
    
    with open(dataset_path, encoding='utf_8') as f:
        f = csv.reader(f)
        output = []
        next(f) # skip the first line
        for line in tqdm(f):
            output.append((' '.join(line[1:5]), line[5], line[6], int(line[-1])-1))
    return output
427678
def cli():
    """"""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "dataset_directory",
        help="Directory with datasets to be served"
    )
    parser.add_argument(
        "-p",
        "--port",
        type=int,
        default=8081,
        help="Port to serve datasets on (default 8081)"
    )
    args = parser.parse_args()
    if not os.path.isdir(args.dataset_directory):
        parser.error("Not a directory: {}".format(args.dataset_directory))

    serve_dtool_directory(args.dataset_directory, args.port)
189388
def _build_data_block(self, lexer):
        
        data_block = OrderedDict()
        header = ''

        while True:
            token = next(lexer)
            key = token.__class__.__name__

            if key == 'DataHeader':
                header = token.header[1:-1]
                data_block.setdefault(header, [])

            elif key == 'DataItem':
                data_block[header].append(token.data_item)

            elif key == 'DataBlockEnd':
                break

            else:
                raise KeyError('SDfile data block does not supposed to have any other information: "{}".'.format(key))

        return data_block
1048138
def summary(dataset_uri, format):
    """"""
    dataset = dtoolcore.DataSet.from_uri(dataset_uri)
    creator_username = dataset._admin_metadata["creator_username"]
    frozen_at = dataset._admin_metadata["frozen_at"]
    num_items = len(dataset.identifiers)
    tot_size = sum([dataset.item_properties(i)["size_in_bytes"]
                    for i in dataset.identifiers])

    if format == "json":
        json_lines = [
            '{',
            '  "name": "{}",'.format(dataset.name),
            '  "uuid": "{}",'.format(dataset.uuid),
            '  "creator_username": "{}",'.format(creator_username),
            '  "number_of_items": {},'.format(num_items),
            '  "size_in_bytes": {},'.format(tot_size),
            '  "frozen_at": {}'.format(frozen_at),
            '}',
        ]
        formatted_json = "\n".join(json_lines)
        colorful_json = pygments.highlight(
            formatted_json,
            pygments.lexers.JsonLexer(),
            pygments.formatters.TerminalFormatter())
        click.secho(colorful_json, nl=False)

    else:
        info = [
            ("name", dataset.name),
            ("uuid", dataset.uuid),
            ("creator_username", creator_username),
            ("number_of_items", str(num_items)),
            ("size", sizeof_fmt(tot_size).strip()),
            ("frozen_at", date_fmt(frozen_at)),
        ]
        for key, value in info:
            click.secho("{}: ".format(key), nl=False)
            click.secho(value, fg="green")
498633
def p_rddl_block(self, p):
        ''''''
        if p[1] is None:
            p[0] = dict()
        else:
            name, block = p[2]
            p[1][name] = block
            p[0] = p[1]
1129159
@Override
  public Set<Dataset> findDistinctDatasets() throws IOException {
    Set<Dataset> datasets = Sets.newHashSet();
    Path inputPath = new Path(this.inputDir);
    Path inputLatePath = new Path(inputPath, MRCompactor.COMPACTION_LATE_DIR_SUFFIX);
    Path outputPath = new Path(this.destDir);
    Path outputLatePath = new Path(outputPath, MRCompactor.COMPACTION_LATE_DIR_SUFFIX);
    Dataset dataset =
        new Dataset.Builder().withPriority(this.getDatasetPriority(inputPath.getName()))
            .addInputPath(this.recompactDatasets ? outputPath : inputPath)
            .addInputLatePath(this.recompactDatasets ? outputLatePath : inputLatePath).withOutputPath(outputPath)
            .withOutputLatePath(outputLatePath).withOutputTmpPath(new Path(this.tmpOutputDir)).build();
    datasets.add(dataset);
    return datasets;
  }
1135533
def generate_dataset(self, cdl_path):
        '''
        
        '''
        if '.cdl' in cdl_path:  # it's possible the filename doesn't have the .cdl extension
            ds_str = cdl_path.replace('.cdl', '.nc')
        else:
            ds_str = cdl_path + '.nc'
        subprocess.call(['ncgen', '-o', ds_str, cdl_path])
        return ds_str
259182
def get_block_bounds(filename):
 
 bs = []
 with open(filename,'rb') as inf:
  while True:
   bytes1 = inf.read(12)
   if len(bytes1) < 12: break
   bs.append([inf.tell()-12])
   gzip_id1,gzip_id2,compression_method,flag,mtime,xfl,osval,xlen=struct.unpack('<BBBBIBBH',bytes1)
   # ready to look in extra field
   bytes2 = inf.read(xlen) # all the extra field stuff 
   s = StringIO.StringIO(bytes2)
   obsslen = 0
   blocksize = 0
   while True:
     v1 = s.read(4)
     if len(v1) == 0: 
       break
     if len(v1) < 4:
       sys.stderr.write("lack header values ERROR\n")
       return False
     s1,s2,slen = struct.unpack('<BBH',v1)
     if s1 == 66 and s2 == 67:
       has_id = True
       obsslen = slen
       blocksize = struct.unpack('<H',s.read(slen))[0]
     else:
       v = s.read(slen)
   chunk = inf.read(blocksize-1-xlen-19)
   inf.read(9)
   bs[-1].append(inf.tell())
 return bs
505736
def shape(cls, dataset):
        
        if not dataset.data:
            return (0, len(dataset.dimensions()))

        rows, cols = 0, 0
        ds = cls._inner_dataset_template(dataset)
        for d in dataset.data:
            ds.data = d
            r, cols = ds.interface.shape(ds)
            rows += r
        return rows+len(dataset.data)-1, cols
1014357
def dumpBlock(self, block_name):
        
        if '%' in block_name or '*' in block_name:
            msg = "No wildcard is allowed in block_name for dumpBlock API"
            dbsExceptionHandler('dbsException-invalid-input', msg, self.logger.exception)

        conn = self.dbi.connection()
        try :
            #block name is unique
            block1 = self.blocklist.execute(conn, block_name=block_name)
            block = []
            for b1 in block1:
		if not b1:
		    return {}
		else:
		    block = b1
            #a block only has one dataset and one primary dataset
            #in order to reduce the number of dao objects, we will not write
            #a special migration one. However, we will have to remove the
            #extras
            #block1 is a generator. When it is empty, it will skip the for loop above. why? 
            #we cannot test on b1 to decide if the generator is empty or not.
            #so have to do below:
            if not block: return {}
            dataset1 = self.datasetlist.execute(conn,
                      dataset=block["dataset"], dataset_access_type="")
	    dataset = []
            for d in dataset1:
		if d:
		    dataset = d
		    dconfig_list = self.outputCoflist.execute(conn, dataset=dataset['dataset'])
                else: return {}
            #get block parentage
            bparent = self.blockparentlist.execute(conn, block['block_name'])
            #get dataset parentage
            dsparent = self.dsparentlist.execute(conn, dataset['dataset'])
            for p in dsparent:
                del p['parent_dataset_id']
                if 'dataset'in p:
                    del p['dataset']
                elif 'this_dataset' in p:
                    del p['this_dataset']
                else:
                    pass

            fparent_list = self.fplist.execute(conn,
                                               block_id=block['block_id'])
	    fparent_list2 = [] 	
	    for fp in fparent_list:
		fparent_list2.append(fp)	
	    #print "---YG file Parent List--"
	    #print fparent_list2	
            fconfig_list = self.outputCoflist.execute(conn,
                                                block_id=block['block_id'])
            acqEra = {}
            prsEra = {}
            if dataset["acquisition_era_name"] not in ( "", None):
                acqEra = self.aelist.execute(conn,
                        acquisitionEra=dataset["acquisition_era_name"])[0]
            if dataset["processing_version"] not in ("", None):
                prsEra = self.pelist.execute(conn,
                        processingV=dataset["processing_version"])[0]
            primds = self.primdslist.execute(conn,
                        primary_ds_name=dataset["primary_ds_name"])[0]
            del dataset["primary_ds_name"], dataset['primary_ds_type']
            files = self.filelist.execute(conn, block_name=block_name)
            for f in files:
                #There are a trade off between json sorting and db query.
                #We keep lumi sec in a file, but the file parentage seperate
                #from file
		file_lumi_list = []
                for item in self.fllist.execute(conn, logical_file_name=f['logical_file_name'], migration=True):
		    file_lumi_list.append(item)
		#print "---YG file lumi list---"	
		f.update(file_lumi_list = file_lumi_list)
		del file_lumi_list #YG 09/2015
                del f['branch_hash_id']
            del dataset["acquisition_era_name"], dataset["processing_version"]
            del block["dataset"]
            result = dict(block=block, dataset=dataset, primds=primds,
                          files=files, block_parent_list=bparent,
                          ds_parent_list=dsparent, file_conf_list=fconfig_list,
                          file_parent_list=fparent_list2, dataset_conf_list=dconfig_list)
            if acqEra:
                result["acquisition_era"] = acqEra
            if prsEra:
                result["processing_era"] = prsEra
            return result
        finally:
            if conn:
                conn.close()
543036
def splits_and_paths(self, data_dir):
    """"""
    filepath_fns = {
        problem.DatasetSplit.TRAIN: self.training_filepaths,
        problem.DatasetSplit.EVAL: self.dev_filepaths,
        problem.DatasetSplit.TEST: self.test_filepaths,
    }

    def append_epoch(paths):
      return [
          "{}.{}".format(path, self.current_epoch)
          for path in paths
      ]

    # We set shuffled=True as we don't want to shuffle on disk later.
    return [
        (split["split"], append_epoch(filepath_fns[split["split"]](
            data_dir, split["shards"], shuffled=True
        )))
        for split in self.dataset_splits
    ]
544274
def pre_process_data(filepath):
    
    positive_path = os.path.join(filepath, 'pos')
    negative_path = os.path.join(filepath, 'neg')

    pos_label = 1
    neg_label = 0

    dataset = []

    for filename in glob.glob(os.path.join(positive_path, '*.txt')):
        with open(filename, 'r') as f:
            dataset.append((pos_label, f.read()))

    for filename in glob.glob(os.path.join(negative_path, '*.txt')):
        with open(filename, 'r') as f:
            dataset.append((neg_label, f.read()))

    shuffle(dataset)

    return dataset
292706
def new_dataset(args):
    
    if not args.args:
        raise ParserError('you must specify an existing directory')
    outdir = Path(args.args.pop(0))
    if not outdir.exists():
        raise ParserError('you must specify an existing directory')

    id_pattern = re.compile('[a-z_0-9]+$')
    md = {}
    if args.args:
        md['id'] = args.args.pop(0)
    else:
        md['id'] = input('Dataset ID: ')

    while not id_pattern.match(md['id']):
        print('dataset id must only consist of lowercase ascii letters, digits and _ (underscore)!')
        md['id'] = input('Dataset ID: ')

    outdir = outdir / md['id']
    if not outdir.exists():
        outdir.mkdir()

    for key in ['title', 'url', 'license', 'conceptlist', 'citation']:
        md[key] = input('Dataset {0}: '.format(key))

    # check license!
    # check conceptlist!

    for path in Path(pylexibank.__file__).parent.joinpath('dataset_template').iterdir():
        if path.is_file():
            if path.suffix in ['.pyc']:
                continue  # pragma: no cover
            target = path.name
            content = read_text(path)
            if '+' in path.name:
                target = re.sub(
                    '\+([a-z]+)\+',
                    lambda m: '{' + m.groups()[0] + '}',
                    path.name
                ).format(**md)
            if target.endswith('_tmpl'):
                target = target[:-5]
                content = content.format(**md)
            write_text(outdir / target, content)
        else:
            target = outdir / path.name
            if target.exists():
                shutil.rmtree(str(target))
            shutil.copytree(str(path), str(target))
    del md['id']
    jsonlib.dump(md, outdir / 'metadata.json', indent=4)
609244
def block(self, group=False, custom=False):
        '''
        
        '''
        reader = BlockReader()
        symbol = os.path.join(self.tdxdir, 'block_zs.dat')

        if symbol is not None:
            return reader.get_df(symbol, group)

        return None