Split the data depending on the folder (`train` or `valid`) in which the filenames are.
170505
def split_by_folder(self, train:str='train', valid:str='valid')->'ItemLists':
        ""
        return self.split_by_idxs(self._get_by_folder(train), self._get_by_folder(valid))
877679
def _redirect_split(self, args):
        
        if ">>" in args:
            append = True
            usable, filename = args.split(">>")
        elif ">" in args:
            append = False
            usable, filename = args.split(">")
        else:
            append = filename = None
            usable = args

        if filename is not None:
            return (usable, filename.strip(), append)
        else:
            return (usable, filename, append)
1038791
def split_path_and_file(s):
    
    _path = s
    _filename = ""
    try:
        x = os.path.split(s)
        _path = x[0]
        _filename = x[1]
    except Exception:
        print("Error: unable to split path")

    return _path, _filename
916105
def split_by_idx(self, valid_idx:Collection[int])->'ItemLists':
        ""
        #train_idx = [i for i in range_of(self.items) if i not in valid_idx]
        train_idx = np.setdiff1d(arange_of(self.items), valid_idx)
        return self.split_by_idxs(train_idx, valid_idx)
1012343
def _split_by_callable_region(data):
    
    batch = tz.get_in(("metadata", "batch"), data)
    jointcaller = tz.get_in(("config", "algorithm", "jointcaller"), data)
    name = batch if batch else tz.get_in(("rgnames", "sample"), data)
    out_dir = utils.safe_makedir(os.path.join(data["dirs"]["work"], "joint", jointcaller, name))
    utils.safe_makedir(os.path.join(out_dir, "inprep"))
    parts = []
    for feat in _get_callable_regions(data):
        region_dir = utils.safe_makedir(os.path.join(out_dir, feat[0]))
        region_prep_dir = os.path.join(region_dir, "inprep")
        if not os.path.exists(region_prep_dir):
            os.symlink(os.path.join(os.pardir, "inprep"), region_prep_dir)
        region_outfile = os.path.join(region_dir, "%s-%s.vcf.gz" % (batch, region.to_safestr(feat)))
        parts.append((feat, data["work_bams"], data["vrn_files"], region_outfile))
    out_file = os.path.join(out_dir, "%s-joint.vcf.gz" % name)
    return out_file, parts
527043
def get_filenames_in_lipd(path, name=""):
    
    _filenames = []
    try:
        # in the top level, list all files and skip the "data" directory
        _top = [os.path.join(name, f) for f in os.listdir(path) if f != "data"]
        # in the data directory, list all files
        _dir_data = [os.path.join(name, "data", f) for f in os.listdir(os.path.join(path, "data"))]
        # combine the two lists
        _filenames = _top + _dir_data
    except Exception:
        pass
    return _filenames
1172792
public static void splitData(final String originalTrainingDataFile) {
        List<Tuple> trainingData = NaiveBayesClassifier.readTrainingData(originalTrainingDataFile, "\\s");
        List<Tuple> wrongData = new ArrayList<>();

        int lastTrainingDataSize;
        int iterCount = 0;
        do {
            System.out.println("Iteration:\t" + (++iterCount));
            lastTrainingDataSize = trainingData.size();

            NaiveBayesClassifier nbc = new NaiveBayesClassifier();
            nbc.train(trainingData);

            Iterator<Tuple> trainingDataIter = trainingData.iterator();
            while (trainingDataIter.hasNext()) {
                Tuple t = trainingDataIter.next();
                String actual = nbc.predictLabel(t);
                if (!t.label.equals(actual) && !t.label.equals("1")) { // preserve 1 since too few.
                    wrongData.add(t);
                    trainingDataIter.remove();
                }
            }

            Iterator<Tuple> wrongDataIter = wrongData.iterator();
            while (wrongDataIter.hasNext()) {
                Tuple t = wrongDataIter.next();
                String actual = nbc.predictLabel(t);
                if (t.label.equals(actual)) {
                    trainingData.add(t);
                    wrongDataIter.remove();
                }
            }
        } while (trainingData.size() != lastTrainingDataSize);

        writeToFile(trainingData, originalTrainingDataFile + ".aligned");
        writeToFile(wrongData, originalTrainingDataFile + ".wrong");
    }
993096
def example_splits(url_file, all_files):
  """"""

  def generate_hash(inp):
    """Generate a sha1 hash to match the raw url to the filename extracted."""
    h = hashlib.sha1()
    h.update(inp)
    return h.hexdigest()

  all_files_map = {f.split("/")[-1]: f for f in all_files}

  urls = [line.strip().encode("utf-8") for line in tf.gfile.Open(url_file)]

  filelist = []
  for url in urls:
    url_hash = generate_hash(url)
    filename = url_hash + ".story"
    if filename not in all_files_map:
      tf.logging.info("Missing file: %s" % url)
      continue
    filelist.append(all_files_map[filename])

  tf.logging.info("Found %d examples" % len(filelist))

  return filelist
995243
def stash_split(fqdn, result, *argl, **argd):
    
    global _splits
    if fqdn == "sklearn.cross_validation.train_test_split":
        key = id(result[1])
        _splits[key] = result

    #We don't actually want to return anything for the analysis; we are using it
    #as a hook to save pointers to the dataset split so that we can easily
    #analyze performance later on.
    return None
861782
def split_input(cls, mapper_spec):
    """"""
    filelists = mapper_spec.params[cls.FILES_PARAM]
    max_values_count = mapper_spec.params.get(cls.MAX_VALUES_COUNT_PARAM, -1)
    max_values_size = mapper_spec.params.get(cls.MAX_VALUES_SIZE_PARAM, -1)
    return [cls([0] * len(files), max_values_count, max_values_size)
            for files in filelists]
722373
def match_filenames(filenames, pattern):
    """"""
    matching = []

    for filename in filenames:
        if fnmatch(get_filebase(filename, pattern), globify(pattern)):
            matching.append(filename)

    return matching
512534
def split_and_save_datasets(X, Y, paths):
    
    shuffled_idxs = np.random.permutation(np.arange(len(X)))

    for i in range(len(paths)):
        # Take every len(paths) item, starting at i.
        # len(paths) is 3, so this would be [0::3], [1::3], [2::3]
        X_i = X[shuffled_idxs[i::len(paths)]]
        Y_i = Y[shuffled_idxs[i::len(paths)]]
        np.savez(paths[i], X=X_i, Y=Y_i)
968518
def merge(self, datasets=None, separate_datasets=False):
        """"""
        self.logger.info("merging")
        if separate_datasets:
            warnings.warn("The option seperate_datasets=True is"
                          "not implemented yet. Performing merging, but"
                          "neglecting the option.")
        else:
            if datasets is None:
                datasets = list(range(len(self.datasets)))
            first = True
            for dataset_number in datasets:
                if first:
                    dataset = self.datasets[dataset_number]
                    first = False
                else:
                    dataset = self._append(dataset, self.datasets[dataset_number])
                    for raw_data_file, file_size in zip(self.datasets[dataset_number].raw_data_files,
                                                        self.datasets[dataset_number].raw_data_files_length):
                        dataset.raw_data_files.append(raw_data_file)
                        dataset.raw_data_files_length.append(file_size)
            self.datasets = [dataset]
            self.number_of_datasets = 1
        return self
1293785
def flattened_filenames(filenames)
      # Expect an array of filenames return otherwise
      return filenames if !filenames.is_a?(Array)
      # Iterate through array
      filenames.map! do |filename|
        # Flatten if its a file, flatten if a dir.
        Dir.exist?(filename) ? directory_to_filenames(filename) : filename
      end
      # Flattern the array and remove all nils
      filenames.flatten.compact
    end
1040474
def split_data(data, subset, splits):
  '''
  '''

  return dict([(k, data[k][splits[subset]]) for k in data])
505945
def ensure_dirs(filename):
    """"""
    dirname, _ = os.path.split(filename)
    if dirname and not os.path.exists(dirname):
        os.makedirs(dirname)
68580
def _split_by_ready_regions(ext, file_key, dir_ext_fn):
    
    def _sort_by_size(region_w_bams):
        region, _ = region_w_bams
        _, start, end = region
        return end - start
    def _assign_bams_to_regions(data):
        """Ensure BAMs aligned with input regions, either global or individual.
        """
        for i, region in enumerate(data["region"]):
            work_bams = []
            for xs in data["region_bams"]:
                if len(xs) == 1:
                    work_bams.append(xs[0])
                else:
                    work_bams.append(xs[i])
            for work_bam in work_bams:
                assert os.path.exists(work_bam), work_bam
            yield region, work_bams
    def _do_work(data):
        if "region" in data:
            name = data["group"][0] if "group" in data else data["description"]
            out_dir = os.path.join(data["dirs"]["work"], dir_ext_fn(data))
            out_file = os.path.join(out_dir, "%s%s" % (name, ext))
            assert isinstance(data["region"], (list, tuple))
            out_parts = []
            for r, work_bams in sorted(_assign_bams_to_regions(data), key=_sort_by_size, reverse=True):
                out_region_dir = os.path.join(out_dir, r[0])
                out_region_file = os.path.join(out_region_dir,
                                               "%s-%s%s" % (name, pregion.to_safestr(r), ext))
                out_parts.append((r, work_bams, out_region_file))
            return out_file, out_parts
        else:
            return None, []
    return _do_work