split a MagIC upload format file into lists.
    the lists are split by the '>>>' lines between file_types.
167977
def split_lines(lines):
    
    container = []
    new_list = []
    for line in lines:
        if '>>>' in line:
            container.append(new_list)
            new_list = []
        else:
            new_list.append(line)
    container.append(new_list)
    return container
87501
def split_file(name, record_count)
      counts = row_count_threshold(record_count)
      file_name = local_file(name)
      `#{RailsRedshiftReplicator.split_command} -l #{counts} #{file_name} #{file_name}.`
    end
1303146
def split_into_segments(data):
    
    if data[0:2] != b"\xff\xd8":
        raise InvalidImageDataError("Given data isn't JPEG.")

    head = 2
    segments = [b"\xff\xd8"]
    while 1:
        if data[head: head + 2] == b"\xff\xda":
            segments.append(data[head:])
            break
        else:
            length = struct.unpack(">H", data[head + 2: head + 4])[0]
            endPoint = head + length + 2
            seg = data[head: endPoint]
            segments.append(seg)
            head = endPoint

        if (head >= len(data)):
            raise InvalidImageDataError("Wrong JPEG data.")
    return segments
1320300
def _upload_files(item, files, upload_kwargs, prev_identifier=None, archive_session=None):
    """"""
    responses = []
    if (upload_kwargs['verbose']) and (prev_identifier != item.identifier):
        print('{0}:'.format(item.identifier))

    try:
        response = item.upload(files, **upload_kwargs)
        responses += response
    except HTTPError as exc:
        responses += [exc.response]
    finally:
        # Debug mode.
        if upload_kwargs['debug']:
            for i, r in enumerate(responses):
                if i != 0:
                    print('---')
                headers = '\n'.join(
                    [' {0}:{1}'.format(k, v) for (k, v) in r.headers.items()]
                )
                print('Endpoint:\n {0}\n'.format(r.url))
                print('HTTP Headers:\n{0}'.format(headers))
                return responses

        # Format error message for any non 200 responses that
        # we haven't caught yet,and write to stderr.
        if responses and responses[-1].status_code and responses[-1].status_code != 200:
            if not responses[-1].status_code:
                return responses
            filename = responses[-1].request.url.split('/')[-1]
            try:
                msg = get_s3_xml_text(responses[-1].content)
            except:
                msg = responses[-1].content
            print(' error uploading {}: {}'.format(filename, msg), file=sys.stderr)

    return responses
579714
def path_file_to_list(path_file):
    
    paths = []
    path_file_fd = file(path_file)

    for line_no, line in enumerate(path_file_fd.readlines(), start=1):
        line = line.strip()

        if not line:
            # Blank line support
            continue

        if line.startswith('#'):
            # Comment support
            continue

        try:
            is_valid_path(line)
            paths.append(line)
        except ValueError, ve:
            args = (ve, path_file, line_no)
            raise ValueError('%s error found in %s:%s.' % args)

    return paths
864445
def upload_slice_file(self, real_file_path, slice_size, file_name, offset=0, dir_name=None):
        
        if dir_name is not None and dir_name[0] == '/':
            dir_name = dir_name[1:len(dir_name)]
        if dir_name is None:
            dir_name = ""
        self.url = 'http://' + self.config.region + '.file.myqcloud.com/files/v2/' + str(
            self.config.app_id) + '/' + self.config.bucket
        if dir_name is not None:
            self.url = self.url + '/' + dir_name
        self.url = self.url + '/' + file_name
        file_size = os.path.getsize(real_file_path)
        session = self._upload_slice_control(file_size=file_size, slice_size=slice_size)
        with open(real_file_path, 'rb') as local_file:
            while offset < file_size:
                file_content = local_file.read(slice_size)
                self._upload_slice_data(filecontent=file_content, session=session, offset=offset)
                offset += slice_size
            r = self._upload_slice_finish(session=session, file_size=file_size)
        return r
12013
def magic_read(self,infile):
        
#        print "calling magic_read(self, infile)", infile
        hold,magic_data,magic_record,magic_keys=[],[],{},[]
        try:
            f=open(infile,"r")
        except:
            return [],'bad_file'
        d = f.readline()[:-1].strip('\n')
        if d[0]=="s" or d[1]=="s":
            delim='space'
        elif d[0]=="t" or d[1]=="t":
            delim='tab'
        else:
            print('error reading ', infile)
            sys.exit()
        if delim=='space':file_type=d.split()[1]
        if delim=='tab':file_type=d.split('\t')[1]
        if file_type=='delimited':
            if delim=='space':file_type=d.split()[2]
            if delim=='tab':file_type=d.split('\t')[2]
        if delim=='space':line =f.readline()[:-1].split()
        if delim=='tab':line =f.readline()[:-1].split('\t')
        for key in line:
            magic_keys.append(key)
        lines=f.readlines()
        for line in lines[:-1]:
            line.replace('\n','')
            if delim=='space':rec=line[:-1].split()
            if delim=='tab':rec=line[:-1].split('\t')
            hold.append(rec)
        line = lines[-1].replace('\n','')
        if delim=='space':rec=line[:-1].split()
        if delim=='tab':rec=line.split('\t')
        hold.append(rec)
        for rec in hold:
            magic_record={}
            if len(magic_keys) != len(rec):

                print("Warning: Uneven record lengths detected: ")
                #print magic_keys
                #print rec
            for k in range(len(rec)):
               magic_record[magic_keys[k]]=rec[k].strip('\n')
            magic_data.append(magic_record)
        magictype=file_type.lower().split("_")
        Types=['er','magic','pmag','rmag']
        if magictype in Types:file_type=file_type.lower()
#        print "magic data from magic_read:"
#        print str(magic_data)[:500] + "..."
#        print "file_type", file_type
        return magic_data,file_type
695709
def file_to_list(in_file):
    '''  '''
    lines = []
    for line in in_file:
        # Strip new line
        line = line.strip('\n')

        # Ignore empty lines
        if line != '':
            # Ignore comments
            if line[0] != '#':
                lines.append(line)

    return lines
821208
def loadFileList(inputFilelist):
    
    f = open(inputFilelist[1:])
    # check the first line in order to determine whether
    # IVM files have been specified in a second column...
    lines = f.readline()
    f.close()

    # If there is a second column...
    if len(line.split()) == 2:
        # ...parse out the names of the IVM files as well
        ivmlist = irafglob.irafglob(input, atfile=atfile_ivm)

    # Parse the @-file with irafglob to extract the input filename
    filelist = irafglob.irafglob(input, atfile=atfile_sci)
    return filelist
781308
def multipart_content(*files):
    
    outer = MIMEMultipart()
    for fname in files:
        mtype = get_type(fname)
        maintype, subtype = mtype.split('/', 1)
        with open(fname) as f:
            msg = MIMEText(f.read(), _subtype=subtype)
        msg.add_header('Content-Disposition', 'attachment', 
                       filename=os.path.basename(fname))
        outer.attach(msg)
    return outer.as_string()
1068828
def format_file_params(files):
        '''
            
        '''
        files_payload = {}
        if files:
            for idx, filename in enumerate(files):
                files_payload["file[" + str(idx) + "]"] = open(filename, 'rb')
        return files_payload
656547
def parse_rule_file(file)
      tags = []
      begin
        tag_text = File.read file
        tags = tag_text.split(/\s/)
      rescue
        raise "ERROR: Could not read or parse the specified rule file at #{file}"
      end
      tags
    end
490749
def _parse_file_spec(self, spec):
        '''
        
        '''
        # separate wildcard specs into more specs
        if '*' in spec['file']:
            expanded_paths = _expand_paths(spec['file'])
            if not expanded_paths:
                return []
            expanded_specs = []
            for p in expanded_paths:
                _spec = copy.copy(spec)
                _spec['file'] = p
                expanded_specs.append(_spec)
            return expanded_specs

        else:
            return [spec]
927311
def upload_file(f):
    
    with FilesOpener(f) as files:
        response = requests.post(
            'https://telegra.ph/upload',
            files=files
        ).json()

    if isinstance(response, list):
        error = response[0].get('error')
    else:
        error = response.get('error')

    if error:
        raise TelegraphException(error)

    return [i['src'] for i in response]
1066872
def upload(
    ctx, files, max_threads, prompt, forward, reverse, tags, metadata, project_id, coerce_ascii
):
    """"""

    appendables = {}
    if tags:
        appendables["tags"] = []
        for tag in tags:
            appendables["tags"].append(tag)

    if metadata:
        appendables["metadata"] = {}
        for metadata_kv in metadata:
            split_metadata = metadata_kv.split("=", 1)
            if len(split_metadata) > 1:
                metadata_value = split_metadata[1]
                appendables["metadata"][split_metadata[0]] = metadata_value

    appendables = validate_appendables(appendables, ctx.obj["API"])

    if (forward or reverse) and not (forward and reverse):
        click.echo("You must specify both forward and reverse files", err=True)
        ctx.exit(1)

    if forward and reverse:
        if len(files) > 0:
            click.echo(
                "You may not pass a FILES argument when using the "
                " --forward and --reverse options.",
                err=True,
            )
            ctx.exit(1)
        files = [(forward, reverse)]
    elif len(files) == 0:
        click.echo(ctx.get_help())
        return
    else:
        files = list(files)

        # "intelligently" find paired files and tuple them
        paired_files = []
        single_files = set(files)

        for filename in files:
            # convert "read 1" filenames into "read 2" and check that they exist; if they do
            # upload the files as a pair, autointerleaving them
            pair = re.sub("[._][Rr]1[._]", lambda x: x.group().replace("1", "2"), filename)

            # we don't necessary need the R2 to have been passed in; we infer it anyways
            if pair != filename and os.path.exists(pair):
                if not prompt and pair not in single_files:
                    # if we're not prompting, don't automatically pull in files
                    # not in the list the user passed in
                    continue

                paired_files.append((filename, pair))

                if pair in single_files:
                    single_files.remove(pair)

                single_files.remove(filename)

        auto_pair = True

        if prompt and len(paired_files) > 0:
            pair_list = ""
            for p in paired_files:
                pair_list += "\n  {}  &  {}".format(os.path.basename(p[0]), os.path.basename(p[1]))

            answer = click.confirm(
                "It appears there are paired files:{}\nInterleave them after upload?".format(
                    pair_list
                ),
                default="Y",
            )

            if not answer:
                auto_pair = False

        if auto_pair:
            files = paired_files + list(single_files)

    total_size = sum(
        [
            (_file_size(x[0], uncompressed=True) + _file_size(x[1], uncompressed=True))
            if isinstance(x, tuple)
            else _file_size(x, uncompressed=False)
            for x in files
        ]
    )

    upload_kwargs = {
        "metadata": appendables["valid_metadata"],
        "tags": appendables["valid_tags"],
        "project": project_id,
        "coerce_ascii": coerce_ascii,
        "progressbar": progressbar(length=total_size, label="Uploading..."),
    }

    run_via_threadpool(
        ctx.obj["API"].Samples.upload,
        files,
        upload_kwargs,
        max_threads=max_threads,
        graceful_exit=False,
    )
422104
def machine_listings_to_file_entries(listings: Iterable[dict]) -> \
        Iterable[FileEntry]:
    ''''''
    for listing in listings:
        yield FileEntry(
            listing['name'],
            type=listing.get('type'),
            size=listing.get('size'),
            date=listing.get('modify')
        )
22099
def file_to_list(path):
    
    if not os.path.exists(path):
        ui.error(c.MESSAGES["path_missing"], path)
        sys.exit(1)

    with codecs.open(path, "r", "UTF-8") as contents:
        lines = contents.read().splitlines()

    return lines