Extract istep and compute list of rows to delete
779581
def _extract_rsnap_isteps(rproffile):
    """"""
    step_regex = re.compile(r'^\*+step:\s*(\d+) ; time =\s*(\S+)')
    isteps = []  # list of (istep, time, nz)
    rows_to_del = set()
    line = ' '
    with rproffile.open() as stream:
        while line[0] != '*':
            line = stream.readline()
        match = step_regex.match(line)
        istep = int(match.group(1))
        time = float(match.group(2))
        nlines = 0
        iline = 0
        for line in stream:
            if line[0] == '*':
                isteps.append((istep, time, nlines))
                match = step_regex.match(line)
                istep = int(match.group(1))
                time = float(match.group(2))
                nlines = 0
                # remove useless lines produced when run is restarted
                nrows_to_del = 0
                while isteps and istep <= isteps[-1][0]:
                    nrows_to_del += isteps.pop()[-1]
                rows_to_del = rows_to_del.union(
                    range(iline - nrows_to_del, iline))
            else:
                nlines += 1
                iline += 1
        isteps.append((istep, time, nlines))
    return isteps, rows_to_del
753758
def _extract(self, stim):
        '''  '''
        props = [(e.text, e.onset, e.duration) for e in stim.elements]
        vals, onsets, durations = map(list, zip(*props))
        return ExtractorResult(vals, stim, self, ['word'], onsets, durations)
414087
def delistify(x):
    
    if isinstance(x, list):
        x = [e.replace("'", "") for e in x]
        return '-'.join(sorted(x))
    return x
345790
def _recursive_cleanup(foo):
    
    if isinstance(foo, dict):
        for (key, val) in list(foo.items()):
            if isinstance(val, dict):
                _recursive_cleanup(val)
            if val == "" or val == [] or val == {}:
                del foo[key]
434057
def remove(self, row_or_row_indices):
        """"""
        if not row_or_row_indices:
            return
        if isinstance(row_or_row_indices, int):
            rows_remove = [row_or_row_indices]
        else:
            rows_remove = row_or_row_indices
        for col in self._columns:
            self._columns[col] = [elem for i, elem in enumerate(self[col]) if i not in rows_remove]
        return self
1205747
def delete_row(self, ind):
        
        self.df = pd.concat([self.df[:ind], self.df[ind+1:]], sort=True)
        return self.df
559936
def cleanup(self):
        
        remove = []
        for handle, assoc in self.assocs.items():
            if assoc.expiresIn == 0:
                remove.append(handle)
        for handle in remove:
            del self.assocs[handle]
        return len(remove), len(self.assocs)
534314
def get_offdiag_vals(dcorr):
    
    del_indexes=[]
    for spc1 in np.unique(dcorr.index.get_level_values(0)):
        for spc2 in np.unique(dcorr.index.get_level_values(0)):
            if (not (spc1,spc2) in del_indexes) and (not (spc2,spc1) in del_indexes):
                del_indexes.append((spc1,spc2))
    #         break
    for spc1 in np.unique(dcorr.index.get_level_values(0)):
        for spc2 in np.unique(dcorr.index.get_level_values(0)):
            if spc1==spc2:
                del_indexes.append((spc1,spc2))
    
    return dcorr.drop(del_indexes)
273664
def detokenize(self, inputs, delim=' '):
        
        detok = delim.join([self.idx2tok[idx] for idx in inputs])
        detok = detok.replace(self.separator + ' ', '')
        detok = detok.replace(self.separator, '')

        detok = detok.replace(config.BOS_TOKEN, '')
        detok = detok.replace(config.EOS_TOKEN, '')
        detok = detok.replace(config.PAD_TOKEN, '')
        detok = detok.strip()
        return detok
423068
def delete(cls, resources, background=False, force=False):
        """"""
        if not isinstance(resources, (list, tuple)):
            resources = [resources]

        ifaces = []
        for item in resources:
            try:
                ip_ = cls.info(item)
            except UsageError:
                cls.error("Can't find this ip %s" % item)

            iface = Iface.info(ip_['iface_id'])
            ifaces.append(iface['id'])

        return Iface.delete(ifaces, background)
579904
def grep_split(pattern, text):
    '''

    '''
    lines = text.splitlines()
    indices = [i for i, line in enumerate(lines)
               if re.search(pattern, line)]
    
    return ['\n'.join(part) for part in partition(lines, indices)]
249263
def extract_rows(self, result={}, selector='', table_headers=[], attr='', connector='', default='', verbosity=0, *args, **kwargs):
		
		result_list = []

		try:
			values = self.get_tree_tag(selector)
			if len(table_headers) >= len(values):
				from itertools import izip_longest
				pairs = izip_longest(table_headers, values, fillvalue=default)
			else:
				from itertools import izip
				pairs = izip(table_headers, values)
			for head, val in pairs:
				if verbosity > 1:
					print("\nExtracting", head, "attribute", sep=' ', end='')
				if attr.lower() == "text":
					try:
						content = connector.join([make_ascii(x).strip() for x in val.itertext()])
					except Exception:
						content = default
					content = content.replace("\n", " ").strip()
				else:
					content = val.get(attr)
					if attr in ["href", "src"]:
						content = urljoin(self.url, content)
				result[head] = content
			result_list.append(result)
		except XPathError:
			raise Exception("Invalid %s selector - %s" % (self.__selector_type__, selector))
		except TypeError:
			raise Exception("Selector expression string to be provided. Got " + selector)

		return result_list
277549
def list_remove_by_index(l, index):
    ''''''
    if index < 0 or index >= len(l):
        raise ValueError('index out of range')
    if index == (len(l) - 1):
        l.pop()
    elif index == 0:
        l = l[1:]
    else:
        l = l[0:index] + l[index+1:]

    return l
412881
def removecontains(args):
    
    p = OptionParser(removecontains.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    contains, gkpStore = args

    s = set()
    fp = open(contains)
    for row in fp:
        if row[0] == '#':
            continue
        iid = int(row.split()[0])
        s.add(iid)

    cmd = "gatekeeper -dumpfragments -lastfragiid {}".format(gkpStore)
    gkpmsg = popen(cmd).read()
    last_iid = int(gkpmsg.strip().split()[-1])

    ndeleted = 0
    editfile = "delete.edit"
    fw = open(editfile, "w")
    for iid in xrange(1, last_iid + 1):
        if iid in s:
            print("frg iid {0} isdeleted 1".format(iid), file=fw)
            ndeleted += 1

    fw.close()
    assert len(s) == ndeleted
    logging.debug("A total of {0} contained reads flagged as deleted."\
                  .format(ndeleted))
    print("Now you can run:", file=sys.stderr)
    print("$ gatekeeper --edit {0} {1}".format(editfile, gkpStore), file=sys.stderr)
415104
def filter_and_copy_table(tab, to_remove):
    
    nsrcs = len(tab)
    mask = np.zeros((nsrcs), '?')
    mask[to_remove] = True
    inv_mask = np.invert(mask)
    out_tab = tab[inv_mask]
    return out_tab
804347
private void extractTablesFromAPage(ArrayList<TextPiece> linesOfAPage,
	    ArrayList<Table> tables, ArrayList<TextPiece> wordsOfAPage, int i,
	    File pdfFile, ArrayList<ArrayList<TextPiece>> linesByPage) {
	if (linesOfAPage.size() > 30) {
	    /*
	     * Checks for table keywords
	     */
	    boolean tableKwdExist = false;
	    int prevTableEndIndex = 0;
	    for (TextPiece t : linesOfAPage) {
		String textWithoutSpace = t.getText().replace(" ", "");
		if (getMatchedTabledKeyword(textWithoutSpace) != null) {
		    tableKwdExist = true;
		    break;
		}
	    }

	    /*
	     * TODO [improve]: keywords are searched twice, no need to do the
	     * first search in the future
	     */
	    if (tableKwdExist == true) {
		for (int k = 0; k < linesOfAPage.size();) {
		    TextPiece line = linesOfAPage.get(k);
		    TextPiece prevLine = (k == 0) ? null : linesOfAPage
			    .get(k - 1);
		    String textWithoutSpace = line.getText().replace(" ", "");
		    String keyword = getMatchedTabledKeyword(textWithoutSpace);

		    if (keyword != null
			    && (textWithoutSpace.startsWith("tableofcontent") == false)
			    && (textWithoutSpace.startsWith("TABLEOFCONTENT") == false)
			    && (textWithoutSpace.startsWith("TableofContent") == false)) {
			char lastCharInKwd = keyword
				.charAt(keyword.length() - 1);
			String oldKwd = line.getText().substring(0,
				line.getText().indexOf(lastCharInKwd) + 1);
			String newLine = "Table"
				+ line.getText()
					.substring(
						line.getText().indexOf(
							lastCharInKwd) + 1,
						line.getText().length());
			// linesOfAPage.get(k).setText(newLine);

			if (prevLine != null && line.getY() == prevLine.getY())
			    k++;
			else {
			    /*
			     * try to extract this table
			     */
			    TableCandidate tc = new TableCandidate();
			    tc.setKeyword(keyword);
			    // tc.setXMLDoc(xml_doc); // to add all table
			    // metadata elements in TableCandidate

			    k = extractOneTable(k, linesOfAPage, keyword, tc,
				    wordsOfAPage, i, prevTableEndIndex,
				    linesByPage);
			    if (tc.isValid()) {
				if (tc.isTopCaption())
				    prevTableEndIndex = tc.getBodyEndLine();
				else
				    prevTableEndIndex = tc.getCaptionEndLine();
				if (Config.DEBUG_MODE)
				    tc.print();
				String metadata_StructureLevel = tc
					.getMetadata_StructureLevel();
				String tableMeta_thisDoc = m_docInfo
					.getTableMetaInThisDoc();
				String tableMetaHighLevel = tc
					.getMetadata_HighLevel();
				String latestMetaData = tableMeta_thisDoc
					+ tableMetaHighLevel
					+ metadata_StructureLevel;
				m_docInfo.setTableMetadata(latestMetaData);

				/*
				 * Generates the extracted table list, then XML
				 * output from each table
				 */
				// System.out.println(Integer.toString(i)+" Adding");
				Table oneTable = new Table();
				oneTable.setValidTable(tc);
				tables.add(oneTable);

				oneTable.setPageNumber(i + 1);
			    }
			}
		    } else
			k++;
		}
	    } else {// TODO: log
	    }
	} else {
	    /*
	     * Impossible to contain tables because of few texts in the page
	     */
	}
    }
75691
def process_row(cls, data, column_map):
        """"""
        row = {}
        for key,value in data.iteritems():
            if not value:
                value = '-'                
            elif isinstance(value, list):
                value = value[1]
            elif isinstance(value, dict):
                if 'type_name' in value:
                    if 'UnixTimeStamp' in value['type_name']:
                        value = datetime.datetime.utcfromtimestamp(value['epoch'])
                        if value == datetime.datetime(1970, 1, 1, 0, 0):
                            value = '-'

            # Assume the value is somehow well formed when we get here
            row[column_map[key]] = value
        return row