convert the idf text to a simple text
767665
def idf2txt(txt):
    """"""
    astr = nocomment(txt)
    objs = astr.split(';')
    objs = [obj.split(',') for obj in objs]
    objs = [[line.strip() for line in obj] for obj in objs]
    objs = [[_tofloat(line) for line in obj] for obj in objs]
    objs = [tuple(obj) for obj in objs]
    objs.sort()

    lst = []
    for obj in objs:
        for field in obj[:-1]:
            lst.append('%s,' % (field, ))
        lst.append('%s;\n' % (obj[-1], ))

    return '\n'.join(lst)
169874
def to_naf(self):
        
        if self.type == 'KAF':
            self.type = 'NAF'
            for node in self.__get_node_terms():
                node.set('id',node.get('tid'))
                del node.attrib['tid']
796836
def fmt_text(text, bg = None, fg = None, attr = None, plain = False):
        
        if not plain:
            if fg is not None:
                text = TEXT_FORMATING['fg'][fg] + text
            if bg is not None:
                text = TEXT_FORMATING['bg'][bg] + text
            if attr is not None:
                text = TEXT_FORMATING['attr'][attr] + text
            if (fg is not None) or (bg is not None) or (attr is not None):
                text += TEXT_FORMATING['rst']
        return text
1274849
def text_to_title(value):
    """"""
    title = None
    if not value:
        return title
    words = value.split(" ")
    keep_words = []
    for word in words:
        if word.endswith(".") or word.endswith(":"):
            keep_words.append(word)
            if len(word) > 1 and "<italic>" not in word and "<i>" not in word:
                break
        else:
            keep_words.append(word)
    if len(keep_words) > 0:
        title = " ".join(keep_words)
        if title.split(" ")[-1] != "spp.":
            title = title.rstrip(" .:")
    return title
813756
def sparse_tensor_value_to_texts(value, alphabet):
    
    """
    return sparse_tuple_to_texts((value.indices, value.values, value.dense_shape), alphabet)
306434
def vectorize(self, token_list):
        '''
        
        '''
        vector_list = [self.__collection.tf_idf(token, self.__collection) for token in token_list]
        return vector_list
953386
def extract_plain_text(self, reduce: bool = False) -> str:
        
        if reduce:
            self.reduce()

        result = ''
        for seg in self:
            if seg.type == 'text':
                result += ' ' + seg.data['text']
        if result:
            result = result[1:]
        return result
415204
def divine_format(text):
    """"""
    try:
        nbformat.reads(text, as_version=4)
        return 'ipynb'
    except nbformat.reader.NotJSONError:
        pass

    lines = text.splitlines()
    for comment in ['', '#'] + _COMMENT_CHARS:
        metadata, _, _, _ = header_to_metadata_and_cell(lines, comment)
        ext = metadata.get('jupytext', {}).get('text_representation', {}).get('extension')
        if ext:
            return ext[1:] + ':' + guess_format(text, ext)[0]

    # No metadata, but ``` on at least one line => markdown
    for line in lines:
        if line == '```':
            return 'md'

    return 'py:' + guess_format(text, '.py')[0]
538327
def tokenize_documents(docs):
    """"""
    sep = '\n\n'
    texts = []
    for doc in docs:
        text = '\n\n'.join(['\n'.join(para[SENTENCES]) for para in doc[PARAGRAPHS]])
        doc[TEXT] = text
        del doc[PARAGRAPHS]
        texts.append(Text(doc))
    return texts
894820
def getidfobjectlist(idf):
    """"""
    idfobjects = idf.idfobjects  
    # idfobjlst = [idfobjects[key] for key in idfobjects if idfobjects[key]]
    idfobjlst = [idfobjects[key] for key in idf.model.dtls if idfobjects[key]]
        # `for key in idf.model.dtls` maintains the order
        # `for key in idfobjects` does not have order
    idfobjlst = itertools.chain.from_iterable(idfobjlst)
    idfobjlst = list(idfobjlst)
    return idfobjlst
934324
def text(self):
        
        text = []
        rows = self.rowCount()
        cols = self.columnCount()

        # handle empty table case
        if rows == 2 and cols == 2:
            item = self.item(0, 0)
            if item is None:
                return ''

        for r in range(rows - 1):
            for c in range(cols - 1):
                item = self.item(r, c)
                if item is not None:
                    value = item.text()
                else:
                    value = '0'

                if not value.strip():
                    value = '0'

                text.append(' ')
                text.append(value)
            text.append(ROW_SEPARATOR)

        return ''.join(text[:-1])
154217
def updateidf(idf, dct):
    """"""
    for key in list(dct.keys()):
        if key.startswith('idf.'):
            idftag, objkey, objname, field = key2elements(key)
            if objname == '':
                try:
                    idfobj = idf.idfobjects[objkey.upper()][0]
                except IndexError as e:
                    idfobj = idf.newidfobject(objkey.upper())
            else:
                idfobj = idf.getobject(objkey.upper(), objname)
                if idfobj == None:
                    idfobj = idf.newidfobject(objkey.upper(), Name=objname)
            idfobj[field] = dct[key]
135232
def as_text(self, max_rows=0, sep=" | "):
        """"""
        if not max_rows or max_rows > self.num_rows:
            max_rows = self.num_rows
        omitted = max(0, self.num_rows - max_rows)
        labels = self._columns.keys()
        fmts = self._get_column_formatters(max_rows, False)
        rows = [[fmt(label, label=True) for fmt, label in zip(fmts, labels)]]
        for row in itertools.islice(self.rows, max_rows):
            rows.append([f(v, label=False) for v, f in zip(row, fmts)])
        lines = [sep.join(row) for row in rows]
        if omitted:
            lines.append('... ({} rows omitted)'.format(omitted))
        return '\n'.join([line.rstrip() for line in lines])
53126
def getrefnames(idf, objname):
    """"""
    iddinfo = idf.idd_info
    dtls = idf.model.dtls
    index = dtls.index(objname)
    fieldidds = iddinfo[index]
    for fieldidd in fieldidds:
        if 'field' in fieldidd:
            if fieldidd['field'][0].endswith('Name'):
                if 'reference' in fieldidd:
                    return fieldidd['reference']
                else:
                    return []
1183088
def iddtxt2grouplist(txt):
    
    def makenone(astr):
        if astr == 'None':
            return None
        else:
            return astr

    txt = nocomment(txt, '!')
    txt = txt.replace("\\group", "!-group") # retains group in next line
    txt = nocomment(txt, '\\') # remove all other idd info
    lines = txt.splitlines()
    lines = [line.strip() for line in lines] # cleanup
    lines = [line for line in lines if line != ''] # cleanup
    txt = '\n'.join(lines)
    gsplits = txt.split('!') # split into groups, since we have !-group
    gsplits = [gsplit.splitlines() for gsplit in gsplits] # split group

    gsplits[0].insert(0, u'-group None')
        # Put None for the first group that does nothave a group name
    
    glist = []
    for gsplit in gsplits:
        glist.append((gsplit[0], gsplit[1:]))
        # makes dict {groupname:[k1, k2], groupname2:[k3, k4]}

    glist = [(k, '\n'.join(v)) for k, v in glist]# joins lines back
    glist = [(k, v.split(';')) for k, v in glist] # splits into idfobjects
    glist = [(k, [i.strip() for i in v]) for k, v in glist] # cleanup
    glist = [(k, [i.splitlines() for i in v]) for k, v in glist]
        # splits idfobjects into lines
    glist = [(k, [i for i in v if len(i) > 0]) for k, v in glist]
        # cleanup - removes blank lines
    glist = [(k, [i[0] for i in v]) for k, v in glist] # use first line
    fglist = []
    for gnamelist in glist:
        gname = gnamelist[0]
        thelist = gnamelist[-1]
        for item in thelist:
            fglist.append((gname, item))
    glist = [(gname[len("-group "):], obj) for gname, obj in fglist] # remove "-group "
    glist = [(makenone(gname), obj) for gname, obj in glist] # make str None into None
    glist = [(gname, obj.split(',')[0]) for gname, obj in glist] # remove comma
    return glist
1158554
def _convertUnitAsWords(self, unitText):
        
        word_list, a, b = re.split(r"[,\s-]+", unitText), 0, 0
        for word in word_list:
            x = self.ptc.small.get(word)
            if x is not None:
                a += x
            elif word == "hundred":
                a *= 100
            else:
                x = self.ptc.magnitude.get(word)
                if x is not None:
                    b += a * x
                    a = 0
                elif word in self.ptc.ignore:
                    pass
                else:
                    raise Exception("Unknown number: " + word)
        return a + b
998924
def normalize(self, text, **kwargs):
        """"""
        identifiers = []
        for ident in super(IdentifierType, self).normalize(text, **kwargs):
            identifiers.append(normalize(ident))
        return identifiers