Convert an html string to data table

    Parameters
    ----------
    html_string : str
    row_count : int
    column_count : int

    Returns
    -------
    data_table : list of lists of str
163214
def extract_table(html_string, row_count, column_count):
    
    try:
        from bs4 import BeautifulSoup
        from bs4.element import Tag
    except ImportError:
        print("ERROR: You must have BeautifulSoup to use html2data")
        return

    #html_string = convertRichText(html_string)

    data_table = []
    for row in range(0, row_count):
        data_table.append([])
        for column in range(0, column_count):
            data_table[-1].append(None)

    soup = BeautifulSoup(html_string, 'html.parser')

    table = soup.find('table')

    if not table:
        return ''

    trs = table.findAll('tr')
    if len(trs) == 0:
        return [['']]

    for tr in range(len(trs)):
        ths = trs[tr].findAll('th')
        if len(ths) == 0:
            tds = trs[tr].findAll('td')
        else:
            tds = ths

        if len(tds) == 0:
            tds = []
            for i in range(0, column_count):
                tds.append(Tag("", name=""))

        for i in range(len(tds)):
            td = tds[i]
            row, column = find_unassigned_table_cell(data_table)

            r_span_count = 1
            c_span_count = 1

            if td.has_attr('rowspan'):
                r_span_count = int(td['rowspan'])
            if td.has_attr('colspan'):
                c_span_count = int(td['colspan'])

            for row_prime in range(row, row + r_span_count):
                for column_prime in range(column, column + c_span_count):
                    if row_prime == row and column_prime == column:

                        items = []
                        for item in td.contents:
                            items.append(str(item))
                        string = ''.join(items).strip()

                        text = restructify(string).rstrip()

                        data_table[row_prime][column_prime] = text
                    else:
                        data_table[row_prime][column_prime] = ""

            if i + 1 < column_count and i == len(tds) - 1:
                for x in range(len(tds), column_count):
                    if data_table[row][x] is None:
                        data_table[row][x] = ""
    for row in range(len(data_table)):
        for column in range(len(data_table[row])):
            if not data_table[row][column]:
                data_table[row][column] = ""

    return data_table
877706
def format_table(columns, rows):
    
    rows = [tuple(str(i) for i in r) for r in rows]
    columns = tuple(str(i).upper() for i in columns)
    if rows:
        widths = tuple(max(max(map(len, x)), len(c))
                       for x, c in zip(zip(*rows), columns))
    else:
        widths = tuple(map(len, columns))
    row_template = ('    '.join('%%-%ds' for _ in columns)) % widths
    header = (row_template % tuple(columns)).strip()
    if rows:
        data = '\n'.join((row_template % r).strip() for r in rows)
        return '\n'.join([header, data])
    else:
        return header
1014398
def _ExtractDataFromShowHtml(self, html):
    
    htmlLines = html.splitlines()
    for count, line in enumerate(htmlLines):
      if line.strip() == r'<pre>':
        startLine = count+1
      if line.strip() == r'</pre>':
        endLine = count

    try:
      dataList = htmlLines[startLine:endLine]
      dataString = '\n'.join(dataList)
      return dataString.strip()
    except:
      raise Exception("Show content not found - check EPGuides html formatting")
1279893
def sheetToHTML(sheet):
    

    assert "SHEET" in str(type(sheet))
    #data,names=None,units=None,bookName=None,sheetName=None,xCol=None
    #sheet=OR.SHEET()
    data=sheet.data
    names=sheet.colDesc
    units=sheet.colUnits
    bookName=sheet.bookName
    sheetName=sheet.sheetName

    def htmlListToTR(l,trClass=None,tdClass=None,td1Class=None):
        """
        turns a list into a <tr><td>something</td></tr>
        call this when generating HTML tables dynamically.
        """
        html="<tr>"
        for item in l:
            html+="<td>%s</td>"%item
        html+="</tr>"
        if trClass:
            html=html.replace("<tr>",'<tr class="%s">'%trClass)
        if td1Class:
            html=html.replace("<td>",'<td class="%s">'%td1Class,1)
        if tdClass:
            html=html.replace("<td>",'<td class="%s">'%tdClass)
        return html

    htmlFname = os.path.expanduser("~")+"/WKS-%s.%s.html"%(bookName,sheetName)
    html="""<body>
    <style>
    body {
          background-color: #ababab;
          padding:20px;
          }
    table {
           font-size:12px;
           border-spacing: 0;
           border-collapse: collapse;
           }
    .name {background-color:#fafac8;text-align:center;}
    .units {background-color:#fafac8;text-align:center;}
    .data0 {background-color:#FFFFFF;font-family: monospace;text-align:center;}
    .data1 {background-color:#FAFAFA;font-family: monospace;text-align:center;}
    .labelRow {background-color:#e0dfe4; text-align:right;border:1px solid #000000;}
    .labelCol {background-color:#e0dfe4; text-align:center;border:1px solid #000000;  padding-left: 20px; padding-right: 20px;}
    td {
        border:1px solid #c0c0c0; padding:5px;
        font-family: Arial, Helvetica, sans-serif;
        }
    </style>
    <html>"""
    html+="<h1>FauxRigin</h1>"
    if bookName or sheetName:
        html+='<code><b>%s / %s</b></code><br><br>'%(bookName,sheetName)
    html+="<table>"
    colNames=['']
    for i in range(len(units)):
        shortName=chr(i%26+ord('A'))
        if i>=26:
            shortName=chr(int(i/26-1)+ord('A'))+shortName
        label="%s(%s)"%(shortName,"X" if sheet.colTypes[i]==3 else "Y")
        colNames.append(label)
    html+=htmlListToTR(colNames,'labelCol','labelCol')
    html+=htmlListToTR(['Long Name']+list(names),'name',td1Class='labelRow')
    html+=htmlListToTR(['Units']+list(units),'units',td1Class='labelRow')
    cutOff=False
    for y in range(len(data)):
        html+=htmlListToTR([y+1]+list(data[y]),trClass='data%d'%(y%2),td1Class='labelRow')
        if y>=200:
            cutOff=True
            break
    html+="</table>"
    html=html.replace(">nan<",">--<")
    html=html.replace(">None<","><")
    if cutOff:
        html+="<h3>... showing only %d of %d rows ...</h3>"%(y,len(data))
    html+="</body></html>"
    with open(htmlFname,'w') as f:
        f.write(html)
    import webbrowser
    webbrowser.open(htmlFname)
    return
793642
def _to_str_columns(self):
        
        frame = self.tr_frame

        # may include levels names also

        str_index = self._get_formatted_index(frame)
        str_columns = self._get_formatted_column_labels(frame)

        if self.header:
            stringified = []
            for i, c in enumerate(frame.columns):
                cheader = str_columns[i]
                max_colwidth = max(self.col_space or 0,
                                   *(self.adj.len(x) for x in cheader))
                fmt_values = self._format_col(i)
                fmt_values = _make_fixed_width(fmt_values, self.justify,
                                               minimum=max_colwidth,
                                               adj=self.adj)

                max_len = max(max([self.adj.len(x) for x in fmt_values]),
                              max_colwidth)
                cheader = self.adj.justify(cheader, max_len, mode=self.justify)
                stringified.append(cheader + fmt_values)
        else:
            stringified = []
            for i, c in enumerate(frame):
                fmt_values = self._format_col(i)
                fmt_values = _make_fixed_width(fmt_values, self.justify,
                                               minimum=(self.col_space or 0),
                                               adj=self.adj)

                stringified.append(fmt_values)

        strcols = stringified
        if self.index:
            strcols.insert(0, str_index)

        # Add ... to signal truncated
        truncate_h = self.truncate_h
        truncate_v = self.truncate_v

        if truncate_h:
            col_num = self.tr_col_num
            col_width = self.adj.len(strcols[self.tr_size_col][0])  # infer from column header
            strcols.insert(self.tr_col_num + 1, ['...'.center(col_width)] * (len(str_index)))
        if truncate_v:
            n_header_rows = len(str_index) - len(frame)
            row_num = self.tr_row_num
            for ix, col in enumerate(strcols):
                cwidth = self.adj.len(strcols[ix][row_num])  # infer from above row
                is_dot_col = False
                if truncate_h:
                    is_dot_col = ix == col_num + 1
                if cwidth > 3 or is_dot_col:
                    my_str = '...'
                else:
                    my_str = '..'

                if ix == 0:
                    dot_mode = 'left'
                elif is_dot_col:
                    cwidth = self.adj.len(strcols[self.tr_size_col][0])
                    dot_mode = 'center'
                else:
                    dot_mode = 'right'
                dot_str = self.adj.justify([my_str], cwidth, mode=dot_mode)[0]
                strcols[ix].insert(row_num + n_header_rows, dot_str)
        return strcols
210261
def pprint_cell(self, row, col):
        
        if col > 2:
            raise Exception("Only two columns available in a ItemTable.")
        elif row >= self.rows:
            raise Exception("Maximum row index is %d" % self.rows-1)
        elif col == 0:
            return self.dimensions('value')[row].pprint_label
        else:
            dim = self.get_dimension(row)
            heading = self.vdims[row]
            return dim.pprint_value(self.data.get(heading.name, np.NaN))
342763
def _string_in_table(self, candidate: str) -> List[str]:
        
        candidate_column_names: List[str] = []
        # First check if the entire candidate occurs as a cell.
        if candidate in self._string_column_mapping:
            candidate_column_names = self._string_column_mapping[candidate]
        # If not, check if it is a substring pf any cell value.
        if not candidate_column_names:
            for cell_value, column_names in self._string_column_mapping.items():
                if candidate in cell_value:
                    candidate_column_names.extend(column_names)
        candidate_column_names = list(set(candidate_column_names))
        return candidate_column_names
309538
def _repr_html_(self):
        
        if self._info_repr():
            buf = StringIO("")
            self.info(buf=buf)
            # need to escape the <class>, should be the first line.
            val = buf.getvalue().replace('<', r'&lt;', 1)
            val = val.replace('>', r'&gt;', 1)
            return '<pre>' + val + '</pre>'

        if get_option("display.notebook_repr_html"):
            max_rows = get_option("display.max_rows")
            max_cols = get_option("display.max_columns")
            show_dimensions = get_option("display.show_dimensions")

            return self.to_html(max_rows=max_rows, max_cols=max_cols,
                                show_dimensions=show_dimensions, notebook=True)
        else:
            return None
646645
def to_json_payload(self, data_rows):
        
        num_rows = len(data_rows)
        num_page = self.num_page
        num_rows_page = self.num_rows_page

        pages = num_rows / num_rows_page
        pages += divmod(num_rows, num_rows_page)[1] and 1 or 0
        start = (num_page - 1) * num_rows_page
        end = num_page * num_rows_page
        payload = {"page": num_page,
                   "total": pages,
                   "records": num_rows,
                   "rows": data_rows[start:end]}
        return json.dumps(payload)
1208482
def _get_table_rows(parent_table, table_name, row_name):
        
        if parent_table is None:
            return []
        _table = parent_table.get(table_name)
        _table_rows = []
        if isinstance(_table, list):
            _table_rows = [_table_row.get(row_name) for _table_row in _table]
        elif isinstance(_table, dict):
            _table_rows = _table.get(row_name)
        if not isinstance(_table_rows, list):
            _table_rows = [_table_rows]
        return _table_rows
305628
def get_data_and_column_widths(self, data_kwargs, width_kwargs):
        
        data_kwargs = data_kwargs.copy()
        safe_str = data_kwargs.pop('safe_str', self._safe_str)
        list_of_list = [[safe_str(col, _is_header=True, **data_kwargs)
                         for col in self.columns]]
        list_of_list+= [[safe_str(row[col], **data_kwargs)
                         for col in self.columns] for row in self]
        column_widths = self._get_column_widths(list_of_list, **width_kwargs)
        return list_of_list, column_widths
216722
def from_df(data_frame):
        
        labels = data_frame.keys().tolist()
        data = data_frame.values.tolist()
        return SqlTable(labels, data, "{:.3f}", "\n")
364612
function tableRow () {
  var args = arrayify(arguments)
  var rows = args.shift()
  if (!rows) return
  var options = args.pop()
  var cols = args
  var output = ''

  if (options.data) {
    var data = handlebars.createFrame(options.data)
    cols.forEach(function (col, index) {
      var colNumber = index + 1
      data['col' + colNumber] = containsData(rows, col)
    })
  }
  rows.forEach(function (row) {
    output += options.fn(row, { data: data })
  })
  return output
}
752150
def table(args):
    
    import csv

    p = OptionParser(table.__doc__)
    p.set_sep(sep=",")
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    htmlfile, = args
    page = open(htmlfile).read()
    soup = BeautifulSoup(page)

    for i, tabl in enumerate(soup.findAll('table')):
        nrows = 0
        csvfile = htmlfile.rsplit(".", 1)[0] + ".{0}.csv".format(i)
        writer = csv.writer(open(csvfile, "w"), delimiter=opts.sep)
        rows = tabl.findAll('tr')
        for tr in rows:
            cols = tr.findAll('td')
            if not cols:
                cols = tr.findAll('th')

            row = []
            for td in cols:
                try:
                    cell = "".join(td.find(text=True))
                    cell = unescape(cell)
                except TypeError:
                    cell = ""
                row.append(cell)
            writer.writerow(row)
            nrows += 1
        logging.debug("Table with {0} rows written to `{1}`.".format(nrows, csvfile))
36357
def _stringify_row(self, row_index):
        '''
        
        '''
        table_row = self.table[row_index]
        prior_cell = None
        for column_index in range(self.start[1], self.end[1]):
            cell, changed = self._check_interpret_cell(table_row[column_index], prior_cell, row_index, column_index)
            if changed:
                table_row[column_index] = cell
            prior_cell = cell
74806
def _read_as_table(self):
        
        rows = list()

        for row in self._rows:
            rows.append([row[i].get() for i in range(self.num_of_columns)])

        return rows
644006
async def display_columns_and_rows(
        self,
        database,
        table,
        description,
        rows,
        link_column=False,
        truncate_cells=0,
    ):
        ""
        table_metadata = self.ds.table_metadata(database, table)
        sortable_columns = await self.sortable_columns_for_table(database, table, True)
        columns = [
            {"name": r[0], "sortable": r[0] in sortable_columns} for r in description
        ]
        pks = await self.ds.execute_against_connection_in_thread(
            database, lambda conn: detect_primary_keys(conn, table)
        )
        column_to_foreign_key_table = {
            fk["column"]: fk["other_table"]
            for fk in await self.ds.foreign_keys_for_table(database, table)
        }

        cell_rows = []
        for row in rows:
            cells = []
            # Unless we are a view, the first column is a link - either to the rowid
            # or to the simple or compound primary key
            if link_column:
                cells.append(
                    {
                        "column": pks[0] if len(pks) == 1 else "Link",
                        "value": jinja2.Markup(
                            '<a href="/{database}/{table}/{flat_pks_quoted}">{flat_pks}</a>'.format(
                                database=database,
                                table=urllib.parse.quote_plus(table),
                                flat_pks=str(
                                    jinja2.escape(
                                        path_from_row_pks(row, pks, not pks, False)
                                    )
                                ),
                                flat_pks_quoted=path_from_row_pks(row, pks, not pks),
                            )
                        ),
                    }
                )

            for value, column_dict in zip(row, columns):
                column = column_dict["name"]
                if link_column and len(pks) == 1 and column == pks[0]:
                    # If there's a simple primary key, don't repeat the value as it's
                    # already shown in the link column.
                    continue

                # First let the plugins have a go
                # pylint: disable=no-member
                plugin_display_value = pm.hook.render_cell(
                    value=value,
                    column=column,
                    table=table,
                    database=database,
                    datasette=self.ds,
                )
                if plugin_display_value is not None:
                    display_value = plugin_display_value
                elif isinstance(value, dict):
                    # It's an expanded foreign key - display link to other row
                    label = value["label"]
                    value = value["value"]
                    # The table we link to depends on the column
                    other_table = column_to_foreign_key_table[column]
                    link_template = (
                        LINK_WITH_LABEL if (label != value) else LINK_WITH_VALUE
                    )
                    display_value = jinja2.Markup(link_template.format(
                        database=database,
                        table=urllib.parse.quote_plus(other_table),
                        link_id=urllib.parse.quote_plus(str(value)),
                        id=str(jinja2.escape(value)),
                        label=str(jinja2.escape(label)),
                    ))
                elif value in ("", None):
                    display_value = jinja2.Markup("&nbsp;")
                elif is_url(str(value).strip()):
                    display_value = jinja2.Markup(
                        '<a href="{url}">{url}</a>'.format(
                            url=jinja2.escape(value.strip())
                        )
                    )
                elif column in table_metadata.get("units", {}) and value != "":
                    # Interpret units using pint
                    value = value * ureg(table_metadata["units"][column])
                    # Pint uses floating point which sometimes introduces errors in the compact
                    # representation, which we have to round off to avoid ugliness. In the vast
                    # majority of cases this rounding will be inconsequential. I hope.
                    value = round(value.to_compact(), 6)
                    display_value = jinja2.Markup(
                        "{:~P}".format(value).replace(" ", "&nbsp;")
                    )
                else:
                    display_value = str(value)
                    if truncate_cells and len(display_value) > truncate_cells:
                        display_value = display_value[:truncate_cells] + u"\u2026"

                cells.append({"column": column, "value": display_value})
            cell_rows.append(cells)

        if link_column:
            # Add the link column header.
            # If it's a simple primary key, we have to remove and re-add that column name at
            # the beginning of the header row.
            if len(pks) == 1:
                columns = [col for col in columns if col["name"] != pks[0]]

            columns = [
                {"name": pks[0] if len(pks) == 1 else "Link", "sortable": len(pks) == 1}
            ] + columns
        return columns, cell_rows