{
    "file_diffs": [
        {
            "old_file_content": "import contextlib\nimport csv\nimport locale\nimport pickle\nimport re\nimport subprocess\nimport sys\nimport warnings\n\nfrom ast import literal_eval\nfrom collections import OrderedDict, Counter, defaultdict\nfrom functools import lru_cache\nfrom importlib import import_module\nfrom itertools import chain, repeat\nfrom math import isnan\n\nfrom os import path, remove\nfrom tempfile import NamedTemporaryFile\nfrom urllib.parse import urlparse, urlsplit, urlunsplit, unquote as urlunquote\nfrom urllib.request import urlopen, Request\nfrom fnmatch import fnmatch\nfrom glob import glob\n\nimport numpy as np\nfrom chardet.universaldetector import UniversalDetector\n\nfrom Orange.data import (\n    _io, is_discrete_values, MISSING_VALUES, Table, Domain, Variable,\n    DiscreteVariable, StringVariable, ContinuousVariable, TimeVariable,\n)\nfrom Orange.util import Registry, flatten, namegen\n\n\n# Support values longer than 128K (i.e. text contents features)\ncsv.field_size_limit(100*1024*1024)\n\n__all__ = [\"Flags\", \"FileFormat\"]\n\n\n_IDENTITY = lambda i: i\n\n\nclass Compression:\n    \"\"\"Supported compression extensions\"\"\"\n    GZIP = '.gz'\n    BZIP2 = '.bz2'\n    XZ = '.xz'\n    all = (GZIP, BZIP2, XZ)\n\n\ndef open_compressed(filename, *args, _open=open, **kwargs):\n    \"\"\"Return seamlessly decompressed open file handle for `filename`\"\"\"\n    if isinstance(filename, str):\n        if filename.endswith(Compression.GZIP):\n            from gzip import open as _open\n        elif filename.endswith(Compression.BZIP2):\n            from bz2 import open as _open\n        elif filename.endswith(Compression.XZ):\n            from lzma import open as _open\n        return _open(filename, *args, **kwargs)\n    # Else already a file, just pass it through\n    return filename\n\n\ndef detect_encoding(filename):\n    \"\"\"\n    Detect encoding of `filename`, which can be a ``str`` filename, a\n    ``file``-like object, or ``bytes``.\n    \"\"\"\n    # Try with Unix file utility first because it's faster (~10ms vs 100ms)\n    if isinstance(filename, str) and not filename.endswith(Compression.all):\n        try:\n            with subprocess.Popen(('file', '--brief', '--mime-encoding', filename),\n                                  stdout=subprocess.PIPE) as process:\n                process.wait()\n                if process.returncode == 0:\n                    encoding = process.stdout.read().strip()\n                    # file only supports these encodings; for others it says\n                    # unknown-8bit or binary. So we give chardet a chance to do\n                    # better\n                    if encoding in (b'utf-8', b'us-ascii', b'iso-8859-1',\n                                    b'utf-7', b'utf-16le', b'utf-16be', b'ebcdic'):\n                        return encoding.decode('us-ascii')\n        except OSError:\n            pass  # windoze\n\n    # file not available or unable to guess the encoding, have chardet do it\n    detector = UniversalDetector()\n    # We examine only first N 4kB blocks of file because chardet is really slow\n    MAX_BYTES = 4*1024*12\n\n    def _from_file(f):\n        detector.feed(f.read(MAX_BYTES))\n        detector.close()\n        return (detector.result.get('encoding')\n                if detector.result.get('confidence', 0) >= .85 else\n                'utf-8')\n\n    if isinstance(filename, str):\n        with open_compressed(filename, 'rb') as f:\n            return _from_file(f)\n    elif isinstance(filename, bytes):\n        detector.feed(filename[:MAX_BYTES])\n        detector.close()\n        return detector.result.get('encoding')\n    elif hasattr(filename, 'encoding'):\n        return filename.encoding\n    else:  # assume file-like object that you can iter through\n        return _from_file(filename)\n\n\n__isnastr = np.frompyfunc(\n    {v for v in MISSING_VALUES if isinstance(v, str)}.__contains__, 1, 1)\n\n# wrapper for __isnastr with proper default out dtype\ndef isnastr(arr, out=None):\n    \"\"\"\n    Given an (object) array of string values, return a boolean mask array\n    that is True where the `arr` contains one of the string constants\n    considered as N/A.\n\n    Parameters\n    ----------\n    arr : np.ndarray\n        Input array of strings.\n    out : Optional[np.ndarray]\n        Optional output array of the same shape as arr\n\n    Returns\n    -------\n    mask : np.ndarray\n    \"\"\"\n    arr = np.asarray(arr)\n    if out is None and arr.shape != ():\n        out = np.empty_like(arr, dtype=bool)\n    return __isnastr(arr, out=out)\n\n\ndef guess_data_type(orig_values, namask=None):\n    \"\"\"\n    Use heuristics to guess data type.\n    \"\"\"\n    valuemap, values = None, orig_values\n    is_discrete = is_discrete_values(orig_values)\n    if is_discrete:\n        valuemap = sorted(is_discrete)\n        coltype = DiscreteVariable\n    else:\n        # try to parse as float\n        orig_values = np.asarray(orig_values)\n        if namask is None:\n            namask = isnastr(orig_values)\n        values = np.empty_like(orig_values, dtype=float)\n        values[namask] = np.nan\n        try:\n            np.copyto(values, orig_values, where=~namask, casting=\"unsafe\")\n        except ValueError:\n            tvar = TimeVariable('_')\n            try:\n                values[~namask] = [tvar.parse(i) for i in orig_values[~namask]]\n            except ValueError:\n                coltype = StringVariable\n                # return original_values\n                values = orig_values\n            else:\n                coltype = TimeVariable\n        else:\n            coltype = ContinuousVariable\n    return valuemap, values, coltype\n\n\ndef sanitize_variable(valuemap, values, orig_values, coltype, coltype_kwargs,\n                      domain_vars=None, existing_var=None, new_var_name=None, data=None, name=None):\n    assert issubclass(coltype, Variable)\n\n    if name is None or existing_var is not None or new_var_name is not None:\n        name = existing_var.strip() if existing_var else new_var_name\n        raise DeprecationWarning(\"Arguments 'existing_var' and 'new_var_name' are \"\\\n                                 \"deprecated since 3.16; use 'name' instead\")\n\n    if domain_vars is not None:\n        raise DeprecationWarning(\"Argument 'domain_vars' is deprecated since 3.16\")\n\n    if data is not None:\n        raise DeprecationWarning(\"Argument 'data' is deprecated since 3.16\")\n\n    def get_number_of_decimals(values):\n        len_ = len\n        ndecimals = max((len_(value) - value.find(\".\")\n                         for value in values if \".\" in value),\n                        default=1)\n        return ndecimals - 1\n\n    if issubclass(coltype, DiscreteVariable) and valuemap is not None:\n        coltype_kwargs.update(values=valuemap)\n\n    var = coltype.make(name, **coltype_kwargs)\n\n    if isinstance(var, DiscreteVariable):\n        # Map discrete data to 'ints' (or at least what passes as int around\n        # here)\n        mapping = defaultdict(\n            lambda: np.nan,\n            {val: i for i, val in enumerate(var.values)},\n        )\n        mapping[\"\"] = np.nan\n        mapvalues_ = np.frompyfunc(mapping.__getitem__, 1, 1)\n\n        def mapvalues(arr):\n            arr = np.asarray(arr, dtype=object)\n            return mapvalues_(arr, out=np.empty_like(arr, dtype=float))\n        values = mapvalues(orig_values)\n\n    if coltype is StringVariable:\n        values = orig_values\n\n    # ContinuousVariable.number_of_decimals is supposed to be handled by\n    # ContinuousVariable.to_val. In the interest of speed, the reader bypasses\n    # it, so we set the number of decimals here.\n    # The number of decimals is increased if not set manually (in which case\n    # var.adjust_decimals would be 0).\n    if isinstance(var, ContinuousVariable) and var.adjust_decimals:\n        ndecimals = get_number_of_decimals(orig_values)\n        if var.adjust_decimals == 2 or ndecimals > var.number_of_decimals:\n            var.number_of_decimals = ndecimals\n            var.adjust_decimals = 1\n\n    if isinstance(var, TimeVariable) or coltype is TimeVariable:\n        # Re-parse the values because only now after coltype.make call\n        # above, variable var is the correct one\n        _var = var if isinstance(var, TimeVariable) else TimeVariable('_')\n        values = [_var.parse(i) for i in orig_values]\n\n    return values, var\n\n\nclass Flags:\n    \"\"\"Parser for column flags (i.e. third header row)\"\"\"\n    DELIMITER = ' '\n    _RE_SPLIT = re.compile(r'(?<!\\\\)' + DELIMITER).split\n    _RE_ATTR_UNQUOTED_STR = re.compile(r'^[a-zA-Z_]').match\n    ALL = OrderedDict((\n        ('class', 'c'),\n        ('ignore', 'i'),\n        ('meta', 'm'),\n        ('weight', 'w'),\n        ('.+?=.*?', ''),  # general key=value attributes\n    ))\n    _RE_ALL = re.compile(r'^({})$'.format('|'.join(filter(None, flatten(ALL.items())))))\n\n    def __init__(self, flags):\n        for v in filter(None, self.ALL.values()):\n            setattr(self, v, False)\n        self.attributes = {}\n        for flag in flags or []:\n            flag = flag.strip()\n            if self._RE_ALL.match(flag):\n                if '=' in flag:\n                    k, v = flag.split('=', 1)\n                    if not Flags._RE_ATTR_UNQUOTED_STR(v):\n                        try:\n                            v = literal_eval(v)\n                        except SyntaxError:\n                            # If parsing failed, treat value as string\n                            pass\n                    self.attributes[k] = v\n                else:\n                    setattr(self, flag, True)\n                    setattr(self, self.ALL.get(flag, ''), True)\n            elif flag:\n                warnings.warn('Invalid attribute flag \\'{}\\''.format(flag))\n\n    @staticmethod\n    def join(iterable, *args):\n        return Flags.DELIMITER.join(i.strip().replace(Flags.DELIMITER, '\\\\' + Flags.DELIMITER)\n                                    for i in chain(iterable, args)).lstrip()\n\n    @staticmethod\n    def split(s):\n        return [i.replace('\\\\' + Flags.DELIMITER, Flags.DELIMITER)\n                for i in Flags._RE_SPLIT(s)]\n\n\n# Matches discrete specification where all the values are listed, space-separated\n_RE_DISCRETE_LIST = re.compile(r'^\\s*[^\\s]+(\\s[^\\s]+)+\\s*$')\n_RE_TYPES = re.compile(r'^\\s*({}|{}|)\\s*$'.format(\n    _RE_DISCRETE_LIST.pattern,\n    '|'.join(flatten(getattr(vartype, 'TYPE_HEADERS') for vartype in Variable.registry.values()))\n    ))\n_RE_FLAGS = re.compile(r'^\\s*( |{}|)*\\s*$'.format(\n    '|'.join(flatten(filter(None, i) for i in Flags.ALL.items()))\n    ))\n\n\nclass FileFormatMeta(Registry):\n\n    def __new__(cls, name, bases, attrs):\n        newcls = super().__new__(cls, name, bases, attrs)\n\n        # Optionally add compressed versions of extensions as supported\n        if getattr(newcls, 'SUPPORT_COMPRESSED', False):\n            new_extensions = list(getattr(newcls, 'EXTENSIONS', ()))\n            for compression in Compression.all:\n                for ext in newcls.EXTENSIONS:\n                    new_extensions.append(ext + compression)\n                if sys.platform in ('darwin', 'win32'):\n                    # OSX file dialog doesn't support filtering on double\n                    # extensions (e.g. .csv.gz)\n                    # https://bugreports.qt.io/browse/QTBUG-38303\n                    # This is just here for OWFile that gets QFileDialog\n                    # filters from FileFormat.readers.keys()\n                    # EDIT: Windows exhibit similar problems:\n                    # while .tab.gz works, .tab.xz and .tab.bz2 do not!\n                    new_extensions.append(compression)\n            newcls.EXTENSIONS = tuple(new_extensions)\n\n        return newcls\n\n    @property\n    def formats(cls):\n        return cls.registry.values()\n\n    @lru_cache(5)\n    def _ext_to_attr_if_attr2(cls, attr, attr2):\n        \"\"\"\n        Return ``{ext: `attr`, ...}`` dict if ``cls`` has `attr2`.\n        If `attr` is '', return ``{ext: cls, ...}`` instead.\n\n        If there are multiple formats for an extension, return a format\n        with the lowest priority.\n        \"\"\"\n        formats = OrderedDict()\n        for format in sorted(cls.registry.values(), key=lambda x: x.PRIORITY):\n            if not hasattr(format, attr2):\n                continue\n            for ext in getattr(format, 'EXTENSIONS', []):\n                # Only adds if not yet registered\n                formats.setdefault(ext, getattr(format, attr, format))\n        return formats\n\n    @property\n    def names(cls):\n        return cls._ext_to_attr_if_attr2('DESCRIPTION', '__class__')\n\n    @property\n    def writers(cls):\n        return cls._ext_to_attr_if_attr2('', 'write_file')\n\n    @property\n    def readers(cls):\n        return cls._ext_to_attr_if_attr2('', 'read')\n\n    @property\n    def img_writers(cls):\n        return cls._ext_to_attr_if_attr2('', 'write_image')\n\n    @property\n    def graph_writers(cls):\n        return cls._ext_to_attr_if_attr2('', 'write_graph')\n\n\nclass FileFormat(metaclass=FileFormatMeta):\n    \"\"\"\n    Subclasses set the following attributes and override the following methods:\n\n        EXTENSIONS = ('.ext1', '.ext2', ...)\n        DESCRIPTION = 'human-readable file format description'\n        SUPPORT_COMPRESSED = False\n        SUPPORT_SPARSE_DATA = False\n\n        def read(self):\n            ...  # load headers, data, ...\n            return self.data_table(data, headers)\n\n        @classmethod\n        def write_file(cls, filename, data):\n            ...\n            self.write_headers(writer.write, data)\n            writer.writerows(data)\n\n    Wrapper FileFormat.data_table() returns Orange.data.Table from `data`\n    iterable (list (rows) of lists of values (cols)).\n    \"\"\"\n\n    # Priority when multiple formats support the same extension. Also\n    # the sort order in file open/save combo boxes. Lower is better.\n    PRIORITY = 10000\n\n    def __init__(self, filename):\n        \"\"\"\n        Parameters\n        ----------\n        filename : str\n            name of the file to open\n        \"\"\"\n        self.filename = filename\n        self.sheet = None\n\n    @property\n    def sheets(self):\n        \"\"\"FileFormats with a notion of sheets should override this property\n        to return a list of sheet names in the file.\n\n        Returns\n        -------\n        a list of sheet names\n        \"\"\"\n        return ()\n\n    def select_sheet(self, sheet):\n        \"\"\"Select sheet to be read\n\n        Parameters\n        ----------\n        sheet : str\n            sheet name\n        \"\"\"\n        self.sheet = sheet\n\n    @classmethod\n    def get_reader(cls, filename):\n        \"\"\"Return reader instance that can be used to read the file\n\n        Parameters\n        ----------\n        filename : str\n\n        Returns\n        -------\n        FileFormat\n        \"\"\"\n        for ext, reader in cls.readers.items():\n            # Skip ambiguous, invalid compression-only extensions added on OSX\n            if ext in Compression.all:\n                continue\n            if fnmatch(path.basename(filename), '*' + ext):\n                return reader(filename)\n\n        raise IOError('No readers for file \"{}\"'.format(filename))\n\n    @classmethod\n    def write(cls, filename, data):\n        return cls.write_file(filename, data)\n\n    @classmethod\n    def write_table_metadata(cls, filename, data):\n\n        def write_file(fn):\n            if all(isinstance(key, str) and isinstance(value, str)\n                   for key, value in data.attributes.items()):\n                with open(fn, 'w', encoding='utf-8') as f:\n                    f.write(\"\\n\".join(\"{}: {}\".format(*kv)\n                                      for kv in data.attributes.items()))\n            else:\n                with open(fn, 'wb') as f:\n                    pickle.dump(data.attributes, f, pickle.HIGHEST_PROTOCOL)\n\n        if isinstance(filename, str):\n            metafile = filename + '.metadata'\n            if getattr(data, 'attributes', None):\n                write_file(metafile)\n            elif path.exists(metafile):\n                remove(metafile)\n\n    @classmethod\n    def set_table_metadata(cls, filename, table):\n        # pylint: disable=bare-except\n        if isinstance(filename, str) and path.exists(filename + '.metadata'):\n            try:\n                with open(filename + '.metadata', 'rb') as f:\n                    table.attributes = pickle.load(f)\n            # Unpickling throws different exceptions, not just UnpickleError\n            except:\n                with open(filename + '.metadata', encoding='utf-8') as f:\n                    table.attributes = OrderedDict(\n                        (k.strip(), v.strip())\n                        for k, v in (line.split(\":\", 1)\n                                     for line in f.readlines()))\n\n    @classmethod\n    def locate(cls, filename, search_dirs=('.',)):\n        \"\"\"Locate a file with given filename that can be opened by one\n        of the available readers.\n\n        Parameters\n        ----------\n        filename : str\n        search_dirs : Iterable[str]\n\n        Returns\n        -------\n        str\n            Absolute path to the file\n        \"\"\"\n        if path.exists(filename):\n            return filename\n\n        for directory in search_dirs:\n            absolute_filename = path.join(directory, filename)\n            if path.exists(absolute_filename):\n                break\n            for ext in cls.readers:\n                if fnmatch(path.basename(filename), '*' + ext):\n                    break\n                # glob uses fnmatch internally\n                matching_files = glob(absolute_filename + ext)\n                if matching_files:\n                    absolute_filename = matching_files[0]\n                    break\n            if path.exists(absolute_filename):\n                break\n        else:\n            absolute_filename = \"\"\n\n        if not path.exists(absolute_filename):\n            raise IOError('File \"{}\" was not found.'.format(filename))\n\n        return absolute_filename\n\n    @staticmethod\n    def open(filename, *args, **kwargs):\n        \"\"\"\n        Format handlers can use this method instead of the builtin ``open()``\n        to transparently (de)compress files if requested (according to\n        `filename` extension). Set ``SUPPORT_COMPRESSED=True`` if you use this.\n        \"\"\"\n        return open_compressed(filename, *args, **kwargs)\n\n    @staticmethod\n    def parse_headers(data):\n        \"\"\"Return (header rows, rest of data) as discerned from `data`\"\"\"\n\n        def is_number(item):\n            try:\n                float(item)\n            except ValueError:\n                return False\n            return True\n        # Second row items are type identifiers\n        def header_test2(items):\n            return all(map(_RE_TYPES.match, items))\n        # Third row items are flags and column attributes (attr=value)\n        def header_test3(items):\n            return all(map(_RE_FLAGS.match, items))\n\n        data = iter(data)\n        header_rows = []\n\n        # Try to parse a three-line header\n        lines = []\n        try:\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n        except StopIteration:\n            lines, data = [], chain(lines, data)\n        if lines:\n            l1, l2, l3 = lines\n            # Three-line header if line 2 & 3 match (1st line can be anything)\n            if header_test2(l2) and header_test3(l3):\n                header_rows = [l1, l2, l3]\n            else:\n                lines, data = [], chain((l1, l2, l3), data)\n\n        # Try to parse a single-line header\n        if not header_rows:\n            try:\n                lines.append(list(next(data)))\n            except StopIteration:\n                pass\n            if lines:\n                # Header if none of the values in line 1 parses as a number\n                if not all(is_number(i) for i in lines[0]):\n                    header_rows = [lines[0]]\n                else:\n                    data = chain(lines, data)\n\n        return header_rows, data\n\n    @classmethod\n    def data_table(cls, data, headers=None):\n        \"\"\"\n        Return Orange.data.Table given rows of `headers` (iterable of iterable)\n        and rows of `data` (iterable of iterable).\n\n        Basically, the idea of subclasses is to produce those two iterables,\n        however they might.\n\n        If `headers` is not provided, the header rows are extracted from `data`,\n        assuming they precede it.\n        \"\"\"\n        if not headers:\n            headers, data = cls.parse_headers(data)\n\n        # Consider various header types (single-row, two-row, three-row, none)\n        if len(headers) == 3:\n            names, types, flags = map(list, headers)\n        else:\n            if len(headers) == 1:\n                HEADER1_FLAG_SEP = '#'\n                # First row format either:\n                #   1) delimited column names\n                #   2) -||- with type and flags prepended, separated by #,\n                #      e.g. d#sex,c#age,cC#IQ\n                _flags, names = zip(*[i.split(HEADER1_FLAG_SEP, 1)\n                                      if HEADER1_FLAG_SEP in i else ('', i)\n                                      for i in headers[0]]\n                                   )\n                names = list(names)\n            elif len(headers) == 2:\n                names, _flags = map(list, headers)\n            else:\n                # Use heuristics for everything\n                names, _flags = [], []\n            types = [''.join(filter(str.isupper, flag)).lower() for flag in _flags]\n            flags = [Flags.join(filter(str.islower, flag)) for flag in _flags]\n\n        # Determine maximum row length\n        rowlen = max(map(len, (names, types, flags)))\n\n        def _equal_length(lst):\n            lst.extend(['']*(rowlen - len(lst)))\n            return lst\n\n        # Ensure all data is of equal width in a column-contiguous array\n        data = [_equal_length([s.strip() for s in row])\n                for row in data if any(row)]\n        data = np.array(data, dtype=object, order='F')\n\n        # Data may actually be longer than headers were\n        try:\n            rowlen = data.shape[1]\n        except IndexError:\n            pass\n        else:\n            for lst in (names, types, flags):\n                _equal_length(lst)\n\n        NAMEGEN = namegen('Feature ', 1)\n        Xcols, attrs = [], []\n        Mcols, metas = [], []\n        Ycols, clses = [], []\n        Wcols = []\n\n        # Rename variables if necessary\n        # Reusing across files still works if both files have same duplicates\n        name_counts = Counter(names)\n        del name_counts[\"\"]\n        if len(name_counts) != len(names) and name_counts:\n            uses = {name: 0 for name, count in name_counts.items() if count > 1}\n            for i, name in enumerate(names):\n                if name in uses:\n                    uses[name] += 1\n                    names[i] = \"{}_{}\".format(name, uses[name])\n\n        namask = np.empty(data.shape[0], dtype=bool)\n        # Iterate through the columns\n        for col in range(rowlen):\n            flag = Flags(Flags.split(flags[col]))\n            if flag.i:\n                continue\n\n            type_flag = types and types[col].strip()\n            try:\n                orig_values = data[:, col]\n            except IndexError:\n                orig_values = np.array([], dtype=object)\n\n            namask = isnastr(orig_values, out=namask)\n\n            coltype_kwargs = {}\n            valuemap = None\n            values = orig_values\n\n            if type_flag in StringVariable.TYPE_HEADERS:\n                coltype = StringVariable\n                values = orig_values\n            elif type_flag in ContinuousVariable.TYPE_HEADERS:\n                coltype = ContinuousVariable\n                values = np.empty(data.shape[0], dtype=float)\n                try:\n                    np.copyto(values, orig_values, casting=\"unsafe\",\n                              where=~namask)\n                    values[namask] = np.nan\n                except ValueError:\n                    for row, num in enumerate(orig_values):\n                        if not isnastr(num):\n                            try:\n                                float(num)\n                            except ValueError:\n                                break\n                    raise ValueError('Non-continuous value in (1-based) '\n                                     'line {}, column {}'.format(row + len(headers) + 1,\n                                                                 col + 1))\n\n            elif type_flag in TimeVariable.TYPE_HEADERS:\n                coltype = TimeVariable\n                values = np.where(namask, \"\", orig_values)\n            elif (type_flag in DiscreteVariable.TYPE_HEADERS or\n                  _RE_DISCRETE_LIST.match(type_flag)):\n                coltype = DiscreteVariable\n                orig_values = values = np.where(namask, \"\", orig_values)\n                if _RE_DISCRETE_LIST.match(type_flag):\n                    valuemap = Flags.split(type_flag)\n                    coltype_kwargs.update(ordered=True)\n                else:\n                    valuemap = sorted(set(orig_values) - {\"\"})\n            else:\n                # No known type specified, use heuristics\n                valuemap, values, coltype = guess_data_type(orig_values, namask)\n\n            if flag.m or coltype is StringVariable:\n                append_to = (Mcols, metas)\n            elif flag.w:\n                append_to = (Wcols, None)\n            elif flag.c:\n                append_to = (Ycols, clses)\n            else:\n                append_to = (Xcols, attrs)\n\n            cols, domain_vars = append_to\n\n            if domain_vars is not None:\n                var_name = names and names[col]\n                if not var_name:\n                    var_name = next(NAMEGEN)\n\n                values, var = sanitize_variable(\n                    valuemap, values, orig_values, coltype, coltype_kwargs,\n                    name=var_name)\n            else:\n                var = None\n            if domain_vars is not None:\n                var.attributes.update(flag.attributes)\n                domain_vars.append(var)\n\n            if isinstance(values, np.ndarray) and not values.flags.owndata:\n                values = values.copy()  # might view `data` (string columns)\n            cols.append(values)\n\n            try:\n                # allow gc to reclaim memory used by string values\n                data[:, col] = None\n            except IndexError:\n                pass\n\n        domain = Domain(attrs, clses, metas)\n\n        if not data.size:\n            return Table.from_domain(domain, 0)\n\n        X = Y = M = W = None\n        if Xcols:\n            X = np.c_[tuple(Xcols)]\n            assert X.dtype == np.float_\n        else:\n            X = np.empty((data.shape[0], 0), dtype=np.float_)\n        if Ycols:\n            Y = np.c_[tuple(Ycols)]\n            assert Y.dtype == np.float_\n        if Mcols:\n            M = np.c_[tuple(Mcols)].astype(object)\n        if Wcols:\n            W = np.c_[tuple(Wcols)].astype(float)\n\n        table = Table.from_numpy(domain, X, Y, M, W)\n        return table\n\n    @staticmethod\n    def header_names(data):\n        return ['weights'] * data.has_weights() + \\\n               [v.name for v in chain(data.domain.attributes,\n                                      data.domain.class_vars,\n                                      data.domain.metas)]\n\n    @staticmethod\n    def header_types(data):\n        def _vartype(var):\n            if var.is_continuous or var.is_string:\n                return var.TYPE_HEADERS[0]\n            elif var.is_discrete:\n                return Flags.join(var.values) if var.ordered else var.TYPE_HEADERS[0]\n            raise NotImplementedError\n        return ['continuous'] * data.has_weights() + \\\n               [_vartype(v) for v in chain(data.domain.attributes,\n                                           data.domain.class_vars,\n                                           data.domain.metas)]\n\n    @staticmethod\n    def header_flags(data):\n        return list(chain(['weight'] * data.has_weights(),\n                          (Flags.join([flag], *('{}={}'.format(*a)\n                                                for a in sorted(var.attributes.items())))\n                           for flag, var in chain(zip(repeat(''), data.domain.attributes),\n                                                  zip(repeat('class'), data.domain.class_vars),\n                                                  zip(repeat('meta'), data.domain.metas)))))\n\n    @classmethod\n    def write_headers(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        write(cls.header_names(data))\n        write(cls.header_types(data))\n        write(cls.header_flags(data))\n\n    @classmethod\n    def write_data(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        vars = list(chain((ContinuousVariable('_w'),) if data.has_weights() else (),\n                          data.domain.attributes,\n                          data.domain.class_vars,\n                          data.domain.metas))\n\n        def formatter(var):\n            # type: (Variable) -> Callable[[Variable], Any]\n            # Return a column 'formatter' function. The function must return\n            # something that `write` knows how to write\n            if var.is_time:\n                return var.repr_val\n            elif var.is_continuous:\n                return lambda value: \"\" if isnan(value) else value\n            elif var.is_discrete:\n                return lambda value: \"\" if isnan(value) else var.values[int(value)]\n            elif var.is_string:\n                return lambda value: value\n            else:\n                return var.repr_val\n\n        formatters = [formatter(v) for v in vars]\n        for row in zip(data.W if data.W.ndim > 1 else data.W[:, np.newaxis],\n                       data.X,\n                       data.Y if data.Y.ndim > 1 else data.Y[:, np.newaxis],\n                       data.metas):\n            write([fmt(v) for fmt, v in zip(formatters, flatten(row))])\n\n    @classmethod\n    def qualified_name(cls):\n        return cls.__module__ + '.' + cls.__name__\n\n\ndef class_from_qualified_name(format_name):\n    \"\"\" File format class from qualified name. \"\"\"\n    elements = format_name.split(\".\")\n    m = import_module(\".\".join(elements[:-1]))\n    return getattr(m, elements[-1])\n\n\nclass CSVReader(FileFormat):\n    \"\"\"Reader for comma separated files\"\"\"\n\n    EXTENSIONS = ('.csv',)\n    DESCRIPTION = 'Comma-separated values'\n    DELIMITERS = ',;:\\t$ '\n    SUPPORT_COMPRESSED = True\n    SUPPORT_SPARSE_DATA = False\n    PRIORITY = 20\n\n    def read(self):\n        for encoding in (lambda: ('us-ascii', None),                 # fast\n                         lambda: (detect_encoding(self.filename), None),  # precise\n                         lambda: (locale.getpreferredencoding(False), None),\n                         lambda: (sys.getdefaultencoding(), None),   # desperate\n                         lambda: ('utf-8', None),                    # ...\n                         lambda: ('utf-8', 'ignore')):               # fallback\n            encoding, errors = encoding()\n            # Clear the error flag for all except the last check, because\n            # the error of second-to-last check is stored and shown as warning in owfile\n            if errors != 'ignore':\n                error = ''\n            with self.open(self.filename, mode='rt', newline='',\n                           encoding=encoding, errors=errors) as file:\n                # Sniff the CSV dialect (delimiter, quotes, ...)\n                try:\n                    dialect = csv.Sniffer().sniff(\n                        # Take first couple of *complete* lines as sample\n                        ''.join(file.readline() for _ in range(10)),\n                        self.DELIMITERS)\n                    delimiter = dialect.delimiter\n                    quotechar = dialect.quotechar\n                except UnicodeDecodeError as e:\n                    error = e\n                    continue\n                except csv.Error:\n                    delimiter = self.DELIMITERS[0]\n                    quotechar = csv.excel.quotechar\n\n                file.seek(0)\n                try:\n                    reader = csv.reader(\n                        file, delimiter=delimiter, quotechar=quotechar,\n                        skipinitialspace=True,\n                    )\n                    data = self.data_table(reader)\n\n                    # TODO: Name can be set unconditionally when/if\n                    # self.filename will always be a string with the file name.\n                    # Currently, some tests pass StringIO instead of\n                    # the file name to a reader.\n                    if isinstance(self.filename, str):\n                        data.name = path.splitext(\n                            path.split(self.filename)[-1])[0]\n                    if error and isinstance(error, UnicodeDecodeError):\n                        pos, endpos = error.args[2], error.args[3]\n                        warning = ('Skipped invalid byte(s) in position '\n                                   '{}{}').format(pos,\n                                                  ('-' + str(endpos)) if (endpos - pos) > 1 else '')\n                        warnings.warn(warning)\n                    self.set_table_metadata(self.filename, data)\n                    return data\n                except Exception as e:\n                    error = e\n                    continue\n        raise ValueError('Cannot parse dataset {}: {}'.format(self.filename, error)) from error\n\n    @classmethod\n    def write_file(cls, filename, data):\n        with cls.open(filename, mode='wt', newline='', encoding='utf-8') as file:\n            writer = csv.writer(file, delimiter=cls.DELIMITERS[0])\n            cls.write_headers(writer.writerow, data)\n            cls.write_data(writer.writerow, data)\n        cls.write_table_metadata(filename, data)\n\n\nclass TabReader(CSVReader):\n    \"\"\"Reader for tab separated files\"\"\"\n    EXTENSIONS = ('.tab', '.tsv')\n    DESCRIPTION = 'Tab-separated values'\n    DELIMITERS = '\\t'\n    PRIORITY = 10\n\n\nclass PickleReader(FileFormat):\n    \"\"\"Reader for pickled Table objects\"\"\"\n    EXTENSIONS = ('.pkl', '.pickle')\n    DESCRIPTION = 'Pickled Orange data'\n    SUPPORT_COMPRESSED = True\n    SUPPORT_SPARSE_DATA = True\n\n    def read(self):\n        with self.open(self.filename, 'rb') as f:\n            table = pickle.load(f)\n            if not isinstance(table, Table):\n                raise TypeError(\"file does not contain a data table\")\n            else:\n                return table\n\n    @classmethod\n    def write_file(cls, filename, data):\n        with cls.open(filename, 'wb') as f:\n            pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)\n\n\nclass BasketReader(FileFormat):\n    \"\"\"Reader for basket (sparse) files\"\"\"\n    EXTENSIONS = ('.basket', '.bsk')\n    DESCRIPTION = 'Basket file'\n    SUPPORT_SPARSE_DATA = True\n\n    def read(self):\n        def constr_vars(inds):\n            if inds:\n                return [ContinuousVariable(x.decode(\"utf-8\")) for _, x in\n                        sorted((ind, name) for name, ind in inds.items())]\n\n        X, Y, metas, attr_indices, class_indices, meta_indices = \\\n            _io.sparse_read_float(self.filename.encode(sys.getdefaultencoding()))\n\n        attrs = constr_vars(attr_indices)\n        classes = constr_vars(class_indices)\n        meta_attrs = constr_vars(meta_indices)\n        domain = Domain(attrs, classes, meta_attrs)\n        table = Table.from_numpy(\n            domain, attrs and X, classes and Y, metas and meta_attrs)\n        table.name = path.splitext(path.split(self.filename)[-1])[0]\n        return table\n\n\nclass ExcelReader(FileFormat):\n    \"\"\"Reader for excel files\"\"\"\n    EXTENSIONS = ('.xls', '.xlsx')\n    DESCRIPTION = 'Mircosoft Excel spreadsheet'\n    SUPPORT_SPARSE_DATA = False\n\n    def __init__(self, filename):\n        super().__init__(filename)\n\n        from xlrd import open_workbook\n        self.workbook = open_workbook(self.filename)\n\n    @property\n    @lru_cache(1)\n    def sheets(self):\n        return self.workbook.sheet_names()\n\n    def read(self):\n        import xlrd\n        wb = xlrd.open_workbook(self.filename, on_demand=True)\n        if self.sheet:\n            ss = wb.sheet_by_name(self.sheet)\n        else:\n            ss = wb.sheet_by_index(0)\n        try:\n            first_row = next(i for i in range(ss.nrows) if any(ss.row_values(i)))\n            first_col = next(i for i in range(ss.ncols) if ss.cell_value(first_row, i))\n            row_len = ss.row_len(first_row)\n            cells = filter(any,\n                           [[str(ss.cell_value(row, col)) if col < ss.row_len(row) else ''\n                             for col in range(first_col, row_len)]\n                            for row in range(first_row, ss.nrows)])\n            table = self.data_table(cells)\n            table.name = path.splitext(path.split(self.filename)[-1])[0]\n            if self.sheet:\n                table.name = '-'.join((table.name, self.sheet))\n        except Exception:\n            raise IOError(\"Couldn't load spreadsheet from \" + self.filename)\n        return table\n\n\nclass DotReader(FileFormat):\n    \"\"\"Writer for dot (graph) files\"\"\"\n    EXTENSIONS = ('.dot', '.gv')\n    DESCRIPTION = 'Dot graph description'\n    SUPPORT_COMPRESSED = True\n    SUPPORT_SPARSE_DATA = False\n\n    @classmethod\n    def write_graph(cls, filename, graph):\n        from sklearn import tree\n        tree.export_graphviz(graph, out_file=cls.open(filename, 'wt'))\n\n    @classmethod\n    def write(cls, filename, tree):\n        if type(tree) == dict:\n            tree = tree['tree']\n        cls.write_graph(filename, tree)\n\n\nclass UrlReader(FileFormat):\n    def __init__(self, filename):\n        filename = filename.strip()\n        if not urlparse(filename).scheme:\n            filename = 'http://' + filename\n        super().__init__(filename)\n\n    @staticmethod\n    def urlopen(url):\n        req = Request(\n            url,\n            # Avoid 403 error with servers that dislike scrapers\n            headers={'User-Agent': 'Mozilla/5.0 (X11; Linux) Gecko/20100101 Firefox/'})\n        return urlopen(req, timeout=10)\n\n    def read(self):\n        self.filename = self._trim(self._resolve_redirects(self.filename))\n        with contextlib.closing(self.urlopen(self.filename)) as response:\n            name = self._suggest_filename(response.headers['content-disposition'])\n            with NamedTemporaryFile(suffix=name, delete=False) as f:\n                f.write(response.read())\n                # delete=False is a workaround for https://bugs.python.org/issue14243\n\n            reader = self.get_reader(f.name)\n            data = reader.read()\n            remove(f.name)\n        # Override name set in from_file() to avoid holding the temp prefix\n        data.name = path.splitext(name)[0]\n        data.origin = self.filename\n        return data\n\n    def _resolve_redirects(self, url):\n        # Resolve (potential) redirects to a final URL\n        with contextlib.closing(self.urlopen(url)) as response:\n            return response.url\n\n    @classmethod\n    def _trim(cls, url):\n        URL_TRIMMERS = (\n            cls._trim_googlesheet,\n            cls._trim_dropbox,\n        )\n        for trim in URL_TRIMMERS:\n            try:\n                url = trim(url)\n            except ValueError:\n                continue\n            else:\n                break\n        return url\n\n    @staticmethod\n    def _trim_googlesheet(url):\n        match = re.match(r'(?:https?://)?(?:www\\.)?'\n                         r'docs\\.google\\.com/spreadsheets/d/'\n                         r'(?P<workbook_id>[-\\w_]+)'\n                         r'(?:/.*?gid=(?P<sheet_id>\\d+).*|.*)?',\n                         url, re.IGNORECASE)\n        try:\n            workbook, sheet = match.group('workbook_id'), match.group('sheet_id')\n            if not workbook:\n                raise ValueError\n        except (AttributeError, ValueError):\n            raise ValueError\n        url = 'https://docs.google.com/spreadsheets/d/{}/export?format=tsv'.format(workbook)\n        if sheet:\n            url += '&gid=' + sheet\n        return url\n\n    @staticmethod\n    def _trim_dropbox(url):\n        parts = urlsplit(url)\n        if not parts.netloc.endswith('dropbox.com'):\n            raise ValueError\n        return urlunsplit(parts._replace(query='dl=1'))\n\n    def _suggest_filename(self, content_disposition):\n        default_name = re.sub(r'[\\\\:/]', '_', urlparse(self.filename).path)\n\n        # See https://tools.ietf.org/html/rfc6266#section-4.1\n        matches = re.findall(r\"filename\\*?=(?:\\\"|.{0,10}?'[^']*')([^\\\"]+)\",\n                             content_disposition or '')\n        return urlunquote(matches[-1]) if matches else default_name\n",
            "new_file_content": "import contextlib\nimport csv\nimport locale\nimport pickle\nimport re\nimport subprocess\nimport sys\nimport warnings\n\nfrom ast import literal_eval\nfrom collections import OrderedDict, Counter, defaultdict\nfrom functools import lru_cache\nfrom importlib import import_module\nfrom itertools import chain, repeat\nfrom math import isnan\n\nfrom os import path, remove\nfrom tempfile import NamedTemporaryFile\nfrom urllib.parse import urlparse, urlsplit, urlunsplit, unquote as urlunquote\nfrom urllib.request import urlopen, Request\nfrom fnmatch import fnmatch\nfrom glob import glob\n\nimport numpy as np\nfrom chardet.universaldetector import UniversalDetector\n\nfrom Orange.data import (\n    _io, is_discrete_values, MISSING_VALUES, Table, Domain, Variable,\n    DiscreteVariable, StringVariable, ContinuousVariable, TimeVariable,\n)\nfrom Orange.util import Registry, flatten, namegen\n\n\n# Support values longer than 128K (i.e. text contents features)\ncsv.field_size_limit(100*1024*1024)\n\n__all__ = [\"Flags\", \"FileFormat\"]\n\n\n_IDENTITY = lambda i: i\n\n\nclass Compression:\n    \"\"\"Supported compression extensions\"\"\"\n    GZIP = '.gz'\n    BZIP2 = '.bz2'\n    XZ = '.xz'\n    all = (GZIP, BZIP2, XZ)\n\n\ndef open_compressed(filename, *args, _open=open, **kwargs):\n    \"\"\"Return seamlessly decompressed open file handle for `filename`\"\"\"\n    if isinstance(filename, str):\n        if filename.endswith(Compression.GZIP):\n            from gzip import open as _open\n        elif filename.endswith(Compression.BZIP2):\n            from bz2 import open as _open\n        elif filename.endswith(Compression.XZ):\n            from lzma import open as _open\n        return _open(filename, *args, **kwargs)\n    # Else already a file, just pass it through\n    return filename\n\n\ndef detect_encoding(filename):\n    \"\"\"\n    Detect encoding of `filename`, which can be a ``str`` filename, a\n    ``file``-like object, or ``bytes``.\n    \"\"\"\n    # Try with Unix file utility first because it's faster (~10ms vs 100ms)\n    if isinstance(filename, str) and not filename.endswith(Compression.all):\n        try:\n            with subprocess.Popen(('file', '--brief', '--mime-encoding', filename),\n                                  stdout=subprocess.PIPE) as process:\n                process.wait()\n                if process.returncode == 0:\n                    encoding = process.stdout.read().strip()\n                    # file only supports these encodings; for others it says\n                    # unknown-8bit or binary. So we give chardet a chance to do\n                    # better\n                    if encoding in (b'utf-8', b'us-ascii', b'iso-8859-1',\n                                    b'utf-7', b'utf-16le', b'utf-16be', b'ebcdic'):\n                        return encoding.decode('us-ascii')\n        except OSError:\n            pass  # windoze\n\n    # file not available or unable to guess the encoding, have chardet do it\n    detector = UniversalDetector()\n    # We examine only first N 4kB blocks of file because chardet is really slow\n    MAX_BYTES = 4*1024*12\n\n    def _from_file(f):\n        detector.feed(f.read(MAX_BYTES))\n        detector.close()\n        return (detector.result.get('encoding')\n                if detector.result.get('confidence', 0) >= .85 else\n                'utf-8')\n\n    if isinstance(filename, str):\n        with open_compressed(filename, 'rb') as f:\n            return _from_file(f)\n    elif isinstance(filename, bytes):\n        detector.feed(filename[:MAX_BYTES])\n        detector.close()\n        return detector.result.get('encoding')\n    elif hasattr(filename, 'encoding'):\n        return filename.encoding\n    else:  # assume file-like object that you can iter through\n        return _from_file(filename)\n\n\n__isnastr = np.frompyfunc(\n    {v for v in MISSING_VALUES if isinstance(v, str)}.__contains__, 1, 1)\n\n# wrapper for __isnastr with proper default out dtype\ndef isnastr(arr, out=None):\n    \"\"\"\n    Given an (object) array of string values, return a boolean mask array\n    that is True where the `arr` contains one of the string constants\n    considered as N/A.\n\n    Parameters\n    ----------\n    arr : np.ndarray\n        Input array of strings.\n    out : Optional[np.ndarray]\n        Optional output array of the same shape as arr\n\n    Returns\n    -------\n    mask : np.ndarray\n    \"\"\"\n    arr = np.asarray(arr)\n    if out is None and arr.shape != ():\n        out = np.empty_like(arr, dtype=bool)\n    return __isnastr(arr, out=out)\n\n\ndef guess_data_type(orig_values, namask=None):\n    \"\"\"\n    Use heuristics to guess data type.\n    \"\"\"\n    valuemap, values = None, orig_values\n    is_discrete = is_discrete_values(orig_values)\n    if is_discrete:\n        valuemap = sorted(is_discrete)\n        coltype = DiscreteVariable\n    else:\n        # try to parse as float\n        orig_values = np.asarray(orig_values)\n        if namask is None:\n            namask = isnastr(orig_values)\n        values = np.empty_like(orig_values, dtype=float)\n        values[namask] = np.nan\n        try:\n            np.copyto(values, orig_values, where=~namask, casting=\"unsafe\")\n        except ValueError:\n            tvar = TimeVariable('_')\n            try:\n                values[~namask] = [tvar.parse(i) for i in orig_values[~namask]]\n            except ValueError:\n                coltype = StringVariable\n                # return original_values\n                values = orig_values\n            else:\n                coltype = TimeVariable\n        else:\n            coltype = ContinuousVariable\n    return valuemap, values, coltype\n\n\ndef sanitize_variable(valuemap, values, orig_values, coltype, coltype_kwargs,\n                      domain_vars=None, existing_var=None, new_var_name=None, data=None, name=None):\n    assert issubclass(coltype, Variable)\n\n    if name is None or existing_var is not None or new_var_name is not None:\n        name = existing_var.strip() if existing_var else new_var_name\n        raise DeprecationWarning(\"Arguments 'existing_var' and 'new_var_name' are \"\\\n                                 \"deprecated since 3.16; use 'name' instead\")\n\n    if domain_vars is not None:\n        raise DeprecationWarning(\"Argument 'domain_vars' is deprecated since 3.16\")\n\n    if data is not None:\n        raise DeprecationWarning(\"Argument 'data' is deprecated since 3.16\")\n\n    def get_number_of_decimals(values):\n        len_ = len\n        ndecimals = max((len_(value) - value.find(\".\")\n                         for value in values if \".\" in value),\n                        default=1)\n        return ndecimals - 1\n\n    if issubclass(coltype, DiscreteVariable) and valuemap is not None:\n        coltype_kwargs.update(values=valuemap)\n\n    var = coltype.make(name, **coltype_kwargs)\n\n    if isinstance(var, DiscreteVariable):\n        # Map discrete data to 'ints' (or at least what passes as int around\n        # here)\n        mapping = defaultdict(\n            lambda: np.nan,\n            {val: i for i, val in enumerate(var.values)},\n        )\n        mapping[\"\"] = np.nan\n        mapvalues_ = np.frompyfunc(mapping.__getitem__, 1, 1)\n\n        def mapvalues(arr):\n            arr = np.asarray(arr, dtype=object)\n            return mapvalues_(arr, out=np.empty_like(arr, dtype=float))\n        values = mapvalues(orig_values)\n\n    if coltype is StringVariable:\n        values = orig_values\n\n    # ContinuousVariable.number_of_decimals is supposed to be handled by\n    # ContinuousVariable.to_val. In the interest of speed, the reader bypasses\n    # it, so we set the number of decimals here.\n    # The number of decimals is increased if not set manually (in which case\n    # var.adjust_decimals would be 0).\n    if isinstance(var, ContinuousVariable) and var.adjust_decimals:\n        ndecimals = get_number_of_decimals(orig_values)\n        if var.adjust_decimals == 2 or ndecimals > var.number_of_decimals:\n            var.number_of_decimals = ndecimals\n            var.adjust_decimals = 1\n\n    if isinstance(var, TimeVariable) or coltype is TimeVariable:\n        # Re-parse the values because only now after coltype.make call\n        # above, variable var is the correct one\n        _var = var if isinstance(var, TimeVariable) else TimeVariable('_')\n        values = [_var.parse(i) for i in orig_values]\n\n    return values, var\n\n\nclass Flags:\n    \"\"\"Parser for column flags (i.e. third header row)\"\"\"\n    DELIMITER = ' '\n    _RE_SPLIT = re.compile(r'(?<!\\\\)' + DELIMITER).split\n    _RE_ATTR_UNQUOTED_STR = re.compile(r'^[a-zA-Z_]').match\n    ALL = OrderedDict((\n        ('class', 'c'),\n        ('ignore', 'i'),\n        ('meta', 'm'),\n        ('weight', 'w'),\n        ('.+?=.*?', ''),  # general key=value attributes\n    ))\n    _RE_ALL = re.compile(r'^({})$'.format('|'.join(filter(None, flatten(ALL.items())))))\n\n    def __init__(self, flags):\n        for v in filter(None, self.ALL.values()):\n            setattr(self, v, False)\n        self.attributes = {}\n        for flag in flags or []:\n            flag = flag.strip()\n            if self._RE_ALL.match(flag):\n                if '=' in flag:\n                    k, v = flag.split('=', 1)\n                    if not Flags._RE_ATTR_UNQUOTED_STR(v):\n                        try:\n                            v = literal_eval(v)\n                        except SyntaxError:\n                            # If parsing failed, treat value as string\n                            pass\n                    self.attributes[k] = v\n                else:\n                    setattr(self, flag, True)\n                    setattr(self, self.ALL.get(flag, ''), True)\n            elif flag:\n                warnings.warn('Invalid attribute flag \\'{}\\''.format(flag))\n\n    @staticmethod\n    def join(iterable, *args):\n        return Flags.DELIMITER.join(i.strip().replace(Flags.DELIMITER, '\\\\' + Flags.DELIMITER)\n                                    for i in chain(iterable, args)).lstrip()\n\n    @staticmethod\n    def split(s):\n        return [i.replace('\\\\' + Flags.DELIMITER, Flags.DELIMITER)\n                for i in Flags._RE_SPLIT(s)]\n\n\n# Matches discrete specification where all the values are listed, space-separated\n_RE_DISCRETE_LIST = re.compile(r'^\\s*[^\\s]+(\\s[^\\s]+)+\\s*$')\n_RE_TYPES = re.compile(r'^\\s*({}|{}|)\\s*$'.format(\n    _RE_DISCRETE_LIST.pattern,\n    '|'.join(flatten(getattr(vartype, 'TYPE_HEADERS') for vartype in Variable.registry.values()))\n    ))\n_RE_FLAGS = re.compile(r'^\\s*( |{}|)*\\s*$'.format(\n    '|'.join(flatten(filter(None, i) for i in Flags.ALL.items()))\n    ))\n\n\nclass FileFormatMeta(Registry):\n\n    def __new__(cls, name, bases, attrs):\n        newcls = super().__new__(cls, name, bases, attrs)\n\n        # Optionally add compressed versions of extensions as supported\n        if getattr(newcls, 'SUPPORT_COMPRESSED', False):\n            new_extensions = list(getattr(newcls, 'EXTENSIONS', ()))\n            for compression in Compression.all:\n                for ext in newcls.EXTENSIONS:\n                    new_extensions.append(ext + compression)\n                if sys.platform in ('darwin', 'win32'):\n                    # OSX file dialog doesn't support filtering on double\n                    # extensions (e.g. .csv.gz)\n                    # https://bugreports.qt.io/browse/QTBUG-38303\n                    # This is just here for OWFile that gets QFileDialog\n                    # filters from FileFormat.readers.keys()\n                    # EDIT: Windows exhibit similar problems:\n                    # while .tab.gz works, .tab.xz and .tab.bz2 do not!\n                    new_extensions.append(compression)\n            newcls.EXTENSIONS = tuple(new_extensions)\n\n        return newcls\n\n    @property\n    def formats(cls):\n        return cls.registry.values()\n\n    @lru_cache(5)\n    def _ext_to_attr_if_attr2(cls, attr, attr2):\n        \"\"\"\n        Return ``{ext: `attr`, ...}`` dict if ``cls`` has `attr2`.\n        If `attr` is '', return ``{ext: cls, ...}`` instead.\n\n        If there are multiple formats for an extension, return a format\n        with the lowest priority.\n        \"\"\"\n        formats = OrderedDict()\n        for format in sorted(cls.registry.values(), key=lambda x: x.PRIORITY):\n            if not hasattr(format, attr2):\n                continue\n            for ext in getattr(format, 'EXTENSIONS', []):\n                # Only adds if not yet registered\n                formats.setdefault(ext, getattr(format, attr, format))\n        return formats\n\n    @property\n    def names(cls):\n        return cls._ext_to_attr_if_attr2('DESCRIPTION', '__class__')\n\n    @property\n    def writers(cls):\n        return cls._ext_to_attr_if_attr2('', 'write_file')\n\n    @property\n    def readers(cls):\n        return cls._ext_to_attr_if_attr2('', 'read')\n\n    @property\n    def img_writers(cls):\n        return cls._ext_to_attr_if_attr2('', 'write_image')\n\n    @property\n    def graph_writers(cls):\n        return cls._ext_to_attr_if_attr2('', 'write_graph')\n\n\nclass FileFormat(metaclass=FileFormatMeta):\n    \"\"\"\n    Subclasses set the following attributes and override the following methods:\n\n        EXTENSIONS = ('.ext1', '.ext2', ...)\n        DESCRIPTION = 'human-readable file format description'\n        SUPPORT_COMPRESSED = False\n        SUPPORT_SPARSE_DATA = False\n\n        def read(self):\n            ...  # load headers, data, ...\n            return self.data_table(data, headers)\n\n        @classmethod\n        def write_file(cls, filename, data):\n            ...\n            self.write_headers(writer.write, data)\n            writer.writerows(data)\n\n    Wrapper FileFormat.data_table() returns Orange.data.Table from `data`\n    iterable (list (rows) of lists of values (cols)).\n    \"\"\"\n\n    # Priority when multiple formats support the same extension. Also\n    # the sort order in file open/save combo boxes. Lower is better.\n    PRIORITY = 10000\n\n    def __init__(self, filename):\n        \"\"\"\n        Parameters\n        ----------\n        filename : str\n            name of the file to open\n        \"\"\"\n        self.filename = filename\n        self.sheet = None\n\n    @property\n    def sheets(self):\n        \"\"\"FileFormats with a notion of sheets should override this property\n        to return a list of sheet names in the file.\n\n        Returns\n        -------\n        a list of sheet names\n        \"\"\"\n        return ()\n\n    def select_sheet(self, sheet):\n        \"\"\"Select sheet to be read\n\n        Parameters\n        ----------\n        sheet : str\n            sheet name\n        \"\"\"\n        self.sheet = sheet\n\n    @classmethod\n    def get_reader(cls, filename):\n        \"\"\"Return reader instance that can be used to read the file\n\n        Parameters\n        ----------\n        filename : str\n\n        Returns\n        -------\n        FileFormat\n        \"\"\"\n        for ext, reader in cls.readers.items():\n            # Skip ambiguous, invalid compression-only extensions added on OSX\n            if ext in Compression.all:\n                continue\n            if fnmatch(path.basename(filename), '*' + ext):\n                return reader(filename)\n\n        raise IOError('No readers for file \"{}\"'.format(filename))\n\n    @classmethod\n    def write(cls, filename, data):\n        return cls.write_file(filename, data)\n\n    @classmethod\n    def write_table_metadata(cls, filename, data):\n\n        def write_file(fn):\n            if all(isinstance(key, str) and isinstance(value, str)\n                   for key, value in data.attributes.items()):\n                with open(fn, 'w', encoding='utf-8') as f:\n                    f.write(\"\\n\".join(\"{}: {}\".format(*kv)\n                                      for kv in data.attributes.items()))\n            else:\n                with open(fn, 'wb') as f:\n                    pickle.dump(data.attributes, f, pickle.HIGHEST_PROTOCOL)\n\n        if isinstance(filename, str):\n            metafile = filename + '.metadata'\n            if getattr(data, 'attributes', None):\n                write_file(metafile)\n            elif path.exists(metafile):\n                remove(metafile)\n\n    @classmethod\n    def set_table_metadata(cls, filename, table):\n        # pylint: disable=bare-except\n        if isinstance(filename, str) and path.exists(filename + '.metadata'):\n            try:\n                with open(filename + '.metadata', 'rb') as f:\n                    table.attributes = pickle.load(f)\n            # Unpickling throws different exceptions, not just UnpickleError\n            except:\n                with open(filename + '.metadata', encoding='utf-8') as f:\n                    table.attributes = OrderedDict(\n                        (k.strip(), v.strip())\n                        for k, v in (line.split(\":\", 1)\n                                     for line in f.readlines()))\n\n    @classmethod\n    def locate(cls, filename, search_dirs=('.',)):\n        \"\"\"Locate a file with given filename that can be opened by one\n        of the available readers.\n\n        Parameters\n        ----------\n        filename : str\n        search_dirs : Iterable[str]\n\n        Returns\n        -------\n        str\n            Absolute path to the file\n        \"\"\"\n        if path.exists(filename):\n            return filename\n\n        for directory in search_dirs:\n            absolute_filename = path.join(directory, filename)\n            if path.exists(absolute_filename):\n                break\n            for ext in cls.readers:\n                if fnmatch(path.basename(filename), '*' + ext):\n                    break\n                # glob uses fnmatch internally\n                matching_files = glob(absolute_filename + ext)\n                if matching_files:\n                    absolute_filename = matching_files[0]\n                    break\n            if path.exists(absolute_filename):\n                break\n        else:\n            absolute_filename = \"\"\n\n        if not path.exists(absolute_filename):\n            raise IOError('File \"{}\" was not found.'.format(filename))\n\n        return absolute_filename\n\n    @staticmethod\n    def open(filename, *args, **kwargs):\n        \"\"\"\n        Format handlers can use this method instead of the builtin ``open()``\n        to transparently (de)compress files if requested (according to\n        `filename` extension). Set ``SUPPORT_COMPRESSED=True`` if you use this.\n        \"\"\"\n        return open_compressed(filename, *args, **kwargs)\n\n    @staticmethod\n    def parse_headers(data):\n        \"\"\"Return (header rows, rest of data) as discerned from `data`\"\"\"\n\n        def is_number(item):\n            try:\n                float(item)\n            except ValueError:\n                return False\n            return True\n        # Second row items are type identifiers\n        def header_test2(items):\n            return all(map(_RE_TYPES.match, items))\n        # Third row items are flags and column attributes (attr=value)\n        def header_test3(items):\n            return all(map(_RE_FLAGS.match, items))\n\n        data = iter(data)\n        header_rows = []\n\n        # Try to parse a three-line header\n        lines = []\n        try:\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n        except StopIteration:\n            lines, data = [], chain(lines, data)\n        if lines:\n            l1, l2, l3 = lines\n            # Three-line header if line 2 & 3 match (1st line can be anything)\n            if header_test2(l2) and header_test3(l3):\n                header_rows = [l1, l2, l3]\n            else:\n                lines, data = [], chain((l1, l2, l3), data)\n\n        # Try to parse a single-line header\n        if not header_rows:\n            try:\n                lines.append(list(next(data)))\n            except StopIteration:\n                pass\n            if lines:\n                # Header if none of the values in line 1 parses as a number\n                if not all(is_number(i) for i in lines[0]):\n                    header_rows = [lines[0]]\n                else:\n                    data = chain(lines, data)\n\n        return header_rows, data\n\n    @classmethod\n    def data_table(cls, data, headers=None):\n        \"\"\"\n        Return Orange.data.Table given rows of `headers` (iterable of iterable)\n        and rows of `data` (iterable of iterable).\n\n        Basically, the idea of subclasses is to produce those two iterables,\n        however they might.\n\n        If `headers` is not provided, the header rows are extracted from `data`,\n        assuming they precede it.\n        \"\"\"\n        if not headers:\n            headers, data = cls.parse_headers(data)\n\n        # Consider various header types (single-row, two-row, three-row, none)\n        if len(headers) == 3:\n            names, types, flags = map(list, headers)\n        else:\n            if len(headers) == 1:\n                HEADER1_FLAG_SEP = '#'\n                # First row format either:\n                #   1) delimited column names\n                #   2) -||- with type and flags prepended, separated by #,\n                #      e.g. d#sex,c#age,cC#IQ\n                _flags, names = zip(*[i.split(HEADER1_FLAG_SEP, 1)\n                                      if HEADER1_FLAG_SEP in i else ('', i)\n                                      for i in headers[0]]\n                                   )\n                names = list(names)\n            elif len(headers) == 2:\n                names, _flags = map(list, headers)\n            else:\n                # Use heuristics for everything\n                names, _flags = [], []\n            types = [''.join(filter(str.isupper, flag)).lower() for flag in _flags]\n            flags = [Flags.join(filter(str.islower, flag)) for flag in _flags]\n\n        # Determine maximum row length\n        rowlen = max(map(len, (names, types, flags)))\n\n        strip = False\n\n        def _equal_length(lst):\n            nonlocal strip\n            if len(lst) > rowlen > 0:\n                lst = lst[:rowlen]\n                strip = True\n            elif len(lst) < rowlen:\n                lst.extend(['']*(rowlen - len(lst)))\n            return lst\n\n        # Ensure all data is of equal width in a column-contiguous array\n        data = [_equal_length([s.strip() for s in row])\n                for row in data if any(row)]\n        data = np.array(data, dtype=object, order='F')\n\n        if strip:\n            warnings.warn(\"Columns with no headers were removed.\")\n\n        # Data may actually be longer than headers were\n        try:\n            rowlen = data.shape[1]\n        except IndexError:\n            pass\n        else:\n            for lst in (names, types, flags):\n                _equal_length(lst)\n\n        NAMEGEN = namegen('Feature ', 1)\n        Xcols, attrs = [], []\n        Mcols, metas = [], []\n        Ycols, clses = [], []\n        Wcols = []\n\n        # Rename variables if necessary\n        # Reusing across files still works if both files have same duplicates\n        name_counts = Counter(names)\n        del name_counts[\"\"]\n        if len(name_counts) != len(names) and name_counts:\n            uses = {name: 0 for name, count in name_counts.items() if count > 1}\n            for i, name in enumerate(names):\n                if name in uses:\n                    uses[name] += 1\n                    names[i] = \"{}_{}\".format(name, uses[name])\n\n        namask = np.empty(data.shape[0], dtype=bool)\n        # Iterate through the columns\n        for col in range(rowlen):\n            flag = Flags(Flags.split(flags[col]))\n            if flag.i:\n                continue\n\n            type_flag = types and types[col].strip()\n            try:\n                orig_values = data[:, col]\n            except IndexError:\n                orig_values = np.array([], dtype=object)\n\n            namask = isnastr(orig_values, out=namask)\n\n            coltype_kwargs = {}\n            valuemap = None\n            values = orig_values\n\n            if type_flag in StringVariable.TYPE_HEADERS:\n                coltype = StringVariable\n                values = orig_values\n            elif type_flag in ContinuousVariable.TYPE_HEADERS:\n                coltype = ContinuousVariable\n                values = np.empty(data.shape[0], dtype=float)\n                try:\n                    np.copyto(values, orig_values, casting=\"unsafe\",\n                              where=~namask)\n                    values[namask] = np.nan\n                except ValueError:\n                    for row, num in enumerate(orig_values):\n                        if not isnastr(num):\n                            try:\n                                float(num)\n                            except ValueError:\n                                break\n                    raise ValueError('Non-continuous value in (1-based) '\n                                     'line {}, column {}'.format(row + len(headers) + 1,\n                                                                 col + 1))\n\n            elif type_flag in TimeVariable.TYPE_HEADERS:\n                coltype = TimeVariable\n                values = np.where(namask, \"\", orig_values)\n            elif (type_flag in DiscreteVariable.TYPE_HEADERS or\n                  _RE_DISCRETE_LIST.match(type_flag)):\n                coltype = DiscreteVariable\n                orig_values = values = np.where(namask, \"\", orig_values)\n                if _RE_DISCRETE_LIST.match(type_flag):\n                    valuemap = Flags.split(type_flag)\n                    coltype_kwargs.update(ordered=True)\n                else:\n                    valuemap = sorted(set(orig_values) - {\"\"})\n            else:\n                # No known type specified, use heuristics\n                valuemap, values, coltype = guess_data_type(orig_values, namask)\n\n            if flag.m or coltype is StringVariable:\n                append_to = (Mcols, metas)\n            elif flag.w:\n                append_to = (Wcols, None)\n            elif flag.c:\n                append_to = (Ycols, clses)\n            else:\n                append_to = (Xcols, attrs)\n\n            cols, domain_vars = append_to\n\n            if domain_vars is not None:\n                var_name = names and names[col]\n                if not var_name:\n                    var_name = next(NAMEGEN)\n\n                values, var = sanitize_variable(\n                    valuemap, values, orig_values, coltype, coltype_kwargs,\n                    name=var_name)\n            else:\n                var = None\n            if domain_vars is not None:\n                var.attributes.update(flag.attributes)\n                domain_vars.append(var)\n\n            if isinstance(values, np.ndarray) and not values.flags.owndata:\n                values = values.copy()  # might view `data` (string columns)\n            cols.append(values)\n\n            try:\n                # allow gc to reclaim memory used by string values\n                data[:, col] = None\n            except IndexError:\n                pass\n\n        domain = Domain(attrs, clses, metas)\n\n        if not data.size:\n            return Table.from_domain(domain, 0)\n\n        X = Y = M = W = None\n        if Xcols:\n            X = np.c_[tuple(Xcols)]\n            assert X.dtype == np.float_\n        else:\n            X = np.empty((data.shape[0], 0), dtype=np.float_)\n        if Ycols:\n            Y = np.c_[tuple(Ycols)]\n            assert Y.dtype == np.float_\n        if Mcols:\n            M = np.c_[tuple(Mcols)].astype(object)\n        if Wcols:\n            W = np.c_[tuple(Wcols)].astype(float)\n\n        table = Table.from_numpy(domain, X, Y, M, W)\n        return table\n\n    @staticmethod\n    def header_names(data):\n        return ['weights'] * data.has_weights() + \\\n               [v.name for v in chain(data.domain.attributes,\n                                      data.domain.class_vars,\n                                      data.domain.metas)]\n\n    @staticmethod\n    def header_types(data):\n        def _vartype(var):\n            if var.is_continuous or var.is_string:\n                return var.TYPE_HEADERS[0]\n            elif var.is_discrete:\n                return Flags.join(var.values) if var.ordered else var.TYPE_HEADERS[0]\n            raise NotImplementedError\n        return ['continuous'] * data.has_weights() + \\\n               [_vartype(v) for v in chain(data.domain.attributes,\n                                           data.domain.class_vars,\n                                           data.domain.metas)]\n\n    @staticmethod\n    def header_flags(data):\n        return list(chain(['weight'] * data.has_weights(),\n                          (Flags.join([flag], *('{}={}'.format(*a)\n                                                for a in sorted(var.attributes.items())))\n                           for flag, var in chain(zip(repeat(''), data.domain.attributes),\n                                                  zip(repeat('class'), data.domain.class_vars),\n                                                  zip(repeat('meta'), data.domain.metas)))))\n\n    @classmethod\n    def write_headers(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        write(cls.header_names(data))\n        write(cls.header_types(data))\n        write(cls.header_flags(data))\n\n    @classmethod\n    def write_data(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        vars = list(chain((ContinuousVariable('_w'),) if data.has_weights() else (),\n                          data.domain.attributes,\n                          data.domain.class_vars,\n                          data.domain.metas))\n\n        def formatter(var):\n            # type: (Variable) -> Callable[[Variable], Any]\n            # Return a column 'formatter' function. The function must return\n            # something that `write` knows how to write\n            if var.is_time:\n                return var.repr_val\n            elif var.is_continuous:\n                return lambda value: \"\" if isnan(value) else value\n            elif var.is_discrete:\n                return lambda value: \"\" if isnan(value) else var.values[int(value)]\n            elif var.is_string:\n                return lambda value: value\n            else:\n                return var.repr_val\n\n        formatters = [formatter(v) for v in vars]\n        for row in zip(data.W if data.W.ndim > 1 else data.W[:, np.newaxis],\n                       data.X,\n                       data.Y if data.Y.ndim > 1 else data.Y[:, np.newaxis],\n                       data.metas):\n            write([fmt(v) for fmt, v in zip(formatters, flatten(row))])\n\n    @classmethod\n    def qualified_name(cls):\n        return cls.__module__ + '.' + cls.__name__\n\n\ndef class_from_qualified_name(format_name):\n    \"\"\" File format class from qualified name. \"\"\"\n    elements = format_name.split(\".\")\n    m = import_module(\".\".join(elements[:-1]))\n    return getattr(m, elements[-1])\n\n\nclass CSVReader(FileFormat):\n    \"\"\"Reader for comma separated files\"\"\"\n\n    EXTENSIONS = ('.csv',)\n    DESCRIPTION = 'Comma-separated values'\n    DELIMITERS = ',;:\\t$ '\n    SUPPORT_COMPRESSED = True\n    SUPPORT_SPARSE_DATA = False\n    PRIORITY = 20\n\n    def read(self):\n        for encoding in (lambda: ('us-ascii', None),                 # fast\n                         lambda: (detect_encoding(self.filename), None),  # precise\n                         lambda: (locale.getpreferredencoding(False), None),\n                         lambda: (sys.getdefaultencoding(), None),   # desperate\n                         lambda: ('utf-8', None),                    # ...\n                         lambda: ('utf-8', 'ignore')):               # fallback\n            encoding, errors = encoding()\n            # Clear the error flag for all except the last check, because\n            # the error of second-to-last check is stored and shown as warning in owfile\n            if errors != 'ignore':\n                error = ''\n            with self.open(self.filename, mode='rt', newline='',\n                           encoding=encoding, errors=errors) as file:\n                # Sniff the CSV dialect (delimiter, quotes, ...)\n                try:\n                    dialect = csv.Sniffer().sniff(\n                        # Take first couple of *complete* lines as sample\n                        ''.join(file.readline() for _ in range(10)),\n                        self.DELIMITERS)\n                    delimiter = dialect.delimiter\n                    quotechar = dialect.quotechar\n                except UnicodeDecodeError as e:\n                    error = e\n                    continue\n                except csv.Error:\n                    delimiter = self.DELIMITERS[0]\n                    quotechar = csv.excel.quotechar\n\n                file.seek(0)\n                try:\n                    reader = csv.reader(\n                        file, delimiter=delimiter, quotechar=quotechar,\n                        skipinitialspace=True,\n                    )\n                    data = self.data_table(reader)\n\n                    # TODO: Name can be set unconditionally when/if\n                    # self.filename will always be a string with the file name.\n                    # Currently, some tests pass StringIO instead of\n                    # the file name to a reader.\n                    if isinstance(self.filename, str):\n                        data.name = path.splitext(\n                            path.split(self.filename)[-1])[0]\n                    if error and isinstance(error, UnicodeDecodeError):\n                        pos, endpos = error.args[2], error.args[3]\n                        warning = ('Skipped invalid byte(s) in position '\n                                   '{}{}').format(pos,\n                                                  ('-' + str(endpos)) if (endpos - pos) > 1 else '')\n                        warnings.warn(warning)\n                    self.set_table_metadata(self.filename, data)\n                    return data\n                except Exception as e:\n                    error = e\n                    continue\n        raise ValueError('Cannot parse dataset {}: {}'.format(self.filename, error)) from error\n\n    @classmethod\n    def write_file(cls, filename, data):\n        with cls.open(filename, mode='wt', newline='', encoding='utf-8') as file:\n            writer = csv.writer(file, delimiter=cls.DELIMITERS[0])\n            cls.write_headers(writer.writerow, data)\n            cls.write_data(writer.writerow, data)\n        cls.write_table_metadata(filename, data)\n\n\nclass TabReader(CSVReader):\n    \"\"\"Reader for tab separated files\"\"\"\n    EXTENSIONS = ('.tab', '.tsv')\n    DESCRIPTION = 'Tab-separated values'\n    DELIMITERS = '\\t'\n    PRIORITY = 10\n\n\nclass PickleReader(FileFormat):\n    \"\"\"Reader for pickled Table objects\"\"\"\n    EXTENSIONS = ('.pkl', '.pickle')\n    DESCRIPTION = 'Pickled Orange data'\n    SUPPORT_COMPRESSED = True\n    SUPPORT_SPARSE_DATA = True\n\n    def read(self):\n        with self.open(self.filename, 'rb') as f:\n            table = pickle.load(f)\n            if not isinstance(table, Table):\n                raise TypeError(\"file does not contain a data table\")\n            else:\n                return table\n\n    @classmethod\n    def write_file(cls, filename, data):\n        with cls.open(filename, 'wb') as f:\n            pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)\n\n\nclass BasketReader(FileFormat):\n    \"\"\"Reader for basket (sparse) files\"\"\"\n    EXTENSIONS = ('.basket', '.bsk')\n    DESCRIPTION = 'Basket file'\n    SUPPORT_SPARSE_DATA = True\n\n    def read(self):\n        def constr_vars(inds):\n            if inds:\n                return [ContinuousVariable(x.decode(\"utf-8\")) for _, x in\n                        sorted((ind, name) for name, ind in inds.items())]\n\n        X, Y, metas, attr_indices, class_indices, meta_indices = \\\n            _io.sparse_read_float(self.filename.encode(sys.getdefaultencoding()))\n\n        attrs = constr_vars(attr_indices)\n        classes = constr_vars(class_indices)\n        meta_attrs = constr_vars(meta_indices)\n        domain = Domain(attrs, classes, meta_attrs)\n        table = Table.from_numpy(\n            domain, attrs and X, classes and Y, metas and meta_attrs)\n        table.name = path.splitext(path.split(self.filename)[-1])[0]\n        return table\n\n\nclass ExcelReader(FileFormat):\n    \"\"\"Reader for excel files\"\"\"\n    EXTENSIONS = ('.xls', '.xlsx')\n    DESCRIPTION = 'Mircosoft Excel spreadsheet'\n    SUPPORT_SPARSE_DATA = False\n\n    def __init__(self, filename):\n        super().__init__(filename)\n\n        from xlrd import open_workbook\n        self.workbook = open_workbook(self.filename)\n\n    @property\n    @lru_cache(1)\n    def sheets(self):\n        return self.workbook.sheet_names()\n\n    def read(self):\n        import xlrd\n        wb = xlrd.open_workbook(self.filename, on_demand=True)\n        if self.sheet:\n            ss = wb.sheet_by_name(self.sheet)\n        else:\n            ss = wb.sheet_by_index(0)\n        try:\n            first_row = next(i for i in range(ss.nrows) if any(ss.row_values(i)))\n            first_col = next(i for i in range(ss.ncols) if ss.cell_value(first_row, i))\n            row_len = ss.row_len(first_row)\n            cells = filter(any,\n                           [[str(ss.cell_value(row, col)) if col < ss.row_len(row) else ''\n                             for col in range(first_col, row_len)]\n                            for row in range(first_row, ss.nrows)])\n            table = self.data_table(cells)\n            table.name = path.splitext(path.split(self.filename)[-1])[0]\n            if self.sheet:\n                table.name = '-'.join((table.name, self.sheet))\n        except Exception:\n            raise IOError(\"Couldn't load spreadsheet from \" + self.filename)\n        return table\n\n\nclass DotReader(FileFormat):\n    \"\"\"Writer for dot (graph) files\"\"\"\n    EXTENSIONS = ('.dot', '.gv')\n    DESCRIPTION = 'Dot graph description'\n    SUPPORT_COMPRESSED = True\n    SUPPORT_SPARSE_DATA = False\n\n    @classmethod\n    def write_graph(cls, filename, graph):\n        from sklearn import tree\n        tree.export_graphviz(graph, out_file=cls.open(filename, 'wt'))\n\n    @classmethod\n    def write(cls, filename, tree):\n        if type(tree) == dict:\n            tree = tree['tree']\n        cls.write_graph(filename, tree)\n\n\nclass UrlReader(FileFormat):\n    def __init__(self, filename):\n        filename = filename.strip()\n        if not urlparse(filename).scheme:\n            filename = 'http://' + filename\n        super().__init__(filename)\n\n    @staticmethod\n    def urlopen(url):\n        req = Request(\n            url,\n            # Avoid 403 error with servers that dislike scrapers\n            headers={'User-Agent': 'Mozilla/5.0 (X11; Linux) Gecko/20100101 Firefox/'})\n        return urlopen(req, timeout=10)\n\n    def read(self):\n        self.filename = self._trim(self._resolve_redirects(self.filename))\n        with contextlib.closing(self.urlopen(self.filename)) as response:\n            name = self._suggest_filename(response.headers['content-disposition'])\n            with NamedTemporaryFile(suffix=name, delete=False) as f:\n                f.write(response.read())\n                # delete=False is a workaround for https://bugs.python.org/issue14243\n\n            reader = self.get_reader(f.name)\n            data = reader.read()\n            remove(f.name)\n        # Override name set in from_file() to avoid holding the temp prefix\n        data.name = path.splitext(name)[0]\n        data.origin = self.filename\n        return data\n\n    def _resolve_redirects(self, url):\n        # Resolve (potential) redirects to a final URL\n        with contextlib.closing(self.urlopen(url)) as response:\n            return response.url\n\n    @classmethod\n    def _trim(cls, url):\n        URL_TRIMMERS = (\n            cls._trim_googlesheet,\n            cls._trim_dropbox,\n        )\n        for trim in URL_TRIMMERS:\n            try:\n                url = trim(url)\n            except ValueError:\n                continue\n            else:\n                break\n        return url\n\n    @staticmethod\n    def _trim_googlesheet(url):\n        match = re.match(r'(?:https?://)?(?:www\\.)?'\n                         r'docs\\.google\\.com/spreadsheets/d/'\n                         r'(?P<workbook_id>[-\\w_]+)'\n                         r'(?:/.*?gid=(?P<sheet_id>\\d+).*|.*)?',\n                         url, re.IGNORECASE)\n        try:\n            workbook, sheet = match.group('workbook_id'), match.group('sheet_id')\n            if not workbook:\n                raise ValueError\n        except (AttributeError, ValueError):\n            raise ValueError\n        url = 'https://docs.google.com/spreadsheets/d/{}/export?format=tsv'.format(workbook)\n        if sheet:\n            url += '&gid=' + sheet\n        return url\n\n    @staticmethod\n    def _trim_dropbox(url):\n        parts = urlsplit(url)\n        if not parts.netloc.endswith('dropbox.com'):\n            raise ValueError\n        return urlunsplit(parts._replace(query='dl=1'))\n\n    def _suggest_filename(self, content_disposition):\n        default_name = re.sub(r'[\\\\:/]', '_', urlparse(self.filename).path)\n\n        # See https://tools.ietf.org/html/rfc6266#section-4.1\n        matches = re.findall(r\"filename\\*?=(?:\\\"|.{0,10}?'[^']*')([^\\\"]+)\",\n                             content_disposition or '')\n        return urlunquote(matches[-1]) if matches else default_name\n",
            "header": {
                "file": {
                    "path": "Orange/data/io.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "77def811e",
                "new_commit_hash": "00ab17f4a",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/Orange/data/io.py"
            },
            "plus_file": {
                "path": "b/Orange/data/io.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 618,
                            "length": 8
                        },
                        "new_range": {
                            "start": 618,
                            "length": 15
                        },
                        "section": "class FileFormat(metaclass=FileFormatMeta):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        # Determine maximum row length",
                                "type": "context"
                            },
                            {
                                "content": "        rowlen = max(map(len, (names, types, flags)))",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        strip = False",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        def _equal_length(lst):",
                                "type": "context"
                            },
                            {
                                "content": "            lst.extend(['']*(rowlen - len(lst)))",
                                "type": "deleted"
                            },
                            {
                                "content": "            nonlocal strip",
                                "type": "added"
                            },
                            {
                                "content": "            if len(lst) > rowlen > 0:",
                                "type": "added"
                            },
                            {
                                "content": "                lst = lst[:rowlen]",
                                "type": "added"
                            },
                            {
                                "content": "                strip = True",
                                "type": "added"
                            },
                            {
                                "content": "            elif len(lst) < rowlen:",
                                "type": "added"
                            },
                            {
                                "content": "                lst.extend(['']*(rowlen - len(lst)))",
                                "type": "added"
                            },
                            {
                                "content": "            return lst",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        # Ensure all data is of equal width in a column-contiguous array",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "Orange/data/io.py",
                            "type": "method",
                            "name": "FileFormat.data_table",
                            "content": "class FileFormat(metaclass=FileFormatMeta):\n    \"\"\"\n    Subclasses set the following attributes and override the following methods:\n\n        EXTENSIONS = ('.ext1', '.ext2', ...)\n        DESCRIPTION = 'human-readable file format description'\n        SUPPORT_COMPRESSED = False\n        SUPPORT_SPARSE_DATA = False\n\n        def read(self):\n            ...  # load headers, data, ...\n            return self.data_table(data, headers)\n\n        @classmethod\n        def write_file(cls, filename, data):\n            ...\n            self.write_headers(writer.write, data)\n            writer.writerows(data)\n\n    Wrapper FileFormat.data_table() returns Orange.data.Table from `data`\n    iterable (list (rows) of lists of values (cols)).\n    \"\"\"\n\n    # Priority when multiple formats support the same extension. Also\n    # the sort order in file open/save combo boxes. Lower is better.\n    PRIORITY = 10000\n\n    def __init__(self, filename):\n        \"\"\"\n        Parameters\n        ----------\n        filename : str\n            name of the file to open\n        \"\"\"\n        self.filename = filename\n        self.sheet = None\n\n    @property\n    def sheets(self):\n        \"\"\"FileFormats with a notion of sheets should override this property\n        to return a list of sheet names in the file.\n\n        Returns\n        -------\n        a list of sheet names\n        \"\"\"\n        return ()\n\n    def select_sheet(self, sheet):\n        \"\"\"Select sheet to be read\n\n        Parameters\n        ----------\n        sheet : str\n            sheet name\n        \"\"\"\n        self.sheet = sheet\n\n    @classmethod\n    def get_reader(cls, filename):\n        \"\"\"Return reader instance that can be used to read the file\n\n        Parameters\n        ----------\n        filename : str\n\n        Returns\n        -------\n        FileFormat\n        \"\"\"\n        for ext, reader in cls.readers.items():\n            # Skip ambiguous, invalid compression-only extensions added on OSX\n            if ext in Compression.all:\n                continue\n            if fnmatch(path.basename(filename), '*' + ext):\n                return reader(filename)\n\n        raise IOError('No readers for file \"{}\"'.format(filename))\n\n    @classmethod\n    def write(cls, filename, data):\n        return cls.write_file(filename, data)\n\n    @classmethod\n    def write_table_metadata(cls, filename, data):\n\n        def write_file(fn):\n            if all(isinstance(key, str) and isinstance(value, str)\n                   for key, value in data.attributes.items()):\n                with open(fn, 'w', encoding='utf-8') as f:\n                    f.write(\"\\n\".join(\"{}: {}\".format(*kv)\n                                      for kv in data.attributes.items()))\n            else:\n                with open(fn, 'wb') as f:\n                    pickle.dump(data.attributes, f, pickle.HIGHEST_PROTOCOL)\n\n        if isinstance(filename, str):\n            metafile = filename + '.metadata'\n            if getattr(data, 'attributes', None):\n                write_file(metafile)\n            elif path.exists(metafile):\n                remove(metafile)\n\n    @classmethod\n    def set_table_metadata(cls, filename, table):\n        # pylint: disable=bare-except\n        if isinstance(filename, str) and path.exists(filename + '.metadata'):\n            try:\n                with open(filename + '.metadata', 'rb') as f:\n                    table.attributes = pickle.load(f)\n            # Unpickling throws different exceptions, not just UnpickleError\n            except:\n                with open(filename + '.metadata', encoding='utf-8') as f:\n                    table.attributes = OrderedDict(\n                        (k.strip(), v.strip())\n                        for k, v in (line.split(\":\", 1)\n                                     for line in f.readlines()))\n\n    @classmethod\n    def locate(cls, filename, search_dirs=('.',)):\n        \"\"\"Locate a file with given filename that can be opened by one\n        of the available readers.\n\n        Parameters\n        ----------\n        filename : str\n        search_dirs : Iterable[str]\n\n        Returns\n        -------\n        str\n            Absolute path to the file\n        \"\"\"\n        if path.exists(filename):\n            return filename\n\n        for directory in search_dirs:\n            absolute_filename = path.join(directory, filename)\n            if path.exists(absolute_filename):\n                break\n            for ext in cls.readers:\n                if fnmatch(path.basename(filename), '*' + ext):\n                    break\n                # glob uses fnmatch internally\n                matching_files = glob(absolute_filename + ext)\n                if matching_files:\n                    absolute_filename = matching_files[0]\n                    break\n            if path.exists(absolute_filename):\n                break\n        else:\n            absolute_filename = \"\"\n\n        if not path.exists(absolute_filename):\n            raise IOError('File \"{}\" was not found.'.format(filename))\n\n        return absolute_filename\n\n    @staticmethod\n    def open(filename, *args, **kwargs):\n        \"\"\"\n        Format handlers can use this method instead of the builtin ``open()``\n        to transparently (de)compress files if requested (according to\n        `filename` extension). Set ``SUPPORT_COMPRESSED=True`` if you use this.\n        \"\"\"\n        return open_compressed(filename, *args, **kwargs)\n\n    @staticmethod\n    def parse_headers(data):\n        \"\"\"Return (header rows, rest of data) as discerned from `data`\"\"\"\n\n        def is_number(item):\n            try:\n                float(item)\n            except ValueError:\n                return False\n            return True\n        # Second row items are type identifiers\n        def header_test2(items):\n            return all(map(_RE_TYPES.match, items))\n        # Third row items are flags and column attributes (attr=value)\n        def header_test3(items):\n            return all(map(_RE_FLAGS.match, items))\n\n        data = iter(data)\n        header_rows = []\n\n        # Try to parse a three-line header\n        lines = []\n        try:\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n        except StopIteration:\n            lines, data = [], chain(lines, data)\n        if lines:\n            l1, l2, l3 = lines\n            # Three-line header if line 2 & 3 match (1st line can be anything)\n            if header_test2(l2) and header_test3(l3):\n                header_rows = [l1, l2, l3]\n            else:\n                lines, data = [], chain((l1, l2, l3), data)\n\n        # Try to parse a single-line header\n        if not header_rows:\n            try:\n                lines.append(list(next(data)))\n            except StopIteration:\n                pass\n            if lines:\n                # Header if none of the values in line 1 parses as a number\n                if not all(is_number(i) for i in lines[0]):\n                    header_rows = [lines[0]]\n                else:\n                    data = chain(lines, data)\n\n        return header_rows, data\n\n    @classmethod\n    def data_table(cls, data, headers=None):\n        \"\"\"\n        Return Orange.data.Table given rows of `headers` (iterable of iterable)\n        and rows of `data` (iterable of iterable).\n\n        Basically, the idea of subclasses is to produce those two iterables,\n        however they might.\n\n        If `headers` is not provided, the header rows are extracted from `data`,\n        assuming they precede it.\n        \"\"\"\n        if not headers:\n            headers, data = cls.parse_headers(data)\n\n        # Consider various header types (single-row, two-row, three-row, none)\n        if len(headers) == 3:\n            names, types, flags = map(list, headers)\n        else:\n            if len(headers) == 1:\n                HEADER1_FLAG_SEP = '#'\n                # First row format either:\n                #   1) delimited column names\n                #   2) -||- with type and flags prepended, separated by #,\n                #      e.g. d#sex,c#age,cC#IQ\n                _flags, names = zip(*[i.split(HEADER1_FLAG_SEP, 1)\n                                      if HEADER1_FLAG_SEP in i else ('', i)\n                                      for i in headers[0]]\n                                   )\n                names = list(names)\n            elif len(headers) == 2:\n                names, _flags = map(list, headers)\n            else:\n                # Use heuristics for everything\n                names, _flags = [], []\n            types = [''.join(filter(str.isupper, flag)).lower() for flag in _flags]\n            flags = [Flags.join(filter(str.islower, flag)) for flag in _flags]\n\n        # Determine maximum row length\n        rowlen = max(map(len, (names, types, flags)))\n\n        def _equal_length(lst):\n            lst.extend(['']*(rowlen - len(lst)))\n            return lst\n\n        # Ensure all data is of equal width in a column-contiguous array\n        data = [_equal_length([s.strip() for s in row])\n                for row in data if any(row)]\n        data = np.array(data, dtype=object, order='F')\n\n        # Data may actually be longer than headers were\n        try:\n            rowlen = data.shape[1]\n        except IndexError:\n            pass\n        else:\n            for lst in (names, types, flags):\n                _equal_length(lst)\n\n        NAMEGEN = namegen('Feature ', 1)\n        Xcols, attrs = [], []\n        Mcols, metas = [], []\n        Ycols, clses = [], []\n        Wcols = []\n\n        # Rename variables if necessary\n        # Reusing across files still works if both files have same duplicates\n        name_counts = Counter(names)\n        del name_counts[\"\"]\n        if len(name_counts) != len(names) and name_counts:\n            uses = {name: 0 for name, count in name_counts.items() if count > 1}\n            for i, name in enumerate(names):\n                if name in uses:\n                    uses[name] += 1\n                    names[i] = \"{}_{}\".format(name, uses[name])\n\n        namask = np.empty(data.shape[0], dtype=bool)\n        # Iterate through the columns\n        for col in range(rowlen):\n            flag = Flags(Flags.split(flags[col]))\n            if flag.i:\n                continue\n\n            type_flag = types and types[col].strip()\n            try:\n                orig_values = data[:, col]\n            except IndexError:\n                orig_values = np.array([], dtype=object)\n\n            namask = isnastr(orig_values, out=namask)\n\n            coltype_kwargs = {}\n            valuemap = None\n            values = orig_values\n\n            if type_flag in StringVariable.TYPE_HEADERS:\n                coltype = StringVariable\n                values = orig_values\n            elif type_flag in ContinuousVariable.TYPE_HEADERS:\n                coltype = ContinuousVariable\n                values = np.empty(data.shape[0], dtype=float)\n                try:\n                    np.copyto(values, orig_values, casting=\"unsafe\",\n                              where=~namask)\n                    values[namask] = np.nan\n                except ValueError:\n                    for row, num in enumerate(orig_values):\n                        if not isnastr(num):\n                            try:\n                                float(num)\n                            except ValueError:\n                                break\n                    raise ValueError('Non-continuous value in (1-based) '\n                                     'line {}, column {}'.format(row + len(headers) + 1,\n                                                                 col + 1))\n\n            elif type_flag in TimeVariable.TYPE_HEADERS:\n                coltype = TimeVariable\n                values = np.where(namask, \"\", orig_values)\n            elif (type_flag in DiscreteVariable.TYPE_HEADERS or\n                  _RE_DISCRETE_LIST.match(type_flag)):\n                coltype = DiscreteVariable\n                orig_values = values = np.where(namask, \"\", orig_values)\n                if _RE_DISCRETE_LIST.match(type_flag):\n                    valuemap = Flags.split(type_flag)\n                    coltype_kwargs.update(ordered=True)\n                else:\n                    valuemap = sorted(set(orig_values) - {\"\"})\n            else:\n                # No known type specified, use heuristics\n                valuemap, values, coltype = guess_data_type(orig_values, namask)\n\n            if flag.m or coltype is StringVariable:\n                append_to = (Mcols, metas)\n            elif flag.w:\n                append_to = (Wcols, None)\n            elif flag.c:\n                append_to = (Ycols, clses)\n            else:\n                append_to = (Xcols, attrs)\n\n            cols, domain_vars = append_to\n\n            if domain_vars is not None:\n                var_name = names and names[col]\n                if not var_name:\n                    var_name = next(NAMEGEN)\n\n                values, var = sanitize_variable(\n                    valuemap, values, orig_values, coltype, coltype_kwargs,\n                    name=var_name)\n            else:\n                var = None\n            if domain_vars is not None:\n                var.attributes.update(flag.attributes)\n                domain_vars.append(var)\n\n            if isinstance(values, np.ndarray) and not values.flags.owndata:\n                values = values.copy()  # might view `data` (string columns)\n            cols.append(values)\n\n            try:\n                # allow gc to reclaim memory used by string values\n                data[:, col] = None\n            except IndexError:\n                pass\n\n        domain = Domain(attrs, clses, metas)\n\n        if not data.size:\n            return Table.from_domain(domain, 0)\n\n        X = Y = M = W = None\n        if Xcols:\n            X = np.c_[tuple(Xcols)]\n            assert X.dtype == np.float_\n        else:\n            X = np.empty((data.shape[0], 0), dtype=np.float_)\n        if Ycols:\n            Y = np.c_[tuple(Ycols)]\n            assert Y.dtype == np.float_\n        if Mcols:\n            M = np.c_[tuple(Mcols)].astype(object)\n        if Wcols:\n            W = np.c_[tuple(Wcols)].astype(float)\n\n        table = Table.from_numpy(domain, X, Y, M, W)\n        return table\n\n    @staticmethod\n    def header_names(data):\n        return ['weights'] * data.has_weights() + \\\n               [v.name for v in chain(data.domain.attributes,\n                                      data.domain.class_vars,\n                                      data.domain.metas)]\n\n    @staticmethod\n    def header_types(data):\n        def _vartype(var):\n            if var.is_continuous or var.is_string:\n                return var.TYPE_HEADERS[0]\n            elif var.is_discrete:\n                return Flags.join(var.values) if var.ordered else var.TYPE_HEADERS[0]\n            raise NotImplementedError\n        return ['continuous'] * data.has_weights() + \\\n               [_vartype(v) for v in chain(data.domain.attributes,\n                                           data.domain.class_vars,\n                                           data.domain.metas)]\n\n    @staticmethod\n    def header_flags(data):\n        return list(chain(['weight'] * data.has_weights(),\n                          (Flags.join([flag], *('{}={}'.format(*a)\n                                                for a in sorted(var.attributes.items())))\n                           for flag, var in chain(zip(repeat(''), data.domain.attributes),\n                                                  zip(repeat('class'), data.domain.class_vars),\n                                                  zip(repeat('meta'), data.domain.metas)))))\n\n    @classmethod\n    def write_headers(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        write(cls.header_names(data))\n        write(cls.header_types(data))\n        write(cls.header_flags(data))\n\n    @classmethod\n    def write_data(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        vars = list(chain((ContinuousVariable('_w'),) if data.has_weights() else (),\n                          data.domain.attributes,\n                          data.domain.class_vars,\n                          data.domain.metas))\n\n        def formatter(var):\n            # type: (Variable) -> Callable[[Variable], Any]\n            # Return a column 'formatter' function. The function must return\n            # something that `write` knows how to write\n            if var.is_time:\n                return var.repr_val\n            elif var.is_continuous:\n                return lambda value: \"\" if isnan(value) else value\n            elif var.is_discrete:\n                return lambda value: \"\" if isnan(value) else var.values[int(value)]\n            elif var.is_string:\n                return lambda value: value\n            else:\n                return var.repr_val\n\n        formatters = [formatter(v) for v in vars]\n        for row in zip(data.W if data.W.ndim > 1 else data.W[:, np.newaxis],\n                       data.X,\n                       data.Y if data.Y.ndim > 1 else data.Y[:, np.newaxis],\n                       data.metas):\n            write([fmt(v) for fmt, v in zip(formatters, flatten(row))])\n\n    @classmethod\n    def qualified_name(cls):\n        return cls.__module__ + '.' + cls.__name__",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 581,
                            "end_lineno": 767,
                            "parent": {
                                "file_name": "Orange/data/io.py",
                                "type": "class",
                                "name": "FileFormat",
                                "content": "class FileFormat(metaclass=FileFormatMeta):\n    \"\"\"\n    Subclasses set the following attributes and override the following methods:\n\n        EXTENSIONS = ('.ext1', '.ext2', ...)\n        DESCRIPTION = 'human-readable file format description'\n        SUPPORT_COMPRESSED = False\n        SUPPORT_SPARSE_DATA = False\n\n        def read(self):\n            ...  # load headers, data, ...\n            return self.data_table(data, headers)\n\n        @classmethod\n        def write_file(cls, filename, data):\n            ...\n            self.write_headers(writer.write, data)\n            writer.writerows(data)\n\n    Wrapper FileFormat.data_table() returns Orange.data.Table from `data`\n    iterable (list (rows) of lists of values (cols)).\n    \"\"\"\n\n    # Priority when multiple formats support the same extension. Also\n    # the sort order in file open/save combo boxes. Lower is better.\n    PRIORITY = 10000\n\n    def __init__(self, filename):\n        \"\"\"\n        Parameters\n        ----------\n        filename : str\n            name of the file to open\n        \"\"\"\n        self.filename = filename\n        self.sheet = None\n\n    @property\n    def sheets(self):\n        \"\"\"FileFormats with a notion of sheets should override this property\n        to return a list of sheet names in the file.\n\n        Returns\n        -------\n        a list of sheet names\n        \"\"\"\n        return ()\n\n    def select_sheet(self, sheet):\n        \"\"\"Select sheet to be read\n\n        Parameters\n        ----------\n        sheet : str\n            sheet name\n        \"\"\"\n        self.sheet = sheet\n\n    @classmethod\n    def get_reader(cls, filename):\n        \"\"\"Return reader instance that can be used to read the file\n\n        Parameters\n        ----------\n        filename : str\n\n        Returns\n        -------\n        FileFormat\n        \"\"\"\n        for ext, reader in cls.readers.items():\n            # Skip ambiguous, invalid compression-only extensions added on OSX\n            if ext in Compression.all:\n                continue\n            if fnmatch(path.basename(filename), '*' + ext):\n                return reader(filename)\n\n        raise IOError('No readers for file \"{}\"'.format(filename))\n\n    @classmethod\n    def write(cls, filename, data):\n        return cls.write_file(filename, data)\n\n    @classmethod\n    def write_table_metadata(cls, filename, data):\n\n        def write_file(fn):\n            if all(isinstance(key, str) and isinstance(value, str)\n                   for key, value in data.attributes.items()):\n                with open(fn, 'w', encoding='utf-8') as f:\n                    f.write(\"\\n\".join(\"{}: {}\".format(*kv)\n                                      for kv in data.attributes.items()))\n            else:\n                with open(fn, 'wb') as f:\n                    pickle.dump(data.attributes, f, pickle.HIGHEST_PROTOCOL)\n\n        if isinstance(filename, str):\n            metafile = filename + '.metadata'\n            if getattr(data, 'attributes', None):\n                write_file(metafile)\n            elif path.exists(metafile):\n                remove(metafile)\n\n    @classmethod\n    def set_table_metadata(cls, filename, table):\n        # pylint: disable=bare-except\n        if isinstance(filename, str) and path.exists(filename + '.metadata'):\n            try:\n                with open(filename + '.metadata', 'rb') as f:\n                    table.attributes = pickle.load(f)\n            # Unpickling throws different exceptions, not just UnpickleError\n            except:\n                with open(filename + '.metadata', encoding='utf-8') as f:\n                    table.attributes = OrderedDict(\n                        (k.strip(), v.strip())\n                        for k, v in (line.split(\":\", 1)\n                                     for line in f.readlines()))\n\n    @classmethod\n    def locate(cls, filename, search_dirs=('.',)):\n        \"\"\"Locate a file with given filename that can be opened by one\n        of the available readers.\n\n        Parameters\n        ----------\n        filename : str\n        search_dirs : Iterable[str]\n\n        Returns\n        -------\n        str\n            Absolute path to the file\n        \"\"\"\n        if path.exists(filename):\n            return filename\n\n        for directory in search_dirs:\n            absolute_filename = path.join(directory, filename)\n            if path.exists(absolute_filename):\n                break\n            for ext in cls.readers:\n                if fnmatch(path.basename(filename), '*' + ext):\n                    break\n                # glob uses fnmatch internally\n                matching_files = glob(absolute_filename + ext)\n                if matching_files:\n                    absolute_filename = matching_files[0]\n                    break\n            if path.exists(absolute_filename):\n                break\n        else:\n            absolute_filename = \"\"\n\n        if not path.exists(absolute_filename):\n            raise IOError('File \"{}\" was not found.'.format(filename))\n\n        return absolute_filename\n\n    @staticmethod\n    def open(filename, *args, **kwargs):\n        \"\"\"\n        Format handlers can use this method instead of the builtin ``open()``\n        to transparently (de)compress files if requested (according to\n        `filename` extension). Set ``SUPPORT_COMPRESSED=True`` if you use this.\n        \"\"\"\n        return open_compressed(filename, *args, **kwargs)\n\n    @staticmethod\n    def parse_headers(data):\n        \"\"\"Return (header rows, rest of data) as discerned from `data`\"\"\"\n\n        def is_number(item):\n            try:\n                float(item)\n            except ValueError:\n                return False\n            return True\n        # Second row items are type identifiers\n        def header_test2(items):\n            return all(map(_RE_TYPES.match, items))\n        # Third row items are flags and column attributes (attr=value)\n        def header_test3(items):\n            return all(map(_RE_FLAGS.match, items))\n\n        data = iter(data)\n        header_rows = []\n\n        # Try to parse a three-line header\n        lines = []\n        try:\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n        except StopIteration:\n            lines, data = [], chain(lines, data)\n        if lines:\n            l1, l2, l3 = lines\n            # Three-line header if line 2 & 3 match (1st line can be anything)\n            if header_test2(l2) and header_test3(l3):\n                header_rows = [l1, l2, l3]\n            else:\n                lines, data = [], chain((l1, l2, l3), data)\n\n        # Try to parse a single-line header\n        if not header_rows:\n            try:\n                lines.append(list(next(data)))\n            except StopIteration:\n                pass\n            if lines:\n                # Header if none of the values in line 1 parses as a number\n                if not all(is_number(i) for i in lines[0]):\n                    header_rows = [lines[0]]\n                else:\n                    data = chain(lines, data)\n\n        return header_rows, data\n\n    @classmethod\n    def data_table(cls, data, headers=None):\n        \"\"\"\n        Return Orange.data.Table given rows of `headers` (iterable of iterable)\n        and rows of `data` (iterable of iterable).\n\n        Basically, the idea of subclasses is to produce those two iterables,\n        however they might.\n\n        If `headers` is not provided, the header rows are extracted from `data`,\n        assuming they precede it.\n        \"\"\"\n        if not headers:\n            headers, data = cls.parse_headers(data)\n\n        # Consider various header types (single-row, two-row, three-row, none)\n        if len(headers) == 3:\n            names, types, flags = map(list, headers)\n        else:\n            if len(headers) == 1:\n                HEADER1_FLAG_SEP = '#'\n                # First row format either:\n                #   1) delimited column names\n                #   2) -||- with type and flags prepended, separated by #,\n                #      e.g. d#sex,c#age,cC#IQ\n                _flags, names = zip(*[i.split(HEADER1_FLAG_SEP, 1)\n                                      if HEADER1_FLAG_SEP in i else ('', i)\n                                      for i in headers[0]]\n                                   )\n                names = list(names)\n            elif len(headers) == 2:\n                names, _flags = map(list, headers)\n            else:\n                # Use heuristics for everything\n                names, _flags = [], []\n            types = [''.join(filter(str.isupper, flag)).lower() for flag in _flags]\n            flags = [Flags.join(filter(str.islower, flag)) for flag in _flags]\n\n        # Determine maximum row length\n        rowlen = max(map(len, (names, types, flags)))\n\n        def _equal_length(lst):\n            lst.extend(['']*(rowlen - len(lst)))\n            return lst\n\n        # Ensure all data is of equal width in a column-contiguous array\n        data = [_equal_length([s.strip() for s in row])\n                for row in data if any(row)]\n        data = np.array(data, dtype=object, order='F')\n\n        # Data may actually be longer than headers were\n        try:\n            rowlen = data.shape[1]\n        except IndexError:\n            pass\n        else:\n            for lst in (names, types, flags):\n                _equal_length(lst)\n\n        NAMEGEN = namegen('Feature ', 1)\n        Xcols, attrs = [], []\n        Mcols, metas = [], []\n        Ycols, clses = [], []\n        Wcols = []\n\n        # Rename variables if necessary\n        # Reusing across files still works if both files have same duplicates\n        name_counts = Counter(names)\n        del name_counts[\"\"]\n        if len(name_counts) != len(names) and name_counts:\n            uses = {name: 0 for name, count in name_counts.items() if count > 1}\n            for i, name in enumerate(names):\n                if name in uses:\n                    uses[name] += 1\n                    names[i] = \"{}_{}\".format(name, uses[name])\n\n        namask = np.empty(data.shape[0], dtype=bool)\n        # Iterate through the columns\n        for col in range(rowlen):\n            flag = Flags(Flags.split(flags[col]))\n            if flag.i:\n                continue\n\n            type_flag = types and types[col].strip()\n            try:\n                orig_values = data[:, col]\n            except IndexError:\n                orig_values = np.array([], dtype=object)\n\n            namask = isnastr(orig_values, out=namask)\n\n            coltype_kwargs = {}\n            valuemap = None\n            values = orig_values\n\n            if type_flag in StringVariable.TYPE_HEADERS:\n                coltype = StringVariable\n                values = orig_values\n            elif type_flag in ContinuousVariable.TYPE_HEADERS:\n                coltype = ContinuousVariable\n                values = np.empty(data.shape[0], dtype=float)\n                try:\n                    np.copyto(values, orig_values, casting=\"unsafe\",\n                              where=~namask)\n                    values[namask] = np.nan\n                except ValueError:\n                    for row, num in enumerate(orig_values):\n                        if not isnastr(num):\n                            try:\n                                float(num)\n                            except ValueError:\n                                break\n                    raise ValueError('Non-continuous value in (1-based) '\n                                     'line {}, column {}'.format(row + len(headers) + 1,\n                                                                 col + 1))\n\n            elif type_flag in TimeVariable.TYPE_HEADERS:\n                coltype = TimeVariable\n                values = np.where(namask, \"\", orig_values)\n            elif (type_flag in DiscreteVariable.TYPE_HEADERS or\n                  _RE_DISCRETE_LIST.match(type_flag)):\n                coltype = DiscreteVariable\n                orig_values = values = np.where(namask, \"\", orig_values)\n                if _RE_DISCRETE_LIST.match(type_flag):\n                    valuemap = Flags.split(type_flag)\n                    coltype_kwargs.update(ordered=True)\n                else:\n                    valuemap = sorted(set(orig_values) - {\"\"})\n            else:\n                # No known type specified, use heuristics\n                valuemap, values, coltype = guess_data_type(orig_values, namask)\n\n            if flag.m or coltype is StringVariable:\n                append_to = (Mcols, metas)\n            elif flag.w:\n                append_to = (Wcols, None)\n            elif flag.c:\n                append_to = (Ycols, clses)\n            else:\n                append_to = (Xcols, attrs)\n\n            cols, domain_vars = append_to\n\n            if domain_vars is not None:\n                var_name = names and names[col]\n                if not var_name:\n                    var_name = next(NAMEGEN)\n\n                values, var = sanitize_variable(\n                    valuemap, values, orig_values, coltype, coltype_kwargs,\n                    name=var_name)\n            else:\n                var = None\n            if domain_vars is not None:\n                var.attributes.update(flag.attributes)\n                domain_vars.append(var)\n\n            if isinstance(values, np.ndarray) and not values.flags.owndata:\n                values = values.copy()  # might view `data` (string columns)\n            cols.append(values)\n\n            try:\n                # allow gc to reclaim memory used by string values\n                data[:, col] = None\n            except IndexError:\n                pass\n\n        domain = Domain(attrs, clses, metas)\n\n        if not data.size:\n            return Table.from_domain(domain, 0)\n\n        X = Y = M = W = None\n        if Xcols:\n            X = np.c_[tuple(Xcols)]\n            assert X.dtype == np.float_\n        else:\n            X = np.empty((data.shape[0], 0), dtype=np.float_)\n        if Ycols:\n            Y = np.c_[tuple(Ycols)]\n            assert Y.dtype == np.float_\n        if Mcols:\n            M = np.c_[tuple(Mcols)].astype(object)\n        if Wcols:\n            W = np.c_[tuple(Wcols)].astype(float)\n\n        table = Table.from_numpy(domain, X, Y, M, W)\n        return table\n\n    @staticmethod\n    def header_names(data):\n        return ['weights'] * data.has_weights() + \\\n               [v.name for v in chain(data.domain.attributes,\n                                      data.domain.class_vars,\n                                      data.domain.metas)]\n\n    @staticmethod\n    def header_types(data):\n        def _vartype(var):\n            if var.is_continuous or var.is_string:\n                return var.TYPE_HEADERS[0]\n            elif var.is_discrete:\n                return Flags.join(var.values) if var.ordered else var.TYPE_HEADERS[0]\n            raise NotImplementedError\n        return ['continuous'] * data.has_weights() + \\\n               [_vartype(v) for v in chain(data.domain.attributes,\n                                           data.domain.class_vars,\n                                           data.domain.metas)]\n\n    @staticmethod\n    def header_flags(data):\n        return list(chain(['weight'] * data.has_weights(),\n                          (Flags.join([flag], *('{}={}'.format(*a)\n                                                for a in sorted(var.attributes.items())))\n                           for flag, var in chain(zip(repeat(''), data.domain.attributes),\n                                                  zip(repeat('class'), data.domain.class_vars),\n                                                  zip(repeat('meta'), data.domain.metas)))))\n\n    @classmethod\n    def write_headers(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        write(cls.header_names(data))\n        write(cls.header_types(data))\n        write(cls.header_flags(data))\n\n    @classmethod\n    def write_data(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        vars = list(chain((ContinuousVariable('_w'),) if data.has_weights() else (),\n                          data.domain.attributes,\n                          data.domain.class_vars,\n                          data.domain.metas))\n\n        def formatter(var):\n            # type: (Variable) -> Callable[[Variable], Any]\n            # Return a column 'formatter' function. The function must return\n            # something that `write` knows how to write\n            if var.is_time:\n                return var.repr_val\n            elif var.is_continuous:\n                return lambda value: \"\" if isnan(value) else value\n            elif var.is_discrete:\n                return lambda value: \"\" if isnan(value) else var.values[int(value)]\n            elif var.is_string:\n                return lambda value: value\n            else:\n                return var.repr_val\n\n        formatters = [formatter(v) for v in vars]\n        for row in zip(data.W if data.W.ndim > 1 else data.W[:, np.newaxis],\n                       data.X,\n                       data.Y if data.Y.ndim > 1 else data.Y[:, np.newaxis],\n                       data.metas):\n            write([fmt(v) for fmt, v in zip(formatters, flatten(row))])\n\n    @classmethod\n    def qualified_name(cls):\n        return cls.__module__ + '.' + cls.__name__",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 362,
                                "end_lineno": 837,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "Orange/data/io.py",
                            "type": "class",
                            "name": "FileFormat",
                            "content": "class FileFormat(metaclass=FileFormatMeta):\n    \"\"\"\n    Subclasses set the following attributes and override the following methods:\n\n        EXTENSIONS = ('.ext1', '.ext2', ...)\n        DESCRIPTION = 'human-readable file format description'\n        SUPPORT_COMPRESSED = False\n        SUPPORT_SPARSE_DATA = False\n\n        def read(self):\n            ...  # load headers, data, ...\n            return self.data_table(data, headers)\n\n        @classmethod\n        def write_file(cls, filename, data):\n            ...\n            self.write_headers(writer.write, data)\n            writer.writerows(data)\n\n    Wrapper FileFormat.data_table() returns Orange.data.Table from `data`\n    iterable (list (rows) of lists of values (cols)).\n    \"\"\"\n\n    # Priority when multiple formats support the same extension. Also\n    # the sort order in file open/save combo boxes. Lower is better.\n    PRIORITY = 10000\n\n    def __init__(self, filename):\n        \"\"\"\n        Parameters\n        ----------\n        filename : str\n            name of the file to open\n        \"\"\"\n        self.filename = filename\n        self.sheet = None\n\n    @property\n    def sheets(self):\n        \"\"\"FileFormats with a notion of sheets should override this property\n        to return a list of sheet names in the file.\n\n        Returns\n        -------\n        a list of sheet names\n        \"\"\"\n        return ()\n\n    def select_sheet(self, sheet):\n        \"\"\"Select sheet to be read\n\n        Parameters\n        ----------\n        sheet : str\n            sheet name\n        \"\"\"\n        self.sheet = sheet\n\n    @classmethod\n    def get_reader(cls, filename):\n        \"\"\"Return reader instance that can be used to read the file\n\n        Parameters\n        ----------\n        filename : str\n\n        Returns\n        -------\n        FileFormat\n        \"\"\"\n        for ext, reader in cls.readers.items():\n            # Skip ambiguous, invalid compression-only extensions added on OSX\n            if ext in Compression.all:\n                continue\n            if fnmatch(path.basename(filename), '*' + ext):\n                return reader(filename)\n\n        raise IOError('No readers for file \"{}\"'.format(filename))\n\n    @classmethod\n    def write(cls, filename, data):\n        return cls.write_file(filename, data)\n\n    @classmethod\n    def write_table_metadata(cls, filename, data):\n\n        def write_file(fn):\n            if all(isinstance(key, str) and isinstance(value, str)\n                   for key, value in data.attributes.items()):\n                with open(fn, 'w', encoding='utf-8') as f:\n                    f.write(\"\\n\".join(\"{}: {}\".format(*kv)\n                                      for kv in data.attributes.items()))\n            else:\n                with open(fn, 'wb') as f:\n                    pickle.dump(data.attributes, f, pickle.HIGHEST_PROTOCOL)\n\n        if isinstance(filename, str):\n            metafile = filename + '.metadata'\n            if getattr(data, 'attributes', None):\n                write_file(metafile)\n            elif path.exists(metafile):\n                remove(metafile)\n\n    @classmethod\n    def set_table_metadata(cls, filename, table):\n        # pylint: disable=bare-except\n        if isinstance(filename, str) and path.exists(filename + '.metadata'):\n            try:\n                with open(filename + '.metadata', 'rb') as f:\n                    table.attributes = pickle.load(f)\n            # Unpickling throws different exceptions, not just UnpickleError\n            except:\n                with open(filename + '.metadata', encoding='utf-8') as f:\n                    table.attributes = OrderedDict(\n                        (k.strip(), v.strip())\n                        for k, v in (line.split(\":\", 1)\n                                     for line in f.readlines()))\n\n    @classmethod\n    def locate(cls, filename, search_dirs=('.',)):\n        \"\"\"Locate a file with given filename that can be opened by one\n        of the available readers.\n\n        Parameters\n        ----------\n        filename : str\n        search_dirs : Iterable[str]\n\n        Returns\n        -------\n        str\n            Absolute path to the file\n        \"\"\"\n        if path.exists(filename):\n            return filename\n\n        for directory in search_dirs:\n            absolute_filename = path.join(directory, filename)\n            if path.exists(absolute_filename):\n                break\n            for ext in cls.readers:\n                if fnmatch(path.basename(filename), '*' + ext):\n                    break\n                # glob uses fnmatch internally\n                matching_files = glob(absolute_filename + ext)\n                if matching_files:\n                    absolute_filename = matching_files[0]\n                    break\n            if path.exists(absolute_filename):\n                break\n        else:\n            absolute_filename = \"\"\n\n        if not path.exists(absolute_filename):\n            raise IOError('File \"{}\" was not found.'.format(filename))\n\n        return absolute_filename\n\n    @staticmethod\n    def open(filename, *args, **kwargs):\n        \"\"\"\n        Format handlers can use this method instead of the builtin ``open()``\n        to transparently (de)compress files if requested (according to\n        `filename` extension). Set ``SUPPORT_COMPRESSED=True`` if you use this.\n        \"\"\"\n        return open_compressed(filename, *args, **kwargs)\n\n    @staticmethod\n    def parse_headers(data):\n        \"\"\"Return (header rows, rest of data) as discerned from `data`\"\"\"\n\n        def is_number(item):\n            try:\n                float(item)\n            except ValueError:\n                return False\n            return True\n        # Second row items are type identifiers\n        def header_test2(items):\n            return all(map(_RE_TYPES.match, items))\n        # Third row items are flags and column attributes (attr=value)\n        def header_test3(items):\n            return all(map(_RE_FLAGS.match, items))\n\n        data = iter(data)\n        header_rows = []\n\n        # Try to parse a three-line header\n        lines = []\n        try:\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n        except StopIteration:\n            lines, data = [], chain(lines, data)\n        if lines:\n            l1, l2, l3 = lines\n            # Three-line header if line 2 & 3 match (1st line can be anything)\n            if header_test2(l2) and header_test3(l3):\n                header_rows = [l1, l2, l3]\n            else:\n                lines, data = [], chain((l1, l2, l3), data)\n\n        # Try to parse a single-line header\n        if not header_rows:\n            try:\n                lines.append(list(next(data)))\n            except StopIteration:\n                pass\n            if lines:\n                # Header if none of the values in line 1 parses as a number\n                if not all(is_number(i) for i in lines[0]):\n                    header_rows = [lines[0]]\n                else:\n                    data = chain(lines, data)\n\n        return header_rows, data\n\n    @classmethod\n    def data_table(cls, data, headers=None):\n        \"\"\"\n        Return Orange.data.Table given rows of `headers` (iterable of iterable)\n        and rows of `data` (iterable of iterable).\n\n        Basically, the idea of subclasses is to produce those two iterables,\n        however they might.\n\n        If `headers` is not provided, the header rows are extracted from `data`,\n        assuming they precede it.\n        \"\"\"\n        if not headers:\n            headers, data = cls.parse_headers(data)\n\n        # Consider various header types (single-row, two-row, three-row, none)\n        if len(headers) == 3:\n            names, types, flags = map(list, headers)\n        else:\n            if len(headers) == 1:\n                HEADER1_FLAG_SEP = '#'\n                # First row format either:\n                #   1) delimited column names\n                #   2) -||- with type and flags prepended, separated by #,\n                #      e.g. d#sex,c#age,cC#IQ\n                _flags, names = zip(*[i.split(HEADER1_FLAG_SEP, 1)\n                                      if HEADER1_FLAG_SEP in i else ('', i)\n                                      for i in headers[0]]\n                                   )\n                names = list(names)\n            elif len(headers) == 2:\n                names, _flags = map(list, headers)\n            else:\n                # Use heuristics for everything\n                names, _flags = [], []\n            types = [''.join(filter(str.isupper, flag)).lower() for flag in _flags]\n            flags = [Flags.join(filter(str.islower, flag)) for flag in _flags]\n\n        # Determine maximum row length\n        rowlen = max(map(len, (names, types, flags)))\n\n        def _equal_length(lst):\n            lst.extend(['']*(rowlen - len(lst)))\n            return lst\n\n        # Ensure all data is of equal width in a column-contiguous array\n        data = [_equal_length([s.strip() for s in row])\n                for row in data if any(row)]\n        data = np.array(data, dtype=object, order='F')\n\n        # Data may actually be longer than headers were\n        try:\n            rowlen = data.shape[1]\n        except IndexError:\n            pass\n        else:\n            for lst in (names, types, flags):\n                _equal_length(lst)\n\n        NAMEGEN = namegen('Feature ', 1)\n        Xcols, attrs = [], []\n        Mcols, metas = [], []\n        Ycols, clses = [], []\n        Wcols = []\n\n        # Rename variables if necessary\n        # Reusing across files still works if both files have same duplicates\n        name_counts = Counter(names)\n        del name_counts[\"\"]\n        if len(name_counts) != len(names) and name_counts:\n            uses = {name: 0 for name, count in name_counts.items() if count > 1}\n            for i, name in enumerate(names):\n                if name in uses:\n                    uses[name] += 1\n                    names[i] = \"{}_{}\".format(name, uses[name])\n\n        namask = np.empty(data.shape[0], dtype=bool)\n        # Iterate through the columns\n        for col in range(rowlen):\n            flag = Flags(Flags.split(flags[col]))\n            if flag.i:\n                continue\n\n            type_flag = types and types[col].strip()\n            try:\n                orig_values = data[:, col]\n            except IndexError:\n                orig_values = np.array([], dtype=object)\n\n            namask = isnastr(orig_values, out=namask)\n\n            coltype_kwargs = {}\n            valuemap = None\n            values = orig_values\n\n            if type_flag in StringVariable.TYPE_HEADERS:\n                coltype = StringVariable\n                values = orig_values\n            elif type_flag in ContinuousVariable.TYPE_HEADERS:\n                coltype = ContinuousVariable\n                values = np.empty(data.shape[0], dtype=float)\n                try:\n                    np.copyto(values, orig_values, casting=\"unsafe\",\n                              where=~namask)\n                    values[namask] = np.nan\n                except ValueError:\n                    for row, num in enumerate(orig_values):\n                        if not isnastr(num):\n                            try:\n                                float(num)\n                            except ValueError:\n                                break\n                    raise ValueError('Non-continuous value in (1-based) '\n                                     'line {}, column {}'.format(row + len(headers) + 1,\n                                                                 col + 1))\n\n            elif type_flag in TimeVariable.TYPE_HEADERS:\n                coltype = TimeVariable\n                values = np.where(namask, \"\", orig_values)\n            elif (type_flag in DiscreteVariable.TYPE_HEADERS or\n                  _RE_DISCRETE_LIST.match(type_flag)):\n                coltype = DiscreteVariable\n                orig_values = values = np.where(namask, \"\", orig_values)\n                if _RE_DISCRETE_LIST.match(type_flag):\n                    valuemap = Flags.split(type_flag)\n                    coltype_kwargs.update(ordered=True)\n                else:\n                    valuemap = sorted(set(orig_values) - {\"\"})\n            else:\n                # No known type specified, use heuristics\n                valuemap, values, coltype = guess_data_type(orig_values, namask)\n\n            if flag.m or coltype is StringVariable:\n                append_to = (Mcols, metas)\n            elif flag.w:\n                append_to = (Wcols, None)\n            elif flag.c:\n                append_to = (Ycols, clses)\n            else:\n                append_to = (Xcols, attrs)\n\n            cols, domain_vars = append_to\n\n            if domain_vars is not None:\n                var_name = names and names[col]\n                if not var_name:\n                    var_name = next(NAMEGEN)\n\n                values, var = sanitize_variable(\n                    valuemap, values, orig_values, coltype, coltype_kwargs,\n                    name=var_name)\n            else:\n                var = None\n            if domain_vars is not None:\n                var.attributes.update(flag.attributes)\n                domain_vars.append(var)\n\n            if isinstance(values, np.ndarray) and not values.flags.owndata:\n                values = values.copy()  # might view `data` (string columns)\n            cols.append(values)\n\n            try:\n                # allow gc to reclaim memory used by string values\n                data[:, col] = None\n            except IndexError:\n                pass\n\n        domain = Domain(attrs, clses, metas)\n\n        if not data.size:\n            return Table.from_domain(domain, 0)\n\n        X = Y = M = W = None\n        if Xcols:\n            X = np.c_[tuple(Xcols)]\n            assert X.dtype == np.float_\n        else:\n            X = np.empty((data.shape[0], 0), dtype=np.float_)\n        if Ycols:\n            Y = np.c_[tuple(Ycols)]\n            assert Y.dtype == np.float_\n        if Mcols:\n            M = np.c_[tuple(Mcols)].astype(object)\n        if Wcols:\n            W = np.c_[tuple(Wcols)].astype(float)\n\n        table = Table.from_numpy(domain, X, Y, M, W)\n        return table\n\n    @staticmethod\n    def header_names(data):\n        return ['weights'] * data.has_weights() + \\\n               [v.name for v in chain(data.domain.attributes,\n                                      data.domain.class_vars,\n                                      data.domain.metas)]\n\n    @staticmethod\n    def header_types(data):\n        def _vartype(var):\n            if var.is_continuous or var.is_string:\n                return var.TYPE_HEADERS[0]\n            elif var.is_discrete:\n                return Flags.join(var.values) if var.ordered else var.TYPE_HEADERS[0]\n            raise NotImplementedError\n        return ['continuous'] * data.has_weights() + \\\n               [_vartype(v) for v in chain(data.domain.attributes,\n                                           data.domain.class_vars,\n                                           data.domain.metas)]\n\n    @staticmethod\n    def header_flags(data):\n        return list(chain(['weight'] * data.has_weights(),\n                          (Flags.join([flag], *('{}={}'.format(*a)\n                                                for a in sorted(var.attributes.items())))\n                           for flag, var in chain(zip(repeat(''), data.domain.attributes),\n                                                  zip(repeat('class'), data.domain.class_vars),\n                                                  zip(repeat('meta'), data.domain.metas)))))\n\n    @classmethod\n    def write_headers(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        write(cls.header_names(data))\n        write(cls.header_types(data))\n        write(cls.header_flags(data))\n\n    @classmethod\n    def write_data(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        vars = list(chain((ContinuousVariable('_w'),) if data.has_weights() else (),\n                          data.domain.attributes,\n                          data.domain.class_vars,\n                          data.domain.metas))\n\n        def formatter(var):\n            # type: (Variable) -> Callable[[Variable], Any]\n            # Return a column 'formatter' function. The function must return\n            # something that `write` knows how to write\n            if var.is_time:\n                return var.repr_val\n            elif var.is_continuous:\n                return lambda value: \"\" if isnan(value) else value\n            elif var.is_discrete:\n                return lambda value: \"\" if isnan(value) else var.values[int(value)]\n            elif var.is_string:\n                return lambda value: value\n            else:\n                return var.repr_val\n\n        formatters = [formatter(v) for v in vars]\n        for row in zip(data.W if data.W.ndim > 1 else data.W[:, np.newaxis],\n                       data.X,\n                       data.Y if data.Y.ndim > 1 else data.Y[:, np.newaxis],\n                       data.metas):\n            write([fmt(v) for fmt, v in zip(formatters, flatten(row))])\n\n    @classmethod\n    def qualified_name(cls):\n        return cls.__module__ + '.' + cls.__name__",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 362,
                            "end_lineno": 837,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 627,
                            "length": 6
                        },
                        "new_range": {
                            "start": 634,
                            "length": 9
                        },
                        "section": "class FileFormat(metaclass=FileFormatMeta):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                for row in data if any(row)]",
                                "type": "context"
                            },
                            {
                                "content": "        data = np.array(data, dtype=object, order='F')",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        if strip:",
                                "type": "added"
                            },
                            {
                                "content": "            warnings.warn(\"Columns with no headers were removed.\")",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        # Data may actually be longer than headers were",
                                "type": "context"
                            },
                            {
                                "content": "        try:",
                                "type": "context"
                            },
                            {
                                "content": "            rowlen = data.shape[1]",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "Orange/data/io.py",
                            "type": "method",
                            "name": "FileFormat.data_table",
                            "content": "class FileFormat(metaclass=FileFormatMeta):\n    \"\"\"\n    Subclasses set the following attributes and override the following methods:\n\n        EXTENSIONS = ('.ext1', '.ext2', ...)\n        DESCRIPTION = 'human-readable file format description'\n        SUPPORT_COMPRESSED = False\n        SUPPORT_SPARSE_DATA = False\n\n        def read(self):\n            ...  # load headers, data, ...\n            return self.data_table(data, headers)\n\n        @classmethod\n        def write_file(cls, filename, data):\n            ...\n            self.write_headers(writer.write, data)\n            writer.writerows(data)\n\n    Wrapper FileFormat.data_table() returns Orange.data.Table from `data`\n    iterable (list (rows) of lists of values (cols)).\n    \"\"\"\n\n    # Priority when multiple formats support the same extension. Also\n    # the sort order in file open/save combo boxes. Lower is better.\n    PRIORITY = 10000\n\n    def __init__(self, filename):\n        \"\"\"\n        Parameters\n        ----------\n        filename : str\n            name of the file to open\n        \"\"\"\n        self.filename = filename\n        self.sheet = None\n\n    @property\n    def sheets(self):\n        \"\"\"FileFormats with a notion of sheets should override this property\n        to return a list of sheet names in the file.\n\n        Returns\n        -------\n        a list of sheet names\n        \"\"\"\n        return ()\n\n    def select_sheet(self, sheet):\n        \"\"\"Select sheet to be read\n\n        Parameters\n        ----------\n        sheet : str\n            sheet name\n        \"\"\"\n        self.sheet = sheet\n\n    @classmethod\n    def get_reader(cls, filename):\n        \"\"\"Return reader instance that can be used to read the file\n\n        Parameters\n        ----------\n        filename : str\n\n        Returns\n        -------\n        FileFormat\n        \"\"\"\n        for ext, reader in cls.readers.items():\n            # Skip ambiguous, invalid compression-only extensions added on OSX\n            if ext in Compression.all:\n                continue\n            if fnmatch(path.basename(filename), '*' + ext):\n                return reader(filename)\n\n        raise IOError('No readers for file \"{}\"'.format(filename))\n\n    @classmethod\n    def write(cls, filename, data):\n        return cls.write_file(filename, data)\n\n    @classmethod\n    def write_table_metadata(cls, filename, data):\n\n        def write_file(fn):\n            if all(isinstance(key, str) and isinstance(value, str)\n                   for key, value in data.attributes.items()):\n                with open(fn, 'w', encoding='utf-8') as f:\n                    f.write(\"\\n\".join(\"{}: {}\".format(*kv)\n                                      for kv in data.attributes.items()))\n            else:\n                with open(fn, 'wb') as f:\n                    pickle.dump(data.attributes, f, pickle.HIGHEST_PROTOCOL)\n\n        if isinstance(filename, str):\n            metafile = filename + '.metadata'\n            if getattr(data, 'attributes', None):\n                write_file(metafile)\n            elif path.exists(metafile):\n                remove(metafile)\n\n    @classmethod\n    def set_table_metadata(cls, filename, table):\n        # pylint: disable=bare-except\n        if isinstance(filename, str) and path.exists(filename + '.metadata'):\n            try:\n                with open(filename + '.metadata', 'rb') as f:\n                    table.attributes = pickle.load(f)\n            # Unpickling throws different exceptions, not just UnpickleError\n            except:\n                with open(filename + '.metadata', encoding='utf-8') as f:\n                    table.attributes = OrderedDict(\n                        (k.strip(), v.strip())\n                        for k, v in (line.split(\":\", 1)\n                                     for line in f.readlines()))\n\n    @classmethod\n    def locate(cls, filename, search_dirs=('.',)):\n        \"\"\"Locate a file with given filename that can be opened by one\n        of the available readers.\n\n        Parameters\n        ----------\n        filename : str\n        search_dirs : Iterable[str]\n\n        Returns\n        -------\n        str\n            Absolute path to the file\n        \"\"\"\n        if path.exists(filename):\n            return filename\n\n        for directory in search_dirs:\n            absolute_filename = path.join(directory, filename)\n            if path.exists(absolute_filename):\n                break\n            for ext in cls.readers:\n                if fnmatch(path.basename(filename), '*' + ext):\n                    break\n                # glob uses fnmatch internally\n                matching_files = glob(absolute_filename + ext)\n                if matching_files:\n                    absolute_filename = matching_files[0]\n                    break\n            if path.exists(absolute_filename):\n                break\n        else:\n            absolute_filename = \"\"\n\n        if not path.exists(absolute_filename):\n            raise IOError('File \"{}\" was not found.'.format(filename))\n\n        return absolute_filename\n\n    @staticmethod\n    def open(filename, *args, **kwargs):\n        \"\"\"\n        Format handlers can use this method instead of the builtin ``open()``\n        to transparently (de)compress files if requested (according to\n        `filename` extension). Set ``SUPPORT_COMPRESSED=True`` if you use this.\n        \"\"\"\n        return open_compressed(filename, *args, **kwargs)\n\n    @staticmethod\n    def parse_headers(data):\n        \"\"\"Return (header rows, rest of data) as discerned from `data`\"\"\"\n\n        def is_number(item):\n            try:\n                float(item)\n            except ValueError:\n                return False\n            return True\n        # Second row items are type identifiers\n        def header_test2(items):\n            return all(map(_RE_TYPES.match, items))\n        # Third row items are flags and column attributes (attr=value)\n        def header_test3(items):\n            return all(map(_RE_FLAGS.match, items))\n\n        data = iter(data)\n        header_rows = []\n\n        # Try to parse a three-line header\n        lines = []\n        try:\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n        except StopIteration:\n            lines, data = [], chain(lines, data)\n        if lines:\n            l1, l2, l3 = lines\n            # Three-line header if line 2 & 3 match (1st line can be anything)\n            if header_test2(l2) and header_test3(l3):\n                header_rows = [l1, l2, l3]\n            else:\n                lines, data = [], chain((l1, l2, l3), data)\n\n        # Try to parse a single-line header\n        if not header_rows:\n            try:\n                lines.append(list(next(data)))\n            except StopIteration:\n                pass\n            if lines:\n                # Header if none of the values in line 1 parses as a number\n                if not all(is_number(i) for i in lines[0]):\n                    header_rows = [lines[0]]\n                else:\n                    data = chain(lines, data)\n\n        return header_rows, data\n\n    @classmethod\n    def data_table(cls, data, headers=None):\n        \"\"\"\n        Return Orange.data.Table given rows of `headers` (iterable of iterable)\n        and rows of `data` (iterable of iterable).\n\n        Basically, the idea of subclasses is to produce those two iterables,\n        however they might.\n\n        If `headers` is not provided, the header rows are extracted from `data`,\n        assuming they precede it.\n        \"\"\"\n        if not headers:\n            headers, data = cls.parse_headers(data)\n\n        # Consider various header types (single-row, two-row, three-row, none)\n        if len(headers) == 3:\n            names, types, flags = map(list, headers)\n        else:\n            if len(headers) == 1:\n                HEADER1_FLAG_SEP = '#'\n                # First row format either:\n                #   1) delimited column names\n                #   2) -||- with type and flags prepended, separated by #,\n                #      e.g. d#sex,c#age,cC#IQ\n                _flags, names = zip(*[i.split(HEADER1_FLAG_SEP, 1)\n                                      if HEADER1_FLAG_SEP in i else ('', i)\n                                      for i in headers[0]]\n                                   )\n                names = list(names)\n            elif len(headers) == 2:\n                names, _flags = map(list, headers)\n            else:\n                # Use heuristics for everything\n                names, _flags = [], []\n            types = [''.join(filter(str.isupper, flag)).lower() for flag in _flags]\n            flags = [Flags.join(filter(str.islower, flag)) for flag in _flags]\n\n        # Determine maximum row length\n        rowlen = max(map(len, (names, types, flags)))\n\n        strip = False\n\n        def _equal_length(lst):\n            nonlocal strip\n            if len(lst) > rowlen > 0:\n                lst = lst[:rowlen]\n                strip = True\n            elif len(lst) < rowlen:\n                lst.extend(['']*(rowlen - len(lst)))\n            return lst\n\n        # Ensure all data is of equal width in a column-contiguous array\n        data = [_equal_length([s.strip() for s in row])\n                for row in data if any(row)]\n        data = np.array(data, dtype=object, order='F')\n\n        if strip:\n            warnings.warn(\"Columns with no headers were removed.\")\n\n        # Data may actually be longer than headers were\n        try:\n            rowlen = data.shape[1]\n        except IndexError:\n            pass\n        else:\n            for lst in (names, types, flags):\n                _equal_length(lst)\n\n        NAMEGEN = namegen('Feature ', 1)\n        Xcols, attrs = [], []\n        Mcols, metas = [], []\n        Ycols, clses = [], []\n        Wcols = []\n\n        # Rename variables if necessary\n        # Reusing across files still works if both files have same duplicates\n        name_counts = Counter(names)\n        del name_counts[\"\"]\n        if len(name_counts) != len(names) and name_counts:\n            uses = {name: 0 for name, count in name_counts.items() if count > 1}\n            for i, name in enumerate(names):\n                if name in uses:\n                    uses[name] += 1\n                    names[i] = \"{}_{}\".format(name, uses[name])\n\n        namask = np.empty(data.shape[0], dtype=bool)\n        # Iterate through the columns\n        for col in range(rowlen):\n            flag = Flags(Flags.split(flags[col]))\n            if flag.i:\n                continue\n\n            type_flag = types and types[col].strip()\n            try:\n                orig_values = data[:, col]\n            except IndexError:\n                orig_values = np.array([], dtype=object)\n\n            namask = isnastr(orig_values, out=namask)\n\n            coltype_kwargs = {}\n            valuemap = None\n            values = orig_values\n\n            if type_flag in StringVariable.TYPE_HEADERS:\n                coltype = StringVariable\n                values = orig_values\n            elif type_flag in ContinuousVariable.TYPE_HEADERS:\n                coltype = ContinuousVariable\n                values = np.empty(data.shape[0], dtype=float)\n                try:\n                    np.copyto(values, orig_values, casting=\"unsafe\",\n                              where=~namask)\n                    values[namask] = np.nan\n                except ValueError:\n                    for row, num in enumerate(orig_values):\n                        if not isnastr(num):\n                            try:\n                                float(num)\n                            except ValueError:\n                                break\n                    raise ValueError('Non-continuous value in (1-based) '\n                                     'line {}, column {}'.format(row + len(headers) + 1,\n                                                                 col + 1))\n\n            elif type_flag in TimeVariable.TYPE_HEADERS:\n                coltype = TimeVariable\n                values = np.where(namask, \"\", orig_values)\n            elif (type_flag in DiscreteVariable.TYPE_HEADERS or\n                  _RE_DISCRETE_LIST.match(type_flag)):\n                coltype = DiscreteVariable\n                orig_values = values = np.where(namask, \"\", orig_values)\n                if _RE_DISCRETE_LIST.match(type_flag):\n                    valuemap = Flags.split(type_flag)\n                    coltype_kwargs.update(ordered=True)\n                else:\n                    valuemap = sorted(set(orig_values) - {\"\"})\n            else:\n                # No known type specified, use heuristics\n                valuemap, values, coltype = guess_data_type(orig_values, namask)\n\n            if flag.m or coltype is StringVariable:\n                append_to = (Mcols, metas)\n            elif flag.w:\n                append_to = (Wcols, None)\n            elif flag.c:\n                append_to = (Ycols, clses)\n            else:\n                append_to = (Xcols, attrs)\n\n            cols, domain_vars = append_to\n\n            if domain_vars is not None:\n                var_name = names and names[col]\n                if not var_name:\n                    var_name = next(NAMEGEN)\n\n                values, var = sanitize_variable(\n                    valuemap, values, orig_values, coltype, coltype_kwargs,\n                    name=var_name)\n            else:\n                var = None\n            if domain_vars is not None:\n                var.attributes.update(flag.attributes)\n                domain_vars.append(var)\n\n            if isinstance(values, np.ndarray) and not values.flags.owndata:\n                values = values.copy()  # might view `data` (string columns)\n            cols.append(values)\n\n            try:\n                # allow gc to reclaim memory used by string values\n                data[:, col] = None\n            except IndexError:\n                pass\n\n        domain = Domain(attrs, clses, metas)\n\n        if not data.size:\n            return Table.from_domain(domain, 0)\n\n        X = Y = M = W = None\n        if Xcols:\n            X = np.c_[tuple(Xcols)]\n            assert X.dtype == np.float_\n        else:\n            X = np.empty((data.shape[0], 0), dtype=np.float_)\n        if Ycols:\n            Y = np.c_[tuple(Ycols)]\n            assert Y.dtype == np.float_\n        if Mcols:\n            M = np.c_[tuple(Mcols)].astype(object)\n        if Wcols:\n            W = np.c_[tuple(Wcols)].astype(float)\n\n        table = Table.from_numpy(domain, X, Y, M, W)\n        return table\n\n    @staticmethod\n    def header_names(data):\n        return ['weights'] * data.has_weights() + \\\n               [v.name for v in chain(data.domain.attributes,\n                                      data.domain.class_vars,\n                                      data.domain.metas)]\n\n    @staticmethod\n    def header_types(data):\n        def _vartype(var):\n            if var.is_continuous or var.is_string:\n                return var.TYPE_HEADERS[0]\n            elif var.is_discrete:\n                return Flags.join(var.values) if var.ordered else var.TYPE_HEADERS[0]\n            raise NotImplementedError\n        return ['continuous'] * data.has_weights() + \\\n               [_vartype(v) for v in chain(data.domain.attributes,\n                                           data.domain.class_vars,\n                                           data.domain.metas)]\n\n    @staticmethod\n    def header_flags(data):\n        return list(chain(['weight'] * data.has_weights(),\n                          (Flags.join([flag], *('{}={}'.format(*a)\n                                                for a in sorted(var.attributes.items())))\n                           for flag, var in chain(zip(repeat(''), data.domain.attributes),\n                                                  zip(repeat('class'), data.domain.class_vars),\n                                                  zip(repeat('meta'), data.domain.metas)))))\n\n    @classmethod\n    def write_headers(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        write(cls.header_names(data))\n        write(cls.header_types(data))\n        write(cls.header_flags(data))\n\n    @classmethod\n    def write_data(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        vars = list(chain((ContinuousVariable('_w'),) if data.has_weights() else (),\n                          data.domain.attributes,\n                          data.domain.class_vars,\n                          data.domain.metas))\n\n        def formatter(var):\n            # type: (Variable) -> Callable[[Variable], Any]\n            # Return a column 'formatter' function. The function must return\n            # something that `write` knows how to write\n            if var.is_time:\n                return var.repr_val\n            elif var.is_continuous:\n                return lambda value: \"\" if isnan(value) else value\n            elif var.is_discrete:\n                return lambda value: \"\" if isnan(value) else var.values[int(value)]\n            elif var.is_string:\n                return lambda value: value\n            else:\n                return var.repr_val\n\n        formatters = [formatter(v) for v in vars]\n        for row in zip(data.W if data.W.ndim > 1 else data.W[:, np.newaxis],\n                       data.X,\n                       data.Y if data.Y.ndim > 1 else data.Y[:, np.newaxis],\n                       data.metas):\n            write([fmt(v) for fmt, v in zip(formatters, flatten(row))])\n\n    @classmethod\n    def qualified_name(cls):\n        return cls.__module__ + '.' + cls.__name__",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 581,
                            "end_lineno": 777,
                            "parent": {
                                "file_name": "Orange/data/io.py",
                                "type": "class",
                                "name": "FileFormat",
                                "content": "class FileFormat(metaclass=FileFormatMeta):\n    \"\"\"\n    Subclasses set the following attributes and override the following methods:\n\n        EXTENSIONS = ('.ext1', '.ext2', ...)\n        DESCRIPTION = 'human-readable file format description'\n        SUPPORT_COMPRESSED = False\n        SUPPORT_SPARSE_DATA = False\n\n        def read(self):\n            ...  # load headers, data, ...\n            return self.data_table(data, headers)\n\n        @classmethod\n        def write_file(cls, filename, data):\n            ...\n            self.write_headers(writer.write, data)\n            writer.writerows(data)\n\n    Wrapper FileFormat.data_table() returns Orange.data.Table from `data`\n    iterable (list (rows) of lists of values (cols)).\n    \"\"\"\n\n    # Priority when multiple formats support the same extension. Also\n    # the sort order in file open/save combo boxes. Lower is better.\n    PRIORITY = 10000\n\n    def __init__(self, filename):\n        \"\"\"\n        Parameters\n        ----------\n        filename : str\n            name of the file to open\n        \"\"\"\n        self.filename = filename\n        self.sheet = None\n\n    @property\n    def sheets(self):\n        \"\"\"FileFormats with a notion of sheets should override this property\n        to return a list of sheet names in the file.\n\n        Returns\n        -------\n        a list of sheet names\n        \"\"\"\n        return ()\n\n    def select_sheet(self, sheet):\n        \"\"\"Select sheet to be read\n\n        Parameters\n        ----------\n        sheet : str\n            sheet name\n        \"\"\"\n        self.sheet = sheet\n\n    @classmethod\n    def get_reader(cls, filename):\n        \"\"\"Return reader instance that can be used to read the file\n\n        Parameters\n        ----------\n        filename : str\n\n        Returns\n        -------\n        FileFormat\n        \"\"\"\n        for ext, reader in cls.readers.items():\n            # Skip ambiguous, invalid compression-only extensions added on OSX\n            if ext in Compression.all:\n                continue\n            if fnmatch(path.basename(filename), '*' + ext):\n                return reader(filename)\n\n        raise IOError('No readers for file \"{}\"'.format(filename))\n\n    @classmethod\n    def write(cls, filename, data):\n        return cls.write_file(filename, data)\n\n    @classmethod\n    def write_table_metadata(cls, filename, data):\n\n        def write_file(fn):\n            if all(isinstance(key, str) and isinstance(value, str)\n                   for key, value in data.attributes.items()):\n                with open(fn, 'w', encoding='utf-8') as f:\n                    f.write(\"\\n\".join(\"{}: {}\".format(*kv)\n                                      for kv in data.attributes.items()))\n            else:\n                with open(fn, 'wb') as f:\n                    pickle.dump(data.attributes, f, pickle.HIGHEST_PROTOCOL)\n\n        if isinstance(filename, str):\n            metafile = filename + '.metadata'\n            if getattr(data, 'attributes', None):\n                write_file(metafile)\n            elif path.exists(metafile):\n                remove(metafile)\n\n    @classmethod\n    def set_table_metadata(cls, filename, table):\n        # pylint: disable=bare-except\n        if isinstance(filename, str) and path.exists(filename + '.metadata'):\n            try:\n                with open(filename + '.metadata', 'rb') as f:\n                    table.attributes = pickle.load(f)\n            # Unpickling throws different exceptions, not just UnpickleError\n            except:\n                with open(filename + '.metadata', encoding='utf-8') as f:\n                    table.attributes = OrderedDict(\n                        (k.strip(), v.strip())\n                        for k, v in (line.split(\":\", 1)\n                                     for line in f.readlines()))\n\n    @classmethod\n    def locate(cls, filename, search_dirs=('.',)):\n        \"\"\"Locate a file with given filename that can be opened by one\n        of the available readers.\n\n        Parameters\n        ----------\n        filename : str\n        search_dirs : Iterable[str]\n\n        Returns\n        -------\n        str\n            Absolute path to the file\n        \"\"\"\n        if path.exists(filename):\n            return filename\n\n        for directory in search_dirs:\n            absolute_filename = path.join(directory, filename)\n            if path.exists(absolute_filename):\n                break\n            for ext in cls.readers:\n                if fnmatch(path.basename(filename), '*' + ext):\n                    break\n                # glob uses fnmatch internally\n                matching_files = glob(absolute_filename + ext)\n                if matching_files:\n                    absolute_filename = matching_files[0]\n                    break\n            if path.exists(absolute_filename):\n                break\n        else:\n            absolute_filename = \"\"\n\n        if not path.exists(absolute_filename):\n            raise IOError('File \"{}\" was not found.'.format(filename))\n\n        return absolute_filename\n\n    @staticmethod\n    def open(filename, *args, **kwargs):\n        \"\"\"\n        Format handlers can use this method instead of the builtin ``open()``\n        to transparently (de)compress files if requested (according to\n        `filename` extension). Set ``SUPPORT_COMPRESSED=True`` if you use this.\n        \"\"\"\n        return open_compressed(filename, *args, **kwargs)\n\n    @staticmethod\n    def parse_headers(data):\n        \"\"\"Return (header rows, rest of data) as discerned from `data`\"\"\"\n\n        def is_number(item):\n            try:\n                float(item)\n            except ValueError:\n                return False\n            return True\n        # Second row items are type identifiers\n        def header_test2(items):\n            return all(map(_RE_TYPES.match, items))\n        # Third row items are flags and column attributes (attr=value)\n        def header_test3(items):\n            return all(map(_RE_FLAGS.match, items))\n\n        data = iter(data)\n        header_rows = []\n\n        # Try to parse a three-line header\n        lines = []\n        try:\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n        except StopIteration:\n            lines, data = [], chain(lines, data)\n        if lines:\n            l1, l2, l3 = lines\n            # Three-line header if line 2 & 3 match (1st line can be anything)\n            if header_test2(l2) and header_test3(l3):\n                header_rows = [l1, l2, l3]\n            else:\n                lines, data = [], chain((l1, l2, l3), data)\n\n        # Try to parse a single-line header\n        if not header_rows:\n            try:\n                lines.append(list(next(data)))\n            except StopIteration:\n                pass\n            if lines:\n                # Header if none of the values in line 1 parses as a number\n                if not all(is_number(i) for i in lines[0]):\n                    header_rows = [lines[0]]\n                else:\n                    data = chain(lines, data)\n\n        return header_rows, data\n\n    @classmethod\n    def data_table(cls, data, headers=None):\n        \"\"\"\n        Return Orange.data.Table given rows of `headers` (iterable of iterable)\n        and rows of `data` (iterable of iterable).\n\n        Basically, the idea of subclasses is to produce those two iterables,\n        however they might.\n\n        If `headers` is not provided, the header rows are extracted from `data`,\n        assuming they precede it.\n        \"\"\"\n        if not headers:\n            headers, data = cls.parse_headers(data)\n\n        # Consider various header types (single-row, two-row, three-row, none)\n        if len(headers) == 3:\n            names, types, flags = map(list, headers)\n        else:\n            if len(headers) == 1:\n                HEADER1_FLAG_SEP = '#'\n                # First row format either:\n                #   1) delimited column names\n                #   2) -||- with type and flags prepended, separated by #,\n                #      e.g. d#sex,c#age,cC#IQ\n                _flags, names = zip(*[i.split(HEADER1_FLAG_SEP, 1)\n                                      if HEADER1_FLAG_SEP in i else ('', i)\n                                      for i in headers[0]]\n                                   )\n                names = list(names)\n            elif len(headers) == 2:\n                names, _flags = map(list, headers)\n            else:\n                # Use heuristics for everything\n                names, _flags = [], []\n            types = [''.join(filter(str.isupper, flag)).lower() for flag in _flags]\n            flags = [Flags.join(filter(str.islower, flag)) for flag in _flags]\n\n        # Determine maximum row length\n        rowlen = max(map(len, (names, types, flags)))\n\n        strip = False\n\n        def _equal_length(lst):\n            nonlocal strip\n            if len(lst) > rowlen > 0:\n                lst = lst[:rowlen]\n                strip = True\n            elif len(lst) < rowlen:\n                lst.extend(['']*(rowlen - len(lst)))\n            return lst\n\n        # Ensure all data is of equal width in a column-contiguous array\n        data = [_equal_length([s.strip() for s in row])\n                for row in data if any(row)]\n        data = np.array(data, dtype=object, order='F')\n\n        if strip:\n            warnings.warn(\"Columns with no headers were removed.\")\n\n        # Data may actually be longer than headers were\n        try:\n            rowlen = data.shape[1]\n        except IndexError:\n            pass\n        else:\n            for lst in (names, types, flags):\n                _equal_length(lst)\n\n        NAMEGEN = namegen('Feature ', 1)\n        Xcols, attrs = [], []\n        Mcols, metas = [], []\n        Ycols, clses = [], []\n        Wcols = []\n\n        # Rename variables if necessary\n        # Reusing across files still works if both files have same duplicates\n        name_counts = Counter(names)\n        del name_counts[\"\"]\n        if len(name_counts) != len(names) and name_counts:\n            uses = {name: 0 for name, count in name_counts.items() if count > 1}\n            for i, name in enumerate(names):\n                if name in uses:\n                    uses[name] += 1\n                    names[i] = \"{}_{}\".format(name, uses[name])\n\n        namask = np.empty(data.shape[0], dtype=bool)\n        # Iterate through the columns\n        for col in range(rowlen):\n            flag = Flags(Flags.split(flags[col]))\n            if flag.i:\n                continue\n\n            type_flag = types and types[col].strip()\n            try:\n                orig_values = data[:, col]\n            except IndexError:\n                orig_values = np.array([], dtype=object)\n\n            namask = isnastr(orig_values, out=namask)\n\n            coltype_kwargs = {}\n            valuemap = None\n            values = orig_values\n\n            if type_flag in StringVariable.TYPE_HEADERS:\n                coltype = StringVariable\n                values = orig_values\n            elif type_flag in ContinuousVariable.TYPE_HEADERS:\n                coltype = ContinuousVariable\n                values = np.empty(data.shape[0], dtype=float)\n                try:\n                    np.copyto(values, orig_values, casting=\"unsafe\",\n                              where=~namask)\n                    values[namask] = np.nan\n                except ValueError:\n                    for row, num in enumerate(orig_values):\n                        if not isnastr(num):\n                            try:\n                                float(num)\n                            except ValueError:\n                                break\n                    raise ValueError('Non-continuous value in (1-based) '\n                                     'line {}, column {}'.format(row + len(headers) + 1,\n                                                                 col + 1))\n\n            elif type_flag in TimeVariable.TYPE_HEADERS:\n                coltype = TimeVariable\n                values = np.where(namask, \"\", orig_values)\n            elif (type_flag in DiscreteVariable.TYPE_HEADERS or\n                  _RE_DISCRETE_LIST.match(type_flag)):\n                coltype = DiscreteVariable\n                orig_values = values = np.where(namask, \"\", orig_values)\n                if _RE_DISCRETE_LIST.match(type_flag):\n                    valuemap = Flags.split(type_flag)\n                    coltype_kwargs.update(ordered=True)\n                else:\n                    valuemap = sorted(set(orig_values) - {\"\"})\n            else:\n                # No known type specified, use heuristics\n                valuemap, values, coltype = guess_data_type(orig_values, namask)\n\n            if flag.m or coltype is StringVariable:\n                append_to = (Mcols, metas)\n            elif flag.w:\n                append_to = (Wcols, None)\n            elif flag.c:\n                append_to = (Ycols, clses)\n            else:\n                append_to = (Xcols, attrs)\n\n            cols, domain_vars = append_to\n\n            if domain_vars is not None:\n                var_name = names and names[col]\n                if not var_name:\n                    var_name = next(NAMEGEN)\n\n                values, var = sanitize_variable(\n                    valuemap, values, orig_values, coltype, coltype_kwargs,\n                    name=var_name)\n            else:\n                var = None\n            if domain_vars is not None:\n                var.attributes.update(flag.attributes)\n                domain_vars.append(var)\n\n            if isinstance(values, np.ndarray) and not values.flags.owndata:\n                values = values.copy()  # might view `data` (string columns)\n            cols.append(values)\n\n            try:\n                # allow gc to reclaim memory used by string values\n                data[:, col] = None\n            except IndexError:\n                pass\n\n        domain = Domain(attrs, clses, metas)\n\n        if not data.size:\n            return Table.from_domain(domain, 0)\n\n        X = Y = M = W = None\n        if Xcols:\n            X = np.c_[tuple(Xcols)]\n            assert X.dtype == np.float_\n        else:\n            X = np.empty((data.shape[0], 0), dtype=np.float_)\n        if Ycols:\n            Y = np.c_[tuple(Ycols)]\n            assert Y.dtype == np.float_\n        if Mcols:\n            M = np.c_[tuple(Mcols)].astype(object)\n        if Wcols:\n            W = np.c_[tuple(Wcols)].astype(float)\n\n        table = Table.from_numpy(domain, X, Y, M, W)\n        return table\n\n    @staticmethod\n    def header_names(data):\n        return ['weights'] * data.has_weights() + \\\n               [v.name for v in chain(data.domain.attributes,\n                                      data.domain.class_vars,\n                                      data.domain.metas)]\n\n    @staticmethod\n    def header_types(data):\n        def _vartype(var):\n            if var.is_continuous or var.is_string:\n                return var.TYPE_HEADERS[0]\n            elif var.is_discrete:\n                return Flags.join(var.values) if var.ordered else var.TYPE_HEADERS[0]\n            raise NotImplementedError\n        return ['continuous'] * data.has_weights() + \\\n               [_vartype(v) for v in chain(data.domain.attributes,\n                                           data.domain.class_vars,\n                                           data.domain.metas)]\n\n    @staticmethod\n    def header_flags(data):\n        return list(chain(['weight'] * data.has_weights(),\n                          (Flags.join([flag], *('{}={}'.format(*a)\n                                                for a in sorted(var.attributes.items())))\n                           for flag, var in chain(zip(repeat(''), data.domain.attributes),\n                                                  zip(repeat('class'), data.domain.class_vars),\n                                                  zip(repeat('meta'), data.domain.metas)))))\n\n    @classmethod\n    def write_headers(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        write(cls.header_names(data))\n        write(cls.header_types(data))\n        write(cls.header_flags(data))\n\n    @classmethod\n    def write_data(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        vars = list(chain((ContinuousVariable('_w'),) if data.has_weights() else (),\n                          data.domain.attributes,\n                          data.domain.class_vars,\n                          data.domain.metas))\n\n        def formatter(var):\n            # type: (Variable) -> Callable[[Variable], Any]\n            # Return a column 'formatter' function. The function must return\n            # something that `write` knows how to write\n            if var.is_time:\n                return var.repr_val\n            elif var.is_continuous:\n                return lambda value: \"\" if isnan(value) else value\n            elif var.is_discrete:\n                return lambda value: \"\" if isnan(value) else var.values[int(value)]\n            elif var.is_string:\n                return lambda value: value\n            else:\n                return var.repr_val\n\n        formatters = [formatter(v) for v in vars]\n        for row in zip(data.W if data.W.ndim > 1 else data.W[:, np.newaxis],\n                       data.X,\n                       data.Y if data.Y.ndim > 1 else data.Y[:, np.newaxis],\n                       data.metas):\n            write([fmt(v) for fmt, v in zip(formatters, flatten(row))])\n\n    @classmethod\n    def qualified_name(cls):\n        return cls.__module__ + '.' + cls.__name__",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 362,
                                "end_lineno": 847,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "Orange/data/io.py",
                            "type": "class",
                            "name": "FileFormat",
                            "content": "class FileFormat(metaclass=FileFormatMeta):\n    \"\"\"\n    Subclasses set the following attributes and override the following methods:\n\n        EXTENSIONS = ('.ext1', '.ext2', ...)\n        DESCRIPTION = 'human-readable file format description'\n        SUPPORT_COMPRESSED = False\n        SUPPORT_SPARSE_DATA = False\n\n        def read(self):\n            ...  # load headers, data, ...\n            return self.data_table(data, headers)\n\n        @classmethod\n        def write_file(cls, filename, data):\n            ...\n            self.write_headers(writer.write, data)\n            writer.writerows(data)\n\n    Wrapper FileFormat.data_table() returns Orange.data.Table from `data`\n    iterable (list (rows) of lists of values (cols)).\n    \"\"\"\n\n    # Priority when multiple formats support the same extension. Also\n    # the sort order in file open/save combo boxes. Lower is better.\n    PRIORITY = 10000\n\n    def __init__(self, filename):\n        \"\"\"\n        Parameters\n        ----------\n        filename : str\n            name of the file to open\n        \"\"\"\n        self.filename = filename\n        self.sheet = None\n\n    @property\n    def sheets(self):\n        \"\"\"FileFormats with a notion of sheets should override this property\n        to return a list of sheet names in the file.\n\n        Returns\n        -------\n        a list of sheet names\n        \"\"\"\n        return ()\n\n    def select_sheet(self, sheet):\n        \"\"\"Select sheet to be read\n\n        Parameters\n        ----------\n        sheet : str\n            sheet name\n        \"\"\"\n        self.sheet = sheet\n\n    @classmethod\n    def get_reader(cls, filename):\n        \"\"\"Return reader instance that can be used to read the file\n\n        Parameters\n        ----------\n        filename : str\n\n        Returns\n        -------\n        FileFormat\n        \"\"\"\n        for ext, reader in cls.readers.items():\n            # Skip ambiguous, invalid compression-only extensions added on OSX\n            if ext in Compression.all:\n                continue\n            if fnmatch(path.basename(filename), '*' + ext):\n                return reader(filename)\n\n        raise IOError('No readers for file \"{}\"'.format(filename))\n\n    @classmethod\n    def write(cls, filename, data):\n        return cls.write_file(filename, data)\n\n    @classmethod\n    def write_table_metadata(cls, filename, data):\n\n        def write_file(fn):\n            if all(isinstance(key, str) and isinstance(value, str)\n                   for key, value in data.attributes.items()):\n                with open(fn, 'w', encoding='utf-8') as f:\n                    f.write(\"\\n\".join(\"{}: {}\".format(*kv)\n                                      for kv in data.attributes.items()))\n            else:\n                with open(fn, 'wb') as f:\n                    pickle.dump(data.attributes, f, pickle.HIGHEST_PROTOCOL)\n\n        if isinstance(filename, str):\n            metafile = filename + '.metadata'\n            if getattr(data, 'attributes', None):\n                write_file(metafile)\n            elif path.exists(metafile):\n                remove(metafile)\n\n    @classmethod\n    def set_table_metadata(cls, filename, table):\n        # pylint: disable=bare-except\n        if isinstance(filename, str) and path.exists(filename + '.metadata'):\n            try:\n                with open(filename + '.metadata', 'rb') as f:\n                    table.attributes = pickle.load(f)\n            # Unpickling throws different exceptions, not just UnpickleError\n            except:\n                with open(filename + '.metadata', encoding='utf-8') as f:\n                    table.attributes = OrderedDict(\n                        (k.strip(), v.strip())\n                        for k, v in (line.split(\":\", 1)\n                                     for line in f.readlines()))\n\n    @classmethod\n    def locate(cls, filename, search_dirs=('.',)):\n        \"\"\"Locate a file with given filename that can be opened by one\n        of the available readers.\n\n        Parameters\n        ----------\n        filename : str\n        search_dirs : Iterable[str]\n\n        Returns\n        -------\n        str\n            Absolute path to the file\n        \"\"\"\n        if path.exists(filename):\n            return filename\n\n        for directory in search_dirs:\n            absolute_filename = path.join(directory, filename)\n            if path.exists(absolute_filename):\n                break\n            for ext in cls.readers:\n                if fnmatch(path.basename(filename), '*' + ext):\n                    break\n                # glob uses fnmatch internally\n                matching_files = glob(absolute_filename + ext)\n                if matching_files:\n                    absolute_filename = matching_files[0]\n                    break\n            if path.exists(absolute_filename):\n                break\n        else:\n            absolute_filename = \"\"\n\n        if not path.exists(absolute_filename):\n            raise IOError('File \"{}\" was not found.'.format(filename))\n\n        return absolute_filename\n\n    @staticmethod\n    def open(filename, *args, **kwargs):\n        \"\"\"\n        Format handlers can use this method instead of the builtin ``open()``\n        to transparently (de)compress files if requested (according to\n        `filename` extension). Set ``SUPPORT_COMPRESSED=True`` if you use this.\n        \"\"\"\n        return open_compressed(filename, *args, **kwargs)\n\n    @staticmethod\n    def parse_headers(data):\n        \"\"\"Return (header rows, rest of data) as discerned from `data`\"\"\"\n\n        def is_number(item):\n            try:\n                float(item)\n            except ValueError:\n                return False\n            return True\n        # Second row items are type identifiers\n        def header_test2(items):\n            return all(map(_RE_TYPES.match, items))\n        # Third row items are flags and column attributes (attr=value)\n        def header_test3(items):\n            return all(map(_RE_FLAGS.match, items))\n\n        data = iter(data)\n        header_rows = []\n\n        # Try to parse a three-line header\n        lines = []\n        try:\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n        except StopIteration:\n            lines, data = [], chain(lines, data)\n        if lines:\n            l1, l2, l3 = lines\n            # Three-line header if line 2 & 3 match (1st line can be anything)\n            if header_test2(l2) and header_test3(l3):\n                header_rows = [l1, l2, l3]\n            else:\n                lines, data = [], chain((l1, l2, l3), data)\n\n        # Try to parse a single-line header\n        if not header_rows:\n            try:\n                lines.append(list(next(data)))\n            except StopIteration:\n                pass\n            if lines:\n                # Header if none of the values in line 1 parses as a number\n                if not all(is_number(i) for i in lines[0]):\n                    header_rows = [lines[0]]\n                else:\n                    data = chain(lines, data)\n\n        return header_rows, data\n\n    @classmethod\n    def data_table(cls, data, headers=None):\n        \"\"\"\n        Return Orange.data.Table given rows of `headers` (iterable of iterable)\n        and rows of `data` (iterable of iterable).\n\n        Basically, the idea of subclasses is to produce those two iterables,\n        however they might.\n\n        If `headers` is not provided, the header rows are extracted from `data`,\n        assuming they precede it.\n        \"\"\"\n        if not headers:\n            headers, data = cls.parse_headers(data)\n\n        # Consider various header types (single-row, two-row, three-row, none)\n        if len(headers) == 3:\n            names, types, flags = map(list, headers)\n        else:\n            if len(headers) == 1:\n                HEADER1_FLAG_SEP = '#'\n                # First row format either:\n                #   1) delimited column names\n                #   2) -||- with type and flags prepended, separated by #,\n                #      e.g. d#sex,c#age,cC#IQ\n                _flags, names = zip(*[i.split(HEADER1_FLAG_SEP, 1)\n                                      if HEADER1_FLAG_SEP in i else ('', i)\n                                      for i in headers[0]]\n                                   )\n                names = list(names)\n            elif len(headers) == 2:\n                names, _flags = map(list, headers)\n            else:\n                # Use heuristics for everything\n                names, _flags = [], []\n            types = [''.join(filter(str.isupper, flag)).lower() for flag in _flags]\n            flags = [Flags.join(filter(str.islower, flag)) for flag in _flags]\n\n        # Determine maximum row length\n        rowlen = max(map(len, (names, types, flags)))\n\n        strip = False\n\n        def _equal_length(lst):\n            nonlocal strip\n            if len(lst) > rowlen > 0:\n                lst = lst[:rowlen]\n                strip = True\n            elif len(lst) < rowlen:\n                lst.extend(['']*(rowlen - len(lst)))\n            return lst\n\n        # Ensure all data is of equal width in a column-contiguous array\n        data = [_equal_length([s.strip() for s in row])\n                for row in data if any(row)]\n        data = np.array(data, dtype=object, order='F')\n\n        if strip:\n            warnings.warn(\"Columns with no headers were removed.\")\n\n        # Data may actually be longer than headers were\n        try:\n            rowlen = data.shape[1]\n        except IndexError:\n            pass\n        else:\n            for lst in (names, types, flags):\n                _equal_length(lst)\n\n        NAMEGEN = namegen('Feature ', 1)\n        Xcols, attrs = [], []\n        Mcols, metas = [], []\n        Ycols, clses = [], []\n        Wcols = []\n\n        # Rename variables if necessary\n        # Reusing across files still works if both files have same duplicates\n        name_counts = Counter(names)\n        del name_counts[\"\"]\n        if len(name_counts) != len(names) and name_counts:\n            uses = {name: 0 for name, count in name_counts.items() if count > 1}\n            for i, name in enumerate(names):\n                if name in uses:\n                    uses[name] += 1\n                    names[i] = \"{}_{}\".format(name, uses[name])\n\n        namask = np.empty(data.shape[0], dtype=bool)\n        # Iterate through the columns\n        for col in range(rowlen):\n            flag = Flags(Flags.split(flags[col]))\n            if flag.i:\n                continue\n\n            type_flag = types and types[col].strip()\n            try:\n                orig_values = data[:, col]\n            except IndexError:\n                orig_values = np.array([], dtype=object)\n\n            namask = isnastr(orig_values, out=namask)\n\n            coltype_kwargs = {}\n            valuemap = None\n            values = orig_values\n\n            if type_flag in StringVariable.TYPE_HEADERS:\n                coltype = StringVariable\n                values = orig_values\n            elif type_flag in ContinuousVariable.TYPE_HEADERS:\n                coltype = ContinuousVariable\n                values = np.empty(data.shape[0], dtype=float)\n                try:\n                    np.copyto(values, orig_values, casting=\"unsafe\",\n                              where=~namask)\n                    values[namask] = np.nan\n                except ValueError:\n                    for row, num in enumerate(orig_values):\n                        if not isnastr(num):\n                            try:\n                                float(num)\n                            except ValueError:\n                                break\n                    raise ValueError('Non-continuous value in (1-based) '\n                                     'line {}, column {}'.format(row + len(headers) + 1,\n                                                                 col + 1))\n\n            elif type_flag in TimeVariable.TYPE_HEADERS:\n                coltype = TimeVariable\n                values = np.where(namask, \"\", orig_values)\n            elif (type_flag in DiscreteVariable.TYPE_HEADERS or\n                  _RE_DISCRETE_LIST.match(type_flag)):\n                coltype = DiscreteVariable\n                orig_values = values = np.where(namask, \"\", orig_values)\n                if _RE_DISCRETE_LIST.match(type_flag):\n                    valuemap = Flags.split(type_flag)\n                    coltype_kwargs.update(ordered=True)\n                else:\n                    valuemap = sorted(set(orig_values) - {\"\"})\n            else:\n                # No known type specified, use heuristics\n                valuemap, values, coltype = guess_data_type(orig_values, namask)\n\n            if flag.m or coltype is StringVariable:\n                append_to = (Mcols, metas)\n            elif flag.w:\n                append_to = (Wcols, None)\n            elif flag.c:\n                append_to = (Ycols, clses)\n            else:\n                append_to = (Xcols, attrs)\n\n            cols, domain_vars = append_to\n\n            if domain_vars is not None:\n                var_name = names and names[col]\n                if not var_name:\n                    var_name = next(NAMEGEN)\n\n                values, var = sanitize_variable(\n                    valuemap, values, orig_values, coltype, coltype_kwargs,\n                    name=var_name)\n            else:\n                var = None\n            if domain_vars is not None:\n                var.attributes.update(flag.attributes)\n                domain_vars.append(var)\n\n            if isinstance(values, np.ndarray) and not values.flags.owndata:\n                values = values.copy()  # might view `data` (string columns)\n            cols.append(values)\n\n            try:\n                # allow gc to reclaim memory used by string values\n                data[:, col] = None\n            except IndexError:\n                pass\n\n        domain = Domain(attrs, clses, metas)\n\n        if not data.size:\n            return Table.from_domain(domain, 0)\n\n        X = Y = M = W = None\n        if Xcols:\n            X = np.c_[tuple(Xcols)]\n            assert X.dtype == np.float_\n        else:\n            X = np.empty((data.shape[0], 0), dtype=np.float_)\n        if Ycols:\n            Y = np.c_[tuple(Ycols)]\n            assert Y.dtype == np.float_\n        if Mcols:\n            M = np.c_[tuple(Mcols)].astype(object)\n        if Wcols:\n            W = np.c_[tuple(Wcols)].astype(float)\n\n        table = Table.from_numpy(domain, X, Y, M, W)\n        return table\n\n    @staticmethod\n    def header_names(data):\n        return ['weights'] * data.has_weights() + \\\n               [v.name for v in chain(data.domain.attributes,\n                                      data.domain.class_vars,\n                                      data.domain.metas)]\n\n    @staticmethod\n    def header_types(data):\n        def _vartype(var):\n            if var.is_continuous or var.is_string:\n                return var.TYPE_HEADERS[0]\n            elif var.is_discrete:\n                return Flags.join(var.values) if var.ordered else var.TYPE_HEADERS[0]\n            raise NotImplementedError\n        return ['continuous'] * data.has_weights() + \\\n               [_vartype(v) for v in chain(data.domain.attributes,\n                                           data.domain.class_vars,\n                                           data.domain.metas)]\n\n    @staticmethod\n    def header_flags(data):\n        return list(chain(['weight'] * data.has_weights(),\n                          (Flags.join([flag], *('{}={}'.format(*a)\n                                                for a in sorted(var.attributes.items())))\n                           for flag, var in chain(zip(repeat(''), data.domain.attributes),\n                                                  zip(repeat('class'), data.domain.class_vars),\n                                                  zip(repeat('meta'), data.domain.metas)))))\n\n    @classmethod\n    def write_headers(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        write(cls.header_names(data))\n        write(cls.header_types(data))\n        write(cls.header_flags(data))\n\n    @classmethod\n    def write_data(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        vars = list(chain((ContinuousVariable('_w'),) if data.has_weights() else (),\n                          data.domain.attributes,\n                          data.domain.class_vars,\n                          data.domain.metas))\n\n        def formatter(var):\n            # type: (Variable) -> Callable[[Variable], Any]\n            # Return a column 'formatter' function. The function must return\n            # something that `write` knows how to write\n            if var.is_time:\n                return var.repr_val\n            elif var.is_continuous:\n                return lambda value: \"\" if isnan(value) else value\n            elif var.is_discrete:\n                return lambda value: \"\" if isnan(value) else var.values[int(value)]\n            elif var.is_string:\n                return lambda value: value\n            else:\n                return var.repr_val\n\n        formatters = [formatter(v) for v in vars]\n        for row in zip(data.W if data.W.ndim > 1 else data.W[:, np.newaxis],\n                       data.X,\n                       data.Y if data.Y.ndim > 1 else data.Y[:, np.newaxis],\n                       data.metas):\n            write([fmt(v) for fmt, v in zip(formatters, flatten(row))])\n\n    @classmethod\n    def qualified_name(cls):\n        return cls.__module__ + '.' + cls.__name__",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 362,
                            "end_lineno": 847,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# Test methods with long descriptive names can omit docstrings\n# pylint: disable=missing-docstring\n\nimport unittest\nimport os\nimport tempfile\nimport shutil\n\nfrom Orange.data import ContinuousVariable\nfrom Orange.data.io import FileFormat, TabReader, CSVReader, PickleReader, \\\n    sanitize_variable\nfrom Orange.data.table import get_sample_datasets_dir\nfrom Orange.version import version\n\n\nclass WildcardReader(FileFormat):\n    EXTENSIONS = ('.wild', '.wild[0-9]')\n    DESCRIPTION = \"Dummy reader for testing extensions\"\n\n    def read(self):\n        pass\n\n\nclass TestChooseReader(unittest.TestCase):\n\n    def test_usual_extensions(self):\n        self.assertIsInstance(FileFormat.get_reader(\"t.tab\"), TabReader)\n        self.assertIsInstance(FileFormat.get_reader(\"t.csv\"), CSVReader)\n        self.assertIsInstance(FileFormat.get_reader(\"t.pkl\"), PickleReader)\n        with self.assertRaises(OSError):\n            FileFormat.get_reader(\"test.undefined_extension\")\n\n    def test_wildcard_extension(self):\n        self.assertIsInstance(FileFormat.get_reader(\"t.wild\"),\n                              WildcardReader)\n        self.assertIsInstance(FileFormat.get_reader(\"t.wild2\"),\n                              WildcardReader)\n        with self.assertRaises(OSError):\n            FileFormat.get_reader(\"t.wild2a\")\n\n\nclass SameExtension(FileFormat):\n    PRIORITY = 100\n    EXTENSIONS = ('.same_extension',)\n    DESCRIPTION = \"Same extension, different priority\"\n\n    def read(self):\n        pass\n\n\nclass SameExtensionPreferred(SameExtension):\n    PRIORITY = 90\n\n\nclass SameExtensionL(SameExtension):\n    PRIORITY = 110\n\n\nclass TestMultipleSameExtension(unittest.TestCase):\n\n    def test_find_reader(self):\n        reader = FileFormat.get_reader(\"some.same_extension\")\n        self.assertIsInstance(reader, SameExtensionPreferred)\n\n\nclass TestLocate(unittest.TestCase):\n\n    def test_locate_sample_datasets(self):\n        with self.assertRaises(OSError):\n            FileFormat.locate(\"iris.tab\",\n                              search_dirs=[os.path.dirname(__file__)])\n        iris = FileFormat.locate(\"iris.tab\",\n                                 search_dirs=[get_sample_datasets_dir()])\n        self.assertEqual(os.path.basename(iris), \"iris.tab\")\n        # test extension adding\n        iris = FileFormat.locate(\"iris\",\n                                 search_dirs=[get_sample_datasets_dir()])\n        self.assertEqual(os.path.basename(iris), \"iris.tab\")\n\n    def test_locate_wildcard_extension(self):\n        tempdir = tempfile.mkdtemp()\n        with self.assertRaises(OSError):\n            FileFormat.locate(\"t.wild9\", search_dirs=[tempdir])\n        fn = os.path.join(tempdir, \"t.wild8\")\n        with open(fn, \"wt\") as f:\n            f.write(\"\\n\")\n        l = FileFormat.locate(\"t.wild8\", search_dirs=[tempdir])\n        self.assertEqual(l, fn)\n        # test extension adding\n        l = FileFormat.locate(\"t\", search_dirs=[tempdir])\n        self.assertEqual(l, fn)\n        shutil.rmtree(tempdir)\n\n\nclass TestReader(unittest.TestCase):\n\n    def test_open_bad_pickle(self):\n        \"\"\"\n        Raise TypeError when PickleReader reads a pickle\n        file without a table (and it suppose to be there).\n        GH-2232\n        \"\"\"\n        reader = PickleReader(\"\")\n        with unittest.mock.patch(\"pickle.load\", return_value=None):\n            self.assertRaises(TypeError, reader.read, \"foo\")\n\n\nclass TestIo(unittest.TestCase):\n    def test_sanitize_variable_deprecated_params(self):\n        \"\"\"In version 3.18 deprecation warnings in function 'sanitize_variable'\n        should be removed along with unused parameters.\"\"\"\n        if version > \"3.18\":\n            _, _ = sanitize_variable(None, None, None, ContinuousVariable,\n                                     {}, name=\"name\", data=\"data\")\n",
            "new_file_content": "# Test methods with long descriptive names can omit docstrings\n# pylint: disable=missing-docstring\n\nimport unittest\nimport os\nimport tempfile\nimport shutil\nimport io\n\nfrom Orange.data import ContinuousVariable\nfrom Orange.data.io import FileFormat, TabReader, CSVReader, PickleReader, \\\n    sanitize_variable\nfrom Orange.data.table import get_sample_datasets_dir\nfrom Orange.version import version\n\n\nclass WildcardReader(FileFormat):\n    EXTENSIONS = ('.wild', '.wild[0-9]')\n    DESCRIPTION = \"Dummy reader for testing extensions\"\n\n    def read(self):\n        pass\n\n\nclass TestChooseReader(unittest.TestCase):\n\n    def test_usual_extensions(self):\n        self.assertIsInstance(FileFormat.get_reader(\"t.tab\"), TabReader)\n        self.assertIsInstance(FileFormat.get_reader(\"t.csv\"), CSVReader)\n        self.assertIsInstance(FileFormat.get_reader(\"t.pkl\"), PickleReader)\n        with self.assertRaises(OSError):\n            FileFormat.get_reader(\"test.undefined_extension\")\n\n    def test_wildcard_extension(self):\n        self.assertIsInstance(FileFormat.get_reader(\"t.wild\"),\n                              WildcardReader)\n        self.assertIsInstance(FileFormat.get_reader(\"t.wild2\"),\n                              WildcardReader)\n        with self.assertRaises(OSError):\n            FileFormat.get_reader(\"t.wild2a\")\n\n\nclass SameExtension(FileFormat):\n    PRIORITY = 100\n    EXTENSIONS = ('.same_extension',)\n    DESCRIPTION = \"Same extension, different priority\"\n\n    def read(self):\n        pass\n\n\nclass SameExtensionPreferred(SameExtension):\n    PRIORITY = 90\n\n\nclass SameExtensionL(SameExtension):\n    PRIORITY = 110\n\n\nclass TestMultipleSameExtension(unittest.TestCase):\n\n    def test_find_reader(self):\n        reader = FileFormat.get_reader(\"some.same_extension\")\n        self.assertIsInstance(reader, SameExtensionPreferred)\n\n\nclass TestLocate(unittest.TestCase):\n\n    def test_locate_sample_datasets(self):\n        with self.assertRaises(OSError):\n            FileFormat.locate(\"iris.tab\",\n                              search_dirs=[os.path.dirname(__file__)])\n        iris = FileFormat.locate(\"iris.tab\",\n                                 search_dirs=[get_sample_datasets_dir()])\n        self.assertEqual(os.path.basename(iris), \"iris.tab\")\n        # test extension adding\n        iris = FileFormat.locate(\"iris\",\n                                 search_dirs=[get_sample_datasets_dir()])\n        self.assertEqual(os.path.basename(iris), \"iris.tab\")\n\n    def test_locate_wildcard_extension(self):\n        tempdir = tempfile.mkdtemp()\n        with self.assertRaises(OSError):\n            FileFormat.locate(\"t.wild9\", search_dirs=[tempdir])\n        fn = os.path.join(tempdir, \"t.wild8\")\n        with open(fn, \"wt\") as f:\n            f.write(\"\\n\")\n        l = FileFormat.locate(\"t.wild8\", search_dirs=[tempdir])\n        self.assertEqual(l, fn)\n        # test extension adding\n        l = FileFormat.locate(\"t\", search_dirs=[tempdir])\n        self.assertEqual(l, fn)\n        shutil.rmtree(tempdir)\n\n\nclass TestReader(unittest.TestCase):\n\n    def test_open_bad_pickle(self):\n        \"\"\"\n        Raise TypeError when PickleReader reads a pickle\n        file without a table (and it suppose to be there).\n        GH-2232\n        \"\"\"\n        reader = PickleReader(\"\")\n        with unittest.mock.patch(\"pickle.load\", return_value=None):\n            self.assertRaises(TypeError, reader.read, \"foo\")\n\n    def test_empty_columns(self):\n        \"\"\"Can't read files with more columns then headers. GH-1417\"\"\"\n        samplefile = \"\"\"\\\n        a, b\n        1, 0,\n        1, 2,\n        \"\"\"\n        c = io.StringIO(samplefile)\n        with self.assertWarns(UserWarning) as cm:\n            table = CSVReader(c).read()\n        self.assertEqual(len(table.domain.attributes), 2)\n        self.assertEqual(cm.warning.args[0],\n                         \"Columns with no headers were removed.\")\n\n\nclass TestIo(unittest.TestCase):\n    def test_sanitize_variable_deprecated_params(self):\n        \"\"\"In version 3.18 deprecation warnings in function 'sanitize_variable'\n        should be removed along with unused parameters.\"\"\"\n        if version > \"3.18\":\n            _, _ = sanitize_variable(None, None, None, ContinuousVariable,\n                                     {}, name=\"name\", data=\"data\")\n",
            "header": {
                "file": {
                    "path": "Orange/tests/test_io.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "ad6731c67",
                "new_commit_hash": "9dee28e52",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/Orange/tests/test_io.py"
            },
            "plus_file": {
                "path": "b/Orange/tests/test_io.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 5,
                            "length": 6
                        },
                        "new_range": {
                            "start": 5,
                            "length": 7
                        },
                        "section": "import unittest"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "import os",
                                "type": "context"
                            },
                            {
                                "content": "import tempfile",
                                "type": "context"
                            },
                            {
                                "content": "import shutil",
                                "type": "context"
                            },
                            {
                                "content": "import io",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "from Orange.data import ContinuousVariable",
                                "type": "context"
                            },
                            {
                                "content": "from Orange.data.io import FileFormat, TabReader, CSVReader, PickleReader, \\",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "Orange/tests/test_io.py",
                            "type": "import",
                            "name": "io",
                            "content": "import io",
                            "ast_type_str": "Import",
                            "start_lineno": 8,
                            "end_lineno": 8,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 104,
                            "length": 6
                        },
                        "new_range": {
                            "start": 105,
                            "length": 20
                        },
                        "section": "class TestReader(unittest.TestCase):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        with unittest.mock.patch(\"pickle.load\", return_value=None):",
                                "type": "context"
                            },
                            {
                                "content": "            self.assertRaises(TypeError, reader.read, \"foo\")",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    def test_empty_columns(self):",
                                "type": "added"
                            },
                            {
                                "content": "        \"\"\"Can't read files with more columns then headers. GH-1417\"\"\"",
                                "type": "added"
                            },
                            {
                                "content": "        samplefile = \"\"\"\\",
                                "type": "added"
                            },
                            {
                                "content": "        a, b",
                                "type": "added"
                            },
                            {
                                "content": "        1, 0,",
                                "type": "added"
                            },
                            {
                                "content": "        1, 2,",
                                "type": "added"
                            },
                            {
                                "content": "        \"\"\"",
                                "type": "added"
                            },
                            {
                                "content": "        c = io.StringIO(samplefile)",
                                "type": "added"
                            },
                            {
                                "content": "        with self.assertWarns(UserWarning) as cm:",
                                "type": "added"
                            },
                            {
                                "content": "            table = CSVReader(c).read()",
                                "type": "added"
                            },
                            {
                                "content": "        self.assertEqual(len(table.domain.attributes), 2)",
                                "type": "added"
                            },
                            {
                                "content": "        self.assertEqual(cm.warning.args[0],",
                                "type": "added"
                            },
                            {
                                "content": "                         \"Columns with no headers were removed.\")",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "class TestIo(unittest.TestCase):",
                                "type": "context"
                            },
                            {
                                "content": "    def test_sanitize_variable_deprecated_params(self):",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "Orange/tests/test_io.py",
                            "type": "class",
                            "name": "TestReader",
                            "content": "class TestReader(unittest.TestCase):\n\n    def test_open_bad_pickle(self):\n        \"\"\"\n        Raise TypeError when PickleReader reads a pickle\n        file without a table (and it suppose to be there).\n        GH-2232\n        \"\"\"\n        reader = PickleReader(\"\")\n        with unittest.mock.patch(\"pickle.load\", return_value=None):\n            self.assertRaises(TypeError, reader.read, \"foo\")\n\n    def test_empty_columns(self):\n        \"\"\"Can't read files with more columns then headers. GH-1417\"\"\"\n        samplefile = \"\"\"\\\n        a, b\n        1, 0,\n        1, 2,\n        \"\"\"\n        c = io.StringIO(samplefile)\n        with self.assertWarns(UserWarning) as cm:\n            table = CSVReader(c).read()\n        self.assertEqual(len(table.domain.attributes), 2)\n        self.assertEqual(cm.warning.args[0],\n                         \"Columns with no headers were removed.\")",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 96,
                            "end_lineno": 120,
                            "parent": null
                        }
                    ],
                    "added_entities": [
                        {
                            "file_name": "Orange/tests/test_io.py",
                            "type": "method",
                            "name": "TestReader.test_empty_columns",
                            "content": "class TestReader(unittest.TestCase):\n\n    def test_open_bad_pickle(self):\n        \"\"\"\n        Raise TypeError when PickleReader reads a pickle\n        file without a table (and it suppose to be there).\n        GH-2232\n        \"\"\"\n        reader = PickleReader(\"\")\n        with unittest.mock.patch(\"pickle.load\", return_value=None):\n            self.assertRaises(TypeError, reader.read, \"foo\")\n\n    def test_empty_columns(self):\n        \"\"\"Can't read files with more columns then headers. GH-1417\"\"\"\n        samplefile = \"\"\"\\\n        a, b\n        1, 0,\n        1, 2,\n        \"\"\"\n        c = io.StringIO(samplefile)\n        with self.assertWarns(UserWarning) as cm:\n            table = CSVReader(c).read()\n        self.assertEqual(len(table.domain.attributes), 2)\n        self.assertEqual(cm.warning.args[0],\n                         \"Columns with no headers were removed.\")",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 108,
                            "end_lineno": 120,
                            "parent": {
                                "file_name": "Orange/tests/test_io.py",
                                "type": "class",
                                "name": "TestReader",
                                "content": "class TestReader(unittest.TestCase):\n\n    def test_open_bad_pickle(self):\n        \"\"\"\n        Raise TypeError when PickleReader reads a pickle\n        file without a table (and it suppose to be there).\n        GH-2232\n        \"\"\"\n        reader = PickleReader(\"\")\n        with unittest.mock.patch(\"pickle.load\", return_value=None):\n            self.assertRaises(TypeError, reader.read, \"foo\")\n\n    def test_empty_columns(self):\n        \"\"\"Can't read files with more columns then headers. GH-1417\"\"\"\n        samplefile = \"\"\"\\\n        a, b\n        1, 0,\n        1, 2,\n        \"\"\"\n        c = io.StringIO(samplefile)\n        with self.assertWarns(UserWarning) as cm:\n            table = CSVReader(c).read()\n        self.assertEqual(len(table.domain.attributes), 2)\n        self.assertEqual(cm.warning.args[0],\n                         \"Columns with no headers were removed.\")",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 96,
                                "end_lineno": 120,
                                "parent": null
                            }
                        }
                    ],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "1a916d7bffde022f04cc4547d35adda82484d151^",
    "new_commit_hash": "1a916d7bffde022f04cc4547d35adda82484d151",
    "commit_message": "Merge pull request #3237 from robertcv/fixes/file_strip_columns\n\n[FIX] io: Handle mismatched number of header/data values",
    "commit_date": "2018-09-12T14:28:17+02:00",
    "metadata": {}
}
{
    "repo_name": "orange3",
    "new_commit_hash": "1a916d7bffde022f04cc4547d35adda82484d151",
    "test_file_codes": [
        "# Test methods with long descriptive names can omit docstrings\n# pylint: disable=missing-docstring\n\nimport unittest\nimport os\nimport tempfile\nimport shutil\nimport io\n\nfrom Orange.data import ContinuousVariable\nfrom Orange.data.io import FileFormat, TabReader, CSVReader, PickleReader, \\\n    sanitize_variable\nfrom Orange.data.table import get_sample_datasets_dir\nfrom Orange.version import version\n\n\nclass WildcardReader(FileFormat):\n    EXTENSIONS = ('.wild', '.wild[0-9]')\n    DESCRIPTION = \"Dummy reader for testing extensions\"\n\n    def read(self):\n        pass\n\n\nclass TestChooseReader(unittest.TestCase):\n\n    def test_usual_extensions(self):\n        self.assertIsInstance(FileFormat.get_reader(\"t.tab\"), TabReader)\n        self.assertIsInstance(FileFormat.get_reader(\"t.csv\"), CSVReader)\n        self.assertIsInstance(FileFormat.get_reader(\"t.pkl\"), PickleReader)\n        with self.assertRaises(OSError):\n            FileFormat.get_reader(\"test.undefined_extension\")\n\n    def test_wildcard_extension(self):\n        self.assertIsInstance(FileFormat.get_reader(\"t.wild\"),\n                              WildcardReader)\n        self.assertIsInstance(FileFormat.get_reader(\"t.wild2\"),\n                              WildcardReader)\n        with self.assertRaises(OSError):\n            FileFormat.get_reader(\"t.wild2a\")\n\n\nclass SameExtension(FileFormat):\n    PRIORITY = 100\n    EXTENSIONS = ('.same_extension',)\n    DESCRIPTION = \"Same extension, different priority\"\n\n    def read(self):\n        pass\n\n\nclass SameExtensionPreferred(SameExtension):\n    PRIORITY = 90\n\n\nclass SameExtensionL(SameExtension):\n    PRIORITY = 110\n\n\nclass TestMultipleSameExtension(unittest.TestCase):\n\n    def test_find_reader(self):\n        reader = FileFormat.get_reader(\"some.same_extension\")\n        self.assertIsInstance(reader, SameExtensionPreferred)\n\n\nclass TestLocate(unittest.TestCase):\n\n    def test_locate_sample_datasets(self):\n        with self.assertRaises(OSError):\n            FileFormat.locate(\"iris.tab\",\n                              search_dirs=[os.path.dirname(__file__)])\n        iris = FileFormat.locate(\"iris.tab\",\n                                 search_dirs=[get_sample_datasets_dir()])\n        self.assertEqual(os.path.basename(iris), \"iris.tab\")\n        # test extension adding\n        iris = FileFormat.locate(\"iris\",\n                                 search_dirs=[get_sample_datasets_dir()])\n        self.assertEqual(os.path.basename(iris), \"iris.tab\")\n\n    def test_locate_wildcard_extension(self):\n        tempdir = tempfile.mkdtemp()\n        with self.assertRaises(OSError):\n            FileFormat.locate(\"t.wild9\", search_dirs=[tempdir])\n        fn = os.path.join(tempdir, \"t.wild8\")\n        with open(fn, \"wt\") as f:\n            f.write(\"\\n\")\n        l = FileFormat.locate(\"t.wild8\", search_dirs=[tempdir])\n        self.assertEqual(l, fn)\n        # test extension adding\n        l = FileFormat.locate(\"t\", search_dirs=[tempdir])\n        self.assertEqual(l, fn)\n        shutil.rmtree(tempdir)\n\n\nclass TestReader(unittest.TestCase):\n\n    def test_open_bad_pickle(self):\n        \"\"\"\n        Raise TypeError when PickleReader reads a pickle\n        file without a table (and it suppose to be there).\n        GH-2232\n        \"\"\"\n        reader = PickleReader(\"\")\n        with unittest.mock.patch(\"pickle.load\", return_value=None):\n            self.assertRaises(TypeError, reader.read, \"foo\")\n\n    def test_empty_columns(self):\n        \"\"\"Can't read files with more columns then headers. GH-1417\"\"\"\n        samplefile = \"\"\"\\\n        a, b\n        1, 0,\n        1, 2,\n        \"\"\"\n        c = io.StringIO(samplefile)\n        with self.assertWarns(UserWarning) as cm:\n            table = CSVReader(c).read()\n        self.assertEqual(len(table.domain.attributes), 2)\n        self.assertEqual(cm.warning.args[0],\n                         \"Columns with no headers were removed.\")\n\n\nclass TestIo(unittest.TestCase):\n    def test_sanitize_variable_deprecated_params(self):\n        \"\"\"In version 3.18 deprecation warnings in function 'sanitize_variable'\n        should be removed along with unused parameters.\"\"\"\n        if version > \"3.18\":\n            _, _ = sanitize_variable(None, None, None, ContinuousVariable,\n                                     {}, name=\"name\", data=\"data\")\n"
    ],
    "test_file_names": [
        "test_1.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Orange installation attempts...\nAttempting installation with Python 3.7...\nWarning: 'keywords' should be a list, got type 'tuple'\nWarning: 'classifiers' should be a list, got type 'tuple'\nWarning: 'keywords' should be a list, got type 'tuple'\nWarning: 'classifiers' should be a list, got type 'tuple'\nrunning build_ext\nrunning build_src\nbuild_src\nbuilding extension \"Orange.classification._simple_tree\" sources\nbuilding extension \"Orange.classification._tree_scorers\" sources\nbuilding extension \"Orange.data._valuecount\" sources\nbuilding extension \"Orange.data._contingency\" sources\nbuilding extension \"Orange.data._io\" sources\nbuilding extension \"Orange.data._variable\" sources\nbuilding extension \"Orange.distance._distance\" sources\nbuilding extension \"Orange.preprocess._discretize\" sources\nbuilding extension \"Orange.preprocess._relieff\" sources\nbuilding extension \"Orange.widgets.utils._grid_density\" sources\nbuild_src: building npy-pkg config files\ncustomize UnixCCompiler\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating /tmp/tmpqkzq2u7l/tmp\ncreating /tmp/tmpqkzq2u7l/tmp/tmpqkzq2u7l\ncompile options: '-MMD -MF /tmp/tmpqkzq2u7l/file.c.d -c'\ncc: /tmp/tmpqkzq2u7l/file.c\ncustomize UnixCCompiler using build_ext\ncustomize UnixCCompiler\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating /tmp/tmpyad8bciz/tmp\ncreating /tmp/tmpyad8bciz/tmp/tmpyad8bciz\ncompile options: '-MMD -MF /tmp/tmpyad8bciz/file.c.d -c'\ncc: /tmp/tmpyad8bciz/file.c\ncustomize UnixCCompiler using build_ext\nbuilding 'Orange.classification._simple_tree' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating build/temp.linux-x86_64-3.7/Orange\ncreating build/temp.linux-x86_64-3.7/Orange/classification\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/classification/_simple_tree.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/classification/_simple_tree.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/classification/_simple_tree.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.classification._tree_scorers' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/classification/_tree_scorers.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/classification/_tree_scorers.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/classification/_tree_scorers.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.data._valuecount' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating build/temp.linux-x86_64-3.7/Orange/data\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/data/_valuecount.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/data/_valuecount.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/data/_valuecount.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.data._contingency' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/data/_contingency.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/data/_contingency.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/data/_contingency.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.data._io' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/data/_io.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/data/_io.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/data/_io.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.data._variable' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/data/_variable.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/data/_variable.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/data/_variable.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.distance._distance' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating build/temp.linux-x86_64-3.7/Orange/distance\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/distance/_distance.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/distance/_distance.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/distance/_distance.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.preprocess._discretize' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating build/temp.linux-x86_64-3.7/Orange/preprocess\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/preprocess/_discretize.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/preprocess/_discretize.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/preprocess/_discretize.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.preprocess._relieff' extension\ncompiling C++ sources\nC compiler: c++ -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\nc++: Orange/preprocess/_relieff.cpp\nc++ -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/preprocess/_relieff.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/preprocess/_relieff.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.widgets.utils._grid_density' extension\ncompiling C++ sources\nC compiler: c++ -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating build/temp.linux-x86_64-3.7/Orange/widgets\ncreating build/temp.linux-x86_64-3.7/Orange/widgets/utils\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\nc++: Orange/widgets/utils/_grid_density.cpp\nc++ -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/widgets/utils/_grid_density.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/widgets/utils/_grid_density.cpython-37m-x86_64-linux-gnu.so\nWarning: 'keywords' should be a list, got type 'tuple'\nWarning: 'classifiers' should be a list, got type 'tuple'\nWarning: 'keywords' should be a list, got type 'tuple'\nWarning: 'classifiers' should be a list, got type 'tuple'\nrunning develop\nrunning build_scripts\nrunning egg_info\nrunning build_src\nbuild_src\nbuilding extension \"Orange.classification._simple_tree\" sources\nbuilding extension \"Orange.classification._tree_scorers\" sources\nbuilding extension \"Orange.data._valuecount\" sources\nbuilding extension \"Orange.data._contingency\" sources\nbuilding extension \"Orange.data._io\" sources\nbuilding extension \"Orange.data._variable\" sources\nbuilding extension \"Orange.distance._distance\" sources\nbuilding extension \"Orange.preprocess._discretize\" sources\nbuilding extension \"Orange.preprocess._relieff\" sources\nbuilding extension \"Orange.widgets.utils._grid_density\" sources\nbuild_src: building npy-pkg config files\ncreating Orange3.egg-info\nwriting Orange3.egg-info/PKG-INFO\nwriting dependency_links to Orange3.egg-info/dependency_links.txt\nwriting entry points to Orange3.egg-info/entry_points.txt\nwriting requirements to Orange3.egg-info/requires.txt\nwriting top-level names to Orange3.egg-info/top_level.txt\nwriting manifest file 'Orange3.egg-info/SOURCES.txt'\nreading manifest file 'Orange3.egg-info/SOURCES.txt'\nreading manifest template 'MANIFEST.in'\nwarning: no files found matching 'README*' under directory 'Orange'\nwarning: no files found matching 'LICENSE' under directory 'Orange'\nwarning: no files found matching '*.basket' under directory 'Orange/datasets'\nwarning: no files found matching '*.html' under directory 'Orange/canvas'\nwarning: no files found matching '*.css' under directory 'Orange/widgets'\nwarning: no files found matching 'COPYING'\nadding license file 'LICENSE'\nwriting manifest file 'Orange3.egg-info/SOURCES.txt'\nrunning build_ext\ncustomize UnixCCompiler\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating /tmp/tmpmrywllz2/tmp\ncreating /tmp/tmpmrywllz2/tmp/tmpmrywllz2\ncompile options: '-MMD -MF /tmp/tmpmrywllz2/file.c.d -c'\ncc: /tmp/tmpmrywllz2/file.c\ncustomize UnixCCompiler using build_ext\ncustomize UnixCCompiler\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating /tmp/tmp8iap2w94/tmp\ncreating /tmp/tmp8iap2w94/tmp/tmp8iap2w94\ncompile options: '-MMD -MF /tmp/tmp8iap2w94/file.c.d -c'\ncc: /tmp/tmp8iap2w94/file.c\ncustomize UnixCCompiler using build_ext\nCreating /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/Orange3.egg-link (link to .)\nAdding Orange3 3.16.0.dev0+1a916d7 to easy-install.pth file\nInstalling orange-canvas script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\n\nInstalled /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151\nProcessing dependencies for Orange3==3.16.0.dev0+1a916d7\nSearching for xlrd==2.0.1\nBest match: xlrd 2.0.1\nAdding xlrd 2.0.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for setuptools==59.8.0\nBest match: setuptools 59.8.0\nAdding setuptools 59.8.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for serverfiles==0.3.1\nBest match: serverfiles 0.3.1\nAdding serverfiles 0.3.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for scipy==1.7.3\nBest match: scipy 1.7.3\nAdding scipy 1.7.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for scikit-learn==1.0.2\nBest match: scikit-learn 1.0.2\nAdding scikit-learn 1.0.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for python-louvain==0.16\nBest match: python-louvain 0.16\nAdding python-louvain 0.16 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for pyqtgraph==0.12.4\nBest match: pyqtgraph 0.12.4\nAdding pyqtgraph 0.12.4 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for pip==24.0\nBest match: pip 24.0\nAdding pip 24.0 to easy-install.pth file\nInstalling pip script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\nInstalling pip3 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\nInstalling pip3.10 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for numpy==1.17.5\nBest match: numpy 1.17.5\nAdding numpy 1.17.5 to easy-install.pth file\nInstalling f2py script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\nInstalling f2py3 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\nInstalling f2py3.7 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for networkx==2.6.3\nBest match: networkx 2.6.3\nAdding networkx 2.6.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for matplotlib==3.5.3\nBest match: matplotlib 3.5.3\nAdding matplotlib 3.5.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for keyrings.alt==4.2.0\nBest match: keyrings.alt 4.2.0\nAdding keyrings.alt 4.2.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for keyring==24.1.1\nBest match: keyring 24.1.1\nAdding keyring 24.1.1 to easy-install.pth file\nInstalling keyring script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for joblib==1.3.2\nBest match: joblib 1.3.2\nAdding joblib 1.3.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for docutils==0.20.1\nBest match: docutils 0.20.1\nAdding docutils 0.20.1 to easy-install.pth file\nInstalling docutils script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for chardet==5.2.0\nBest match: chardet 5.2.0\nAdding chardet 5.2.0 to easy-install.pth file\nInstalling chardetect script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for Bottleneck==1.4.0\nBest match: Bottleneck 1.4.0\nAdding Bottleneck 1.4.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for AnyQt==0.2.0\nBest match: AnyQt 0.2.0\nAdding AnyQt 0.2.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for requests==2.31.0\nBest match: requests 2.31.0\nAdding requests 2.31.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for threadpoolctl==3.1.0\nBest match: threadpoolctl 3.1.0\nAdding threadpoolctl 3.1.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for pyparsing==3.1.4\nBest match: pyparsing 3.1.4\nAdding pyparsing 3.1.4 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for fonttools==4.38.0\nBest match: fonttools 4.38.0\nAdding fonttools 4.38.0 to easy-install.pth file\nInstalling fonttools script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\nInstalling pyftmerge script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\nInstalling pyftsubset script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\nInstalling ttx script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for cycler==0.11.0\nBest match: cycler 0.11.0\nAdding cycler 0.11.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for Pillow==9.5.0\nBest match: Pillow 9.5.0\nAdding Pillow 9.5.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for packaging==24.0\nBest match: packaging 24.0\nAdding packaging 24.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for python-dateutil==2.9.0.post0\nBest match: python-dateutil 2.9.0.post0\nAdding python-dateutil 2.9.0.post0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for kiwisolver==1.4.5\nBest match: kiwisolver 1.4.5\nAdding kiwisolver 1.4.5 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for jaraco.classes==3.2.3\nBest match: jaraco.classes 3.2.3\nAdding jaraco.classes 3.2.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for SecretStorage==3.3.3\nBest match: SecretStorage 3.3.3\nAdding SecretStorage 3.3.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for importlib-metadata==6.7.0\nBest match: importlib-metadata 6.7.0\nAdding importlib-metadata 6.7.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for importlib-resources==5.12.0\nBest match: importlib-resources 5.12.0\nAdding importlib-resources 5.12.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for jeepney==0.8.0\nBest match: jeepney 0.8.0\nAdding jeepney 0.8.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for idna==3.10\nBest match: idna 3.10\nAdding idna 3.10 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for certifi==2024.12.14\nBest match: certifi 2024.12.14\nAdding certifi 2024.12.14 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for urllib3==2.0.7\nBest match: urllib3 2.0.7\nAdding urllib3 2.0.7 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for charset-normalizer==3.4.1\nBest match: charset-normalizer 3.4.1\nAdding charset-normalizer 3.4.1 to easy-install.pth file\nInstalling normalizer script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for six==1.17.0\nBest match: six 1.17.0\nAdding six 1.17.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for typing-extensions==4.7.1\nBest match: typing-extensions 4.7.1\nAdding typing-extensions 4.7.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for more-itertools==9.1.0\nBest match: more-itertools 9.1.0\nAdding more-itertools 9.1.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for cryptography==44.0.0\nBest match: cryptography 44.0.0\nAdding cryptography 44.0.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for zipp==3.15.0\nBest match: zipp 3.15.0\nAdding zipp 3.15.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for cffi==1.15.1\nBest match: cffi 1.15.1\nAdding cffi 1.15.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nSearching for pycparser==2.21\nBest match: pycparser 2.21\nAdding pycparser 2.21 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages\nFinished processing dependencies for Orange3==3.16.0.dev0+1a916d7\nVerifying Orange installation...\n✅ Orange installation successful!\nSuccessfully installed orange using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 18 packages in 284ms\nPrepared 18 packages in 10ms\nInstalled 18 packages in 347ms\n + cython==0.29.37\n + exceptiongroup==1.2.2\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + numpy==1.17.5\n + packaging==24.0\n + pluggy==1.2.0\n + pyqt5==5.15.10\n + pyqt5-qt5==5.15.16\n + pyqt5-sip==12.13.0\n + pyqtwebengine==5.15.6\n + pyqtwebengine-qt5==5.15.16\n + pytest==7.4.4\n + setuptools==59.8.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + wheel==0.42.0\n + zipp==3.15.0\nResolved 31 packages in 109ms\nInstalled 26 packages in 417ms\n + bottleneck==1.4.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + cryptography==44.0.0\n + idna==3.10\n + importlib-resources==5.12.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + joblib==1.3.2\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + more-itertools==9.1.0\n + networkx==2.6.3\n + pip==24.0\n + pycparser==2.21\n + python-louvain==0.16\n + requests==2.31.0\n + scikit-learn==1.0.2\n + scipy==1.7.3\n + secretstorage==3.3.3\n + serverfiles==0.3.1\n + threadpoolctl==3.1.0\n + urllib3==2.0.7\n + xlrd==2.0.1\nResolved 14 packages in 73ms\nInstalled 11 packages in 35ms\n + anyqt==0.2.0\n + cycler==0.11.0\n + docutils==0.20.1\n + fonttools==4.38.0\n + kiwisolver==1.4.5\n + matplotlib==3.5.3\n + pillow==9.5.0\n + pyparsing==3.1.4\n + pyqtgraph==0.12.4\n + python-dateutil==2.9.0.post0\n + six==1.17.0\nResolved 2 packages in 21ms\nInstalled 2 packages in 49ms\n + psycopg2==2.9.9\n + pymssql==2.3.2\nResolved 5 packages in 20ms\nInstalled 2 packages in 95ms\n + pandas==1.3.5\n + pytz==2024.2\nAudited 2 packages in 4ms\nOrange/classification/_simple_tree.c: In function ‘gain_ratio_c’:\nOrange/classification/_simple_tree.c:63:9: warning: implicit declaration of function ‘qsort_r’; did you mean ‘qsort’? [-Wimplicit-function-declaration]\n   63 |         qsort_r(base, nel, size, compar, thunk)\n      |         ^~~~~~~\nOrange/classification/_simple_tree.c:147:9: note: in expansion of macro ‘QSORT_R_FUNC’\n  147 |         QSORT_R_FUNC(examples, size, sizeof(struct Example), (void*) &compar_attr, compar_examples);\n      |         ^~~~~~~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/classification/_tree_scorers.c:567:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_2find_threshold_entropy’:\nOrange/classification/_tree_scorers.c:3115:27: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 3115 |   __pyx_t_7 = ((__pyx_v_N <= __pyx_v_min_leaf) != 0);\n      |                           ^~\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_6find_threshold_MSE’:\nOrange/classification/_tree_scorers.c:4619:27: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 4619 |   __pyx_t_1 = ((__pyx_v_N <= __pyx_v_min_leaf) != 0);\n      |                           ^~\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_8find_binarization_MSE’:\nOrange/classification/_tree_scorers.c:5396:43: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 5396 |           for (__pyx_t_22 = 0; __pyx_t_22 < __pyx_t_21; __pyx_t_22+=1) {\n      |                                           ^\nOrange/classification/_tree_scorers.c:5517:39: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 5517 |           __pyx_t_27 = ((__pyx_v_left >= __pyx_v_min_leaf) != 0);\n      |                                       ^~\nOrange/classification/_tree_scorers.c:5523:53: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 5523 |           __pyx_t_27 = (((__pyx_v_N - __pyx_v_left) >= __pyx_v_min_leaf) != 0);\n      |                                                     ^~\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_14compute_predictions_csr’:\nOrange/classification/_tree_scorers.c:7051:41: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 7051 |         for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_10; __pyx_t_11+=1) {\n      |                                         ^\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_16compute_predictions_csc’:\nOrange/classification/_tree_scorers.c:7741:41: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 7741 |         for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_10; __pyx_t_11+=1) {\n      |                                         ^\nOrange/classification/_tree_scorers.c:7801:143: warning: comparison of integer expressions of different signedness: ‘__pyx_t_5numpy_int32_t’ {aka ‘int’} and ‘unsigned int’ [-Wsign-compare]\n 7801 |               __pyx_t_17 = (((*((__pyx_t_5numpy_int32_t *) ( /* dim=0 */ (__pyx_v_indices.data + __pyx_t_18 * __pyx_v_indices.strides[0]) ))) != __pyx_v_i) != 0);\n      |                                                                                                                                               ^~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/data/_valuecount.c:567:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/data/_contingency.c:567:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nOrange/data/_io.c: In function ‘__pyx_pf_6Orange_4data_3_io_sparse_read_float.constprop’:\nOrange/data/_io.c:3881:24: warning: ‘__pyx_v_decs’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 3881 |           __pyx_v_decs = (__pyx_v_decs / 10.0);\n      |           ~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~\nOrange/data/_io.c:4070:24: warning: ‘__pyx_v_value’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 4070 |           __pyx_t_10 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 221, __pyx_L5_error)\n      |                        ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nOrange/data/_io.c:480:40: warning: ‘__pyx_v_attr_index’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n  480 |   #define PyInt_FromLong               PyLong_FromLong\n      |                                        ^~~~~~~~~~~~~~~\nOrange/data/_io.c:1301:7: note: ‘__pyx_v_attr_index’ was declared here\n 1301 |   int __pyx_v_attr_index;\n      |       ^~~~~~~~~~~~~~~~~~\nOrange/data/_io.c:2935:25: warning: ‘__pyx_v_atomp’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 2935 |           __pyx_v_atomp = (__pyx_v_atomp + 1);\n      |           ~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/distance/_distance.c:567:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/preprocess/_discretize.c:567:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/preprocess/_relieff.cpp:581:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/setuptools/command/install.py:37: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools.\n  setuptools.SetuptoolsDeprecationWarning,\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/setuptools/command/easy_install.py:159: EasyInstallDeprecationWarning: easy_install command is deprecated. Use build and pip and other standards-based tools.\n  EasyInstallDeprecationWarning,\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/setuptools/command/install.py:37: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools.\n  setuptools.SetuptoolsDeprecationWarning,\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151/.venv/lib/python3.7/site-packages/setuptools/command/egg_info.py:628: SetuptoolsDeprecationWarning: Custom 'build_py' does not implement 'get_data_files_without_manifest'.\nPlease extend command classes from setuptools instead of distutils.\n  SetuptoolsDeprecationWarning\n",
    "new_commit_res_code": 0,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151\ncollected 8 items\n\nr2e_tests/test_1.py ........                                             [100%]\n\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::TestChooseReader::test_usual_extensions\nPASSED r2e_tests/test_1.py::TestChooseReader::test_wildcard_extension\nPASSED r2e_tests/test_1.py::TestMultipleSameExtension::test_find_reader\nPASSED r2e_tests/test_1.py::TestLocate::test_locate_sample_datasets\nPASSED r2e_tests/test_1.py::TestLocate::test_locate_wildcard_extension\nPASSED r2e_tests/test_1.py::TestReader::test_empty_columns\nPASSED r2e_tests/test_1.py::TestReader::test_open_bad_pickle\nPASSED r2e_tests/test_1.py::TestIo::test_sanitize_variable_deprecated_params\n============================== 8 passed in 0.52s ===============================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_1a916d7bffde022f04cc4547d35adda82484d151\ncollected 8 items\n\nr2e_tests/test_1.py .....F..                                             [100%]\n\n=================================== FAILURES ===================================\n________________________ TestReader.test_empty_columns _________________________\n\nself = <r2e_tests.test_1.TestReader testMethod=test_empty_columns>\n\n    def test_empty_columns(self):\n        \"\"\"Can't read files with more columns then headers. GH-1417\"\"\"\n        samplefile = \"\"\"\\\n        a, b\n        1, 0,\n        1, 2,\n        \"\"\"\n        c = io.StringIO(samplefile)\n        with self.assertWarns(UserWarning) as cm:\n>           table = CSVReader(c).read()\nE           AssertionError: UserWarning not triggered\n\nr2e_tests/test_1.py:117: AssertionError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::TestChooseReader::test_usual_extensions\nPASSED r2e_tests/test_1.py::TestChooseReader::test_wildcard_extension\nPASSED r2e_tests/test_1.py::TestMultipleSameExtension::test_find_reader\nPASSED r2e_tests/test_1.py::TestLocate::test_locate_sample_datasets\nPASSED r2e_tests/test_1.py::TestLocate::test_locate_wildcard_extension\nPASSED r2e_tests/test_1.py::TestReader::test_open_bad_pickle\nPASSED r2e_tests/test_1.py::TestIo::test_sanitize_variable_deprecated_params\nFAILED r2e_tests/test_1.py::TestReader::test_empty_columns - AssertionError: ...\n========================= 1 failed, 7 passed in 0.52s ==========================\n",
    "old_commit_res_stderr": ""
}
Orange/data/io.py Orange/tests/test_io.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]

**Title:** CSV Reader Fails to Handle Files with More Data Columns Than Headers

**Description:**
When reading a CSV file where data rows contain more columns than the header row, the CSV Reader does not issue a warning and fails to remove the extra columns.

**Example Buggy Code:**
```python
import io
from Orange.data.io import CSVReader

samplefile = """\
a, b
1, 0,
1, 2,
"""
c = io.StringIO(samplefile)
table = CSVReader(c).read()
print(len(table.domain.attributes))  # Expected: 2 attributes
```

**Expected Behavior:**
A warning stating "Columns with no headers were removed." should be issued, and the extra data columns should be stripped, resulting in only the defined header columns being present.

**Actual Behavior:**
No warning is triggered, and the extra columns remain, leading to inconsistencies between the header and data rows.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### Orange/data/io.py
import contextlib
import csv
import locale
import pickle
import re
import subprocess
import sys
import warnings

from ast import literal_eval
from collections import OrderedDict, Counter, defaultdict
from functools import lru_cache
from importlib import import_module
from itertools import chain, repeat
from math import isnan

from os import path, remove
from tempfile import NamedTemporaryFile
from urllib.parse import urlparse, urlsplit, urlunsplit, unquote as urlunquote
from urllib.request import urlopen, Request
from fnmatch import fnmatch
from glob import glob

import numpy as np
from chardet.universaldetector import UniversalDetector

from Orange.data import (
    _io, is_discrete_values, MISSING_VALUES, Table, Domain, Variable,
    DiscreteVariable, StringVariable, ContinuousVariable, TimeVariable,
)
from Orange.util import Registry, flatten, namegen


# Support values longer than 128K (i.e. text contents features)
csv.field_size_limit(100*1024*1024)

__all__ = ["Flags", "FileFormat"]


_IDENTITY = lambda i: i


class Compression:
    """Supported compression extensions"""
    GZIP = '.gz'
    BZIP2 = '.bz2'
    XZ = '.xz'
    all = (GZIP, BZIP2, XZ)


def open_compressed(filename, *args, _open=open, **kwargs):
    """Return seamlessly decompressed open file handle for `filename`"""
    if isinstance(filename, str):
        if filename.endswith(Compression.GZIP):
            from gzip import open as _open
        elif filename.endswith(Compression.BZIP2):
            from bz2 import open as _open
        elif filename.endswith(Compression.XZ):
            from lzma import open as _open
        return _open(filename, *args, **kwargs)
    # Else already a file, just pass it through
    return filename


def detect_encoding(filename):
    """
    Detect encoding of `filename`, which can be a ``str`` filename, a
    ``file``-like object, or ``bytes``.
    """
    # Try with Unix file utility first because it's faster (~10ms vs 100ms)
    if isinstance(filename, str) and not filename.endswith(Compression.all):
        try:
            with subprocess.Popen(('file', '--brief', '--mime-encoding', filename),
                                  stdout=subprocess.PIPE) as process:
                process.wait()
                if process.returncode == 0:
                    encoding = process.stdout.read().strip()
                    # file only supports these encodings; for others it says
                    # unknown-8bit or binary. So we give chardet a chance to do
                    # better
                    if encoding in (b'utf-8', b'us-ascii', b'iso-8859-1',
                                    b'utf-7', b'utf-16le', b'utf-16be', b'ebcdic'):
                        return encoding.decode('us-ascii')
        except OSError:
            pass  # windoze

    # file not available or unable to guess the encoding, have chardet do it
    detector = UniversalDetector()
    # We examine only first N 4kB blocks of file because chardet is really slow
    MAX_BYTES = 4*1024*12

    def _from_file(f):
        detector.feed(f.read(MAX_BYTES))
        detector.close()
        return (detector.result.get('encoding')
                if detector.result.get('confidence', 0) >= .85 else
                'utf-8')

    if isinstance(filename, str):
        with open_compressed(filename, 'rb') as f:
            return _from_file(f)
    elif isinstance(filename, bytes):
        detector.feed(filename[:MAX_BYTES])
        detector.close()
        return detector.result.get('encoding')
    elif hasattr(filename, 'encoding'):
        return filename.encoding
    else:  # assume file-like object that you can iter through
        return _from_file(filename)


__isnastr = np.frompyfunc(
    {v for v in MISSING_VALUES if isinstance(v, str)}.__contains__, 1, 1)

# wrapper for __isnastr with proper default out dtype
def isnastr(arr, out=None):
    """
    Given an (object) array of string values, return a boolean mask array
    that is True where the `arr` contains one of the string constants
    considered as N/A.

    Parameters
    ----------
    arr : np.ndarray
        Input array of strings.
    out : Optional[np.ndarray]
        Optional output array of the same shape as arr

    Returns
    -------
    mask : np.ndarray
    """
    arr = np.asarray(arr)
    if out is None and arr.shape != ():
        out = np.empty_like(arr, dtype=bool)
    return __isnastr(arr, out=out)


def guess_data_type(orig_values, namask=None):
    """
    Use heuristics to guess data type.
    """
    valuemap, values = None, orig_values
    is_discrete = is_discrete_values(orig_values)
    if is_discrete:
        valuemap = sorted(is_discrete)
        coltype = DiscreteVariable
    else:
        # try to parse as float
        orig_values = np.asarray(orig_values)
        if namask is None:
            namask = isnastr(orig_values)
        values = np.empty_like(orig_values, dtype=float)
        values[namask] = np.nan
        try:
            np.copyto(values, orig_values, where=~namask, casting="unsafe")
        except ValueError:
            tvar = TimeVariable('_')
            try:
                values[~namask] = [tvar.parse(i) for i in orig_values[~namask]]
            except ValueError:
                coltype = StringVariable
                # return original_values
                values = orig_values
            else:
                coltype = TimeVariable
        else:
            coltype = ContinuousVariable
    return valuemap, values, coltype


def sanitize_variable(valuemap, values, orig_values, coltype, coltype_kwargs,
                      domain_vars=None, existing_var=None, new_var_name=None, data=None, name=None):
    assert issubclass(coltype, Variable)

    if name is None or existing_var is not None or new_var_name is not None:
        name = existing_var.strip() if existing_var else new_var_name
        raise DeprecationWarning("Arguments 'existing_var' and 'new_var_name' are "\
                                 "deprecated since 3.16; use 'name' instead")

    if domain_vars is not None:
        raise DeprecationWarning("Argument 'domain_vars' is deprecated since 3.16")

    if data is not None:
        raise DeprecationWarning("Argument 'data' is deprecated since 3.16")

    def get_number_of_decimals(values):
        len_ = len
        ndecimals = max((len_(value) - value.find(".")
                         for value in values if "." in value),
                        default=1)
        return ndecimals - 1

    if issubclass(coltype, DiscreteVariable) and valuemap is not None:
        coltype_kwargs.update(values=valuemap)

    var = coltype.make(name, **coltype_kwargs)

    if isinstance(var, DiscreteVariable):
        # Map discrete data to 'ints' (or at least what passes as int around
        # here)
        mapping = defaultdict(
            lambda: np.nan,
            {val: i for i, val in enumerate(var.values)},
        )
        mapping[""] = np.nan
        mapvalues_ = np.frompyfunc(mapping.__getitem__, 1, 1)

        def mapvalues(arr):
            arr = np.asarray(arr, dtype=object)
            return mapvalues_(arr, out=np.empty_like(arr, dtype=float))
        values = mapvalues(orig_values)

    if coltype is StringVariable:
        values = orig_values

    # ContinuousVariable.number_of_decimals is supposed to be handled by
    # ContinuousVariable.to_val. In the interest of speed, the reader bypasses
    # it, so we set the number of decimals here.
    # The number of decimals is increased if not set manually (in which case
    # var.adjust_decimals would be 0).
    if isinstance(var, ContinuousVariable) and var.adjust_decimals:
        ndecimals = get_number_of_decimals(orig_values)
        if var.adjust_decimals == 2 or ndecimals > var.number_of_decimals:
            var.number_of_decimals = ndecimals
            var.adjust_decimals = 1

    if isinstance(var, TimeVariable) or coltype is TimeVariable:
        # Re-parse the values because only now after coltype.make call
        # above, variable var is the correct one
        _var = var if isinstance(var, TimeVariable) else TimeVariable('_')
        values = [_var.parse(i) for i in orig_values]

    return values, var


class Flags:
    """Parser for column flags (i.e. third header row)"""
    DELIMITER = ' '
    _RE_SPLIT = re.compile(r'(?<!\\)' + DELIMITER).split
    _RE_ATTR_UNQUOTED_STR = re.compile(r'^[a-zA-Z_]').match
    ALL = OrderedDict((
        ('class', 'c'),
        ('ignore', 'i'),
        ('meta', 'm'),
        ('weight', 'w'),
        ('.+?=.*?', ''),  # general key=value attributes
    ))
    _RE_ALL = re.compile(r'^({})$'.format('|'.join(filter(None, flatten(ALL.items())))))

    def __init__(self, flags):
        for v in filter(None, self.ALL.values()):
            setattr(self, v, False)
        self.attributes = {}
        for flag in flags or []:
            flag = flag.strip()
            if self._RE_ALL.match(flag):
                if '=' in flag:
                    k, v = flag.split('=', 1)
                    if not Flags._RE_ATTR_UNQUOTED_STR(v):
                        try:
                            v = literal_eval(v)
                        except SyntaxError:
                            # If parsing failed, treat value as string
                            pass
                    self.attributes[k] = v
                else:
                    setattr(self, flag, True)
                    setattr(self, self.ALL.get(flag, ''), True)
            elif flag:
                warnings.warn('Invalid attribute flag \'{}\''.format(flag))

    @staticmethod
    def join(iterable, *args):
        return Flags.DELIMITER.join(i.strip().replace(Flags.DELIMITER, '\\' + Flags.DELIMITER)
                                    for i in chain(iterable, args)).lstrip()

    @staticmethod
    def split(s):
        return [i.replace('\\' + Flags.DELIMITER, Flags.DELIMITER)
                for i in Flags._RE_SPLIT(s)]


# Matches discrete specification where all the values are listed, space-separated
_RE_DISCRETE_LIST = re.compile(r'^\s*[^\s]+(\s[^\s]+)+\s*$')
_RE_TYPES = re.compile(r'^\s*({}|{}|)\s*$'.format(
    _RE_DISCRETE_LIST.pattern,
    '|'.join(flatten(getattr(vartype, 'TYPE_HEADERS') for vartype in Variable.registry.values()))
    ))
_RE_FLAGS = re.compile(r'^\s*( |{}|)*\s*$'.format(
    '|'.join(flatten(filter(None, i) for i in Flags.ALL.items()))
    ))


class FileFormatMeta(Registry):

    def __new__(cls, name, bases, attrs):
        newcls = super().__new__(cls, name, bases, attrs)

        # Optionally add compressed versions of extensions as supported
        if getattr(newcls, 'SUPPORT_COMPRESSED', False):
            new_extensions = list(getattr(newcls, 'EXTENSIONS', ()))
            for compression in Compression.all:
                for ext in newcls.EXTENSIONS:
                    new_extensions.append(ext + compression)
                if sys.platform in ('darwin', 'win32'):
                    # OSX file dialog doesn't support filtering on double
                    # extensions (e.g. .csv.gz)
                    # https://bugreports.qt.io/browse/QTBUG-38303
                    # This is just here for OWFile that gets QFileDialog
                    # filters from FileFormat.readers.keys()
                    # EDIT: Windows exhibit similar problems:
                    # while .tab.gz works, .tab.xz and .tab.bz2 do not!
                    new_extensions.append(compression)
            newcls.EXTENSIONS = tuple(new_extensions)

        return newcls

    @property
    def formats(cls):
        return cls.registry.values()

    @lru_cache(5)
    def _ext_to_attr_if_attr2(cls, attr, attr2):
        """
        Return ``{ext: `attr`, ...}`` dict if ``cls`` has `attr2`.
        If `attr` is '', return ``{ext: cls, ...}`` instead.

        If there are multiple formats for an extension, return a format
        with the lowest priority.
        """
        formats = OrderedDict()
        for format in sorted(cls.registry.values(), key=lambda x: x.PRIORITY):
            if not hasattr(format, attr2):
                continue
            for ext in getattr(format, 'EXTENSIONS', []):
                # Only adds if not yet registered
                formats.setdefault(ext, getattr(format, attr, format))
        return formats

    @property
    def names(cls):
        return cls._ext_to_attr_if_attr2('DESCRIPTION', '__class__')

    @property
    def writers(cls):
        return cls._ext_to_attr_if_attr2('', 'write_file')

    @property
    def readers(cls):
        return cls._ext_to_attr_if_attr2('', 'read')

    @property
    def img_writers(cls):
        return cls._ext_to_attr_if_attr2('', 'write_image')

    @property
    def graph_writers(cls):
        return cls._ext_to_attr_if_attr2('', 'write_graph')


class FileFormat(metaclass=FileFormatMeta):
    """
    Subclasses set the following attributes and override the following methods:

        EXTENSIONS = ('.ext1', '.ext2', ...)
        DESCRIPTION = 'human-readable file format description'
        SUPPORT_COMPRESSED = False
        SUPPORT_SPARSE_DATA = False

        def read(self):
            ...  # load headers, data, ...
            return self.data_table(data, headers)

        @classmethod
        def write_file(cls, filename, data):
            ...
            self.write_headers(writer.write, data)
            writer.writerows(data)

    Wrapper FileFormat.data_table() returns Orange.data.Table from `data`
    iterable (list (rows) of lists of values (cols)).
    """

    # Priority when multiple formats support the same extension. Also
    # the sort order in file open/save combo boxes. Lower is better.
    PRIORITY = 10000

    def __init__(self, filename):
        """
        Parameters
        ----------
        filename : str
            name of the file to open
        """
        self.filename = filename
        self.sheet = None

    @property
    def sheets(self):
        """FileFormats with a notion of sheets should override this property
        to return a list of sheet names in the file.

        Returns
        -------
        a list of sheet names
        """
        return ()

    def select_sheet(self, sheet):
        """Select sheet to be read

        Parameters
        ----------
        sheet : str
            sheet name
        """
        self.sheet = sheet

    @classmethod
    def get_reader(cls, filename):
        """Return reader instance that can be used to read the file

        Parameters
        ----------
        filename : str

        Returns
        -------
        FileFormat
        """
        for ext, reader in cls.readers.items():
            # Skip ambiguous, invalid compression-only extensions added on OSX
            if ext in Compression.all:
                continue
            if fnmatch(path.basename(filename), '*' + ext):
                return reader(filename)

        raise IOError('No readers for file "{}"'.format(filename))

    @classmethod
    def write(cls, filename, data):
        return cls.write_file(filename, data)

    @classmethod
    def write_table_metadata(cls, filename, data):

        def write_file(fn):
            if all(isinstance(key, str) and isinstance(value, str)
                   for key, value in data.attributes.items()):
                with open(fn, 'w', encoding='utf-8') as f:
                    f.write("\n".join("{}: {}".format(*kv)
                                      for kv in data.attributes.items()))
            else:
                with open(fn, 'wb') as f:
                    pickle.dump(data.attributes, f, pickle.HIGHEST_PROTOCOL)

        if isinstance(filename, str):
            metafile = filename + '.metadata'
            if getattr(data, 'attributes', None):
                write_file(metafile)
            elif path.exists(metafile):
                remove(metafile)

    @classmethod
    def set_table_metadata(cls, filename, table):
        # pylint: disable=bare-except
        if isinstance(filename, str) and path.exists(filename + '.metadata'):
            try:
                with open(filename + '.metadata', 'rb') as f:
                    table.attributes = pickle.load(f)
            # Unpickling throws different exceptions, not just UnpickleError
            except:
                with open(filename + '.metadata', encoding='utf-8') as f:
                    table.attributes = OrderedDict(
                        (k.strip(), v.strip())
                        for k, v in (line.split(":", 1)
                                     for line in f.readlines()))

    @classmethod
    def locate(cls, filename, search_dirs=('.',)):
        """Locate a file with given filename that can be opened by one
        of the available readers.

        Parameters
        ----------
        filename : str
        search_dirs : Iterable[str]

        Returns
        -------
        str
            Absolute path to the file
        """
        if path.exists(filename):
            return filename

        for directory in search_dirs:
            absolute_filename = path.join(directory, filename)
            if path.exists(absolute_filename):
                break
            for ext in cls.readers:
                if fnmatch(path.basename(filename), '*' + ext):
                    break
                # glob uses fnmatch internally
                matching_files = glob(absolute_filename + ext)
                if matching_files:
                    absolute_filename = matching_files[0]
                    break
            if path.exists(absolute_filename):
                break
        else:
            absolute_filename = ""

        if not path.exists(absolute_filename):
            raise IOError('File "{}" was not found.'.format(filename))

        return absolute_filename

    @staticmethod
    def open(filename, *args, **kwargs):
        """
        Format handlers can use this method instead of the builtin ``open()``
        to transparently (de)compress files if requested (according to
        `filename` extension). Set ``SUPPORT_COMPRESSED=True`` if you use this.
        """
        return open_compressed(filename, *args, **kwargs)

    @staticmethod
    def parse_headers(data):
        """Return (header rows, rest of data) as discerned from `data`"""

        def is_number(item):
            try:
                float(item)
            except ValueError:
                return False
            return True
        # Second row items are type identifiers
        def header_test2(items):
            return all(map(_RE_TYPES.match, items))
        # Third row items are flags and column attributes (attr=value)
        def header_test3(items):
            return all(map(_RE_FLAGS.match, items))

        data = iter(data)
        header_rows = []

        # Try to parse a three-line header
        lines = []
        try:
            lines.append(list(next(data)))
            lines.append(list(next(data)))
            lines.append(list(next(data)))
        except StopIteration:
            lines, data = [], chain(lines, data)
        if lines:
            l1, l2, l3 = lines
            # Three-line header if line 2 & 3 match (1st line can be anything)
            if header_test2(l2) and header_test3(l3):
                header_rows = [l1, l2, l3]
            else:
                lines, data = [], chain((l1, l2, l3), data)

        # Try to parse a single-line header
        if not header_rows:
            try:
                lines.append(list(next(data)))
            except StopIteration:
                pass
            if lines:
                # Header if none of the values in line 1 parses as a number
                if not all(is_number(i) for i in lines[0]):
                    header_rows = [lines[0]]
                else:
                    data = chain(lines, data)

        return header_rows, data

    @classmethod
    def data_table(cls, data, headers=None):
        """
        Return Orange.data.Table given rows of `headers` (iterable of iterable)
        and rows of `data` (iterable of iterable).

        Basically, the idea of subclasses is to produce those two iterables,
        however they might.

        If `headers` is not provided, the header rows are extracted from `data`,
        assuming they precede it.
        """
        if not headers:
            headers, data = cls.parse_headers(data)

        # Consider various header types (single-row, two-row, three-row, none)
        if len(headers) == 3:
            names, types, flags = map(list, headers)
        else:
            if len(headers) == 1:
                HEADER1_FLAG_SEP = '#'
                # First row format either:
                #   1) delimited column names
                #   2) -||- with type and flags prepended, separated by #,
                #      e.g. d#sex,c#age,cC#IQ
                _flags, names = zip(*[i.split(HEADER1_FLAG_SEP, 1)
                                      if HEADER1_FLAG_SEP in i else ('', i)
                                      for i in headers[0]]
                                   )
                names = list(names)
            elif len(headers) == 2:
                names, _flags = map(list, headers)
            else:
                # Use heuristics for everything
                names, _flags = [], []
            types = [''.join(filter(str.isupper, flag)).lower() for flag in _flags]
            flags = [Flags.join(filter(str.islower, flag)) for flag in _flags]

        # Determine maximum row length
        rowlen = max(map(len, (names, types, flags)))

        def _equal_length(lst):
            lst.extend(['']*(rowlen - len(lst)))
            return lst

        # Ensure all data is of equal width in a column-contiguous array
        data = [_equal_length([s.strip() for s in row])
                for row in data if any(row)]
        data = np.array(data, dtype=object, order='F')

        # Data may actually be longer than headers were
        try:
            rowlen = data.shape[1]
        except IndexError:
            pass
        else:
            for lst in (names, types, flags):
                _equal_length(lst)

        NAMEGEN = namegen('Feature ', 1)
        Xcols, attrs = [], []
        Mcols, metas = [], []
        Ycols, clses = [], []
        Wcols = []

        # Rename variables if necessary
        # Reusing across files still works if both files have same duplicates
        name_counts = Counter(names)
        del name_counts[""]
        if len(name_counts) != len(names) and name_counts:
            uses = {name: 0 for name, count in name_counts.items() if count > 1}
            for i, name in enumerate(names):
                if name in uses:
                    uses[name] += 1
                    names[i] = "{}_{}".format(name, uses[name])

        namask = np.empty(data.shape[0], dtype=bool)
        # Iterate through the columns
        for col in range(rowlen):
            flag = Flags(Flags.split(flags[col]))
            if flag.i:
                continue

            type_flag = types and types[col].strip()
            try:
                orig_values = data[:, col]
            except IndexError:
                orig_values = np.array([], dtype=object)

            namask = isnastr(orig_values, out=namask)

            coltype_kwargs = {}
            valuemap = None
            values = orig_values

            if type_flag in StringVariable.TYPE_HEADERS:
                coltype = StringVariable
                values = orig_values
            elif type_flag in ContinuousVariable.TYPE_HEADERS:
                coltype = ContinuousVariable
                values = np.empty(data.shape[0], dtype=float)
                try:
                    np.copyto(values, orig_values, casting="unsafe",
                              where=~namask)
                    values[namask] = np.nan
                except ValueError:
                    for row, num in enumerate(orig_values):
                        if not isnastr(num):
                            try:
                                float(num)
                            except ValueError:
                                break
                    raise ValueError('Non-continuous value in (1-based) '
                                     'line {}, column {}'.format(row + len(headers) + 1,
                                                                 col + 1))

            elif type_flag in TimeVariable.TYPE_HEADERS:
                coltype = TimeVariable
                values = np.where(namask, "", orig_values)
            elif (type_flag in DiscreteVariable.TYPE_HEADERS or
                  _RE_DISCRETE_LIST.match(type_flag)):
                coltype = DiscreteVariable
                orig_values = values = np.where(namask, "", orig_values)
                if _RE_DISCRETE_LIST.match(type_flag):
                    valuemap = Flags.split(type_flag)
                    coltype_kwargs.update(ordered=True)
                else:
                    valuemap = sorted(set(orig_values) - {""})
            else:
                # No known type specified, use heuristics
                valuemap, values, coltype = guess_data_type(orig_values, namask)

            if flag.m or coltype is StringVariable:
                append_to = (Mcols, metas)
            elif flag.w:
                append_to = (Wcols, None)
            elif flag.c:
                append_to = (Ycols, clses)
            else:
                append_to = (Xcols, attrs)

            cols, domain_vars = append_to

            if domain_vars is not None:
                var_name = names and names[col]
                if not var_name:
                    var_name = next(NAMEGEN)

                values, var = sanitize_variable(
                    valuemap, values, orig_values, coltype, coltype_kwargs,
                    name=var_name)
            else:
                var = None
            if domain_vars is not None:
                var.attributes.update(flag.attributes)
                domain_vars.append(var)

            if isinstance(values, np.ndarray) and not values.flags.owndata:
                values = values.copy()  # might view `data` (string columns)
            cols.append(values)

            try:
                # allow gc to reclaim memory used by string values
                data[:, col] = None
            except IndexError:
                pass

        domain = Domain(attrs, clses, metas)

        if not data.size:
            return Table.from_domain(domain, 0)

        X = Y = M = W = None
        if Xcols:
            X = np.c_[tuple(Xcols)]
            assert X.dtype == np.float_
        else:
            X = np.empty((data.shape[0], 0), dtype=np.float_)
        if Ycols:
            Y = np.c_[tuple(Ycols)]
            assert Y.dtype == np.float_
        if Mcols:
            M = np.c_[tuple(Mcols)].astype(object)
        if Wcols:
            W = np.c_[tuple(Wcols)].astype(float)

        table = Table.from_numpy(domain, X, Y, M, W)
        return table

    @staticmethod
    def header_names(data):
        return ['weights'] * data.has_weights() + \
               [v.name for v in chain(data.domain.attributes,
                                      data.domain.class_vars,
                                      data.domain.metas)]

    @staticmethod
    def header_types(data):
        def _vartype(var):
            if var.is_continuous or var.is_string:
                return var.TYPE_HEADERS[0]
            elif var.is_discrete:
                return Flags.join(var.values) if var.ordered else var.TYPE_HEADERS[0]
            raise NotImplementedError
        return ['continuous'] * data.has_weights() + \
               [_vartype(v) for v in chain(data.domain.attributes,
                                           data.domain.class_vars,
                                           data.domain.metas)]

    @staticmethod
    def header_flags(data):
        return list(chain(['weight'] * data.has_weights(),
                          (Flags.join([flag], *('{}={}'.format(*a)
                                                for a in sorted(var.attributes.items())))
                           for flag, var in chain(zip(repeat(''), data.domain.attributes),
                                                  zip(repeat('class'), data.domain.class_vars),
                                                  zip(repeat('meta'), data.domain.metas)))))

    @classmethod
    def write_headers(cls, write, data):
        """`write` is a callback that accepts an iterable"""
        write(cls.header_names(data))
        write(cls.header_types(data))
        write(cls.header_flags(data))

    @classmethod
    def write_data(cls, write, data):
        """`write` is a callback that accepts an iterable"""
        vars = list(chain((ContinuousVariable('_w'),) if data.has_weights() else (),
                          data.domain.attributes,
                          data.domain.class_vars,
                          data.domain.metas))

        def formatter(var):
            # type: (Variable) -> Callable[[Variable], Any]
            # Return a column 'formatter' function. The function must return
            # something that `write` knows how to write
            if var.is_time:
                return var.repr_val
            elif var.is_continuous:
                return lambda value: "" if isnan(value) else value
            elif var.is_discrete:
                return lambda value: "" if isnan(value) else var.values[int(value)]
            elif var.is_string:
                return lambda value: value
            else:
                return var.repr_val

        formatters = [formatter(v) for v in vars]
        for row in zip(data.W if data.W.ndim > 1 else data.W[:, np.newaxis],
                       data.X,
                       data.Y if data.Y.ndim > 1 else data.Y[:, np.newaxis],
                       data.metas):
            write([fmt(v) for fmt, v in zip(formatters, flatten(row))])

    @classmethod
    def qualified_name(cls):
        return cls.__module__ + '.' + cls.__name__


def class_from_qualified_name(format_name):
    """ File format class from qualified name. """
    elements = format_name.split(".")
    m = import_module(".".join(elements[:-1]))
    return getattr(m, elements[-1])


class CSVReader(FileFormat):
    """Reader for comma separated files"""

    EXTENSIONS = ('.csv',)
    DESCRIPTION = 'Comma-separated values'
    DELIMITERS = ',;:\t$ '
    SUPPORT_COMPRESSED = True
    SUPPORT_SPARSE_DATA = False
    PRIORITY = 20

    def read(self):
        for encoding in (lambda: ('us-ascii', None),                 # fast
                         lambda: (detect_encoding(self.filename), None),  # precise
                         lambda: (locale.getpreferredencoding(False), None),
                         lambda: (sys.getdefaultencoding(), None),   # desperate
                         lambda: ('utf-8', None),                    # ...
                         lambda: ('utf-8', 'ignore')):               # fallback
            encoding, errors = encoding()
            # Clear the error flag for all except the last check, because
            # the error of second-to-last check is stored and shown as warning in owfile
            if errors != 'ignore':
                error = ''
            with self.open(self.filename, mode='rt', newline='',
                           encoding=encoding, errors=errors) as file:
                # Sniff the CSV dialect (delimiter, quotes, ...)
                try:
                    dialect = csv.Sniffer().sniff(
                        # Take first couple of *complete* lines as sample
                        ''.join(file.readline() for _ in range(10)),
                        self.DELIMITERS)
                    delimiter = dialect.delimiter
                    quotechar = dialect.quotechar
                except UnicodeDecodeError as e:
                    error = e
                    continue
                except csv.Error:
                    delimiter = self.DELIMITERS[0]
                    quotechar = csv.excel.quotechar

                file.seek(0)
                try:
                    reader = csv.reader(
                        file, delimiter=delimiter, quotechar=quotechar,
                        skipinitialspace=True,
                    )
                    data = self.data_table(reader)

                    # TODO: Name can be set unconditionally when/if
                    # self.filename will always be a string with the file name.
                    # Currently, some tests pass StringIO instead of
                    # the file name to a reader.
                    if isinstance(self.filename, str):
                        data.name = path.splitext(
                            path.split(self.filename)[-1])[0]
                    if error and isinstance(error, UnicodeDecodeError):
                        pos, endpos = error.args[2], error.args[3]
                        warning = ('Skipped invalid byte(s) in position '
                                   '{}{}').format(pos,
                                                  ('-' + str(endpos)) if (endpos - pos) > 1 else '')
                        warnings.warn(warning)
                    self.set_table_metadata(self.filename, data)
                    return data
                except Exception as e:
                    error = e
                    continue
        raise ValueError('Cannot parse dataset {}: {}'.format(self.filename, error)) from error

    @classmethod
    def write_file(cls, filename, data):
        with cls.open(filename, mode='wt', newline='', encoding='utf-8') as file:
            writer = csv.writer(file, delimiter=cls.DELIMITERS[0])
            cls.write_headers(writer.writerow, data)
            cls.write_data(writer.writerow, data)
        cls.write_table_metadata(filename, data)


class TabReader(CSVReader):
    """Reader for tab separated files"""
    EXTENSIONS = ('.tab', '.tsv')
    DESCRIPTION = 'Tab-separated values'
    DELIMITERS = '\t'
    PRIORITY = 10


class PickleReader(FileFormat):
    """Reader for pickled Table objects"""
    EXTENSIONS = ('.pkl', '.pickle')
    DESCRIPTION = 'Pickled Orange data'
    SUPPORT_COMPRESSED = True
    SUPPORT_SPARSE_DATA = True

    def read(self):
        with self.open(self.filename, 'rb') as f:
            table = pickle.load(f)
            if not isinstance(table, Table):
                raise TypeError("file does not contain a data table")
            else:
                return table

    @classmethod
    def write_file(cls, filename, data):
        with cls.open(filename, 'wb') as f:
            pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)


class BasketReader(FileFormat):
    """Reader for basket (sparse) files"""
    EXTENSIONS = ('.basket', '.bsk')
    DESCRIPTION = 'Basket file'
    SUPPORT_SPARSE_DATA = True

    def read(self):
        def constr_vars(inds):
            if inds:
                return [ContinuousVariable(x.decode("utf-8")) for _, x in
                        sorted((ind, name) for name, ind in inds.items())]

        X, Y, metas, attr_indices, class_indices, meta_indices = \
            _io.sparse_read_float(self.filename.encode(sys.getdefaultencoding()))

        attrs = constr_vars(attr_indices)
        classes = constr_vars(class_indices)
        meta_attrs = constr_vars(meta_indices)
        domain = Domain(attrs, classes, meta_attrs)
        table = Table.from_numpy(
            domain, attrs and X, classes and Y, metas and meta_attrs)
        table.name = path.splitext(path.split(self.filename)[-1])[0]
        return table


class ExcelReader(FileFormat):
    """Reader for excel files"""
    EXTENSIONS = ('.xls', '.xlsx')
    DESCRIPTION = 'Mircosoft Excel spreadsheet'
    SUPPORT_SPARSE_DATA = False

    def __init__(self, filename):
        super().__init__(filename)

        from xlrd import open_workbook
        self.workbook = open_workbook(self.filename)

    @property
    @lru_cache(1)
    def sheets(self):
        return self.workbook.sheet_names()

    def read(self):
        import xlrd
        wb = xlrd.open_workbook(self.filename, on_demand=True)
        if self.sheet:
            ss = wb.sheet_by_name(self.sheet)
        else:
            ss = wb.sheet_by_index(0)
        try:
            first_row = next(i for i in range(ss.nrows) if any(ss.row_values(i)))
            first_col = next(i for i in range(ss.ncols) if ss.cell_value(first_row, i))
            row_len = ss.row_len(first_row)
            cells = filter(any,
                           [[str(ss.cell_value(row, col)) if col < ss.row_len(row) else ''
                             for col in range(first_col, row_len)]
                            for row in range(first_row, ss.nrows)])
            table = self.data_table(cells)
            table.name = path.splitext(path.split(self.filename)[-1])[0]
            if self.sheet:
                table.name = '-'.join((table.name, self.sheet))
        except Exception:
            raise IOError("Couldn't load spreadsheet from " + self.filename)
        return table


class DotReader(FileFormat):
    """Writer for dot (graph) files"""
    EXTENSIONS = ('.dot', '.gv')
    DESCRIPTION = 'Dot graph description'
    SUPPORT_COMPRESSED = True
    SUPPORT_SPARSE_DATA = False

    @classmethod
    def write_graph(cls, filename, graph):
        from sklearn import tree
        tree.export_graphviz(graph, out_file=cls.open(filename, 'wt'))

    @classmethod
    def write(cls, filename, tree):
        if type(tree) == dict:
            tree = tree['tree']
        cls.write_graph(filename, tree)


class UrlReader(FileFormat):
    def __init__(self, filename):
        filename = filename.strip()
        if not urlparse(filename).scheme:
            filename = 'http://' + filename
        super().__init__(filename)

    @staticmethod
    def urlopen(url):
        req = Request(
            url,
            # Avoid 403 error with servers that dislike scrapers
            headers={'User-Agent': 'Mozilla/5.0 (X11; Linux) Gecko/20100101 Firefox/'})
        return urlopen(req, timeout=10)

    def read(self):
        self.filename = self._trim(self._resolve_redirects(self.filename))
        with contextlib.closing(self.urlopen(self.filename)) as response:
            name = self._suggest_filename(response.headers['content-disposition'])
            with NamedTemporaryFile(suffix=name, delete=False) as f:
                f.write(response.read())
                # delete=False is a workaround for https://bugs.python.org/issue14243

            reader = self.get_reader(f.name)
            data = reader.read()
            remove(f.name)
        # Override name set in from_file() to avoid holding the temp prefix
        data.name = path.splitext(name)[0]
        data.origin = self.filename
        return data

    def _resolve_redirects(self, url):
        # Resolve (potential) redirects to a final URL
        with contextlib.closing(self.urlopen(url)) as response:
            return response.url

    @classmethod
    def _trim(cls, url):
        URL_TRIMMERS = (
            cls._trim_googlesheet,
            cls._trim_dropbox,
        )
        for trim in URL_TRIMMERS:
            try:
                url = trim(url)
            except ValueError:
                continue
            else:
                break
        return url

    @staticmethod
    def _trim_googlesheet(url):
        match = re.match(r'(?:https?://)?(?:www\.)?'
                         r'docs\.google\.com/spreadsheets/d/'
                         r'(?P<workbook_id>[-\w_]+)'
                         r'(?:/.*?gid=(?P<sheet_id>\d+).*|.*)?',
                         url, re.IGNORECASE)
        try:
            workbook, sheet = match.group('workbook_id'), match.group('sheet_id')
            if not workbook:
                raise ValueError
        except (AttributeError, ValueError):
            raise ValueError
        url = 'https://docs.google.com/spreadsheets/d/{}/export?format=tsv'.format(workbook)
        if sheet:
            url += '&gid=' + sheet
        return url

    @staticmethod
    def _trim_dropbox(url):
        parts = urlsplit(url)
        if not parts.netloc.endswith('dropbox.com'):
            raise ValueError
        return urlunsplit(parts._replace(query='dl=1'))

    def _suggest_filename(self, content_disposition):
        default_name = re.sub(r'[\\:/]', '_', urlparse(self.filename).path)

        # See https://tools.ietf.org/html/rfc6266#section-4.1
        matches = re.findall(r"filename\*?=(?:\"|.{0,10}?'[^']*')([^\"]+)",
                             content_disposition or '')
        return urlunquote(matches[-1]) if matches else default_name

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]

**Title:** CSV Reader Fails to Handle Files with More Data Columns Than Headers

**Description:**
When reading a CSV file where data rows contain more columns than the header row, the CSV Reader does not issue a warning and fails to remove the extra columns.

**Example Buggy Code:**
```python
import io
from Orange.data.io import CSVReader

samplefile = """\
a, b
1, 0,
1, 2,
"""
c = io.StringIO(samplefile)
table = CSVReader(c).read()
print(len(table.domain.attributes))  # Expected: 2 attributes
```

**Expected Behavior:**
A warning stating "Columns with no headers were removed." should be issued, and the extra data columns should be stripped, resulting in only the defined header columns being present.

**Actual Behavior:**
No warning is triggered, and the extra columns remain, leading to inconsistencies between the header and data rows.

[/ISSUE]
{
    "TestChooseReader.test_usual_extensions": "PASSED",
    "TestChooseReader.test_wildcard_extension": "PASSED",
    "TestMultipleSameExtension.test_find_reader": "PASSED",
    "TestLocate.test_locate_sample_datasets": "PASSED",
    "TestLocate.test_locate_wildcard_extension": "PASSED",
    "TestReader.test_empty_columns": "PASSED",
    "TestReader.test_open_bad_pickle": "PASSED",
    "TestIo.test_sanitize_variable_deprecated_params": "PASSED"
}
diff --git a/Orange/data/io.py b/Orange/data/io.py
index 77def811e..00ab17f4a 100644
--- a/Orange/data/io.py
+++ b/Orange/data/io.py
@@ -618,8 +618,15 @@ class FileFormat(metaclass=FileFormatMeta):
         # Determine maximum row length
         rowlen = max(map(len, (names, types, flags)))
 
+        strip = False
+
         def _equal_length(lst):
-            lst.extend(['']*(rowlen - len(lst)))
+            nonlocal strip
+            if len(lst) > rowlen > 0:
+                lst = lst[:rowlen]
+                strip = True
+            elif len(lst) < rowlen:
+                lst.extend(['']*(rowlen - len(lst)))
             return lst
 
         # Ensure all data is of equal width in a column-contiguous array
@@ -627,6 +634,9 @@ class FileFormat(metaclass=FileFormatMeta):
                 for row in data if any(row)]
         data = np.array(data, dtype=object, order='F')
 
+        if strip:
+            warnings.warn("Columns with no headers were removed.")
+
         # Data may actually be longer than headers were
         try:
             rowlen = data.shape[1]

{"Orange/data/io.py": "import contextlib\nimport csv\nimport locale\nimport pickle\nimport re\nimport subprocess\nimport sys\nimport warnings\n\nfrom ast import literal_eval\nfrom collections import OrderedDict, Counter, defaultdict\nfrom functools import lru_cache\nfrom importlib import import_module\nfrom itertools import chain, repeat\nfrom math import isnan\n\nfrom os import path, remove\nfrom tempfile import NamedTemporaryFile\nfrom urllib.parse import urlparse, urlsplit, urlunsplit, unquote as urlunquote\nfrom urllib.request import urlopen, Request\nfrom fnmatch import fnmatch\nfrom glob import glob\n\nimport numpy as np\nfrom chardet.universaldetector import UniversalDetector\n\nfrom Orange.data import (\n    _io, is_discrete_values, MISSING_VALUES, Table, Domain, Variable,\n    DiscreteVariable, StringVariable, ContinuousVariable, TimeVariable,\n)\nfrom Orange.util import Registry, flatten, namegen\n\n\n# Support values longer than 128K (i.e. text contents features)\ncsv.field_size_limit(100*1024*1024)\n\n__all__ = [\"Flags\", \"FileFormat\"]\n\n\n_IDENTITY = lambda i: i\n\n\nclass Compression:\n    \"\"\"Supported compression extensions\"\"\"\n    GZIP = '.gz'\n    BZIP2 = '.bz2'\n    XZ = '.xz'\n    all = (GZIP, BZIP2, XZ)\n\n\ndef open_compressed(filename, *args, _open=open, **kwargs):\n    \"\"\"Return seamlessly decompressed open file handle for `filename`\"\"\"\n    if isinstance(filename, str):\n        if filename.endswith(Compression.GZIP):\n            from gzip import open as _open\n        elif filename.endswith(Compression.BZIP2):\n            from bz2 import open as _open\n        elif filename.endswith(Compression.XZ):\n            from lzma import open as _open\n        return _open(filename, *args, **kwargs)\n    # Else already a file, just pass it through\n    return filename\n\n\ndef detect_encoding(filename):\n    \"\"\"\n    Detect encoding of `filename`, which can be a ``str`` filename, a\n    ``file``-like object, or ``bytes``.\n    \"\"\"\n    # Try with Unix file utility first because it's faster (~10ms vs 100ms)\n    if isinstance(filename, str) and not filename.endswith(Compression.all):\n        try:\n            with subprocess.Popen(('file', '--brief', '--mime-encoding', filename),\n                                  stdout=subprocess.PIPE) as process:\n                process.wait()\n                if process.returncode == 0:\n                    encoding = process.stdout.read().strip()\n                    # file only supports these encodings; for others it says\n                    # unknown-8bit or binary. So we give chardet a chance to do\n                    # better\n                    if encoding in (b'utf-8', b'us-ascii', b'iso-8859-1',\n                                    b'utf-7', b'utf-16le', b'utf-16be', b'ebcdic'):\n                        return encoding.decode('us-ascii')\n        except OSError:\n            pass  # windoze\n\n    # file not available or unable to guess the encoding, have chardet do it\n    detector = UniversalDetector()\n    # We examine only first N 4kB blocks of file because chardet is really slow\n    MAX_BYTES = 4*1024*12\n\n    def _from_file(f):\n        detector.feed(f.read(MAX_BYTES))\n        detector.close()\n        return (detector.result.get('encoding')\n                if detector.result.get('confidence', 0) >= .85 else\n                'utf-8')\n\n    if isinstance(filename, str):\n        with open_compressed(filename, 'rb') as f:\n            return _from_file(f)\n    elif isinstance(filename, bytes):\n        detector.feed(filename[:MAX_BYTES])\n        detector.close()\n        return detector.result.get('encoding')\n    elif hasattr(filename, 'encoding'):\n        return filename.encoding\n    else:  # assume file-like object that you can iter through\n        return _from_file(filename)\n\n\n__isnastr = np.frompyfunc(\n    {v for v in MISSING_VALUES if isinstance(v, str)}.__contains__, 1, 1)\n\n# wrapper for __isnastr with proper default out dtype\ndef isnastr(arr, out=None):\n    \"\"\"\n    Given an (object) array of string values, return a boolean mask array\n    that is True where the `arr` contains one of the string constants\n    considered as N/A.\n\n    Parameters\n    ----------\n    arr : np.ndarray\n        Input array of strings.\n    out : Optional[np.ndarray]\n        Optional output array of the same shape as arr\n\n    Returns\n    -------\n    mask : np.ndarray\n    \"\"\"\n    arr = np.asarray(arr)\n    if out is None and arr.shape != ():\n        out = np.empty_like(arr, dtype=bool)\n    return __isnastr(arr, out=out)\n\n\ndef guess_data_type(orig_values, namask=None):\n    \"\"\"\n    Use heuristics to guess data type.\n    \"\"\"\n    valuemap, values = None, orig_values\n    is_discrete = is_discrete_values(orig_values)\n    if is_discrete:\n        valuemap = sorted(is_discrete)\n        coltype = DiscreteVariable\n    else:\n        # try to parse as float\n        orig_values = np.asarray(orig_values)\n        if namask is None:\n            namask = isnastr(orig_values)\n        values = np.empty_like(orig_values, dtype=float)\n        values[namask] = np.nan\n        try:\n            np.copyto(values, orig_values, where=~namask, casting=\"unsafe\")\n        except ValueError:\n            tvar = TimeVariable('_')\n            try:\n                values[~namask] = [tvar.parse(i) for i in orig_values[~namask]]\n            except ValueError:\n                coltype = StringVariable\n                # return original_values\n                values = orig_values\n            else:\n                coltype = TimeVariable\n        else:\n            coltype = ContinuousVariable\n    return valuemap, values, coltype\n\n\ndef sanitize_variable(valuemap, values, orig_values, coltype, coltype_kwargs,\n                      domain_vars=None, existing_var=None, new_var_name=None, data=None, name=None):\n    assert issubclass(coltype, Variable)\n\n    if name is None or existing_var is not None or new_var_name is not None:\n        name = existing_var.strip() if existing_var else new_var_name\n        raise DeprecationWarning(\"Arguments 'existing_var' and 'new_var_name' are \"\\\n                                 \"deprecated since 3.16; use 'name' instead\")\n\n    if domain_vars is not None:\n        raise DeprecationWarning(\"Argument 'domain_vars' is deprecated since 3.16\")\n\n    if data is not None:\n        raise DeprecationWarning(\"Argument 'data' is deprecated since 3.16\")\n\n    def get_number_of_decimals(values):\n        len_ = len\n        ndecimals = max((len_(value) - value.find(\".\")\n                         for value in values if \".\" in value),\n                        default=1)\n        return ndecimals - 1\n\n    if issubclass(coltype, DiscreteVariable) and valuemap is not None:\n        coltype_kwargs.update(values=valuemap)\n\n    var = coltype.make(name, **coltype_kwargs)\n\n    if isinstance(var, DiscreteVariable):\n        # Map discrete data to 'ints' (or at least what passes as int around\n        # here)\n        mapping = defaultdict(\n            lambda: np.nan,\n            {val: i for i, val in enumerate(var.values)},\n        )\n        mapping[\"\"] = np.nan\n        mapvalues_ = np.frompyfunc(mapping.__getitem__, 1, 1)\n\n        def mapvalues(arr):\n            arr = np.asarray(arr, dtype=object)\n            return mapvalues_(arr, out=np.empty_like(arr, dtype=float))\n        values = mapvalues(orig_values)\n\n    if coltype is StringVariable:\n        values = orig_values\n\n    # ContinuousVariable.number_of_decimals is supposed to be handled by\n    # ContinuousVariable.to_val. In the interest of speed, the reader bypasses\n    # it, so we set the number of decimals here.\n    # The number of decimals is increased if not set manually (in which case\n    # var.adjust_decimals would be 0).\n    if isinstance(var, ContinuousVariable) and var.adjust_decimals:\n        ndecimals = get_number_of_decimals(orig_values)\n        if var.adjust_decimals == 2 or ndecimals > var.number_of_decimals:\n            var.number_of_decimals = ndecimals\n            var.adjust_decimals = 1\n\n    if isinstance(var, TimeVariable) or coltype is TimeVariable:\n        # Re-parse the values because only now after coltype.make call\n        # above, variable var is the correct one\n        _var = var if isinstance(var, TimeVariable) else TimeVariable('_')\n        values = [_var.parse(i) for i in orig_values]\n\n    return values, var\n\n\nclass Flags:\n    \"\"\"Parser for column flags (i.e. third header row)\"\"\"\n    DELIMITER = ' '\n    _RE_SPLIT = re.compile(r'(?<!\\\\)' + DELIMITER).split\n    _RE_ATTR_UNQUOTED_STR = re.compile(r'^[a-zA-Z_]').match\n    ALL = OrderedDict((\n        ('class', 'c'),\n        ('ignore', 'i'),\n        ('meta', 'm'),\n        ('weight', 'w'),\n        ('.+?=.*?', ''),  # general key=value attributes\n    ))\n    _RE_ALL = re.compile(r'^({})$'.format('|'.join(filter(None, flatten(ALL.items())))))\n\n    def __init__(self, flags):\n        for v in filter(None, self.ALL.values()):\n            setattr(self, v, False)\n        self.attributes = {}\n        for flag in flags or []:\n            flag = flag.strip()\n            if self._RE_ALL.match(flag):\n                if '=' in flag:\n                    k, v = flag.split('=', 1)\n                    if not Flags._RE_ATTR_UNQUOTED_STR(v):\n                        try:\n                            v = literal_eval(v)\n                        except SyntaxError:\n                            # If parsing failed, treat value as string\n                            pass\n                    self.attributes[k] = v\n                else:\n                    setattr(self, flag, True)\n                    setattr(self, self.ALL.get(flag, ''), True)\n            elif flag:\n                warnings.warn('Invalid attribute flag \\'{}\\''.format(flag))\n\n    @staticmethod\n    def join(iterable, *args):\n        return Flags.DELIMITER.join(i.strip().replace(Flags.DELIMITER, '\\\\' + Flags.DELIMITER)\n                                    for i in chain(iterable, args)).lstrip()\n\n    @staticmethod\n    def split(s):\n        return [i.replace('\\\\' + Flags.DELIMITER, Flags.DELIMITER)\n                for i in Flags._RE_SPLIT(s)]\n\n\n# Matches discrete specification where all the values are listed, space-separated\n_RE_DISCRETE_LIST = re.compile(r'^\\s*[^\\s]+(\\s[^\\s]+)+\\s*$')\n_RE_TYPES = re.compile(r'^\\s*({}|{}|)\\s*$'.format(\n    _RE_DISCRETE_LIST.pattern,\n    '|'.join(flatten(getattr(vartype, 'TYPE_HEADERS') for vartype in Variable.registry.values()))\n    ))\n_RE_FLAGS = re.compile(r'^\\s*( |{}|)*\\s*$'.format(\n    '|'.join(flatten(filter(None, i) for i in Flags.ALL.items()))\n    ))\n\n\nclass FileFormatMeta(Registry):\n\n    def __new__(cls, name, bases, attrs):\n        newcls = super().__new__(cls, name, bases, attrs)\n\n        # Optionally add compressed versions of extensions as supported\n        if getattr(newcls, 'SUPPORT_COMPRESSED', False):\n            new_extensions = list(getattr(newcls, 'EXTENSIONS', ()))\n            for compression in Compression.all:\n                for ext in newcls.EXTENSIONS:\n                    new_extensions.append(ext + compression)\n                if sys.platform in ('darwin', 'win32'):\n                    # OSX file dialog doesn't support filtering on double\n                    # extensions (e.g. .csv.gz)\n                    # https://bugreports.qt.io/browse/QTBUG-38303\n                    # This is just here for OWFile that gets QFileDialog\n                    # filters from FileFormat.readers.keys()\n                    # EDIT: Windows exhibit similar problems:\n                    # while .tab.gz works, .tab.xz and .tab.bz2 do not!\n                    new_extensions.append(compression)\n            newcls.EXTENSIONS = tuple(new_extensions)\n\n        return newcls\n\n    @property\n    def formats(cls):\n        return cls.registry.values()\n\n    @lru_cache(5)\n    def _ext_to_attr_if_attr2(cls, attr, attr2):\n        \"\"\"\n        Return ``{ext: `attr`, ...}`` dict if ``cls`` has `attr2`.\n        If `attr` is '', return ``{ext: cls, ...}`` instead.\n\n        If there are multiple formats for an extension, return a format\n        with the lowest priority.\n        \"\"\"\n        formats = OrderedDict()\n        for format in sorted(cls.registry.values(), key=lambda x: x.PRIORITY):\n            if not hasattr(format, attr2):\n                continue\n            for ext in getattr(format, 'EXTENSIONS', []):\n                # Only adds if not yet registered\n                formats.setdefault(ext, getattr(format, attr, format))\n        return formats\n\n    @property\n    def names(cls):\n        return cls._ext_to_attr_if_attr2('DESCRIPTION', '__class__')\n\n    @property\n    def writers(cls):\n        return cls._ext_to_attr_if_attr2('', 'write_file')\n\n    @property\n    def readers(cls):\n        return cls._ext_to_attr_if_attr2('', 'read')\n\n    @property\n    def img_writers(cls):\n        return cls._ext_to_attr_if_attr2('', 'write_image')\n\n    @property\n    def graph_writers(cls):\n        return cls._ext_to_attr_if_attr2('', 'write_graph')\n\n\nclass FileFormat(metaclass=FileFormatMeta):\n    \"\"\"\n    Subclasses set the following attributes and override the following methods:\n\n        EXTENSIONS = ('.ext1', '.ext2', ...)\n        DESCRIPTION = 'human-readable file format description'\n        SUPPORT_COMPRESSED = False\n        SUPPORT_SPARSE_DATA = False\n\n        def read(self):\n            ...  # load headers, data, ...\n            return self.data_table(data, headers)\n\n        @classmethod\n        def write_file(cls, filename, data):\n            ...\n            self.write_headers(writer.write, data)\n            writer.writerows(data)\n\n    Wrapper FileFormat.data_table() returns Orange.data.Table from `data`\n    iterable (list (rows) of lists of values (cols)).\n    \"\"\"\n\n    # Priority when multiple formats support the same extension. Also\n    # the sort order in file open/save combo boxes. Lower is better.\n    PRIORITY = 10000\n\n    def __init__(self, filename):\n        \"\"\"\n        Parameters\n        ----------\n        filename : str\n            name of the file to open\n        \"\"\"\n        self.filename = filename\n        self.sheet = None\n\n    @property\n    def sheets(self):\n        \"\"\"FileFormats with a notion of sheets should override this property\n        to return a list of sheet names in the file.\n\n        Returns\n        -------\n        a list of sheet names\n        \"\"\"\n        return ()\n\n    def select_sheet(self, sheet):\n        \"\"\"Select sheet to be read\n\n        Parameters\n        ----------\n        sheet : str\n            sheet name\n        \"\"\"\n        self.sheet = sheet\n\n    @classmethod\n    def get_reader(cls, filename):\n        \"\"\"Return reader instance that can be used to read the file\n\n        Parameters\n        ----------\n        filename : str\n\n        Returns\n        -------\n        FileFormat\n        \"\"\"\n        for ext, reader in cls.readers.items():\n            # Skip ambiguous, invalid compression-only extensions added on OSX\n            if ext in Compression.all:\n                continue\n            if fnmatch(path.basename(filename), '*' + ext):\n                return reader(filename)\n\n        raise IOError('No readers for file \"{}\"'.format(filename))\n\n    @classmethod\n    def write(cls, filename, data):\n        return cls.write_file(filename, data)\n\n    @classmethod\n    def write_table_metadata(cls, filename, data):\n\n        def write_file(fn):\n            if all(isinstance(key, str) and isinstance(value, str)\n                   for key, value in data.attributes.items()):\n                with open(fn, 'w', encoding='utf-8') as f:\n                    f.write(\"\\n\".join(\"{}: {}\".format(*kv)\n                                      for kv in data.attributes.items()))\n            else:\n                with open(fn, 'wb') as f:\n                    pickle.dump(data.attributes, f, pickle.HIGHEST_PROTOCOL)\n\n        if isinstance(filename, str):\n            metafile = filename + '.metadata'\n            if getattr(data, 'attributes', None):\n                write_file(metafile)\n            elif path.exists(metafile):\n                remove(metafile)\n\n    @classmethod\n    def set_table_metadata(cls, filename, table):\n        # pylint: disable=bare-except\n        if isinstance(filename, str) and path.exists(filename + '.metadata'):\n            try:\n                with open(filename + '.metadata', 'rb') as f:\n                    table.attributes = pickle.load(f)\n            # Unpickling throws different exceptions, not just UnpickleError\n            except:\n                with open(filename + '.metadata', encoding='utf-8') as f:\n                    table.attributes = OrderedDict(\n                        (k.strip(), v.strip())\n                        for k, v in (line.split(\":\", 1)\n                                     for line in f.readlines()))\n\n    @classmethod\n    def locate(cls, filename, search_dirs=('.',)):\n        \"\"\"Locate a file with given filename that can be opened by one\n        of the available readers.\n\n        Parameters\n        ----------\n        filename : str\n        search_dirs : Iterable[str]\n\n        Returns\n        -------\n        str\n            Absolute path to the file\n        \"\"\"\n        if path.exists(filename):\n            return filename\n\n        for directory in search_dirs:\n            absolute_filename = path.join(directory, filename)\n            if path.exists(absolute_filename):\n                break\n            for ext in cls.readers:\n                if fnmatch(path.basename(filename), '*' + ext):\n                    break\n                # glob uses fnmatch internally\n                matching_files = glob(absolute_filename + ext)\n                if matching_files:\n                    absolute_filename = matching_files[0]\n                    break\n            if path.exists(absolute_filename):\n                break\n        else:\n            absolute_filename = \"\"\n\n        if not path.exists(absolute_filename):\n            raise IOError('File \"{}\" was not found.'.format(filename))\n\n        return absolute_filename\n\n    @staticmethod\n    def open(filename, *args, **kwargs):\n        \"\"\"\n        Format handlers can use this method instead of the builtin ``open()``\n        to transparently (de)compress files if requested (according to\n        `filename` extension). Set ``SUPPORT_COMPRESSED=True`` if you use this.\n        \"\"\"\n        return open_compressed(filename, *args, **kwargs)\n\n    @staticmethod\n    def parse_headers(data):\n        \"\"\"Return (header rows, rest of data) as discerned from `data`\"\"\"\n\n        def is_number(item):\n            try:\n                float(item)\n            except ValueError:\n                return False\n            return True\n        # Second row items are type identifiers\n        def header_test2(items):\n            return all(map(_RE_TYPES.match, items))\n        # Third row items are flags and column attributes (attr=value)\n        def header_test3(items):\n            return all(map(_RE_FLAGS.match, items))\n\n        data = iter(data)\n        header_rows = []\n\n        # Try to parse a three-line header\n        lines = []\n        try:\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n            lines.append(list(next(data)))\n        except StopIteration:\n            lines, data = [], chain(lines, data)\n        if lines:\n            l1, l2, l3 = lines\n            # Three-line header if line 2 & 3 match (1st line can be anything)\n            if header_test2(l2) and header_test3(l3):\n                header_rows = [l1, l2, l3]\n            else:\n                lines, data = [], chain((l1, l2, l3), data)\n\n        # Try to parse a single-line header\n        if not header_rows:\n            try:\n                lines.append(list(next(data)))\n            except StopIteration:\n                pass\n            if lines:\n                # Header if none of the values in line 1 parses as a number\n                if not all(is_number(i) for i in lines[0]):\n                    header_rows = [lines[0]]\n                else:\n                    data = chain(lines, data)\n\n        return header_rows, data\n\n    @classmethod\n    def data_table(cls, data, headers=None):\n        \"\"\"\n        Return Orange.data.Table given rows of `headers` (iterable of iterable)\n        and rows of `data` (iterable of iterable).\n\n        Basically, the idea of subclasses is to produce those two iterables,\n        however they might.\n\n        If `headers` is not provided, the header rows are extracted from `data`,\n        assuming they precede it.\n        \"\"\"\n        if not headers:\n            headers, data = cls.parse_headers(data)\n\n        # Consider various header types (single-row, two-row, three-row, none)\n        if len(headers) == 3:\n            names, types, flags = map(list, headers)\n        else:\n            if len(headers) == 1:\n                HEADER1_FLAG_SEP = '#'\n                # First row format either:\n                #   1) delimited column names\n                #   2) -||- with type and flags prepended, separated by #,\n                #      e.g. d#sex,c#age,cC#IQ\n                _flags, names = zip(*[i.split(HEADER1_FLAG_SEP, 1)\n                                      if HEADER1_FLAG_SEP in i else ('', i)\n                                      for i in headers[0]]\n                                   )\n                names = list(names)\n            elif len(headers) == 2:\n                names, _flags = map(list, headers)\n            else:\n                # Use heuristics for everything\n                names, _flags = [], []\n            types = [''.join(filter(str.isupper, flag)).lower() for flag in _flags]\n            flags = [Flags.join(filter(str.islower, flag)) for flag in _flags]\n\n        # Determine maximum row length\n        rowlen = max(map(len, (names, types, flags)))\n\n        def _equal_length(lst):\n            lst.extend(['']*(rowlen - len(lst)))\n            return lst\n\n        # Ensure all data is of equal width in a column-contiguous array\n        data = [_equal_length([s.strip() for s in row])\n                for row in data if any(row)]\n        data = np.array(data, dtype=object, order='F')\n\n        # Data may actually be longer than headers were\n        try:\n            rowlen = data.shape[1]\n        except IndexError:\n            pass\n        else:\n            for lst in (names, types, flags):\n                _equal_length(lst)\n\n        NAMEGEN = namegen('Feature ', 1)\n        Xcols, attrs = [], []\n        Mcols, metas = [], []\n        Ycols, clses = [], []\n        Wcols = []\n\n        # Rename variables if necessary\n        # Reusing across files still works if both files have same duplicates\n        name_counts = Counter(names)\n        del name_counts[\"\"]\n        if len(name_counts) != len(names) and name_counts:\n            uses = {name: 0 for name, count in name_counts.items() if count > 1}\n            for i, name in enumerate(names):\n                if name in uses:\n                    uses[name] += 1\n                    names[i] = \"{}_{}\".format(name, uses[name])\n\n        namask = np.empty(data.shape[0], dtype=bool)\n        # Iterate through the columns\n        for col in range(rowlen):\n            flag = Flags(Flags.split(flags[col]))\n            if flag.i:\n                continue\n\n            type_flag = types and types[col].strip()\n            try:\n                orig_values = data[:, col]\n            except IndexError:\n                orig_values = np.array([], dtype=object)\n\n            namask = isnastr(orig_values, out=namask)\n\n            coltype_kwargs = {}\n            valuemap = None\n            values = orig_values\n\n            if type_flag in StringVariable.TYPE_HEADERS:\n                coltype = StringVariable\n                values = orig_values\n            elif type_flag in ContinuousVariable.TYPE_HEADERS:\n                coltype = ContinuousVariable\n                values = np.empty(data.shape[0], dtype=float)\n                try:\n                    np.copyto(values, orig_values, casting=\"unsafe\",\n                              where=~namask)\n                    values[namask] = np.nan\n                except ValueError:\n                    for row, num in enumerate(orig_values):\n                        if not isnastr(num):\n                            try:\n                                float(num)\n                            except ValueError:\n                                break\n                    raise ValueError('Non-continuous value in (1-based) '\n                                     'line {}, column {}'.format(row + len(headers) + 1,\n                                                                 col + 1))\n\n            elif type_flag in TimeVariable.TYPE_HEADERS:\n                coltype = TimeVariable\n                values = np.where(namask, \"\", orig_values)\n            elif (type_flag in DiscreteVariable.TYPE_HEADERS or\n                  _RE_DISCRETE_LIST.match(type_flag)):\n                coltype = DiscreteVariable\n                orig_values = values = np.where(namask, \"\", orig_values)\n                if _RE_DISCRETE_LIST.match(type_flag):\n                    valuemap = Flags.split(type_flag)\n                    coltype_kwargs.update(ordered=True)\n                else:\n                    valuemap = sorted(set(orig_values) - {\"\"})\n            else:\n                # No known type specified, use heuristics\n                valuemap, values, coltype = guess_data_type(orig_values, namask)\n\n            if flag.m or coltype is StringVariable:\n                append_to = (Mcols, metas)\n            elif flag.w:\n                append_to = (Wcols, None)\n            elif flag.c:\n                append_to = (Ycols, clses)\n            else:\n                append_to = (Xcols, attrs)\n\n            cols, domain_vars = append_to\n\n            if domain_vars is not None:\n                var_name = names and names[col]\n                if not var_name:\n                    var_name = next(NAMEGEN)\n\n                values, var = sanitize_variable(\n                    valuemap, values, orig_values, coltype, coltype_kwargs,\n                    name=var_name)\n            else:\n                var = None\n            if domain_vars is not None:\n                var.attributes.update(flag.attributes)\n                domain_vars.append(var)\n\n            if isinstance(values, np.ndarray) and not values.flags.owndata:\n                values = values.copy()  # might view `data` (string columns)\n            cols.append(values)\n\n            try:\n                # allow gc to reclaim memory used by string values\n                data[:, col] = None\n            except IndexError:\n                pass\n\n        domain = Domain(attrs, clses, metas)\n\n        if not data.size:\n            return Table.from_domain(domain, 0)\n\n        X = Y = M = W = None\n        if Xcols:\n            X = np.c_[tuple(Xcols)]\n            assert X.dtype == np.float_\n        else:\n            X = np.empty((data.shape[0], 0), dtype=np.float_)\n        if Ycols:\n            Y = np.c_[tuple(Ycols)]\n            assert Y.dtype == np.float_\n        if Mcols:\n            M = np.c_[tuple(Mcols)].astype(object)\n        if Wcols:\n            W = np.c_[tuple(Wcols)].astype(float)\n\n        table = Table.from_numpy(domain, X, Y, M, W)\n        return table\n\n    @staticmethod\n    def header_names(data):\n        return ['weights'] * data.has_weights() + \\\n               [v.name for v in chain(data.domain.attributes,\n                                      data.domain.class_vars,\n                                      data.domain.metas)]\n\n    @staticmethod\n    def header_types(data):\n        def _vartype(var):\n            if var.is_continuous or var.is_string:\n                return var.TYPE_HEADERS[0]\n            elif var.is_discrete:\n                return Flags.join(var.values) if var.ordered else var.TYPE_HEADERS[0]\n            raise NotImplementedError\n        return ['continuous'] * data.has_weights() + \\\n               [_vartype(v) for v in chain(data.domain.attributes,\n                                           data.domain.class_vars,\n                                           data.domain.metas)]\n\n    @staticmethod\n    def header_flags(data):\n        return list(chain(['weight'] * data.has_weights(),\n                          (Flags.join([flag], *('{}={}'.format(*a)\n                                                for a in sorted(var.attributes.items())))\n                           for flag, var in chain(zip(repeat(''), data.domain.attributes),\n                                                  zip(repeat('class'), data.domain.class_vars),\n                                                  zip(repeat('meta'), data.domain.metas)))))\n\n    @classmethod\n    def write_headers(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        write(cls.header_names(data))\n        write(cls.header_types(data))\n        write(cls.header_flags(data))\n\n    @classmethod\n    def write_data(cls, write, data):\n        \"\"\"`write` is a callback that accepts an iterable\"\"\"\n        vars = list(chain((ContinuousVariable('_w'),) if data.has_weights() else (),\n                          data.domain.attributes,\n                          data.domain.class_vars,\n                          data.domain.metas))\n\n        def formatter(var):\n            # type: (Variable) -> Callable[[Variable], Any]\n            # Return a column 'formatter' function. The function must return\n            # something that `write` knows how to write\n            if var.is_time:\n                return var.repr_val\n            elif var.is_continuous:\n                return lambda value: \"\" if isnan(value) else value\n            elif var.is_discrete:\n                return lambda value: \"\" if isnan(value) else var.values[int(value)]\n            elif var.is_string:\n                return lambda value: value\n            else:\n                return var.repr_val\n\n        formatters = [formatter(v) for v in vars]\n        for row in zip(data.W if data.W.ndim > 1 else data.W[:, np.newaxis],\n                       data.X,\n                       data.Y if data.Y.ndim > 1 else data.Y[:, np.newaxis],\n                       data.metas):\n            write([fmt(v) for fmt, v in zip(formatters, flatten(row))])\n\n    @classmethod\n    def qualified_name(cls):\n        return cls.__module__ + '.' + cls.__name__\n\n\ndef class_from_qualified_name(format_name):\n    \"\"\" File format class from qualified name. \"\"\"\n    elements = format_name.split(\".\")\n    m = import_module(\".\".join(elements[:-1]))\n    return getattr(m, elements[-1])\n\n\nclass CSVReader(FileFormat):\n    \"\"\"Reader for comma separated files\"\"\"\n\n    EXTENSIONS = ('.csv',)\n    DESCRIPTION = 'Comma-separated values'\n    DELIMITERS = ',;:\\t$ '\n    SUPPORT_COMPRESSED = True\n    SUPPORT_SPARSE_DATA = False\n    PRIORITY = 20\n\n    def read(self):\n        for encoding in (lambda: ('us-ascii', None),                 # fast\n                         lambda: (detect_encoding(self.filename), None),  # precise\n                         lambda: (locale.getpreferredencoding(False), None),\n                         lambda: (sys.getdefaultencoding(), None),   # desperate\n                         lambda: ('utf-8', None),                    # ...\n                         lambda: ('utf-8', 'ignore')):               # fallback\n            encoding, errors = encoding()\n            # Clear the error flag for all except the last check, because\n            # the error of second-to-last check is stored and shown as warning in owfile\n            if errors != 'ignore':\n                error = ''\n            with self.open(self.filename, mode='rt', newline='',\n                           encoding=encoding, errors=errors) as file:\n                # Sniff the CSV dialect (delimiter, quotes, ...)\n                try:\n                    dialect = csv.Sniffer().sniff(\n                        # Take first couple of *complete* lines as sample\n                        ''.join(file.readline() for _ in range(10)),\n                        self.DELIMITERS)\n                    delimiter = dialect.delimiter\n                    quotechar = dialect.quotechar\n                except UnicodeDecodeError as e:\n                    error = e\n                    continue\n                except csv.Error:\n                    delimiter = self.DELIMITERS[0]\n                    quotechar = csv.excel.quotechar\n\n                file.seek(0)\n                try:\n                    reader = csv.reader(\n                        file, delimiter=delimiter, quotechar=quotechar,\n                        skipinitialspace=True,\n                    )\n                    data = self.data_table(reader)\n\n                    # TODO: Name can be set unconditionally when/if\n                    # self.filename will always be a string with the file name.\n                    # Currently, some tests pass StringIO instead of\n                    # the file name to a reader.\n                    if isinstance(self.filename, str):\n                        data.name = path.splitext(\n                            path.split(self.filename)[-1])[0]\n                    if error and isinstance(error, UnicodeDecodeError):\n                        pos, endpos = error.args[2], error.args[3]\n                        warning = ('Skipped invalid byte(s) in position '\n                                   '{}{}').format(pos,\n                                                  ('-' + str(endpos)) if (endpos - pos) > 1 else '')\n                        warnings.warn(warning)\n                    self.set_table_metadata(self.filename, data)\n                    return data\n                except Exception as e:\n                    error = e\n                    continue\n        raise ValueError('Cannot parse dataset {}: {}'.format(self.filename, error)) from error\n\n    @classmethod\n    def write_file(cls, filename, data):\n        with cls.open(filename, mode='wt', newline='', encoding='utf-8') as file:\n            writer = csv.writer(file, delimiter=cls.DELIMITERS[0])\n            cls.write_headers(writer.writerow, data)\n            cls.write_data(writer.writerow, data)\n        cls.write_table_metadata(filename, data)\n\n\nclass TabReader(CSVReader):\n    \"\"\"Reader for tab separated files\"\"\"\n    EXTENSIONS = ('.tab', '.tsv')\n    DESCRIPTION = 'Tab-separated values'\n    DELIMITERS = '\\t'\n    PRIORITY = 10\n\n\nclass PickleReader(FileFormat):\n    \"\"\"Reader for pickled Table objects\"\"\"\n    EXTENSIONS = ('.pkl', '.pickle')\n    DESCRIPTION = 'Pickled Orange data'\n    SUPPORT_COMPRESSED = True\n    SUPPORT_SPARSE_DATA = True\n\n    def read(self):\n        with self.open(self.filename, 'rb') as f:\n            table = pickle.load(f)\n            if not isinstance(table, Table):\n                raise TypeError(\"file does not contain a data table\")\n            else:\n                return table\n\n    @classmethod\n    def write_file(cls, filename, data):\n        with cls.open(filename, 'wb') as f:\n            pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)\n\n\nclass BasketReader(FileFormat):\n    \"\"\"Reader for basket (sparse) files\"\"\"\n    EXTENSIONS = ('.basket', '.bsk')\n    DESCRIPTION = 'Basket file'\n    SUPPORT_SPARSE_DATA = True\n\n    def read(self):\n        def constr_vars(inds):\n            if inds:\n                return [ContinuousVariable(x.decode(\"utf-8\")) for _, x in\n                        sorted((ind, name) for name, ind in inds.items())]\n\n        X, Y, metas, attr_indices, class_indices, meta_indices = \\\n            _io.sparse_read_float(self.filename.encode(sys.getdefaultencoding()))\n\n        attrs = constr_vars(attr_indices)\n        classes = constr_vars(class_indices)\n        meta_attrs = constr_vars(meta_indices)\n        domain = Domain(attrs, classes, meta_attrs)\n        table = Table.from_numpy(\n            domain, attrs and X, classes and Y, metas and meta_attrs)\n        table.name = path.splitext(path.split(self.filename)[-1])[0]\n        return table\n\n\nclass ExcelReader(FileFormat):\n    \"\"\"Reader for excel files\"\"\"\n    EXTENSIONS = ('.xls', '.xlsx')\n    DESCRIPTION = 'Mircosoft Excel spreadsheet'\n    SUPPORT_SPARSE_DATA = False\n\n    def __init__(self, filename):\n        super().__init__(filename)\n\n        from xlrd import open_workbook\n        self.workbook = open_workbook(self.filename)\n\n    @property\n    @lru_cache(1)\n    def sheets(self):\n        return self.workbook.sheet_names()\n\n    def read(self):\n        import xlrd\n        wb = xlrd.open_workbook(self.filename, on_demand=True)\n        if self.sheet:\n            ss = wb.sheet_by_name(self.sheet)\n        else:\n            ss = wb.sheet_by_index(0)\n        try:\n            first_row = next(i for i in range(ss.nrows) if any(ss.row_values(i)))\n            first_col = next(i for i in range(ss.ncols) if ss.cell_value(first_row, i))\n            row_len = ss.row_len(first_row)\n            cells = filter(any,\n                           [[str(ss.cell_value(row, col)) if col < ss.row_len(row) else ''\n                             for col in range(first_col, row_len)]\n                            for row in range(first_row, ss.nrows)])\n            table = self.data_table(cells)\n            table.name = path.splitext(path.split(self.filename)[-1])[0]\n            if self.sheet:\n                table.name = '-'.join((table.name, self.sheet))\n        except Exception:\n            raise IOError(\"Couldn't load spreadsheet from \" + self.filename)\n        return table\n\n\nclass DotReader(FileFormat):\n    \"\"\"Writer for dot (graph) files\"\"\"\n    EXTENSIONS = ('.dot', '.gv')\n    DESCRIPTION = 'Dot graph description'\n    SUPPORT_COMPRESSED = True\n    SUPPORT_SPARSE_DATA = False\n\n    @classmethod\n    def write_graph(cls, filename, graph):\n        from sklearn import tree\n        tree.export_graphviz(graph, out_file=cls.open(filename, 'wt'))\n\n    @classmethod\n    def write(cls, filename, tree):\n        if type(tree) == dict:\n            tree = tree['tree']\n        cls.write_graph(filename, tree)\n\n\nclass UrlReader(FileFormat):\n    def __init__(self, filename):\n        filename = filename.strip()\n        if not urlparse(filename).scheme:\n            filename = 'http://' + filename\n        super().__init__(filename)\n\n    @staticmethod\n    def urlopen(url):\n        req = Request(\n            url,\n            # Avoid 403 error with servers that dislike scrapers\n            headers={'User-Agent': 'Mozilla/5.0 (X11; Linux) Gecko/20100101 Firefox/'})\n        return urlopen(req, timeout=10)\n\n    def read(self):\n        self.filename = self._trim(self._resolve_redirects(self.filename))\n        with contextlib.closing(self.urlopen(self.filename)) as response:\n            name = self._suggest_filename(response.headers['content-disposition'])\n            with NamedTemporaryFile(suffix=name, delete=False) as f:\n                f.write(response.read())\n                # delete=False is a workaround for https://bugs.python.org/issue14243\n\n            reader = self.get_reader(f.name)\n            data = reader.read()\n            remove(f.name)\n        # Override name set in from_file() to avoid holding the temp prefix\n        data.name = path.splitext(name)[0]\n        data.origin = self.filename\n        return data\n\n    def _resolve_redirects(self, url):\n        # Resolve (potential) redirects to a final URL\n        with contextlib.closing(self.urlopen(url)) as response:\n            return response.url\n\n    @classmethod\n    def _trim(cls, url):\n        URL_TRIMMERS = (\n            cls._trim_googlesheet,\n            cls._trim_dropbox,\n        )\n        for trim in URL_TRIMMERS:\n            try:\n                url = trim(url)\n            except ValueError:\n                continue\n            else:\n                break\n        return url\n\n    @staticmethod\n    def _trim_googlesheet(url):\n        match = re.match(r'(?:https?://)?(?:www\\.)?'\n                         r'docs\\.google\\.com/spreadsheets/d/'\n                         r'(?P<workbook_id>[-\\w_]+)'\n                         r'(?:/.*?gid=(?P<sheet_id>\\d+).*|.*)?',\n                         url, re.IGNORECASE)\n        try:\n            workbook, sheet = match.group('workbook_id'), match.group('sheet_id')\n            if not workbook:\n                raise ValueError\n        except (AttributeError, ValueError):\n            raise ValueError\n        url = 'https://docs.google.com/spreadsheets/d/{}/export?format=tsv'.format(workbook)\n        if sheet:\n            url += '&gid=' + sheet\n        return url\n\n    @staticmethod\n    def _trim_dropbox(url):\n        parts = urlsplit(url)\n        if not parts.netloc.endswith('dropbox.com'):\n            raise ValueError\n        return urlunsplit(parts._replace(query='dl=1'))\n\n    def _suggest_filename(self, content_disposition):\n        default_name = re.sub(r'[\\\\:/]', '_', urlparse(self.filename).path)\n\n        # See https://tools.ietf.org/html/rfc6266#section-4.1\n        matches = re.findall(r\"filename\\*?=(?:\\\"|.{0,10}?'[^']*')([^\\\"]+)\",\n                             content_disposition or '')\n        return urlunquote(matches[-1]) if matches else default_name\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]

**Title:** CSV Reader Fails to Handle Files with More Data Columns Than Headers

**Description:**
When reading a CSV file where data rows contain more columns than the header row, the CSV Reader does not issue a warning and fails to remove the extra columns.

**Example Buggy Code:**
```python
import io
from Orange.data.io import CSVReader

samplefile = """\
a, b
1, 0,
1, 2,
"""
c = io.StringIO(samplefile)
table = CSVReader(c).read()
print(len(table.domain.attributes))  # Expected: 2 attributes
```

**Expected Behavior:**
A warning stating "Columns with no headers were removed." should be issued, and the extra data columns should be stripped, resulting in only the defined header columns being present.

**Actual Behavior:**
No warning is triggered, and the extra columns remain, leading to inconsistencies between the header and data rows.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### Orange/data/io.py
import contextlib
import csv
import locale
import pickle
import re
import subprocess
import sys
import warnings

from ast import literal_eval
from collections import OrderedDict, Counter, defaultdict
from functools import lru_cache
from importlib import import_module
from itertools import chain, repeat
from math import isnan

from os import path, remove
from tempfile import NamedTemporaryFile
from urllib.parse import urlparse, urlsplit, urlunsplit, unquote as urlunquote
from urllib.request import urlopen, Request
from fnmatch import fnmatch
from glob import glob

import numpy as np
from chardet.universaldetector import UniversalDetector

from Orange.data import (
    _io, is_discrete_values, MISSING_VALUES, Table, Domain, Variable,
    DiscreteVariable, StringVariable, ContinuousVariable, TimeVariable,
)
from Orange.util import Registry, flatten, namegen


# Support values longer than 128K (i.e. text contents features)
csv.field_size_limit(100*1024*1024)

__all__ = ["Flags", "FileFormat"]


_IDENTITY = lambda i: i


class Compression:
    """Supported compression extensions"""
    GZIP = '.gz'
    BZIP2 = '.bz2'
    XZ = '.xz'
    all = (GZIP, BZIP2, XZ)


def open_compressed(filename, *args, _open=open, **kwargs):
    """Return seamlessly decompressed open file handle for `filename`"""
    if isinstance(filename, str):
        if filename.endswith(Compression.GZIP):
            from gzip import open as _open
        elif filename.endswith(Compression.BZIP2):
            from bz2 import open as _open
        elif filename.endswith(Compression.XZ):
            from lzma import open as _open
        return _open(filename, *args, **kwargs)
    # Else already a file, just pass it through
    return filename


def detect_encoding(filename):
    """
    Detect encoding of `filename`, which can be a ``str`` filename, a
    ``file``-like object, or ``bytes``.
    """
    # Try with Unix file utility first because it's faster (~10ms vs 100ms)
    if isinstance(filename, str) and not filename.endswith(Compression.all):
        try:
            with subprocess.Popen(('file', '--brief', '--mime-encoding', filename),
                                  stdout=subprocess.PIPE) as process:
                process.wait()
                if process.returncode == 0:
                    encoding = process.stdout.read().strip()
                    # file only supports these encodings; for others it says
                    # unknown-8bit or binary. So we give chardet a chance to do
                    # better
                    if encoding in (b'utf-8', b'us-ascii', b'iso-8859-1',
                                    b'utf-7', b'utf-16le', b'utf-16be', b'ebcdic'):
                        return encoding.decode('us-ascii')
        except OSError:
            pass  # windoze

    # file not available or unable to guess the encoding, have chardet do it
    detector = UniversalDetector()
    # We examine only first N 4kB blocks of file because chardet is really slow
    MAX_BYTES = 4*1024*12

    def _from_file(f):
        detector.feed(f.read(MAX_BYTES))
        detector.close()
        return (detector.result.get('encoding')
                if detector.result.get('confidence', 0) >= .85 else
                'utf-8')

    if isinstance(filename, str):
        with open_compressed(filename, 'rb') as f:
            return _from_file(f)
    elif isinstance(filename, bytes):
        detector.feed(filename[:MAX_BYTES])
        detector.close()
        return detector.result.get('encoding')
    elif hasattr(filename, 'encoding'):
        return filename.encoding
    else:  # assume file-like object that you can iter through
        return _from_file(filename)


__isnastr = np.frompyfunc(
    {v for v in MISSING_VALUES if isinstance(v, str)}.__contains__, 1, 1)

# wrapper for __isnastr with proper default out dtype
def isnastr(arr, out=None):
    """
    Given an (object) array of string values, return a boolean mask array
    that is True where the `arr` contains one of the string constants
    considered as N/A.

    Parameters
    ----------
    arr : np.ndarray
        Input array of strings.
    out : Optional[np.ndarray]
        Optional output array of the same shape as arr

    Returns
    -------
    mask : np.ndarray
    """
    arr = np.asarray(arr)
    if out is None and arr.shape != ():
        out = np.empty_like(arr, dtype=bool)
    return __isnastr(arr, out=out)


def guess_data_type(orig_values, namask=None):
    """
    Use heuristics to guess data type.
    """
    valuemap, values = None, orig_values
    is_discrete = is_discrete_values(orig_values)
    if is_discrete:
        valuemap = sorted(is_discrete)
        coltype = DiscreteVariable
    else:
        # try to parse as float
        orig_values = np.asarray(orig_values)
        if namask is None:
            namask = isnastr(orig_values)
        values = np.empty_like(orig_values, dtype=float)
        values[namask] = np.nan
        try:
            np.copyto(values, orig_values, where=~namask, casting="unsafe")
        except ValueError:
            tvar = TimeVariable('_')
            try:
                values[~namask] = [tvar.parse(i) for i in orig_values[~namask]]
            except ValueError:
                coltype = StringVariable
                # return original_values
                values = orig_values
            else:
                coltype = TimeVariable
        else:
            coltype = ContinuousVariable
    return valuemap, values, coltype


def sanitize_variable(valuemap, values, orig_values, coltype, coltype_kwargs,
                      domain_vars=None, existing_var=None, new_var_name=None, data=None, name=None):
    assert issubclass(coltype, Variable)

    if name is None or existing_var is not None or new_var_name is not None:
        name = existing_var.strip() if existing_var else new_var_name
        raise DeprecationWarning("Arguments 'existing_var' and 'new_var_name' are "\
                                 "deprecated since 3.16; use 'name' instead")

    if domain_vars is not None:
        raise DeprecationWarning("Argument 'domain_vars' is deprecated since 3.16")

    if data is not None:
        raise DeprecationWarning("Argument 'data' is deprecated since 3.16")

    def get_number_of_decimals(values):
        len_ = len
        ndecimals = max((len_(value) - value.find(".")
                         for value in values if "." in value),
                        default=1)
        return ndecimals - 1

    if issubclass(coltype, DiscreteVariable) and valuemap is not None:
        coltype_kwargs.update(values=valuemap)

    var = coltype.make(name, **coltype_kwargs)

    if isinstance(var, DiscreteVariable):
        # Map discrete data to 'ints' (or at least what passes as int around
        # here)
        mapping = defaultdict(
            lambda: np.nan,
            {val: i for i, val in enumerate(var.values)},
        )
        mapping[""] = np.nan
        mapvalues_ = np.frompyfunc(mapping.__getitem__, 1, 1)

        def mapvalues(arr):
            arr = np.asarray(arr, dtype=object)
            return mapvalues_(arr, out=np.empty_like(arr, dtype=float))
        values = mapvalues(orig_values)

    if coltype is StringVariable:
        values = orig_values

    # ContinuousVariable.number_of_decimals is supposed to be handled by
    # ContinuousVariable.to_val. In the interest of speed, the reader bypasses
    # it, so we set the number of decimals here.
    # The number of decimals is increased if not set manually (in which case
    # var.adjust_decimals would be 0).
    if isinstance(var, ContinuousVariable) and var.adjust_decimals:
        ndecimals = get_number_of_decimals(orig_values)
        if var.adjust_decimals == 2 or ndecimals > var.number_of_decimals:
            var.number_of_decimals = ndecimals
            var.adjust_decimals = 1

    if isinstance(var, TimeVariable) or coltype is TimeVariable:
        # Re-parse the values because only now after coltype.make call
        # above, variable var is the correct one
        _var = var if isinstance(var, TimeVariable) else TimeVariable('_')
        values = [_var.parse(i) for i in orig_values]

    return values, var


class Flags:
    """Parser for column flags (i.e. third header row)"""
    DELIMITER = ' '
    _RE_SPLIT = re.compile(r'(?<!\\)' + DELIMITER).split
    _RE_ATTR_UNQUOTED_STR = re.compile(r'^[a-zA-Z_]').match
    ALL = OrderedDict((
        ('class', 'c'),
        ('ignore', 'i'),
        ('meta', 'm'),
        ('weight', 'w'),
        ('.+?=.*?', ''),  # general key=value attributes
    ))
    _RE_ALL = re.compile(r'^({})$'.format('|'.join(filter(None, flatten(ALL.items())))))

    def __init__(self, flags):
        for v in filter(None, self.ALL.values()):
            setattr(self, v, False)
        self.attributes = {}
        for flag in flags or []:
            flag = flag.strip()
            if self._RE_ALL.match(flag):
                if '=' in flag:
                    k, v = flag.split('=', 1)
                    if not Flags._RE_ATTR_UNQUOTED_STR(v):
                        try:
                            v = literal_eval(v)
                        except SyntaxError:
                            # If parsing failed, treat value as string
                            pass
                    self.attributes[k] = v
                else:
                    setattr(self, flag, True)
                    setattr(self, self.ALL.get(flag, ''), True)
            elif flag:
                warnings.warn('Invalid attribute flag \'{}\''.format(flag))

    @staticmethod
    def join(iterable, *args):
        return Flags.DELIMITER.join(i.strip().replace(Flags.DELIMITER, '\\' + Flags.DELIMITER)
                                    for i in chain(iterable, args)).lstrip()

    @staticmethod
    def split(s):
        return [i.replace('\\' + Flags.DELIMITER, Flags.DELIMITER)
                for i in Flags._RE_SPLIT(s)]


# Matches discrete specification where all the values are listed, space-separated
_RE_DISCRETE_LIST = re.compile(r'^\s*[^\s]+(\s[^\s]+)+\s*$')
_RE_TYPES = re.compile(r'^\s*({}|{}|)\s*$'.format(
    _RE_DISCRETE_LIST.pattern,
    '|'.join(flatten(getattr(vartype, 'TYPE_HEADERS') for vartype in Variable.registry.values()))
    ))
_RE_FLAGS = re.compile(r'^\s*( |{}|)*\s*$'.format(
    '|'.join(flatten(filter(None, i) for i in Flags.ALL.items()))
    ))


class FileFormatMeta(Registry):

    def __new__(cls, name, bases, attrs):
        newcls = super().__new__(cls, name, bases, attrs)

        # Optionally add compressed versions of extensions as supported
        if getattr(newcls, 'SUPPORT_COMPRESSED', False):
            new_extensions = list(getattr(newcls, 'EXTENSIONS', ()))
            for compression in Compression.all:
                for ext in newcls.EXTENSIONS:
                    new_extensions.append(ext + compression)
                if sys.platform in ('darwin', 'win32'):
                    # OSX file dialog doesn't support filtering on double
                    # extensions (e.g. .csv.gz)
                    # https://bugreports.qt.io/browse/QTBUG-38303
                    # This is just here for OWFile that gets QFileDialog
                    # filters from FileFormat.readers.keys()
                    # EDIT: Windows exhibit similar problems:
                    # while .tab.gz works, .tab.xz and .tab.bz2 do not!
                    new_extensions.append(compression)
            newcls.EXTENSIONS = tuple(new_extensions)

        return newcls

    @property
    def formats(cls):
        return cls.registry.values()

    @lru_cache(5)
    def _ext_to_attr_if_attr2(cls, attr, attr2):
        """
        Return ``{ext: `attr`, ...}`` dict if ``cls`` has `attr2`.
        If `attr` is '', return ``{ext: cls, ...}`` instead.

        If there are multiple formats for an extension, return a format
        with the lowest priority.
        """
        formats = OrderedDict()
        for format in sorted(cls.registry.values(), key=lambda x: x.PRIORITY):
            if not hasattr(format, attr2):
                continue
            for ext in getattr(format, 'EXTENSIONS', []):
                # Only adds if not yet registered
                formats.setdefault(ext, getattr(format, attr, format))
        return formats

    @property
    def names(cls):
        return cls._ext_to_attr_if_attr2('DESCRIPTION', '__class__')

    @property
    def writers(cls):
        return cls._ext_to_attr_if_attr2('', 'write_file')

    @property
    def readers(cls):
        return cls._ext_to_attr_if_attr2('', 'read')

    @property
    def img_writers(cls):
        return cls._ext_to_attr_if_attr2('', 'write_image')

    @property
    def graph_writers(cls):
        return cls._ext_to_attr_if_attr2('', 'write_graph')


class FileFormat(metaclass=FileFormatMeta):
    """
    Subclasses set the following attributes and override the following methods:

        EXTENSIONS = ('.ext1', '.ext2', ...)
        DESCRIPTION = 'human-readable file format description'
        SUPPORT_COMPRESSED = False
        SUPPORT_SPARSE_DATA = False

        def read(self):
            ...  # load headers, data, ...
            return self.data_table(data, headers)

        @classmethod
        def write_file(cls, filename, data):
            ...
            self.write_headers(writer.write, data)
            writer.writerows(data)

    Wrapper FileFormat.data_table() returns Orange.data.Table from `data`
    iterable (list (rows) of lists of values (cols)).
    """

    # Priority when multiple formats support the same extension. Also
    # the sort order in file open/save combo boxes. Lower is better.
    PRIORITY = 10000

    def __init__(self, filename):
        """
        Parameters
        ----------
        filename : str
            name of the file to open
        """
        self.filename = filename
        self.sheet = None

    @property
    def sheets(self):
        """FileFormats with a notion of sheets should override this property
        to return a list of sheet names in the file.

        Returns
        -------
        a list of sheet names
        """
        return ()

    def select_sheet(self, sheet):
        """Select sheet to be read

        Parameters
        ----------
        sheet : str
            sheet name
        """
        self.sheet = sheet

    @classmethod
    def get_reader(cls, filename):
        """Return reader instance that can be used to read the file

        Parameters
        ----------
        filename : str

        Returns
        -------
        FileFormat
        """
        for ext, reader in cls.readers.items():
            # Skip ambiguous, invalid compression-only extensions added on OSX
            if ext in Compression.all:
                continue
            if fnmatch(path.basename(filename), '*' + ext):
                return reader(filename)

        raise IOError('No readers for file "{}"'.format(filename))

    @classmethod
    def write(cls, filename, data):
        return cls.write_file(filename, data)

    @classmethod
    def write_table_metadata(cls, filename, data):

        def write_file(fn):
            if all(isinstance(key, str) and isinstance(value, str)
                   for key, value in data.attributes.items()):
                with open(fn, 'w', encoding='utf-8') as f:
                    f.write("\n".join("{}: {}".format(*kv)
                                      for kv in data.attributes.items()))
            else:
                with open(fn, 'wb') as f:
                    pickle.dump(data.attributes, f, pickle.HIGHEST_PROTOCOL)

        if isinstance(filename, str):
            metafile = filename + '.metadata'
            if getattr(data, 'attributes', None):
                write_file(metafile)
            elif path.exists(metafile):
                remove(metafile)

    @classmethod
    def set_table_metadata(cls, filename, table):
        # pylint: disable=bare-except
        if isinstance(filename, str) and path.exists(filename + '.metadata'):
            try:
                with open(filename + '.metadata', 'rb') as f:
                    table.attributes = pickle.load(f)
            # Unpickling throws different exceptions, not just UnpickleError
            except:
                with open(filename + '.metadata', encoding='utf-8') as f:
                    table.attributes = OrderedDict(
                        (k.strip(), v.strip())
                        for k, v in (line.split(":", 1)
                                     for line in f.readlines()))

    @classmethod
    def locate(cls, filename, search_dirs=('.',)):
        """Locate a file with given filename that can be opened by one
        of the available readers.

        Parameters
        ----------
        filename : str
        search_dirs : Iterable[str]

        Returns
        -------
        str
            Absolute path to the file
        """
        if path.exists(filename):
            return filename

        for directory in search_dirs:
            absolute_filename = path.join(directory, filename)
            if path.exists(absolute_filename):
                break
            for ext in cls.readers:
                if fnmatch(path.basename(filename), '*' + ext):
                    break
                # glob uses fnmatch internally
                matching_files = glob(absolute_filename + ext)
                if matching_files:
                    absolute_filename = matching_files[0]
                    break
            if path.exists(absolute_filename):
                break
        else:
            absolute_filename = ""

        if not path.exists(absolute_filename):
            raise IOError('File "{}" was not found.'.format(filename))

        return absolute_filename

    @staticmethod
    def open(filename, *args, **kwargs):
        """
        Format handlers can use this method instead of the builtin ``open()``
        to transparently (de)compress files if requested (according to
        `filename` extension). Set ``SUPPORT_COMPRESSED=True`` if you use this.
        """
        return open_compressed(filename, *args, **kwargs)

    @staticmethod
    def parse_headers(data):
        """Return (header rows, rest of data) as discerned from `data`"""

        def is_number(item):
            try:
                float(item)
            except ValueError:
                return False
            return True
        # Second row items are type identifiers
        def header_test2(items):
            return all(map(_RE_TYPES.match, items))
        # Third row items are flags and column attributes (attr=value)
        def header_test3(items):
            return all(map(_RE_FLAGS.match, items))

        data = iter(data)
        header_rows = []

        # Try to parse a three-line header
        lines = []
        try:
            lines.append(list(next(data)))
            lines.append(list(next(data)))
            lines.append(list(next(data)))
        except StopIteration:
            lines, data = [], chain(lines, data)
        if lines:
            l1, l2, l3 = lines
            # Three-line header if line 2 & 3 match (1st line can be anything)
            if header_test2(l2) and header_test3(l3):
                header_rows = [l1, l2, l3]
            else:
                lines, data = [], chain((l1, l2, l3), data)

        # Try to parse a single-line header
        if not header_rows:
            try:
                lines.append(list(next(data)))
            except StopIteration:
                pass
            if lines:
                # Header if none of the values in line 1 parses as a number
                if not all(is_number(i) for i in lines[0]):
                    header_rows = [lines[0]]
                else:
                    data = chain(lines, data)

        return header_rows, data

    @classmethod
    def data_table(cls, data, headers=None):
        """
        Return Orange.data.Table given rows of `headers` (iterable of iterable)
        and rows of `data` (iterable of iterable).

        Basically, the idea of subclasses is to produce those two iterables,
        however they might.

        If `headers` is not provided, the header rows are extracted from `data`,
        assuming they precede it.
        """
        if not headers:
            headers, data = cls.parse_headers(data)

        # Consider various header types (single-row, two-row, three-row, none)
        if len(headers) == 3:
            names, types, flags = map(list, headers)
        else:
            if len(headers) == 1:
                HEADER1_FLAG_SEP = '#'
                # First row format either:
                #   1) delimited column names
                #   2) -||- with type and flags prepended, separated by #,
                #      e.g. d#sex,c#age,cC#IQ
                _flags, names = zip(*[i.split(HEADER1_FLAG_SEP, 1)
                                      if HEADER1_FLAG_SEP in i else ('', i)
                                      for i in headers[0]]
                                   )
                names = list(names)
            elif len(headers) == 2:
                names, _flags = map(list, headers)
            else:
                # Use heuristics for everything
                names, _flags = [], []
            types = [''.join(filter(str.isupper, flag)).lower() for flag in _flags]
            flags = [Flags.join(filter(str.islower, flag)) for flag in _flags]

        # Determine maximum row length
        rowlen = max(map(len, (names, types, flags)))

        def _equal_length(lst):
            lst.extend(['']*(rowlen - len(lst)))
            return lst

        # Ensure all data is of equal width in a column-contiguous array
        data = [_equal_length([s.strip() for s in row])
                for row in data if any(row)]
        data = np.array(data, dtype=object, order='F')

        # Data may actually be longer than headers were
        try:
            rowlen = data.shape[1]
        except IndexError:
            pass
        else:
            for lst in (names, types, flags):
                _equal_length(lst)

        NAMEGEN = namegen('Feature ', 1)
        Xcols, attrs = [], []
        Mcols, metas = [], []
        Ycols, clses = [], []
        Wcols = []

        # Rename variables if necessary
        # Reusing across files still works if both files have same duplicates
        name_counts = Counter(names)
        del name_counts[""]
        if len(name_counts) != len(names) and name_counts:
            uses = {name: 0 for name, count in name_counts.items() if count > 1}
            for i, name in enumerate(names):
                if name in uses:
                    uses[name] += 1
                    names[i] = "{}_{}".format(name, uses[name])

        namask = np.empty(data.shape[0], dtype=bool)
        # Iterate through the columns
        for col in range(rowlen):
            flag = Flags(Flags.split(flags[col]))
            if flag.i:
                continue

            type_flag = types and types[col].strip()
            try:
                orig_values = data[:, col]
            except IndexError:
                orig_values = np.array([], dtype=object)

            namask = isnastr(orig_values, out=namask)

            coltype_kwargs = {}
            valuemap = None
            values = orig_values

            if type_flag in StringVariable.TYPE_HEADERS:
                coltype = StringVariable
                values = orig_values
            elif type_flag in ContinuousVariable.TYPE_HEADERS:
                coltype = ContinuousVariable
                values = np.empty(data.shape[0], dtype=float)
                try:
                    np.copyto(values, orig_values, casting="unsafe",
                              where=~namask)
                    values[namask] = np.nan
                except ValueError:
                    for row, num in enumerate(orig_values):
                        if not isnastr(num):
                            try:
                                float(num)
                            except ValueError:
                                break
                    raise ValueError('Non-continuous value in (1-based) '
                                     'line {}, column {}'.format(row + len(headers) + 1,
                                                                 col + 1))

            elif type_flag in TimeVariable.TYPE_HEADERS:
                coltype = TimeVariable
                values = np.where(namask, "", orig_values)
            elif (type_flag in DiscreteVariable.TYPE_HEADERS or
                  _RE_DISCRETE_LIST.match(type_flag)):
                coltype = DiscreteVariable
                orig_values = values = np.where(namask, "", orig_values)
                if _RE_DISCRETE_LIST.match(type_flag):
                    valuemap = Flags.split(type_flag)
                    coltype_kwargs.update(ordered=True)
                else:
                    valuemap = sorted(set(orig_values) - {""})
            else:
                # No known type specified, use heuristics
                valuemap, values, coltype = guess_data_type(orig_values, namask)

            if flag.m or coltype is StringVariable:
                append_to = (Mcols, metas)
            elif flag.w:
                append_to = (Wcols, None)
            elif flag.c:
                append_to = (Ycols, clses)
            else:
                append_to = (Xcols, attrs)

            cols, domain_vars = append_to

            if domain_vars is not None:
                var_name = names and names[col]
                if not var_name:
                    var_name = next(NAMEGEN)

                values, var = sanitize_variable(
                    valuemap, values, orig_values, coltype, coltype_kwargs,
                    name=var_name)
            else:
                var = None
            if domain_vars is not None:
                var.attributes.update(flag.attributes)
                domain_vars.append(var)

            if isinstance(values, np.ndarray) and not values.flags.owndata:
                values = values.copy()  # might view `data` (string columns)
            cols.append(values)

            try:
                # allow gc to reclaim memory used by string values
                data[:, col] = None
            except IndexError:
                pass

        domain = Domain(attrs, clses, metas)

        if not data.size:
            return Table.from_domain(domain, 0)

        X = Y = M = W = None
        if Xcols:
            X = np.c_[tuple(Xcols)]
            assert X.dtype == np.float_
        else:
            X = np.empty((data.shape[0], 0), dtype=np.float_)
        if Ycols:
            Y = np.c_[tuple(Ycols)]
            assert Y.dtype == np.float_
        if Mcols:
            M = np.c_[tuple(Mcols)].astype(object)
        if Wcols:
            W = np.c_[tuple(Wcols)].astype(float)

        table = Table.from_numpy(domain, X, Y, M, W)
        return table

    @staticmethod
    def header_names(data):
        return ['weights'] * data.has_weights() + \
               [v.name for v in chain(data.domain.attributes,
                                      data.domain.class_vars,
                                      data.domain.metas)]

    @staticmethod
    def header_types(data):
        def _vartype(var):
            if var.is_continuous or var.is_string:
                return var.TYPE_HEADERS[0]
            elif var.is_discrete:
                return Flags.join(var.values) if var.ordered else var.TYPE_HEADERS[0]
            raise NotImplementedError
        return ['continuous'] * data.has_weights() + \
               [_vartype(v) for v in chain(data.domain.attributes,
                                           data.domain.class_vars,
                                           data.domain.metas)]

    @staticmethod
    def header_flags(data):
        return list(chain(['weight'] * data.has_weights(),
                          (Flags.join([flag], *('{}={}'.format(*a)
                                                for a in sorted(var.attributes.items())))
                           for flag, var in chain(zip(repeat(''), data.domain.attributes),
                                                  zip(repeat('class'), data.domain.class_vars),
                                                  zip(repeat('meta'), data.domain.metas)))))

    @classmethod
    def write_headers(cls, write, data):
        """`write` is a callback that accepts an iterable"""
        write(cls.header_names(data))
        write(cls.header_types(data))
        write(cls.header_flags(data))

    @classmethod
    def write_data(cls, write, data):
        """`write` is a callback that accepts an iterable"""
        vars = list(chain((ContinuousVariable('_w'),) if data.has_weights() else (),
                          data.domain.attributes,
                          data.domain.class_vars,
                          data.domain.metas))

        def formatter(var):
            # type: (Variable) -> Callable[[Variable], Any]
            # Return a column 'formatter' function. The function must return
            # something that `write` knows how to write
            if var.is_time:
                return var.repr_val
            elif var.is_continuous:
                return lambda value: "" if isnan(value) else value
            elif var.is_discrete:
                return lambda value: "" if isnan(value) else var.values[int(value)]
            elif var.is_string:
                return lambda value: value
            else:
                return var.repr_val

        formatters = [formatter(v) for v in vars]
        for row in zip(data.W if data.W.ndim > 1 else data.W[:, np.newaxis],
                       data.X,
                       data.Y if data.Y.ndim > 1 else data.Y[:, np.newaxis],
                       data.metas):
            write([fmt(v) for fmt, v in zip(formatters, flatten(row))])

    @classmethod
    def qualified_name(cls):
        return cls.__module__ + '.' + cls.__name__


def class_from_qualified_name(format_name):
    """ File format class from qualified name. """
    elements = format_name.split(".")
    m = import_module(".".join(elements[:-1]))
    return getattr(m, elements[-1])


class CSVReader(FileFormat):
    """Reader for comma separated files"""

    EXTENSIONS = ('.csv',)
    DESCRIPTION = 'Comma-separated values'
    DELIMITERS = ',;:\t$ '
    SUPPORT_COMPRESSED = True
    SUPPORT_SPARSE_DATA = False
    PRIORITY = 20

    def read(self):
        for encoding in (lambda: ('us-ascii', None),                 # fast
                         lambda: (detect_encoding(self.filename), None),  # precise
                         lambda: (locale.getpreferredencoding(False), None),
                         lambda: (sys.getdefaultencoding(), None),   # desperate
                         lambda: ('utf-8', None),                    # ...
                         lambda: ('utf-8', 'ignore')):               # fallback
            encoding, errors = encoding()
            # Clear the error flag for all except the last check, because
            # the error of second-to-last check is stored and shown as warning in owfile
            if errors != 'ignore':
                error = ''
            with self.open(self.filename, mode='rt', newline='',
                           encoding=encoding, errors=errors) as file:
                # Sniff the CSV dialect (delimiter, quotes, ...)
                try:
                    dialect = csv.Sniffer().sniff(
                        # Take first couple of *complete* lines as sample
                        ''.join(file.readline() for _ in range(10)),
                        self.DELIMITERS)
                    delimiter = dialect.delimiter
                    quotechar = dialect.quotechar
                except UnicodeDecodeError as e:
                    error = e
                    continue
                except csv.Error:
                    delimiter = self.DELIMITERS[0]
                    quotechar = csv.excel.quotechar

                file.seek(0)
                try:
                    reader = csv.reader(
                        file, delimiter=delimiter, quotechar=quotechar,
                        skipinitialspace=True,
                    )
                    data = self.data_table(reader)

                    # TODO: Name can be set unconditionally when/if
                    # self.filename will always be a string with the file name.
                    # Currently, some tests pass StringIO instead of
                    # the file name to a reader.
                    if isinstance(self.filename, str):
                        data.name = path.splitext(
                            path.split(self.filename)[-1])[0]
                    if error and isinstance(error, UnicodeDecodeError):
                        pos, endpos = error.args[2], error.args[3]
                        warning = ('Skipped invalid byte(s) in position '
                                   '{}{}').format(pos,
                                                  ('-' + str(endpos)) if (endpos - pos) > 1 else '')
                        warnings.warn(warning)
                    self.set_table_metadata(self.filename, data)
                    return data
                except Exception as e:
                    error = e
                    continue
        raise ValueError('Cannot parse dataset {}: {}'.format(self.filename, error)) from error

    @classmethod
    def write_file(cls, filename, data):
        with cls.open(filename, mode='wt', newline='', encoding='utf-8') as file:
            writer = csv.writer(file, delimiter=cls.DELIMITERS[0])
            cls.write_headers(writer.writerow, data)
            cls.write_data(writer.writerow, data)
        cls.write_table_metadata(filename, data)


class TabReader(CSVReader):
    """Reader for tab separated files"""
    EXTENSIONS = ('.tab', '.tsv')
    DESCRIPTION = 'Tab-separated values'
    DELIMITERS = '\t'
    PRIORITY = 10


class PickleReader(FileFormat):
    """Reader for pickled Table objects"""
    EXTENSIONS = ('.pkl', '.pickle')
    DESCRIPTION = 'Pickled Orange data'
    SUPPORT_COMPRESSED = True
    SUPPORT_SPARSE_DATA = True

    def read(self):
        with self.open(self.filename, 'rb') as f:
            table = pickle.load(f)
            if not isinstance(table, Table):
                raise TypeError("file does not contain a data table")
            else:
                return table

    @classmethod
    def write_file(cls, filename, data):
        with cls.open(filename, 'wb') as f:
            pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)


class BasketReader(FileFormat):
    """Reader for basket (sparse) files"""
    EXTENSIONS = ('.basket', '.bsk')
    DESCRIPTION = 'Basket file'
    SUPPORT_SPARSE_DATA = True

    def read(self):
        def constr_vars(inds):
            if inds:
                return [ContinuousVariable(x.decode("utf-8")) for _, x in
                        sorted((ind, name) for name, ind in inds.items())]

        X, Y, metas, attr_indices, class_indices, meta_indices = \
            _io.sparse_read_float(self.filename.encode(sys.getdefaultencoding()))

        attrs = constr_vars(attr_indices)
        classes = constr_vars(class_indices)
        meta_attrs = constr_vars(meta_indices)
        domain = Domain(attrs, classes, meta_attrs)
        table = Table.from_numpy(
            domain, attrs and X, classes and Y, metas and meta_attrs)
        table.name = path.splitext(path.split(self.filename)[-1])[0]
        return table


class ExcelReader(FileFormat):
    """Reader for excel files"""
    EXTENSIONS = ('.xls', '.xlsx')
    DESCRIPTION = 'Mircosoft Excel spreadsheet'
    SUPPORT_SPARSE_DATA = False

    def __init__(self, filename):
        super().__init__(filename)

        from xlrd import open_workbook
        self.workbook = open_workbook(self.filename)

    @property
    @lru_cache(1)
    def sheets(self):
        return self.workbook.sheet_names()

    def read(self):
        import xlrd
        wb = xlrd.open_workbook(self.filename, on_demand=True)
        if self.sheet:
            ss = wb.sheet_by_name(self.sheet)
        else:
            ss = wb.sheet_by_index(0)
        try:
            first_row = next(i for i in range(ss.nrows) if any(ss.row_values(i)))
            first_col = next(i for i in range(ss.ncols) if ss.cell_value(first_row, i))
            row_len = ss.row_len(first_row)
            cells = filter(any,
                           [[str(ss.cell_value(row, col)) if col < ss.row_len(row) else ''
                             for col in range(first_col, row_len)]
                            for row in range(first_row, ss.nrows)])
            table = self.data_table(cells)
            table.name = path.splitext(path.split(self.filename)[-1])[0]
            if self.sheet:
                table.name = '-'.join((table.name, self.sheet))
        except Exception:
            raise IOError("Couldn't load spreadsheet from " + self.filename)
        return table


class DotReader(FileFormat):
    """Writer for dot (graph) files"""
    EXTENSIONS = ('.dot', '.gv')
    DESCRIPTION = 'Dot graph description'
    SUPPORT_COMPRESSED = True
    SUPPORT_SPARSE_DATA = False

    @classmethod
    def write_graph(cls, filename, graph):
        from sklearn import tree
        tree.export_graphviz(graph, out_file=cls.open(filename, 'wt'))

    @classmethod
    def write(cls, filename, tree):
        if type(tree) == dict:
            tree = tree['tree']
        cls.write_graph(filename, tree)


class UrlReader(FileFormat):
    def __init__(self, filename):
        filename = filename.strip()
        if not urlparse(filename).scheme:
            filename = 'http://' + filename
        super().__init__(filename)

    @staticmethod
    def urlopen(url):
        req = Request(
            url,
            # Avoid 403 error with servers that dislike scrapers
            headers={'User-Agent': 'Mozilla/5.0 (X11; Linux) Gecko/20100101 Firefox/'})
        return urlopen(req, timeout=10)

    def read(self):
        self.filename = self._trim(self._resolve_redirects(self.filename))
        with contextlib.closing(self.urlopen(self.filename)) as response:
            name = self._suggest_filename(response.headers['content-disposition'])
            with NamedTemporaryFile(suffix=name, delete=False) as f:
                f.write(response.read())
                # delete=False is a workaround for https://bugs.python.org/issue14243

            reader = self.get_reader(f.name)
            data = reader.read()
            remove(f.name)
        # Override name set in from_file() to avoid holding the temp prefix
        data.name = path.splitext(name)[0]
        data.origin = self.filename
        return data

    def _resolve_redirects(self, url):
        # Resolve (potential) redirects to a final URL
        with contextlib.closing(self.urlopen(url)) as response:
            return response.url

    @classmethod
    def _trim(cls, url):
        URL_TRIMMERS = (
            cls._trim_googlesheet,
            cls._trim_dropbox,
        )
        for trim in URL_TRIMMERS:
            try:
                url = trim(url)
            except ValueError:
                continue
            else:
                break
        return url

    @staticmethod
    def _trim_googlesheet(url):
        match = re.match(r'(?:https?://)?(?:www\.)?'
                         r'docs\.google\.com/spreadsheets/d/'
                         r'(?P<workbook_id>[-\w_]+)'
                         r'(?:/.*?gid=(?P<sheet_id>\d+).*|.*)?',
                         url, re.IGNORECASE)
        try:
            workbook, sheet = match.group('workbook_id'), match.group('sheet_id')
            if not workbook:
                raise ValueError
        except (AttributeError, ValueError):
            raise ValueError
        url = 'https://docs.google.com/spreadsheets/d/{}/export?format=tsv'.format(workbook)
        if sheet:
            url += '&gid=' + sheet
        return url

    @staticmethod
    def _trim_dropbox(url):
        parts = urlsplit(url)
        if not parts.netloc.endswith('dropbox.com'):
            raise ValueError
        return urlunsplit(parts._replace(query='dl=1'))

    def _suggest_filename(self, content_disposition):
        default_name = re.sub(r'[\\:/]', '_', urlparse(self.filename).path)

        # See https://tools.ietf.org/html/rfc6266#section-4.1
        matches = re.findall(r"filename\*?=(?:\"|.{0,10}?'[^']*')([^\"]+)",
                             content_disposition or '')
        return urlunquote(matches[-1]) if matches else default_name

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user