set a encoding header as suggested in PEP-0263. This
    is not entirely correct because we don't know the encoding of the
    given file but it's at least a chance to get metadata from the setup.py
779831
def _set_file_encoding_utf8(filename):
    """"""
    with open(filename, 'r+') as f:
        content = f.read()
        f.seek(0, 0)
        f.write("# -*- coding: utf-8 -*-\n" + content)
247574
public void setMetadata(String moduleKeyWithBranch, final DefaultInputFile inputFile, Charset defaultEncoding) {
    CharsetDetector charsetDetector = new CharsetDetector(inputFile.path(), defaultEncoding);
    try {
      Charset charset;
      if (charsetDetector.run()) {
        charset = charsetDetector.charset();
      } else {
        LOG.debug("Failed to detect a valid charset for file '{}'. Using default charset.", inputFile);
        charset = defaultEncoding;
      }
      InputStream is = charsetDetector.inputStream();
      inputFile.setCharset(charset);
      Metadata metadata = fileMetadata.readMetadata(is, charset, inputFile.absolutePath(), exclusionsScanner.createCharHandlerFor(inputFile));
      inputFile.setMetadata(metadata);
      inputFile.setStatus(statusDetection.status(moduleKeyWithBranch, inputFile, metadata.hash()));
      LOG.debug("'{}' generated metadata{} with charset '{}'", inputFile, inputFile.type() == Type.TEST ? " as test " : "", charset);
    } catch (Exception e) {
      throw new IllegalStateException(e);
    }
  }
718347
@Override
	protected void generateFileHeader(QualifiedName qualifiedName, ExtraLanguageAppendable appendable,
			IExtraLanguageGeneratorContext context) {
		appendable.append(PYTHON_FILE_HEADER);
		appendable.newLine();
		appendable.append(getGenerationComment(context));
		appendable.newLine().newLine();
	}
949510
def get_encoding_from_headers(headers):
    

    content_type = headers.get('content-type')

    if not content_type:
        return None

    content_type, params = _parse_content_type_header(content_type)

    if 'charset' in params:
        return params['charset'].strip("'\"")

    if 'text' in content_type:
        return 'ISO-8859-1'
137022
def _detect_encoding(data=None):
    
    import locale
    enc_list = ['utf-8', 'latin-1', 'iso8859-1', 'iso8859-2',
                'utf-16', 'cp720']
    code = locale.getpreferredencoding(False)
    if data is None:
        return code
    if code.lower() not in enc_list:
        enc_list.insert(0, code.lower())
    for c in enc_list:
        try:
            for line in data:
                line.decode(c)
        except (UnicodeDecodeError, UnicodeError, AttributeError):
            continue
        return c
    print("Encoding not detected. Please pass encoding value manually")
1261432
def get_heading_encoding(response):
    '''
    '''
    encoding = wpull.protocol.http.util.parse_charset(
        response.fields.get('content-type', ''))

    if encoding:
        return wpull.string.normalize_codec_name(encoding)
    else:
        return None
1286754
def determine_encoding(buf):
    
    # The ultimate default is utf8; bravo, W3C
    bom_encoding = 'UTF-8'

    if not buf:
        # What
        return bom_encoding

    if isinstance(buf, six.text_type):
        # We got a file that, for whatever reason, produces already-decoded
        # text.  Check for the BOM (which is useless now) and believe
        # whatever's in the @charset.
        if buf[0] == '\ufeff':
            buf = buf[0:]

        # This is pretty similar to the code below, but without any encoding
        # double-checking.
        charset_start = '@charset "'
        charset_end = '";'
        if buf.startswith(charset_start):
            start = len(charset_start)
            end = buf.index(charset_end, start)
            return buf[start:end]
        else:
            return bom_encoding

    # BOMs
    if buf[:3] == b'\xef\xbb\xbf':
        bom_encoding = 'UTF-8'
        buf = buf[3:]
    if buf[:4] == b'\x00\x00\xfe\xff':
        bom_encoding = 'UTF-32BE'
        buf = buf[4:]
    elif buf[:4] == b'\xff\xfe\x00\x00':
        bom_encoding = 'UTF-32LE'
        buf = buf[4:]
    if buf[:4] == b'\x00\x00\xff\xfe':
        raise UnicodeError("UTF-32-2143 is not supported")
    elif buf[:4] == b'\xfe\xff\x00\x00':
        raise UnicodeError("UTF-32-2143 is not supported")
    elif buf[:2] == b'\xfe\xff':
        bom_encoding = 'UTF-16BE'
        buf = buf[2:]
    elif buf[:2] == b'\xff\xfe':
        bom_encoding = 'UTF-16LE'
        buf = buf[2:]

    # The spec requires exactly this syntax; no escapes or extra spaces or
    # other shenanigans, thank goodness.
    charset_start = '@charset "'.encode(bom_encoding)
    charset_end = '";'.encode(bom_encoding)
    if buf.startswith(charset_start):
        start = len(charset_start)
        end = buf.index(charset_end, start)
        encoded_encoding = buf[start:end]
        encoding = encoded_encoding.decode(bom_encoding)

        # Ensure that decoding with the specified encoding actually produces
        # the same @charset rule
        encoded_charset = buf[:end + len(charset_end)]
        if (encoded_charset.decode(encoding) !=
                encoded_charset.decode(bom_encoding)):
            raise UnicodeError(
                "@charset {0} is incompatible with detected encoding {1}"
                .format(bom_encoding, encoding))
    else:
        # With no @charset, believe the BOM
        encoding = bom_encoding

    return encoding
984561
def parse_source_file(filename):
    

    # can't use codecs.open(filename, 'r', 'utf-8') here b/c ast doesn't
    # work with unicode strings in Python2.7 "SyntaxError: encoding
    # declaration in Unicode string" In python 2.7 the string can't be
    # encoded and have information about its encoding. That is particularly
    # problematic since source files include in their header information
    # about the file encoding.
    # Minimal example to fail: ast.parse(u'# -*- coding: utf-8 -*-')

    with open(filename, 'rb') as fid:
        content = fid.read()
    # change from Windows format to UNIX for uniformity
    content = content.replace(b'\r\n', b'\n')

    try:
        node = ast.parse(content)
        return node, content.decode('utf-8')
    except SyntaxError:
        return None, content.decode('utf-8')
644408
def __getfilesystemencoding():
    '''
    
    '''
    try:
        ret = sys.getfilesystemencoding()
        if not ret:
            raise RuntimeError('Unable to get encoding.')
        return ret
    except:
        try:
            #Handle Jython
            from java.lang import System  # @UnresolvedImport
            env = System.getProperty("os.name").lower()
            if env.find('win') != -1:
                return 'ISO-8859-1'  #mbcs does not work on Jython, so, use a (hopefully) suitable replacement
            return 'utf-8'
        except:
            pass

        #Only available from 2.3 onwards.
        if sys.platform == 'win32':
            return 'mbcs'
        return 'utf-8'
1280412
public static void setContentTypeHeader(HttpResponse response, File file) {
		String mimeType = MimeTypes.getMimeTypeForFileName(file.getName());
		String mimeFinal = mimeType != null ? mimeType : MimeTypes.getDefaultMimeType();
		response.headers().set(CONTENT_TYPE, mimeFinal);
	}
548566
def to_utf8(path, output_path=None):
    
    if output_path is None:
        basename, ext = os.path.splitext(path)
        output_path = basename + "-UTF8Encode" + ext
    
    text = smartread(path)
    write(text, output_path)
142950
def process_config(config_path, py3_wrapper=None):
    

    def notify_user(error):
        if py3_wrapper:
            py3_wrapper.notify_user(error)
        else:
            print(error)

    def parse_config(config):
        """
        Parse text or file as a py3status config file.
        """

        if hasattr(config, "readlines"):
            config = "".join(config.readlines())
        parser = ConfigParser(config, py3_wrapper)
        parser.parse()
        parsed = parser.config
        del parser
        return parsed

    def parse_config_error(e, config_path):
        # There was a problem use our special error config
        error = e.one_line(config_path)
        notify_user(error)
        # to display correctly in i3bar we need to do some substitutions
        for char in ['"', "{", "|"]:
            error = error.replace(char, "\\" + char)
        error_config = Template(ERROR_CONFIG).substitute(error=error)
        return parse_config(error_config)

    config = {}

    # get the file encoding this is important with multi-byte unicode chars
    try:
        encoding = check_output(
            ["file", "-b", "--mime-encoding", "--dereference", config_path]
        )
        encoding = encoding.strip().decode("utf-8")
    except CalledProcessError:
        # bsd does not have the --mime-encoding so assume utf-8
        encoding = "utf-8"
    try:
        with codecs.open(config_path, "r", encoding) as f:
            try:
                config_info = parse_config(f)
            except ParseException as e:
                config_info = parse_config_error(e, config_path)
    except LookupError:
        with codecs.open(config_path) as f:
            try:
                config_info = parse_config(f)
            except ParseException as e:
                config_info = parse_config_error(e, config_path)

    # update general section with defaults
    general_defaults = GENERAL_DEFAULTS.copy()
    if "general" in config_info:
        general_defaults.update(config_info["general"])
    config["general"] = general_defaults

    config["py3status"] = config_info.get("py3status", {})
    modules = {}
    on_click = {}
    i3s_modules = []
    py3_modules = []
    module_groups = {}

    def process_onclick(key, value, group_name):
        """
        Check on_click events are valid.  Store if they are good
        """
        button_error = False
        button = ""
        try:
            button = key.split()[1]
            if int(button) not in range(1, 20):
                button_error = True
        except (ValueError, IndexError):
            button_error = True

        if button_error:
            err = "Invalid on_click for `{}`. Number not in range 1-20: `{}`."
            notify_user(err.format(group_name, button))
            return False
        clicks = on_click.setdefault(group_name, {})
        clicks[button] = value
        return True

    def get_module_type(name):
        """
        i3status or py3status?
        """
        if name.split()[0] in I3S_MODULE_NAMES:
            return "i3status"
        return "py3status"

    def process_module(name, module, parent):
        if parent:
            modules[parent]["items"].append(name)
            mg = module_groups.setdefault(name, [])
            mg.append(parent)
            if get_module_type(name) == "py3status":
                module[".group"] = parent

        # check module content
        for k, v in list(module.items()):
            if k.startswith("on_click"):
                # on_click event
                process_onclick(k, v, name)
                # on_click should not be passed to the module via the config.
                del module[k]
            if isinstance(v, ModuleDefinition):
                # we are a container
                module["items"] = []
        return module

    def get_modules(data, parent=None):
        for k, v in data.items():
            if isinstance(v, ModuleDefinition):
                module = process_module(k, v, parent)
                modules[k] = module
                get_modules(v, parent=k)

    get_modules(config_info)

    config["order"] = []

    def remove_any_contained_modules(module):
        """
        takes a module definition and returns a dict without any modules that
        may be defined with it.
        """
        fixed = {}
        for k, v in module.items():
            if not isinstance(v, ModuleDefinition):
                fixed[k] = v
        return fixed

    def append_modules(item):
        module_type = get_module_type(item)
        if module_type == "i3status":
            if item not in i3s_modules:
                i3s_modules.append(item)
        else:
            if item not in py3_modules:
                py3_modules.append(item)

    def add_container_items(module_name):
        module = modules.get(module_name, {})
        items = module.get("items", [])
        for item in items:
            if item in config:
                continue

            append_modules(item)
            module = modules.get(item, {})
            config[item] = remove_any_contained_modules(module)
            # add any children
            add_container_items(item)

    # create config for modules in order
    for name in config_info.get("order", []):
        if name in module_groups:
            msg = "Module `{}` should not be listed in the 'order' directive, use"
            msg += " its parent group instead."
            notify_user(msg.format(name))
            continue
        module_name = name.split(" ")[0]
        if module_name in RETIRED_MODULES:
            notify_user(
                "Module `{}` is no longer available".format(module_name)
                + ". Alternative modules are: {}.".format(
                    ", ".join("`{}`".format(x) for x in RETIRED_MODULES[module_name])
                )
            )
            continue
        module = modules.get(name, {})
        config["order"].append(name)
        add_container_items(name)
        append_modules(name)

        config[name] = remove_any_contained_modules(module)

    config["on_click"] = on_click
    config["i3s_modules"] = i3s_modules
    config["py3_modules"] = py3_modules
    config[".module_groups"] = module_groups

    # time and tztime modules need a format for correct processing
    for name in config:
        if name.split()[0] in TIME_MODULES and "format" not in config[name]:
            if name.split()[0] == "time":
                config[name]["format"] = TIME_FORMAT
            else:
                config[name]["format"] = TZTIME_FORMAT

    if not config["order"]:
        notify_user(
            "Your configuration file does not list any module"
            ' to be loaded with the "order" directive.'
        )
    return config
592307
def open_with_encoding(filename, encoding, mode='r'):
    """"""
    return io.open(filename, mode=mode, encoding=encoding,
                   newline='')
406588
def process_module(self, node):
        '''
        
        '''
        pep263 = re.compile(six.b(self.RE_PEP263))

        try:
            file_stream = node.file_stream
        except AttributeError:
            # Pylint >= 1.8.1
            file_stream = node.stream()

        # Store a reference to the node's file stream position
        current_stream_position = file_stream.tell()

        # Go to the start of stream to achieve our logic
        file_stream.seek(0)

        # Grab the first two lines
        twolines = list(itertools.islice(file_stream, 2))
        pep263_encoding = [m.group(1).lower() for l in twolines for m in [pep263.search(l)] if m]

        multiple_encodings = len(pep263_encoding) > 1
        file_empty = len(twolines) == 0

        # Reset the node's file stream position
        file_stream.seek(current_stream_position)

        # - If the file has an UTF-8 BOM and yet uses any other
        #   encoding, it will be caught by F0002
        # - If the file has a PEP263 UTF-8 encoding and yet uses any
        #   other encoding, it will be caught by W0512
        # - If there are non-ASCII characters and no PEP263, or UTF-8
        #   BOM, it will be caught by W0512
        # - If there are ambiguous PEP263 encodings it will be caught
        #   by E0001, we still test for this
        if multiple_encodings:
            self.add_message('W9901', line=1)

        if node.file_encoding:
            pylint_encoding = node.file_encoding.lower()
            if six.PY3:
                pylint_encoding = pylint_encoding.encode('utf-8')
            if pep263_encoding and pylint_encoding not in pep263_encoding:
                self.add_message('W9902', line=1)
        if not pep263_encoding:
            if file_empty:
                self.add_message('W9905', line=1)
            else:
                self.add_message('W9903', line=1)
        elif self.REQ_ENCOD not in pep263_encoding:
            self.add_message('W9904', line=1)
455137
def _read_metadata(self):
        

        filetype = self.fp.readline().strip()
        filetitle = self.fp.readline().strip()
        # Decode to UTF string, if possible
        try:
            filetype = str(filetype, 'utf-8')
            filetitle = str(filetitle, 'utf-8')
        except:
            # TODO: Handle this edge-case of converting file metadata more elegantly.
            pass

        self.__setattr__('filetype', filetype)
        self.__setattr__('filetitle', filetitle)
944377
def _set_details(self, content):
        
        try:
            self.details = str(content)
        except UnicodeEncodeError:
            if sys.version_info < (3, 0):
                # If Py2 threw encode error, convert to unicode.
                self.details = unicode(content)
            else:
                # We should never hit this in Py3, if this happens, record
                # an encoded version of the content for users to handle.
                logging.error(
                    'Unable to decode "%s" in Py3, encoding in utf-8.',
                    content)
                self.details = content.encode('utf-8')
1147689
def read_zipfile(self, encoding='utf8'):
        
        from zipfile import ZipFile
        with ZipFile(self.abspath) as zipped:
            for num, zip_name in enumerate(zipped.namelist()):
                return zipped.open(zip_name).read().decode(encoding)