Identify files that might contain data

    See function IP_verified() for details about optinoal parmeters
503273
def identify_datafiles(root,
                       extensions_to_ignore=None,
                       directories_to_ignore=None,
                       files_to_ignore=None):
    

    for dirpath, dirnames, filenames in walk(root):

        for ignore in directories_to_ignore:
            if ignore in dirnames:
                dirnames.remove(ignore)  # don't visit ignored directories


        for filename in filenames:


            # Ignore extensions that need no IP check
            ignore = False
            for ext in extensions_to_ignore:
                if filename.endswith(ext):
                    ignore = True

            if filename in files_to_ignore:
                ignore = True

            if ignore is False:
                yield dirpath, filename
1108997
def is_private_addr(x):
    """"""
    paddrs = ['10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16']
    found = False
    for ipr in paddrs:
        try:
            if ipaddress.ip_address(x) in ipaddress.ip_network(ipr):
                found = True
                continue
        except:
            break
    return found
1345414
def list_files(self, secure_data_path, limit=None, offset=None):
        """"""

        # Make sure that limit and offset are in range.
        # Set the normal defaults
        if not limit or limit <= 0:
            limit = 100

        if not offset or offset < 0:
            offset = 0

        payload = {'limit': str(limit), 'offset': str(offset)}

        # Because of the addition of versionId and the way URLs are constructed, secure_data_path should
        #  always end in a '/'.
        secure_data_path = self._add_slash(secure_data_path)
        secret_resp = get_with_retry(self.cerberus_url + '/v1/secure-files/' + secure_data_path,
                                   params=payload, headers=self.HEADERS)
        throw_if_bad_response(secret_resp)
        return secret_resp.json()
1209603
def ip_check(*args, func=None):
    """"""
    func = func or inspect.stack()[2][3]
    for var in args:
        if not isinstance(var, ipaddress._IPAddressBase):
            name = type(var).__name__
            raise IPError(
                f'Function {func} expected IP address, {name} got instead.')
1096668
def do_verify(marfile, keyfiles=None):
    """"""
    try:
        with open(marfile, 'rb') as f:
            with MarReader(f) as m:
                # Check various parts of the mar file
                # e.g. signature algorithms and additional block sections
                errors = m.get_errors()
                if errors:
                    print("File is not well formed: {}".format(errors))
                    sys.exit(1)

                if keyfiles:
                    try:
                        keys = get_keys(keyfiles, m.signature_type)
                    except ValueError as e:
                        print(e)
                        sys.exit(1)

                    if any(m.verify(key) for key in keys):
                        print("Verification OK")
                        return True
                    else:
                        print("Verification failed")
                        sys.exit(1)
                else:
                    print("Verification OK")
                    return True

    except Exception as e:
        print("Error opening or parsing file: {}".format(e))
        sys.exit(1)
36610
def _show_psrs():
    
    sep = os.linesep

    types = "Supported types: " + ", ".join(API.list_types())
    cids = "IDs: " + ", ".join(c for c, _ps in API.list_by_cid())

    x_vs_ps = ["  %s: %s" % (x, ", ".join(p.cid() for p in ps))
               for x, ps in API.list_by_extension()]
    exts = "File extensions:" + sep + sep.join(x_vs_ps)

    _exit_with_output(sep.join([types, exts, cids]))
290391
def verify(path):
        """"""
        valid = False
        try:
            tf = SingleTifPhasics._get_tif(path)
        except (ValueError, IsADirectoryError):
            pass
        else:
            if (len(tf) == 3 and
                "61243" in tf.pages[0].tags and
                "61242" in tf.pages[0].tags and
                "61238" in tf.pages[0].tags and
                "61243" in tf.pages[1].tags and
                "61242" in tf.pages[1].tags and
                "max_sample_value" in tf.pages[0].tags and
                (tf.pages[0].tags["61242"].value !=
                 tf.pages[1].tags["61242"].value)):
                valid = True
        return valid
1293449
def read_ip_ranges(filename, local_file = True, ip_only = False, conditions = []):
    
    targets = []
    data = load_data(filename, local_file = local_file)
    if 'source' in data:
        # Filtered IP ranges
        conditions = data['conditions']
        local_file = data['local_file'] if 'local_file' in data else False
        data = load_data(data['source'], local_file = local_file, key_name = 'prefixes')
    else:
        # Plain IP ranges
        data = data['prefixes']
    for d in data:
        condition_passed = True
        for condition in conditions:
            if type(condition) != list or len(condition) < 3:
                continue
            condition_passed = pass_condition(d[condition[0]], condition[1], condition[2])
            if not condition_passed:
                break
        if condition_passed:
            targets.append(d)
    if ip_only:
        ips = []
        for t in targets:
            ips.append(t['ip_prefix'])
        return ips
    else:
        return targets
697577
public static function ipCidrCheck($p_ip, $p_cidr)
    {
        if (strpos($p_cidr, '/') !== false) {
            list ($net, $mask) = split("/", $p_cidr);
            $ip_net            = ip2long($net);
            $ip_mask           = ~((1 << (32 - $mask)) - 1);
            $ip_ip             = ip2long($p_ip);
            $ip_ip_net         = $ip_ip & $ip_mask;
            
            return ($ip_ip_net == $ip_net);
        } else {
            return (ip2long($p_ip) == ip2long($p_cidr));
        }
    }
499016
def check_data(self):
        """"""
        assert os.path.exists(self.data_fp)
        if gis:
            with fiona.drivers():
                with fiona.open(self.faces_fp) as src:
                    assert src.meta

        gpkg_hash = json.load(open(self.data_fp))['metadata']['sha256']
        assert gpkg_hash == sha256(self.faces_fp)
463015
def verify(path):
        """"""
        valid = False
        try:
            zf = zipfile.ZipFile(path)
        except (zipfile.BadZipfile, IsADirectoryError):
            pass
        else:
            names = sorted(zf.namelist())
            names = [nn for nn in names if nn.endswith(".tif")]
            names = [nn for nn in names if nn.startswith("SID PHA")]
            for name in names:
                with zf.open(name) as pt:
                    fd = io.BytesIO(pt.read())
                    if SingleTifPhasics.verify(fd):
                        valid = True
                        break
            zf.close()
        return valid
969649
def verify(self, data, signature=None, keyrings=None, homedir=None):
        '''
        
        '''

        if isinstance(data, six.text_type):
            data = data.encode('utf-8')

        tmpdir = tempfile.mkdtemp()
        data_file, data_path = tempfile.mkstemp(dir=tmpdir)
        data_file = os.fdopen(data_file, 'wb')
        data_file.write(data)
        data_file.close()
        if signature:
            sig_file, sig_path = tempfile.mkstemp(dir=tmpdir)
            sig_file = os.fdopen(sig_file, 'wb')
            sig_file.write(signature)
            sig_file.close()
        else:
            sig_path = None
        try:
            return self.verify_from_file(
                data_path,
                sig_path=sig_path,
                keyrings=keyrings,
                homedir=homedir
            )
        finally:
            shutil.rmtree(tmpdir)
544569
def verify_log(opts):
    '''
    
    '''
    level = LOG_LEVELS.get(str(opts.get('log_level')).lower(), logging.NOTSET)

    if level < logging.INFO:
        log.warning('Insecure logging configuration detected! Sensitive data may be logged.')
403380
public function verify()
    {
        $phpversion = phpversion();
        $extensions = get_loaded_extensions();
        $ini        = array_only(ini_get_all(), ['max_execution_time', 'max_file_uploads', 'max_input_time', 'post_max_size', 'upload_max_filesize']);

        $return              = [
            'version'    => $phpversion,
            'transports' => implode(', ', stream_get_transports()),
            'filters'    => implode(', ', stream_get_filters()),
            'doc_root'   => getcwd(),
            'extensions' => $extensions,
            'ini'        => $ini,
        ];
        $this->configuration = $return;
        $this->validate($return);
        return $return;
    }
360896
public function detectEnvName()
    {
        // Detect from local env file
        if ($this->envFile && is_file($this->envFile)) {
            $this->name = require $this->envFile;
            if ($this->name) {
                return;
            }
        }

        // Return if env name is detected, or, or continue detecting by IP
        if ($this->detector && $this->name = call_user_func($this->detector)) {
            return;
        }

        // Executes in web server, like Apache, Nginx
        if (isset($this->server['SERVER_ADDR'])) {
            $ip = $this->server['SERVER_ADDR'];
            if (isset($this->ipMap[$ip])) {
                $this->name = $this->ipMap[$ip];
            } else {
                $this->name = 'prod';
            }
            return;
        }

        // Executes in CLI
        if (php_sapi_name() == 'cli' && $ips = $this->getServerIps()) {
            foreach ($ips as $ip) {
                if (isset($this->ipMap[$ip])) {
                    $this->name = $this->ipMap[$ip];
                    return;
                }
            }
        }

        $this->name = 'prod';
        return;
    }
675666
private function checkVisitorDataIsReliable()
    {
        $data = $this->getVisitorData();

        foreach (['user_id', 'client_ip', 'user_agent'] as $key) {
            if ($this->checkDataIsUnreliable($data, $key)) return false;
        }

        return true;
    }
282023
def _identify_all(header, footer, ext=None):
    

    # Capture the length of the data
    # That way we do not try to identify bytes that don't exist
    matches = list()
    for magic_row in magic_header_array:
        start = magic_row.offset
        end = magic_row.offset + len(magic_row.byte_match)
        if end > len(header):
            continue
        if header[start:end] == magic_row.byte_match:
            matches.append(magic_row)

    for magic_row in magic_footer_array:
        start = magic_row.offset
        if footer[start:] == magic_row.byte_match:
            matches.append(magic_row)
    if not matches:
        raise PureError("Could not identify file")

    return _confidence(matches, ext)