get a HTML formatted view of all Python programs
    in all subfolders of AIKIF, including imports and
    lists of functions and classes
10974
def get_program_list():
    
    colList = ['FileName','FileSize','Functions', 'Imports']

    txt = '<TABLE width=90% border=0>'
    txt += format_file_table_header(colList)
    fl = web.GetFileList(aikif_folder, ['*.py'], 'N')
    for f in fl:
        if '__init__.py' in f:
            txt += '<TR><TD colspan=4><HR><H3>' + get_subfolder(f) + '</h3></td></tr>\n'
        else:
            txt += format_file_to_html_row(f, colList)
    txt += '</TABLE>\n\n'
    return txt
278857
def plugins(all, plugins_dir):
    ""
    app = Datasette([], plugins_dir=plugins_dir)
    click.echo(json.dumps(app.plugins(all), indent=4))
79343
def _parse_programs(self, string, parent, filepath=None):
        """"""
        #First, get hold of the docstrings  for all the modules so that we can
        #attach them as we parse them.
        moddocs = self.docparser.parse_docs(string)
        #Now look for modules in the file and then match them to their decorators.
        matches = self.RE_PROGRAM.finditer(string)
        result = []
        for rmodule in matches:
            name = rmodule.group("name").lower()
            contents = re.sub("&[ ]*\n", "", rmodule.group("contents"))
            module = self._process_module(name, contents, parent, rmodule, filepath)
            #Check whether the docparser found docstrings for the module.
            if name in moddocs:                
                module.docstring = self.docparser.to_doc(moddocs[name][0], name)
                module.docstart, module.docend = module.absolute_charindex(string, moddocs[name][1],
                                                                           moddocs[name][2])
            result.append(module)
        return result
1192287
def index():
    """"""
    kwdb = current_app.kwdb

    libraries = get_collections(kwdb, libtype="library")
    resource_files = get_collections(kwdb, libtype="resource")

    return flask.render_template("libraryNames.html",
                                 data={"libraries": libraries,
                                       "version": __version__,
                                       "resource_files": resource_files
                                   })
292777
def generate_xhtml(path, dirs, files):
    """"""
    # Prepare the path to display.
    if path != '/':
        dirs.insert(0, '..')
    if not path.endswith('/'):
        path += '/'

    def itemize(item):
        return '<a href="%s">%s</a>' % (item, path+item)
    dirs = [d + '/' for d in dirs]
    return """
    <html>
     <body>
      <h1>%s</h1>
       <pre>%s\n%s</pre>
     </body>
    </html>
    """ % (path, '\n'.join(itemize(dir) for dir in dirs), '\n'.join(itemize(file) for file in files))
1340931
private static List<File> findPythonLibDirectories(File parentDirectory) {
        List<File> foundDirectories = new ArrayList<>();
        if (parentDirectory != null && parentDirectory.exists()) {
            for (File dir : parentDirectory.listFiles(DIRECTORY_FILE_FILTER)) {
                if (dir.getName().equals("pythonlib")) {
                    foundDirectories.add(dir);
                } else {
                    foundDirectories.addAll(findPythonLibDirectories(dir));
                }
            }
        }
        return foundDirectories;
    }
434168
def find_all_templates(pattern='*.html', ignore_private=True):
    
    templates = []
    template_loaders = flatten_template_loaders(settings.TEMPLATE_LOADERS)
    for loader_name in template_loaders:
        module, klass = loader_name.rsplit('.', 1)
        if loader_name in (
            'django.template.loaders.app_directories.Loader',
            'django.template.loaders.filesystem.Loader',
        ):
            loader_class = getattr(import_module(module), klass)
            if getattr(loader_class, '_accepts_engine_in_init', False):
                loader = loader_class(Engine.get_default())
            else:
                loader = loader_class()
            for dir in loader.get_template_sources(''):
                for root, dirnames, filenames in os.walk(dir):
                    for basename in filenames:
                        if ignore_private and basename.startswith("_"):
                            continue
                        filename = os.path.join(root, basename)
                        rel_filename = filename[len(dir)+1:]
                        if fnmatch.fnmatch(filename, pattern) or \
                           fnmatch.fnmatch(basename, pattern) or \
                           fnmatch.fnmatch(rel_filename, pattern):
                            templates.append(rel_filename)
        else:
            LOGGER.debug('%s is not supported' % loader_name)
    return sorted(set(templates))
727202
def _get_template_list(self):
        "  "
        t_list = []
        if hasattr(self.obj, 'category_id') and self.obj.category_id:
            cat = self.obj.category
            base_path = 'box/category/%s/content_type/%s/' % (cat.path, self.name)
            if hasattr(self.obj, 'slug'):
                t_list.append(base_path + '%s/%s.html' % (self.obj.slug, self.box_type,))
            t_list.append(base_path + '%s.html' % (self.box_type,))
            t_list.append(base_path + 'box.html')

        base_path = 'box/content_type/%s/' % self.name
        if hasattr(self.obj, 'slug'):
            t_list.append(base_path + '%s/%s.html' % (self.obj.slug, self.box_type,))
        t_list.append(base_path + '%s.html' % (self.box_type,))
        t_list.append(base_path + 'box.html')

        t_list.append('box/%s.html' % self.box_type)
        t_list.append('box/box.html')

        return t_list
1026923
def _get_template_dirs():
    """"""
    return filter(lambda x: os.path.exists(x), [
        # user dir
        os.path.join(os.path.expanduser('~'), '.py2pack', 'templates'),
        # system wide dir
        os.path.join('/', 'usr', 'share', 'py2pack', 'templates'),
        # usually inside the site-packages dir
        os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates'),
    ])
585959
def main():
    
    fldr = mod_cfg.fldrs['program_path']
    p = mod_prg.Programs('AIKIF Programs', fldr)
    document_core_programs(p) 
    document_agents(p) 
    document_examples(p) 
  
    # p.list()	# get list of all programs
    p.save(fldr + os.sep + 'examples' + os.sep + 'document_AIKIF.csv')
    p.collect_program_info('progress.md')
1137416
def _get_loaded_modules():
        """"""
        output = subprocess.check_output(['apache2ctl', '-M'])
        if six.PY3:
            output = output.decode('utf-8')
        modules = []
        for line in output.splitlines():
            # Each line of the enabled module output looks like:
            #  module_name (static|shared)
            # Plus a header line at the top of the output which is stripped
            # out by the regex.
            matcher = re.search(r'^ (\S*)_module (\S*)', line)
            if matcher:
                modules.append(matcher.group(1))
        return modules
366492
def docs(output=DOC_OUTPUT, proj_settings=PROJ_SETTINGS, github=False):
    

    local("export PYTHONPATH='' && "
          "export DJANGO_SETTINGS_MODULE=%s && "
          "sphinx-build -b html %s %s" % (proj_settings, DOC_INPUT, output),
          capture=False)

    if _parse_bool(github):
        local("touch %s/.nojekyll" % output, capture=False)
137770
def main():
    """"""
    if (sys.version_info < (2, 7)):
        sys.exit("This program requires python version 2.7 or later")

    # Options and arguments
    p = optparse.OptionParser(description='IIIF Image API static file generator',
                              usage='usage: %prog [options] file [[file2..]] (-h for help)',
                              version='%prog ' + __version__)

    p.add_option('--dst', '-d', action='store', default='/tmp',
                 help="Destination directory for images [default '%default']")
    p.add_option('--tilesize', '-t', action='store', type='int', default=512,
                 help="Tilesize in pixels [default %default]")
    p.add_option('--api-version', '--api', '-a', action='store', default='2.1',
                 help="API version, may be 1.1, 2.0 or 2.1 [default %default]")
    p.add_option('--prefix', '-p', action='store', default=None,
                 help="URI prefix for where the images will be served from (default '%default'). "
                      "An empty prefix may be OK if the HTML page including the image shares the "
                      "the same root on the same server as the images, otherwise a full URL should "
                      "be specified. This is used to construct the @id in the info.json")
    p.add_option('--identifier', '-i', action='store', default=None,
                 help="Identifier for the image that will be used in place of the filename "
                      "(minus extension). Notes that this option cannot be used if more than "
                      "one image file is to be processed")
    p.add_option('--extra', '-e', action='append', default=[],
                 help="Extra request parameters to be used to generate static files, may be "
                      "repeated (e.g. '/full/90,/0/default.jpg' for a 90 wide thumnail)")
    p.add_option('--write-html', action='store', default=None,
                 help="Write HTML page to the specified directory using the 'identifier.html' "
                      "as the filename. HTML will launch OpenSeadragon for this image and to "
                      "display some of information about info.json and tile locations. HTML will "
                      "assume OpenSeadragon at relative path openseadragonVVV/openseadragon.min.js "
                      "and user-interface icons in openseadragonVVV/images, where VVV are the "
                      "three parts of the version number. The --include-osd flag is also specified "
                      "then OpenSeadragon will be copied to these locations")
    p.add_option('--include-osd', action='store_true',
                 help="Include OpenSeadragon files with --write-html flag")
    p.add_option('--osd-version', action='store', default='2.0.0',
                 help="Generate static images for older versions of OpenSeadragon. Use of versions "
                      "prior to 1.2.1 will force use of /w,h/ for size parameter instead of /w,/. "
                      "Likely useful only in combination with --api-version=1.1 "
                      "[default %default]")
    p.add_option('--osd-width', action='store', type='int', default='500',
                 help="Width of OpenSeadragon pane in pixels. Applies only with "
                      "--write-html [default %default]")
    p.add_option('--osd-height', action='store', type='int', default='500',
                 help="Height of OpenSeadragon pane in pixels. Applies only with "
                      "--write-html [default %default]")
    p.add_option('--generator', action='store_true', default=False,
                 help="Use named generator modules in iiif.generators package instead "
                      "of a starting image [default %default]")
    p.add_option('--max-image-pixels', action='store', type='int', default=0,
                 help="Set the maximum number of pixels in an image. A non-zero value "
                      "will set a hard limit on the image size. If left unset then the "
                      "default configuration of the Python Image Libary (PIL) will give "
                      "a DecompressionBombWarning if the image size exceeds a default "
                      "maximum, but otherwise continue as normal")
    p.add_option('--dryrun', '-n', action='store_true',
                 help="Do not write anything, say what would be done")
    p.add_option('--quiet', '-q', action='store_true',
                 help="Quite (no output unless there is a warning/error)")
    p.add_option('--verbose', '-v', action='store_true',
                 help="Verbose")

    (opt, sources) = p.parse_args()

    level = logging.DEBUG if (opt.verbose) else \
        logging.WARNING if (opt.quiet) else logging.INFO
    logging.basicConfig(format='%(name)s: %(message)s',
                        level=level)
    logger = logging.getLogger(os.path.basename(__file__))

    if (not opt.write_html and opt.include_osd):
        logger.warn(
            "--include-osd has no effect without --write-html, ignoring")
    if (len(sources) == 0):
        logger.warn("No sources specified, nothing to do, bye! (-h for help)")
    elif (len(sources) > 1 and opt.identifier):
        logger.error(
            "Cannot use --identifier/-i option with multiple sources, aborting.")
    else:
        try:
            sg = IIIFStatic(dst=opt.dst, tilesize=opt.tilesize,
                            api_version=opt.api_version, dryrun=opt.dryrun,
                            prefix=opt.prefix, osd_version=opt.osd_version,
                            generator=opt.generator,
                            max_image_pixels=opt.max_image_pixels,
                            extras=opt.extra)
            for source in sources:
                # File or directory (or neither)?
                if (os.path.isfile(source) or opt.generator):
                    logger.info("source file: %s" % (source))
                    sg.generate(source, identifier=opt.identifier)
                    if (opt.write_html):
                        sg.write_html(html_dir=opt.write_html, include_osd=opt.include_osd,
                                      osd_width=opt.osd_width, osd_height=opt.osd_height)
                elif (os.path.isdir(source)):
                    logger.warn(
                        "Ignoring source '%s': directory coversion not supported" % (source))
                else:
                    logger.warn(
                        "Ignoring source '%s': neither file nor path" % (source))
        except (IIIFStaticError, IIIFError) as e:
            # catch known errors and report nicely...
            logger.error("Error: " + str(e))
207027
def generateClassView(self):
        '''
        
        '''
        class_view_stream = StringIO()

        for n in self.namespaces:
            n.toHierarchy(True, 0, class_view_stream)

        # Add everything that was not nested in a namespace.
        missing = []
        # class-like objects (structs and classes)
        for cl in sorted(self.class_like):
            if not cl.in_class_hierarchy:
                missing.append(cl)
        # enums
        for e in sorted(self.enums):
            if not e.in_class_hierarchy:
                missing.append(e)
        # unions
        for u in sorted(self.unions):
            if not u.in_class_hierarchy:
                missing.append(u)

        if len(missing) > 0:
            idx = 0
            last_missing_child = len(missing) - 1
            for m in missing:
                m.toHierarchy(True, 0, class_view_stream, idx == last_missing_child)
                idx += 1
        elif configs.createTreeView:
            # need to restart since there were no missing children found, otherwise the
            # last namespace will not correctly have a lastChild
            class_view_stream.close()
            class_view_stream = StringIO()

            last_nspace_index = len(self.namespaces) - 1
            for idx in range(last_nspace_index + 1):
                nspace = self.namespaces[idx]
                nspace.toHierarchy(True, 0, class_view_stream, idx == last_nspace_index)

        # extract the value from the stream and close it down
        class_view_string = class_view_stream.getvalue()
        class_view_stream.close()
        return class_view_string
72540
def _html_find_urls(bytes, mimetype, base_url=None):
    

    def _recursive_tag_values(tag, values=[]):
        """ This sub-function recursively loops through all of the tags to get all of the attribute values. """

        if hasattr(tag, 'children'):
            for child in tag.children:
                if hasattr(child, 'attrs'):
                    for key in child.attrs:
                        if isinstance(child.attrs[key], list):
                            for value in child.attrs[key]:
                                values.append(value)
                        elif isinstance(child.attrs[key], str):
                            values.append(child.attrs[key])

                    values = _recursive_tag_values(child, values)

        return values

    # Only convert the ASCII bytes to HTML.
    ascii_bytes = b''.join(re.compile(b'[\x00\x09\x0A\x0D\x20-\x7E]{8,}').findall(bytes))
    ascii_bytes = ascii_bytes.replace(b'\x00', b'')

    # Store all of the URLs we find.
    urls = []

    # Convert the bytes into soup. Also try url decoding the bytes to bypass some obfuscation.
    soups = []
    soups.append(BeautifulSoup(ascii_bytes, 'lxml'))
    try:
        soups.append(BeautifulSoup(urllib.parse.unquote(str(ascii_bytes)), 'lxml'))
    except:
        pass

    # Loop over both soups.
    for soup in soups:

        # Find any URLs embedded in script + document.write JavaScript.
        # \r\n\r\ndocument.write(unescape('<meta HTTP-EQUIV="REFRESH" content="0; url=http://somebadsite.com/catalog/index.php">'));\r\n\r\n
        script_urls = []
        script_tags = soup.find_all('script')
        for script_tag in script_tags:
            for tag_content in script_tag.contents:
                if 'document.write' in tag_content.lower():
                    # Find the last position of the ( character, which should denote where the
                    # code that they are writing to the page begins. Also find the position of
                    # the first ) character which should denote where the code ends.
                    code_begin = tag_content.rfind('(')
                    code_end = tag_content.find(')')
                    code = tag_content[code_begin+1:code_end]
                    # Strip off any ' or " quotes.
                    if code.startswith('\'') and code.endswith('\''):
                        code = code[1:-1]
                    if code.startswith('"') and code.endswith('"'):
                        code = code[1:-1]
                    # Turn the string into bytes and feed it back into the _html_find_urls function.
                    code = code.encode()
                    script_urls += _html_find_urls(code, mimetype)

        # Find any URLs embedded in <script> tags. Use the _ascii_find_urls function to try and catch everything.
        for script_tag in script_tags:
            script_urls += _ascii_find_urls(str(script_tag).encode('utf-8'), mimetype)

        # Find any meta-refresh URLs.
        # <meta HTTP-Equiv="refresh" content="0; URL=UntitledNotebook1.html">
        meta_urls = []
        meta_tags = soup.find_all('meta')
        for meta_tag in meta_tags:
            for key in meta_tag.attrs:
                if key.lower() == 'content':
                    value = meta_tag.attrs[key]
                    if 'url=' in value.lower():
                        split_value = value.split('=', 1)
                        url = split_value[1]
                        # Remove any quotes around the URL.
                        if url.startswith('"') and url.endswith('"'):
                            url = url[1:-1]
                        if url.startswith("'") and url.endswith("'"):
                            url = url[1:-1]
                        meta_urls.append(url)

        # Hacky way to find URLs in the CSS.
        css_urls = re.compile(r'url\((.*?)\)').findall(str(soup))
        
        # Make sure none of the CSS URLs are in quotes.
        for u in css_urls[:]:
            if u.startswith('\'') and u.endswith('\''):
                css_urls.remove(u)
                css_urls.append(u[1:-1])
            if u.startswith('\"') and u.endswith('\"'):
                css_urls.remove(u)
                css_urls.append(u[1:-1])

        # Look to see if there is a "base" HTML tag specified. This is the "baseStriker" method.
        for tag in soup.find_all('base'):
            try:
                if tag['href']:
                    base_url = tag['href'].replace('\\\\', '//')
            except:
                pass

        # If we were given a base URL, only extract specific tag values that are likely
        # to be URLs. Otherwise, we would end up with joined URLs for every single tag
        # attribute value that exists in the HTML, which is not the correct behavior.
        if base_url:

            # Join any of the CSS URLs we found.
            for css_url in css_urls:
                urls.append(urljoin(base_url, css_url))

            # Join any of the script URLs we found.
            for script_url in script_urls:
                urls.append(urljoin(base_url, script_url))

            # Join any of the meta-refresh URLs we found.
            for meta_url in meta_urls:
                urls.append(urljoin(base_url, meta_url))

            # Get all of the action URLs.
            for tag in soup.find_all(action=True):
                urls.append(urljoin(base_url, tag['action']))

            # Get all of the background URLs.
            for tag in soup.find_all(background=True):
                urls.append(urljoin(base_url, tag['background']))

            # Get all of the href URLs.
            for tag in soup.find_all(href=True):
                urls.append(urljoin(base_url, tag['href']))

            # Get all of the src URLs.
            for tag in soup.find_all(src=True):
                urls.append(urljoin(base_url, tag['src']))

            # Get all of the xmlns URLs.
            for tag in soup.find_all(xmlns=True):
                urls.append(urljoin(base_url, tag['xmlns']))

        # We weren't given a base URL, so just search for URLs by getting every single
        # tag attribute value. That way we will catch everything regardless of the attribute name.
        else:
            urls = _recursive_tag_values(soup)
            urls += css_urls
            urls += meta_urls
            urls += script_urls

        # As a last-ditch effort, find URLs in the visible text of the HTML. However,
        # we only want to add strings that are valid URLs as they are. What we do not
        # want is to add every string as a potential URL, since if a base_url was given,
        # we will likely end up with a joined URL for every string found in the HTML.
        for s in soup.stripped_strings:
            if is_valid(s):
                urls.append(s)

        # Remove any newlines from the potential URLs. Some HTML has newlines
        # after the href=" part before the actual URL begins. This renders
        # correctly in web browsers but is otherwise considered an invalid
        # URL by the is_valid function.
        urls = [u.strip() for u in urls]

        # Remove any leading //'s from the URLs.
        urls = [u[2:] if u.startswith('//') else u for u in urls]

    return urls
227224
def load_soups(config):
  """"""
  soups = {}
  for page, path in config['sources'].items():
    with open(path, 'r') as orig_file:
      soups[page] = beautiful_soup(orig_file.read(), features='html.parser')
  return soups
197192
def main(directories):
    ''''''
    msg = 'Checking module "{}" from directory "{}" for coding errors.'
    api_checker = ApiChecker()
    resource_checker = ResourceChecker()
    errors = []
    modules = []
    for loader, mname, _ in pkgutil.walk_packages(directories):
        sys.path.append(os.path.abspath(loader.path))
        log.info(msg.format(mname, loader.path))
        modules.append(mname)
        import_module(mname)
    for api in Api:
        if api.__module__.split('.')[-1] not in modules:
            continue
        log.debug('Anlysing Api class: {}'.format(api.__name__))
        errors.extend(api_checker(api))
    for res in Resource:
        if res.__module__.split('.')[-1] not in modules:
            continue
        log.debug('Anlysing Resource class: {}'.format(res.__name__))
        errors.extend(resource_checker(res))
    else:
        log.info('All modules tested, no problem detected.')
    return errors