Crawl the given directory and build the tree object model for further use
16720
function(treeRoot) {
      var stat = fs.statSync(treeRoot);
      if (stat.isDirectory()) {
        return buildTree(treeRoot);
      } else {
        throw new Error("path: " + treeRoot + " is not a directory");
      }
    }
969141
def build_tree(self, *args, **kwargs):
        """"""
        # Check length #
        assert len(self) > 3
        # Default option #
        algorithm = kwargs.pop(kwargs, None)
        if algorithm is None: algorithm = 'raxml'
        # Dispatch #
        if algorithm is 'raxml':    return self.build_tree_raxml(*args, **kwargs)
        if algorithm is 'fasttree': return self.build_tree_fast(*args, **kwargs)
830872
def dict_crawl(entry, key):
    ''' 
    '''
    value = None
    key_list = make_list(key)
    try:
        result = None
        success_flag = True
        temp = entry
        for next_key in key_list:
            parent = temp
            actual_type = detect_type(temp)
            if actual_type=="mongoengine":
                if next_key in temp.__dict__['_data']:
                    temp = temp.__dict__['_data'][next_key]
                else:
                    break
            elif actual_type=="class":
                if next_key in temp.__dict__:
                    temp = temp.__dict__[next_key]
                else:
                    break
            elif actual_type=="dict":
                if next_key in temp:
                    temp = temp[next_key]
                else:
                    break
            elif actual_type=="iterable_dict":
                if next_key in convert_to_dict(temp): # 'contains' method might not be there
                    temp = temp[next_key]
                else:
                    break
            elif actual_type=="list":
                if next_key in temp:
                    temp = temp[next_key]
                else:
                    break
            else:  ## I'm unsure we can really 'crawl' an 'object'
                break
        else:
            return (parent, key_list[-1], temp)
        return (None, None, None)
    except:
        pass
    return (None, None, None)
210894
def list_to_tree(cls, files):
        """"""

        def attach(branch, trunk):
            """Insert a branch of directories on its trunk."""
            parts = branch.split('/', 1)
            if len(parts) == 1:  # branch is a file
                trunk[FILE_MARKER].append(parts[0])
            else:
                node, others = parts
                if node not in trunk:
                    trunk[node] = defaultdict(dict, ((FILE_MARKER, []), ))
                attach(others, trunk[node])

        tree = defaultdict(dict, ((FILE_MARKER, []), ))
        for line in files:
            attach(line, tree)
        return tree
558383
public function crawl()
    {
        $this->getQueueManager()->addUri($this->seed);
        $this->getDownloader()->getPersistenceHandler()->setSpiderId($this->spiderId);

        $this->doCrawl();
    }
192867
function buildTree(directory, barrelName, logger) {
    logger(`Building directory tree for ${utilities_1.convertPathSeparator(directory)}`);
    const names = fs_1.default.readdirSync(directory);
    const result = {
        directories: [],
        files: [],
        name: path_1.default.basename(directory),
        path: utilities_1.convertPathSeparator(directory)
    };
    names.forEach((name) => {
        const fullPath = path_1.default.join(directory, name);
        if (fs_1.default.statSync(fullPath).isDirectory()) {
            result.directories.push(buildTree(fullPath, barrelName, logger));
        }
        else {
            const convertedPath = utilities_1.convertPathSeparator(fullPath);
            const file = {
                name,
                path: convertedPath
            };
            result.files.push(file);
            if (file.name === barrelName) {
                logger(`Found existing barrel @ ${convertedPath}`);
                result.barrel = file;
            }
        }
    });
    return result;
}
685121
def _spider(self, subfolder, depth):
        '''
            
        '''

        # The following is some funky shit that deals with the way impacket treats file paths

        if subfolder in ['', '.']:
            subfolder = '*'

        elif subfolder.startswith('*/'):
            subfolder = subfolder[2:] + '/*'
        else:
            subfolder = subfolder.replace('/*/', '/') + '/*'

        # End of the funky shit... or is it? Surprise! This whole thing is funky

        filelist = None
        try:
            filelist = self.smbconnection.listPath(self.share, subfolder)
            self.dir_list(filelist, subfolder)
            if depth == 0:
                return
        except SessionError as e:
            if not filelist:
                if 'STATUS_ACCESS_DENIED' not in str(e):
                    logging.debug("Failed listing files on share {} in directory {}: {}".format(self.share, subfolder, e))
                return

        for result in filelist:
            if result.is_directory() and result.get_longname() not in ['.','..']:
                if subfolder == '*':
                    self._spider(subfolder.replace('*', '') + result.get_longname(), depth-1 if depth else None)
                elif subfolder != '*' and (subfolder[:-2].split('/')[-1] not in self.exclude_dirs):
                    self._spider(subfolder.replace('*', '') + result.get_longname(), depth-1 if depth else None)
        return
737845
def read_dir_tree(self, file_hash):
        

        json_d = self.read_index_object(file_hash, 'tree')
        node = {'files' : json_d['files'], 'dirs' : {}}
        for name, hsh in json_d['dirs'].iteritems(): node['dirs'][name] = self.read_dir_tree(hsh)
        return node
396662
function crawl(type, handler) {
		gui.Object.each(type, istype).forEach(
			function(type) {
				handle(type, handler);
				crawl(type, handler);
			}
		);
	}
452764
def crawl_fig(self, fig):
        """"""
        with self.renderer.draw_figure(fig=fig,
                                       props=utils.get_figure_properties(fig)):
            for ax in fig.axes:
                self.crawl_ax(ax)
499418
function buildTree(index, api) {
  const modules = {};

  const dependency = (module, parent) => {
    const filepath = api.resolve(module, parent);
    if (modules[filepath] != null) {
      return modules[filepath];
    }

    const src = api.getFileContents(module, parent, "utf8");
    const basic = api.parser(src);
    const [nestedImports, hasMemory] = parseImports(basic, api);

    const deps = {};

    Object.keys(nestedImports).forEach(mod => {
      if (mod.indexOf(".") === 0) {
        const dep = dependency(mod, filepath);
        deps[mod] = dep;
      }
    });

    const patched = inferImportTypes(basic, deps, api);
    const ast = api.semantics(patched);

    api.validate(ast, {
      lines: src.split("\n"),
      filename: module.split("/").pop(),
    });

    const result = {
      ast,
      deps,
      filepath,
      hasMemory,
    };
    modules[filepath] = result;

    return result;
  };

  const root = dependency(index, null);

  return {
    root,
    modules,
  };
}
619546
function walkTree(directory, callback) {
    callback(directory);
    directory.directories.forEach(childDirectory => walkTree(childDirectory, callback));
}
507559
private void indexDirectory(String path) throws Exception {
        fr.pilato.elasticsearch.crawler.fs.beans.Path pathObject = new fr.pilato.elasticsearch.crawler.fs.beans.Path();
        // The real and complete path
        pathObject.setReal(path);
        String rootdir = path.substring(0, path.lastIndexOf(File.separator));
        // Encoded version of the parent dir
        pathObject.setRoot(SignTool.sign(rootdir));
        // The virtual URL (not including the initial root dir)
        pathObject.setVirtual(computeVirtualPathName(stats.getRootPath(), path));

        indexDirectory(SignTool.sign(path), pathObject);
    }
251060
def _recursive_search_file_in_directory(self, directory, searched_file):
        
        s = os.path.split(searched_file)

        for root, dirs, files in os.walk(directory):
            for file in files:
                # If filename is the same
                if s[1] == file and (s[0] == '' or s[0] == os.path.basename(root)):
                    path = os.path.normpath(os.path.join(root, s[1]))
                    if os.path.exists(path):
                        return path
        return None
669582
function handleTree(username, repos, branch, tree, download) {
    let filterList = tree.filter(item => {
        return item.type === 'blob';
    })
    if (download !== '') {
        filterList = filterList.filter(item => {
            // create reg
            const downRepl = download.replace(/\//g, '\\\/').replace(/\./g, '\\\.');
            const reg = new RegExp(`^${downRepl}`)
            return reg.test(item.path);
        })
    }
    // request list is ready
    spinner.stop();
    bar = new ProgressBar(':bar :current/:total', {
        total: filterList.length
    });

    filterList.map(item => {
        downloadFile(username, repos, branch, item.path)
    });
}
335521
def _run(self, url, auth):
        '''
        
        '''
        if url in self.visited:
            logger.debug("Skipping %s (already crawled)" % url)
            return
        self.visited.append(url)

        logger.info("Crawling: %s" % url)
        url = self._get_catalog_url(url)

        # Get an etree object
        xml_content = request_xml(url, auth)
        for ds in self._build_catalog(url, xml_content):
            yield ds
704516
def write_dir_tree(self, tree):
        

        dirs  = tree['dirs']; files = tree['files']
        child_dirs = {name : self.write_dir_tree(contents) for name, contents in dirs.iteritems()}
        return self.write_index_object('tree', {'files' : files, 'dirs': child_dirs})