Rebuild kevent operations by removing open files that no longer need to
        be watched, and adding new files if they are not currently being watched.

        This is done by comparing self.paths to self.paths_open.
765778
def commit(self, **params):
        
        log = self._getparam('log', self._discard, **params)

        #  Find all the modules that no longer need watching
        #
        removed = 0
        added = 0
        for path in list(self.paths_open):
            if path not in self.paths:
                fd = self.paths_open[path]
                if self._mode == WF_KQUEUE:
                    #  kevent automatically deletes the event when the fd is closed
                    try:
                        os.close(fd)
                    except Exception as e:
                        log.warning("close failed on watched file %r -- %s", path, e)
                elif self._mode == WF_INOTIFYX:
                    try:
                        pynotifyx.rm_watch(self._inx_fd, fd)
                    except Exception as e:
                        log.warning("remove failed on watched file %r -- %s", path, e)
                    if path in self._inx_inode:
                        del self._inx_inode[path]
                elif self._mode == WF_POLLING:
                    if fd in self._poll_stat:
                        del self._poll_stat[fd]
                    else:
                        log.warning("fd watched path %r missing from _poll_stat map", path)
                    try:
                        os.close(fd)
                    except Exception as e:
                        log.warning("close failed on watched file %r -- %s", path, e)
                if fd in self.fds_open:
                    del self.fds_open[fd]
                else:
                    log.warning("fd watched path %r missing from fd map", path)
                del self.paths_open[path]
                log.debug("Removed watch for path %r", path)
                removed += 1

        #  Find all the paths that are new and should be watched
        #
        fdlist = []
        failed = []
        last_exc = None
        log.debug("%d watched path%s", len(self.paths), ses(len(self.paths)))
        for path in list(self.paths):
            if path not in self.paths_open:
                try:
                    if not self._add_file(path, **params):
                        continue
                except Exception as e:
                    last_exc = e
                    failed.append(path)
                    continue
                fdlist.append(self.paths_open[path])

                if path in self.paths_pending:
                    log.debug("pending path %r has now appeared", path)
                    del self.paths_pending[path]
                    self._trigger(self.paths_open[path], **params)

                added += 1
                log.debug("Added watch for path %r with ident %d", path, self.paths_open[path])
        if failed:
            self._clean_failed_fds(fdlist)
            raise Exception("Failed to set watch on %s -- %s" % (str(failed), str(last_exc)))
        log.debug("%d added, %d removed", added, removed)
438592
def _rebuild_entries(self):
        
        self.entries = []
        def collapse_entries(group):
            for entry in group.entries:
                self.entries.append(entry)
            for subgroup in group.children:
                collapse_entries(subgroup)
                
        collapse_entries(self.root)
437790
def _rebuild_blknos_and_blklocs(self):
        
        new_blknos = np.empty(self.shape[0], dtype=np.int64)
        new_blklocs = np.empty(self.shape[0], dtype=np.int64)
        new_blknos.fill(-1)
        new_blklocs.fill(-1)

        for blkno, blk in enumerate(self.blocks):
            rl = blk.mgr_locs
            new_blknos[rl.indexer] = blkno
            new_blklocs[rl.indexer] = np.arange(len(rl))

        if (new_blknos == -1).any():
            raise AssertionError("Gaps in blk ref_locs")

        self._blknos = new_blknos
        self._blklocs = new_blklocs
396653
def rescan(self):
        
        if not self.attached:
            return
        dprint(5, "starting rescan")
        newstuff = {};  # this accumulates names of new or changed files. Keys are paths, values are 'quiet' flag.
        # store timestamp of scan
        self.last_scan_timestamp = time.time()
        # go through watched files/directories, check for mtime changes
        for path, watcher in list(self.watchers.items()):
            # get list of new files from watcher
            newfiles = watcher.newFiles()
            # None indicates access error, so drop it from watcher set
            if newfiles is None:
                if watcher.survive_deletion:
                    dprintf(5, "access error on %s, but will still be watched\n", watcher.path)
                else:
                    dprintf(2, "access error on %s, will no longer be watched\n", watcher.path)
                    del self.watchers[path]
                if not watcher.disappeared:
                    self.emit(SIGNAL("disappearedFile"), path)
                    watcher.disappeared = True
                continue
            dprintf(5, "%s: %d new file(s)\n", watcher.path, len(newfiles))
            # if a file has its own watcher, and is independently reported by a directory watcher, skip the directory's
            # version and let the file's watcher report it. Reason for this is that the file watcher may have a more
            # up-to-date timestamp, so we trust it over the dir watcher.
            newfiles = [p for p in newfiles if p is path or p not in self.watchers]
            # skip files in self._unwatched_paths
            newfiles = [filename for filename in newfiles if
                        self._watching_state.get(os.path.dirname(filename)) > Purr.UNWATCHED]
            # Now go through files and add them to the newstuff dict
            for newfile in newfiles:
                # if quiet flag is explicitly set on watcher, enforce it
                # if not pouncing on directory, also add quietly
                if watcher.quiet or self._watching_state.get(os.path.dirname(newfile)) < Purr.POUNCE:
                    quiet = True
                # else add quietly if file is not in the quiet patterns
                else:
                    quiet = matches_patterns(os.path.basename(newfile), self._quiet_patterns)
                # add file to list of new products. Since a file may be reported by multiple
                # watchers, make the quiet flag a logical AND of all the quiet flags (i.e. DP will be
                # marked as quiet only if all watchers report it as quiet).
                newstuff[newfile] = quiet and newstuff.get(newfile, True)
                dprintf(4, "%s: new data product, quiet=%d (watcher quiet: %s)\n", newfile, quiet, watcher.quiet)
                # add a watcher for this file to the temp_watchers list. this is used below
                # to detect renamed and deleted files
                self.temp_watchers[newfile] = Purrer.WatchedFile(newfile)
        # now, go through temp_watchers to see if any newly pounced-on files have disappeared
        for path, watcher in list(self.temp_watchers.items()):
            # get list of new files from watcher
            if watcher.newFiles() is None:
                dprintf(2, "access error on %s, marking as disappeared", watcher.path)
                del self.temp_watchers[path]
                self.emit(SIGNAL("disappearedFile"), path)
        # if we have new data products, send them to the main window
        return self.makeDataProducts(iter(newstuff.items()))
124213
def cleanup_all_temporary_files(self):
        # type: (Descriptor) -> None
        
        # delete local file
        try:
            self.final_path.unlink()
        except OSError:
            pass
        # iterate unchecked chunks and delete
        for key in self._unchecked_chunks:
            ucc = self._unchecked_chunks[key]['ucc']
            if ucc.temp:
                try:
                    ucc.file_path.unlink()
                except OSError:
                    pass
273402
def cleanup_temporary_directories(self):
        """"""
        while self.build_directories:
            shutil.rmtree(self.build_directories.pop())
        for requirement in self.reported_requirements:
            requirement.remove_temporary_source()
        while self.eggs_links:
            symbolic_link = self.eggs_links.pop()
            if os.path.islink(symbolic_link):
                os.unlink(symbolic_link)
333631
def open_closed_files(self):
        
        if not self.can_open_more_files:
            # If we can't open any more files. Close all of the files.
            self.close_all_files()

        files_with_no_updates = []
        while len(self.closed_file_infos) > 0:
            if (len(self.open_file_infos) >=
                    ray_constants.LOG_MONITOR_MAX_OPEN_FILES):
                self.can_open_more_files = False
                break

            file_info = self.closed_file_infos.pop(0)
            assert file_info.file_handle is None
            # Get the file size to see if it has gotten bigger since we last
            # opened it.
            try:
                file_size = os.path.getsize(file_info.filename)
            except (IOError, OSError) as e:
                # Catch "file not found" errors.
                if e.errno == errno.ENOENT:
                    logger.warning("Warning: The file {} was not "
                                   "found.".format(file_info.filename))
                    self.log_filenames.remove(file_info.filename)
                    continue
                raise e

            # If some new lines have been added to this file, try to reopen the
            # file.
            if file_size > file_info.size_when_last_opened:
                try:
                    f = open(file_info.filename, "r")
                except (IOError, OSError) as e:
                    if e.errno == errno.ENOENT:
                        logger.warning("Warning: The file {} was not "
                                       "found.".format(file_info.filename))
                        self.log_filenames.remove(file_info.filename)
                        continue
                    else:
                        raise e

                f.seek(file_info.file_position)
                file_info.filesize_when_last_opened = file_size
                file_info.file_handle = f
                self.open_file_infos.append(file_info)
            else:
                files_with_no_updates.append(file_info)

        # Add the files with no changes back to the list of closed files.
        self.closed_file_infos += files_with_no_updates
702012
def rebuild(self):
        """"""
        movi = self.riff.find('LIST', 'movi')
        movi.chunks = self.combine_streams()
        self.rebuild_index()
805439
def _add_file(self, path, **params):
        
        log = self._getparam('log', self._discard, **params)
        fd = None
        try:
            fd = os.open(path, os.O_RDONLY)
        except Exception as e:
            if not self.paths[path]:
                log.error("Open failed on watched path %r -- %s", path, e, exc_info=log.isEnabledFor(logging.DEBUG))
                raise e
            elif path in self.paths_pending:
                log.debug("path %r is still pending -- %s", path, e)
            else:
                self.paths_pending[path] = True
                log.debug("Added %r to pending list after open failure -- %s", path, e)
            return False
        if self._mode == WF_KQUEUE:
            log.debug("path %s opened as fd %d", path, fd)
            try:
                ev = select.kevent(fd,
                    filter=select.KQ_FILTER_VNODE,
                    flags=select.KQ_EV_ADD | select.KQ_EV_CLEAR,
                    fflags=select.KQ_NOTE_WRITE | select.KQ_NOTE_ATTRIB | select.KQ_NOTE_LINK |
                                select.KQ_NOTE_DELETE | select.KQ_NOTE_RENAME)
                self._kq.control([ev], 0, 0)
            except Exception as e:
                log.error("kevent failed on watched path %r -- %s", path, e)
                try: os.close(fd)
                except: pass
                raise e

        elif self._mode == WF_INOTIFYX:
            #  inotify doesn't need the target paths open, so now it is known to be
            #  accessible, close the actual fd and use the watch-descriptor as the fd.
            #
            #  However, due to an apparent simfs bug where inotify does not fire either
            #  IN_DELETE_SELF or IN_MOVE_SELF, we need to record the inode so that we
            #  can detect deletes and renames internally.  simfs is used in containers.
            #
            try:
                s = os.fstat(fd)
                self._inx_inode[path] = s.st_ino
            except Exception as e:
                log.error("fstat(%d) failed on open path %r -- %s", fd, path, e)
                try: os.close(fd)
                except: pass
                raise e
            try: os.close(fd)
            except: pass
            try:
                fd = pynotifyx.add_watch(self._inx_fd, path, self._inx_mask)
                log.debug("path %s watched with wd %d", path, fd)
            except Exception as e:
                log.error("inotify failed on watched path %r -- %s", path, e)
                raise e

        elif self._mode == WF_POLLING:
            log.debug("path %s opened as fd %d", path, fd)
            fstate = self._poll_get_stat(fd, path)
            if fstate:
                self._poll_stat[fd] = fstate

        self.paths_open[path] = fd
        self.fds_open[fd] = path
        return True
9543
def rebuild(args):
    
    p = OptionParser(rebuild.__doc__)
    p.add_option("--header", default=False, action="store_true",
                 help="First line is header [default: %default]")
    p.add_option("--write_blast", default=False, action="store_true",
                 help="Get blast records of rebuilt anchors [default: %default]")
    p.set_beds()

    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    blocksfile, blastfile = args
    bk = BlockFile(blocksfile, header=opts.header)
    fw = open("pairs", "w")
    for a, b, h in bk.iter_all_pairs():
        print("\t".join((a, b)), file=fw)
    fw.close()

    if opts.write_blast:
        AnchorFile("pairs").blast(blastfile, "pairs.blast")

    fw = open("tracks", "w")
    for g, col in bk.iter_gene_col():
        print("\t".join(str(x) for x in (g, col)), file=fw)
    fw.close()
673149
def update_paths(self):
        
        new_paths = []
        with self.lock:
            for path in expand_source_paths(iter_module_paths()):
                if path not in self.paths:
                    self.paths.add(path)
                    new_paths.append(path)
        if new_paths:
            self.watch_paths(new_paths)
750563
def remove_files():
    
    logger.info("Removing local track files that were not downloaded...")
    files = [f for f in os.listdir('.') if os.path.isfile(f)]
    for f in files:
        if f not in fileToKeep:
            os.remove(f)
11877
def fixup_paths():
    
    original_paths = os.environ.get('PATH', "").split(os.path.pathsep)
    original_dirs = set(added_dirs)
    yield

    # Fix PATH environment variable
    current_paths = os.environ.get('PATH', "").split(os.path.pathsep)
    if original_paths != current_paths:
        changed_paths = set(current_paths).difference(set(original_paths))
        # rebuild PATH env var
        fixed_paths = []
        for path in current_paths:
            if path in changed_paths:
                fixed_paths.append(env_t(fix_path(path)))
            else:
                fixed_paths.append(env_t(path))
        os.environ['PATH'] = os.pathsep.join(fixed_paths)

    # Fix added_dirs
    if added_dirs != original_dirs:
        for path in set(added_dirs.difference(original_dirs)):
            fixed_path = fix_path(path)
            if fixed_path != path:
                print("Fix %s >> %s" % (path, fixed_path))
                added_dirs.remove(path)
                added_dirs.add(fixed_path)

                i = sys.path.index(path)  # not efficient... but shouldn't happen often
                sys.path[i] = fixed_path

                if env_t(fixed_path) not in os.environ['PATH']:
                    os.environ['PATH'].append(os.pathsep + env_t(fixed_path))
796086
def run_on_removals(paths)
      Runner.remove(Inspector.clean(paths, :missing_ok => true), watchers, options)
    end
895231
def visit((prefix, strip, found), dirname, names):
    
    files = []
    # Iterate over a copy of names, modify names
    for name in names[:]:
        path = os.path.join(dirname, name)
        # Ignore directories -  we will visit later
        if os.path.isdir(path):
            # Remove directories we don't want to visit later
            if isbad(name):
                names.remove(name)
            continue
        elif isgood(name):
            files.append(path)
    destination = os.path.join(prefix, dirname[strip:])
    found.append((destination, files))
1325577
def reset_files(self):
        
        confirm = self.no_confirm

        path = SavepathParser.get_base_path(
            SavepathParser.get_abs_path_static(
                self.cfg.section("Files")["local_data_directory"],
                os.path.dirname(self.cfg_file_path)
            )
        )

        print("""
Cleanup files:
    This will delete {path} and all its contents.
""".format(path=path))

        if not confirm:
            confirm = 'yes' in builtins.input(
                """
    Do you really want to do this? Write 'yes' to confirm: {yes}"""
                    .format(yes='yes' if confirm else ''))

        if not confirm:
            print("Did not type yes. Thus aborting.")
            return

        print("Removing: {}".format(path))

        try:
            shutil.rmtree(path)
        except OSError as error:
            if not os.path.exists(path):
                self.log.error("%s does not exist.", path)
            self.log.error(error)
65610
def cleanup(self):
        
        if self.sock is not None:
            self.sock.close()
        if self.outfile is not None:
            self.outfile.close()
        if self.bar is not None:
            self.update_progress(complete=True)