Syncs the local logdir on driver to worker if possible.

        Requires ray cluster to be started with the autoscaler. Also requires
        rsync to be installed.
769465
def sync_to_worker_if_possible(self):
        
        if self.worker_ip == self.local_ip:
            return
        ssh_key = get_ssh_key()
        ssh_user = get_ssh_user()
        global _log_sync_warned
        if ssh_key is None or ssh_user is None:
            if not _log_sync_warned:
                logger.error("Log sync requires cluster to be setup with "
                             "`ray up`.")
                _log_sync_warned = True
            return
        if not distutils.spawn.find_executable("rsync"):
            logger.error("Log sync requires rsync to be installed.")
            return
        source = "{}/".format(self.local_dir)
        target = "{}@{}:{}/".format(ssh_user, self.worker_ip, self.local_dir)
        final_cmd = (("""rsync -savz -e "ssh -i {} -o ConnectTimeout=120s -o StrictHostKeyChecking=no" {} {}""").format(
                          quote(ssh_key), quote(source), quote(target)))
        logger.info("Syncing results to %s", str(self.worker_ip))
        sync_process = subprocess.Popen(
            final_cmd, shell=True, stdout=self.logfile)
        sync_process.wait()
328964
public static void fixLoggingSystem(File basedir) {
        ILoggerFactory factory = LoggerFactory.getILoggerFactory();
        if (factory instanceof LoggerContext) {
            // We know that we are using logback from here.
            LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory();
            ch.qos.logback.classic.Logger logbackLogger = lc.getLogger(Logger.ROOT_LOGGER_NAME);
            if (logbackLogger == null) {
                return;
            }

            Iterator<Appender<ILoggingEvent>> iterator = logbackLogger.iteratorForAppenders();
            while (iterator.hasNext()) {
                Appender<ILoggingEvent> appender = iterator.next();

                if (appender instanceof AsyncAppender) {
                    appender = ((AsyncAppender) appender).getAppender("FILE");
                }

                if (appender instanceof RollingFileAppender) {
                    RollingFileAppender<ILoggingEvent> fileAppender =
                            (RollingFileAppender<ILoggingEvent>) appender;
                    String file = new File(basedir, "logs/wisdom.log").getAbsolutePath();
                    fileAppender.stop();
                    // Remove the created log directory.
                    // We do that afterwards because on Windows the file cannot be deleted while we still have a logger
                    // using it.
                    FileUtils.deleteQuietly(new File("logs"));
                    fileAppender.setFile(file);
                    fileAppender.setContext(lc);
                    fileAppender.start();
                }
            }
        }
    }
92293
def sync(self):
        """"""
        if self.filepath:
            serialised_file = open(self.filepath, "w")
            json.dump(self.state, serialised_file)
            serialised_file.close()
        else:
            print("Filepath to the persistence file is not set. State cannot be synced to disc.")
243285
def synchronize! updater
      source_hash[updater.filename] = source_hash[updater.filename].synchronize(updater)
    end
1189832
def _worker_process(self):
        # type: (LocalFileMd5Offload) -> None
        
        while not self.terminated:
            try:
                key, lpath, fpath, remote_md5, pagealign, lpview = \
                    self._task_queue.get(True, 0.1)
            except queue.Empty:
                continue
            if lpview is None:
                start = None
                end = None
                size = None
            else:
                start = lpview.fd_start
                end = lpview.fd_end
                size = end - start
            md5 = blobxfer.operations.md5.compute_md5_for_file_asbase64(
                fpath, pagealign, start, end)
            logger.debug('pre-transfer MD5 check: {} <L..R> {} {}'.format(
                md5, remote_md5, fpath))
            self._done_cv.acquire()
            self._done_queue.put((key, lpath, size, md5 == remote_md5))
            self._done_cv.notify()
            self._done_cv.release()
374952
private String write(String logDir) throws IOException, ExecutionException, InterruptedException {
        final File file = new File(logDir, "trace_" + System.currentTimeMillis() + ".json.gz");
        if (file.exists()) {
            throw new IOException("Trace file " + file.getAbsolutePath() + " already exists");
        }
        if (!file.getParentFile().canWrite() || !file.getParentFile().canExecute()) {
            throw new IOException("Trace file " + file.getAbsolutePath() + " is not writable");
        }

        SettableFuture<Future<?>> f = SettableFuture.create();
        m_work.offer(() -> f.set(dumpEvents(file)));
        final Future<?> writeFuture = f.get();
        if (writeFuture != null) {
            writeFuture.get(); // Wait for the write to finish without blocking new events
            return file.getAbsolutePath();
        } else {
            // A write is already in progress, ignore this request
            return null;
        }
    }
497125
def sync(filename, connection=None):
    
    c = connection or connect()
    rev = c.ls(filename)
    if rev:
        rev[0].sync()
104131
def sync_path_to_host(path, host, user, verbose=False, cmd=None, gid=None,
                      fatal=False):
    
    cmd = cmd or copy(BASE_CMD)
    if not verbose:
        cmd.append('-silent')

    # removing trailing slash from directory paths, unison
    # doesn't like these.
    if path.endswith('/'):
        path = path[:(len(path) - 1)]

    cmd = cmd + [path, 'ssh://%s@%s/%s' % (user, host, path)]

    try:
        log('Syncing local path %s to %s@%s:%s' % (path, user, host, path))
        run_as_user(user, cmd, gid)
    except Exception:
        log('Error syncing remote files')
        if fatal:
            raise
480053
def setup_logging(filename, log_dir=None, force_setup=False):
    ''' 
    '''
    if not force_setup and ChirpCLI.SETUP_COMPLETED:
        logging.debug("Master logging has been setup. This call will be ignored.")
        return
    if log_dir and not os.path.exists(log_dir):
        os.makedirs(log_dir)
    if os.path.isfile(filename):
        with open(filename) as config_file:
            try:
                config = json.load(config_file)
                logging.config.dictConfig(config)
                logging.info("logging was setup using {}".format(filename))
                ChirpCLI.SETUP_COMPLETED = True
            except Exception as e:
                logging.exception("Could not load logging config")
                # default logging config
                logging.basicConfig(level=logging.INFO)
    else:
        logging.basicConfig(level=logging.INFO)
873087
private void rolloverLogTask()
  {
    try {
      if (_isInit) {
        flush();
      }
    } catch (Exception e) {
      log.log(Level.WARNING, e.toString(), e);
    }

    _isRollingOver = true;
    
    try {
      if (! _isInit)
        return;
      
      Path savedPath = null;

      long now = CurrentTime.currentTime();

      long lastPeriodEnd = _nextPeriodEnd;

      _nextPeriodEnd = nextRolloverTime(now);

      Path path = getPath();

      synchronized (_logLock) {
        flushTempStream();

        long length = Files.size(path);
        
        if (lastPeriodEnd <= now && lastPeriodEnd > 0) {
          closeLogStream();

          savedPath = getSavedPath(lastPeriodEnd - 1);
        }
        else if (path != null && getRolloverSize() <= length) {
          closeLogStream();

          savedPath = getSavedPath(now);
        }
      }
      
      // archiving of path is outside of the synchronized block to
      // avoid freezing during archive
      if (savedPath != null) {
        movePathToArchive(savedPath);
      }
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      synchronized (_logLock) {
        _isRollingOver = false;
        flushTempStream();
      }
      
      _rolloverListener.requeue(_rolloverAlarm);
    }
  }
1327112
private boolean setup() throws IOException {
    estimateSavings();

    final String randomId = getRandomId();
    JobClient jClient = new JobClient(jobconf);
    Path jobdir = new Path(jClient.getSystemDir(), NAME + "_" + randomId);

    LOG.info(JOB_DIR_LABEL + "=" + jobdir);
    jobconf.set(JOB_DIR_LABEL, jobdir.toString());
    Path log = new Path(jobdir, "_logs");

    // The control file should have small size blocks. This helps
    // in spreading out the load from mappers that will be spawned.
    jobconf.setInt("dfs.blocks.size",  OP_LIST_BLOCK_SIZE);

    FileOutputFormat.setOutputPath(jobconf, log);
    LOG.info("log=" + log);

    // create operation list
    FileSystem fs = jobdir.getFileSystem(jobconf);
    Path opList = new Path(jobdir, "_" + OP_LIST_LABEL);
    jobconf.set(OP_LIST_LABEL, opList.toString());
    int opCount = 0, synCount = 0;
    SequenceFile.Writer opWriter = null;

    try {
      opWriter = SequenceFile.createWriter(fs, jobconf, opList, Text.class,
          PolicyInfo.class, SequenceFile.CompressionType.NONE);
      for (RaidPolicyPathPair p : raidPolicyPathPairList) {
        // If a large set of files are Raided for the first time, files
        // in the same directory that tend to have the same size will end up
        // with the same map. This shuffle mixes things up, allowing a better
        // mix of files.
        java.util.Collections.shuffle(p.srcPaths);
        for (EncodingCandidate ec : p.srcPaths) {
          opWriter.append(new Text(ec.toString()), p.policy);
          opCount++;
          if (++synCount > SYNC_FILE_MAX) {
            opWriter.sync();
            synCount = 0;
          }
        }
      }

    } finally {
      if (opWriter != null) {
        opWriter.close();
      }
      fs.setReplication(opList, OP_LIST_REPLICATION); // increase replication for control file
    }
    raidPolicyPathPairList.clear();
    
    jobconf.setInt(OP_COUNT_LABEL, opCount);
    LOG.info("Number of files=" + opCount);
    jobconf.setNumMapTasks(getMapCount(opCount));
    LOG.info("jobName= " + jobName + " numMapTasks=" + jobconf.getNumMapTasks());
    return opCount != 0;

  }
115080
def _startup(self):
        
        global _ac_config_logs
        global sconf_global
        global SConfFS

        self.lastEnvFs = self.env.fs
        self.env.fs = SConfFS
        self._createDir(self.confdir)
        self.confdir.up().add_ignore( [self.confdir] )

        if self.logfile is not None and not dryrun:
            # truncate logfile, if SConf.Configure is called for the first time
            # in a build
            if self.logfile in _ac_config_logs:
                log_mode = "a"
            else:
                _ac_config_logs[self.logfile] = None
                log_mode = "w"
            fp = open(str(self.logfile), log_mode)
            self.logstream = SCons.Util.Unbuffered(fp)
            # logfile may stay in a build directory, so we tell
            # the build system not to override it with a eventually
            # existing file with the same name in the source directory
            self.logfile.dir.add_ignore( [self.logfile] )

            tb = traceback.extract_stack()[-3-self.depth]
            old_fs_dir = SConfFS.getcwd()
            SConfFS.chdir(SConfFS.Top, change_os_dir=0)
            self.logstream.write('file %s,line %d:\n\tConfigure(confdir = %s)\n' %
                                 (tb[0], tb[1], str(self.confdir)) )
            SConfFS.chdir(old_fs_dir)
        else:
            self.logstream = None
        # we use a special builder to create source files from TEXT
        action = SCons.Action.Action(_createSource,
                                     _stringSource)
        sconfSrcBld = SCons.Builder.Builder(action=action)
        self.env.Append( BUILDERS={'SConfSourceBuilder':sconfSrcBld} )
        self.config_h_text = _ac_config_hs.get(self.config_h, "")
        self.active = 1
        # only one SConf instance should be active at a time ...
        sconf_global = self
26350
void initFromDir() {
    for (File file : this.logDir.listFiles()) {
      if (!file.isDirectory() &&
          file.getName().matches("transaction\\.\\d+_\\d+")) {
        // Appends the file with valid name to log file list.
        this.logFiles.add(file);
      }
    }
    if (!this.logFiles.isEmpty()) {
      // Sorts the file by the zxid order.
      Collections.sort(this.logFiles);
    }
  }
669256
def _worker_thread_disk(self):
        # type: (Downloader) -> None
        
        while not self.termination_check:
            try:
                dd, offsets, data = self._disk_queue.get(
                    block=False, timeout=0.1)
            except queue.Empty:
                continue
            try:
                self._process_data(dd, offsets, data)
            except Exception as e:
                with self._transfer_lock:
                    self._exceptions.append(e)
416284
public synchronized void saveWorkdir() throws IOException {
    Preconditions.checkState(mState == State.STARTED,
        "cluster must be started before you can save its work directory");
    ARTIFACTS_DIR.mkdirs();

    File tarball = new File(mWorkDir.getParentFile(), mWorkDir.getName() + ".tar.gz");
    // Tar up the work directory.
    ProcessBuilder pb =
        new ProcessBuilder("tar", "-czf", tarball.getName(), mWorkDir.getName());
    pb.directory(mWorkDir.getParentFile());
    pb.redirectOutput(Redirect.appendTo(TESTS_LOG));
    pb.redirectError(Redirect.appendTo(TESTS_LOG));
    Process p = pb.start();
    try {
      p.waitFor();
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
      throw new RuntimeException(e);
    }
    // Move tarball to artifacts directory.
    File finalTarball = new File(ARTIFACTS_DIR, tarball.getName());
    FileUtils.moveFile(tarball, finalTarball);
    LOG.info("Saved cluster {} to {}", mClusterName, finalTarball.getAbsolutePath());
  }
961266
def start(self):
        """"""
        self.t = threading.Thread(target=self._run, name="ray_import_thread")
        # Making the thread a daemon causes it to exit
        # when the main thread exits.
        self.t.daemon = True
        self.t.start()
134245
def work(options):
    """"""
    # pylint: disable=too-many-locals
    record = get_record(options)
    _, mainv, dailyv, _, _, _, safebrowsingv, bytecodev = record.split(':')
    versions = {'main': mainv, 'daily': dailyv,
                'safebrowsing': safebrowsingv,
                'bytecode': bytecodev}
    dqueue = Queue(maxsize=0)
    dqueue_workers = 3
    info("[+] \033[92mStarting workers\033[0m")
    for index in range(dqueue_workers):
        info("=> Starting diff download worker: %d" % (index + 1))
        worker = Thread(target=download_diffs, args=(dqueue,))
        worker.setDaemon(True)
        worker.start()
    mqueue = Queue(maxsize=0)
    mqueue_workers = 4
    for index in range(mqueue_workers):
        info("=> Starting signature download worker: %d" % (index + 1))
        worker = Thread(target=update_sig, args=(mqueue,))
        worker.setDaemon(True)
        worker.start()
    for signature_type in ['main', 'daily', 'bytecode', 'safebrowsing']:
        if signature_type in ['daily', 'bytecode', 'safebrowsing']:
            # cdiff downloads
            localver = get_local_version(options.mirrordir, signature_type)
            remotever = versions[signature_type]
            if localver is not None:
                dqueue.put(
                    (
                        options,
                        signature_type,
                        localver,
                        remotever
                    )
                )
        mqueue.put((options, signature_type, versions))
    info("=> Waiting on workers to complete tasks")
    dqueue.join()
    mqueue.join()
    info("=> Workers done processing queues")
    create_dns_file(options, record)
    sys.exit(0)