Generate status rows for each file
168658
private void populateStatusRows()
  {
    for (Map.Entry<String, FileMetadata> entry : fileMetadataMap.entrySet())
    {
      FileMetadata fileMetadata = entry.getValue();

      if (commandType == CommandType.UPLOAD)
      {
        statusRows.add(showEncryptionParameter ?
                       new UploadCommandEncryptionFacade(
                           fileMetadata.srcFileName,
                           fileMetadata.destFileName,
                           fileMetadata.resultStatus.name(),
                           fileMetadata.errorDetails,
                           fileMetadata.srcFileSize,
                           fileMetadata.destFileSize,
                           (fileMetadata.srcCompressionType == null) ?
                           "NONE" : fileMetadata.srcCompressionType.name(),
                           (fileMetadata.destCompressionType == null) ?
                           "NONE" : fileMetadata.destCompressionType.name(),
                           fileMetadata.isEncrypted) :
                       new UploadCommandFacade(
                           fileMetadata.srcFileName,
                           fileMetadata.destFileName,
                           fileMetadata.resultStatus.name(),
                           fileMetadata.errorDetails,
                           fileMetadata.srcFileSize,
                           fileMetadata.destFileSize,
                           (fileMetadata.srcCompressionType == null) ?
                           "NONE" : fileMetadata.srcCompressionType.name(),
                           (fileMetadata.destCompressionType == null) ?
                           "NONE" : fileMetadata.destCompressionType.name()));
      }
      else if (commandType == CommandType.DOWNLOAD)
      {
        statusRows.add(showEncryptionParameter ?
                       new DownloadCommandEncryptionFacade(
                           fileMetadata.srcFileName.startsWith("/") ?
                           fileMetadata.srcFileName.substring(1) :
                           fileMetadata.srcFileName,
                           fileMetadata.resultStatus.name(),
                           fileMetadata.errorDetails,
                           fileMetadata.destFileSize,
                           fileMetadata.isEncrypted) :
                       new DownloadCommandFacade(
                           fileMetadata.srcFileName.startsWith("/") ?
                           fileMetadata.srcFileName.substring(1) :
                           fileMetadata.srcFileName,
                           fileMetadata.resultStatus.name(),
                           fileMetadata.errorDetails,
                           fileMetadata.destFileSize));
      }

    }

    /* we sort the result if the connection is in sorting mode
     */
    Object sortProperty = null;

    sortProperty =
        connection.getSFSessionProperty("sort");

    boolean sortResult = sortProperty != null && (Boolean) sortProperty;

    if (sortResult)
    {
      Comparator comparator =
          (commandType == CommandType.UPLOAD) ?
          new Comparator<Object>()
          {
            public int compare(Object a, Object b)
            {
              String srcFileNameA = ((UploadCommandFacade) a).srcFile;
              String srcFileNameB = ((UploadCommandFacade) b).srcFile;

              return srcFileNameA.compareTo(srcFileNameB);
            }
          } :
          new Comparator<Object>()
          {
            public int compare(Object a, Object b)
            {
              String srcFileNameA = ((DownloadCommandFacade) a).file;
              String srcFileNameB = ((DownloadCommandFacade) b).file;

              return srcFileNameA.compareTo(srcFileNameB);
            }
          };

      // sort the rows by source file names
      Collections.sort(statusRows, comparator);
    }
  }
1365681
def iter_rows(self, start=None, end=None):
        """"""
        start = start or 0
        end = end or self.nrows
        for i in range(start, end):
            yield self.iloc[i, :]
1302642
def generate_files_distributed(generator,
                               output_name,
                               output_dir,
                               num_shards=1,
                               max_cases=None,
                               task_id=0):
  """"""
  assert task_id < num_shards
  output_filename = sharded_name(output_name, task_id, num_shards)
  output_file = os.path.join(output_dir, output_filename)
  tf.logging.info("Writing to file %s", output_file)
  writer = tf.python_io.TFRecordWriter(output_file)

  counter = 0
  for case in generator:
    if counter % 100000 == 0:
      tf.logging.info("Generating case %d for %s." % (counter, output_name))
    counter += 1
    if max_cases and counter > max_cases:
      break
    example = to_example(case)
    writer.write(example.SerializeToString())

  writer.close()
  return output_file
1235152
def status():
    ''''''
    with Session() as session:
        resp = session.Manager.status()
        print(tabulate([('Status', 'Active Sessions'),
                        (resp['status'], resp['active_sessions'])],
                       headers='firstrow'))
710530
def echo_results_file(results_file, no_color, verbose=False):
    """"""
    try:
        data = collect_results(results_file)
    except ValueError:
        echo_style(
            'The results file is not the proper json format.',
            no_color,
            fg='red'
        )
        sys.exit(1)
    except Exception as error:
        echo_style(
            'Unable to process results file: %s' % error,
            no_color,
            fg='red'
        )
        sys.exit(1)

    echo_results(data, no_color, verbose)
1231728
def generate_pretty_output(self, stat, verbose, output_function, logs=True):
        

        has_check = False
        for r in self.results:
            has_check = True
            if stat:
                output_function(OUTPUT_CHARS[r.status],
                                fg=COLOURS[r.status],
                                nl=False)
            else:
                output_function(str(r), fg=COLOURS[r.status])
                if verbose:
                    output_function("  -> {}\n"
                                    "  -> {}".format(r.description,
                                                     r.reference_url),
                                    fg=COLOURS[r.status])
                    if logs and r.logs:
                        output_function("  -> logs:",
                                        fg=COLOURS[r.status])
                        for l in r.logs:
                            output_function("    -> {}".format(l),
                                            fg=COLOURS[r.status])

        if not has_check:
            output_function("No check found.")
        elif stat and not verbose:
            output_function("")
        else:
            output_function("")
            for status, count in six.iteritems(self.statistics):
                output_function("{}:{} ".format(status, count), nl=False)
            output_function("")
823987
def checkfiles(args):
  """"""

  from .query import Database
  db = Database()

  r = db.objects()

  # go through all files, check if they are available on the filesystem
  good = []
  bad = []
  for f in r:
    if os.path.exists(f.make_path(args.directory, args.extension)):
      good.append(f)
    else:
      bad.append(f)

  # report
  output = sys.stdout
  if args.selftest:
    from bob.db.utils import null
    output = null()

  if bad:
    for f in bad:
      output.write('Cannot find file "%s"\n' % (f.make_path(args.directory, args.extension),))
    output.write('%d files (out of %d) were not found at "%s"\n' % \
      (len(bad), len(r), args.directory))

  return 0
326999
def generate_data(self, data_dir, tmp_dir, task_id=-1):
    """"""
    filepath_fns = {
        problem.DatasetSplit.TRAIN: self.training_filepaths,
        problem.DatasetSplit.EVAL: self.dev_filepaths,
        problem.DatasetSplit.TEST: self.test_filepaths,
    }

    # We set shuffled=True as we don't want to shuffle on disk later.
    split_paths = [(split["split"], filepath_fns[split["split"]](
        data_dir, split["shards"], shuffled=True))
                   for split in self.dataset_splits]
    all_paths = []
    for _, paths in split_paths:
      all_paths.extend(paths)

    if self.is_generate_per_split:
      for split, paths in split_paths:
        generator_utils.generate_files(
            self.generate_encoded_samples(data_dir, tmp_dir, split),
            paths,
            cycle_every_n=self.total_number_of_frames // len(paths))
    else:
      generator_utils.generate_files(
          self.generate_encoded_samples(data_dir, tmp_dir,
                                        problem.DatasetSplit.TRAIN),
          all_paths,
          cycle_every_n=self.total_number_of_frames // len(all_paths))
843480
def _build_status(data, item):
    '''
    
    '''
    stream = item['stream']
    if 'Running in' in stream:
        data.setdefault('Intermediate_Containers', []).append(
            stream.rstrip().split()[-1])
    if 'Successfully built' in stream:
        data['Id'] = stream.rstrip().split()[-1]
1356493
def status(ctx, client, revision, no_output, path):
    """"""
    graph = Graph(client)
    # TODO filter only paths = {graph.normalize_path(p) for p in path}
    status = graph.build_status(revision=revision, can_be_cwl=no_output)

    click.echo('On branch {0}'.format(client.repo.active_branch))
    if status['outdated']:
        click.echo(
            'Files generated from newer inputs:\n'
            '  (use "renku log [<file>...]" to see the full lineage)\n'
            '  (use "renku update [<file>...]" to '
            'generate the file from its latest inputs)\n'
        )

        for filepath, stts in sorted(status['outdated'].items()):
            outdated = (
                ', '.join(
                    '{0}#{1}'.format(
                        click.style(
                            graph._format_path(n.path), fg='blue', bold=True
                        ),
                        _format_sha1(graph, n),
                    ) for n in stts
                    if n.path and n.path not in status['outdated']
                )
            )

            click.echo(
                '\t{0}: {1}'.format(
                    click.style(
                        graph._format_path(filepath), fg='red', bold=True
                    ), outdated
                )
            )

        click.echo()

    else:
        click.secho(
            'All files were generated from the latest inputs.', fg='green'
        )

    if status['multiple-versions']:
        click.echo(
            'Input files used in different versions:\n'
            '  (use "renku log --revision <sha1> <file>" to see a lineage '
            'for the given revision)\n'
        )

        for filepath, files in sorted(status['multiple-versions'].items()):
            # Do not show duplicated commits!  (see #387)
            commits = {_format_sha1(graph, key) for key in files}
            click.echo(
                '\t{0}: {1}'.format(
                    click.style(
                        graph._format_path(filepath), fg='blue', bold=True
                    ), ', '.join(commits)
                )
            )

        click.echo()

    if status['deleted']:
        click.echo(
            'Deleted files used to generate outputs:\n'
            '  (use "git show <sha1>:<file>" to see the file content '
            'for the given revision)\n'
        )

        for filepath, node in status['deleted'].items():
            click.echo(
                '\t{0}: {1}'.format(
                    click.style(
                        graph._format_path(filepath), fg='blue', bold=True
                    ), _format_sha1(graph, node)
                )
            )

        click.echo()

    ctx.exit(1 if status['outdated'] else 0)
1015382
def lines_stats(dir_path, file_filter):
    
    n_files = 0
    n_lines = 0
    for p in Path(dir_path).select_file(file_filter):
        n_files += 1
        n_lines += count_lines(p.abspath)
    return n_files, n_lines
75010
def generate_data(self, data_dir, tmp_dir=None, task_id=-1):
    """"""
    if not self._rollouts_by_epoch_and_split[self.current_epoch]:
      # Data not loaded from disk.
      self._split_current_epoch()

    rollouts_by_split = self._rollouts_by_epoch_and_split[self.current_epoch]
    splits_and_paths = self.splits_and_paths(data_dir)

    for (split, paths) in splits_and_paths:
      rollouts = rollouts_by_split[split]
      num_frames = self._calc_num_frames(rollouts)
      shard_size = num_frames // len(paths)

      frame_gen = self._generate_frames(rollouts)
      for (path_index, path) in enumerate(paths):
        limit = shard_size
        # Put the remainder in the last shard to preserve the ordering.
        if path_index == len(paths) - 1:
          limit = None
        generator_utils.generate_files(
            itertools.islice(frame_gen, limit), [path],
            cycle_every_n=float("inf")
        )
406459
def down_by_arrival(*filters, local_dir=".", remote_dir=DEFAULT_REMOTE_DIR):
    """"""
    remote_monitor = watch_remote_files(*filters, remote_dir=remote_dir)
    _, file_set = next(remote_monitor)
    _notify_sync_ready(len(file_set), remote_dir, local_dir)
    for new_arrivals, file_set in remote_monitor:
        if new_arrivals:
            _notify_sync(Direction.down, new_arrivals)
            down_by_files(new_arrivals, local_dir)
            _notify_sync_ready(len(file_set), remote_dir, local_dir)
        yield Direction.down, new_arrivals
1183433
def write_summary(all_procs, summary_file):
    
    if not summary_file:
        return

    with summary_file:
        writer = csv.writer(summary_file, delimiter='\t', lineterminator='\n')
        writer.writerow(('directory', 'command', 'start_time', 'end_time',
            'run_time', 'exit_status', 'result'))
        rows = ((p.working_dir, ' '.join(p.command), p.start_time, p.end_time,
                p.running_time, p.return_code, p.status) for p in all_procs)
        writer.writerows(rows)
171539
def progress_status(self):
        """"""

        if self.line % self.freq == 0:
            text = self.statustext.format(nele=self.line,
                                          totalele=self.total_lines)

            if self.main_window.grid.actions.pasting:
                try:
                    post_command_event(self.main_window,
                                       self.main_window.StatusBarMsg,
                                       text=text)
                except TypeError:
                    # The main window does not exist any more
                    pass
            else:
                # Write directly to the status bar because the event queue
                # is not emptied during file access

                self.main_window.GetStatusBar().SetStatusText(text)

            # Now wait for the statusbar update to be written on screen
            if is_gtk():
                try:
                    wx.Yield()
                except:
                    pass

        self.line += 1
632289
function status() {
  var tasks = [],
      cwd = process.cwd();

  findRepos(function(err, repos) {
    if (err) return console.log(err);
    repos.forEach(function(repo) {
      tasks.push(function(done) {
        repo.status(function(err, status) {
          if (err) return done(err);
          if (argv.b) {
            repo.branch(function(err, head) {
              if (err) return done(err);
              done(null, {status: status, head: head});
            });
          }
          else {
            done(null, {status: status});
          }
        });
      });
    });
    async.series(tasks, function(err, results) {
      if (err) return console.log(err);

      var row,
          col,
          res,
          file,
          head = [],
          table,
          symbol;

      head.push('Repo');
      if (argv.b) {
        head.push('Branch');
      }
      head.push('Status');

      table = new Table({
        head: head,
        style: {
          compact: true,
          'padding-left': 1,
          'padding-right': 1,
          head: ['white', 'bold']
        }
      });

      results.forEach(function(res) {
        var row = [];

        row.push(res.status.repo.path.replace(cwd + '/', ''));
        if (argv.b) {
          row.push(res.head.name);
        }
        row.push(res.status.clean ? 'Clean'.green : 'Changed'.red);
        table.push(row);

        if (argv.v) {
          for (var name in res.status.files) {
            row = [];
            row.push('');
            if (argv.b) {
              row.push('');
            }
            switch (res.status.files[name].type) {
              case 'A':
                symbol = '+'.green; break;
              case 'D':
                symbol = '-'.red; break;
              case 'M':
                symbol = '*'.yellow; break;
              default:
                symbol = '~'.grey; break;
            }
            row.push(' ' + symbol + ' ' + name);
            table.push(row);
          }
        }
      });

      console.log(table.toString());
    });
  });
}
1012422
def format_op_row(ipFile, totLines, totWords, uniqueWords):
    
    txt = os.path.basename(ipFile).ljust(36) + ' '
    txt += str(totLines).rjust(7) + ' '
    txt += str(totWords).rjust(7) + ' '
    txt += str(len(uniqueWords)).rjust(7) + ' '
    return txt