Reads/verifies previously created output
770771
def readMNIST(sc, output, format):
  """"""

  output_images = output + "/images"
  output_labels = output + "/labels"
  imageRDD = None
  labelRDD = None

  if format == "pickle":
    imageRDD = sc.pickleFile(output_images)
    labelRDD = sc.pickleFile(output_labels)
  elif format == "csv":
    imageRDD = sc.textFile(output_images).map(fromCSV)
    labelRDD = sc.textFile(output_labels).map(fromCSV)
  else:  # format.startswith("tf"):
    # requires: --jars tensorflow-hadoop-1.0-SNAPSHOT.jar
    tfRDD = sc.newAPIHadoopFile(output, "org.tensorflow.hadoop.io.TFRecordFileInputFormat",
                                keyClass="org.apache.hadoop.io.BytesWritable",
                                valueClass="org.apache.hadoop.io.NullWritable")
    imageRDD = tfRDD.map(lambda x: fromTFExample(bytes(x[0])))

  num_images = imageRDD.count()
  num_labels = labelRDD.count() if labelRDD is not None else num_images
  samples = imageRDD.take(10)
  print("num_images: ", num_images)
  print("num_labels: ", num_labels)
  print("samples: ", samples)
899558
def check_finished(self):
        """"""
        changed = False
        for key in list(self.processes.keys()):
            # Poll process and check if it finshed
            process = self.processes[key]
            process.poll()
            if process.returncode is not None:
                # If a process is terminated by `stop` or `kill`
                # we want to queue it again instead closing it as failed.
                if key not in self.stopping:
                    # Get std_out and err_out
                    output, error_output = process.communicate()

                    descriptor = self.descriptors[key]
                    descriptor['stdout'].seek(0)
                    descriptor['stderr'].seek(0)
                    output = get_descriptor_output(descriptor['stdout'], key, handler=self)
                    error_output = get_descriptor_output(descriptor['stderr'], key, handler=self)

                    # Mark queue entry as finished and save returncode
                    self.queue[key]['returncode'] = process.returncode
                    if process.returncode != 0:
                        self.queue[key]['status'] = 'failed'
                    else:
                        self.queue[key]['status'] = 'done'

                    # Add outputs to queue
                    self.queue[key]['stdout'] = output
                    self.queue[key]['stderr'] = error_output
                    self.queue[key]['end'] = str(datetime.now().strftime("%H:%M"))

                    self.queue.write()
                    changed = True
                else:
                    self.stopping.remove(key)
                    if key in self.to_remove:
                        self.to_remove.remove(key)
                        del self.queue[key]
                    else:
                        if key in self.to_stash:
                            self.to_stash.remove(key)
                            self.queue[key]['status'] = 'stashed'
                        else:
                            self.queue[key]['status'] = 'queued'
                        self.queue[key]['start'] = ''
                        self.queue[key]['end'] = ''

                    self.queue.write()

                self.clean_descriptor(key)
                del self.processes[key]

        # If anything should be logged we return True
        return changed
184472
def handle_raw_output(ctx, data):
    """"""
    if ctx.obj['format'] == 'json':
        print(json_dump(data))
        exit(0)
    if ctx.obj['format'] == 'yaml':
        print(yaml_dump(data), end='')
        exit(0)
511687
def output(stream):
    """"""
    while True:
        content = stream.read(1024)
        if len(content) == 0: break
        sys.stdout.write(content)
26114
private static boolean verifyFile(Configuration conf,
      FileSystem srcFs, FileSystem parityFs,
      FileStatus stat, Path parityPath, Codec codec,
      long blockOffset, Progressable reporter) 
    throws IOException, InterruptedException {
    Path srcPath = stat.getPath();
    LOG.info("Verify file: " + srcPath + " at offset: " + blockOffset);
    int limit = (int) Math.min(stat.getBlockSize(), DEFAULT_VERIFY_LEN);
    if (reporter == null) {
      reporter = RaidUtils.NULL_PROGRESSABLE;
    }
    
    // try to decode.
    Decoder decoder = new Decoder(conf, codec);
    if (codec.isDirRaid) {
      decoder.connectToStore(srcPath);
    }
    
    List<Long> errorOffsets = new ArrayList<Long>();
    // first limit bytes
    errorOffsets.add(blockOffset);
    long left = Math.min(stat.getBlockSize(), stat.getLen() - blockOffset);
    if (left > limit) {
      // last limit bytes
      errorOffsets.add(blockOffset + left - limit);
      // random limit bytes.
      errorOffsets.add(blockOffset + 
          rand.nextInt((int)(left - limit)));
    }
 
    byte[] buffer = new byte[limit];
    FSDataInputStream is = srcFs.open(srcPath);
    try {
      for (long errorOffset : errorOffsets) {
        is.seek(errorOffset);
        is.read(buffer);
        // calculate the oldCRC.
        CRC32 oldCrc = new CRC32();
        oldCrc.update(buffer);
        
        CRC32 newCrc = new CRC32();
        DecoderInputStream stream = decoder.new DecoderInputStream(
            RaidUtils.NULL_PROGRESSABLE, limit, stat.getBlockSize(), errorOffset, 
            srcFs, srcPath, parityFs, parityPath, null, null, false);
        try {
          stream.read(buffer);
          newCrc.update(buffer);
          if (oldCrc.getValue() != newCrc.getValue()) {
            LogUtils.logFileCheckMetrics(LOGRESULTS.FAILURE, codec, srcPath, 
                srcFs, errorOffset, limit, null, reporter);
            LOG.error("mismatch crc, old " + oldCrc.getValue() + 
                ", new " + newCrc.getValue() + ", for file: " + srcPath
                + " at offset " + errorOffset + ", read limit " + limit);
            return false;
          }
        } finally {
          reporter.progress();
          if (stream != null) {
            stream.close();
          }
        }
      }
      return true;
    } finally {
      is.close();
    }
  }
682354
def check_output_file(self, path):
        
        if not gf.file_can_be_written(path):
            self.print_error(u"Unable to create file '%s'" % (path))
            self.print_error(u"Make sure the file path is written/escaped correctly and that you have write permission on it")
            return False
        return True
422015
def verify(ctx, file, account):
    
    if not file:
        print_message("Prompting for message. Terminate with CTRL-D", "info")
        file = click.get_text_stream("stdin")
    m = Message(file.read(), bitshares_instance=ctx.bitshares)
    try:
        if m.verify():
            print_message("Verified", "success")
        else:
            print_message("not verified", "error")
    except InvalidMessageSignature:
        print_message("Signature INVALID!", "error")
173169
private void readData(StringBuilder powerShellOutput) throws IOException {
        String line;
        while (null != (line = this.reader.readLine())) {

            //In the case of script mode it finish when the last line is read
            if (this.scriptMode) {
                if (line.equals(PowerShell.END_SCRIPT_STRING)) {
                    break;
                }
            }

            powerShellOutput.append(line).append(CRLF);

            //When not in script mode, it exits when the command is finished
            if (!this.scriptMode) {
                try {
                    if (this.closed || !canContinueReading()) {
                        break;
                    }
                } catch (InterruptedException ex) {
                    Logger.getLogger(PowerShellCommandProcessor.class.getName()).log(Level.SEVERE, "Error executing command and reading result", ex);
                }
            }
        }
    }
291725
public static void verify(final ClassReader cr, final boolean dump,
            final PrintWriter pw) {
        verify(cr, null, dump, pw);
    }
352675
def _read_output(self, command):
        
        cmd = []
        cmd.extend(self._prefix)
        cmd.extend([self._path, "-iP"])
        cmd.extend(command)
        command = cmd

        logger.debug("Cmd %s" % command)
        null = open('/dev/null', 'w')
        process = subprocess.Popen(
            command, stdout=subprocess.PIPE, stderr=null)
        null.close()

        results = []
        reader = csv.reader(_input_csv(process.stdout), delimiter=str("|"))

        try:
            headers = next(reader)
            logger.debug("<-- headers %s" % headers)
        except StopIteration:
            logger.debug("Cmd %s headers not found" % command)
            headers = []

        for row in reader:
            _output_csv(row)

            logger.debug("<-- row %s" % row)
            this_row = {}

            i = 0
            for i in range(0, len(headers)):
                key = headers[i]
                value = row[i]
                this_row[key] = value

            results.append(this_row)

        process.stdout.close()
        retcode = process.wait()
        if retcode != 0:
            logger.error("<-- Cmd %s returned %d (error)" % (command, retcode))
            raise subprocess.CalledProcessError(retcode, command)

        if len(headers) == 0:
            logger.error("Cmd %s didn't return any headers." % command)
            raise RuntimeError("Cmd %s didn't return any headers." % command)

        logger.debug("<-- Returned: %d (good)" % retcode)
        return results
575436
def _verify_output(self, submission_type):
    
    result = True
    if submission_type == 'defense':
      try:
        image_classification = load_defense_output(
            os.path.join(self._sample_output_dir, 'result.csv'))
        expected_keys = [IMAGE_NAME_PATTERN.format(i)
                         for i in range(BATCH_SIZE)]
        if set(image_classification.keys()) != set(expected_keys):
          logging.error('Classification results are not saved for all images')
          result = False
      except IOError as e:
        logging.error('Failed to read defense output file: %s', e)
        result = False
    else:
      for i in range(BATCH_SIZE):
        image_filename = os.path.join(self._sample_output_dir,
                                      IMAGE_NAME_PATTERN.format(i))
        try:
          img = np.array(Image.open(image_filename).convert('RGB'))
          if list(img.shape) != [299, 299, 3]:
            logging.error('Invalid image size %s for image %s',
                          str(img.shape), image_filename)
            result = False
        except IOError as e:
          result = False
    return result
1235192
def check_existing_results(self, benchmark):
        
        if os.path.exists(benchmark.log_folder):
            sys.exit('Output directory {0} already exists, will not overwrite existing results.'.format(benchmark.log_folder))
        if os.path.exists(benchmark.log_zip):
            sys.exit('Output archive {0} already exists, will not overwrite existing results.'.format(benchmark.log_zip))
1247632
def FromTXOutputsConfirmed(outputs):
        
        uns = UnspentCoinState()
        uns.Items = [0] * len(outputs)
        for i in range(0, len(outputs)):
            uns.Items[i] = int(CoinState.Confirmed)
        return uns
309986
def _read_stdout(self):
        ''''''
        try:
            while self._process.returncode is None:
                line = yield from self._process.stdout.readline()

                _logger.debug('Read stdout line %s', repr(line))

                if not line:
                    break

                if self._stdout_callback:
                    yield from self._stdout_callback(line)

        except Exception:
            _logger.exception('Unhandled read stdout exception.')
            raise
996790
def regenerateToOutputFile(outFile, checkNoChanges = false)
      executeRubyComponents
      @pageObject.process
      writeRegeneratedFile(outFile, checkNoChanges, checkNoChanges)
    end
701673
def capture_output(self, with_hook=True):
        """"""
        self.hooked = ''

        def display_hook(obj):
            # That's some dirty hack
            self.hooked += self.safe_better_repr(obj)
            self.last_obj = obj

        stdout, stderr = sys.stdout, sys.stderr
        if with_hook:
            d_hook = sys.displayhook
            sys.displayhook = display_hook

        sys.stdout, sys.stderr = StringIO(), StringIO()
        out, err = [], []
        try:
            yield out, err
        finally:
            out.extend(sys.stdout.getvalue().splitlines())
            err.extend(sys.stderr.getvalue().splitlines())
            if with_hook:
                sys.displayhook = d_hook

            sys.stdout, sys.stderr = stdout, stderr
1065259
def read(self):
        
        if self._output_buffer:
            b, self._output_buffer = (self._output_buffer[0:1],
                                      self._output_buffer[1:])
            return b

        return b''