Compress an input stream with GZIP and return the result size, digest and
compressed stream.

@param inputStream
@return the compressed stream
@throws SnowflakeSQLException
@deprecated Can be removed when all accounts are encrypted
6358
private static InputStreamWithMetadata compressStreamWithGZIPNoDigest(
      InputStream inputStream) throws SnowflakeSQLException
  {
    try
    {
      FileBackedOutputStream tempStream =
          new FileBackedOutputStream(MAX_BUFFER_SIZE, true);

      CountingOutputStream countingStream =
          new CountingOutputStream(tempStream);

      // construct a gzip stream with sync_flush mode
      GZIPOutputStream gzipStream;

      gzipStream = new GZIPOutputStream(countingStream, true);

      IOUtils.copy(inputStream, gzipStream);

      inputStream.close();

      gzipStream.finish();
      gzipStream.flush();

      countingStream.flush();

      return new InputStreamWithMetadata(countingStream.getCount(),
                                         null, tempStream);

    }
    catch (IOException ex)
    {
      logger.error("Exception compressing input stream", ex);

      throw new SnowflakeSQLException(ex, SqlState.INTERNAL_ERROR,
                                      ErrorCode.INTERNAL_ERROR.getMessageCode(),
                                      "error encountered for compression");
    }

  }
373491
private static byte[] gzip(final InputStream input)
        throws IOException {
        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
        // @checkstyle MagicNumberCheck (1 line)
        final byte[] buf = new byte[4096];
        final OutputStream gzip = new GZIPOutputStream(baos);
        try {
            while (true) {
                final int len = input.read(buf);
                if (len < 0) {
                    break;
                }
                gzip.write(buf, 0, len);
            }
        } finally {
            gzip.close();
            input.close();
        }
        return baos.toByteArray();
    }
286262
private InputStream prepareInputStream(InputStream stream) throws IOException
   {
      InputStream result;
      BufferedInputStream bis = new BufferedInputStream(stream);
      readHeaderProperties(bis);
      if (isCompressed())
      {
         result = new InflaterInputStream(bis);
      }
      else
      {
         result = bis;
      }
      return result;
   }
40516
public void encodeAndApply(InputStream inputStream, String ssid, String msgId, Consumer<StreamFragmentBean> consumer) {
        try (final InputStream _inputStream = inputStream) {
            final int LENGTH = StreamManager.BUF_SIZE_IN_BYTE;
            int read;
            boolean first = true;
            boolean last;
            int current = -1;
            do {
                byte[] buffer = new byte[LENGTH];
                read = _inputStream.read(buffer);
                if (read == -1) {
                    last = true;
                    buffer = new byte[0];
                } else if (read < LENGTH) {
                    buffer = Arrays.copyOf(buffer, read);
                    last = true;
                } else
                    last = false;
                StreamFragmentBean.Header header = new StreamFragmentBean.Header();
                header.setFirst(first);
                header.setId(ssid);
                header.setLast(last);
                header.setMsgId(msgId);
                header.setIndex(++current);
                StreamFragmentBean streamFragmentBean = new StreamFragmentBean();
                streamFragmentBean.setBody(buffer);
                streamFragmentBean.setHeader(header);
                streamFragmentBean.setMessageType(MessageType.responseStream);
                consumer.accept(streamFragmentBean);
                first = false;
            }
            while (!last);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
1211297
InputStream downloadStream(String fileName) throws SnowflakeSQLException
  {
    if (stageInfo.getStageType() == StageInfo.StageType.LOCAL_FS)
    {
      logger.error("downloadStream function doesn't support local file system");

      throw new SnowflakeSQLException(SqlState.INTERNAL_ERROR,
                                      ErrorCode.INTERNAL_ERROR.getMessageCode(),
                                      "downloadStream function only supported in remote stages");
    }

    remoteLocation remoteLocation =
        extractLocationAndPath(stageInfo.getLocation());

    String stageFilePath = fileName;

    if (!remoteLocation.path.isEmpty())
    {
      stageFilePath = SnowflakeUtil.concatFilePathNames(remoteLocation.path,
                                                        fileName, "/");
    }

    RemoteStoreFileEncryptionMaterial encMat = srcFileToEncMat.get(fileName);

    return storageFactory.createClient(stageInfo, parallel, encMat)
        .downloadToStream(connection, command, parallel, remoteLocation.location,
                          stageFilePath, stageInfo.getRegion());
  }
1264505
public static ByteBuffer getByteBufferFromInputStreamAndCloseStream(InputStream is) throws IOException {
        try {
            // is.available() may return 0, or 1, or the total number of bytes in the stream,
            // or some other number.
            // Do not try to use is.available() == 0 to find the end of the stream!
            byte[] bytes;
            int avail = is.available();
            if (avail > 32) {
                // There are more bytes available than just the ICU data header length.
                // With luck, it is the total number of bytes.
                bytes = new byte[avail];
            } else {
                bytes = new byte[128];  // empty .res files are even smaller
            }
            // Call is.read(...) until one returns a negative value.
            int length = 0;
            for(;;) {
                if (length < bytes.length) {
                    int numRead = is.read(bytes, length, bytes.length - length);
                    if (numRead < 0) {
                        break;  // end of stream
                    }
                    length += numRead;
                } else {
                    // See if we are at the end of the stream before we grow the array.
                    int nextByte = is.read();
                    if (nextByte < 0) {
                        break;
                    }
                    int capacity = 2 * bytes.length;
                    if (capacity < 128) {
                        capacity = 128;
                    } else if (capacity < 0x4000) {
                        capacity *= 2;  // Grow faster until we reach 16kB.
                    }
                    // TODO Java 6 replace new byte[] and arraycopy(): bytes = Arrays.copyOf(bytes, capacity);
                    byte[] newBytes = new byte[capacity];
                    System.arraycopy(bytes, 0, newBytes, 0, length);
                    bytes = newBytes;
                    bytes[length++] = (byte) nextByte;
                }
            }
            return ByteBuffer.wrap(bytes, 0, length);
        } finally {
            is.close();
        }
    }
294960
public static InputStream decrypt(InputStream inputStream, byte[] key) {
        checkNotNull(inputStream);
        checkNotNull(key);
        AESFastEngine aesEngine = new AESFastEngine();
        PaddedBufferedBlockCipher cipher =
                new PaddedBufferedBlockCipher(aesEngine, new PKCS7Padding());
        cipher.init(false, new KeyParameter(key));
        return new CipherInputStream(inputStream, cipher);
    }
982023
public static CipherInputStream encrypt(StorageObjectMetadata meta,
                                          long originalContentLength,
                                          InputStream src,
                                          RemoteStoreFileEncryptionMaterial encMat,
                                          SnowflakeStorageClient client)
  throws InvalidKeyException,
         InvalidAlgorithmParameterException,
         NoSuchAlgorithmException,
         NoSuchProviderException,
         NoSuchPaddingException,
         FileNotFoundException,
         IllegalBlockSizeException,
         BadPaddingException
  {
    final byte[] decodedKey = Base64.decode(encMat.getQueryStageMasterKey());
    final int keySize = decodedKey.length;
    final byte[] fileKeyBytes = new byte[keySize];
    final byte[] ivData;
    final CipherInputStream cis;
    final int blockSz;
    {
      final Cipher fileCipher = Cipher.getInstance(FILE_CIPHER);
      blockSz = fileCipher.getBlockSize();

      // Create IV
      ivData = new byte[blockSz];
      getSecRnd().nextBytes(ivData);
      final IvParameterSpec iv = new IvParameterSpec(ivData);

      // Create file key
      getSecRnd().nextBytes(fileKeyBytes);
      SecretKey fileKey = new SecretKeySpec(fileKeyBytes, 0, keySize, AES);

      // Init cipher
      fileCipher.init(Cipher.ENCRYPT_MODE, fileKey, iv);

      // Create encrypting input stream
      cis = new CipherInputStream(src, fileCipher);
    }

    // Encrypt the file key with the QRMK
    {
      final Cipher keyCipher = Cipher.getInstance(KEY_CIPHER);
      SecretKey queryStageMasterKey =
          new SecretKeySpec(decodedKey, 0, keySize, AES);

      // Init cipher
      keyCipher.init(Cipher.ENCRYPT_MODE, queryStageMasterKey);
      byte[] encKeK = keyCipher.doFinal(fileKeyBytes);

      // Store metadata
      MatDesc matDesc =
          new MatDesc(encMat.getSmkId(), encMat.getQueryId(), keySize * 8);
      // Round up length to next multiple of the block size
      // Sizes that are multiples of the block size need to be padded to next
      // multiple
      long contentLength = ((originalContentLength + blockSz) / blockSz) * blockSz;
      client.addEncryptionMetadata(meta, matDesc, ivData, encKeK, contentLength);
    }

    return cis;
  }
465944
public static byte[] decompressGZIP(byte[] buffer) throws IOException {
        // Wrap the buffer in a ByteArrayInputStream and extract the decompressed pieces
        // into a ByteArrayOutputStream.
        ByteArrayInputStream bytesIn = new ByteArrayInputStream(buffer);
        GZIPInputStream gzipIn = new GZIPInputStream(bytesIn);
        ByteArrayOutputStream bytesBuffer = new ByteArrayOutputStream();
        byte[] chunkBuffer = new byte[65536];
        for (int bytesRead = gzipIn.read(chunkBuffer);
             bytesRead > 0;
             bytesRead = gzipIn.read(chunkBuffer)) {
            // Push this chunk into the ByteArrayOutputStream
            bytesBuffer.write(chunkBuffer, 0, bytesRead);
        }
        gzipIn.close();
        return bytesBuffer.toByteArray();
    }
877980
public static byte[] inputStreamAsByteArray(InputStream inputStream) {
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    try {
      byte[] buffer = new byte[16 * 1024];
      int read;
      while((read = inputStream.read(buffer)) > 0) {
        os.write(buffer, 0, read);
      }
      return os.toByteArray();
    } catch (IOException e) {
      throw LOG.unableToReadInputStream(e);
    }
    finally {
      closeSilently(inputStream);
    }
  }
1072956
InputStream getInputStream() throws IOException {
        switch (filetype) {
            case GZ:
                LOG.debug("Decompressing .gz file");
                return new GzipCompressorInputStream(new FileInputStream(compressedFile));
            case BZ2:
                LOG.debug("Decompressing .bz2 file");
                return new BZip2CompressorInputStream(new FileInputStream(compressedFile));
        }
        return null;
    }
1066868
public static byte[] encode(InputStream in) {
        Base64OutputStream out = null;
        try {
            ByteArrayOutputStream bytes = new ByteArrayOutputStream();
            out = new Base64OutputStream(bytes);
            byte[] buf = new byte[4096];
            int len;
            while ((len = in.read(buf)) > 0) {
                out.write(buf, 0, len);
            }
            out.flush();
            return bytes.toByteArray();
        } catch (IOException e) {
            return null;
        } finally {
            try {
                in.close();
                if (out != null) out.close();
            } catch (IOException e) {
                throw new FaultException(e);
            }
        }
    }
547055
public static void compressString(InputStream inputStream, OutputStream outputStream) throws IOException {
        try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
            fillExportStreamCompress(inputStream, byteArrayOutputStream);
            // to do -> make it in chunks
            String result = getStringFromEncode(byteArrayOutputStream.toByteArray());
            try (PrintWriter printWriter = new PrintWriter(outputStream)) {
                printWriter.write(result);
            }
        }
    }
780536
@Override
    public void compress(final InputStream in, final OutputStream out, final String targetName) {
        ZipOutputStream zos = new ZipOutputStream(out);
        ZipEntry ze = new ZipEntry(targetName);
        try {
            zos.putNextEntry(ze);
            int len;
            while ((len = in.read(buffer)) > 0) {
                zos.write(buffer, 0, len);
            }
        } catch (IOException e) {
            throw new Audit4jRuntimeException("Could not compress the given data", e);
        } finally {
            try {
                in.close();
            } catch (IOException e) {
                throw new Audit4jRuntimeException("Could not compress the given data", e);
            } finally {
                try {
                    zos.closeEntry();
                } catch (IOException e) {
                    throw new Audit4jRuntimeException("Could not compress the given data", e);
                } finally {
                    try {
                        zos.close();
                    } catch (IOException e) {
                        throw new Audit4jRuntimeException("Could not compress the given data", e);
                    }
                }

            }

        }

    }
347900
private InputStream decompressBody() {
        if (!decompress) {
            return body;
        }
        // if has no body, some server still set content-encoding header,
        // GZIPInputStream wrap empty input stream will cause exception. we should check this
        if (method.equals(Methods.HEAD)
                || (statusCode >= 100 && statusCode < 200) || statusCode == NOT_MODIFIED || statusCode == NO_CONTENT) {
            return body;
        }

        String contentEncoding = headers.getHeader(NAME_CONTENT_ENCODING);
        if (contentEncoding == null) {
            return body;
        }

        //we should remove the content-encoding header here?
        switch (contentEncoding) {
            case "gzip":
                try {
                    return new GZIPInputStream(body);
                } catch (IOException e) {
                    Closeables.closeQuietly(body);
                    throw new RequestsException(e);
                }
            case "deflate":
                // Note: deflate implements may or may not wrap in zlib due to rfc confusing.
                // here deal with deflate without zlib header
                return new InflaterInputStream(body, new Inflater(true));
            case "identity":
            case "compress": //historic; deprecated in most applications and replaced by gzip or deflate
            default:
                return body;
        }
    }
253987
public static byte[] inputStreamToBytes(InputStream in)
    {
        try
        {
            FastByteArrayOutputStream out = new FastByteArrayOutputStream(16384);
            transfer(in, out);
            return Arrays.copyOf(out.buffer, out.size);
        }
        catch (Exception e)
        {
            return null;
        }
    }
1132239
public static MD5Hash digest(InputStream in) throws IOException {
    final byte[] buffer = new byte[4*1024]; 
    int fileLength = 0;

    final MessageDigest digester = DIGESTER_FACTORY.get();
    for(int n; (n = in.read(buffer)) != -1; ) {
      digester.update(buffer, 0, n);
      fileLength += n;
    }

    return new MD5Hash(digester.digest(), fileLength);
  }