Transform stream parses JSON strings and base64 decodes into buffers
163411
function Parser() {
  var parser = new stream.Transform({ highWaterMark: 100 });
  parser._writableState.objectMode = false;
  parser._readableState.objectMode = true;

  var firstline = true;
  parser._transform = function(record, enc, callback) {
    if (!record || record.length === 0) return;

    if (firstline) {
      firstline = false;

      var parsed = Dyno.deserialize(record);
      if (!Object.keys(parsed).every(function(key) {
        return !!parsed[key];
      })) return this.push(JSON.parse(record.toString()));
    }

    record = Dyno.deserialize(record);

    this.push(record);
    setImmediate(callback);
  };

  return parser;
}
894562
function __deserializeObjectStreaming(buffer, start, options) {
	let end = start;
	let dataView = new DataView(buffer);
	let endData = new Uint16Array(TYPE_HEADER.OBJECT_END);
	let data = {};
	while (start < buffer.byteLength) {
		let tmpType = dataView.getUint16(start, true);
		if (tmpType === endData[0]) {
			end += 2;
			break;
		}

		let subType, subKey, subData;
		({ anchor: start, value: subType } = __deserializeType(buffer, start, options));
		({ anchor: start, value: subKey } = __deserializeShortString(buffer, start, options));
		({ anchor: start, value: subData } = __deserializeData(subType, buffer, start, options));
		data[subKey] = subData;
		end = start;
	}
	return { anchor: end, value: data };
}
239650
function base64toBlob(base64Data, contentType, sliceSize) {
        contentType = contentType || '';
        sliceSize = sliceSize || 512;

        var byteCharacters = atob(base64Data);
        var byteArrays = [];

        for (var offset = 0; offset < byteCharacters.length; offset += sliceSize) {
            var slice = byteCharacters.slice(offset, offset + sliceSize);
            var byteNumbers = new Array(slice.length);
            for (var i = 0; i < slice.length; i++) {
                byteNumbers[i] = slice.charCodeAt(i);
            }
            var byteArray = new Uint8Array(byteNumbers);
            byteArrays.push(byteArray);
        }

        return new Blob(byteArrays, {type: contentType});
    }
856313
function decode(object) {
        if (defaults.decodeBase64 && object && object.content > '' &&
            object.encoding && object.encoding === 'base64') {
            object.content = b64decode(object.content)
        }
        return object
    }
211689
function MetaCharsetReplacerStream(options) {
    options = options || {};
    this.encoding = options.encoding = 'utf8'; // this is the *output* encoding
    options.decodeStrings = false; // don't turn my strings back into a buffer!
    Transform.call(this, options);
}
153362
function dataURItoBlob(dataURI) {
    // convert base64 to raw binary data held in a string
    var byteString = atob(dataURI.split(',')[1]),
        // separate out the mime component
        mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0],
        // write the bytes of the string to an ArrayBuffer
        arrayBuffer = new ArrayBuffer(byteString.length),
        _ia = new Uint8Array(arrayBuffer),
        i, dataView, blob;

    for (i = 0; i < byteString.length; i++) {
        _ia[i] = byteString.charCodeAt(i);
    }

    dataView = new DataView(arrayBuffer);
    blob = new Blob([dataView], { type: mimeString });

    return blob;
}
604826
function bytes2base64(buffer) {
            var str = '';
            var length;
            var i;
            // ArrayBuffer
            if (buffer instanceof ArrayBuffer) {
                length = buffer.byteLength;
                var view = new DataView(buffer, 0, length);
                for (i = 0; i < length; i++) {
                    str += String.fromCharCode(view.getUint8(i, false));
                }
            }
            // Array
            else if (buffer.length) {
                length = buffer.length;
                for (i = 0; i < length; i++) {
                    str += String.fromCharCode(buffer[i]);
                }
            }

            return btoa(str);
        }
491001
function decodeJson(basePath, value) {
    var data = {},
        path = basePath,       // [TODO] bn is optional, so we need basePath to check type
        pathType = cutils.getPathDateType(path),
        oid = cutils.getPathIdKey(path).oid,
        rid;

    if (_.isBuffer(value) || _.isString(value))
        value = JSON.parse(value);

    if (!_.isPlainObject(value) || !value.e) 
        throw new TypeError('value should be an object.');

    switch (pathType) {
        case 'object':         // obj
            _.forEach(value.e, function (resrc) {
                var pathArray = cutils.getPathArray(resrc.n),          // [iid, rid[, riid]]
                    val;

                if (!_.isUndefined(resrc.v)) {
                    val = resrc.v;
                } else if (!_.isUndefined(resrc.sv)) {
                    val = resrc.sv;
                } else if (!_.isUndefined(resrc.bv)) {
                    val = resrc.bv;
                } else if (!_.isUndefined(resrc.ov)) {
                    val = resrc.ov;     // [TODO] objlnk
                }

                if (pathArray[0] === '')
                    pathArray = pathArray.slice(1);

                if (pathArray[pathArray.length - 1] === '')
                    pathArray = pathArray.slice(0, pathArray.length - 1);

                if (pathArray[0] && !_.has(data, pathArray[0]))
                    data[pathArray[0]] = {};

                rid = cutils.ridKey(oid, pathArray[1]);

                if (rid && !_.has(data, [pathArray[0], rid])) {
                    if (pathArray[2]) {
                        data[pathArray[0]][rid] = {};
                        data[pathArray[0]][rid][pathArray[2]] = val;
                    } else {
                        data[pathArray[0]][rid] = val;
                    }
                }
            });
            break;

        case 'instance':         // inst
            _.forEach(value.e, function (resrc) {
                var pathArray = cutils.getPathArray(resrc.n),          // [rid[, riid]]
                    val;

                if (!_.isUndefined(resrc.v)) {
                    val = resrc.v;
                } else if (!_.isUndefined(resrc.sv)) {
                    val = resrc.sv;
                } else if (!_.isUndefined(resrc.bv)) {
                    val = resrc.bv;
                } else if (!_.isUndefined(resrc.ov)) {
                    val = resrc.ov;     // [TODO] objlnk
                }

                if (pathArray[0] === '')
                    pathArray = pathArray.slice(1);

                if (pathArray[pathArray.length - 1] === '')
                    pathArray = pathArray.slice(0, pathArray.length - 1);

                rid = cutils.ridKey(oid, pathArray[0]);

                if (rid && !_.has(data, rid)) {
                    if (pathArray[1]) {
                        data[rid] = {};
                        data[rid][pathArray[1]] = val;
                    } else {
                        data[rid] = val;
                    }
                }
            });
            break;

        case 'resource':         // resrc
            _.forEach(value.e, function (resrc) {
                var riid = resrc.n,          // [[riid]]
                    isMultiResrc = false,
                    val;

                if (riid)
                    isMultiResrc = true;

                if (!_.isUndefined(resrc.v)) {
                    val = resrc.v;
                } else if (!_.isUndefined(resrc.sv)) {
                    val = resrc.sv;
                } else if (!_.isUndefined(resrc.bv)) {
                    val = resrc.bv;
                } else if (!_.isUndefined(resrc.ov)) {
                    val = resrc.ov;     // [TODO] objlnk
                }

                if (!isMultiResrc) {
                    data = val;
                } else if (isMultiResrc && !_.has(data, riid)) {
                    data[riid] = val;
                }
            });
            break;

        default:
            break;
     }
    
    return data;
}
1023914
function __serializeObjectStreaming(data, options=DEFAULT_OPTIONS) {
	let dataBuffers = [];
	let allKeys = (options.sort_key === true) ? Object.keys(data).sort() : Object.keys(data);
	// ignore undefined value
	for (let key of allKeys) {
		let subData = data[key];
		if (subData === undefined) continue;

		let subType = __getType(subData, options);
		let subTypeBuffer = __serializeType(subType);
		let keyBuffers = __serializeShortString(key);
		let subDataBuffers = __serializeData(subType, subData, options);
		dataBuffers.push(subTypeBuffer, ...keyBuffers, ...subDataBuffers);
	}
	return [...dataBuffers, TYPE_HEADER.OBJECT_END];
}
947871
function b64encode(str) {
    const buf = Buffer.isBuffer(str) ? str : Buffer.from(str);
    return b64escape(buf.toString('base64'));
}
1229715
def decode_bytes(string):
    

    if is_string_type(type(string)):
        string = bytes(string, "utf-8")
    return base64.decodebytes(string)
424915
def parse(self, stream, media_type=None, parser_context=None):
        
        parser_context = parser_context or {}
        encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET)
        data = QueryDict(stream.read(), encoding=encoding)
        return data
1075290
function __serializeArrayStreaming(data, options=DEFAULT_OPTIONS) {
	let dataBuffers = [];
	// ignore undefined value
	for (let key in data) {
		let subData = data[key];
		let subType = __getType(subData, options);
		let subTypeBuffer = __serializeType(subType);
		let subDataBuffers = __serializeData(subType, subData, options);
		dataBuffers.push(subTypeBuffer, ...subDataBuffers);
	}
	return [...dataBuffers, TYPE_HEADER.ARRAY_END];
}
165498
function readStream (stream, encoding, length, limit, callback) {
  var complete = false
  var sync = true

  // check the length and limit options.
  // note: we intentionally leave the stream paused,
  // so users should handle the stream themselves.
  if (limit !== null && length !== null && length > limit) {
    return done(createError(413, 'request entity too large', {
      expected: length,
      length: length,
      limit: limit,
      type: 'entity.too.large'
    }))
  }

  // streams1: assert request encoding is buffer.
  // streams2+: assert the stream encoding is buffer.
  //   stream._decoder: streams1
  //   state.encoding: streams2
  //   state.decoder: streams2, specifically < 0.10.6
  var state = stream._readableState
  if (stream._decoder || (state && (state.encoding || state.decoder))) {
    // developer error
    return done(createError(500, 'stream encoding should not be set', {
      type: 'stream.encoding.set'
    }))
  }

  var received = 0
  var decoder

  try {
    decoder = getDecoder(encoding)
  } catch (err) {
    return done(err)
  }

  var buffer = decoder
    ? ''
    : []

  // attach listeners
  stream.on('aborted', onAborted)
  stream.on('close', cleanup)
  stream.on('data', onData)
  stream.on('end', onEnd)
  stream.on('error', onEnd)

  // mark sync section complete
  sync = false

  function done () {
    var args = new Array(arguments.length)

    // copy arguments
    for (var i = 0; i < args.length; i++) {
      args[i] = arguments[i]
    }

    // mark complete
    complete = true

    if (sync) {
      process.nextTick(invokeCallback)
    } else {
      invokeCallback()
    }

    function invokeCallback () {
      cleanup()

      if (args[0]) {
        // halt the stream on error
        halt(stream)
      }

      callback.apply(null, args)
    }
  }

  function onAborted () {
    if (complete) return

    done(createError(400, 'request aborted', {
      code: 'ECONNABORTED',
      expected: length,
      length: length,
      received: received,
      type: 'request.aborted'
    }))
  }

  function onData (chunk) {
    if (complete) return

    received += chunk.length

    if (limit !== null && received > limit) {
      done(createError(413, 'request entity too large', {
        limit: limit,
        received: received,
        type: 'entity.too.large'
      }))
    } else if (decoder) {
      buffer += decoder.write(chunk)
    } else {
      buffer.push(chunk)
    }
  }

  function onEnd (err) {
    if (complete) return
    if (err) return done(err)

    if (length !== null && received !== length) {
      done(createError(400, 'request size did not match content length', {
        expected: length,
        length: length,
        received: received,
        type: 'request.size.invalid'
      }))
    } else {
      var string = decoder
        ? buffer + (decoder.end() || '')
        : Buffer.concat(buffer)
      done(null, string)
    }
  }

  function cleanup () {
    buffer = null

    stream.removeListener('aborted', onAborted)
    stream.removeListener('data', onData)
    stream.removeListener('end', onEnd)
    stream.removeListener('error', onEnd)
    stream.removeListener('close', cleanup)
  }
}
343911
function extractNestedStreams(obj) {
  return flatten(
    Object.keys(obj).map(key => {
      // DEPRECATED I can't think of a usecase
      // if (typeof obj[key] === 'object') {
      // 	return extractNestedStreams(obj[key]);
      // }
      if (obj[key] === null || obj[key] === undefined) {
        return [];
      }
      if (isStream(obj[key])) {
        return [
          {
            parent: obj,
            key,
            stream: obj[key]
          }
        ];
      }
      if (typeof obj[key] === "object") {
        return extractNestedStreams(obj[key]);
      }
      return [];
    })
  );
}
857192
function base64ArrayBuffer(bytes) {
    var base64    = ''
    var encodings = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'

    var byteLength    = bytes.byteLength
    var byteRemainder = byteLength % 3
    var mainLength    = byteLength - byteRemainder
   
    var a, b, c, d
    var chunk
   
    // Main loop deals with bytes in chunks of 3
    for (var i = 0; i < mainLength; i = i + 3) {
      // Combine the three bytes into a single integer
      chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2]
   
      // Use bitmasks to extract 6-bit segments from the triplet
      a = (chunk & 16515072) >> 18 // 16515072 = (2^6 - 1) << 18
      b = (chunk & 258048)   >> 12 // 258048   = (2^6 - 1) << 12
      c = (chunk & 4032)     >>  6 // 4032     = (2^6 - 1) << 6
      d = chunk & 63               // 63       = 2^6 - 1
   
      // Convert the raw binary segments to the appropriate ASCII encoding
      base64 += encodings[a] + encodings[b] + encodings[c] + encodings[d]
    }
   
    // Deal with the remaining bytes and padding
    if (byteRemainder == 1) {
      chunk = bytes[mainLength]
   
      a = (chunk & 252) >> 2 // 252 = (2^6 - 1) << 2
   
      // Set the 4 least significant bits to zero
      b = (chunk & 3)   << 4 // 3   = 2^2 - 1
   
      base64 += encodings[a] + encodings[b] + '=='
    } else if (byteRemainder == 2) {
      chunk = (bytes[mainLength] << 8) | bytes[mainLength + 1]
   
      a = (chunk & 64512) >> 10 // 64512 = (2^6 - 1) << 10
      b = (chunk & 1008)  >>  4 // 1008  = (2^6 - 1) << 4
   
      // Set the 2 least significant bits to zero
      c = (chunk & 15)    <<  2 // 15    = 2^4 - 1
   
      base64 += encodings[a] + encodings[b] + encodings[c] + '='
    }
    
    return base64
  }
564574
def parse_stream(response):
    
    stream_data = []
    stream = stdout

    for data in response:
        if data:
            try:
                data = data.decode('utf-8')
            except AttributeError as e:
                logger.exception("Unable to parse stream, Attribute Error Raised: {0}".format(e))
                stream.write(data)
                continue

            try:
                normalized_data = normalize_keys(json.loads(data))
            except ValueError:
                stream.write(data)
                continue
            except TypeError:
                stream.write(data)
                continue

            if 'progress' in normalized_data:
                stream_data.append(normalized_data)
                _display_progress(normalized_data, stream)

            elif 'error' in normalized_data:
                _display_error(normalized_data, stream)

            elif 'status' in normalized_data:
                stream_data.append(normalized_data)
                _display_status(normalized_data, stream)

            elif 'stream' in normalized_data:
                stream_data.append(normalized_data)
                _display_stream(normalized_data, stream)
            else:
                stream.write(data)

    stream.flush()
    return stream_data