Repair a message, whatever its size is, by using chunking
17960
def decode(self, data, erase_pos=None, only_erasures=False):
        ''''''
        # erase_pos is a list of positions where you know (or greatly suspect at least) there is an erasure (ie, wrong character but you know it's at this position). Just input the list of all positions you know there are errors, and this method will automatically split the erasures positions to attach to the corresponding data chunk.
        if isinstance(data, str):
            data = bytearray(data, "latin-1")
        dec = bytearray()
        for i in xrange(0, len(data), self.nsize):
            # Split the long message in a chunk
            chunk = data[i:i+self.nsize]
            # Extract the erasures for this chunk
            e_pos = []
            if erase_pos:
                # First extract the erasures for this chunk (all erasures below the maximum chunk length)
                e_pos = [x for x in erase_pos if x <= self.nsize]
                # Then remove the extract erasures from the big list and also decrement all subsequent positions values by nsize (the current chunk's size) so as to prepare the correct alignment for the next iteration
                erase_pos = [x - (self.nsize+1) for x in erase_pos if x > self.nsize]
            # Decode/repair this chunk!
            dec.extend(rs_correct_msg(chunk, self.nsym, fcr=self.fcr, generator=self.generator, erase_pos=e_pos, only_erasures=only_erasures)[0])
        return dec
844319
def parse(self):
        

        if not self.message:
            return self

        # reset and start parsing
        self._reset()
        parts = []  # Normal parts plus defects

        # walk all mail parts to search defects
        for p in self.message.walk():
            part_content_type = p.get_content_type()
            self._append_defects(p, part_content_type)
            parts.append(p)

        # If defects are in epilogue defects get epilogue
        if self.defects_categories & EPILOGUE_DEFECTS:
            log.debug("Found defects in emails")
            epilogue = find_between(
                self.message.epilogue,
                "{}".format("--" + self.message.get_boundary()),
                "{}".format("--" + self.message.get_boundary() + "--"))

            try:
                p = email.message_from_string(epilogue)
                parts.append(p)
            except TypeError:
                log.debug("Failed to get epilogue part for TypeError")
            except Exception:
                log.error("Failed to get epilogue part. Check raw mail.")

        # walk all mail parts
        for i, p in enumerate(parts):
            if not p.is_multipart():
                filename = decode_header_part(p.get_filename())
                charset = p.get_content_charset('utf-8')
                charset_raw = p.get_content_charset()
                log.debug("Charset {!r} part {!r}".format(charset, i))

                if filename:
                    log.debug("Email part {!r} is an attachment".format(i))
                    log.debug("Filename {!r} part {!r}".format(filename, i))
                    binary = False
                    mail_content_type = ported_string(p.get_content_type())
                    log.debug("Mail content type {!r} part {!r}".format(
                        mail_content_type, i))
                    transfer_encoding = ported_string(
                        p.get('content-transfer-encoding', '')).lower()
                    log.debug("Transfer encoding {!r} part {!r}".format(
                        transfer_encoding, i))
                    content_id = ported_string(p.get('content-id'))
                    log.debug("content-id {!r} part {!r}".format(
                        content_id, i))

                    if transfer_encoding == "base64" or (
                       transfer_encoding == "quoted-\
                       printable" and "application" in mail_content_type):

                        payload = p.get_payload(decode=False)
                        binary = True
                        log.debug("Filename {!r} part {!r} is binary".format(
                            filename, i))
                    else:
                        payload = ported_string(
                            p.get_payload(decode=True), encoding=charset)
                        log.debug(
                            "Filename {!r} part {!r} is not binary".format(
                                filename, i))

                    self._attachments.append({
                        "filename": filename,
                        "payload": payload,
                        "binary": binary,
                        "mail_content_type": mail_content_type,
                        "content-id": content_id,
                        "charset": charset_raw,
                        "content_transfer_encoding": transfer_encoding})
                else:
                    log.debug("Email part {!r} is not an attachment".format(i))
                    payload = ported_string(
                        p.get_payload(decode=True), encoding=charset)
                    if payload:
                        if p.get_content_subtype() == 'html':
                            self._text_html.append(payload)
                        else:
                            self._text_plain.append(payload)
        else:
            # Parsed object mail with all parts
            self._mail = self._make_mail()

            # Parsed object mail with mains parts
            self._mail_partial = self._make_mail(complete=False)
717041
function( tmp ) {
      var _length = tmp.val.length;
      return _length === 0 || _length >= tmp.arg || messages.minLength.replace( '{count}', tmp.arg );
    }
979306
function formatMessage(message) {
  return message
    // Make some common errors shorter:
    .replace(
      // Babel syntax error
      'Module build failed: SyntaxError:',
      friendlySyntaxErrorLabel
    )
    .replace(
      // Webpack file not found error
      /Module not found: Error: Cannot resolve 'file' or 'directory'/,
      'Module not found:'
    )
    // Internal stacks are generally useless so we strip them
    .replace(/^\s*at\s.*:\d+:\d+[\s\)]*\n/gm, '') // at ... ...:x:y
    // Webpack loader names obscure CSS filenames
    .replace('./~/css-loader!./~/postcss-loader!', '');
}
752790
def from_full_message(commit_msg_str):
        
        all_lines = commit_msg_str.splitlines()
        try:
            cutline_index = all_lines.index(GitCommitMessage.CUTLINE)
        except ValueError:
            cutline_index = None
        lines = [line for line in all_lines[:cutline_index] if not line.startswith(GitCommitMessage.COMMENT_CHAR)]
        full = "\n".join(lines)
        title = lines[0] if lines else ""
        body = lines[1:] if len(lines) > 1 else []
        return GitCommitMessage(original=commit_msg_str, full=full, title=title, body=body)
1281931
def msg(self, target, message, formatted=True, tags=None):
        """"""
        if formatted:
            message = unescape(message)

        self.send('PRIVMSG', params=[target, message], source=self.nick, tags=tags)
1348038
def canonicalize(message):
    

    if message.is_multipart() \
            or message.get('Content-Transfer-Encoding') != 'binary':

        return mime_to_bytes(message, 0).replace(
            b'\r\n', b'\n').replace(b'\r', b'\n').replace(b'\n', b'\r\n')
    else:
        message_header = ''
        message_body = message.get_payload(decode=True)
        for k, v in message.items():
            message_header += '{}: {}\r\n'.format(k, v)
        message_header += '\r\n'
        return message_header.encode('utf-8') + message_body
548995
function digest(message) {
    return message.id || sha1(serializeNodes(message.nodes).join('') + ("[" + message.meaning + "]"));
}
290942
def _ReformatMessageString(self, message_string):
    
    def _PlaceHolderSpecifierReplacer(match_object):
      """Replaces message string place holders into Python format() style."""
      expanded_groups = []
      for group in match_object.groups():
        try:
          place_holder_number = int(group, 10) - 1
          expanded_group = '{{{0:d}:s}}'.format(place_holder_number)
        except ValueError:
          expanded_group = group

        expanded_groups.append(expanded_group)

      return ''.join(expanded_groups)

    if not message_string:
      return None

    message_string = self._WHITE_SPACE_SPECIFIER_RE.sub(r'', message_string)
    message_string = self._TEXT_SPECIFIER_RE.sub(r'\\\1', message_string)
    message_string = self._CURLY_BRACKETS.sub(r'\1\1', message_string)
    return self._PLACE_HOLDER_SPECIFIER_RE.sub(
        _PlaceHolderSpecifierReplacer, message_string)
41549
function processMessage(conn, message) {
  const msgHeader = parseHeader(message);
  if (msgHeader.opCode !== OP_COMPRESSED) {
    const ResponseConstructor = msgHeader.opCode === OP_MSG ? BinMsg : Response;
    conn.emit(
      'message',
      new ResponseConstructor(
        conn.bson,
        message,
        msgHeader,
        message.slice(MESSAGE_HEADER_SIZE),
        conn.responseOptions
      ),
      conn
    );

    return;
  }

  msgHeader.fromCompressed = true;
  let index = MESSAGE_HEADER_SIZE;
  msgHeader.opCode = message.readInt32LE(index);
  index += 4;
  msgHeader.length = message.readInt32LE(index);
  index += 4;
  const compressorID = message[index];
  index++;

  decompress(compressorID, message.slice(index), (err, decompressedMsgBody) => {
    if (err) {
      conn.emit('error', err);
      return;
    }

    if (decompressedMsgBody.length !== msgHeader.length) {
      conn.emit(
        'error',
        new MongoError(
          'Decompressing a compressed message from the server failed. The message is corrupt.'
        )
      );

      return;
    }

    const ResponseConstructor = msgHeader.opCode === OP_MSG ? BinMsg : Response;
    conn.emit(
      'message',
      new ResponseConstructor(
        conn.bson,
        message,
        msgHeader,
        decompressedMsgBody,
        conn.responseOptions
      ),
      conn
    );
  });
}
1091718
def clean_message(message: Message, topmost: bool = False) -> Message:
    
    if message.is_multipart():
        # Don't recurse in already-deleted attachments
        if message.get_content_type() != 'message/external-body':
            parts = message.get_payload()
            parts[:] = map(clean_message, parts)
    elif message_is_binary(message):
        # Don't gut if this is the topmost message
        if not topmost:
            message = gut_message(message)

    return message
806952
function _transform(chunk, encoding, cb) {

  this.emit('lines', chunk);

  try {
    this.parse(chunk);
  }catch(e) {
    this.emit('error', e);
  }

  cb();
}
1307064
function teardownChunk(chunk, encoding) {
	if (!(chunk instanceof Buffer)) {
		chunk = new Buffer(chunk);
	}
	if (encoding == 'gbk') {
		chunk = iconv.encode(iconv.decode(chunk, 'utf8'), 'gbk');
	}
	return chunk;
}
1277272
def raw_messages(self):
        ''''''
        if self.data:
            # offset for first message, at end of Mail data file header
            last_offset = 24
            self.skipped_chunks = 0

            for msginfo in self.index.messages:
                msg = self.data.get_message(msginfo.offset, msginfo.size)
                # Index file seems to references messages in order by
                # offset; check for data skipped between messages.
                if msginfo.offset > last_offset:
                    logger.debug('Skipped %d bytes between %s (%s) and %s (%s)',
                           msginfo.offset - last_offset,
                           last_offset, hex(last_offset),
                           msginfo.offset, hex(msginfo.offset))

                    self.skipped_chunks += 1
                last_offset = msginfo.offset + msginfo.size

                yield msg
281449
def messages(count, size):
    ''''''
    import string
    # Make sure we have at least 'size' letters
    letters = islice(cycle(chain(string.lowercase, string.uppercase)), size)
    return islice(cycle(''.join(l) for l in permutations(letters, size)), count)
467737
function(self, message) {
  var msgHeader = parseHeader(message);
  if (msgHeader.opCode === OP_COMPRESSED) {
    msgHeader.fromCompressed = true;
    var index = MESSAGE_HEADER_SIZE;
    msgHeader.opCode = message.readInt32LE(index);
    index += 4;
    msgHeader.length = message.readInt32LE(index);
    index += 4;
    var compressorID = message[index];
    index++;
    decompress(compressorID, message.slice(index), function(err, decompressedMsgBody) {
      if (err) {
        throw err;
      }
      if (decompressedMsgBody.length !== msgHeader.length) {
        throw new Error(
          'Decompressing a compressed message from the server failed. The message is corrupt.'
        );
      }
      self.messageHandler(
        new Response(self.bson, message, msgHeader, decompressedMsgBody, self.responseOptions),
        self
      );
    });
  } else {
    self.messageHandler(
      new Response(
        self.bson,
        message,
        msgHeader,
        message.slice(MESSAGE_HEADER_SIZE),
        self.responseOptions
      ),
      self
    );
  }
}
927122
def each_message_chunk
      LOG 'reading message...'
      LOG_off()
      read_bytes = 0
      while (line = readuntil("\r\n")) != ".\r\n"
        read_bytes += line.size
        yield line.sub(/\A\./, '')
      end
      LOG_on()
      LOG "read message (#{read_bytes} bytes)"
    end