Take a dict-like fedmsg envelope and store the headers and message
        in the table.
359304
def add(envelope):
    
    message = envelope['body']
    timestamp = message.get('timestamp', None)
    try:
        if timestamp:
            timestamp = datetime.datetime.utcfromtimestamp(timestamp)
        else:
            timestamp = datetime.datetime.utcnow()
    except Exception:
        pass

    headers = envelope.get('headers', None)
    msg_id = message.get('msg_id', None)
    if not msg_id and headers:
        msg_id = headers.get('message-id', None)
    if not msg_id:
        msg_id = six.text_type(timestamp.year) + six.u('-') + six.text_type(uuid.uuid4())
    obj = Message(
        i=message.get('i', 0),
        msg_id=msg_id,
        topic=message['topic'],
        timestamp=timestamp,
        username=message.get('username', None),
        crypto=message.get('crypto', None),
        certificate=message.get('certificate', None),
        signature=message.get('signature', None),
    )

    obj.msg = message['msg']
    obj.headers = headers

    try:
        session.add(obj)
        session.flush()
    except IntegrityError:
        log.warning('Skipping message from %s with duplicate id: %s',
                    message['topic'], msg_id)
        session.rollback()
        return

    usernames = fedmsg.meta.msg2usernames(message)
    packages = fedmsg.meta.msg2packages(message)

    # Do a little sanity checking on fedmsg.meta results
    if None in usernames:
        # Notify developers so they can fix msg2usernames
        log.error('NoneType found in usernames of %r' % msg_id)
        # And prune out the bad value
        usernames = [name for name in usernames if name is not None]

    if None in packages:
        # Notify developers so they can fix msg2packages
        log.error('NoneType found in packages of %r' % msg_id)
        # And prune out the bad value
        packages = [pkg for pkg in packages if pkg is not None]

    # If we've never seen one of these users before, then:
    # 1) make sure they exist in the db (create them if necessary)
    # 2) mark an in memory cache so we can remember that they exist without
    #    having to hit the db.
    for username in usernames:
        if username not in _users_seen:
            # Create the user in the DB if necessary
            User.get_or_create(username)
            # Then just mark an in memory cache noting that we've seen them.
            _users_seen.add(username)

    for package in packages:
        if package not in _packages_seen:
            Package.get_or_create(package)
            _packages_seen.add(package)

    session.flush()

    # These two blocks would normally be a simple "obj.users.append(user)" kind
    # of statement, but here we drop down out of sqlalchemy's ORM and into the
    # sql abstraction in order to gain a little performance boost.
    values = [{'username': username, 'msg': obj.id} for username in usernames]
    if values:
        session.execute(user_assoc_table.insert(), values)

    values = [{'package': package, 'msg': obj.id} for package in packages]
    if values:
        session.execute(pack_assoc_table.insert(), values)

    # TODO -- can we avoid committing every time?
    session.flush()
    session.commit()
398771
def mk_set_headers(self, data, columns):
        
        columns = tuple(columns)
        lens = []

        for key in columns:
            value_len = max(len(str(each.get(key, ''))) for each in data)
            # account for header lengths
            lens.append(max(value_len, len(self._get_name(key))))

        fmt = self.mk_fmt(*lens)
        return fmt
1339043
def generate_messages_info(msgs):
    """"""

    for msg in msgs:
        msg.swift_name = camel_case_from_underscores(msg.name)

        msg.formatted_description = ""
        if msg.description:
            msg.description = " ".join(msg.description.split())
            msg.formatted_description = "\n/**\n    %s\n*/\n" % " ".join(msg.description.split())
        msg.message_description = msg.description.replace('"','\\"')

        for field in msg.ordered_fields:
            field.swift_name = lower_camel_case_from_underscores(field.name)
            field.return_type = swift_types[field.type][0]
            
            # configure fields initializers
            if field.enum:
                # handle enums
                field.return_type = camel_case_from_underscores(field.enum)
                field.initial_value = "try data.mavEnumeration(offset: %u)" % field.wire_offset
            elif field.array_length > 0:
                if field.return_type == "String":
                    # handle strings
                    field.initial_value = "data." + swift_types[field.type][2] % (field.wire_offset, field.array_length)
                else:
                    # other array types
                    field.return_type = "[%s]" % field.return_type
                    field.initial_value = "try data.mavArray(offset: %u, count: %u)" % (field.wire_offset, field.array_length)
            else:
                # simple type field
                field.initial_value = "try data." + swift_types[field.type][2] % field.wire_offset
            
            field.formatted_description = ""
            if field.description:
                field.description = " ".join(field.description.split())
                field.formatted_description = "\n\t/// " + field.description + "\n"
         
        fields_info = map(lambda field: '("%s", %u, "%s", "%s")' % (field.swift_name, field.wire_offset, field.return_type, field.description.replace('"','\\"')), msg.fields)
        msg.fields_info = ", ".join(fields_info)

    msgs.sort(key = lambda msg : msg.id)
1313005
def toDict(self, msg: Dict) -> Dict:
        

        if isinstance(msg, Request):
            tmsg = msg.as_dict
        elif hasattr(msg, "_asdict"):
            tmsg = dict(msg._asdict())
        elif hasattr(msg, "__dict__"):
            tmsg = dict(msg.__dict__)
        elif self.allowDictOnly:
            raise ValueError("Message cannot be converted to an appropriate "
                             "format for transmission")
        else:
            tmsg = msg
        return tmsg
213510
def store_magic_envelope_doc(self, payload):
        """"""
        try:
            json_payload = json.loads(decode_if_bytes(payload))
        except ValueError:
            # XML payload
            xml = unquote(decode_if_bytes(payload))
            xml = xml.lstrip().encode("utf-8")
            logger.debug("diaspora.protocol.store_magic_envelope_doc: xml payload: %s", xml)
            self.doc = etree.fromstring(xml)
        else:
            logger.debug("diaspora.protocol.store_magic_envelope_doc: json payload: %s", json_payload)
            self.doc = self.get_json_payload_magic_envelope(json_payload)
807320
def _structure(msg, fp=None, level=0, include_default=False):
    """"""
    if fp is None:
        fp = sys.stdout
    tab = ' ' * (level * 4)
    print(tab + msg.get_content_type(), end='', file=fp)
    if include_default:
        print(' [%s]' % msg.get_default_type(), file=fp)
    else:
        print(file=fp)
    if msg.is_multipart():
        for subpart in msg.get_payload():
            _structure(subpart, fp, level+1, include_default)
691004
def headers(self, serialize_cell=None):
    """"""
    if serialize_cell is None:
      serialize_cell = self.get_cell_value
    return [serialize_cell(cell) for cell in self.rows[0]]
1120389
def serialize(message):
        
        fmt = "!BBH"

        if message.token is None or message.token == "":
            tkl = 0
        else:
            tkl = len(message.token)
        tmp = (defines.VERSION << 2)
        tmp |= message.type
        tmp <<= 4
        tmp |= tkl

        values = [tmp, message.code, message.mid]

        if message.token is not None and tkl > 0:

            for b in str(message.token):
                fmt += "c"
                values.append(bytes(b, "utf-8"))

        options = Serializer.as_sorted_list(message.options)  # already sorted
        lastoptionnumber = 0
        for option in options:

            # write 4-bit option delta
            optiondelta = option.number - lastoptionnumber
            optiondeltanibble = Serializer.get_option_nibble(optiondelta)
            tmp = (optiondeltanibble << defines.OPTION_DELTA_BITS)

            # write 4-bit option length
            optionlength = option.length
            optionlengthnibble = Serializer.get_option_nibble(optionlength)
            tmp |= optionlengthnibble
            fmt += "B"
            values.append(tmp)

            # write extended option delta field (0 - 2 bytes)
            if optiondeltanibble == 13:
                fmt += "B"
                values.append(optiondelta - 13)
            elif optiondeltanibble == 14:
                fmt += "H"
                values.append(optiondelta - 269)

            # write extended option length field (0 - 2 bytes)
            if optionlengthnibble == 13:
                fmt += "B"
                values.append(optionlength - 13)
            elif optionlengthnibble == 14:
                fmt += "H"
                values.append(optionlength - 269)

            # write option value
            if optionlength > 0:
                opt_type = defines.OptionRegistry.LIST[option.number].value_type
                if opt_type == defines.INTEGER:
                    words = Serializer.int_to_words(option.value, optionlength, 8)
                    for num in range(0, optionlength):
                        fmt += "B"
                        values.append(words[num])
                elif opt_type == defines.STRING:
                    fmt += str(len(bytes(option.value, "utf-8"))) + "s"
                    values.append(bytes(option.value, "utf-8"))

                else:  # OPAQUE
                    for b in option.value:
                        fmt += "B"
                        values.append(b)

            # update last option number
            lastoptionnumber = option.number

        payload = message.payload

        if payload is not None and len(payload) > 0:
            # if payload is present and of non-zero length, it is prefixed by
            # an one-byte Payload Marker (0xFF) which indicates the end of
            # options and the start of the payload

            fmt += "B"
            values.append(defines.PAYLOAD_MARKER)

            if isinstance(payload, bytes):
                fmt += str(len(payload)) + "s"
                values.append(payload)
            else:
                fmt += str(len(bytes(payload, "utf-8"))) + "s"
                values.append(bytes(payload, "utf-8"))
            # for b in str(payload):
            #     fmt += "c"
            #     values.append(bytes(b, "utf-8"))

        datagram = None
        if values[1] is None:
            values[1] = 0
        if values[2] is None:
            values[2] = 0
        try:
            s = struct.Struct(fmt)
            datagram = ctypes.create_string_buffer(s.size)
            s.pack_into(datagram, 0, *values)
        except struct.error:
            # The .exception method will report on the exception encountered
            # and provide a traceback.
            logger.debug(fmt)
            logger.debug(values)
            logging.exception('Failed to pack structure')

        return datagram
986
def as_dict(self):
        
        data = super(BaseEmail, self).as_dict()
        data["Headers"] = [{"Name": name, "Value": value} for name, value in data["Headers"].items()]
        for field in ("To", "Cc", "Bcc"):
            if field in data:
                data[field] = list_to_csv(data[field])
        data["Attachments"] = [prepare_attachments(attachment) for attachment in data["Attachments"]]
        return data
250724
def serialize_dict(self, msg_dict):
        """"""
        serialized = json.dumps(msg_dict, namedtuple_as_object=False)
        if PY2:
            serialized = serialized.decode("utf-8")
        serialized = "{}\nend\n".format(serialized)
        return serialized
627742
def pack_msg(method, msg, pickle_protocol=PICKLE_PROTOCOL):
    """"""
    dump = io.BytesIO()
    pickle.dump(msg, dump, pickle_protocol)
    size = dump.tell()
    return (struct.pack(METHOD_STRUCT_FORMAT, method) +
            struct.pack(SIZE_STRUCT_FORMAT, size) + dump.getvalue())
385207
def FromDictionary(cls, msg_dict):
        

        level = msg_dict.get('level')
        msg = msg_dict.get('message')
        now = msg_dict.get('now_time')
        created = msg_dict.get('created_time')
        count = msg_dict.get('count', 1)
        msg_id = msg_dict.get('id', 0)

        new_msg = ServiceMessage(level, msg, msg_id, created, now)
        if count > 1:
            new_msg.count = count

        return new_msg
681416
def write_table(mboxfile, mailTable):
    
    mail_box_contents = mailbox.mbox(mboxfile)

    m_pbar = tqdm.tqdm(range(0,len(mail_box_contents)))
    m_pbar.set_description('Extracting mbox messages...')

    count = 0
    update_interval = min(50,len(mail_box_contents))

    for message in mail_box_contents:
        count += 1
        if count % update_interval == 0:
            m_pbar.update(update_interval)
        clean_from = clean_address(message['From'])
        clean_to = clean_addresses(message['To'])
        clean_cc = clean_addresses(message['Cc'])

        try:
            clean_date = email.parsedate_to_datetime(message['Date'])
        except:
            clean_date = None

        mailTable.append([
            clean_from,
            clean_to,
            clean_cc,
            clean_date,
            message['Subject'],
            get_body(message)
            ])
848505
async def message_data(request):
    
    messages = []
    # {% if database.is_none %}
    if request.app['settings'].MESSAGE_FILE.exists():
        # read the message file, process it and populate the "messages" list
        with request.app['settings'].MESSAGE_FILE.open() as msg_file:
            for line in msg_file:
                if not line:
                    # ignore blank lines eg. end of file
                    continue
                # split the line into it constituent parts, see process_form above
                username, ts, message = line.split('|', 2)
                # parse the datetime string and render it in a more readable format.
                ts = '{:%Y-%m-%d %H:%M:%S}'.format(datetime.strptime(ts, '%Y-%m-%dT%H:%M:%S.%f'))
                messages.append({'username': username, 'timestamp': ts, 'message': message})
        messages.reverse()
    # {% elif database.is_pg_sqlalchemy %}

    async with request.app['pg_engine'].acquire() as conn:
        async for row in conn.execute(sa_messages.select().order_by(sa_messages.c.timestamp.desc())):
            ts = '{:%Y-%m-%d %H:%M:%S}'.format(row.timestamp)
            messages.append({'username': row.username, 'timestamp': ts, 'message': row.message})
    # {% endif %}
    return json_response(messages)
437933
def header(msg):
    
    # Accounting for '| ' and ' |'
    width = len(msg) + 4
    s = []
    s.append('-' * width)
    s.append("| %s |" % msg)
    s.append('-' * width)
    return '\n'.join(s)
374011
def prepare_message(user, useralias, addresses, subject, contents, attachments, headers, encoding, newline_to_break=True):
    # check if closed!!!!!! XXX
    
    if isinstance(contents, text_type):
        contents = [contents]
    if isinstance(attachments, text_type):
        attachments = [attachments]

    # merge contents and attachments for now.
    if attachments is not None:
        for a in attachments:
            if not os.path.isfile(a):
                raise TypeError("'{0}' is not a valid filepath".format(a))
        contents = attachments if contents is None else contents + attachments

    has_included_images, content_objects = prepare_contents(contents, encoding)
    msg = MIMEMultipart()
    if headers is not None:
        # Strangely, msg does not have an update method, so then manually.
        for k, v in headers.items():
            msg[k] = v
    if headers is None or "Date" not in headers:
        msg["Date"] = formatdate()

    msg_alternative = MIMEMultipart("alternative")
    msg_related = MIMEMultipart("related")
    msg_related.attach("-- HTML goes here --")
    msg.attach(msg_alternative)
    add_subject(msg, subject)
    add_recipients_headers(user, useralias, msg, addresses)
    htmlstr = ""
    altstr = []
    if has_included_images:
        msg.preamble = "This message is best displayed using a MIME capable email reader."

    if contents is not None:
        for content_object, content_string in zip(content_objects, contents):
            if content_object["main_type"] == "image":
                # all image objects need base64 encoding, so do it now
                email.encoders.encode_base64(content_object["mime_object"])
                # aliased image {'path' : 'alias'}
                if isinstance(content_string, dict) and len(content_string) == 1:
                    for key in content_string:
                        hashed_ref = str(abs(hash(key)))
                        alias = content_string[key]
                    # pylint: disable=undefined-loop-variable
                    content_string = key
                else:
                    alias = os.path.basename(str(content_string))
                    hashed_ref = str(abs(hash(alias)))

                # TODO: I should probably remove inline now that there is "attachments"
                # if string is `inline`, inline, else, attach
                # pylint: disable=unidiomatic-typecheck
                if type(content_string) == inline:
                    htmlstr += '<img src="cid:{0}" title="{1}"/>'.format(hashed_ref, alias)
                    content_object["mime_object"].add_header(
                        "Content-ID", "<{0}>".format(hashed_ref)
                    )
                    altstr.append("-- img {0} should be here -- ".format(alias))
                    # inline images should be in related MIME block
                    msg_related.attach(content_object["mime_object"])
                else:
                    # non-inline images get attached like any other attachment
                    msg.attach(content_object["mime_object"])

            else:
                if content_object["encoding"] == "base64":
                    email.encoders.encode_base64(content_object["mime_object"])
                    msg.attach(content_object["mime_object"])
                elif content_object["sub_type"] not in ["html", "plain"]:
                    msg.attach(content_object["mime_object"])
                else:
                    if newline_to_break:
                        content_string = content_string.replace("\n", "<br>")
                    try:
                        htmlstr += "<div>{0}</div>".format(content_string)
                    except UnicodeEncodeError:
                        htmlstr += u"<div>{0}</div>".format(content_string)
                    altstr.append(content_string)

    msg_related.get_payload()[0] = MIMEText(htmlstr, "html", _charset=encoding)
    msg_alternative.attach(MIMEText("\n".join(altstr), _charset=encoding))
    msg_alternative.attach(msg_related)
    return msg
744116
def _build_message(self, message):
        """"""
        if not message.recipients():
            return False
        recipients = ','.join(message.to)
        recipients_cc = ','.join(message.cc)
        recipients_bcc = ','.join(message.bcc)

        text_body = message.body
        html_body = None
        if isinstance(message, EmailMultiAlternatives):
            for alt in message.alternatives:
                if alt[1] == "text/html":
                    html_body = alt[0]
                    break

        elif getattr(message, 'content_subtype', None) == 'html':
            # Don't send html content as plain text
            text_body = None
            html_body = message.body

        reply_to = ','.join(message.reply_to)
        custom_headers = {}
        if message.extra_headers and isinstance(message.extra_headers, dict):
            if 'Reply-To' in message.extra_headers:
                reply_to = message.extra_headers.pop('Reply-To')
            if len(message.extra_headers):
                custom_headers = message.extra_headers
        attachments = []
        if message.attachments and isinstance(message.attachments, list):
            if len(message.attachments):
                for item in message.attachments:
                    if isinstance(item, tuple):
                        (f, content, m) = item
                        content = base64.b64encode(content)
                        # b64decode returns bytes on Python 3. PMMail needs a
                        # str (for JSON serialization). Convert on Python 3
                        # only to avoid a useless performance hit on Python 2.
                        if not isinstance(content, str):
                            content = content.decode()
                        attachments.append((f, content, m))
                    else:
                        attachments.append(item)

        postmark_message = PMMail(api_key=self.api_key,
                                  subject=message.subject,
                                  sender=message.from_email,
                                  to=recipients,
                                  cc=recipients_cc,
                                  bcc=recipients_bcc,
                                  text_body=text_body,
                                  html_body=html_body,
                                  reply_to=reply_to,
                                  custom_headers=custom_headers,
                                  attachments=attachments)

        postmark_message.tag = getattr(message, 'tag', None)
        postmark_message.track_opens = getattr(message, 'track_opens', False)

        return postmark_message