1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507 |
- """
- Handle the email *forward* and *reply*. phase. There are 3 actors:
- - contact: who sends emails to alias@sl.co address
- - SL email handler (this script)
- - user personal email: to be protected. Should never leak to contact.
- This script makes sure that in the forward phase, the email that is forwarded to user personal email has the following
- envelope and header fields:
- Envelope:
- mail from: @contact
- rcpt to: @personal_email
- Header:
- From: @contact
- To: alias@sl.co # so user knows this email is sent to alias
- Reply-to: special@sl.co # magic HERE
- And in the reply phase:
- Envelope:
- mail from: @contact
- rcpt to: @contact
- Header:
- From: alias@sl.co # so for contact the email comes from alias. magic HERE
- To: @contact
- The special@sl.co allows to hide user personal email when user clicks "Reply" to the forwarded email.
- It should contain the following info:
- - alias
- - @contact
- """
- import asyncio
- import email
- import os
- import time
- import uuid
- from email import encoders
- from email.message import Message
- from email.mime.application import MIMEApplication
- from email.mime.multipart import MIMEMultipart
- from email.utils import parseaddr, formataddr, make_msgid
- from io import BytesIO
- from smtplib import SMTP, SMTPRecipientsRefused
- from typing import List, Tuple
- import aiosmtpd
- import aiospamc
- import arrow
- import spf
- from aiosmtpd.smtp import Envelope
- from app import pgp_utils, s3
- from app.alias_utils import try_auto_create
- from app.config import (
- EMAIL_DOMAIN,
- POSTFIX_SERVER,
- URL,
- ALIAS_DOMAINS,
- POSTFIX_SUBMISSION_TLS,
- UNSUBSCRIBER,
- LOAD_PGP_EMAIL_HANDLER,
- ENFORCE_SPF,
- ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
- ALERT_BOUNCE_EMAIL,
- ALERT_SPAM_EMAIL,
- ALERT_SPF,
- POSTFIX_PORT,
- SENDER,
- SENDER_DIR,
- SPAMASSASSIN_HOST,
- MAX_SPAM_SCORE,
- MAX_REPLY_PHASE_SPAM_SCORE,
- ALERT_SEND_EMAIL_CYCLE,
- ALERT_MAILBOX_IS_ALIAS,
- )
- from app.email_utils import (
- send_email,
- add_dkim_signature,
- add_or_replace_header,
- delete_header,
- email_belongs_to_alias_domains,
- render,
- get_orig_message_from_bounce,
- delete_all_headers_except,
- get_addrs_from_header,
- get_spam_info,
- get_orig_message_from_spamassassin_report,
- parseaddr_unicode,
- send_email_with_rate_control,
- get_email_domain_part,
- copy,
- to_bytes,
- get_header_from_bounce,
- send_email_at_most_times,
- )
- from app.extensions import db
- from app.greylisting import greylisting_needed
- from app.log import LOG
- from app.models import (
- Alias,
- Contact,
- EmailLog,
- CustomDomain,
- User,
- RefusedEmail,
- Mailbox,
- )
- from app.pgp_utils import PGPException
- from app.utils import random_string
- from init_app import load_pgp_public_keys
- from server import create_app, create_light_app
- # forward or reply
- _DIRECTION = "X-SimpleLogin-Type"
- _IP_HEADER = "X-SimpleLogin-Client-IP"
- _MAILBOX_ID_HEADER = "X-SimpleLogin-Mailbox-ID"
- _EMAIL_LOG_ID_HEADER = "X-SimpleLogin-EmailLog-ID"
- _MESSAGE_ID = "Message-ID"
- # fix the database connection leak issue
- # use this method instead of create_app
- def new_app():
- app = create_light_app()
- @app.teardown_appcontext
- def shutdown_session(response_or_exc):
- # same as shutdown_session() in flask-sqlalchemy but this is not enough
- db.session.remove()
- # dispose the engine too
- db.engine.dispose()
- return app
- def get_or_create_contact(
- contact_from_header: str, mail_from: str, alias: Alias
- ) -> Contact:
- """
- contact_from_header is the RFC 2047 format FROM header
- """
- # contact_from_header can be None, use mail_from in this case instead
- contact_from_header = contact_from_header or mail_from
- # force convert header to string, sometimes contact_from_header is Header object
- contact_from_header = str(contact_from_header)
- contact_name, contact_email = parseaddr_unicode(contact_from_header)
- if not contact_email:
- # From header is wrongly formatted, try with mail_from
- LOG.warning("From header is empty, parse mail_from %s %s", mail_from, alias)
- contact_name, contact_email = parseaddr_unicode(mail_from)
- if not contact_email:
- LOG.exception(
- "Cannot parse contact from from_header:%s, mail_from:%s",
- contact_from_header,
- mail_from,
- )
- contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
- if contact:
- if contact.name != contact_name:
- LOG.d(
- "Update contact %s name %s to %s",
- contact,
- contact.name,
- contact_name,
- )
- contact.name = contact_name
- db.session.commit()
- else:
- LOG.debug(
- "create contact for alias %s and contact %s",
- alias,
- contact_from_header,
- )
- reply_email = generate_reply_email()
- contact = Contact.create(
- user_id=alias.user_id,
- alias_id=alias.id,
- website_email=contact_email,
- name=contact_name,
- reply_email=reply_email,
- )
- db.session.commit()
- return contact
- def replace_header_when_forward(msg: Message, alias: Alias, header: str):
- """
- Replace CC or To header by Reply emails in forward phase
- """
- addrs = get_addrs_from_header(msg, header)
- # Nothing to do
- if not addrs:
- return
- new_addrs: [str] = []
- for addr in addrs:
- contact_name, contact_email = parseaddr_unicode(addr)
- # no transformation when alias is already in the header
- if contact_email == alias.email:
- new_addrs.append(addr)
- continue
- contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
- if contact:
- # update the contact name if needed
- if contact.name != contact_name:
- LOG.d(
- "Update contact %s name %s to %s",
- contact,
- contact.name,
- contact_name,
- )
- contact.name = contact_name
- db.session.commit()
- else:
- LOG.debug(
- "create contact for alias %s and email %s, header %s",
- alias,
- contact_email,
- header,
- )
- reply_email = generate_reply_email()
- contact = Contact.create(
- user_id=alias.user_id,
- alias_id=alias.id,
- website_email=contact_email,
- name=contact_name,
- reply_email=reply_email,
- is_cc=header.lower() == "cc",
- )
- db.session.commit()
- new_addrs.append(contact.new_addr())
- if new_addrs:
- new_header = ",".join(new_addrs)
- LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
- add_or_replace_header(msg, header, new_header)
- else:
- LOG.d("Delete %s header, old value %s", header, msg[header])
- delete_header(msg, header)
- def replace_header_when_reply(msg: Message, alias: Alias, header: str):
- """
- Replace CC or To Reply emails by original emails
- """
- addrs = get_addrs_from_header(msg, header)
- # Nothing to do
- if not addrs:
- return
- new_addrs: [str] = []
- for addr in addrs:
- name, reply_email = parseaddr(addr)
- # no transformation when alias is already in the header
- if reply_email == alias.email:
- continue
- contact = Contact.get_by(reply_email=reply_email)
- if not contact:
- LOG.warning(
- "%s email in reply phase %s must be reply emails", header, reply_email
- )
- # still keep this email in header
- new_addrs.append(addr)
- else:
- new_addrs.append(formataddr((contact.name, contact.website_email)))
- if new_addrs:
- new_header = ",".join(new_addrs)
- LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
- add_or_replace_header(msg, header, new_header)
- else:
- LOG.d("delete the %s header. Old value %s", header, msg[header])
- delete_header(msg, header)
- def replace_str_in_msg(msg: Message, fr: str, to: str):
- if msg.get_content_maintype() != "text":
- return msg
- new_body = msg.get_payload(decode=True).replace(fr.encode(), to.encode())
- # If utf-8 decoding fails, do not touch message part
- try:
- new_body = new_body.decode("utf-8")
- except:
- return msg
- cte = (
- msg["Content-Transfer-Encoding"].lower()
- if msg["Content-Transfer-Encoding"]
- else None
- )
- subtype = msg.get_content_subtype()
- delete_header(msg, "Content-Transfer-Encoding")
- delete_header(msg, "Content-Type")
- email.contentmanager.set_text_content(msg, new_body, subtype=subtype, cte=cte)
- return msg
- def generate_reply_email():
- # generate a reply_email, make sure it is unique
- # not use while loop to avoid infinite loop
- reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}"
- for _ in range(1000):
- if not Contact.get_by(reply_email=reply_email):
- # found!
- break
- reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}"
- return reply_email
- def should_append_alias(msg: Message, address: str):
- """whether an alias should be appended to TO header in message"""
- # # force convert header to string, sometimes addrs is Header object
- if msg["To"] and address.lower() in str(msg["To"]).lower():
- return False
- if msg["Cc"] and address.lower() in str(msg["Cc"]).lower():
- return False
- return True
- _MIME_HEADERS = [
- "MIME-Version",
- "Content-Type",
- "Content-Disposition",
- "Content-Transfer-Encoding",
- ]
- _MIME_HEADERS = [h.lower() for h in _MIME_HEADERS]
- def prepare_pgp_message(orig_msg: Message, pgp_fingerprint: str):
- msg = MIMEMultipart("encrypted", protocol="application/pgp-encrypted")
- # copy all headers from original message except all standard MIME headers
- for i in reversed(range(len(orig_msg._headers))):
- header_name = orig_msg._headers[i][0].lower()
- if header_name.lower() not in _MIME_HEADERS:
- msg[header_name] = orig_msg._headers[i][1]
- # Delete unnecessary headers in orig_msg except to save space
- delete_all_headers_except(
- orig_msg,
- _MIME_HEADERS,
- )
- first = MIMEApplication(
- _subtype="pgp-encrypted", _encoder=encoders.encode_7or8bit, _data=""
- )
- first.set_payload("Version: 1")
- msg.attach(first)
- second = MIMEApplication("octet-stream", _encoder=encoders.encode_7or8bit)
- second.add_header("Content-Disposition", "inline")
- # encrypt original message
- encrypted_data = pgp_utils.encrypt_file(
- BytesIO(orig_msg.as_bytes()), pgp_fingerprint
- )
- second.set_payload(encrypted_data)
- msg.attach(second)
- return msg
- def handle_email_sent_to_ourself(alias, mailbox, msg: Message, user):
- # store the refused email
- random_name = str(uuid.uuid4())
- full_report_path = f"refused-emails/cycle-{random_name}.eml"
- s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name)
- refused_email = RefusedEmail.create(
- path=None, full_report_path=full_report_path, user_id=alias.user_id
- )
- db.session.commit()
- LOG.d("Create refused email %s", refused_email)
- # link available for 6 days as it gets deleted in 7 days
- refused_email_url = refused_email.get_url(expires_in=518400)
- send_email_at_most_times(
- user,
- ALERT_SEND_EMAIL_CYCLE,
- mailbox.email,
- f"Email sent to {alias.email} from its own mailbox {mailbox.email}",
- render(
- "transactional/cycle-email.txt",
- name=user.name or "",
- alias=alias,
- mailbox=mailbox,
- refused_email_url=refused_email_url,
- ),
- render(
- "transactional/cycle-email.html",
- name=user.name or "",
- alias=alias,
- mailbox=mailbox,
- refused_email_url=refused_email_url,
- ),
- )
- async def handle_forward(
- envelope, smtp: SMTP, msg: Message, rcpt_to: str
- ) -> List[Tuple[bool, str]]:
- """return an array of SMTP status (is_success, smtp_status)
- is_success indicates whether an email has been delivered and
- smtp_status is the SMTP Status ("250 Message accepted", "550 Non-existent email address", etc)
- """
- address = rcpt_to.lower().strip().replace(" ", "") # alias@SL
- alias = Alias.get_by(email=address)
- if not alias:
- LOG.d("alias %s not exist. Try to see if it can be created on the fly", address)
- alias = try_auto_create(address)
- if not alias:
- LOG.d("alias %s cannot be created on-the-fly, return 550", address)
- return [(False, "550 SL E3 Email not exist")]
- mail_from = envelope.mail_from.lower().strip()
- for mb in alias.mailboxes:
- # email send from a mailbox to alias
- if mb.email.lower().strip() == mail_from:
- LOG.warning("cycle email sent from %s to %s", mb, alias)
- handle_email_sent_to_ourself(alias, mb, msg, alias.user)
- return [(True, "250 Message accepted for delivery")]
- contact = get_or_create_contact(msg["From"], envelope.mail_from, alias)
- email_log = EmailLog.create(contact_id=contact.id, user_id=contact.user_id)
- db.session.commit()
- if not alias.enabled:
- LOG.d("%s is disabled, do not forward", alias)
- email_log.blocked = True
- db.session.commit()
- # do not return 5** to allow user to receive emails later when alias is enabled
- return [(True, "250 Message accepted for delivery")]
- user = alias.user
- ret = []
- mailboxes = alias.mailboxes
- # no valid mailbox
- if not mailboxes:
- return [(False, "550 SL E16 invalid mailbox")]
- # no need to create a copy of message
- if len(mailboxes) == 1:
- mailbox = mailboxes[0]
- ret.append(
- await forward_email_to_mailbox(
- alias, msg, email_log, contact, envelope, smtp, mailbox, user
- )
- )
- # create a copy of message for each forward
- else:
- for mailbox in mailboxes:
- ret.append(
- await forward_email_to_mailbox(
- alias, copy(msg), email_log, contact, envelope, smtp, mailbox, user
- )
- )
- return ret
- async def forward_email_to_mailbox(
- alias,
- msg: Message,
- email_log: EmailLog,
- contact: Contact,
- envelope,
- smtp: SMTP,
- mailbox,
- user,
- ) -> (bool, str):
- LOG.d("Forward %s -> %s -> %s", contact, alias, mailbox)
- # sanity check: make sure mailbox is not actually an alias
- if get_email_domain_part(alias.email) == get_email_domain_part(mailbox.email):
- LOG.warning(
- "Mailbox has the same domain as alias. %s -> %s -> %s",
- contact,
- alias,
- mailbox,
- )
- mailbox_url = f"{URL}/dashboard/mailbox/{mailbox.id}/"
- send_email_with_rate_control(
- user,
- ALERT_MAILBOX_IS_ALIAS,
- user.email,
- f"Your SimpleLogin mailbox {mailbox.email} cannot be an email alias",
- render(
- "transactional/mailbox-invalid.txt",
- name=user.name or "",
- mailbox=mailbox,
- mailbox_url=mailbox_url,
- ),
- render(
- "transactional/mailbox-invalid.html",
- name=user.name or "",
- mailbox=mailbox,
- mailbox_url=mailbox_url,
- ),
- max_nb_alert=1,
- )
- # retry later
- # so when user fixes the mailbox, the email can be delivered
- return False, "421 SL E14"
- # Spam check
- spam_status = ""
- is_spam = False
- if SPAMASSASSIN_HOST:
- start = time.time()
- spam_score = await get_spam_score(msg)
- LOG.d(
- "%s -> %s - spam score %s in %s seconds",
- contact,
- alias,
- spam_score,
- time.time() - start,
- )
- email_log.spam_score = spam_score
- db.session.commit()
- if (user.max_spam_score and spam_score > user.max_spam_score) or (
- not user.max_spam_score and spam_score > MAX_SPAM_SCORE
- ):
- is_spam = True
- spam_status = "Spam detected by SpamAssassin server"
- else:
- is_spam, spam_status = get_spam_info(msg, max_score=user.max_spam_score)
- if is_spam:
- LOG.warning("Email detected as spam. Alias: %s, from: %s", alias, contact)
- email_log.is_spam = True
- email_log.spam_status = spam_status
- db.session.commit()
- handle_spam(contact, alias, msg, user, mailbox, email_log)
- return False, "550 SL E1 Email detected as spam"
- # create PGP email if needed
- if mailbox.pgp_finger_print and user.is_premium() and not alias.disable_pgp:
- LOG.d("Encrypt message using mailbox %s", mailbox)
- try:
- msg = prepare_pgp_message(msg, mailbox.pgp_finger_print)
- except PGPException:
- LOG.exception(
- "Cannot encrypt message %s -> %s. %s %s", contact, alias, mailbox, user
- )
- # so the client can retry later
- return False, "421 SL E12 Retry later"
- # add custom header
- add_or_replace_header(msg, _DIRECTION, "Forward")
- # remove reply-to & sender header if present
- delete_header(msg, "Reply-To")
- delete_header(msg, "Sender")
- delete_header(msg, _IP_HEADER)
- add_or_replace_header(msg, _MAILBOX_ID_HEADER, str(mailbox.id))
- add_or_replace_header(msg, _EMAIL_LOG_ID_HEADER, str(email_log.id))
- add_or_replace_header(msg, _MESSAGE_ID, make_msgid(str(email_log.id), EMAIL_DOMAIN))
- # change the from header so the sender comes from @SL
- # so it can pass DMARC check
- # replace the email part in from: header
- contact_from_header = msg["From"]
- new_from_header = contact.new_addr()
- add_or_replace_header(msg, "From", new_from_header)
- LOG.d("new_from_header:%s, old header %s", new_from_header, contact_from_header)
- # replace CC & To emails by reply-email for all emails that are not alias
- replace_header_when_forward(msg, alias, "Cc")
- replace_header_when_forward(msg, alias, "To")
- # append alias into the TO header if it's not present in To or CC
- if should_append_alias(msg, alias.email):
- LOG.d("append alias %s to TO header %s", alias, msg["To"])
- if msg["To"]:
- to_header = msg["To"] + "," + alias.email
- else:
- to_header = alias.email
- add_or_replace_header(msg, "To", to_header.strip())
- # add List-Unsubscribe header
- if UNSUBSCRIBER:
- unsubscribe_link = f"mailto:{UNSUBSCRIBER}?subject={alias.id}="
- add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>")
- else:
- unsubscribe_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
- add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>")
- add_or_replace_header(
- msg, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click"
- )
- add_dkim_signature(msg, EMAIL_DOMAIN)
- LOG.d(
- "Forward mail from %s to %s, mail_options %s, rcpt_options %s ",
- contact.website_email,
- mailbox.email,
- envelope.mail_options,
- envelope.rcpt_options,
- )
- # smtp.send_message has UnicodeEncodeErroremail issue
- # encode message raw directly instead
- try:
- smtp.sendmail(
- contact.reply_email,
- mailbox.email,
- msg.as_bytes(),
- envelope.mail_options,
- envelope.rcpt_options,
- )
- except SMTPRecipientsRefused:
- # that means the mailbox is maybe invalid
- LOG.warning(
- "SMTPRecipientsRefused forward phase %s -> %s -> %s",
- contact,
- alias,
- mailbox,
- )
- # return 421 so Postfix can retry later
- return False, "421 SL E17 Retry later"
- else:
- db.session.commit()
- return True, "250 Message accepted for delivery"
- async def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str):
- """
- return whether an email has been delivered and
- the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
- """
- reply_email = rcpt_to.lower().strip()
- # reply_email must end with EMAIL_DOMAIN
- if not reply_email.endswith(EMAIL_DOMAIN):
- LOG.warning(f"Reply email {reply_email} has wrong domain")
- return False, "550 SL E2"
- contact = Contact.get_by(reply_email=reply_email)
- if not contact:
- LOG.warning(f"No such forward-email with {reply_email} as reply-email")
- return False, "550 SL E4 Email not exist"
- alias = contact.alias
- address: str = contact.alias.email
- alias_domain = address[address.find("@") + 1 :]
- # alias must end with one of the ALIAS_DOMAINS or custom-domain
- if not email_belongs_to_alias_domains(alias.email):
- if not CustomDomain.get_by(domain=alias_domain):
- return False, "550 SL E5"
- user = alias.user
- mail_from = envelope.mail_from.lower().strip()
- # bounce email initiated by Postfix
- # can happen in case emails cannot be delivered to user-email
- # in this case Postfix will try to send a bounce report to original sender, which is
- # the "reply email"
- if mail_from == "<>":
- LOG.warning(
- "Bounce when sending to alias %s from %s, user %s",
- alias,
- contact,
- user,
- )
- handle_bounce(contact, alias, msg, user)
- return False, "550 SL E6"
- mailbox = Mailbox.get_by(email=mail_from, user_id=user.id)
- if not mailbox or mailbox not in alias.mailboxes:
- if alias.disable_email_spoofing_check:
- # ignore this error, use default alias mailbox
- LOG.warning(
- "ignore unknown sender to reverse-alias %s: %s -> %s",
- mail_from,
- alias,
- contact,
- )
- mailbox = alias.mailbox
- else:
- # only mailbox can send email to the reply-email
- handle_unknown_mailbox(envelope, msg, reply_email, user, alias)
- return False, "550 SL E7"
- if ENFORCE_SPF and mailbox.force_spf and not alias.disable_email_spoofing_check:
- ip = msg[_IP_HEADER]
- if not spf_pass(ip, envelope, mailbox, user, alias, contact.website_email, msg):
- # cannot use 4** here as sender will retry. 5** because that generates bounce report
- return True, "250 SL E11"
- email_log = EmailLog.create(
- contact_id=contact.id, is_reply=True, user_id=contact.user_id
- )
- # Spam check
- spam_status = ""
- is_spam = False
- # do not use user.max_spam_score here
- if SPAMASSASSIN_HOST:
- start = time.time()
- spam_score = await get_spam_score(msg)
- LOG.d(
- "%s -> %s - spam score %s in %s seconds",
- alias,
- contact,
- spam_score,
- time.time() - start,
- )
- email_log.spam_score = spam_score
- if spam_score > MAX_REPLY_PHASE_SPAM_SCORE:
- is_spam = True
- spam_status = "Spam detected by SpamAssassin server"
- else:
- is_spam, spam_status = get_spam_info(msg, max_score=MAX_REPLY_PHASE_SPAM_SCORE)
- if is_spam:
- LOG.exception(
- "Reply phase - email sent from %s to %s detected as spam", alias, contact
- )
- email_log.is_spam = True
- email_log.spam_status = spam_status
- db.session.commit()
- handle_spam(contact, alias, msg, user, mailbox, email_log, is_reply=True)
- return False, "550 SL E15 Email detected as spam"
- delete_header(msg, _IP_HEADER)
- delete_header(msg, "DKIM-Signature")
- delete_header(msg, "Received")
- # make the email comes from alias
- from_header = alias.email
- # add alias name from alias
- if alias.name:
- LOG.d("Put alias name in from header")
- from_header = formataddr((alias.name, alias.email))
- elif alias.custom_domain:
- LOG.d("Put domain default alias name in from header")
- # add alias name from domain
- if alias.custom_domain.name:
- from_header = formataddr((alias.custom_domain.name, alias.email))
- add_or_replace_header(msg, "From", from_header)
- # some email providers like ProtonMail adds automatically the Reply-To field
- # make sure to delete it
- delete_header(msg, "Reply-To")
- # remove sender header if present as this could reveal user real email
- delete_header(msg, "Sender")
- delete_header(msg, "X-Sender")
- replace_header_when_reply(msg, alias, "To")
- replace_header_when_reply(msg, alias, "Cc")
- add_or_replace_header(
- msg,
- _MESSAGE_ID,
- make_msgid(str(email_log.id), get_email_domain_part(alias.email)),
- )
- add_or_replace_header(msg, _EMAIL_LOG_ID_HEADER, str(email_log.id))
- add_or_replace_header(msg, _DIRECTION, "Reply")
- # Received-SPF is injected by postfix-policyd-spf-python can reveal user original email
- delete_header(msg, "Received-SPF")
- LOG.d(
- "send email from %s to %s, mail_options:%s,rcpt_options:%s",
- alias.email,
- contact.website_email,
- envelope.mail_options,
- envelope.rcpt_options,
- )
- # replace "ra+string@simplelogin.co" by the contact email in the email body
- # as this is usually included when replying
- if user.replace_reverse_alias:
- if msg.is_multipart():
- for part in msg.walk():
- if part.get_content_maintype() != "text":
- continue
- part = replace_str_in_msg(part, reply_email, contact.website_email)
- else:
- msg = replace_str_in_msg(msg, reply_email, contact.website_email)
- if alias_domain in ALIAS_DOMAINS:
- add_dkim_signature(msg, alias_domain)
- # add DKIM-Signature for custom-domain alias
- else:
- custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain)
- if custom_domain.dkim_verified:
- add_dkim_signature(msg, alias_domain)
- # create PGP email if needed
- if contact.pgp_finger_print and user.is_premium():
- LOG.d("Encrypt message for contact %s", contact)
- try:
- msg = prepare_pgp_message(msg, contact.pgp_finger_print)
- except PGPException:
- LOG.exception(
- "Cannot encrypt message %s -> %s. %s %s", alias, contact, mailbox, user
- )
- # to not save the email_log
- db.session.rollback()
- # return 421 so the client can retry later
- return False, "421 SL E13 Retry later"
- try:
- smtp.sendmail(
- alias.email,
- contact.website_email,
- msg.as_bytes(),
- envelope.mail_options,
- envelope.rcpt_options,
- )
- except Exception:
- # to not save the email_log
- db.session.rollback()
- LOG.warning("Cannot send email from %s to %s", alias, contact)
- send_email(
- mailbox.email,
- f"Email cannot be sent to {contact.email} from {alias.email}",
- render(
- "transactional/reply-error.txt",
- user=user,
- alias=alias,
- contact=contact,
- contact_domain=get_email_domain_part(contact.email),
- ),
- render(
- "transactional/reply-error.html",
- user=user,
- alias=alias,
- contact=contact,
- contact_domain=get_email_domain_part(contact.email),
- ),
- )
- # return 250 even if error as user is already informed of the incident and can retry sending the email
- db.session.commit()
- return True, "250 Message accepted for delivery"
- def spf_pass(
- ip: str,
- envelope,
- mailbox: Mailbox,
- user: User,
- alias: Alias,
- contact_email: str,
- msg: Message,
- ) -> bool:
- if ip:
- LOG.d("Enforce SPF")
- try:
- r = spf.check2(i=ip, s=envelope.mail_from.lower(), h=None)
- except Exception:
- LOG.exception("SPF error, mailbox %s, ip %s", mailbox.email, ip)
- else:
- # TODO: Handle temperr case (e.g. dns timeout)
- # only an absolute pass, or no SPF policy at all is 'valid'
- if r[0] not in ["pass", "none"]:
- LOG.warning(
- "SPF fail for mailbox %s, reason %s, failed IP %s",
- mailbox.email,
- r[0],
- ip,
- )
- send_email_with_rate_control(
- user,
- ALERT_SPF,
- mailbox.email,
- f"SimpleLogin Alert: attempt to send emails from your alias {alias.email} from unknown IP Address",
- render(
- "transactional/spf-fail.txt",
- name=user.name,
- alias=alias.email,
- ip=ip,
- mailbox_url=URL + f"/dashboard/mailbox/{mailbox.id}#spf",
- to_email=contact_email,
- subject=msg["Subject"],
- time=arrow.now(),
- ),
- render(
- "transactional/spf-fail.html",
- name=user.name,
- alias=alias.email,
- ip=ip,
- mailbox_url=URL + f"/dashboard/mailbox/{mailbox.id}#spf",
- to_email=contact_email,
- subject=msg["Subject"],
- time=arrow.now(),
- ),
- )
- return False
- else:
- LOG.warning(
- "Could not find %s header %s -> %s",
- _IP_HEADER,
- mailbox.email,
- contact_email,
- )
- return True
- def handle_unknown_mailbox(envelope, msg, reply_email: str, user: User, alias: Alias):
- LOG.warning(
- f"Reply email can only be used by mailbox. "
- f"Actual mail_from: %s. msg from header: %s, reverse-alias %s, %s %s",
- envelope.mail_from,
- msg["From"],
- reply_email,
- alias,
- user,
- )
- send_email_with_rate_control(
- user,
- ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
- user.email,
- f"Attempt to use your alias {alias.email} from {envelope.mail_from}",
- render(
- "transactional/reply-must-use-personal-email.txt",
- name=user.name,
- alias=alias,
- sender=envelope.mail_from,
- ),
- render(
- "transactional/reply-must-use-personal-email.html",
- name=user.name,
- alias=alias,
- sender=envelope.mail_from,
- ),
- )
- # Notify sender that they cannot send emails to this address
- send_email_with_rate_control(
- user,
- ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
- envelope.mail_from,
- f"Your email ({envelope.mail_from}) is not allowed to send emails to {reply_email}",
- render(
- "transactional/send-from-alias-from-unknown-sender.txt",
- sender=envelope.mail_from,
- reply_email=reply_email,
- ),
- render(
- "transactional/send-from-alias-from-unknown-sender.html",
- sender=envelope.mail_from,
- reply_email=reply_email,
- ),
- )
- def handle_bounce(contact: Contact, alias: Alias, msg: Message, user: User):
- disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
- # Store the bounced email
- # generate a name for the email
- random_name = str(uuid.uuid4())
- full_report_path = f"refused-emails/full-{random_name}.eml"
- s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name)
- file_path = None
- mailbox = None
- email_log: EmailLog = None
- orig_msg = get_orig_message_from_bounce(msg)
- if not orig_msg:
- # Some MTA does not return the original message in bounce message
- # nothing we can do here
- LOG.warning(
- "Cannot parse original message from bounce message %s %s %s %s",
- alias,
- user,
- contact,
- full_report_path,
- )
- else:
- file_path = f"refused-emails/{random_name}.eml"
- s3.upload_email_from_bytesio(
- file_path, BytesIO(orig_msg.as_bytes()), random_name
- )
- try:
- mailbox_id = int(orig_msg[_MAILBOX_ID_HEADER])
- except TypeError:
- LOG.exception(
- "cannot parse mailbox from original message header %s",
- orig_msg[_MAILBOX_ID_HEADER],
- )
- else:
- mailbox = Mailbox.get(mailbox_id)
- if not mailbox or mailbox.user_id != user.id:
- LOG.exception(
- "Tampered message mailbox_id %s, %s, %s, %s %s",
- mailbox_id,
- user,
- alias,
- contact,
- full_report_path,
- )
- # cannot use this tampered mailbox, reset it
- mailbox = None
- # try to get the original email_log
- try:
- email_log_id = int(orig_msg[_EMAIL_LOG_ID_HEADER])
- except TypeError:
- LOG.exception(
- "cannot parse email log from original message header %s",
- orig_msg[_EMAIL_LOG_ID_HEADER],
- )
- else:
- email_log = EmailLog.get(email_log_id)
- refused_email = RefusedEmail.create(
- path=file_path, full_report_path=full_report_path, user_id=user.id
- )
- db.session.flush()
- LOG.d("Create refused email %s", refused_email)
- if not mailbox:
- LOG.debug("Try to get mailbox from bounce report")
- try:
- mailbox_id = int(get_header_from_bounce(msg, _MAILBOX_ID_HEADER))
- except Exception:
- LOG.exception("cannot get mailbox-id from bounce report, %s", refused_email)
- else:
- mailbox = Mailbox.get(mailbox_id)
- if not mailbox or mailbox.user_id != user.id:
- LOG.exception(
- "Tampered message mailbox_id %s, %s, %s, %s %s",
- mailbox_id,
- user,
- alias,
- contact,
- full_report_path,
- )
- mailbox = None
- if not email_log:
- LOG.d("Try to get email log from bounce report")
- try:
- email_log_id = int(get_header_from_bounce(msg, _EMAIL_LOG_ID_HEADER))
- except Exception:
- LOG.warning("cannot get email log id from bounce report, %s", refused_email)
- else:
- email_log = EmailLog.get(email_log_id)
- # use the default mailbox as the last option
- if not mailbox:
- LOG.warning("Use %s default mailbox %s", alias, refused_email)
- mailbox = alias.mailbox
- # create a new email log as the last option
- if not email_log:
- LOG.warning("cannot get the original email_log, create a new one")
- email_log: EmailLog = EmailLog.create(
- contact_id=contact.id, user_id=contact.user_id
- )
- email_log.bounced = True
- email_log.refused_email_id = refused_email.id
- email_log.bounced_mailbox_id = mailbox.id
- db.session.commit()
- refused_email_url = (
- URL + f"/dashboard/refused_email?highlight_id=" + str(email_log.id)
- )
- nb_bounced = EmailLog.filter_by(contact_id=contact.id, bounced=True).count()
- if nb_bounced >= 2 and alias.cannot_be_disabled:
- LOG.warning("%s cannot be disabled", alias)
- # inform user if this is the first bounced email
- if nb_bounced == 1 or (nb_bounced >= 2 and alias.cannot_be_disabled):
- LOG.d(
- "Inform user %s about bounced email sent by %s to alias %s",
- user,
- contact.website_email,
- alias,
- )
- send_email_with_rate_control(
- user,
- ALERT_BOUNCE_EMAIL,
- user.email,
- f"Email from {contact.website_email} to {alias.email} cannot be delivered to your inbox",
- render(
- "transactional/bounced-email.txt",
- name=user.name,
- alias=alias,
- website_email=contact.website_email,
- disable_alias_link=disable_alias_link,
- refused_email_url=refused_email_url,
- mailbox_email=mailbox.email,
- ),
- render(
- "transactional/bounced-email.html",
- name=user.name,
- alias=alias,
- website_email=contact.website_email,
- disable_alias_link=disable_alias_link,
- refused_email_url=refused_email_url,
- mailbox_email=mailbox.email,
- ),
- )
- # disable the alias the second time email is bounced
- elif nb_bounced >= 2:
- LOG.d(
- "Bounce happens again with alias %s from %s. Disable alias now ",
- alias,
- contact.website_email,
- )
- alias.enabled = False
- db.session.commit()
- send_email_with_rate_control(
- user,
- ALERT_BOUNCE_EMAIL,
- user.email,
- f"Alias {alias.email} has been disabled due to second undelivered email from {contact.website_email}",
- render(
- "transactional/automatic-disable-alias.txt",
- name=user.name,
- alias=alias,
- website_email=contact.website_email,
- refused_email_url=refused_email_url,
- mailbox_email=mailbox.email,
- ),
- render(
- "transactional/automatic-disable-alias.html",
- name=user.name,
- alias=alias,
- website_email=contact.website_email,
- refused_email_url=refused_email_url,
- mailbox_email=mailbox.email,
- ),
- )
- def handle_spam(
- contact: Contact,
- alias: Alias,
- msg: Message,
- user: User,
- mailbox: Mailbox,
- email_log: EmailLog,
- is_reply=False, # whether the email is in forward or reply phase
- ):
- # Store the report & original email
- orig_msg = get_orig_message_from_spamassassin_report(msg)
- # generate a name for the email
- random_name = str(uuid.uuid4())
- full_report_path = f"spams/full-{random_name}.eml"
- s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name)
- file_path = None
- if orig_msg:
- file_path = f"spams/{random_name}.eml"
- s3.upload_email_from_bytesio(
- file_path, BytesIO(orig_msg.as_bytes()), random_name
- )
- refused_email = RefusedEmail.create(
- path=file_path, full_report_path=full_report_path, user_id=user.id
- )
- db.session.flush()
- email_log.refused_email_id = refused_email.id
- db.session.commit()
- LOG.d("Create spam email %s", refused_email)
- refused_email_url = (
- URL + f"/dashboard/refused_email?highlight_id=" + str(email_log.id)
- )
- disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
- if is_reply:
- LOG.d(
- "Inform %s (%s) about spam email sent from alias %s to %s",
- mailbox,
- user,
- alias,
- contact,
- )
- send_email_with_rate_control(
- user,
- ALERT_SPAM_EMAIL,
- mailbox.email,
- f"Email from {contact.website_email} to {alias.email} is detected as spam",
- render(
- "transactional/spam-email-reply-phase.txt",
- name=user.name,
- alias=alias,
- website_email=contact.website_email,
- disable_alias_link=disable_alias_link,
- refused_email_url=refused_email_url,
- ),
- render(
- "transactional/spam-email-reply-phase.html",
- name=user.name,
- alias=alias,
- website_email=contact.website_email,
- disable_alias_link=disable_alias_link,
- refused_email_url=refused_email_url,
- ),
- )
- else:
- # inform user
- LOG.d(
- "Inform %s (%s) about spam email sent by %s to alias %s",
- mailbox,
- user,
- contact,
- alias,
- )
- send_email_with_rate_control(
- user,
- ALERT_SPAM_EMAIL,
- mailbox.email,
- f"Email from {contact.website_email} to {alias.email} is detected as spam",
- render(
- "transactional/spam-email.txt",
- name=user.name,
- alias=alias,
- website_email=contact.website_email,
- disable_alias_link=disable_alias_link,
- refused_email_url=refused_email_url,
- ),
- render(
- "transactional/spam-email.html",
- name=user.name,
- alias=alias,
- website_email=contact.website_email,
- disable_alias_link=disable_alias_link,
- refused_email_url=refused_email_url,
- ),
- )
- def handle_unsubscribe(envelope: Envelope):
- msg = email.message_from_bytes(envelope.original_content)
- # format: alias_id:
- subject = msg["Subject"]
- try:
- # subject has the format {alias.id}=
- if subject.endswith("="):
- alias_id = int(subject[:-1])
- # some email providers might strip off the = suffix
- else:
- alias_id = int(subject)
- alias = Alias.get(alias_id)
- except Exception:
- LOG.warning("Cannot parse alias from subject %s", msg["Subject"])
- return "550 SL E8 Wrongly formatted subject"
- if not alias:
- LOG.warning("No such alias %s", alias_id)
- return "550 SL E9 Email not exist"
- # This sender cannot unsubscribe
- mail_from = envelope.mail_from.lower().strip()
- mailbox = Mailbox.get_by(user_id=alias.user_id, email=mail_from)
- if not mailbox or mailbox not in alias.mailboxes:
- LOG.d("%s cannot disable alias %s", envelope.mail_from, alias)
- return "550 SL E10 unauthorized"
- # Sender is owner of this alias
- alias.enabled = False
- db.session.commit()
- user = alias.user
- enable_alias_url = URL + f"/dashboard/?highlight_alias_id={alias.id}"
- for mailbox in alias.mailboxes:
- send_email(
- mailbox.email,
- f"Alias {alias.email} has been disabled successfully",
- render(
- "transactional/unsubscribe-disable-alias.txt",
- user=user,
- alias=alias.email,
- enable_alias_url=enable_alias_url,
- ),
- render(
- "transactional/unsubscribe-disable-alias.html",
- user=user,
- alias=alias.email,
- enable_alias_url=enable_alias_url,
- ),
- )
- return "250 Unsubscribe request accepted"
- def handle_sender_email(envelope: Envelope):
- filename = (
- arrow.now().format("YYYY-MM-DD_HH-mm-ss") + "_" + random_string(10) + ".eml"
- )
- filepath = os.path.join(SENDER_DIR, filename)
- with open(filepath, "wb") as f:
- f.write(envelope.original_content)
- LOG.d("Write email to sender at %s", filepath)
- msg = email.message_from_bytes(envelope.original_content)
- orig = get_orig_message_from_bounce(msg)
- if orig:
- LOG.warning(
- "Original message %s -> %s saved at %s", orig["From"], orig["To"], filepath
- )
- return "250 email to sender accepted"
- async def handle(envelope: Envelope, smtp: SMTP) -> str:
- """Return SMTP status"""
- # unsubscribe request
- if UNSUBSCRIBER and envelope.rcpt_tos == [UNSUBSCRIBER]:
- LOG.d("Handle unsubscribe request from %s", envelope.mail_from)
- return handle_unsubscribe(envelope)
- # emails sent to sender. Probably bounce emails
- if SENDER and envelope.rcpt_tos == [SENDER]:
- LOG.d("Handle email sent to sender from %s", envelope.mail_from)
- return handle_sender_email(envelope)
- # Whether it's necessary to apply greylisting
- if greylisting_needed(envelope.mail_from, envelope.rcpt_tos):
- LOG.warning(
- "Grey listing applied for %s %s", envelope.mail_from, envelope.rcpt_tos
- )
- return "421 SL Retry later"
- # result of all deliveries
- # each element is a couple of whether the delivery is successful and the smtp status
- res: [(bool, str)] = []
- for rcpt_to in envelope.rcpt_tos:
- msg = email.message_from_bytes(envelope.original_content)
- # Reply case
- # recipient starts with "reply+" or "ra+" (ra=reverse-alias) prefix
- if rcpt_to.startswith("reply+") or rcpt_to.startswith("ra+"):
- LOG.debug(
- ">>> Reply phase %s(%s) -> %s", envelope.mail_from, msg["From"], rcpt_to
- )
- is_delivered, smtp_status = await handle_reply(envelope, smtp, msg, rcpt_to)
- res.append((is_delivered, smtp_status))
- else: # Forward case
- LOG.debug(
- ">>> Forward phase %s(%s) -> %s",
- envelope.mail_from,
- msg["From"],
- rcpt_to,
- )
- for is_delivered, smtp_status in await handle_forward(
- envelope, smtp, msg, rcpt_to
- ):
- res.append((is_delivered, smtp_status))
- for (is_success, smtp_status) in res:
- # Consider all deliveries successful if 1 delivery is successful
- if is_success:
- return smtp_status
- # Failed delivery for all, return the first failure
- return res[0][1]
- async def get_spam_score(message: Message) -> float:
- sa_input = to_bytes(message)
- # Spamassassin requires to have an ending linebreak
- if not sa_input.endswith(b"\n"):
- LOG.d("add linebreak to spamassassin input")
- sa_input += b"\n"
- try:
- # wait for at max 300s which is the default spamd timeout-child
- response = await asyncio.wait_for(
- aiospamc.check(sa_input, host=SPAMASSASSIN_HOST), timeout=300
- )
- return response.headers["Spam"].score
- except asyncio.TimeoutError:
- LOG.exception("SpamAssassin timeout")
- # return a negative score so the message is always considered as ham
- return -999
- except Exception:
- LOG.exception("SpamAssassin exception")
- return -999
- class MailHandler:
- def __init__(self, lock):
- self.lock = lock
- async def handle_DATA(self, server, session, envelope: Envelope):
- try:
- ret = await self._handle(envelope)
- return ret
- except Exception:
- LOG.exception(
- "email handling fail %s -> %s",
- envelope.mail_from,
- envelope.rcpt_tos,
- )
- return "421 SL Retry later"
- async def _handle(self, envelope: Envelope):
- async with self.lock:
- start = time.time()
- LOG.info(
- "===>> New message, mail from %s, rctp tos %s ",
- envelope.mail_from,
- envelope.rcpt_tos,
- )
- if POSTFIX_SUBMISSION_TLS:
- smtp = SMTP(POSTFIX_SERVER, 587)
- smtp.starttls()
- else:
- smtp = SMTP(POSTFIX_SERVER, POSTFIX_PORT or 25)
- app = new_app()
- with app.app_context():
- ret = await handle(envelope, smtp)
- LOG.info("takes %s seconds <<===", time.time() - start)
- return ret
- if __name__ == "__main__":
- if LOAD_PGP_EMAIL_HANDLER:
- LOG.warning("LOAD PGP keys")
- app = create_app()
- with app.app_context():
- load_pgp_public_keys()
- loop = asyncio.new_event_loop()
- asyncio.set_event_loop(loop)
- lock = asyncio.Lock()
- handler = MailHandler(lock)
- def factory():
- return aiosmtpd.smtp.SMTP(handler, enable_SMTPUTF8=True)
- server = loop.run_until_complete(
- loop.create_server(factory, host="0.0.0.0", port=20381)
- )
- try:
- loop.run_forever()
- except KeyboardInterrupt:
- pass
- # Close the server
- LOG.info("Close SMTP server")
- server.close()
- loop.run_until_complete(server.wait_closed())
- loop.close()
|