email_handler.py 32 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013
  1. """
  2. Handle the email *forward* and *reply*. phase. There are 3 actors:
  3. - contact: who sends emails to alias@sl.co address
  4. - SL email handler (this script)
  5. - user personal email: to be protected. Should never leak to contact.
  6. This script makes sure that in the forward phase, the email that is forwarded to user personal email has the following
  7. envelope and header fields:
  8. Envelope:
  9. mail from: @contact
  10. rcpt to: @personal_email
  11. Header:
  12. From: @contact
  13. To: alias@sl.co # so user knows this email is sent to alias
  14. Reply-to: special@sl.co # magic HERE
  15. And in the reply phase:
  16. Envelope:
  17. mail from: @contact
  18. rcpt to: @contact
  19. Header:
  20. From: alias@sl.co # so for contact the email comes from alias. magic HERE
  21. To: @contact
  22. The special@sl.co allows to hide user personal email when user clicks "Reply" to the forwarded email.
  23. It should contain the following info:
  24. - alias
  25. - @contact
  26. """
  27. import email
  28. import time
  29. import uuid
  30. from email import encoders
  31. from email.message import Message
  32. from email.mime.application import MIMEApplication
  33. from email.mime.multipart import MIMEMultipart
  34. from email.utils import parseaddr, formataddr
  35. from io import BytesIO
  36. from smtplib import SMTP
  37. from typing import List, Tuple
  38. import arrow
  39. import spf
  40. from aiosmtpd.controller import Controller
  41. from aiosmtpd.smtp import Envelope
  42. from app import pgp_utils, s3
  43. from app.alias_utils import try_auto_create
  44. from app.config import (
  45. EMAIL_DOMAIN,
  46. POSTFIX_SERVER,
  47. URL,
  48. ALIAS_DOMAINS,
  49. POSTFIX_SUBMISSION_TLS,
  50. UNSUBSCRIBER,
  51. LOAD_PGP_EMAIL_HANDLER,
  52. ENFORCE_SPF,
  53. ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
  54. ALERT_BOUNCE_EMAIL,
  55. ALERT_SPAM_EMAIL,
  56. ALERT_SPF,
  57. )
  58. from app.email_utils import (
  59. send_email,
  60. add_dkim_signature,
  61. add_or_replace_header,
  62. delete_header,
  63. email_belongs_to_alias_domains,
  64. render,
  65. get_orig_message_from_bounce,
  66. delete_all_headers_except,
  67. get_addrs_from_header,
  68. get_spam_info,
  69. get_orig_message_from_spamassassin_report,
  70. parseaddr_unicode,
  71. send_email_with_rate_control,
  72. )
  73. from app.extensions import db
  74. from app.greylisting import greylisting_needed
  75. from app.log import LOG
  76. from app.models import (
  77. Alias,
  78. Contact,
  79. EmailLog,
  80. CustomDomain,
  81. User,
  82. RefusedEmail,
  83. Mailbox,
  84. )
  85. from app.utils import random_string
  86. from init_app import load_pgp_public_keys
  87. from server import create_app
  88. _IP_HEADER = "X-SimpleLogin-Client-IP"
  89. _MAILBOX_ID_HEADER = "X-SimpleLogin-Mailbox-ID"
  90. # fix the database connection leak issue
  91. # use this method instead of create_app
  92. def new_app():
  93. app = create_app()
  94. @app.teardown_appcontext
  95. def shutdown_session(response_or_exc):
  96. # same as shutdown_session() in flask-sqlalchemy but this is not enough
  97. db.session.remove()
  98. # dispose the engine too
  99. db.engine.dispose()
  100. return app
  101. def get_or_create_contact(contact_from_header: str, alias: Alias) -> Contact:
  102. """
  103. contact_from_header is the RFC 2047 format FROM header
  104. """
  105. # force convert header to string, sometimes contact_from_header is Header object
  106. contact_from_header = str(contact_from_header)
  107. contact_name, contact_email = parseaddr_unicode(contact_from_header)
  108. contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
  109. if contact:
  110. if contact.name != contact_name:
  111. LOG.d(
  112. "Update contact %s name %s to %s", contact, contact.name, contact_name,
  113. )
  114. contact.name = contact_name
  115. db.session.commit()
  116. else:
  117. LOG.debug(
  118. "create contact for alias %s and contact %s", alias, contact_from_header,
  119. )
  120. reply_email = generate_reply_email()
  121. contact = Contact.create(
  122. user_id=alias.user_id,
  123. alias_id=alias.id,
  124. website_email=contact_email,
  125. name=contact_name,
  126. reply_email=reply_email,
  127. )
  128. db.session.commit()
  129. return contact
  130. def replace_header_when_forward(msg: Message, alias: Alias, header: str):
  131. """
  132. Replace CC or To header by Reply emails in forward phase
  133. """
  134. addrs = get_addrs_from_header(msg, header)
  135. # Nothing to do
  136. if not addrs:
  137. return
  138. new_addrs: [str] = []
  139. need_replace = False
  140. for addr in addrs:
  141. contact_name, contact_email = parseaddr_unicode(addr)
  142. # no transformation when alias is already in the header
  143. if contact_email == alias.email:
  144. new_addrs.append(addr)
  145. continue
  146. contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
  147. if contact:
  148. # update the contact name if needed
  149. if contact.name != contact_name:
  150. LOG.d(
  151. "Update contact %s name %s to %s",
  152. contact,
  153. contact.name,
  154. contact_name,
  155. )
  156. contact.name = contact_name
  157. db.session.commit()
  158. else:
  159. LOG.debug(
  160. "create contact for alias %s and email %s, header %s",
  161. alias,
  162. contact_email,
  163. header,
  164. )
  165. reply_email = generate_reply_email()
  166. contact = Contact.create(
  167. user_id=alias.user_id,
  168. alias_id=alias.id,
  169. website_email=contact_email,
  170. name=contact_name,
  171. reply_email=reply_email,
  172. is_cc=header.lower() == "cc",
  173. )
  174. db.session.commit()
  175. new_addrs.append(contact.new_addr())
  176. need_replace = True
  177. if need_replace:
  178. new_header = ",".join(new_addrs)
  179. LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
  180. add_or_replace_header(msg, header, new_header)
  181. else:
  182. LOG.d("No need to replace %s header", header)
  183. def replace_header_when_reply(msg: Message, alias: Alias, header: str):
  184. """
  185. Replace CC or To Reply emails by original emails
  186. """
  187. addrs = get_addrs_from_header(msg, header)
  188. # Nothing to do
  189. if not addrs:
  190. return
  191. new_addrs: [str] = []
  192. for addr in addrs:
  193. name, reply_email = parseaddr(addr)
  194. # no transformation when alias is already in the header
  195. if reply_email == alias.email:
  196. continue
  197. contact = Contact.get_by(reply_email=reply_email)
  198. if not contact:
  199. LOG.warning(
  200. "%s email in reply phase %s must be reply emails", header, reply_email
  201. )
  202. # still keep this email in header
  203. new_addrs.append(addr)
  204. else:
  205. new_addrs.append(formataddr((contact.name, contact.website_email)))
  206. new_header = ",".join(new_addrs)
  207. LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
  208. add_or_replace_header(msg, header, new_header)
  209. def generate_reply_email():
  210. # generate a reply_email, make sure it is unique
  211. # not use while loop to avoid infinite loop
  212. reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}"
  213. for _ in range(1000):
  214. if not Contact.get_by(reply_email=reply_email):
  215. # found!
  216. break
  217. reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}"
  218. return reply_email
  219. def should_append_alias(msg: Message, address: str):
  220. """whether an alias should be appended to TO header in message"""
  221. if msg["To"] and address in msg["To"]:
  222. return False
  223. if msg["Cc"] and address in msg["Cc"]:
  224. return False
  225. return True
  226. _MIME_HEADERS = [
  227. "MIME-Version",
  228. "Content-Type",
  229. "Content-Disposition",
  230. "Content-Transfer-Encoding",
  231. ]
  232. _MIME_HEADERS = [h.lower() for h in _MIME_HEADERS]
  233. def prepare_pgp_message(orig_msg: Message, pgp_fingerprint: str):
  234. msg = MIMEMultipart("encrypted", protocol="application/pgp-encrypted")
  235. # copy all headers from original message except all standard MIME headers
  236. for i in reversed(range(len(orig_msg._headers))):
  237. header_name = orig_msg._headers[i][0].lower()
  238. if header_name.lower() not in _MIME_HEADERS:
  239. msg[header_name] = orig_msg._headers[i][1]
  240. # Delete unnecessary headers in orig_msg except to save space
  241. delete_all_headers_except(
  242. orig_msg, _MIME_HEADERS,
  243. )
  244. first = MIMEApplication(
  245. _subtype="pgp-encrypted", _encoder=encoders.encode_7or8bit, _data=""
  246. )
  247. first.set_payload("Version: 1")
  248. msg.attach(first)
  249. second = MIMEApplication("octet-stream", _encoder=encoders.encode_7or8bit)
  250. second.add_header("Content-Disposition", "inline")
  251. # encrypt original message
  252. encrypted_data = pgp_utils.encrypt_file(
  253. BytesIO(orig_msg.as_bytes()), pgp_fingerprint
  254. )
  255. second.set_payload(encrypted_data)
  256. msg.attach(second)
  257. return msg
  258. def handle_forward(
  259. envelope, smtp: SMTP, msg: Message, rcpt_to: str
  260. ) -> List[Tuple[bool, str]]:
  261. """return whether an email has been delivered and
  262. the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
  263. """
  264. address = rcpt_to.lower().strip() # alias@SL
  265. alias = Alias.get_by(email=address)
  266. if not alias:
  267. LOG.d("alias %s not exist. Try to see if it can be created on the fly", address)
  268. alias = try_auto_create(address)
  269. if not alias:
  270. LOG.d("alias %s cannot be created on-the-fly, return 550", address)
  271. return [(False, "550 SL E3")]
  272. contact = get_or_create_contact(msg["From"], alias)
  273. email_log = EmailLog.create(contact_id=contact.id, user_id=contact.user_id)
  274. if not alias.enabled:
  275. LOG.d("%s is disabled, do not forward", alias)
  276. email_log.blocked = True
  277. db.session.commit()
  278. # do not return 5** to allow user to receive emails later when alias is enabled
  279. return [(True, "250 Message accepted for delivery")]
  280. user = alias.user
  281. ret = []
  282. for mailbox in alias.mailboxes:
  283. ret.append(
  284. forward_email_to_mailbox(
  285. alias, msg, email_log, contact, envelope, smtp, mailbox, user
  286. )
  287. )
  288. return ret
  289. def forward_email_to_mailbox(
  290. alias,
  291. msg: Message,
  292. email_log: EmailLog,
  293. contact: Contact,
  294. envelope,
  295. smtp: SMTP,
  296. mailbox,
  297. user,
  298. ) -> (bool, str):
  299. LOG.d("Forward %s -> %s -> %s", contact, alias, mailbox)
  300. spam_check = True
  301. # create PGP email if needed
  302. if mailbox.pgp_finger_print and user.is_premium():
  303. LOG.d("Encrypt message using mailbox %s", mailbox)
  304. msg = prepare_pgp_message(msg, mailbox.pgp_finger_print)
  305. # no need to spam check for encrypted message
  306. spam_check = False
  307. if spam_check:
  308. is_spam, spam_status = get_spam_info(msg)
  309. if is_spam:
  310. LOG.warning("Email detected as spam. Alias: %s, from: %s", alias, contact)
  311. email_log.is_spam = True
  312. email_log.spam_status = spam_status
  313. handle_spam(contact, alias, msg, user, mailbox.email, email_log)
  314. return False, "550 SL E1"
  315. # add custom header
  316. add_or_replace_header(msg, "X-SimpleLogin-Type", "Forward")
  317. # remove reply-to & sender header if present
  318. delete_header(msg, "Reply-To")
  319. delete_header(msg, "Sender")
  320. delete_header(msg, _IP_HEADER)
  321. add_or_replace_header(msg, _MAILBOX_ID_HEADER, str(mailbox.id))
  322. # change the from header so the sender comes from @SL
  323. # so it can pass DMARC check
  324. # replace the email part in from: header
  325. contact_from_header = msg["From"]
  326. new_from_header = contact.new_addr()
  327. add_or_replace_header(msg, "From", new_from_header)
  328. LOG.d("new_from_header:%s, old header %s", new_from_header, contact_from_header)
  329. # replace CC & To emails by reply-email for all emails that are not alias
  330. replace_header_when_forward(msg, alias, "Cc")
  331. replace_header_when_forward(msg, alias, "To")
  332. # append alias into the TO header if it's not present in To or CC
  333. if should_append_alias(msg, alias.email):
  334. LOG.d("append alias %s to TO header %s", alias, msg["To"])
  335. if msg["To"]:
  336. to_header = msg["To"] + "," + alias.email
  337. else:
  338. to_header = alias.email
  339. add_or_replace_header(msg, "To", to_header.strip())
  340. # add List-Unsubscribe header
  341. if UNSUBSCRIBER:
  342. unsubscribe_link = f"mailto:{UNSUBSCRIBER}?subject={alias.id}="
  343. add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>")
  344. else:
  345. unsubscribe_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
  346. add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>")
  347. add_or_replace_header(
  348. msg, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click"
  349. )
  350. add_dkim_signature(msg, EMAIL_DOMAIN)
  351. LOG.d(
  352. "Forward mail from %s to %s, mail_options %s, rcpt_options %s ",
  353. contact.website_email,
  354. mailbox.email,
  355. envelope.mail_options,
  356. envelope.rcpt_options,
  357. )
  358. # smtp.send_message has UnicodeEncodeErroremail issue
  359. # encode message raw directly instead
  360. smtp.sendmail(
  361. contact.reply_email,
  362. mailbox.email,
  363. msg.as_bytes(),
  364. envelope.mail_options,
  365. envelope.rcpt_options,
  366. )
  367. db.session.commit()
  368. return True, "250 Message accepted for delivery"
  369. def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str):
  370. """
  371. return whether an email has been delivered and
  372. the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
  373. """
  374. reply_email = rcpt_to.lower().strip()
  375. # reply_email must end with EMAIL_DOMAIN
  376. if not reply_email.endswith(EMAIL_DOMAIN):
  377. LOG.warning(f"Reply email {reply_email} has wrong domain")
  378. return False, "550 SL E2"
  379. contact = Contact.get_by(reply_email=reply_email)
  380. if not contact:
  381. LOG.warning(f"No such forward-email with {reply_email} as reply-email")
  382. return False, "550 SL E4"
  383. alias = contact.alias
  384. address: str = contact.alias.email
  385. alias_domain = address[address.find("@") + 1 :]
  386. # alias must end with one of the ALIAS_DOMAINS or custom-domain
  387. if not email_belongs_to_alias_domains(alias.email):
  388. if not CustomDomain.get_by(domain=alias_domain):
  389. return False, "550 SL E5"
  390. user = alias.user
  391. mail_from = envelope.mail_from.lower().strip()
  392. # bounce email initiated by Postfix
  393. # can happen in case emails cannot be delivered to user-email
  394. # in this case Postfix will try to send a bounce report to original sender, which is
  395. # the "reply email"
  396. if mail_from == "<>":
  397. LOG.warning(
  398. "Bounce when sending to alias %s from %s, user %s", alias, contact, user,
  399. )
  400. handle_bounce(contact, alias, msg, user)
  401. return False, "550 SL E6"
  402. mailbox = Mailbox.get_by(email=mail_from, user_id=user.id)
  403. if not mailbox or mailbox not in alias.mailboxes:
  404. # only mailbox can send email to the reply-email
  405. handle_unknown_mailbox(envelope, msg, reply_email, user, alias)
  406. return False, "550 SL E7"
  407. if ENFORCE_SPF and mailbox.force_spf:
  408. ip = msg[_IP_HEADER]
  409. if not spf_pass(ip, envelope, mailbox, user, alias, contact.website_email, msg):
  410. # cannot use 4** here as sender will retry. 5** because that generates bounce report
  411. return True, "250 SL E11"
  412. delete_header(msg, _IP_HEADER)
  413. delete_header(msg, "DKIM-Signature")
  414. delete_header(msg, "Received")
  415. # make the email comes from alias
  416. from_header = alias.email
  417. # add alias name from alias
  418. if alias.name:
  419. LOG.d("Put alias name in from header")
  420. from_header = formataddr((alias.name, alias.email))
  421. elif alias.custom_domain:
  422. LOG.d("Put domain default alias name in from header")
  423. # add alias name from domain
  424. if alias.custom_domain.name:
  425. from_header = formataddr((alias.custom_domain.name, alias.email))
  426. add_or_replace_header(msg, "From", from_header)
  427. # some email providers like ProtonMail adds automatically the Reply-To field
  428. # make sure to delete it
  429. delete_header(msg, "Reply-To")
  430. # remove sender header if present as this could reveal user real email
  431. delete_header(msg, "Sender")
  432. delete_header(msg, "X-Sender")
  433. replace_header_when_reply(msg, alias, "To")
  434. replace_header_when_reply(msg, alias, "Cc")
  435. # Received-SPF is injected by postfix-policyd-spf-python can reveal user original email
  436. delete_header(msg, "Received-SPF")
  437. LOG.d(
  438. "send email from %s to %s, mail_options:%s,rcpt_options:%s",
  439. alias.email,
  440. contact.website_email,
  441. envelope.mail_options,
  442. envelope.rcpt_options,
  443. )
  444. if alias_domain in ALIAS_DOMAINS:
  445. add_dkim_signature(msg, alias_domain)
  446. # add DKIM-Signature for custom-domain alias
  447. else:
  448. custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain)
  449. if custom_domain.dkim_verified:
  450. add_dkim_signature(msg, alias_domain)
  451. smtp.sendmail(
  452. alias.email,
  453. contact.website_email,
  454. msg.as_bytes(),
  455. envelope.mail_options,
  456. envelope.rcpt_options,
  457. )
  458. EmailLog.create(contact_id=contact.id, is_reply=True, user_id=contact.user_id)
  459. db.session.commit()
  460. return True, "250 Message accepted for delivery"
  461. def spf_pass(
  462. ip: str,
  463. envelope,
  464. mailbox: Mailbox,
  465. user: User,
  466. alias: Alias,
  467. contact_email: str,
  468. msg: Message,
  469. ) -> bool:
  470. if ip:
  471. LOG.d("Enforce SPF")
  472. try:
  473. r = spf.check2(i=ip, s=envelope.mail_from.lower(), h=None)
  474. except Exception:
  475. LOG.error("SPF error, mailbox %s, ip %s", mailbox.email, ip)
  476. else:
  477. # TODO: Handle temperr case (e.g. dns timeout)
  478. # only an absolute pass, or no SPF policy at all is 'valid'
  479. if r[0] not in ["pass", "none"]:
  480. LOG.error(
  481. "SPF fail for mailbox %s, reason %s, failed IP %s",
  482. mailbox.email,
  483. r[0],
  484. ip,
  485. )
  486. send_email_with_rate_control(
  487. user,
  488. ALERT_SPF,
  489. mailbox.email,
  490. f"SimpleLogin Alert: attempt to send emails from your alias {alias.email} from unknown IP Address",
  491. render(
  492. "transactional/spf-fail.txt",
  493. name=user.name,
  494. alias=alias.email,
  495. ip=ip,
  496. mailbox_url=URL + f"/dashboard/mailbox/{mailbox.id}#spf",
  497. to_email=contact_email,
  498. subject=msg["Subject"],
  499. time=arrow.now(),
  500. ),
  501. render(
  502. "transactional/spf-fail.html",
  503. name=user.name,
  504. alias=alias.email,
  505. ip=ip,
  506. mailbox_url=URL + f"/dashboard/mailbox/{mailbox.id}#spf",
  507. to_email=contact_email,
  508. subject=msg["Subject"],
  509. time=arrow.now(),
  510. ),
  511. # as the returned error status is 4**,
  512. # the sender will try to resend the email. Send the error message only once
  513. max_alert_24h=1,
  514. )
  515. return False
  516. else:
  517. LOG.warning(
  518. "Could not find %s header %s -> %s",
  519. _IP_HEADER,
  520. mailbox.email,
  521. contact_email,
  522. )
  523. return True
  524. def handle_unknown_mailbox(envelope, msg, reply_email: str, user: User, alias: Alias):
  525. LOG.warning(
  526. f"Reply email can only be used by mailbox. "
  527. f"Actual mail_from: %s. msg from header: %s, reverse-alias %s, %s %s",
  528. envelope.mail_from,
  529. msg["From"],
  530. reply_email,
  531. alias,
  532. user,
  533. )
  534. send_email_with_rate_control(
  535. user,
  536. ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
  537. user.email,
  538. f"Reply from your alias {alias.email} only works from your mailbox",
  539. render(
  540. "transactional/reply-must-use-personal-email.txt",
  541. name=user.name,
  542. alias=alias,
  543. sender=envelope.mail_from,
  544. ),
  545. render(
  546. "transactional/reply-must-use-personal-email.html",
  547. name=user.name,
  548. alias=alias,
  549. sender=envelope.mail_from,
  550. ),
  551. )
  552. # Notify sender that they cannot send emails to this address
  553. send_email_with_rate_control(
  554. user,
  555. ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
  556. envelope.mail_from,
  557. f"Your email ({envelope.mail_from}) is not allowed to send emails to {reply_email}",
  558. render(
  559. "transactional/send-from-alias-from-unknown-sender.txt",
  560. sender=envelope.mail_from,
  561. reply_email=reply_email,
  562. ),
  563. render(
  564. "transactional/send-from-alias-from-unknown-sender.html",
  565. sender=envelope.mail_from,
  566. reply_email=reply_email,
  567. ),
  568. )
  569. def handle_bounce(contact: Contact, alias: Alias, msg: Message, user: User):
  570. address = alias.email
  571. email_log: EmailLog = EmailLog.create(
  572. contact_id=contact.id, bounced=True, user_id=contact.user_id
  573. )
  574. db.session.commit()
  575. nb_bounced = EmailLog.filter_by(contact_id=contact.id, bounced=True).count()
  576. disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
  577. # Store the bounced email
  578. orig_msg = get_orig_message_from_bounce(msg)
  579. # generate a name for the email
  580. random_name = str(uuid.uuid4())
  581. full_report_path = f"refused-emails/full-{random_name}.eml"
  582. s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name)
  583. if not orig_msg:
  584. LOG.error(
  585. "Cannot parse original message from bounce message %s %s %s",
  586. alias,
  587. user,
  588. contact,
  589. )
  590. return
  591. file_path = f"refused-emails/{random_name}.eml"
  592. s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()), random_name)
  593. mailbox_id = int(orig_msg[_MAILBOX_ID_HEADER])
  594. mailbox = Mailbox.get(mailbox_id)
  595. if not mailbox or mailbox.user_id != user.id:
  596. LOG.error(
  597. "Tampered message mailbox_id %s, %s, %s, %s",
  598. mailbox_id,
  599. user,
  600. alias,
  601. contact,
  602. )
  603. return
  604. refused_email = RefusedEmail.create(
  605. path=file_path, full_report_path=full_report_path, user_id=user.id
  606. )
  607. db.session.flush()
  608. email_log.refused_email_id = refused_email.id
  609. email_log.bounced_mailbox_id = mailbox.id
  610. db.session.commit()
  611. LOG.d("Create refused email %s", refused_email)
  612. refused_email_url = (
  613. URL + f"/dashboard/refused_email?highlight_id=" + str(email_log.id)
  614. )
  615. # inform user if this is the first bounced email
  616. if nb_bounced == 1:
  617. LOG.d(
  618. "Inform user %s about bounced email sent by %s to alias %s",
  619. user,
  620. contact.website_email,
  621. address,
  622. )
  623. send_email_with_rate_control(
  624. user,
  625. ALERT_BOUNCE_EMAIL,
  626. # use user mail here as only user is authenticated to see the refused email
  627. user.email,
  628. f"Email from {contact.website_email} to {address} cannot be delivered to your inbox",
  629. render(
  630. "transactional/bounced-email.txt",
  631. name=user.name,
  632. alias=alias,
  633. website_email=contact.website_email,
  634. disable_alias_link=disable_alias_link,
  635. refused_email_url=refused_email_url,
  636. mailbox_email=mailbox.email,
  637. ),
  638. render(
  639. "transactional/bounced-email.html",
  640. name=user.name,
  641. alias=alias,
  642. website_email=contact.website_email,
  643. disable_alias_link=disable_alias_link,
  644. refused_email_url=refused_email_url,
  645. mailbox_email=mailbox.email,
  646. ),
  647. # cannot include bounce email as it can contain spammy text
  648. # bounced_email=msg,
  649. )
  650. # disable the alias the second time email is bounced
  651. elif nb_bounced >= 2:
  652. LOG.d(
  653. "Bounce happens again with alias %s from %s. Disable alias now ",
  654. address,
  655. contact.website_email,
  656. )
  657. alias.enabled = False
  658. db.session.commit()
  659. send_email_with_rate_control(
  660. user,
  661. ALERT_BOUNCE_EMAIL,
  662. # use user mail here as only user is authenticated to see the refused email
  663. user.email,
  664. f"Alias {address} has been disabled due to second undelivered email from {contact.website_email}",
  665. render(
  666. "transactional/automatic-disable-alias.txt",
  667. name=user.name,
  668. alias=alias,
  669. website_email=contact.website_email,
  670. refused_email_url=refused_email_url,
  671. mailbox_email=mailbox.email,
  672. ),
  673. render(
  674. "transactional/automatic-disable-alias.html",
  675. name=user.name,
  676. alias=alias,
  677. website_email=contact.website_email,
  678. refused_email_url=refused_email_url,
  679. mailbox_email=mailbox.email,
  680. ),
  681. # cannot include bounce email as it can contain spammy text
  682. # bounced_email=msg,
  683. )
  684. def handle_spam(
  685. contact: Contact,
  686. alias: Alias,
  687. msg: Message,
  688. user: User,
  689. mailbox_email: str,
  690. email_log: EmailLog,
  691. ):
  692. # Store the report & original email
  693. orig_msg = get_orig_message_from_spamassassin_report(msg)
  694. # generate a name for the email
  695. random_name = str(uuid.uuid4())
  696. full_report_path = f"spams/full-{random_name}.eml"
  697. s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name)
  698. file_path = None
  699. if orig_msg:
  700. file_path = f"spams/{random_name}.eml"
  701. s3.upload_email_from_bytesio(
  702. file_path, BytesIO(orig_msg.as_bytes()), random_name
  703. )
  704. refused_email = RefusedEmail.create(
  705. path=file_path, full_report_path=full_report_path, user_id=user.id
  706. )
  707. db.session.flush()
  708. email_log.refused_email_id = refused_email.id
  709. db.session.commit()
  710. LOG.d("Create spam email %s", refused_email)
  711. refused_email_url = (
  712. URL + f"/dashboard/refused_email?highlight_id=" + str(email_log.id)
  713. )
  714. disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
  715. # inform user
  716. LOG.d(
  717. "Inform user %s about spam email sent by %s to alias %s",
  718. user,
  719. contact.website_email,
  720. alias.email,
  721. )
  722. send_email_with_rate_control(
  723. user,
  724. ALERT_SPAM_EMAIL,
  725. mailbox_email,
  726. f"Email from {contact.website_email} to {alias.email} is detected as spam",
  727. render(
  728. "transactional/spam-email.txt",
  729. name=user.name,
  730. alias=alias,
  731. website_email=contact.website_email,
  732. disable_alias_link=disable_alias_link,
  733. refused_email_url=refused_email_url,
  734. ),
  735. render(
  736. "transactional/spam-email.html",
  737. name=user.name,
  738. alias=alias,
  739. website_email=contact.website_email,
  740. disable_alias_link=disable_alias_link,
  741. refused_email_url=refused_email_url,
  742. ),
  743. )
  744. def handle_unsubscribe(envelope: Envelope):
  745. msg = email.message_from_bytes(envelope.original_content)
  746. # format: alias_id:
  747. subject = msg["Subject"]
  748. try:
  749. # subject has the format {alias.id}=
  750. if subject.endswith("="):
  751. alias_id = int(subject[:-1])
  752. # some email providers might strip off the = suffix
  753. else:
  754. alias_id = int(subject)
  755. alias = Alias.get(alias_id)
  756. except Exception:
  757. LOG.warning("Cannot parse alias from subject %s", msg["Subject"])
  758. return "550 SL E8"
  759. if not alias:
  760. LOG.warning("No such alias %s", alias_id)
  761. return "550 SL E9"
  762. # This sender cannot unsubscribe
  763. mail_from = envelope.mail_from.lower().strip()
  764. mailbox = Mailbox.get_by(user_id=alias.user_id, email=mail_from)
  765. if not mailbox or mailbox not in alias.mailboxes:
  766. LOG.d("%s cannot disable alias %s", envelope.mail_from, alias)
  767. return "550 SL E10"
  768. # Sender is owner of this alias
  769. alias.enabled = False
  770. db.session.commit()
  771. user = alias.user
  772. enable_alias_url = URL + f"/dashboard/?highlight_alias_id={alias.id}"
  773. for mailbox in alias.mailboxes:
  774. send_email(
  775. mailbox.email,
  776. f"Alias {alias.email} has been disabled successfully",
  777. render(
  778. "transactional/unsubscribe-disable-alias.txt",
  779. user=user,
  780. alias=alias.email,
  781. enable_alias_url=enable_alias_url,
  782. ),
  783. render(
  784. "transactional/unsubscribe-disable-alias.html",
  785. user=user,
  786. alias=alias.email,
  787. enable_alias_url=enable_alias_url,
  788. ),
  789. )
  790. return "250 Unsubscribe request accepted"
  791. def handle(envelope: Envelope, smtp: SMTP) -> str:
  792. """Return SMTP status"""
  793. # unsubscribe request
  794. if UNSUBSCRIBER and envelope.rcpt_tos == [UNSUBSCRIBER]:
  795. LOG.d("Handle unsubscribe request from %s", envelope.mail_from)
  796. return handle_unsubscribe(envelope)
  797. # Whether it's necessary to apply greylisting
  798. if greylisting_needed(envelope.mail_from, envelope.rcpt_tos):
  799. LOG.warning(
  800. "Grey listing applied for %s %s", envelope.mail_from, envelope.rcpt_tos
  801. )
  802. return "421 SL Retry later"
  803. # result of all deliveries
  804. # each element is a couple of whether the delivery is successful and the smtp status
  805. res: [(bool, str)] = []
  806. for rcpt_to in envelope.rcpt_tos:
  807. msg = email.message_from_bytes(envelope.original_content)
  808. # Reply case
  809. # recipient starts with "reply+" or "ra+" (ra=reverse-alias) prefix
  810. if rcpt_to.startswith("reply+") or rcpt_to.startswith("ra+"):
  811. LOG.debug(">>> Reply phase %s -> %s", envelope.mail_from, rcpt_to)
  812. is_delivered, smtp_status = handle_reply(envelope, smtp, msg, rcpt_to)
  813. res.append((is_delivered, smtp_status))
  814. else: # Forward case
  815. LOG.debug(">>> Forward phase %s -> %s", envelope.mail_from, rcpt_to)
  816. for is_delivered, smtp_status in handle_forward(
  817. envelope, smtp, msg, rcpt_to
  818. ):
  819. res.append((is_delivered, smtp_status))
  820. for (is_success, smtp_status) in res:
  821. # Consider all deliveries successful if 1 delivery is successful
  822. if is_success:
  823. return smtp_status
  824. # Failed delivery for all, return the first failure
  825. return res[0][1]
  826. class MailHandler:
  827. async def handle_DATA(self, server, session, envelope: Envelope):
  828. LOG.debug(
  829. "===>> New message, mail from %s, rctp tos %s ",
  830. envelope.mail_from,
  831. envelope.rcpt_tos,
  832. )
  833. if POSTFIX_SUBMISSION_TLS:
  834. smtp = SMTP(POSTFIX_SERVER, 587)
  835. smtp.starttls()
  836. else:
  837. smtp = SMTP(POSTFIX_SERVER, 25)
  838. app = new_app()
  839. with app.app_context():
  840. return handle(envelope, smtp)
  841. if __name__ == "__main__":
  842. controller = Controller(MailHandler(), hostname="0.0.0.0", port=20381)
  843. controller.start()
  844. LOG.d("Start mail controller %s %s", controller.hostname, controller.port)
  845. if LOAD_PGP_EMAIL_HANDLER:
  846. LOG.warning("LOAD PGP keys")
  847. app = create_app()
  848. with app.app_context():
  849. load_pgp_public_keys(app)
  850. while True:
  851. time.sleep(2)