email_handler.py 54 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701
  1. """
  2. Handle the email *forward* and *reply*. phase. There are 3 actors:
  3. - contact: who sends emails to alias@sl.co address
  4. - SL email handler (this script)
  5. - user personal email: to be protected. Should never leak to contact.
  6. This script makes sure that in the forward phase, the email that is forwarded to user personal email has the following
  7. envelope and header fields:
  8. Envelope:
  9. mail from: @contact
  10. rcpt to: @personal_email
  11. Header:
  12. From: @contact
  13. To: alias@sl.co # so user knows this email is sent to alias
  14. Reply-to: special@sl.co # magic HERE
  15. And in the reply phase:
  16. Envelope:
  17. mail from: @contact
  18. rcpt to: @contact
  19. Header:
  20. From: alias@sl.co # so for contact the email comes from alias. magic HERE
  21. To: @contact
  22. The special@sl.co allows to hide user personal email when user clicks "Reply" to the forwarded email.
  23. It should contain the following info:
  24. - alias
  25. - @contact
  26. """
  27. import argparse
  28. import asyncio
  29. import email
  30. import os
  31. import time
  32. import uuid
  33. from email import encoders
  34. from email.message import Message
  35. from email.mime.application import MIMEApplication
  36. from email.mime.multipart import MIMEMultipart
  37. from email.utils import formataddr, make_msgid
  38. from io import BytesIO
  39. from smtplib import SMTP, SMTPRecipientsRefused
  40. from typing import List, Tuple, Optional
  41. import aiosmtpd
  42. import aiospamc
  43. import arrow
  44. import spf
  45. from aiosmtpd.controller import Controller
  46. from aiosmtpd.smtp import Envelope
  47. from sqlalchemy.exc import IntegrityError
  48. from app import pgp_utils, s3
  49. from app.alias_utils import try_auto_create
  50. from app.config import (
  51. EMAIL_DOMAIN,
  52. POSTFIX_SERVER,
  53. URL,
  54. ALIAS_DOMAINS,
  55. POSTFIX_SUBMISSION_TLS,
  56. UNSUBSCRIBER,
  57. LOAD_PGP_EMAIL_HANDLER,
  58. ENFORCE_SPF,
  59. ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
  60. ALERT_BOUNCE_EMAIL,
  61. ALERT_SPAM_EMAIL,
  62. ALERT_SPF,
  63. POSTFIX_PORT,
  64. SENDER,
  65. SENDER_DIR,
  66. SPAMASSASSIN_HOST,
  67. MAX_SPAM_SCORE,
  68. MAX_REPLY_PHASE_SPAM_SCORE,
  69. ALERT_SEND_EMAIL_CYCLE,
  70. ALERT_MAILBOX_IS_ALIAS,
  71. PREMIUM_ALIAS_DOMAINS,
  72. )
  73. from app.email_utils import (
  74. send_email,
  75. add_dkim_signature,
  76. add_or_replace_header,
  77. delete_header,
  78. can_create_directory_for_address,
  79. render,
  80. get_orig_message_from_bounce,
  81. delete_all_headers_except,
  82. get_addrs_from_header,
  83. get_spam_info,
  84. get_orig_message_from_spamassassin_report,
  85. parseaddr_unicode,
  86. send_email_with_rate_control,
  87. get_email_domain_part,
  88. copy,
  89. to_bytes,
  90. get_header_from_bounce,
  91. send_email_at_most_times,
  92. is_valid_alias_address_domain,
  93. should_add_dkim_signature,
  94. )
  95. from app.extensions import db
  96. from app.greylisting import greylisting_needed
  97. from app.log import LOG
  98. from app.models import (
  99. Alias,
  100. Contact,
  101. EmailLog,
  102. CustomDomain,
  103. User,
  104. RefusedEmail,
  105. Mailbox,
  106. )
  107. from app.pgp_utils import PGPException
  108. from app.spamassassin_utils import SpamAssassin
  109. from app.utils import random_string
  110. from init_app import load_pgp_public_keys
  111. from server import create_app, create_light_app
  112. # forward or reply
  113. _DIRECTION = "X-SimpleLogin-Type"
  114. _IP_HEADER = "X-SimpleLogin-Client-IP"
  115. _MAILBOX_ID_HEADER = "X-SimpleLogin-Mailbox-ID"
  116. _EMAIL_LOG_ID_HEADER = "X-SimpleLogin-EmailLog-ID"
  117. _MESSAGE_ID = "Message-ID"
  118. _ENVELOPE_FROM = "X-SimpleLogin-Envelope-From"
  119. # fix the database connection leak issue
  120. # use this method instead of create_app
  121. def new_app():
  122. app = create_light_app()
  123. @app.teardown_appcontext
  124. def shutdown_session(response_or_exc):
  125. # same as shutdown_session() in flask-sqlalchemy but this is not enough
  126. db.session.remove()
  127. # dispose the engine too
  128. db.engine.dispose()
  129. return app
  130. def get_or_create_contact(
  131. contact_from_header: str, mail_from: str, alias: Alias
  132. ) -> Contact:
  133. """
  134. contact_from_header is the RFC 2047 format FROM header
  135. """
  136. # contact_from_header can be None, use mail_from in this case instead
  137. contact_from_header = contact_from_header or mail_from
  138. # force convert header to string, sometimes contact_from_header is Header object
  139. contact_from_header = str(contact_from_header)
  140. contact_name, contact_email = parseaddr_unicode(contact_from_header)
  141. if not contact_email:
  142. # From header is wrongly formatted, try with mail_from
  143. LOG.warning("From header is empty, parse mail_from %s %s", mail_from, alias)
  144. contact_name, contact_email = parseaddr_unicode(mail_from)
  145. if not contact_email:
  146. LOG.exception(
  147. "Cannot parse contact from from_header:%s, mail_from:%s",
  148. contact_from_header,
  149. mail_from,
  150. )
  151. contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
  152. if contact:
  153. if contact.name != contact_name:
  154. LOG.d(
  155. "Update contact %s name %s to %s",
  156. contact,
  157. contact.name,
  158. contact_name,
  159. )
  160. contact.name = contact_name
  161. db.session.commit()
  162. if contact.mail_from != mail_from:
  163. LOG.d(
  164. "Update contact %s mail_from %s to %s",
  165. contact,
  166. contact.mail_from,
  167. mail_from,
  168. )
  169. contact.mail_from = mail_from
  170. db.session.commit()
  171. if contact.from_header != contact_from_header:
  172. LOG.d(
  173. "Update contact %s from_header %s to %s",
  174. contact,
  175. contact.from_header,
  176. contact_from_header,
  177. )
  178. contact.from_header = contact_from_header
  179. db.session.commit()
  180. else:
  181. LOG.debug(
  182. "create contact for alias %s and contact %s",
  183. alias,
  184. contact_from_header,
  185. )
  186. reply_email = generate_reply_email()
  187. try:
  188. contact = Contact.create(
  189. user_id=alias.user_id,
  190. alias_id=alias.id,
  191. website_email=contact_email,
  192. name=contact_name,
  193. reply_email=reply_email,
  194. mail_from=mail_from,
  195. from_header=contact_from_header,
  196. )
  197. db.session.commit()
  198. except IntegrityError:
  199. LOG.warning("Contact %s %s already exist", alias, contact_email)
  200. db.session.rollback()
  201. contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
  202. return contact
  203. def replace_header_when_forward(msg: Message, alias: Alias, header: str):
  204. """
  205. Replace CC or To header by Reply emails in forward phase
  206. """
  207. addrs = get_addrs_from_header(msg, header)
  208. # Nothing to do
  209. if not addrs:
  210. return
  211. new_addrs: [str] = []
  212. for addr in addrs:
  213. contact_name, contact_email = parseaddr_unicode(addr)
  214. # no transformation when alias is already in the header
  215. if contact_email == alias.email:
  216. new_addrs.append(addr)
  217. continue
  218. contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
  219. if contact:
  220. # update the contact name if needed
  221. if contact.name != contact_name:
  222. LOG.d(
  223. "Update contact %s name %s to %s",
  224. contact,
  225. contact.name,
  226. contact_name,
  227. )
  228. contact.name = contact_name
  229. db.session.commit()
  230. else:
  231. LOG.debug(
  232. "create contact for alias %s and email %s, header %s",
  233. alias,
  234. contact_email,
  235. header,
  236. )
  237. reply_email = generate_reply_email()
  238. try:
  239. contact = Contact.create(
  240. user_id=alias.user_id,
  241. alias_id=alias.id,
  242. website_email=contact_email,
  243. name=contact_name,
  244. reply_email=reply_email,
  245. is_cc=header.lower() == "cc",
  246. from_header=addr,
  247. )
  248. db.session.commit()
  249. except IntegrityError:
  250. LOG.warning("Contact %s %s already exist", alias, contact_email)
  251. db.session.rollback()
  252. contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
  253. new_addrs.append(contact.new_addr())
  254. if new_addrs:
  255. new_header = ",".join(new_addrs)
  256. LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
  257. add_or_replace_header(msg, header, new_header)
  258. else:
  259. LOG.d("Delete %s header, old value %s", header, msg[header])
  260. delete_header(msg, header)
  261. def replace_header_when_reply(msg: Message, alias: Alias, header: str):
  262. """
  263. Replace CC or To Reply emails by original emails
  264. """
  265. addrs = get_addrs_from_header(msg, header)
  266. # Nothing to do
  267. if not addrs:
  268. return
  269. new_addrs: [str] = []
  270. for addr in addrs:
  271. _, reply_email = parseaddr_unicode(addr)
  272. # no transformation when alias is already in the header
  273. if reply_email == alias.email:
  274. continue
  275. contact = Contact.get_by(reply_email=reply_email)
  276. if not contact:
  277. LOG.warning(
  278. "%s email in reply phase %s must be reply emails", header, reply_email
  279. )
  280. # still keep this email in header
  281. new_addrs.append(addr)
  282. else:
  283. new_addrs.append(formataddr((contact.name, contact.website_email)))
  284. if new_addrs:
  285. new_header = ",".join(new_addrs)
  286. LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
  287. add_or_replace_header(msg, header, new_header)
  288. else:
  289. LOG.d("delete the %s header. Old value %s", header, msg[header])
  290. delete_header(msg, header)
  291. def replace_str_in_msg(msg: Message, fr: str, to: str):
  292. if msg.get_content_maintype() != "text":
  293. return msg
  294. new_body = msg.get_payload(decode=True).replace(fr.encode(), to.encode())
  295. # If utf-8 decoding fails, do not touch message part
  296. try:
  297. new_body = new_body.decode("utf-8")
  298. except:
  299. return msg
  300. cte = (
  301. msg["Content-Transfer-Encoding"].lower()
  302. if msg["Content-Transfer-Encoding"]
  303. else None
  304. )
  305. subtype = msg.get_content_subtype()
  306. delete_header(msg, "Content-Transfer-Encoding")
  307. delete_header(msg, "Content-Type")
  308. email.contentmanager.set_text_content(msg, new_body, subtype=subtype, cte=cte)
  309. return msg
  310. def generate_reply_email():
  311. # generate a reply_email, make sure it is unique
  312. # not use while loop to avoid infinite loop
  313. reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}"
  314. for _ in range(1000):
  315. if not Contact.get_by(reply_email=reply_email):
  316. # found!
  317. break
  318. reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}"
  319. return reply_email
  320. def should_append_alias(msg: Message, address: str):
  321. """whether an alias should be appended to TO header in message"""
  322. # # force convert header to string, sometimes addrs is Header object
  323. if msg["To"] and address.lower() in str(msg["To"]).lower():
  324. return False
  325. if msg["Cc"] and address.lower() in str(msg["Cc"]).lower():
  326. return False
  327. return True
  328. _MIME_HEADERS = [
  329. "MIME-Version",
  330. "Content-Type",
  331. "Content-Disposition",
  332. "Content-Transfer-Encoding",
  333. ]
  334. _MIME_HEADERS = [h.lower() for h in _MIME_HEADERS]
  335. def prepare_pgp_message(orig_msg: Message, pgp_fingerprint: str):
  336. msg = MIMEMultipart("encrypted", protocol="application/pgp-encrypted")
  337. # copy all headers from original message except all standard MIME headers
  338. for i in reversed(range(len(orig_msg._headers))):
  339. header_name = orig_msg._headers[i][0].lower()
  340. if header_name.lower() not in _MIME_HEADERS:
  341. msg[header_name] = orig_msg._headers[i][1]
  342. # Delete unnecessary headers in orig_msg except to save space
  343. delete_all_headers_except(
  344. orig_msg,
  345. _MIME_HEADERS,
  346. )
  347. first = MIMEApplication(
  348. _subtype="pgp-encrypted", _encoder=encoders.encode_7or8bit, _data=""
  349. )
  350. first.set_payload("Version: 1")
  351. msg.attach(first)
  352. second = MIMEApplication(
  353. "octet-stream", _encoder=encoders.encode_7or8bit, name="encrypted.asc"
  354. )
  355. second.add_header("Content-Disposition", 'inline; filename="encrypted.asc"')
  356. # encrypt original message
  357. encrypted_data = pgp_utils.encrypt_file(
  358. BytesIO(orig_msg.as_bytes()), pgp_fingerprint
  359. )
  360. second.set_payload(encrypted_data)
  361. msg.attach(second)
  362. return msg
  363. def handle_email_sent_to_ourself(alias, mailbox, msg: Message, user):
  364. # store the refused email
  365. random_name = str(uuid.uuid4())
  366. full_report_path = f"refused-emails/cycle-{random_name}.eml"
  367. s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name)
  368. refused_email = RefusedEmail.create(
  369. path=None, full_report_path=full_report_path, user_id=alias.user_id
  370. )
  371. db.session.commit()
  372. LOG.d("Create refused email %s", refused_email)
  373. # link available for 6 days as it gets deleted in 7 days
  374. refused_email_url = refused_email.get_url(expires_in=518400)
  375. send_email_at_most_times(
  376. user,
  377. ALERT_SEND_EMAIL_CYCLE,
  378. mailbox.email,
  379. f"Email sent to {alias.email} from its own mailbox {mailbox.email}",
  380. render(
  381. "transactional/cycle-email.txt",
  382. name=user.name or "",
  383. alias=alias,
  384. mailbox=mailbox,
  385. refused_email_url=refused_email_url,
  386. ),
  387. render(
  388. "transactional/cycle-email.html",
  389. name=user.name or "",
  390. alias=alias,
  391. mailbox=mailbox,
  392. refused_email_url=refused_email_url,
  393. ),
  394. )
  395. def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str]]:
  396. """return an array of SMTP status (is_success, smtp_status)
  397. is_success indicates whether an email has been delivered and
  398. smtp_status is the SMTP Status ("250 Message accepted", "550 Non-existent email address", etc)
  399. """
  400. address = rcpt_to # alias@SL
  401. alias = Alias.get_by(email=address)
  402. if not alias:
  403. LOG.d("alias %s not exist. Try to see if it can be created on the fly", address)
  404. alias = try_auto_create(address)
  405. if not alias:
  406. LOG.d("alias %s cannot be created on-the-fly, return 550", address)
  407. return [(False, "550 SL E3 Email not exist")]
  408. if alias.user.disabled:
  409. LOG.warning(
  410. "User %s disabled, disable forwarding emails for %s", alias.user, alias
  411. )
  412. return [(False, "550 SL E20 Account disabled")]
  413. mail_from = envelope.mail_from
  414. for mb in alias.mailboxes:
  415. # email send from a mailbox to alias
  416. if mb.email == mail_from:
  417. LOG.warning("cycle email sent from %s to %s", mb, alias)
  418. handle_email_sent_to_ourself(alias, mb, msg, alias.user)
  419. return [(True, "250 Message accepted for delivery")]
  420. contact = get_or_create_contact(msg["From"], envelope.mail_from, alias)
  421. email_log = EmailLog.create(contact_id=contact.id, user_id=contact.user_id)
  422. db.session.commit()
  423. if not alias.enabled:
  424. LOG.d("%s is disabled, do not forward", alias)
  425. email_log.blocked = True
  426. db.session.commit()
  427. # do not return 5** to allow user to receive emails later when alias is enabled
  428. return [(True, "250 Message accepted for delivery")]
  429. user = alias.user
  430. ret = []
  431. mailboxes = alias.mailboxes
  432. # no valid mailbox
  433. if not mailboxes:
  434. return [(False, "550 SL E16 invalid mailbox")]
  435. # no need to create a copy of message
  436. if len(mailboxes) == 1:
  437. mailbox = mailboxes[0]
  438. if not mailbox.verified:
  439. LOG.debug("Mailbox %s unverified, do not forward", mailbox)
  440. return [(False, "550 SL E18 unverified mailbox")]
  441. else:
  442. ret.append(
  443. forward_email_to_mailbox(
  444. alias, msg, email_log, contact, envelope, mailbox, user
  445. )
  446. )
  447. # create a copy of message for each forward
  448. else:
  449. for mailbox in mailboxes:
  450. if not mailbox.verified:
  451. LOG.debug("Mailbox %s unverified, do not forward", mailbox)
  452. ret.append((False, "550 SL E19 unverified mailbox"))
  453. else:
  454. ret.append(
  455. forward_email_to_mailbox(
  456. alias,
  457. copy(msg),
  458. email_log,
  459. contact,
  460. envelope,
  461. mailbox,
  462. user,
  463. )
  464. )
  465. return ret
  466. def forward_email_to_mailbox(
  467. alias,
  468. msg: Message,
  469. email_log: EmailLog,
  470. contact: Contact,
  471. envelope,
  472. mailbox,
  473. user,
  474. ) -> (bool, str):
  475. LOG.d("Forward %s -> %s -> %s", contact, alias, mailbox)
  476. if mailbox.disabled:
  477. LOG.debug("%s disabled, do not forward")
  478. return False, "550 SL E21 Disabled mailbox"
  479. # sanity check: make sure mailbox is not actually an alias
  480. if get_email_domain_part(alias.email) == get_email_domain_part(mailbox.email):
  481. LOG.warning(
  482. "Mailbox has the same domain as alias. %s -> %s -> %s",
  483. contact,
  484. alias,
  485. mailbox,
  486. )
  487. mailbox_url = f"{URL}/dashboard/mailbox/{mailbox.id}/"
  488. send_email_with_rate_control(
  489. user,
  490. ALERT_MAILBOX_IS_ALIAS,
  491. user.email,
  492. f"Your SimpleLogin mailbox {mailbox.email} cannot be an email alias",
  493. render(
  494. "transactional/mailbox-invalid.txt",
  495. name=user.name or "",
  496. mailbox=mailbox,
  497. mailbox_url=mailbox_url,
  498. ),
  499. render(
  500. "transactional/mailbox-invalid.html",
  501. name=user.name or "",
  502. mailbox=mailbox,
  503. mailbox_url=mailbox_url,
  504. ),
  505. max_nb_alert=1,
  506. )
  507. # retry later
  508. # so when user fixes the mailbox, the email can be delivered
  509. return False, "421 SL E14"
  510. # Spam check
  511. spam_status = ""
  512. is_spam = False
  513. if SPAMASSASSIN_HOST:
  514. start = time.time()
  515. spam_score = get_spam_score(msg)
  516. LOG.d(
  517. "%s -> %s - spam score %s in %s seconds",
  518. contact,
  519. alias,
  520. spam_score,
  521. time.time() - start,
  522. )
  523. email_log.spam_score = spam_score
  524. db.session.commit()
  525. if (user.max_spam_score and spam_score > user.max_spam_score) or (
  526. not user.max_spam_score and spam_score > MAX_SPAM_SCORE
  527. ):
  528. is_spam = True
  529. spam_status = "Spam detected by SpamAssassin server"
  530. else:
  531. is_spam, spam_status = get_spam_info(msg, max_score=user.max_spam_score)
  532. if is_spam:
  533. LOG.warning("Email detected as spam. Alias: %s, from: %s", alias, contact)
  534. email_log.is_spam = True
  535. email_log.spam_status = spam_status
  536. db.session.commit()
  537. handle_spam(contact, alias, msg, user, mailbox, email_log)
  538. return False, "550 SL E1 Email detected as spam"
  539. # create PGP email if needed
  540. if mailbox.pgp_finger_print and user.is_premium() and not alias.disable_pgp:
  541. LOG.d("Encrypt message using mailbox %s", mailbox)
  542. try:
  543. msg = prepare_pgp_message(msg, mailbox.pgp_finger_print)
  544. except PGPException:
  545. LOG.exception(
  546. "Cannot encrypt message %s -> %s. %s %s", contact, alias, mailbox, user
  547. )
  548. # so the client can retry later
  549. return False, "421 SL E12 Retry later"
  550. # add custom header
  551. add_or_replace_header(msg, _DIRECTION, "Forward")
  552. # remove reply-to & sender header if present
  553. delete_header(msg, "Reply-To")
  554. delete_header(msg, "Sender")
  555. delete_header(msg, _IP_HEADER)
  556. add_or_replace_header(msg, _MAILBOX_ID_HEADER, str(mailbox.id))
  557. add_or_replace_header(msg, _EMAIL_LOG_ID_HEADER, str(email_log.id))
  558. add_or_replace_header(msg, _MESSAGE_ID, make_msgid(str(email_log.id), EMAIL_DOMAIN))
  559. add_or_replace_header(msg, _ENVELOPE_FROM, envelope.mail_from)
  560. # change the from header so the sender comes from @SL
  561. # so it can pass DMARC check
  562. # replace the email part in from: header
  563. contact_from_header = msg["From"]
  564. new_from_header = contact.new_addr()
  565. add_or_replace_header(msg, "From", new_from_header)
  566. LOG.d("new_from_header:%s, old header %s", new_from_header, contact_from_header)
  567. # replace CC & To emails by reply-email for all emails that are not alias
  568. replace_header_when_forward(msg, alias, "Cc")
  569. replace_header_when_forward(msg, alias, "To")
  570. # append alias into the TO header if it's not present in To or CC
  571. if should_append_alias(msg, alias.email):
  572. LOG.d("append alias %s to TO header %s", alias, msg["To"])
  573. if msg["To"]:
  574. to_header = msg["To"] + "," + alias.email
  575. else:
  576. to_header = alias.email
  577. add_or_replace_header(msg, "To", to_header.strip())
  578. # add List-Unsubscribe header
  579. unsubscribe_link, via_email = alias.unsubscribe_link()
  580. add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>")
  581. if not via_email:
  582. add_or_replace_header(
  583. msg, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click"
  584. )
  585. add_dkim_signature(msg, EMAIL_DOMAIN)
  586. LOG.d(
  587. "Forward mail from %s to %s, mail_options %s, rcpt_options %s ",
  588. contact.website_email,
  589. mailbox.email,
  590. envelope.mail_options,
  591. envelope.rcpt_options,
  592. )
  593. try:
  594. sl_sendmail(
  595. contact.reply_email,
  596. mailbox.email,
  597. msg,
  598. envelope.mail_options,
  599. envelope.rcpt_options,
  600. )
  601. except SMTPRecipientsRefused:
  602. # that means the mailbox is maybe invalid
  603. LOG.warning(
  604. "SMTPRecipientsRefused forward phase %s -> %s -> %s",
  605. contact,
  606. alias,
  607. mailbox,
  608. )
  609. # return 421 so Postfix can retry later
  610. return False, "421 SL E17 Retry later"
  611. else:
  612. db.session.commit()
  613. return True, "250 Message accepted for delivery"
  614. def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str):
  615. """
  616. return whether an email has been delivered and
  617. the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
  618. """
  619. reply_email = rcpt_to
  620. # reply_email must end with EMAIL_DOMAIN
  621. if not reply_email.endswith(EMAIL_DOMAIN):
  622. LOG.warning(f"Reply email {reply_email} has wrong domain")
  623. return False, "550 SL E2"
  624. contact = Contact.get_by(reply_email=reply_email)
  625. if not contact:
  626. LOG.warning(f"No such forward-email with {reply_email} as reply-email")
  627. return False, "550 SL E4 Email not exist"
  628. alias = contact.alias
  629. address: str = contact.alias.email
  630. alias_domain = address[address.find("@") + 1 :]
  631. # Sanity check: verify alias domain is managed by SimpleLogin
  632. # scenario: a user have removed a domain but due to a bug, the aliases are still there
  633. if not is_valid_alias_address_domain(alias.email):
  634. LOG.exception("%s domain isn't known", alias)
  635. return False, "550 SL E5"
  636. user = alias.user
  637. mail_from = envelope.mail_from
  638. if user.disabled:
  639. LOG.exception(
  640. "User %s disabled, disable sending emails from %s to %s",
  641. user,
  642. alias,
  643. contact,
  644. )
  645. return [(False, "550 SL E20 Account disabled")]
  646. # bounce email initiated by Postfix
  647. # can happen in case emails cannot be delivered to user-email
  648. # in this case Postfix will try to send a bounce report to original sender, which is
  649. # the "reply email"
  650. if mail_from == "<>":
  651. LOG.warning(
  652. "Bounce when sending to alias %s from %s, user %s",
  653. alias,
  654. contact,
  655. user,
  656. )
  657. handle_bounce(contact, alias, msg, user)
  658. return False, "550 SL E6"
  659. # Anti-spoofing
  660. mailbox = get_mailbox_from_mail_from(mail_from, alias)
  661. if not mailbox:
  662. if alias.disable_email_spoofing_check:
  663. # ignore this error, use default alias mailbox
  664. LOG.warning(
  665. "ignore unknown sender to reverse-alias %s: %s -> %s",
  666. mail_from,
  667. alias,
  668. contact,
  669. )
  670. mailbox = alias.mailbox
  671. else:
  672. # only mailbox can send email to the reply-email
  673. handle_unknown_mailbox(envelope, msg, reply_email, user, alias, contact)
  674. return False, "550 SL E7"
  675. if ENFORCE_SPF and mailbox.force_spf and not alias.disable_email_spoofing_check:
  676. ip = msg[_IP_HEADER]
  677. if not spf_pass(ip, envelope, mailbox, user, alias, contact.website_email, msg):
  678. # cannot use 4** here as sender will retry. 5** because that generates bounce report
  679. return True, "250 SL E11"
  680. email_log = EmailLog.create(
  681. contact_id=contact.id, is_reply=True, user_id=contact.user_id
  682. )
  683. # Spam check
  684. spam_status = ""
  685. is_spam = False
  686. # do not use user.max_spam_score here
  687. if SPAMASSASSIN_HOST:
  688. start = time.time()
  689. spam_score = get_spam_score(msg)
  690. LOG.d(
  691. "%s -> %s - spam score %s in %s seconds",
  692. alias,
  693. contact,
  694. spam_score,
  695. time.time() - start,
  696. )
  697. email_log.spam_score = spam_score
  698. if spam_score > MAX_REPLY_PHASE_SPAM_SCORE:
  699. is_spam = True
  700. spam_status = "Spam detected by SpamAssassin server"
  701. else:
  702. is_spam, spam_status = get_spam_info(msg, max_score=MAX_REPLY_PHASE_SPAM_SCORE)
  703. if is_spam:
  704. LOG.exception(
  705. "Reply phase - email sent from %s to %s detected as spam", alias, contact
  706. )
  707. email_log.is_spam = True
  708. email_log.spam_status = spam_status
  709. db.session.commit()
  710. handle_spam(contact, alias, msg, user, mailbox, email_log, is_reply=True)
  711. return False, "550 SL E15 Email detected as spam"
  712. delete_header(msg, _IP_HEADER)
  713. delete_header(msg, "DKIM-Signature")
  714. delete_header(msg, "Received")
  715. # make the email comes from alias
  716. from_header = alias.email
  717. # add alias name from alias
  718. if alias.name:
  719. LOG.d("Put alias name in from header")
  720. from_header = formataddr((alias.name, alias.email))
  721. elif alias.custom_domain:
  722. LOG.d("Put domain default alias name in from header")
  723. # add alias name from domain
  724. if alias.custom_domain.name:
  725. from_header = formataddr((alias.custom_domain.name, alias.email))
  726. add_or_replace_header(msg, "From", from_header)
  727. # some email providers like ProtonMail adds automatically the Reply-To field
  728. # make sure to delete it
  729. delete_header(msg, "Reply-To")
  730. # remove sender header if present as this could reveal user real email
  731. delete_header(msg, "Sender")
  732. delete_header(msg, "X-Sender")
  733. replace_header_when_reply(msg, alias, "To")
  734. replace_header_when_reply(msg, alias, "Cc")
  735. add_or_replace_header(
  736. msg,
  737. _MESSAGE_ID,
  738. make_msgid(str(email_log.id), get_email_domain_part(alias.email)),
  739. )
  740. add_or_replace_header(msg, _EMAIL_LOG_ID_HEADER, str(email_log.id))
  741. add_or_replace_header(msg, _DIRECTION, "Reply")
  742. # Received-SPF is injected by postfix-policyd-spf-python can reveal user original email
  743. delete_header(msg, "Received-SPF")
  744. LOG.d(
  745. "send email from %s to %s, mail_options:%s,rcpt_options:%s",
  746. alias.email,
  747. contact.website_email,
  748. envelope.mail_options,
  749. envelope.rcpt_options,
  750. )
  751. # replace "ra+string@simplelogin.co" by the contact email in the email body
  752. # as this is usually included when replying
  753. if user.replace_reverse_alias:
  754. if msg.is_multipart():
  755. for part in msg.walk():
  756. if part.get_content_maintype() != "text":
  757. continue
  758. part = replace_str_in_msg(part, reply_email, contact.website_email)
  759. else:
  760. msg = replace_str_in_msg(msg, reply_email, contact.website_email)
  761. if should_add_dkim_signature(alias_domain):
  762. add_dkim_signature(msg, alias_domain)
  763. # create PGP email if needed
  764. if contact.pgp_finger_print and user.is_premium():
  765. LOG.d("Encrypt message for contact %s", contact)
  766. try:
  767. msg = prepare_pgp_message(msg, contact.pgp_finger_print)
  768. except PGPException:
  769. LOG.exception(
  770. "Cannot encrypt message %s -> %s. %s %s", alias, contact, mailbox, user
  771. )
  772. # to not save the email_log
  773. db.session.rollback()
  774. # return 421 so the client can retry later
  775. return False, "421 SL E13 Retry later"
  776. try:
  777. sl_sendmail(
  778. alias.email,
  779. contact.website_email,
  780. msg,
  781. envelope.mail_options,
  782. envelope.rcpt_options,
  783. )
  784. except Exception:
  785. # to not save the email_log
  786. db.session.rollback()
  787. LOG.warning("Cannot send email from %s to %s", alias, contact)
  788. send_email(
  789. mailbox.email,
  790. f"Email cannot be sent to {contact.email} from {alias.email}",
  791. render(
  792. "transactional/reply-error.txt",
  793. user=user,
  794. alias=alias,
  795. contact=contact,
  796. contact_domain=get_email_domain_part(contact.email),
  797. ),
  798. render(
  799. "transactional/reply-error.html",
  800. user=user,
  801. alias=alias,
  802. contact=contact,
  803. contact_domain=get_email_domain_part(contact.email),
  804. ),
  805. )
  806. # return 250 even if error as user is already informed of the incident and can retry sending the email
  807. db.session.commit()
  808. return True, "250 Message accepted for delivery"
  809. def get_mailbox_from_mail_from(mail_from: str, alias) -> Optional[Mailbox]:
  810. """return the corresponding mailbox given the mail_from and alias
  811. Usually the mail_from=mailbox.email but it can also be one of the authorized address
  812. """
  813. for mailbox in alias.mailboxes:
  814. if mailbox.email == mail_from:
  815. return mailbox
  816. for address in mailbox.authorized_addresses:
  817. if address.email == mail_from:
  818. LOG.debug(
  819. "Found an authorized address for %s %s %s", alias, mailbox, address
  820. )
  821. return mailbox
  822. return None
  823. def spf_pass(
  824. ip: str,
  825. envelope,
  826. mailbox: Mailbox,
  827. user: User,
  828. alias: Alias,
  829. contact_email: str,
  830. msg: Message,
  831. ) -> bool:
  832. if ip:
  833. LOG.d("Enforce SPF")
  834. try:
  835. r = spf.check2(i=ip, s=envelope.mail_from, h=None)
  836. except Exception:
  837. LOG.exception("SPF error, mailbox %s, ip %s", mailbox.email, ip)
  838. else:
  839. # TODO: Handle temperr case (e.g. dns timeout)
  840. # only an absolute pass, or no SPF policy at all is 'valid'
  841. if r[0] not in ["pass", "none"]:
  842. LOG.warning(
  843. "SPF fail for mailbox %s, reason %s, failed IP %s",
  844. mailbox.email,
  845. r[0],
  846. ip,
  847. )
  848. send_email_with_rate_control(
  849. user,
  850. ALERT_SPF,
  851. mailbox.email,
  852. f"SimpleLogin Alert: attempt to send emails from your alias {alias.email} from unknown IP Address",
  853. render(
  854. "transactional/spf-fail.txt",
  855. name=user.name,
  856. alias=alias.email,
  857. ip=ip,
  858. mailbox_url=URL + f"/dashboard/mailbox/{mailbox.id}#spf",
  859. to_email=contact_email,
  860. subject=msg["Subject"],
  861. time=arrow.now(),
  862. ),
  863. render(
  864. "transactional/spf-fail.html",
  865. name=user.name,
  866. alias=alias.email,
  867. ip=ip,
  868. mailbox_url=URL + f"/dashboard/mailbox/{mailbox.id}#spf",
  869. to_email=contact_email,
  870. subject=msg["Subject"],
  871. time=arrow.now(),
  872. ),
  873. )
  874. return False
  875. else:
  876. LOG.warning(
  877. "Could not find %s header %s -> %s",
  878. _IP_HEADER,
  879. mailbox.email,
  880. contact_email,
  881. )
  882. return True
  883. def handle_unknown_mailbox(
  884. envelope, msg, reply_email: str, user: User, alias: Alias, contact: Contact
  885. ):
  886. LOG.warning(
  887. f"Reply email can only be used by mailbox. "
  888. f"Actual mail_from: %s. msg from header: %s, reverse-alias %s, %s %s %s",
  889. envelope.mail_from,
  890. msg["From"],
  891. reply_email,
  892. alias,
  893. user,
  894. contact,
  895. )
  896. authorize_address_link = (
  897. f"{URL}/dashboard/mailbox/{alias.mailbox_id}/#authorized-address"
  898. )
  899. mailbox_emails = [mailbox.email for mailbox in alias.mailboxes]
  900. send_email_with_rate_control(
  901. user,
  902. ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
  903. user.email,
  904. f"Attempt to use your alias {alias.email} from {envelope.mail_from}",
  905. render(
  906. "transactional/reply-must-use-personal-email.txt",
  907. name=user.name,
  908. alias=alias,
  909. sender=envelope.mail_from,
  910. authorize_address_link=authorize_address_link,
  911. mailbox_emails=mailbox_emails,
  912. ),
  913. render(
  914. "transactional/reply-must-use-personal-email.html",
  915. name=user.name,
  916. alias=alias,
  917. sender=envelope.mail_from,
  918. authorize_address_link=authorize_address_link,
  919. mailbox_emails=mailbox_emails,
  920. ),
  921. )
  922. # Notify sender that they cannot send emails to this address
  923. send_email_with_rate_control(
  924. user,
  925. ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
  926. envelope.mail_from,
  927. f"Your email ({envelope.mail_from}) is not allowed to send emails to {reply_email}",
  928. render(
  929. "transactional/send-from-alias-from-unknown-sender.txt",
  930. sender=envelope.mail_from,
  931. reply_email=reply_email,
  932. ),
  933. render(
  934. "transactional/send-from-alias-from-unknown-sender.html",
  935. sender=envelope.mail_from,
  936. reply_email=reply_email,
  937. ),
  938. )
  939. def handle_bounce(contact: Contact, alias: Alias, msg: Message, user: User):
  940. disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
  941. # Store the bounced email
  942. # generate a name for the email
  943. random_name = str(uuid.uuid4())
  944. full_report_path = f"refused-emails/full-{random_name}.eml"
  945. s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name)
  946. file_path = None
  947. mailbox = None
  948. email_log: EmailLog = None
  949. orig_msg = get_orig_message_from_bounce(msg)
  950. if not orig_msg:
  951. # Some MTA does not return the original message in bounce message
  952. # nothing we can do here
  953. LOG.warning(
  954. "Cannot parse original message from bounce message %s %s %s %s",
  955. alias,
  956. user,
  957. contact,
  958. full_report_path,
  959. )
  960. else:
  961. file_path = f"refused-emails/{random_name}.eml"
  962. s3.upload_email_from_bytesio(
  963. file_path, BytesIO(orig_msg.as_bytes()), random_name
  964. )
  965. try:
  966. mailbox_id = int(orig_msg[_MAILBOX_ID_HEADER])
  967. except TypeError:
  968. LOG.warning(
  969. "cannot parse mailbox from original message header %s",
  970. orig_msg[_MAILBOX_ID_HEADER],
  971. )
  972. else:
  973. mailbox = Mailbox.get(mailbox_id)
  974. if not mailbox or mailbox.user_id != user.id:
  975. LOG.exception(
  976. "Tampered message mailbox_id %s, %s, %s, %s %s",
  977. mailbox_id,
  978. user,
  979. alias,
  980. contact,
  981. full_report_path,
  982. )
  983. # cannot use this tampered mailbox, reset it
  984. mailbox = None
  985. # try to get the original email_log
  986. try:
  987. email_log_id = int(orig_msg[_EMAIL_LOG_ID_HEADER])
  988. except TypeError:
  989. LOG.warning(
  990. "cannot parse email log from original message header %s",
  991. orig_msg[_EMAIL_LOG_ID_HEADER],
  992. )
  993. else:
  994. email_log = EmailLog.get(email_log_id)
  995. refused_email = RefusedEmail.create(
  996. path=file_path, full_report_path=full_report_path, user_id=user.id
  997. )
  998. db.session.flush()
  999. LOG.d("Create refused email %s", refused_email)
  1000. if not mailbox:
  1001. LOG.debug("Try to get mailbox from bounce report")
  1002. try:
  1003. mailbox_id = int(get_header_from_bounce(msg, _MAILBOX_ID_HEADER))
  1004. except Exception:
  1005. LOG.warning("cannot get mailbox-id from bounce report, %s", refused_email)
  1006. else:
  1007. mailbox = Mailbox.get(mailbox_id)
  1008. if not mailbox or mailbox.user_id != user.id:
  1009. LOG.exception(
  1010. "Tampered message mailbox_id %s, %s, %s, %s %s",
  1011. mailbox_id,
  1012. user,
  1013. alias,
  1014. contact,
  1015. full_report_path,
  1016. )
  1017. mailbox = None
  1018. if not email_log:
  1019. LOG.d("Try to get email log from bounce report")
  1020. try:
  1021. email_log_id = int(get_header_from_bounce(msg, _EMAIL_LOG_ID_HEADER))
  1022. except Exception:
  1023. LOG.warning("cannot get email log id from bounce report, %s", refused_email)
  1024. else:
  1025. email_log = EmailLog.get(email_log_id)
  1026. # use the default mailbox as the last option
  1027. if not mailbox:
  1028. LOG.warning("Use %s default mailbox %s", alias, refused_email)
  1029. mailbox = alias.mailbox
  1030. # create a new email log as the last option
  1031. if not email_log:
  1032. LOG.warning("cannot get the original email_log, create a new one")
  1033. email_log: EmailLog = EmailLog.create(
  1034. contact_id=contact.id, user_id=contact.user_id
  1035. )
  1036. email_log.bounced = True
  1037. email_log.refused_email_id = refused_email.id
  1038. email_log.bounced_mailbox_id = mailbox.id
  1039. db.session.commit()
  1040. refused_email_url = (
  1041. URL + f"/dashboard/refused_email?highlight_id=" + str(email_log.id)
  1042. )
  1043. nb_bounced = EmailLog.filter_by(contact_id=contact.id, bounced=True).count()
  1044. if nb_bounced >= 2 and alias.cannot_be_disabled:
  1045. LOG.warning("%s cannot be disabled", alias)
  1046. # inform user if this is the first bounced email
  1047. if nb_bounced == 1 or (nb_bounced >= 2 and alias.cannot_be_disabled):
  1048. LOG.d(
  1049. "Inform user %s about bounced email sent by %s to alias %s",
  1050. user,
  1051. contact.website_email,
  1052. alias,
  1053. )
  1054. send_email_with_rate_control(
  1055. user,
  1056. ALERT_BOUNCE_EMAIL,
  1057. user.email,
  1058. f"Email from {contact.website_email} to {alias.email} cannot be delivered to your inbox",
  1059. render(
  1060. "transactional/bounced-email.txt",
  1061. name=user.name,
  1062. alias=alias,
  1063. website_email=contact.website_email,
  1064. disable_alias_link=disable_alias_link,
  1065. refused_email_url=refused_email_url,
  1066. mailbox_email=mailbox.email,
  1067. ),
  1068. render(
  1069. "transactional/bounced-email.html",
  1070. name=user.name,
  1071. alias=alias,
  1072. website_email=contact.website_email,
  1073. disable_alias_link=disable_alias_link,
  1074. refused_email_url=refused_email_url,
  1075. mailbox_email=mailbox.email,
  1076. ),
  1077. )
  1078. # disable the alias the second time email is bounced
  1079. elif nb_bounced >= 2:
  1080. LOG.d(
  1081. "Bounce happens again with alias %s from %s. Disable alias now ",
  1082. alias,
  1083. contact.website_email,
  1084. )
  1085. alias.enabled = False
  1086. db.session.commit()
  1087. send_email_with_rate_control(
  1088. user,
  1089. ALERT_BOUNCE_EMAIL,
  1090. user.email,
  1091. f"Alias {alias.email} has been disabled due to second undelivered email from {contact.website_email}",
  1092. render(
  1093. "transactional/automatic-disable-alias.txt",
  1094. name=user.name,
  1095. alias=alias,
  1096. website_email=contact.website_email,
  1097. refused_email_url=refused_email_url,
  1098. mailbox_email=mailbox.email,
  1099. ),
  1100. render(
  1101. "transactional/automatic-disable-alias.html",
  1102. name=user.name,
  1103. alias=alias,
  1104. website_email=contact.website_email,
  1105. refused_email_url=refused_email_url,
  1106. mailbox_email=mailbox.email,
  1107. ),
  1108. )
  1109. def handle_spam(
  1110. contact: Contact,
  1111. alias: Alias,
  1112. msg: Message,
  1113. user: User,
  1114. mailbox: Mailbox,
  1115. email_log: EmailLog,
  1116. is_reply=False, # whether the email is in forward or reply phase
  1117. ):
  1118. # Store the report & original email
  1119. orig_msg = get_orig_message_from_spamassassin_report(msg)
  1120. # generate a name for the email
  1121. random_name = str(uuid.uuid4())
  1122. full_report_path = f"spams/full-{random_name}.eml"
  1123. s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name)
  1124. file_path = None
  1125. if orig_msg:
  1126. file_path = f"spams/{random_name}.eml"
  1127. s3.upload_email_from_bytesio(
  1128. file_path, BytesIO(orig_msg.as_bytes()), random_name
  1129. )
  1130. refused_email = RefusedEmail.create(
  1131. path=file_path, full_report_path=full_report_path, user_id=user.id
  1132. )
  1133. db.session.flush()
  1134. email_log.refused_email_id = refused_email.id
  1135. db.session.commit()
  1136. LOG.d("Create spam email %s", refused_email)
  1137. refused_email_url = (
  1138. URL + f"/dashboard/refused_email?highlight_id=" + str(email_log.id)
  1139. )
  1140. disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
  1141. if is_reply:
  1142. LOG.d(
  1143. "Inform %s (%s) about spam email sent from alias %s to %s",
  1144. mailbox,
  1145. user,
  1146. alias,
  1147. contact,
  1148. )
  1149. send_email_with_rate_control(
  1150. user,
  1151. ALERT_SPAM_EMAIL,
  1152. mailbox.email,
  1153. f"Email from {contact.website_email} to {alias.email} is detected as spam",
  1154. render(
  1155. "transactional/spam-email-reply-phase.txt",
  1156. name=user.name,
  1157. alias=alias,
  1158. website_email=contact.website_email,
  1159. disable_alias_link=disable_alias_link,
  1160. refused_email_url=refused_email_url,
  1161. ),
  1162. render(
  1163. "transactional/spam-email-reply-phase.html",
  1164. name=user.name,
  1165. alias=alias,
  1166. website_email=contact.website_email,
  1167. disable_alias_link=disable_alias_link,
  1168. refused_email_url=refused_email_url,
  1169. ),
  1170. )
  1171. else:
  1172. # inform user
  1173. LOG.d(
  1174. "Inform %s (%s) about spam email sent by %s to alias %s",
  1175. mailbox,
  1176. user,
  1177. contact,
  1178. alias,
  1179. )
  1180. send_email_with_rate_control(
  1181. user,
  1182. ALERT_SPAM_EMAIL,
  1183. mailbox.email,
  1184. f"Email from {contact.website_email} to {alias.email} is detected as spam",
  1185. render(
  1186. "transactional/spam-email.txt",
  1187. name=user.name,
  1188. alias=alias,
  1189. website_email=contact.website_email,
  1190. disable_alias_link=disable_alias_link,
  1191. refused_email_url=refused_email_url,
  1192. ),
  1193. render(
  1194. "transactional/spam-email.html",
  1195. name=user.name,
  1196. alias=alias,
  1197. website_email=contact.website_email,
  1198. disable_alias_link=disable_alias_link,
  1199. refused_email_url=refused_email_url,
  1200. ),
  1201. )
  1202. def handle_unsubscribe(envelope: Envelope) -> str:
  1203. """return the SMTP status"""
  1204. msg = email.message_from_bytes(envelope.original_content)
  1205. # format: alias_id:
  1206. subject = msg["Subject"]
  1207. try:
  1208. # subject has the format {alias.id}=
  1209. if subject.endswith("="):
  1210. alias_id = int(subject[:-1])
  1211. # {user.id}*
  1212. elif subject.endswith("*"):
  1213. user_id = int(subject[:-1])
  1214. return handle_unsubscribe_user(user_id, envelope.mail_from)
  1215. # some email providers might strip off the = suffix
  1216. else:
  1217. alias_id = int(subject)
  1218. alias = Alias.get(alias_id)
  1219. except Exception:
  1220. LOG.warning("Cannot parse alias from subject %s", msg["Subject"])
  1221. return "550 SL E8 Wrongly formatted subject"
  1222. if not alias:
  1223. LOG.warning("No such alias %s", alias_id)
  1224. return "550 SL E9 Email not exist"
  1225. # This sender cannot unsubscribe
  1226. mail_from = envelope.mail_from
  1227. # Only alias's owning mailbox can send the unsubscribe request
  1228. mailbox = get_mailbox_from_mail_from(mail_from, alias)
  1229. if not mailbox:
  1230. LOG.d("%s cannot disable alias %s", envelope.mail_from, alias)
  1231. return "550 SL E10 unauthorized"
  1232. # Sender is owner of this alias
  1233. alias.enabled = False
  1234. db.session.commit()
  1235. user = alias.user
  1236. enable_alias_url = URL + f"/dashboard/?highlight_alias_id={alias.id}"
  1237. for mailbox in alias.mailboxes:
  1238. send_email(
  1239. mailbox.email,
  1240. f"Alias {alias.email} has been disabled successfully",
  1241. render(
  1242. "transactional/unsubscribe-disable-alias.txt",
  1243. user=user,
  1244. alias=alias.email,
  1245. enable_alias_url=enable_alias_url,
  1246. ),
  1247. render(
  1248. "transactional/unsubscribe-disable-alias.html",
  1249. user=user,
  1250. alias=alias.email,
  1251. enable_alias_url=enable_alias_url,
  1252. ),
  1253. )
  1254. return "250 Unsubscribe request accepted"
  1255. def handle_unsubscribe_user(user_id: int, mail_from: str) -> str:
  1256. """return the SMTP status"""
  1257. user = User.get(user_id)
  1258. if not user:
  1259. LOG.exception("No such user %s %s", user_id, mail_from)
  1260. return "550 SL E22 so such user"
  1261. if mail_from != user.email:
  1262. LOG.exception("Unauthorized mail_from %s %s", user, mail_from)
  1263. return "550 SL E23 unsubscribe error"
  1264. user.notification = False
  1265. db.session.commit()
  1266. send_email(
  1267. user.email,
  1268. f"You have been unsubscribed from SimpleLogin newsletter",
  1269. render(
  1270. "transactional/unsubscribe-newsletter.txt",
  1271. user=user,
  1272. ),
  1273. render(
  1274. "transactional/unsubscribe-newsletter.html",
  1275. user=user,
  1276. ),
  1277. )
  1278. return "250 Unsubscribe request accepted"
  1279. def handle_sender_email(envelope: Envelope):
  1280. filename = (
  1281. arrow.now().format("YYYY-MM-DD_HH-mm-ss") + "_" + random_string(10) + ".eml"
  1282. )
  1283. filepath = os.path.join(SENDER_DIR, filename)
  1284. with open(filepath, "wb") as f:
  1285. f.write(envelope.original_content)
  1286. LOG.d("Write email to sender at %s", filepath)
  1287. msg = email.message_from_bytes(envelope.original_content)
  1288. orig = get_orig_message_from_bounce(msg)
  1289. if orig:
  1290. LOG.warning(
  1291. "Original message %s -> %s saved at %s", orig["From"], orig["To"], filepath
  1292. )
  1293. return "250 email to sender accepted"
  1294. def handle(envelope: Envelope) -> str:
  1295. """Return SMTP status"""
  1296. # sanitize mail_from, rcpt_tos
  1297. mail_from = envelope.mail_from.lower().strip().replace(" ", "")
  1298. rcpt_tos = [
  1299. rcpt_to.lower().strip().replace(" ", "") for rcpt_to in envelope.rcpt_tos
  1300. ]
  1301. envelope.mail_from = mail_from
  1302. envelope.rcpt_tos = rcpt_tos
  1303. # unsubscribe request
  1304. if UNSUBSCRIBER and rcpt_tos == [UNSUBSCRIBER]:
  1305. LOG.d("Handle unsubscribe request from %s", mail_from)
  1306. return handle_unsubscribe(envelope)
  1307. # emails sent to sender. Probably bounce emails
  1308. if SENDER and rcpt_tos == [SENDER]:
  1309. LOG.d("Handle email sent to sender from %s", mail_from)
  1310. return handle_sender_email(envelope)
  1311. # Whether it's necessary to apply greylisting
  1312. if greylisting_needed(mail_from, rcpt_tos):
  1313. LOG.warning("Grey listing applied for %s %s", mail_from, rcpt_tos)
  1314. return "421 SL Retry later"
  1315. # result of all deliveries
  1316. # each element is a couple of whether the delivery is successful and the smtp status
  1317. res: [(bool, str)] = []
  1318. for rcpt_to in rcpt_tos:
  1319. msg = email.message_from_bytes(envelope.original_content)
  1320. # Reply case
  1321. # recipient starts with "reply+" or "ra+" (ra=reverse-alias) prefix
  1322. if rcpt_to.startswith("reply+") or rcpt_to.startswith("ra+"):
  1323. LOG.debug(">>> Reply phase %s(%s) -> %s", mail_from, msg["From"], rcpt_to)
  1324. is_delivered, smtp_status = handle_reply(envelope, msg, rcpt_to)
  1325. res.append((is_delivered, smtp_status))
  1326. else: # Forward case
  1327. LOG.debug(
  1328. ">>> Forward phase %s(%s) -> %s",
  1329. mail_from,
  1330. msg["From"],
  1331. rcpt_to,
  1332. )
  1333. for is_delivered, smtp_status in handle_forward(envelope, msg, rcpt_to):
  1334. res.append((is_delivered, smtp_status))
  1335. for (is_success, smtp_status) in res:
  1336. # Consider all deliveries successful if 1 delivery is successful
  1337. if is_success:
  1338. return smtp_status
  1339. # Failed delivery for all, return the first failure
  1340. return res[0][1]
  1341. async def get_spam_score_async(message: Message) -> float:
  1342. LOG.debug("get spam score for %s", message[_MESSAGE_ID])
  1343. sa_input = to_bytes(message)
  1344. # Spamassassin requires to have an ending linebreak
  1345. if not sa_input.endswith(b"\n"):
  1346. LOG.d("add linebreak to spamassassin input")
  1347. sa_input += b"\n"
  1348. try:
  1349. # wait for at max 300s which is the default spamd timeout-child
  1350. response = await asyncio.wait_for(
  1351. aiospamc.check(sa_input, host=SPAMASSASSIN_HOST), timeout=300
  1352. )
  1353. return response.headers["Spam"].score
  1354. except asyncio.TimeoutError:
  1355. LOG.exception("SpamAssassin timeout")
  1356. # return a negative score so the message is always considered as ham
  1357. return -999
  1358. except Exception:
  1359. LOG.exception("SpamAssassin exception")
  1360. return -999
  1361. def get_spam_score(message: Message) -> float:
  1362. LOG.debug("get spam score for %s", message[_MESSAGE_ID])
  1363. sa_input = to_bytes(message)
  1364. # Spamassassin requires to have an ending linebreak
  1365. if not sa_input.endswith(b"\n"):
  1366. LOG.d("add linebreak to spamassassin input")
  1367. sa_input += b"\n"
  1368. try:
  1369. # wait for at max 300s which is the default spamd timeout-child
  1370. sa = SpamAssassin(sa_input, host=SPAMASSASSIN_HOST, timeout=300)
  1371. return sa.get_score()
  1372. except Exception:
  1373. LOG.exception("SpamAssassin exception")
  1374. # return a negative score so the message is always considered as ham
  1375. return -999
  1376. def sl_sendmail(from_addr, to_addr, msg: Message, mail_options, rcpt_options):
  1377. """replace smtp.sendmail"""
  1378. if POSTFIX_SUBMISSION_TLS:
  1379. smtp = SMTP(POSTFIX_SERVER, 587)
  1380. smtp.starttls()
  1381. else:
  1382. smtp = SMTP(POSTFIX_SERVER, POSTFIX_PORT or 25)
  1383. # smtp.send_message has UnicodeEncodeErroremail issue
  1384. # encode message raw directly instead
  1385. smtp.sendmail(
  1386. from_addr,
  1387. to_addr,
  1388. msg.as_bytes(),
  1389. mail_options,
  1390. rcpt_options,
  1391. )
  1392. class MailHandler:
  1393. async def handle_DATA(self, server, session, envelope: Envelope):
  1394. try:
  1395. ret = self._handle(envelope)
  1396. return ret
  1397. except Exception:
  1398. LOG.exception(
  1399. "email handling fail %s -> %s",
  1400. envelope.mail_from,
  1401. envelope.rcpt_tos,
  1402. )
  1403. return "421 SL Retry later"
  1404. def _handle(self, envelope: Envelope):
  1405. start = time.time()
  1406. LOG.info(
  1407. "===>> New message, mail from %s, rctp tos %s ",
  1408. envelope.mail_from,
  1409. envelope.rcpt_tos,
  1410. )
  1411. app = new_app()
  1412. with app.app_context():
  1413. ret = handle(envelope)
  1414. LOG.info("takes %s seconds <<===", time.time() - start)
  1415. return ret
  1416. def main(port: int):
  1417. """Use aiosmtpd Controller"""
  1418. controller = Controller(MailHandler(), hostname="0.0.0.0", port=port)
  1419. controller.start()
  1420. LOG.d("Start mail controller %s %s", controller.hostname, controller.port)
  1421. if LOAD_PGP_EMAIL_HANDLER:
  1422. LOG.warning("LOAD PGP keys")
  1423. app = create_app()
  1424. with app.app_context():
  1425. load_pgp_public_keys()
  1426. while True:
  1427. time.sleep(2)
  1428. def asyncio_main(port: int):
  1429. """
  1430. Main entrypoint using asyncio directly without passing by aiosmtpd Controller
  1431. """
  1432. if LOAD_PGP_EMAIL_HANDLER:
  1433. LOG.warning("LOAD PGP keys")
  1434. app = create_app()
  1435. with app.app_context():
  1436. load_pgp_public_keys()
  1437. loop = asyncio.new_event_loop()
  1438. asyncio.set_event_loop(loop)
  1439. lock = asyncio.Lock()
  1440. handler = MailHandler(lock)
  1441. def factory():
  1442. return aiosmtpd.smtp.SMTP(handler, enable_SMTPUTF8=True)
  1443. server = loop.run_until_complete(
  1444. loop.create_server(factory, host="0.0.0.0", port=port)
  1445. )
  1446. try:
  1447. loop.run_forever()
  1448. except KeyboardInterrupt:
  1449. pass
  1450. # Close the server
  1451. LOG.info("Close SMTP server")
  1452. server.close()
  1453. loop.run_until_complete(server.wait_closed())
  1454. loop.close()
  1455. if __name__ == "__main__":
  1456. parser = argparse.ArgumentParser()
  1457. parser.add_argument(
  1458. "-p", "--port", help="SMTP port to listen for", type=int, default=20381
  1459. )
  1460. args = parser.parse_args()
  1461. LOG.info("Listen for port %s", args.port)
  1462. main(port=args.port)