email_handler.py 53 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679
  1. """
  2. Handle the email *forward* and *reply*. phase. There are 3 actors:
  3. - contact: who sends emails to alias@sl.co address
  4. - SL email handler (this script)
  5. - user personal email: to be protected. Should never leak to contact.
  6. This script makes sure that in the forward phase, the email that is forwarded to user personal email has the following
  7. envelope and header fields:
  8. Envelope:
  9. mail from: @contact
  10. rcpt to: @personal_email
  11. Header:
  12. From: @contact
  13. To: alias@sl.co # so user knows this email is sent to alias
  14. Reply-to: special@sl.co # magic HERE
  15. And in the reply phase:
  16. Envelope:
  17. mail from: @contact
  18. rcpt to: @contact
  19. Header:
  20. From: alias@sl.co # so for contact the email comes from alias. magic HERE
  21. To: @contact
  22. The special@sl.co allows to hide user personal email when user clicks "Reply" to the forwarded email.
  23. It should contain the following info:
  24. - alias
  25. - @contact
  26. """
  27. import argparse
  28. import asyncio
  29. import email
  30. import os
  31. import time
  32. import uuid
  33. from email import encoders
  34. from email.message import Message
  35. from email.mime.application import MIMEApplication
  36. from email.mime.multipart import MIMEMultipart
  37. from email.utils import formataddr, make_msgid
  38. from io import BytesIO
  39. from smtplib import SMTP, SMTPRecipientsRefused
  40. from typing import List, Tuple, Optional
  41. import aiosmtpd
  42. import aiospamc
  43. import arrow
  44. import spf
  45. from aiosmtpd.controller import Controller
  46. from aiosmtpd.smtp import Envelope
  47. from sqlalchemy.exc import IntegrityError
  48. from app import pgp_utils, s3
  49. from app.alias_utils import try_auto_create
  50. from app.config import (
  51. EMAIL_DOMAIN,
  52. POSTFIX_SERVER,
  53. URL,
  54. ALIAS_DOMAINS,
  55. POSTFIX_SUBMISSION_TLS,
  56. UNSUBSCRIBER,
  57. LOAD_PGP_EMAIL_HANDLER,
  58. ENFORCE_SPF,
  59. ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
  60. ALERT_BOUNCE_EMAIL,
  61. ALERT_SPAM_EMAIL,
  62. ALERT_SPF,
  63. POSTFIX_PORT,
  64. SENDER,
  65. SENDER_DIR,
  66. SPAMASSASSIN_HOST,
  67. MAX_SPAM_SCORE,
  68. MAX_REPLY_PHASE_SPAM_SCORE,
  69. ALERT_SEND_EMAIL_CYCLE,
  70. ALERT_MAILBOX_IS_ALIAS,
  71. PREMIUM_ALIAS_DOMAINS,
  72. )
  73. from app.email_utils import (
  74. send_email,
  75. add_dkim_signature,
  76. add_or_replace_header,
  77. delete_header,
  78. can_create_directory_for_address,
  79. render,
  80. get_orig_message_from_bounce,
  81. delete_all_headers_except,
  82. get_addrs_from_header,
  83. get_spam_info,
  84. get_orig_message_from_spamassassin_report,
  85. parseaddr_unicode,
  86. send_email_with_rate_control,
  87. get_email_domain_part,
  88. copy,
  89. to_bytes,
  90. get_header_from_bounce,
  91. send_email_at_most_times,
  92. is_valid_alias_address_domain,
  93. should_add_dkim_signature,
  94. )
  95. from app.extensions import db
  96. from app.greylisting import greylisting_needed
  97. from app.log import LOG
  98. from app.models import (
  99. Alias,
  100. Contact,
  101. EmailLog,
  102. CustomDomain,
  103. User,
  104. RefusedEmail,
  105. Mailbox,
  106. )
  107. from app.pgp_utils import PGPException
  108. from app.spamassassin_utils import SpamAssassin
  109. from app.utils import random_string
  110. from init_app import load_pgp_public_keys
  111. from server import create_app, create_light_app
  112. # forward or reply
  113. _DIRECTION = "X-SimpleLogin-Type"
  114. _IP_HEADER = "X-SimpleLogin-Client-IP"
  115. _MAILBOX_ID_HEADER = "X-SimpleLogin-Mailbox-ID"
  116. _EMAIL_LOG_ID_HEADER = "X-SimpleLogin-EmailLog-ID"
  117. _MESSAGE_ID = "Message-ID"
  118. # fix the database connection leak issue
  119. # use this method instead of create_app
  120. def new_app():
  121. app = create_light_app()
  122. @app.teardown_appcontext
  123. def shutdown_session(response_or_exc):
  124. # same as shutdown_session() in flask-sqlalchemy but this is not enough
  125. db.session.remove()
  126. # dispose the engine too
  127. db.engine.dispose()
  128. return app
  129. def get_or_create_contact(
  130. contact_from_header: str, mail_from: str, alias: Alias
  131. ) -> Contact:
  132. """
  133. contact_from_header is the RFC 2047 format FROM header
  134. """
  135. # contact_from_header can be None, use mail_from in this case instead
  136. contact_from_header = contact_from_header or mail_from
  137. # force convert header to string, sometimes contact_from_header is Header object
  138. contact_from_header = str(contact_from_header)
  139. contact_name, contact_email = parseaddr_unicode(contact_from_header)
  140. if not contact_email:
  141. # From header is wrongly formatted, try with mail_from
  142. LOG.warning("From header is empty, parse mail_from %s %s", mail_from, alias)
  143. contact_name, contact_email = parseaddr_unicode(mail_from)
  144. if not contact_email:
  145. LOG.exception(
  146. "Cannot parse contact from from_header:%s, mail_from:%s",
  147. contact_from_header,
  148. mail_from,
  149. )
  150. contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
  151. if contact:
  152. if contact.name != contact_name:
  153. LOG.d(
  154. "Update contact %s name %s to %s",
  155. contact,
  156. contact.name,
  157. contact_name,
  158. )
  159. contact.name = contact_name
  160. db.session.commit()
  161. else:
  162. LOG.debug(
  163. "create contact for alias %s and contact %s",
  164. alias,
  165. contact_from_header,
  166. )
  167. reply_email = generate_reply_email()
  168. try:
  169. contact = Contact.create(
  170. user_id=alias.user_id,
  171. alias_id=alias.id,
  172. website_email=contact_email,
  173. name=contact_name,
  174. reply_email=reply_email,
  175. mail_from=mail_from,
  176. from_header=contact_from_header,
  177. )
  178. db.session.commit()
  179. except IntegrityError:
  180. LOG.warning("Contact %s %s already exist", alias, contact_email)
  181. db.session.rollback()
  182. contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
  183. return contact
  184. def replace_header_when_forward(msg: Message, alias: Alias, header: str):
  185. """
  186. Replace CC or To header by Reply emails in forward phase
  187. """
  188. addrs = get_addrs_from_header(msg, header)
  189. # Nothing to do
  190. if not addrs:
  191. return
  192. new_addrs: [str] = []
  193. for addr in addrs:
  194. contact_name, contact_email = parseaddr_unicode(addr)
  195. # no transformation when alias is already in the header
  196. if contact_email == alias.email:
  197. new_addrs.append(addr)
  198. continue
  199. contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
  200. if contact:
  201. # update the contact name if needed
  202. if contact.name != contact_name:
  203. LOG.d(
  204. "Update contact %s name %s to %s",
  205. contact,
  206. contact.name,
  207. contact_name,
  208. )
  209. contact.name = contact_name
  210. db.session.commit()
  211. else:
  212. LOG.debug(
  213. "create contact for alias %s and email %s, header %s",
  214. alias,
  215. contact_email,
  216. header,
  217. )
  218. reply_email = generate_reply_email()
  219. try:
  220. contact = Contact.create(
  221. user_id=alias.user_id,
  222. alias_id=alias.id,
  223. website_email=contact_email,
  224. name=contact_name,
  225. reply_email=reply_email,
  226. is_cc=header.lower() == "cc",
  227. from_header=addr,
  228. )
  229. db.session.commit()
  230. except IntegrityError:
  231. LOG.warning("Contact %s %s already exist", alias, contact_email)
  232. db.session.rollback()
  233. contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
  234. new_addrs.append(contact.new_addr())
  235. if new_addrs:
  236. new_header = ",".join(new_addrs)
  237. LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
  238. add_or_replace_header(msg, header, new_header)
  239. else:
  240. LOG.d("Delete %s header, old value %s", header, msg[header])
  241. delete_header(msg, header)
  242. def replace_header_when_reply(msg: Message, alias: Alias, header: str):
  243. """
  244. Replace CC or To Reply emails by original emails
  245. """
  246. addrs = get_addrs_from_header(msg, header)
  247. # Nothing to do
  248. if not addrs:
  249. return
  250. new_addrs: [str] = []
  251. for addr in addrs:
  252. _, reply_email = parseaddr_unicode(addr)
  253. # no transformation when alias is already in the header
  254. if reply_email == alias.email:
  255. continue
  256. contact = Contact.get_by(reply_email=reply_email)
  257. if not contact:
  258. LOG.warning(
  259. "%s email in reply phase %s must be reply emails", header, reply_email
  260. )
  261. # still keep this email in header
  262. new_addrs.append(addr)
  263. else:
  264. new_addrs.append(formataddr((contact.name, contact.website_email)))
  265. if new_addrs:
  266. new_header = ",".join(new_addrs)
  267. LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
  268. add_or_replace_header(msg, header, new_header)
  269. else:
  270. LOG.d("delete the %s header. Old value %s", header, msg[header])
  271. delete_header(msg, header)
  272. def replace_str_in_msg(msg: Message, fr: str, to: str):
  273. if msg.get_content_maintype() != "text":
  274. return msg
  275. new_body = msg.get_payload(decode=True).replace(fr.encode(), to.encode())
  276. # If utf-8 decoding fails, do not touch message part
  277. try:
  278. new_body = new_body.decode("utf-8")
  279. except:
  280. return msg
  281. cte = (
  282. msg["Content-Transfer-Encoding"].lower()
  283. if msg["Content-Transfer-Encoding"]
  284. else None
  285. )
  286. subtype = msg.get_content_subtype()
  287. delete_header(msg, "Content-Transfer-Encoding")
  288. delete_header(msg, "Content-Type")
  289. email.contentmanager.set_text_content(msg, new_body, subtype=subtype, cte=cte)
  290. return msg
  291. def generate_reply_email():
  292. # generate a reply_email, make sure it is unique
  293. # not use while loop to avoid infinite loop
  294. reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}"
  295. for _ in range(1000):
  296. if not Contact.get_by(reply_email=reply_email):
  297. # found!
  298. break
  299. reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}"
  300. return reply_email
  301. def should_append_alias(msg: Message, address: str):
  302. """whether an alias should be appended to TO header in message"""
  303. # # force convert header to string, sometimes addrs is Header object
  304. if msg["To"] and address.lower() in str(msg["To"]).lower():
  305. return False
  306. if msg["Cc"] and address.lower() in str(msg["Cc"]).lower():
  307. return False
  308. return True
  309. _MIME_HEADERS = [
  310. "MIME-Version",
  311. "Content-Type",
  312. "Content-Disposition",
  313. "Content-Transfer-Encoding",
  314. ]
  315. _MIME_HEADERS = [h.lower() for h in _MIME_HEADERS]
  316. def prepare_pgp_message(orig_msg: Message, pgp_fingerprint: str):
  317. msg = MIMEMultipart("encrypted", protocol="application/pgp-encrypted")
  318. # copy all headers from original message except all standard MIME headers
  319. for i in reversed(range(len(orig_msg._headers))):
  320. header_name = orig_msg._headers[i][0].lower()
  321. if header_name.lower() not in _MIME_HEADERS:
  322. msg[header_name] = orig_msg._headers[i][1]
  323. # Delete unnecessary headers in orig_msg except to save space
  324. delete_all_headers_except(
  325. orig_msg,
  326. _MIME_HEADERS,
  327. )
  328. first = MIMEApplication(
  329. _subtype="pgp-encrypted", _encoder=encoders.encode_7or8bit, _data=""
  330. )
  331. first.set_payload("Version: 1")
  332. msg.attach(first)
  333. second = MIMEApplication(
  334. "octet-stream", _encoder=encoders.encode_7or8bit, name="encrypted.asc"
  335. )
  336. second.add_header("Content-Disposition", 'inline; filename="encrypted.asc"')
  337. # encrypt original message
  338. encrypted_data = pgp_utils.encrypt_file(
  339. BytesIO(orig_msg.as_bytes()), pgp_fingerprint
  340. )
  341. second.set_payload(encrypted_data)
  342. msg.attach(second)
  343. return msg
  344. def handle_email_sent_to_ourself(alias, mailbox, msg: Message, user):
  345. # store the refused email
  346. random_name = str(uuid.uuid4())
  347. full_report_path = f"refused-emails/cycle-{random_name}.eml"
  348. s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name)
  349. refused_email = RefusedEmail.create(
  350. path=None, full_report_path=full_report_path, user_id=alias.user_id
  351. )
  352. db.session.commit()
  353. LOG.d("Create refused email %s", refused_email)
  354. # link available for 6 days as it gets deleted in 7 days
  355. refused_email_url = refused_email.get_url(expires_in=518400)
  356. send_email_at_most_times(
  357. user,
  358. ALERT_SEND_EMAIL_CYCLE,
  359. mailbox.email,
  360. f"Email sent to {alias.email} from its own mailbox {mailbox.email}",
  361. render(
  362. "transactional/cycle-email.txt",
  363. name=user.name or "",
  364. alias=alias,
  365. mailbox=mailbox,
  366. refused_email_url=refused_email_url,
  367. ),
  368. render(
  369. "transactional/cycle-email.html",
  370. name=user.name or "",
  371. alias=alias,
  372. mailbox=mailbox,
  373. refused_email_url=refused_email_url,
  374. ),
  375. )
  376. def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str]]:
  377. """return an array of SMTP status (is_success, smtp_status)
  378. is_success indicates whether an email has been delivered and
  379. smtp_status is the SMTP Status ("250 Message accepted", "550 Non-existent email address", etc)
  380. """
  381. address = rcpt_to # alias@SL
  382. alias = Alias.get_by(email=address)
  383. if not alias:
  384. LOG.d("alias %s not exist. Try to see if it can be created on the fly", address)
  385. alias = try_auto_create(address)
  386. if not alias:
  387. LOG.d("alias %s cannot be created on-the-fly, return 550", address)
  388. return [(False, "550 SL E3 Email not exist")]
  389. if alias.user.disabled:
  390. LOG.exception(
  391. "User %s disabled, disable forwarding emails for %s", alias.user, alias
  392. )
  393. return [(False, "550 SL E20 Account disabled")]
  394. mail_from = envelope.mail_from
  395. for mb in alias.mailboxes:
  396. # email send from a mailbox to alias
  397. if mb.email == mail_from:
  398. LOG.warning("cycle email sent from %s to %s", mb, alias)
  399. handle_email_sent_to_ourself(alias, mb, msg, alias.user)
  400. return [(True, "250 Message accepted for delivery")]
  401. contact = get_or_create_contact(msg["From"], envelope.mail_from, alias)
  402. email_log = EmailLog.create(contact_id=contact.id, user_id=contact.user_id)
  403. db.session.commit()
  404. if not alias.enabled:
  405. LOG.d("%s is disabled, do not forward", alias)
  406. email_log.blocked = True
  407. db.session.commit()
  408. # do not return 5** to allow user to receive emails later when alias is enabled
  409. return [(True, "250 Message accepted for delivery")]
  410. user = alias.user
  411. ret = []
  412. mailboxes = alias.mailboxes
  413. # no valid mailbox
  414. if not mailboxes:
  415. return [(False, "550 SL E16 invalid mailbox")]
  416. # no need to create a copy of message
  417. if len(mailboxes) == 1:
  418. mailbox = mailboxes[0]
  419. if not mailbox.verified:
  420. LOG.debug("Mailbox %s unverified, do not forward", mailbox)
  421. return [(False, "550 SL E18 unverified mailbox")]
  422. else:
  423. ret.append(
  424. forward_email_to_mailbox(
  425. alias, msg, email_log, contact, envelope, mailbox, user
  426. )
  427. )
  428. # create a copy of message for each forward
  429. else:
  430. for mailbox in mailboxes:
  431. if not mailbox.verified:
  432. LOG.debug("Mailbox %s unverified, do not forward", mailbox)
  433. ret.append((False, "550 SL E19 unverified mailbox"))
  434. else:
  435. ret.append(
  436. forward_email_to_mailbox(
  437. alias,
  438. copy(msg),
  439. email_log,
  440. contact,
  441. envelope,
  442. mailbox,
  443. user,
  444. )
  445. )
  446. return ret
  447. def forward_email_to_mailbox(
  448. alias,
  449. msg: Message,
  450. email_log: EmailLog,
  451. contact: Contact,
  452. envelope,
  453. mailbox,
  454. user,
  455. ) -> (bool, str):
  456. LOG.d("Forward %s -> %s -> %s", contact, alias, mailbox)
  457. if mailbox.disabled:
  458. LOG.debug("%s disabled, do not forward")
  459. return False, "550 SL E21 Disabled mailbox"
  460. # sanity check: make sure mailbox is not actually an alias
  461. if get_email_domain_part(alias.email) == get_email_domain_part(mailbox.email):
  462. LOG.warning(
  463. "Mailbox has the same domain as alias. %s -> %s -> %s",
  464. contact,
  465. alias,
  466. mailbox,
  467. )
  468. mailbox_url = f"{URL}/dashboard/mailbox/{mailbox.id}/"
  469. send_email_with_rate_control(
  470. user,
  471. ALERT_MAILBOX_IS_ALIAS,
  472. user.email,
  473. f"Your SimpleLogin mailbox {mailbox.email} cannot be an email alias",
  474. render(
  475. "transactional/mailbox-invalid.txt",
  476. name=user.name or "",
  477. mailbox=mailbox,
  478. mailbox_url=mailbox_url,
  479. ),
  480. render(
  481. "transactional/mailbox-invalid.html",
  482. name=user.name or "",
  483. mailbox=mailbox,
  484. mailbox_url=mailbox_url,
  485. ),
  486. max_nb_alert=1,
  487. )
  488. # retry later
  489. # so when user fixes the mailbox, the email can be delivered
  490. return False, "421 SL E14"
  491. # Spam check
  492. spam_status = ""
  493. is_spam = False
  494. if SPAMASSASSIN_HOST:
  495. start = time.time()
  496. spam_score = get_spam_score(msg)
  497. LOG.d(
  498. "%s -> %s - spam score %s in %s seconds",
  499. contact,
  500. alias,
  501. spam_score,
  502. time.time() - start,
  503. )
  504. email_log.spam_score = spam_score
  505. db.session.commit()
  506. if (user.max_spam_score and spam_score > user.max_spam_score) or (
  507. not user.max_spam_score and spam_score > MAX_SPAM_SCORE
  508. ):
  509. is_spam = True
  510. spam_status = "Spam detected by SpamAssassin server"
  511. else:
  512. is_spam, spam_status = get_spam_info(msg, max_score=user.max_spam_score)
  513. if is_spam:
  514. LOG.warning("Email detected as spam. Alias: %s, from: %s", alias, contact)
  515. email_log.is_spam = True
  516. email_log.spam_status = spam_status
  517. db.session.commit()
  518. handle_spam(contact, alias, msg, user, mailbox, email_log)
  519. return False, "550 SL E1 Email detected as spam"
  520. # create PGP email if needed
  521. if mailbox.pgp_finger_print and user.is_premium() and not alias.disable_pgp:
  522. LOG.d("Encrypt message using mailbox %s", mailbox)
  523. try:
  524. msg = prepare_pgp_message(msg, mailbox.pgp_finger_print)
  525. except PGPException:
  526. LOG.exception(
  527. "Cannot encrypt message %s -> %s. %s %s", contact, alias, mailbox, user
  528. )
  529. # so the client can retry later
  530. return False, "421 SL E12 Retry later"
  531. # add custom header
  532. add_or_replace_header(msg, _DIRECTION, "Forward")
  533. # remove reply-to & sender header if present
  534. delete_header(msg, "Reply-To")
  535. delete_header(msg, "Sender")
  536. delete_header(msg, _IP_HEADER)
  537. add_or_replace_header(msg, _MAILBOX_ID_HEADER, str(mailbox.id))
  538. add_or_replace_header(msg, _EMAIL_LOG_ID_HEADER, str(email_log.id))
  539. add_or_replace_header(msg, _MESSAGE_ID, make_msgid(str(email_log.id), EMAIL_DOMAIN))
  540. # change the from header so the sender comes from @SL
  541. # so it can pass DMARC check
  542. # replace the email part in from: header
  543. contact_from_header = msg["From"]
  544. new_from_header = contact.new_addr()
  545. add_or_replace_header(msg, "From", new_from_header)
  546. LOG.d("new_from_header:%s, old header %s", new_from_header, contact_from_header)
  547. # replace CC & To emails by reply-email for all emails that are not alias
  548. replace_header_when_forward(msg, alias, "Cc")
  549. replace_header_when_forward(msg, alias, "To")
  550. # append alias into the TO header if it's not present in To or CC
  551. if should_append_alias(msg, alias.email):
  552. LOG.d("append alias %s to TO header %s", alias, msg["To"])
  553. if msg["To"]:
  554. to_header = msg["To"] + "," + alias.email
  555. else:
  556. to_header = alias.email
  557. add_or_replace_header(msg, "To", to_header.strip())
  558. # add List-Unsubscribe header
  559. unsubscribe_link, via_email = alias.unsubscribe_link()
  560. add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>")
  561. if not via_email:
  562. add_or_replace_header(
  563. msg, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click"
  564. )
  565. add_dkim_signature(msg, EMAIL_DOMAIN)
  566. LOG.d(
  567. "Forward mail from %s to %s, mail_options %s, rcpt_options %s ",
  568. contact.website_email,
  569. mailbox.email,
  570. envelope.mail_options,
  571. envelope.rcpt_options,
  572. )
  573. try:
  574. sl_sendmail(
  575. contact.reply_email,
  576. mailbox.email,
  577. msg,
  578. envelope.mail_options,
  579. envelope.rcpt_options,
  580. )
  581. except SMTPRecipientsRefused:
  582. # that means the mailbox is maybe invalid
  583. LOG.warning(
  584. "SMTPRecipientsRefused forward phase %s -> %s -> %s",
  585. contact,
  586. alias,
  587. mailbox,
  588. )
  589. # return 421 so Postfix can retry later
  590. return False, "421 SL E17 Retry later"
  591. else:
  592. db.session.commit()
  593. return True, "250 Message accepted for delivery"
  594. def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str):
  595. """
  596. return whether an email has been delivered and
  597. the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
  598. """
  599. reply_email = rcpt_to
  600. # reply_email must end with EMAIL_DOMAIN
  601. if not reply_email.endswith(EMAIL_DOMAIN):
  602. LOG.warning(f"Reply email {reply_email} has wrong domain")
  603. return False, "550 SL E2"
  604. contact = Contact.get_by(reply_email=reply_email)
  605. if not contact:
  606. LOG.warning(f"No such forward-email with {reply_email} as reply-email")
  607. return False, "550 SL E4 Email not exist"
  608. alias = contact.alias
  609. address: str = contact.alias.email
  610. alias_domain = address[address.find("@") + 1 :]
  611. # Sanity check: verify alias domain is managed by SimpleLogin
  612. # scenario: a user have removed a domain but due to a bug, the aliases are still there
  613. if not is_valid_alias_address_domain(alias.email):
  614. LOG.exception("%s domain isn't known", alias)
  615. return False, "550 SL E5"
  616. user = alias.user
  617. mail_from = envelope.mail_from
  618. if user.disabled:
  619. LOG.exception(
  620. "User %s disabled, disable sending emails from %s to %s",
  621. user,
  622. alias,
  623. contact,
  624. )
  625. return [(False, "550 SL E20 Account disabled")]
  626. # bounce email initiated by Postfix
  627. # can happen in case emails cannot be delivered to user-email
  628. # in this case Postfix will try to send a bounce report to original sender, which is
  629. # the "reply email"
  630. if mail_from == "<>":
  631. LOG.warning(
  632. "Bounce when sending to alias %s from %s, user %s",
  633. alias,
  634. contact,
  635. user,
  636. )
  637. handle_bounce(contact, alias, msg, user)
  638. return False, "550 SL E6"
  639. # Anti-spoofing
  640. mailbox = get_mailbox_from_mail_from(mail_from, alias)
  641. if not mailbox:
  642. if alias.disable_email_spoofing_check:
  643. # ignore this error, use default alias mailbox
  644. LOG.warning(
  645. "ignore unknown sender to reverse-alias %s: %s -> %s",
  646. mail_from,
  647. alias,
  648. contact,
  649. )
  650. mailbox = alias.mailbox
  651. else:
  652. # only mailbox can send email to the reply-email
  653. handle_unknown_mailbox(envelope, msg, reply_email, user, alias, contact)
  654. return False, "550 SL E7"
  655. if ENFORCE_SPF and mailbox.force_spf and not alias.disable_email_spoofing_check:
  656. ip = msg[_IP_HEADER]
  657. if not spf_pass(ip, envelope, mailbox, user, alias, contact.website_email, msg):
  658. # cannot use 4** here as sender will retry. 5** because that generates bounce report
  659. return True, "250 SL E11"
  660. email_log = EmailLog.create(
  661. contact_id=contact.id, is_reply=True, user_id=contact.user_id
  662. )
  663. # Spam check
  664. spam_status = ""
  665. is_spam = False
  666. # do not use user.max_spam_score here
  667. if SPAMASSASSIN_HOST:
  668. start = time.time()
  669. spam_score = get_spam_score(msg)
  670. LOG.d(
  671. "%s -> %s - spam score %s in %s seconds",
  672. alias,
  673. contact,
  674. spam_score,
  675. time.time() - start,
  676. )
  677. email_log.spam_score = spam_score
  678. if spam_score > MAX_REPLY_PHASE_SPAM_SCORE:
  679. is_spam = True
  680. spam_status = "Spam detected by SpamAssassin server"
  681. else:
  682. is_spam, spam_status = get_spam_info(msg, max_score=MAX_REPLY_PHASE_SPAM_SCORE)
  683. if is_spam:
  684. LOG.exception(
  685. "Reply phase - email sent from %s to %s detected as spam", alias, contact
  686. )
  687. email_log.is_spam = True
  688. email_log.spam_status = spam_status
  689. db.session.commit()
  690. handle_spam(contact, alias, msg, user, mailbox, email_log, is_reply=True)
  691. return False, "550 SL E15 Email detected as spam"
  692. delete_header(msg, _IP_HEADER)
  693. delete_header(msg, "DKIM-Signature")
  694. delete_header(msg, "Received")
  695. # make the email comes from alias
  696. from_header = alias.email
  697. # add alias name from alias
  698. if alias.name:
  699. LOG.d("Put alias name in from header")
  700. from_header = formataddr((alias.name, alias.email))
  701. elif alias.custom_domain:
  702. LOG.d("Put domain default alias name in from header")
  703. # add alias name from domain
  704. if alias.custom_domain.name:
  705. from_header = formataddr((alias.custom_domain.name, alias.email))
  706. add_or_replace_header(msg, "From", from_header)
  707. # some email providers like ProtonMail adds automatically the Reply-To field
  708. # make sure to delete it
  709. delete_header(msg, "Reply-To")
  710. # remove sender header if present as this could reveal user real email
  711. delete_header(msg, "Sender")
  712. delete_header(msg, "X-Sender")
  713. replace_header_when_reply(msg, alias, "To")
  714. replace_header_when_reply(msg, alias, "Cc")
  715. add_or_replace_header(
  716. msg,
  717. _MESSAGE_ID,
  718. make_msgid(str(email_log.id), get_email_domain_part(alias.email)),
  719. )
  720. add_or_replace_header(msg, _EMAIL_LOG_ID_HEADER, str(email_log.id))
  721. add_or_replace_header(msg, _DIRECTION, "Reply")
  722. # Received-SPF is injected by postfix-policyd-spf-python can reveal user original email
  723. delete_header(msg, "Received-SPF")
  724. LOG.d(
  725. "send email from %s to %s, mail_options:%s,rcpt_options:%s",
  726. alias.email,
  727. contact.website_email,
  728. envelope.mail_options,
  729. envelope.rcpt_options,
  730. )
  731. # replace "ra+string@simplelogin.co" by the contact email in the email body
  732. # as this is usually included when replying
  733. if user.replace_reverse_alias:
  734. if msg.is_multipart():
  735. for part in msg.walk():
  736. if part.get_content_maintype() != "text":
  737. continue
  738. part = replace_str_in_msg(part, reply_email, contact.website_email)
  739. else:
  740. msg = replace_str_in_msg(msg, reply_email, contact.website_email)
  741. if should_add_dkim_signature(alias_domain):
  742. add_dkim_signature(msg, alias_domain)
  743. # create PGP email if needed
  744. if contact.pgp_finger_print and user.is_premium():
  745. LOG.d("Encrypt message for contact %s", contact)
  746. try:
  747. msg = prepare_pgp_message(msg, contact.pgp_finger_print)
  748. except PGPException:
  749. LOG.exception(
  750. "Cannot encrypt message %s -> %s. %s %s", alias, contact, mailbox, user
  751. )
  752. # to not save the email_log
  753. db.session.rollback()
  754. # return 421 so the client can retry later
  755. return False, "421 SL E13 Retry later"
  756. try:
  757. sl_sendmail(
  758. alias.email,
  759. contact.website_email,
  760. msg,
  761. envelope.mail_options,
  762. envelope.rcpt_options,
  763. )
  764. except Exception:
  765. # to not save the email_log
  766. db.session.rollback()
  767. LOG.warning("Cannot send email from %s to %s", alias, contact)
  768. send_email(
  769. mailbox.email,
  770. f"Email cannot be sent to {contact.email} from {alias.email}",
  771. render(
  772. "transactional/reply-error.txt",
  773. user=user,
  774. alias=alias,
  775. contact=contact,
  776. contact_domain=get_email_domain_part(contact.email),
  777. ),
  778. render(
  779. "transactional/reply-error.html",
  780. user=user,
  781. alias=alias,
  782. contact=contact,
  783. contact_domain=get_email_domain_part(contact.email),
  784. ),
  785. )
  786. # return 250 even if error as user is already informed of the incident and can retry sending the email
  787. db.session.commit()
  788. return True, "250 Message accepted for delivery"
  789. def get_mailbox_from_mail_from(mail_from: str, alias) -> Optional[Mailbox]:
  790. """return the corresponding mailbox given the mail_from and alias
  791. Usually the mail_from=mailbox.email but it can also be one of the authorized address
  792. """
  793. for mailbox in alias.mailboxes:
  794. if mailbox.email == mail_from:
  795. return mailbox
  796. for address in mailbox.authorized_addresses:
  797. if address.email == mail_from:
  798. LOG.debug(
  799. "Found an authorized address for %s %s %s", alias, mailbox, address
  800. )
  801. return mailbox
  802. return None
  803. def spf_pass(
  804. ip: str,
  805. envelope,
  806. mailbox: Mailbox,
  807. user: User,
  808. alias: Alias,
  809. contact_email: str,
  810. msg: Message,
  811. ) -> bool:
  812. if ip:
  813. LOG.d("Enforce SPF")
  814. try:
  815. r = spf.check2(i=ip, s=envelope.mail_from, h=None)
  816. except Exception:
  817. LOG.exception("SPF error, mailbox %s, ip %s", mailbox.email, ip)
  818. else:
  819. # TODO: Handle temperr case (e.g. dns timeout)
  820. # only an absolute pass, or no SPF policy at all is 'valid'
  821. if r[0] not in ["pass", "none"]:
  822. LOG.warning(
  823. "SPF fail for mailbox %s, reason %s, failed IP %s",
  824. mailbox.email,
  825. r[0],
  826. ip,
  827. )
  828. send_email_with_rate_control(
  829. user,
  830. ALERT_SPF,
  831. mailbox.email,
  832. f"SimpleLogin Alert: attempt to send emails from your alias {alias.email} from unknown IP Address",
  833. render(
  834. "transactional/spf-fail.txt",
  835. name=user.name,
  836. alias=alias.email,
  837. ip=ip,
  838. mailbox_url=URL + f"/dashboard/mailbox/{mailbox.id}#spf",
  839. to_email=contact_email,
  840. subject=msg["Subject"],
  841. time=arrow.now(),
  842. ),
  843. render(
  844. "transactional/spf-fail.html",
  845. name=user.name,
  846. alias=alias.email,
  847. ip=ip,
  848. mailbox_url=URL + f"/dashboard/mailbox/{mailbox.id}#spf",
  849. to_email=contact_email,
  850. subject=msg["Subject"],
  851. time=arrow.now(),
  852. ),
  853. )
  854. return False
  855. else:
  856. LOG.warning(
  857. "Could not find %s header %s -> %s",
  858. _IP_HEADER,
  859. mailbox.email,
  860. contact_email,
  861. )
  862. return True
  863. def handle_unknown_mailbox(
  864. envelope, msg, reply_email: str, user: User, alias: Alias, contact: Contact
  865. ):
  866. LOG.warning(
  867. f"Reply email can only be used by mailbox. "
  868. f"Actual mail_from: %s. msg from header: %s, reverse-alias %s, %s %s %s",
  869. envelope.mail_from,
  870. msg["From"],
  871. reply_email,
  872. alias,
  873. user,
  874. contact,
  875. )
  876. authorize_address_link = (
  877. f"{URL}/dashboard/mailbox/{alias.mailbox_id}/#authorized-address"
  878. )
  879. mailbox_emails = [mailbox.email for mailbox in alias.mailboxes]
  880. send_email_with_rate_control(
  881. user,
  882. ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
  883. user.email,
  884. f"Attempt to use your alias {alias.email} from {envelope.mail_from}",
  885. render(
  886. "transactional/reply-must-use-personal-email.txt",
  887. name=user.name,
  888. alias=alias,
  889. sender=envelope.mail_from,
  890. authorize_address_link=authorize_address_link,
  891. mailbox_emails=mailbox_emails,
  892. ),
  893. render(
  894. "transactional/reply-must-use-personal-email.html",
  895. name=user.name,
  896. alias=alias,
  897. sender=envelope.mail_from,
  898. authorize_address_link=authorize_address_link,
  899. mailbox_emails=mailbox_emails,
  900. ),
  901. )
  902. # Notify sender that they cannot send emails to this address
  903. send_email_with_rate_control(
  904. user,
  905. ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
  906. envelope.mail_from,
  907. f"Your email ({envelope.mail_from}) is not allowed to send emails to {reply_email}",
  908. render(
  909. "transactional/send-from-alias-from-unknown-sender.txt",
  910. sender=envelope.mail_from,
  911. reply_email=reply_email,
  912. ),
  913. render(
  914. "transactional/send-from-alias-from-unknown-sender.html",
  915. sender=envelope.mail_from,
  916. reply_email=reply_email,
  917. ),
  918. )
  919. def handle_bounce(contact: Contact, alias: Alias, msg: Message, user: User):
  920. disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
  921. # Store the bounced email
  922. # generate a name for the email
  923. random_name = str(uuid.uuid4())
  924. full_report_path = f"refused-emails/full-{random_name}.eml"
  925. s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name)
  926. file_path = None
  927. mailbox = None
  928. email_log: EmailLog = None
  929. orig_msg = get_orig_message_from_bounce(msg)
  930. if not orig_msg:
  931. # Some MTA does not return the original message in bounce message
  932. # nothing we can do here
  933. LOG.warning(
  934. "Cannot parse original message from bounce message %s %s %s %s",
  935. alias,
  936. user,
  937. contact,
  938. full_report_path,
  939. )
  940. else:
  941. file_path = f"refused-emails/{random_name}.eml"
  942. s3.upload_email_from_bytesio(
  943. file_path, BytesIO(orig_msg.as_bytes()), random_name
  944. )
  945. try:
  946. mailbox_id = int(orig_msg[_MAILBOX_ID_HEADER])
  947. except TypeError:
  948. LOG.warning(
  949. "cannot parse mailbox from original message header %s",
  950. orig_msg[_MAILBOX_ID_HEADER],
  951. )
  952. else:
  953. mailbox = Mailbox.get(mailbox_id)
  954. if not mailbox or mailbox.user_id != user.id:
  955. LOG.exception(
  956. "Tampered message mailbox_id %s, %s, %s, %s %s",
  957. mailbox_id,
  958. user,
  959. alias,
  960. contact,
  961. full_report_path,
  962. )
  963. # cannot use this tampered mailbox, reset it
  964. mailbox = None
  965. # try to get the original email_log
  966. try:
  967. email_log_id = int(orig_msg[_EMAIL_LOG_ID_HEADER])
  968. except TypeError:
  969. LOG.warning(
  970. "cannot parse email log from original message header %s",
  971. orig_msg[_EMAIL_LOG_ID_HEADER],
  972. )
  973. else:
  974. email_log = EmailLog.get(email_log_id)
  975. refused_email = RefusedEmail.create(
  976. path=file_path, full_report_path=full_report_path, user_id=user.id
  977. )
  978. db.session.flush()
  979. LOG.d("Create refused email %s", refused_email)
  980. if not mailbox:
  981. LOG.debug("Try to get mailbox from bounce report")
  982. try:
  983. mailbox_id = int(get_header_from_bounce(msg, _MAILBOX_ID_HEADER))
  984. except Exception:
  985. LOG.warning("cannot get mailbox-id from bounce report, %s", refused_email)
  986. else:
  987. mailbox = Mailbox.get(mailbox_id)
  988. if not mailbox or mailbox.user_id != user.id:
  989. LOG.exception(
  990. "Tampered message mailbox_id %s, %s, %s, %s %s",
  991. mailbox_id,
  992. user,
  993. alias,
  994. contact,
  995. full_report_path,
  996. )
  997. mailbox = None
  998. if not email_log:
  999. LOG.d("Try to get email log from bounce report")
  1000. try:
  1001. email_log_id = int(get_header_from_bounce(msg, _EMAIL_LOG_ID_HEADER))
  1002. except Exception:
  1003. LOG.warning("cannot get email log id from bounce report, %s", refused_email)
  1004. else:
  1005. email_log = EmailLog.get(email_log_id)
  1006. # use the default mailbox as the last option
  1007. if not mailbox:
  1008. LOG.warning("Use %s default mailbox %s", alias, refused_email)
  1009. mailbox = alias.mailbox
  1010. # create a new email log as the last option
  1011. if not email_log:
  1012. LOG.warning("cannot get the original email_log, create a new one")
  1013. email_log: EmailLog = EmailLog.create(
  1014. contact_id=contact.id, user_id=contact.user_id
  1015. )
  1016. email_log.bounced = True
  1017. email_log.refused_email_id = refused_email.id
  1018. email_log.bounced_mailbox_id = mailbox.id
  1019. db.session.commit()
  1020. refused_email_url = (
  1021. URL + f"/dashboard/refused_email?highlight_id=" + str(email_log.id)
  1022. )
  1023. nb_bounced = EmailLog.filter_by(contact_id=contact.id, bounced=True).count()
  1024. if nb_bounced >= 2 and alias.cannot_be_disabled:
  1025. LOG.warning("%s cannot be disabled", alias)
  1026. # inform user if this is the first bounced email
  1027. if nb_bounced == 1 or (nb_bounced >= 2 and alias.cannot_be_disabled):
  1028. LOG.d(
  1029. "Inform user %s about bounced email sent by %s to alias %s",
  1030. user,
  1031. contact.website_email,
  1032. alias,
  1033. )
  1034. send_email_with_rate_control(
  1035. user,
  1036. ALERT_BOUNCE_EMAIL,
  1037. user.email,
  1038. f"Email from {contact.website_email} to {alias.email} cannot be delivered to your inbox",
  1039. render(
  1040. "transactional/bounced-email.txt",
  1041. name=user.name,
  1042. alias=alias,
  1043. website_email=contact.website_email,
  1044. disable_alias_link=disable_alias_link,
  1045. refused_email_url=refused_email_url,
  1046. mailbox_email=mailbox.email,
  1047. ),
  1048. render(
  1049. "transactional/bounced-email.html",
  1050. name=user.name,
  1051. alias=alias,
  1052. website_email=contact.website_email,
  1053. disable_alias_link=disable_alias_link,
  1054. refused_email_url=refused_email_url,
  1055. mailbox_email=mailbox.email,
  1056. ),
  1057. )
  1058. # disable the alias the second time email is bounced
  1059. elif nb_bounced >= 2:
  1060. LOG.d(
  1061. "Bounce happens again with alias %s from %s. Disable alias now ",
  1062. alias,
  1063. contact.website_email,
  1064. )
  1065. alias.enabled = False
  1066. db.session.commit()
  1067. send_email_with_rate_control(
  1068. user,
  1069. ALERT_BOUNCE_EMAIL,
  1070. user.email,
  1071. f"Alias {alias.email} has been disabled due to second undelivered email from {contact.website_email}",
  1072. render(
  1073. "transactional/automatic-disable-alias.txt",
  1074. name=user.name,
  1075. alias=alias,
  1076. website_email=contact.website_email,
  1077. refused_email_url=refused_email_url,
  1078. mailbox_email=mailbox.email,
  1079. ),
  1080. render(
  1081. "transactional/automatic-disable-alias.html",
  1082. name=user.name,
  1083. alias=alias,
  1084. website_email=contact.website_email,
  1085. refused_email_url=refused_email_url,
  1086. mailbox_email=mailbox.email,
  1087. ),
  1088. )
  1089. def handle_spam(
  1090. contact: Contact,
  1091. alias: Alias,
  1092. msg: Message,
  1093. user: User,
  1094. mailbox: Mailbox,
  1095. email_log: EmailLog,
  1096. is_reply=False, # whether the email is in forward or reply phase
  1097. ):
  1098. # Store the report & original email
  1099. orig_msg = get_orig_message_from_spamassassin_report(msg)
  1100. # generate a name for the email
  1101. random_name = str(uuid.uuid4())
  1102. full_report_path = f"spams/full-{random_name}.eml"
  1103. s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name)
  1104. file_path = None
  1105. if orig_msg:
  1106. file_path = f"spams/{random_name}.eml"
  1107. s3.upload_email_from_bytesio(
  1108. file_path, BytesIO(orig_msg.as_bytes()), random_name
  1109. )
  1110. refused_email = RefusedEmail.create(
  1111. path=file_path, full_report_path=full_report_path, user_id=user.id
  1112. )
  1113. db.session.flush()
  1114. email_log.refused_email_id = refused_email.id
  1115. db.session.commit()
  1116. LOG.d("Create spam email %s", refused_email)
  1117. refused_email_url = (
  1118. URL + f"/dashboard/refused_email?highlight_id=" + str(email_log.id)
  1119. )
  1120. disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
  1121. if is_reply:
  1122. LOG.d(
  1123. "Inform %s (%s) about spam email sent from alias %s to %s",
  1124. mailbox,
  1125. user,
  1126. alias,
  1127. contact,
  1128. )
  1129. send_email_with_rate_control(
  1130. user,
  1131. ALERT_SPAM_EMAIL,
  1132. mailbox.email,
  1133. f"Email from {contact.website_email} to {alias.email} is detected as spam",
  1134. render(
  1135. "transactional/spam-email-reply-phase.txt",
  1136. name=user.name,
  1137. alias=alias,
  1138. website_email=contact.website_email,
  1139. disable_alias_link=disable_alias_link,
  1140. refused_email_url=refused_email_url,
  1141. ),
  1142. render(
  1143. "transactional/spam-email-reply-phase.html",
  1144. name=user.name,
  1145. alias=alias,
  1146. website_email=contact.website_email,
  1147. disable_alias_link=disable_alias_link,
  1148. refused_email_url=refused_email_url,
  1149. ),
  1150. )
  1151. else:
  1152. # inform user
  1153. LOG.d(
  1154. "Inform %s (%s) about spam email sent by %s to alias %s",
  1155. mailbox,
  1156. user,
  1157. contact,
  1158. alias,
  1159. )
  1160. send_email_with_rate_control(
  1161. user,
  1162. ALERT_SPAM_EMAIL,
  1163. mailbox.email,
  1164. f"Email from {contact.website_email} to {alias.email} is detected as spam",
  1165. render(
  1166. "transactional/spam-email.txt",
  1167. name=user.name,
  1168. alias=alias,
  1169. website_email=contact.website_email,
  1170. disable_alias_link=disable_alias_link,
  1171. refused_email_url=refused_email_url,
  1172. ),
  1173. render(
  1174. "transactional/spam-email.html",
  1175. name=user.name,
  1176. alias=alias,
  1177. website_email=contact.website_email,
  1178. disable_alias_link=disable_alias_link,
  1179. refused_email_url=refused_email_url,
  1180. ),
  1181. )
  1182. def handle_unsubscribe(envelope: Envelope) -> str:
  1183. """return the SMTP status"""
  1184. msg = email.message_from_bytes(envelope.original_content)
  1185. # format: alias_id:
  1186. subject = msg["Subject"]
  1187. try:
  1188. # subject has the format {alias.id}=
  1189. if subject.endswith("="):
  1190. alias_id = int(subject[:-1])
  1191. # {user.id}*
  1192. elif subject.endswith("*"):
  1193. user_id = int(subject[:-1])
  1194. return handle_unsubscribe_user(user_id, envelope.mail_from)
  1195. # some email providers might strip off the = suffix
  1196. else:
  1197. alias_id = int(subject)
  1198. alias = Alias.get(alias_id)
  1199. except Exception:
  1200. LOG.warning("Cannot parse alias from subject %s", msg["Subject"])
  1201. return "550 SL E8 Wrongly formatted subject"
  1202. if not alias:
  1203. LOG.warning("No such alias %s", alias_id)
  1204. return "550 SL E9 Email not exist"
  1205. # This sender cannot unsubscribe
  1206. mail_from = envelope.mail_from
  1207. # Only alias's owning mailbox can send the unsubscribe request
  1208. mailbox = get_mailbox_from_mail_from(mail_from, alias)
  1209. if not mailbox:
  1210. LOG.d("%s cannot disable alias %s", envelope.mail_from, alias)
  1211. return "550 SL E10 unauthorized"
  1212. # Sender is owner of this alias
  1213. alias.enabled = False
  1214. db.session.commit()
  1215. user = alias.user
  1216. enable_alias_url = URL + f"/dashboard/?highlight_alias_id={alias.id}"
  1217. for mailbox in alias.mailboxes:
  1218. send_email(
  1219. mailbox.email,
  1220. f"Alias {alias.email} has been disabled successfully",
  1221. render(
  1222. "transactional/unsubscribe-disable-alias.txt",
  1223. user=user,
  1224. alias=alias.email,
  1225. enable_alias_url=enable_alias_url,
  1226. ),
  1227. render(
  1228. "transactional/unsubscribe-disable-alias.html",
  1229. user=user,
  1230. alias=alias.email,
  1231. enable_alias_url=enable_alias_url,
  1232. ),
  1233. )
  1234. return "250 Unsubscribe request accepted"
  1235. def handle_unsubscribe_user(user_id: int, mail_from: str) -> str:
  1236. """return the SMTP status"""
  1237. user = User.get(user_id)
  1238. if not user:
  1239. LOG.exception("No such user %s %s", user_id, mail_from)
  1240. return "550 SL E22 so such user"
  1241. if mail_from != user.email:
  1242. LOG.exception("Unauthorized mail_from %s %s", user, mail_from)
  1243. return "550 SL E23 unsubscribe error"
  1244. user.notification = False
  1245. db.session.commit()
  1246. send_email(
  1247. user.email,
  1248. f"You have been unsubscribed from SimpleLogin newsletter",
  1249. render(
  1250. "transactional/unsubscribe-newsletter.txt",
  1251. user=user,
  1252. ),
  1253. render(
  1254. "transactional/unsubscribe-newsletter.html",
  1255. user=user,
  1256. ),
  1257. )
  1258. return "250 Unsubscribe request accepted"
  1259. def handle_sender_email(envelope: Envelope):
  1260. filename = (
  1261. arrow.now().format("YYYY-MM-DD_HH-mm-ss") + "_" + random_string(10) + ".eml"
  1262. )
  1263. filepath = os.path.join(SENDER_DIR, filename)
  1264. with open(filepath, "wb") as f:
  1265. f.write(envelope.original_content)
  1266. LOG.d("Write email to sender at %s", filepath)
  1267. msg = email.message_from_bytes(envelope.original_content)
  1268. orig = get_orig_message_from_bounce(msg)
  1269. if orig:
  1270. LOG.warning(
  1271. "Original message %s -> %s saved at %s", orig["From"], orig["To"], filepath
  1272. )
  1273. return "250 email to sender accepted"
  1274. def handle(envelope: Envelope) -> str:
  1275. """Return SMTP status"""
  1276. # sanitize mail_from, rcpt_tos
  1277. mail_from = envelope.mail_from.lower().strip().replace(" ", "")
  1278. rcpt_tos = [
  1279. rcpt_to.lower().strip().replace(" ", "") for rcpt_to in envelope.rcpt_tos
  1280. ]
  1281. envelope.mail_from = mail_from
  1282. envelope.rcpt_tos = rcpt_tos
  1283. # unsubscribe request
  1284. if UNSUBSCRIBER and rcpt_tos == [UNSUBSCRIBER]:
  1285. LOG.d("Handle unsubscribe request from %s", mail_from)
  1286. return handle_unsubscribe(envelope)
  1287. # emails sent to sender. Probably bounce emails
  1288. if SENDER and rcpt_tos == [SENDER]:
  1289. LOG.d("Handle email sent to sender from %s", mail_from)
  1290. return handle_sender_email(envelope)
  1291. # Whether it's necessary to apply greylisting
  1292. if greylisting_needed(mail_from, rcpt_tos):
  1293. LOG.warning("Grey listing applied for %s %s", mail_from, rcpt_tos)
  1294. return "421 SL Retry later"
  1295. # result of all deliveries
  1296. # each element is a couple of whether the delivery is successful and the smtp status
  1297. res: [(bool, str)] = []
  1298. for rcpt_to in rcpt_tos:
  1299. msg = email.message_from_bytes(envelope.original_content)
  1300. # Reply case
  1301. # recipient starts with "reply+" or "ra+" (ra=reverse-alias) prefix
  1302. if rcpt_to.startswith("reply+") or rcpt_to.startswith("ra+"):
  1303. LOG.debug(">>> Reply phase %s(%s) -> %s", mail_from, msg["From"], rcpt_to)
  1304. is_delivered, smtp_status = handle_reply(envelope, msg, rcpt_to)
  1305. res.append((is_delivered, smtp_status))
  1306. else: # Forward case
  1307. LOG.debug(
  1308. ">>> Forward phase %s(%s) -> %s",
  1309. mail_from,
  1310. msg["From"],
  1311. rcpt_to,
  1312. )
  1313. for is_delivered, smtp_status in handle_forward(envelope, msg, rcpt_to):
  1314. res.append((is_delivered, smtp_status))
  1315. for (is_success, smtp_status) in res:
  1316. # Consider all deliveries successful if 1 delivery is successful
  1317. if is_success:
  1318. return smtp_status
  1319. # Failed delivery for all, return the first failure
  1320. return res[0][1]
  1321. async def get_spam_score_async(message: Message) -> float:
  1322. LOG.debug("get spam score for %s", message[_MESSAGE_ID])
  1323. sa_input = to_bytes(message)
  1324. # Spamassassin requires to have an ending linebreak
  1325. if not sa_input.endswith(b"\n"):
  1326. LOG.d("add linebreak to spamassassin input")
  1327. sa_input += b"\n"
  1328. try:
  1329. # wait for at max 300s which is the default spamd timeout-child
  1330. response = await asyncio.wait_for(
  1331. aiospamc.check(sa_input, host=SPAMASSASSIN_HOST), timeout=300
  1332. )
  1333. return response.headers["Spam"].score
  1334. except asyncio.TimeoutError:
  1335. LOG.exception("SpamAssassin timeout")
  1336. # return a negative score so the message is always considered as ham
  1337. return -999
  1338. except Exception:
  1339. LOG.exception("SpamAssassin exception")
  1340. return -999
  1341. def get_spam_score(message: Message) -> float:
  1342. LOG.debug("get spam score for %s", message[_MESSAGE_ID])
  1343. sa_input = to_bytes(message)
  1344. # Spamassassin requires to have an ending linebreak
  1345. if not sa_input.endswith(b"\n"):
  1346. LOG.d("add linebreak to spamassassin input")
  1347. sa_input += b"\n"
  1348. try:
  1349. # wait for at max 300s which is the default spamd timeout-child
  1350. sa = SpamAssassin(sa_input, host=SPAMASSASSIN_HOST, timeout=300)
  1351. return sa.get_score()
  1352. except Exception:
  1353. LOG.exception("SpamAssassin exception")
  1354. # return a negative score so the message is always considered as ham
  1355. return -999
  1356. def sl_sendmail(from_addr, to_addr, msg: Message, mail_options, rcpt_options):
  1357. """replace smtp.sendmail"""
  1358. if POSTFIX_SUBMISSION_TLS:
  1359. smtp = SMTP(POSTFIX_SERVER, 587)
  1360. smtp.starttls()
  1361. else:
  1362. smtp = SMTP(POSTFIX_SERVER, POSTFIX_PORT or 25)
  1363. # smtp.send_message has UnicodeEncodeErroremail issue
  1364. # encode message raw directly instead
  1365. smtp.sendmail(
  1366. from_addr,
  1367. to_addr,
  1368. msg.as_bytes(),
  1369. mail_options,
  1370. rcpt_options,
  1371. )
  1372. class MailHandler:
  1373. async def handle_DATA(self, server, session, envelope: Envelope):
  1374. try:
  1375. ret = self._handle(envelope)
  1376. return ret
  1377. except Exception:
  1378. LOG.exception(
  1379. "email handling fail %s -> %s",
  1380. envelope.mail_from,
  1381. envelope.rcpt_tos,
  1382. )
  1383. return "421 SL Retry later"
  1384. def _handle(self, envelope: Envelope):
  1385. start = time.time()
  1386. LOG.info(
  1387. "===>> New message, mail from %s, rctp tos %s ",
  1388. envelope.mail_from,
  1389. envelope.rcpt_tos,
  1390. )
  1391. app = new_app()
  1392. with app.app_context():
  1393. ret = handle(envelope)
  1394. LOG.info("takes %s seconds <<===", time.time() - start)
  1395. return ret
  1396. def main(port: int):
  1397. """Use aiosmtpd Controller"""
  1398. controller = Controller(MailHandler(), hostname="0.0.0.0", port=port)
  1399. controller.start()
  1400. LOG.d("Start mail controller %s %s", controller.hostname, controller.port)
  1401. if LOAD_PGP_EMAIL_HANDLER:
  1402. LOG.warning("LOAD PGP keys")
  1403. app = create_app()
  1404. with app.app_context():
  1405. load_pgp_public_keys()
  1406. while True:
  1407. time.sleep(2)
  1408. def asyncio_main(port: int):
  1409. """
  1410. Main entrypoint using asyncio directly without passing by aiosmtpd Controller
  1411. """
  1412. if LOAD_PGP_EMAIL_HANDLER:
  1413. LOG.warning("LOAD PGP keys")
  1414. app = create_app()
  1415. with app.app_context():
  1416. load_pgp_public_keys()
  1417. loop = asyncio.new_event_loop()
  1418. asyncio.set_event_loop(loop)
  1419. lock = asyncio.Lock()
  1420. handler = MailHandler(lock)
  1421. def factory():
  1422. return aiosmtpd.smtp.SMTP(handler, enable_SMTPUTF8=True)
  1423. server = loop.run_until_complete(
  1424. loop.create_server(factory, host="0.0.0.0", port=port)
  1425. )
  1426. try:
  1427. loop.run_forever()
  1428. except KeyboardInterrupt:
  1429. pass
  1430. # Close the server
  1431. LOG.info("Close SMTP server")
  1432. server.close()
  1433. loop.run_until_complete(server.wait_closed())
  1434. loop.close()
  1435. if __name__ == "__main__":
  1436. parser = argparse.ArgumentParser()
  1437. parser.add_argument(
  1438. "-p", "--port", help="SMTP port to listen for", type=int, default=20381
  1439. )
  1440. args = parser.parse_args()
  1441. LOG.info("Listen for port %s", args.port)
  1442. main(port=args.port)