From d557885aab8a9fcd7e6fceb06e4ad80085aa4415 Mon Sep 17 00:00:00 2001 From: David Duque Date: Mon, 23 Aug 2021 02:06:10 +0100 Subject: [PATCH] SMTP Relay feature rework (#23) --- management/daemon.py | 129 ++++++++++-- management/dns_update.py | 69 ++++++- management/status_checks.py | 85 +++++++- management/templates/smtp-relays.html | 280 ++++++++++++++++++++++---- setup/mail-postfix.sh | 6 +- 5 files changed, 495 insertions(+), 74 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 18d8a2b..636b815 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -681,43 +681,138 @@ def privacy_status_set(): @authorized_personnel_only def smtp_relay_get(): config = utils.load_settings(env) + + dkim_rrtxt = "" + rr = config.get("SMTP_RELAY_DKIM_RR", None) + if rr is not None: + if rr.get("p") is None: + raise ValueError("Key doesn't exist!") + for c, d in (("v", "DKIM1"), ("h", None), ("k", "rsa"), ("n", None), ("s", None), ("t", None)): + txt = rr.get(c, d) + if txt is None: + continue + else: + dkim_rrtxt += f"{c}={txt}; " + dkim_rrtxt += f"p={rr.get('p')}" + return { - "enabled": config.get("SMTP_RELAY_ENABLED", True), + "enabled": config.get("SMTP_RELAY_ENABLED", False), "host": config.get("SMTP_RELAY_HOST", ""), - "auth_enabled": config.get("SMTP_RELAY_AUTH", False), - "user": config.get("SMTP_RELAY_USER", "") + "port": config.get("SMTP_RELAY_PORT", None), + "user": config.get("SMTP_RELAY_USER", ""), + "authorized_servers": config.get("SMTP_RELAY_AUTHORIZED_SERVERS", []), + "dkim_selector": config.get("SMTP_RELAY_DKIM_SELECTOR", None), + "dkim_rr": dkim_rrtxt } @app.route('/system/smtp/relay', methods=["POST"]) @authorized_personnel_only def smtp_relay_set(): from editconf import edit_conf + from os import chmod + import re, socket, ssl + config = utils.load_settings(env) newconf = request.form + + # Is DKIM configured? + sel = newconf.get("dkim_selector") + if sel is None or sel.strip() == "": + config["SMTP_RELAY_DKIM_SELECTOR"] = None + config["SMTP_RELAY_DKIM_RR"] = None + elif re.fullmatch(r"[a-z\d\._]+", sel.strip()) is None: + return ("The DKIM selector is invalid!", 400) + elif sel.strip() == config.get("local_dkim_selector", "mail"): + return (f"The DKIM selector {sel.strip()} is already in use by the box!", 400) + else: + # DKIM selector looks good, try processing the RR + rr = newconf.get("dkim_rr", "") + if rr.strip() == "": + return ("Cannot publish a selector with an empty key!", 400) + + components = {} + for r in re.split(r"[;\s]+", rr): + sp = re.split(r"\=", r) + if len(sp) != 2: + return ("DKIM public key RR is malformed!", 400) + components[sp[0]] = sp[1] + + if not components.get("p"): + return ("The DKIM public key doesn't exist!", 400) + + config["SMTP_RELAY_DKIM_SELECTOR"] = sel + config["SMTP_RELAY_DKIM_RR"] = components + + relay_on = False + implicit_tls = False + + if newconf.get("enabled") == "true": + relay_on = True + + # Try negotiating TLS directly. We need to know this because we need to configure Postfix + # to be aware of this detail. + try: + ctx = ssl.create_default_context() + with socket.create_connection((newconf.get("host"), int(newconf.get("port"))), 5) as sock: + with ctx.wrap_socket(sock, server_hostname=newconf.get("host")): + implicit_tls = True + except ssl.SSLError as sle: + # Couldn't connect via TLS, configure Postfix to send via STARTTLS + print(sle.reason) + except (socket.herror, socket.gaierror) as he: + return (f"Unable to resolve hostname (it probably is incorrect): {he.strerror}", 400) + except socket.timeout: + return ("We couldn't connect to the server. Is it down or did you write the wrong port number?", 400) + + pw_file = "/etc/postfix/sasl_passwd" + modify_password = True + # Check that if the provided password is empty, that there was a password saved before + if (newconf.get("key", "") == ""): + if os.path.isfile(pw_file): + modify_password = False + else: + return ("Please provide a password/key (there is no existing password to retain).", 400) + try: # Write on daemon settings - config["SMTP_RELAY_ENABLED"] = (newconf.get("enabled") == "true") + config["SMTP_RELAY_ENABLED"] = relay_on config["SMTP_RELAY_HOST"] = newconf.get("host") - config["SMTP_RELAY_AUTH"] = (newconf.get("auth_enabled") == "true") + config["SMTP_RELAY_PORT"] = int(newconf.get("port")) config["SMTP_RELAY_USER"] = newconf.get("user") + config["SMTP_RELAY_AUTHORIZED_SERVERS"] = [s.strip() for s in re.split(r"[, ]+", newconf.get("authorized_servers", []) or "") if s.strip() != ""] utils.write_settings(config, env) + # Write on Postfix configs edit_conf("/etc/postfix/main.cf", [ - "relayhost=" + (f"[{config['SMTP_RELAY_HOST']}]:587" if config["SMTP_RELAY_ENABLED"] else ""), - "smtp_sasl_auth_enable=" + ("yes" if config["SMTP_RELAY_AUTH"] else "no"), - "smtp_sasl_security_options=" + ("noanonymous" if config["SMTP_RELAY_AUTH"] else "anonymous"), - "smtp_sasl_tls_security_options=" + ("noanonymous" if config["SMTP_RELAY_AUTH"] else "anonymous") + "relayhost=" + (f"[{config['SMTP_RELAY_HOST']}]:{config['SMTP_RELAY_PORT']}" if config["SMTP_RELAY_ENABLED"] else ""), + f"smtp_tls_wrappermode={'yes' if implicit_tls else 'no'}" ], delimiter_re=r"\s*=\s*", delimiter="=", comment_char="#") - if config["SMTP_RELAY_AUTH"]: - # Edit the sasl password - with open("/etc/postfix/sasl_passwd", "w") as f: - f.write(f"[{config['SMTP_RELAY_HOST']}]:587 {config['SMTP_RELAY_USER']}:{newconf.get('key')}\n") - utils.shell("check_output", ["/usr/bin/chmod", "600", "/etc/postfix/sasl_passwd"], capture_stderr=True) - utils.shell("check_output", ["/usr/sbin/postmap", "/etc/postfix/sasl_passwd"], capture_stderr=True) + + # Edit the sasl password (still will edit the file, but keep the pw) + + with open(pw_file, "a+") as f: + f.seek(0) + pwm = re.match(r"\[.+\]\:[0-9]+\s.+\:(.*)", f.readline()) + if (pwm is None or len(pwm.groups()) != 1) and not modify_password: + # Well if this isn't a bruh moment + return ("Please provide a password/key (there is no existing password to retain).", 400) + + f.truncate(0) + f.write( + f"[{config['SMTP_RELAY_HOST']}]:{config['SMTP_RELAY_PORT']} {config['SMTP_RELAY_USER']}:{newconf.get('key') if modify_password else pwm[1]}\n" + ) + chmod(pw_file, 0o600) + utils.shell("check_output", ["/usr/sbin/postmap", pw_file], capture_stderr=True) + + # Regenerate DNS (to apply whatever changes need to be made) + from dns_update import do_dns_update + do_dns_update(env) + # Restart Postfix - return utils.shell("check_output", ["/usr/bin/systemctl", "restart", "postfix"], capture_stderr=True) + return utils.shell("check_output", ["/usr/sbin/postfix", "reload"], capture_stderr=True) except Exception as e: - return (str(e), 500) + return (str(e), 400) + # PGP diff --git a/management/dns_update.py b/management/dns_update.py index ebbc6bd..20c9aad 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -7,9 +7,10 @@ import sys, os, os.path, urllib.parse, datetime, re, hashlib, base64 import ipaddress import rtyaml +import idna import dns.resolver -from utils import shell, load_env_vars_from_file, safe_domain_name, sort_domains +from utils import shell, load_env_vars_from_file, safe_domain_name, sort_domains, load_settings from ssl_certificates import get_ssl_certificates, check_certificate # From https://stackoverflow.com/questions/3026957/how-to-validate-a-domain-name-using-regex-php/16491074#16491074 @@ -168,6 +169,36 @@ def build_zones(env): def build_zone(domain, domain_properties, additional_records, env, is_zone=True): records = [] + + # Are there any other authorized servers for this domain? + settings = load_settings(env) + spf_extra = None + if settings.get("SMTP_RELAY_ENABLED", False): + spf_extra = "" + # Convert settings to spf elements + for r in settings.get("SMTP_RELAY_AUTHORIZED_SERVERS", []): + sr = "" + if r[0:4] == "spf:": + sr = f"include:{r[4:]}" + elif "/" in r: + net = ipaddress.ip_network(r) + if isinstance(net, ipaddress.IPv4Network): + sr = "ip4:" + net.compressed + elif isinstance(net, ipaddress.IPv6Network): + sr = "ip6:" + net.compressed + elif not (re.fullmatch(r"[0-9\.\:]+", r) is None): + addr = ipaddress.ip_address(r) + if isinstance(addr, ipaddress.IPv4Address): + sr = "ip4:" + addr.compressed + elif isinstance(addr, ipaddress.IPv6Address): + sr = "ip6:" + addr.compressed + elif idna.encode(r): + sr = "a:" + idna.encode(r).decode() + else: + raise ValueError(f"Unexpected entry on authorized servers: {r}") + spf_extra += f"{sr} " + if spf_extra.strip() == "": + spf_extra = None # For top-level zones, define the authoritative name servers. # @@ -293,10 +324,13 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True) records.append((qname, rtype, value, explanation)) # SPF record: Permit the box ('mx', see above) to send mail on behalf of - # the domain, and no one else. + # the domain, and no one else (unless the user is using an SMTP relay and authorized other servers). # Skip if the user has set a custom SPF record. if not has_rec(None, "TXT", prefix="v=spf1 "): - records.append((None, "TXT", 'v=spf1 mx -all', "Recommended. Specifies that only the box is permitted to send @%s mail." % domain)) + if spf_extra is None: + records.append((None, "TXT", 'v=spf1 mx -all', "Recommended. Specifies that only the box is permitted to send @%s mail." % domain)) + else: + records.append((None, "TXT", f'v=spf1 mx {spf_extra}-all', "Recommended. Specifies that only the box and the server(s) you authorized are permitted to send @%s mail." % domain)) # Append the DKIM TXT record to the zone as generated by OpenDKIM. # Skip if the user has set a DKIM record already. @@ -304,8 +338,25 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True) with open(opendkim_record_file) as orf: m = re.match(r'(\S+)\s+IN\s+TXT\s+\( ((?:"[^"]+"\s+)+)\)', orf.read(), re.S) val = "".join(re.findall(r'"([^"]+)"', m.group(2))) - if not has_rec(m.group(1), "TXT", prefix="v=DKIM1; "): - records.append((m.group(1), "TXT", val, "Recommended. Provides a way for recipients to verify that this machine sent @%s mail." % domain)) + rname = f"{settings.get('local_dkim_selector', 'mail')}._domainkey" + + if not has_rec(rname, "TXT", prefix="v=DKIM1; "): + records.append((rname, "TXT", val, "Recommended. Provides a way for recipients to verify that this machine sent @%s mail." % domain)) + + # Append the DKIM TXT record relative to the SMTP relay, if applicable. + # Skip if manually set by the user. + relay_ds = settings.get("SMTP_RELAY_DKIM_SELECTOR") + rr = settings.get("SMTP_RELAY_DKIM_RR", {}) + if relay_ds is not None and not has_rec(f"{relay_ds}._domainkey", "TXT", prefix="v=DKIM1; ") and rr.get("p") is not None: + dkim_rrtxt = "" + for c, d in (("v", "DKIM1"), ("h", None), ("k", "rsa"), ("n", None), ("s", None), ("t", None)): + txt = rr.get(c, d) + if txt is None: + continue + else: + dkim_rrtxt += f"{c}={txt}; " + dkim_rrtxt += f"p={rr.get('p')}" + records.append((f"{relay_ds}._domainkey", "TXT", dkim_rrtxt, "Recommended. Provides a way for recipients to verify that the SMTP relay you set up sent @%s mail." % domain)) # Append a DMARC record. # Skip if the user has set a DMARC record already. @@ -768,6 +819,7 @@ def write_opendkim_tables(domains, env): # that we send mail from (zones and all subdomains). opendkim_key_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/mail.private') + config = load_settings(env) if not os.path.exists(opendkim_key_file): # Looks like OpenDKIM is not installed. @@ -792,8 +844,11 @@ def write_opendkim_tables(domains, env): # signing domain must match the sender's From: domain. "KeyTable": "".join( - "{domain} {domain}:mail:{key_file}\n".format(domain=domain, key_file=opendkim_key_file) - for domain in domains + "{domain} {domain}:{selector}:{key_file}\n".format( + domain=domain, + key_file=opendkim_key_file, + selector = config.get("local_dkim_selector", "mail") + ) for domain in domains ), } diff --git a/management/status_checks.py b/management/status_checks.py index 77fb6b5..7549625 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -378,14 +378,85 @@ def run_network_checks(env, output): # it might be needed. config = load_settings(env) if config.get("SMTP_RELAY_ENABLED"): - if config.get("SMTP_RELAY_AUTH"): - output.print_ok("An authenticated SMTP relay has been set up via port 587.") - else: - output.print_warning("A SMTP relay has been set up, but it is not authenticated.") - elif ret == 0: - output.print_na("No SMTP relay has been set up (but that's ok since port 25 is not blocked).") + test_smtp_relay(env, output) else: - output.print_error("No SMTP relay has been set up. Since port 25 is blocked, you will probably not be able to send any mail.") + output.print_na("No SMTP relay has been set up.") + +def test_smtp_relay(env, output): + # Test whether the relay configuration works - this is done by: + # 1. Connect to the relay endpoint + # 2. Try to log in with the credentials given + # 3. Successful? We're done. We can close. + config = load_settings(env) + import smtplib, ssl, socket + + # Grab the password + pw = "" + try: + with open("/etc/postfix/sasl_passwd", "r") as cf: + matches = re.match(r"\[.+\]\:[0-9]+\s.+\:(.*)", cf.readline()) + if matches is None or len(matches.groups()) != 1: + output.print_error("Couldn't fetch the relay password. Configuration may be broken or incomplete.") + return + else: + pw = matches[1] + except OSError: + output.print_error("Couldn't fetch the relay password. Configuration may be broken or incomplete.") + + # Try first using implicit TLS, then STARTTLS + client = {} + login_attempted = False + try: + client["tls"] = smtplib.SMTP_SSL(config.get("SMTP_RELAY_HOST"), config.get("SMTP_RELAY_PORT"), timeout=5) + client["tls"].ehlo(env["PRIMARY_HOSTNAME"]) + login_attempted = True + client["tls"].login(config.get("SMTP_RELAY_USER"), pw) + output.print_ok("The SMTP relay is configured correctly (uses implicit TLS).") + except (socket.gaierror, socket.timeout): + output.print_error("Unable to connect to SMTP Relay. The host may be down, or the hostname and/or port number are wrong.") + except (smtplib.SMTPConnectError, smtplib.SMTPServerDisconnected, ssl.SSLError): + # Some providers shut the connection down after an unsuccessful login attempt + if login_attempted: + output.print_error("Authentication on the SMTP relay failed. It's likely the configuration is incorrect, or credentials might have been revoked.") + return + + # This endpoint doesn't seem to support implicit TLS, let's try STARTTLS instead + try: + client["starttls"] = smtplib.SMTP(config.get("SMTP_RELAY_HOST"), config.get("SMTP_RELAY_PORT"), timeout=5) + client["starttls"].starttls() + client["starttls"].ehlo(env["PRIMARY_HOSTNAME"]) + login_attempted = True + client["starttls"].login(config.get("SMTP_RELAY_USER"), pw) + output.print_ok("SMTP Relay is configured correctly (uses STARTTLS).") + except (smtplib.SMTPConnectError, smtplib.SMTPServerDisconnected) as e: + if login_attempted: + output.print_error("Authentication on the SMTP relay failed. It's likely the configuration is incorrect, or credentials might have been revoked.") + return + + output.print_error("Couldn't connect to the SMTP relay or connection was suddenly closed. The host may be down or the configuration might be incorrect.") + except smtplib.SMTPNotSupportedError as err: + if str(err)[0:8] == "STARTTLS": + output.print_error("The SMTP relay doesn't support either implicit TLS or STARTTLS.") + else: + output.print_error("The SMTP relay doesn't support username/password authentication.") + except smtplib.SMTPAuthenticationError: + output.print_error("Authentication on the SMTP relay failed. It's likely the configuration is incorrect, or credentials might have been revoked.") + except Exception as e: + output.print_error("Some unrecognized error happened while testing the SMTP relay configuration.") + output.print_line(str(e)) + finally: + if not client.get("starttls") is None and client["starttls"].sock: + client["starttls"].quit() + except smtplib.SMTPNotSupportedError: + output.print_error("The SMTP relay doesn't support username/password authentication.") + except smtplib.SMTPAuthenticationError: + output.print_error("Authentication on the SMTP relay failed. It's likely the configuration is incorrect, or credentials might have been revoked.") + except Exception as e: + output.print_error("Some unrecognized error happened while testing the SMTP relay configuration.") + output.print_line(str(e)) + finally: + if not client.get("tls") is None and client["tls"].sock: + client["tls"].quit() def run_domain_checks(rounded_time, env, output, pool, domains_to_check=None): # Get the list of domains we handle mail for. diff --git a/management/templates/smtp-relays.html b/management/templates/smtp-relays.html index d87ea36..9213a17 100644 --- a/management/templates/smtp-relays.html +++ b/management/templates/smtp-relays.html @@ -3,77 +3,122 @@

SMTP Relays

-

SMTP Relays are third-party services you can hand off the responsability of getting the mail delivered. They - can be useful when, for example, port 25 is blocked.

+

SMTP Relays are third-party services that can deliver email on your behalf. They + can be useful when, for example, port 25 is blocked, the cloud provider/ISP doesn't provide Reverse DNS, or the IP + address + has a low reputation, among other situations where deliverablity isn't great.

-

Here, you can configure an authenticated SMTP relay (for example, SendGrid) over port 587.

+

These services are governed by their own terms and as such limits can be imposed in the usage of those services.

+ +

Here, you can configure an authenticated SMTP relay and authorize it's associated servers to send mail for you.

-

SMTP Relay Configuration

+

SMTP Relay Configuration

- +
- - - - - - - - + - -
- + +
- +
- + + -
- +
+
:587
- - -
- +
: +
+
- + + -
+
- + + -
+
+

If you've already set up a relay before on this box, you can leave this field blank if you don't want to change it's password.
+
+

Authorized Servers

+

The relay service should specify the servers where the email will be sent from, please add them below. These + will probably be published in the form of a SPF record. Failure to do so will potentially have your email + sent to spam or even rejected altogether by recipients. +

+ +

You can use the button below to attempt to localize the SPF record associated with the service you're using. +

+ + + +
+
+
+

Add your SPF configuration/authorized servers here

+ +

You can separate multiple servers with commas or spaces. You can also add IP addresses or + subnets using 10.20.30.40 or 10.0.0.0/8. You can "import" SPF records using + spf:example.com. +

+
+ +

DKIM Configuration

+

DKIM allows receivers to verify that the email was sent by the relay you configured (this is, somebody you trust). Not doing so will have your email sent to spam.

+ +
+ + + + + +
._domainkey.{{hostname}}
+ +

Paste the DKIM key here:

+

+
+ +

After configuration

+

By that time you should be good to go. If your relay provider provides their own custom DNS verification methods, feel free to publish them on DNS.

+
@@ -83,18 +128,31 @@ + + function add_spf(domain) { + document.getElementById(`smtp_relay_spfinclude_${domain}`).disabled = true + + let impl = false + relay_authorized_servers.value.split(/[\s,]+/).forEach(s => { + if (s === `spf:${domain}`) { + impl = true + } + }); + + if (!impl) { + relay_authorized_servers.value += ` spf:${domain}` + } + } + + async function autodetect_spf() { + let btn = $("#smtp_relay_autospf_btn") + let results = $("#smtp_relay_autospf") + + let host = relay_host.value + if (host.trim() == "") { + results.html("Error: No hostname specified.") + return + } + + let hmatches = host.match(/([^\s.\\\/]+\.)+([^\s.\\\/]+)/) + if (!hmatches || hmatches[0] != host) { + results.html(`Error: ${host} is not a valid hostname.`) + return + } + + btn.html("Working...") + btn.prop("disabled", true) + + results.html("") + + let base_host = hmatches[hmatches.length - 2] + hmatches[hmatches.length - 1] + + function record_html(name, rec) { + if (rec.error) { + return `${name} - ${rec.error} Error (${rec.msg})` + } else { + return ` ${name} ${rec.msg}` + } + } + + let records = [] + + await Promise.all([ + query_spf(base_host).then(rr => { records[0] = record_html(base_host, rr) }), + query_spf(`spf.${base_host}`).then(rr => { records[1] = record_html(`spf.${base_host}`, rr) }), + query_spf(`_spf.${base_host}`).then(rr => { records[2] = record_html(`_spf.${base_host}`, rr) }) + ]) + + let txt = "

Here's what I've found:

    " + records.forEach((r) => { + txt += `
  • ${r}
    • ` + }) + + txt += "
Only some common subdomains are tested here, so it's possible I've missed some. You SHOULD NOT include records you do not recognize, else there will be servers that can send mail as you, but that you will not use." + + results.html(txt) + + btn.html("Detect SPF Records") + btn.prop("disabled", false) + } + + function query_spf(hostname) { + // We use Cloudflare's DNS servers for this (DNS over HTTPS) + return new Promise((resolve, _) => { + $.ajax({ + url: "https://cloudflare-dns.com/dns-query", + headers: { + accept: "application/dns-json" + }, + method: "GET", + data: { + name: hostname, + type: "TXT", + do: false, + cd: false + }, + success: (data) => { + const RRTXT = 16 + const status_description = [ + // From https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-6 + // (last checked: 3rd June 2021) + "Success", + "Format Error", + "Server Failure", + "Non-Existent Domain", + "Not Implemented", + "Query Refused", + "Name exists when it should not", + "RR set exists when it should not", + "RR set that should exist does not", + "Server is not authoritative for zone", + "Not Authorized", + "Name not contained in zone", + "DSO-TYPE Not Implemented" + ] + if (data.Status != 0) { + if (data.Status > 11) { + return resolve({ error: "DNS", msg: "Unknown Error" }) + } else { + return resolve({ error: "DNS", msg: status_description[data.Status] }) + } + } + + if (data.Answer) { + data.Answer.forEach(ans => { + if (ans.type == 16 && ans.name == hostname && ans.data.substring(1, 7) == "v=spf1") { + return resolve({ msg: ans.data.substring(1, ans.data.length - 1) }) + } + }) + } + + return resolve({ error: "DNS", msg: "No SPF record found" }) + }, + error: (_, __, err) => { + resolve({ error: "HTTP", msg: err }) + } + }) + }) + } + \ No newline at end of file diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 7090835..3e3fb6e 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -270,10 +270,10 @@ management/editconf.py /etc/postfix/main.cf \ # Store default configurations for SMTP relays: management/editconf.py /etc/postfix/main.cf \ - smtp_sasl_auth_enable=no \ + smtp_sasl_auth_enable=yes \ smtp_sasl_password_maps="hash:/etc/postfix/sasl_passwd" \ - smtp_sasl_security_options=anonymous \ - smtp_sasl_tls_security_options=anonymous \ + smtp_sasl_security_options=noanonymous \ + smtp_sasl_tls_security_options=noanonymous \ smtp_tls_security_level=encrypt \ header_size_limit=4096000