From 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 17 Jan 2015 13:41:53 +0000 Subject: [PATCH] internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names) * For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing). * Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII. * When checking SSL certificates, treat the CN and SANs as IDNA. * Since Chrome has an interesting feature of converting Unicode to IDNA in form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode. * Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding). --- CHANGELOG.md | 10 +++--- management/dns_update.py | 18 ++++++++-- management/mailconfig.py | 59 ++++++++++++++++++++++++++++--- management/status_checks.py | 1 + management/templates/aliases.html | 1 + management/templates/users.html | 12 ++++--- management/web_update.py | 4 +-- 7 files changed, 87 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64fb76e..2f6a365 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,19 +4,21 @@ CHANGELOG Development ----------- -DNS: - -* If a custom CNAME record is set, don't add a default A/AAAA record, e.g. for 'www', which end up preventing the CNAME record from working. - Control panel: * Status checks now check that system services are actually running by pinging each port that should have something running on it. +* If a custom CNAME record is set on a 'www' subdomain, the default A/AAAA records were preventing the CNAME from working. Setup: * Install cron if it isn't already installed. * Fix a units problem in the minimum memory check. +Miscellaneous: + +* Internationalized domain names (IDNs) are now supported for DNS and web, but email is not yet tested. + + v0.06 (January 4, 2015) ----------------------- diff --git a/management/dns_update.py b/management/dns_update.py index cb0eb86..9925840 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -382,17 +382,26 @@ $TTL 1800 ; default time to live """ # Replace replacement strings. - zone = zone.format(domain=domain, primary_domain=env["PRIMARY_HOSTNAME"]) + zone = zone.format(domain=domain.encode("idna").decode("ascii"), primary_domain=env["PRIMARY_HOSTNAME"].encode("idna").decode("ascii")) # Add records. for subdomain, querytype, value, explanation in records: if subdomain: - zone += subdomain + zone += subdomain.encode("idna").decode("ascii") zone += "\tIN\t" + querytype + "\t" if querytype == "TXT": + # Quote and escape. value = value.replace('\\', '\\\\') # escape backslashes value = value.replace('"', '\\"') # escape quotes value = '"' + value + '"' # wrap in quotes + elif querytype in ("NS", "CNAME"): + # These records must be IDNA-encoded. + value = value.encode("idna").decode("ascii") + elif querytype == "MX": + # Also IDNA-encoded, but must parse first. + priority, host = value.split(" ", 1) + host = host.encode("idna").decode("ascii") + value = priority + " " + host zone += value + "\n" # DNSSEC requires re-signing a zone periodically. That requires @@ -486,7 +495,7 @@ server: zone: name: %s zonefile: %s -""" % (domain, zonefile) +""" % (domain.encode("idna").decode("ascii"), zonefile) # If a custom secondary nameserver has been set, allow zone transfers # and notifies to that nameserver. @@ -531,6 +540,9 @@ def sign_zone(domain, zonefile, env): algo = dnssec_choose_algo(domain, env) dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % algo)) + # From here, use the IDNA encoding of the domain name. + domain = domain.encode("idna").decode("ascii") + # In order to use the same keys for all domains, we have to generate # a new .key file with a DNSSEC record for the specific domain. We # can reuse the same key, but it won't validate without a DNSSEC diff --git a/management/mailconfig.py b/management/mailconfig.py index 1ddcd47..6fbbbde 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -14,7 +14,7 @@ def validate_email(email, mode=None): if mode == 'user': # For Dovecot's benefit, only allow basic characters. - ATEXT = r'[\w\-]' + ATEXT = r'[a-zA-Z0-9_\-]' elif mode in (None, 'alias'): # For aliases, we can allow any valid email address. # Based on RFC 2822 and https://github.com/SyrusAkbary/validate_email/blob/master/validate_email.py, @@ -36,9 +36,34 @@ def validate_email(email, mode=None): DOT_ATOM_TEXT_HOST = ATEXT + r'+(?:\.' + ATEXT + r'+)+' # per RFC 2822 3.4.1 - ADDR_SPEC = '^%s@%s$' % (DOT_ATOM_TEXT_LOCAL, DOT_ATOM_TEXT_HOST) + ADDR_SPEC = '^(%s)@(%s)$' % (DOT_ATOM_TEXT_LOCAL, DOT_ATOM_TEXT_HOST) - return re.match(ADDR_SPEC, email) + # Check the regular expression. + m = re.match(ADDR_SPEC, email) + if not m: return False + + # Check that the domain part is IDNA-encodable. + localpart, domainpart = m.groups() + try: + domainpart.encode("idna") + except: + return False + + return True + +def sanitize_idn_email_address(email): + # Convert an IDNA-encoded email address (domain part) into Unicode + # before storing in our database. Chrome may IDNA-ize + # values before POSTing, so we want to normalize before putting + # values into the database. + try: + localpart, domainpart = email.split("@") + domainpart = domainpart.encode("ascii").decode("idna") + return localpart + "@" + domainpart + except: + # Domain part is already Unicode or not IDNA-valid, so + # leave unchanged. + return email def open_database(env, with_connection=False): conn = sqlite3.connect(env["STORAGE_ROOT"] + "/mail/users.sqlite") @@ -230,6 +255,9 @@ def get_mail_domains(env, filter_aliases=lambda alias : True): ) def add_mail_user(email, pw, privs, env): + # accept IDNA domain names but normalize to Unicode before going into database + email = sanitize_idn_email_address(email) + # validate email if email.strip() == "": return ("No email address provided.", 400) @@ -284,6 +312,10 @@ def add_mail_user(email, pw, privs, env): return kick(env, "mail user added") def set_mail_password(email, pw, env): + # accept IDNA domain names but normalize to Unicode before going into database + email = sanitize_idn_email_address(email) + + # validate that password is acceptable validate_password(pw) # hash the password @@ -298,6 +330,10 @@ def set_mail_password(email, pw, env): return "OK" def remove_mail_user(email, env): + # accept IDNA domain names but normalize to Unicode before going into database + email = sanitize_idn_email_address(email) + + # remove conn, c = open_database(env, with_connection=True) c.execute("DELETE FROM users WHERE email=?", (email,)) if c.rowcount != 1: @@ -311,6 +347,10 @@ def parse_privs(value): return [p for p in value.split("\n") if p.strip() != ""] def get_mail_user_privileges(email, env): + # accept IDNA domain names but normalize to Unicode before going into database + email = sanitize_idn_email_address(email) + + # get privs c = open_database(env) c.execute('SELECT privileges FROM users WHERE email=?', (email,)) rows = c.fetchall() @@ -324,6 +364,9 @@ def validate_privilege(priv): return None def add_remove_mail_user_privilege(email, priv, action, env): + # accept IDNA domain names but normalize to Unicode before going into database + email = sanitize_idn_email_address(email) + # validate validation = validate_privilege(priv) if validation: return validation @@ -351,6 +394,9 @@ def add_remove_mail_user_privilege(email, priv, action, env): return "OK" def add_mail_alias(source, destination, env, update_if_exists=False, do_kick=True): + # accept IDNA domain names but normalize to Unicode before going into database + source = sanitize_idn_email_address(source) + # validate source if source.strip() == "": return ("No incoming email address provided.", 400) @@ -363,13 +409,14 @@ def add_mail_alias(source, destination, env, update_if_exists=False, do_kick=Tru if validate_email(destination, mode='alias'): # Oostfix allows a single @domain.tld as the destination, which means # the local part on the address is preserved in the rewrite. - dests.append(destination) + dests.append(sanitize_idn_email_address(destination)) else: # Parse comma and \n-separated destination emails & validate. In this # case, the recipients must be complete email addresses. for line in destination.split("\n"): for email in line.split(","): email = email.strip() + email = sanitize_idn_email_address(email) # Unicode => IDNA if email == "": continue if not validate_email(email): return ("Invalid destination email address (%s)." % email, 400) @@ -397,6 +444,10 @@ def add_mail_alias(source, destination, env, update_if_exists=False, do_kick=Tru return kick(env, return_status) def remove_mail_alias(source, env, do_kick=True): + # accept IDNA domain names but normalize to Unicode before going into database + source = sanitize_idn_email_address(source) + + # remove conn, c = open_database(env, with_connection=True) c.execute("DELETE FROM aliases WHERE source=?", (source,)) if c.rowcount != 1: diff --git a/management/status_checks.py b/management/status_checks.py index db7a466..c9b82ea 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -552,6 +552,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key): if m: cert_expiration_date = dateutil.parser.parse(m.group(1)) + domain = domain.encode("idna").decode("ascii") wildcard_domain = re.sub("^[^\.]+", "*", domain) if domain is not None and domain not in certificate_names and wildcard_domain not in certificate_names: return ("The certificate is for the wrong domain name. It is for %s." diff --git a/management/templates/aliases.html b/management/templates/aliases.html index 86c8c4d..5ec85c5 100644 --- a/management/templates/aliases.html +++ b/management/templates/aliases.html @@ -27,6 +27,7 @@
+
You may use international (non-ASCII) characters, but this has not yet been well tested.
diff --git a/management/templates/users.html b/management/templates/users.html index 3f54cac..ce9f663 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -12,7 +12,7 @@

Add a mail user

-

Add an email address to this system. This will create a new login username/password. (Use aliases to create email addresses that forward to existing accounts.)

+

Add an email address to this system. This will create a new login username/password.

@@ -31,10 +31,12 @@
-

- Passwords must be at least four characters and may not contain spaces. - Administrators get access to this control panel. -

+

Existing mail users

diff --git a/management/web_update.py b/management/web_update.py index 229ca16..cb07ec8 100644 --- a/management/web_update.py +++ b/management/web_update.py @@ -89,7 +89,7 @@ def make_domain_config(domain, template, template_for_primaryhost, env): # Replace substitution strings in the template & return. nginx_conf = nginx_conf.replace("$STORAGE_ROOT", env['STORAGE_ROOT']) - nginx_conf = nginx_conf.replace("$HOSTNAME", domain) + nginx_conf = nginx_conf.replace("$HOSTNAME", domain.encode("idna").decode("ascii")) nginx_conf = nginx_conf.replace("$ROOT", root) nginx_conf = nginx_conf.replace("$SSL_KEY", ssl_key) nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", ssl_certificate) @@ -210,7 +210,7 @@ def create_csr(domain, ssl_key, env): "-key", ssl_key, "-out", "/dev/stdout", "-sha256", - "-subj", "/C=%s/ST=/L=/O=/CN=%s" % (env["CSR_COUNTRY"], domain)]) + "-subj", "/C=%s/ST=/L=/O=/CN=%s" % (env["CSR_COUNTRY"], domain.encode("idna").decode("ascii"))]) def install_cert(domain, ssl_cert, ssl_chain, env): if domain not in get_web_domains(env):