From 178c5876542d3b9c756e4f153d02167dd169bf86 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 11 Apr 2021 16:07:09 -0400 Subject: [PATCH 01/43] Migrate to the ECDSAP256SHA256 (13) DNSSEC algorithm * Stop generating RSASHA1-NSEC3-SHA1 keys on new installs since it is no longer recommended, but preserve the key on existing installs so that we continue to sign zones with existing keys to retain the chain of trust with existing DS records. * Start generating ECDSAP256SHA256 keys during setup, the current best practice (in addition to RSASHA256 which is also ok). See https://www.iana.org/assignments/dns-sec-alg-numbers/dns-sec-alg-numbers.xhtml#dns-sec-alg-numbers-1 and https://www.cloudflare.com/dns/dnssec/ecdsa-and-dnssec/. * Sign zones using all available keys rather than choosing just one based on the TLD to enable rotation/migration to the new key and to give the user some options since not every registrar/TLD supports every algorithm. * Allow a user to drop a key from signing specific domains using DOMAINS= in our key configuration file. Signing the zones with extraneous keys may increase the size of DNS responses, which isn't ideal, although I don't know if this is a problem in practice. (Although a user can delete the RSASHA1-NSEC3-SHA1 key file, the other keys will be re-generated on upgrade.) * When generating zonefiles, add a hash of all of the DNSSEC signing keys so that when the keys change the zone is definitely regenerated and re-signed. * In status checks, if DNSSEC is not active (or not valid), offer to use all of the keys that have been generated (for RSASHA1-NSEC3-SHA1 on existing installs, RSASHA256, and now ECDSAP256SHA256) with all digest types, since not all registers support everything, but list them in an order that guides users to the best practice. * In status checks, if the deployed DS record doesn't use a ECDSAP256SHA256 key, prompt the user to update their DS record. * In status checks, if multiple DS records are set, only fail if none are valid. If some use ECDSAP256SHA256 and some don't, remind the user to delete the DS records that don't. * Don't fail if the DS record uses the SHA384 digest (by pre-generating a DS record with that digest type) but don't recommend it because it is not in the IANA mandatory list yet (https://www.iana.org/assignments/ds-rr-types/ds-rr-types.xhtml). See #1953 --- CHANGELOG.md | 5 ++ management/dns_update.py | 148 ++++++++++++++++++++++-------------- management/status_checks.py | 134 ++++++++++++++++++++++---------- setup/dns.sh | 43 +++++------ 4 files changed, 207 insertions(+), 123 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a75a9a4..8098898 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ CHANGELOG ========= +In Development +-------------- + +* Migrate to the ECDSAP256SHA256 DNSSEC algorithm. If a DS record is set for any of your domain names that have DNS hosted on your box, you will be prompted by status checks to update the DS record. + v0.53 (April 12, 2021) ---------------------- diff --git a/management/dns_update.py b/management/dns_update.py index b2901bc..89777fb 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -429,6 +429,7 @@ def build_sshfp_records(): # to the zone file (that trigger bumping the serial number). However, # if SSH has been configured to listen on a nonstandard port, we must # specify that port to sshkeyscan. + port = 22 with open('/etc/ssh/sshd_config', 'r') as f: for line in f: @@ -439,8 +440,11 @@ def build_sshfp_records(): except ValueError: pass break + keys = shell("check_output", ["ssh-keyscan", "-t", "rsa,dsa,ecdsa,ed25519", "-p", str(port), "localhost"]) - for key in sorted(keys.split("\n")): + keys = sorted(keys.split("\n")) + + for key in keys: if key.strip() == "" or key[0] == "#": continue try: host, keytype, pubkey = key.split(" ") @@ -460,13 +464,16 @@ def write_nsd_zone(domain, zonefile, records, env, force): # On the $ORIGIN line, there's typically a ';' comment at the end explaining # what the $ORIGIN line does. Any further data after the domain confuses # ldns-signzone, however. It used to say '; default zone domain'. - + # # The SOA contact address for all of the domains on this system is hostmaster # @ the PRIMARY_HOSTNAME. Hopefully that's legit. - + # # For the refresh through TTL fields, a good reference is: # http://www.peerwisdom.org/2013/05/15/dns-understanding-the-soa-record/ - + # + # A hash of the available DNSSEC keys are added in a comment so that when + # the keys change we force a re-generation of the zone which triggers + # re-signing it. zone = """ $ORIGIN {domain}. @@ -502,6 +509,9 @@ $TTL 86400 ; default time to live value = v2 zone += value + "\n" + # Append a stable hash of DNSSEC signing keys in a comment. + zone += "\n; DNSSEC signing keys hash: {}\n".format(hash_dnssec_keys(domain, env)) + # DNSSEC requires re-signing a zone periodically. That requires # bumping the serial number even if no other records have changed. # We don't see the DNSSEC records yet, so we have to figure out @@ -612,53 +622,77 @@ zone: ######################################################################## -def dnssec_choose_algo(domain, env): - if '.' in domain and domain.rsplit('.')[-1] in \ - ("email", "guide", "fund", "be", "lv"): - # At GoDaddy, RSASHA256 is the only algorithm supported - # for .email and .guide. - # A variety of algorithms are supported for .fund. This - # is preferred. - # Gandi tells me that .be does not support RSASHA1-NSEC3-SHA1 - # Nic.lv does not support RSASHA1-NSEC3-SHA1 for .lv tld's - return "RSASHA256" +def find_dnssec_signing_keys(domain, env): + # For key that we generated (one per algorithm)... + d = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec') + keyconfs = [f for f in os.listdir(d) if f.endswith(".conf")] + for keyconf in keyconfs: + # Load the file holding the KSK and ZSK key filenames. + keyconf_fn = os.path.join(d, keyconf) + keyinfo = load_env_vars_from_file(keyconf_fn) - # For any domain we were able to sign before, don't change the algorithm - # on existing users. We'll probably want to migrate to SHA256 later. - return "RSASHA1-NSEC3-SHA1" + # Skip this key if the conf file has a setting named DOMAINS, + # holding a comma-separated list of domain names, and if this + # domain is not in the list. This allows easily disabling a + # key by setting "DOMAINS=" or "DOMAINS=none", other than + # deleting the key's .conf file, which might result in the key + # being regenerated next upgrade. Keys should be disabled if + # they are not needed to reduce the DNSSEC query response size. + if "DOMAINS" in keyinfo and domain not in [dd.strip() for dd in keyinfo["DOMAINS"].split(",")]: + continue + + for keytype in ("KSK", "ZSK"): + yield keytype, keyinfo[keytype] + +def hash_dnssec_keys(domain, env): + # Create a stable (by sorting the items) hash of all of the private keys + # that will be used to sign this domain. + keydata = [] + for keytype, keyfn in sorted(find_dnssec_signing_keys(domain, env)): + oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec', keyfn + ".private") + keydata.append(keytype) + keydata.append(keyfn) + with open(oldkeyfn, "r") as fr: + keydata.append( fr.read() ) + keydata = "".join(keydata).encode("utf8") + return hashlib.sha1(keydata).hexdigest() def sign_zone(domain, zonefile, env): - algo = dnssec_choose_algo(domain, env) - dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % algo)) + # Sign the zone with all of the keys that were generated during + # setup so that the user can choose which to use in their DS record at + # their registrar, and also to support migration to newer algorithms. - # In order to use the same keys for all domains, we have to generate - # a new .key file with a DNSSEC record for the specific domain. We - # can reuse the same key, but it won't validate without a DNSSEC - # record specifically for the domain. + # In order to use the key files generated at setup which are for + # the domain _domain_, we have to re-write the files and place + # the actual domain name in it, so that ldns-signzone works. # - # Copy the .key and .private files to /tmp to patch them up. - # - # Use os.umask and open().write() to securely create a copy that only - # we (root) can read. - files_to_kill = [] - for key in ("KSK", "ZSK"): - if dnssec_keys.get(key, "").strip() == "": raise Exception("DNSSEC is not properly set up.") - oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys[key]) - newkeyfn = '/tmp/' + dnssec_keys[key].replace("_domain_", domain) - dnssec_keys[key] = newkeyfn + # Patch each key, storing the patched version in /tmp for now. + # Each key has a .key and .private file. Collect a list of filenames + # for all of the keys (and separately just the key-signing keys). + all_keys = [] + ksk_keys = [] + for keytype, keyfn in find_dnssec_signing_keys(domain, env): + newkeyfn = '/tmp/' + keyfn.replace("_domain_", domain) + for ext in (".private", ".key"): - if not os.path.exists(oldkeyfn + ext): raise Exception("DNSSEC is not properly set up.") - with open(oldkeyfn + ext, "r") as fr: + # Copy the .key and .private files to /tmp to patch them up. + # + # Use os.umask and open().write() to securely create a copy that only + # we (root) can read. + oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec', keyfn + ext) + with open(oldkeyfn, "r") as fr: keydata = fr.read() - keydata = keydata.replace("_domain_", domain) # trick ldns-signkey into letting our generic key be used by this zone - fn = newkeyfn + ext + keydata = keydata.replace("_domain_", domain) prev_umask = os.umask(0o77) # ensure written file is not world-readable try: - with open(fn, "w") as fw: + with open(newkeyfn + ext, "w") as fw: fw.write(keydata) finally: os.umask(prev_umask) # other files we write should be world-readable - files_to_kill.append(fn) + + # Put the patched key filename base (without extension) into the list of keys we'll sign with. + all_keys.append(newkeyfn) + if keytype == "KSK": ksk_keys.append(newkeyfn) # Do the signing. expiry_date = (datetime.datetime.now() + datetime.timedelta(days=30)).strftime("%Y%m%d") @@ -671,32 +705,34 @@ def sign_zone(domain, zonefile, env): # zonefile to sign "/etc/nsd/zones/" + zonefile, - + ] # keys to sign with (order doesn't matter -- it'll figure it out) - dnssec_keys["KSK"], - dnssec_keys["ZSK"], - ]) + + all_keys + ) # Create a DS record based on the patched-up key files. The DS record is specific to the # zone being signed, so we can't use the .ds files generated when we created the keys. # The DS record points to the KSK only. Write this next to the zone file so we can # get it later to give to the user with instructions on what to do with it. # - # We want to be able to validate DS records too, but multiple forms may be valid depending - # on the digest type. So we'll write all (both) valid records. Only one DS record should - # actually be deployed. Preferebly the first. + # Generate a DS record for each key. There are also several possible hash algorithms that may + # be used, so we'll pre-generate all for each key. One DS record per line. Only one + # needs to actually be deployed at the registrar. We'll select the preferred one + # in the status checks. with open("/etc/nsd/zones/" + zonefile + ".ds", "w") as f: - for digest_type in ('2', '1'): - rr_ds = shell('check_output', ["/usr/bin/ldns-key2ds", - "-n", # output to stdout - "-" + digest_type, # 1=SHA1, 2=SHA256 - dnssec_keys["KSK"] + ".key" - ]) - f.write(rr_ds) + for key in ksk_keys: + for digest_type in ('1', '2', '4'): + rr_ds = shell('check_output', ["/usr/bin/ldns-key2ds", + "-n", # output to stdout + "-" + digest_type, # 1=SHA1, 2=SHA256, 4=SHA384 + key + ".key" + ]) + f.write(rr_ds) - # Remove our temporary file. - for fn in files_to_kill: - os.unlink(fn) + # Remove the temporary patched key files. + for fn in all_keys: + os.unlink(fn + ".private") + os.unlink(fn + ".key") ######################################################################## diff --git a/management/status_checks.py b/management/status_checks.py index 631a82a..607fd57 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -42,7 +42,7 @@ def get_services(): { "name": "HTTPS Web (nginx)", "port": 443, "public": True, }, ] -def run_checks(rounded_values, env, output, pool): +def run_checks(rounded_values, env, output, pool, domains_to_check=None): # run systems checks output.add_heading("System") @@ -63,7 +63,7 @@ def run_checks(rounded_values, env, output, pool): # perform other checks asynchronously run_network_checks(env, output) - run_domain_checks(rounded_values, env, output, pool) + run_domain_checks(rounded_values, env, output, pool, domains_to_check=domains_to_check) def get_ssh_port(): # Returns ssh port @@ -300,7 +300,7 @@ def run_network_checks(env, output): which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/%s.""" % (env['PUBLIC_IP'], zen, env['PUBLIC_IP'])) -def run_domain_checks(rounded_time, env, output, pool): +def run_domain_checks(rounded_time, env, output, pool, domains_to_check=None): # Get the list of domains we handle mail for. mail_domains = get_mail_domains(env) @@ -311,7 +311,8 @@ def run_domain_checks(rounded_time, env, output, pool): # Get the list of domains we serve HTTPS for. web_domains = set(get_web_domains(env)) - domains_to_check = mail_domains | dns_domains | web_domains + if domains_to_check is None: + domains_to_check = mail_domains | dns_domains | web_domains # Remove "www", "autoconfig", "autodiscover", and "mta-sts" subdomains, which we group with their parent, # if their parent is in the domains to check list. @@ -557,61 +558,103 @@ def check_dns_zone_suggestions(domain, env, output, dns_zonefiles, domains_with_ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False): - # See if the domain has a DS record set at the registrar. The DS record may have - # several forms. We have to be prepared to check for any valid record. We've - # pre-generated all of the valid digests --- read them in. + # See if the domain has a DS record set at the registrar. The DS record must + # match one of the keys that we've used to sign the zone. It may use one of + # several hashing algorithms. We've pre-generated all possible valid DS + # records, although some will be preferred. + + alg_name_map = { '7': 'RSASHA1-NSEC3-SHA1', '8': 'RSASHA256', '13': 'ECDSAP256SHA256' } + digalg_name_map = { '1': 'SHA-1', '2': 'SHA-256', '4': 'SHA-384' } + + # Read in the pre-generated DS records + expected_ds_records = { } ds_file = '/etc/nsd/zones/' + dns_zonefiles[domain] + '.ds' if not os.path.exists(ds_file): return # Domain is in our database but DNS has not yet been updated. - ds_correct = open(ds_file).read().strip().split("\n") - digests = { } - for rr_ds in ds_correct: - ds_keytag, ds_alg, ds_digalg, ds_digest = rr_ds.split("\t")[4].split(" ") - digests[ds_digalg] = ds_digest + with open(ds_file) as f: + for rr_ds in f: + rr_ds = rr_ds.rstrip() + ds_keytag, ds_alg, ds_digalg, ds_digest = rr_ds.split("\t")[4].split(" ") - # Some registrars may want the public key so they can compute the digest. The DS - # record that we suggest using is for the KSK (and that's how the DS records were generated). - alg_name_map = { '7': 'RSASHA1-NSEC3-SHA1', '8': 'RSASHA256' } - dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % alg_name_map[ds_alg])) - dnsssec_pubkey = open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key')).read().split("\t")[3].split(" ")[3] + # Some registrars may want the public key so they can compute the digest. The DS + # record that we suggest using is for the KSK (and that's how the DS records were generated). + # We'll also give the nice name for the key algorithm. + dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % alg_name_map[ds_alg])) + dnsssec_pubkey = open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key')).read().split("\t")[3].split(" ")[3] + + expected_ds_records[ (ds_keytag, ds_alg, ds_digalg, ds_digest) ] = { + "record": rr_ds, + "keytag": ds_keytag, + "alg": ds_alg, + "alg_name": alg_name_map[ds_alg], + "digalg": ds_digalg, + "digalg_name": digalg_name_map[ds_digalg], + "digest": ds_digest, + "pubkey": dnsssec_pubkey, + } # Query public DNS for the DS record at the registrar. - ds = query_dns(domain, "DS", nxdomain=None) - ds_looks_valid = ds and len(ds.split(" ")) == 4 - if ds_looks_valid: ds = ds.split(" ") - if ds_looks_valid and ds[0] == ds_keytag and ds[1] == ds_alg and ds[3] == digests.get(ds[2]): - if is_checking_primary: return - output.print_ok("DNSSEC 'DS' record is set correctly at registrar.") + ds = query_dns(domain, "DS", nxdomain=None, as_list=True) + if ds is None or isinstance(ds, str): ds = [] + + # There may be more that one record, so we get the result as a list. + # Filter out records that don't look valid, just in case, and split + # each record on spaces. + ds = [tuple(str(rr).split(" ")) for rr in ds if len(str(rr).split(" ")) == 4] + + if len(ds) == 0: + output.print_warning("""This domain's DNSSEC DS record is not set. The DS record is optional. The DS record activates DNSSEC. See below for instructions.""") else: - if ds == None: - if is_checking_primary: return - output.print_warning("""This domain's DNSSEC DS record is not set. The DS record is optional. The DS record activates DNSSEC. - To set a DS record, you must follow the instructions provided by your domain name registrar and provide to them this information:""") + matched_ds = set(ds) & set(expected_ds_records) + if matched_ds: + # At least one DS record matches one that corresponds with one of the ways we signed + # the zone, so it is valid. + # + # But it may not be preferred. Only algorithm 13 is preferred. Warn if any of the + # matched zones uses a different algorithm. + if set(r[1] for r in matched_ds) == { '13' }: # all are alg 13 + output.print_ok("DNSSEC 'DS' record is set correctly at registrar.") + return + elif '13' in set(r[1] for r in matched_ds): # some but not all are alg 13 + output.print_ok("DNSSEC 'DS' record is set correctly at registrar. (Records using algorithm other than ECDSAP256SHA256 should be removed.)") + return + else: # no record uses alg 13 + output.print_warning("DNSSEC 'DS' record set at registrar is valid but should be updated to ECDSAP256SHA256 (see below).") else: if is_checking_primary: output.print_error("""The DNSSEC 'DS' record for %s is incorrect. See further details below.""" % domain) return output.print_error("""This domain's DNSSEC DS record is incorrect. The chain of trust is broken between the public DNS system and this machine's DNS server. It may take several hours for public DNS to update after a change. If you did not recently - make a change, you must resolve this immediately by following the instructions provided by your domain name registrar and - provide to them this information:""") + make a change, you must resolve this immediately (see below).""") + + output.print_line("""Follow the instructions provided by your domain name registrar to set a DS record. + Registrars support different sorts of DS records. Use the first option that works:""") + preferred_ds_order = [(7, 1), (7, 2), (8, 4), (13, 4), (8, 1), (8, 2), (13, 1), (13, 2)] # low to high + def preferred_ds_order_func(ds_suggestion): + k = (int(ds_suggestion['alg']), int(ds_suggestion['digalg'])) + if k in preferred_ds_order: + return preferred_ds_order.index(k) + return -1 # index before first item + output.print_line("") + for i, ds_suggestion in enumerate(sorted(expected_ds_records.values(), key=preferred_ds_order_func, reverse=True)): output.print_line("") - output.print_line("Key Tag: " + ds_keytag + ("" if not ds_looks_valid or ds[0] == ds_keytag else " (Got '%s')" % ds[0])) + output.print_line("Option " + str(i+1) + ":") + output.print_line("----------") + output.print_line("Key Tag: " + ds_suggestion['keytag']) output.print_line("Key Flags: KSK") - output.print_line( - ("Algorithm: %s / %s" % (ds_alg, alg_name_map[ds_alg])) - + ("" if not ds_looks_valid or ds[1] == ds_alg else " (Got '%s')" % ds[1])) - # see http://www.iana.org/assignments/dns-sec-alg-numbers/dns-sec-alg-numbers.xhtml - output.print_line("Digest Type: 2 / SHA-256") - # http://www.ietf.org/assignments/ds-rr-types/ds-rr-types.xml - output.print_line("Digest: " + digests['2']) - if ds_looks_valid and ds[3] != digests.get(ds[2]): - output.print_line("(Got digest type %s and digest %s which do not match.)" % (ds[2], ds[3])) + output.print_line("Algorithm: %s / %s" % (ds_suggestion['alg'], ds_suggestion['alg_name'])) + output.print_line("Digest Type: %s / %s" % (ds_suggestion['digalg'], ds_suggestion['digalg_name'])) + output.print_line("Digest: " + ds_suggestion['digest']) output.print_line("Public Key: ") - output.print_line(dnsssec_pubkey, monospace=True) + output.print_line(ds_suggestion['pubkey'], monospace=True) output.print_line("") output.print_line("Bulk/Record Format:") - output.print_line("" + ds_correct[0]) + output.print_line(ds_suggestion['record'], monospace=True) + if len(ds) > 0: output.print_line("") + output.print_line("The DS record is currently set to:") + for rr in ds: + output.print_line("Key Tag: {0}, Algorithm: {1}, Digest Type: {2}, Digest: {3}".format(*rr)) def check_mail_domain(domain, env, output): # Check the MX record. @@ -713,7 +756,7 @@ def check_web_domain(domain, rounded_time, ssl_certificates, env, output): # website for also needs a signed certificate. check_ssl_cert(domain, rounded_time, ssl_certificates, env, output) -def query_dns(qname, rtype, nxdomain='[Not Set]', at=None): +def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False): # Make the qname absolute by appending a period. Without this, dns.resolver.query # will fall back a failed lookup to a second query with this machine's hostname # appended. This has been causing some false-positive Spamhaus reports. The @@ -750,6 +793,9 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None): if rtype in ("A", "AAAA"): response = [normalize_ip(str(r)) for r in response] + if as_list: + return response + # There may be multiple answers; concatenate the response. Remove trailing # periods from responses since that's how qnames are encoded in DNS but is # confusing for us. The order of the answers doesn't matter, so sort so we @@ -1050,3 +1096,7 @@ if __name__ == "__main__": elif sys.argv[1] == "--version": print(what_version_is_this(env)) + + elif sys.argv[1] == "--only": + with multiprocessing.pool.Pool(processes=10) as pool: + run_checks(False, env, ConsoleOutput(), pool, domains_to_check=sys.argv[2:]) diff --git a/setup/dns.sh b/setup/dns.sh index 5d86227..2a1b6da 100755 --- a/setup/dns.sh +++ b/setup/dns.sh @@ -68,27 +68,15 @@ echo "include: /etc/nsd/zones.conf" >> /etc/nsd/nsd.conf; mkdir -p "$STORAGE_ROOT/dns/dnssec"; -# TLDs don't all support the same algorithms, so we'll generate keys using a few -# different algorithms. RSASHA1-NSEC3-SHA1 was possibly the first widely used -# algorithm that supported NSEC3, which is a security best practice. However TLDs -# will probably be moving away from it to a a SHA256-based algorithm. -# -# Supports `RSASHA1-NSEC3-SHA1` (didn't test with `RSASHA256`): -# -# * .info -# * .me -# -# Requires `RSASHA256` -# -# * .email -# * .guide -# -# Supports `RSASHA256` (and defaulting to this) -# -# * .fund - +# TLDs, registrars, and validating nameservers don't all support the same algorithms, +# so we'll generate keys using a few different algorithms so that dns_update.py can +# choose which algorithm to use when generating the zonefiles. See #1953 for recent +# discussion. File for previously used algorithms (i.e. RSASHA1-NSEC3-SHA1) may still +# be in the output directory, and we'll continue to support signing zones with them +# so that trust isn't broken with deployed DS records, but we won't generate those +# keys on new systems. FIRST=1 #NODOC -for algo in RSASHA1-NSEC3-SHA1 RSASHA256; do +for algo in RSASHA256 ECDSAP256SHA256; do if [ ! -f "$STORAGE_ROOT/dns/dnssec/$algo.conf" ]; then if [ $FIRST == 1 ]; then echo "Generating DNSSEC signing keys..." @@ -97,7 +85,7 @@ if [ ! -f "$STORAGE_ROOT/dns/dnssec/$algo.conf" ]; then # Create the Key-Signing Key (KSK) (with `-k`) which is the so-called # Secure Entry Point. The domain name we provide ("_domain_") doesn't - # matter -- we'll use the same keys for all our domains. + # matter -- we'll use the same keys for all our domains. # # `ldns-keygen` outputs the new key's filename to stdout, which # we're capturing into the `KSK` variable. @@ -105,17 +93,22 @@ if [ ! -f "$STORAGE_ROOT/dns/dnssec/$algo.conf" ]; then # ldns-keygen uses /dev/random for generating random numbers by default. # This is slow and unecessary if we ensure /dev/urandom is seeded properly, # so we use /dev/urandom. See system.sh for an explanation. See #596, #115. - KSK=$(umask 077; cd $STORAGE_ROOT/dns/dnssec; ldns-keygen -r /dev/urandom -a $algo -b 2048 -k _domain_); + # (This previously used -b 2048 but it's unclear if this setting makes sense + # for non-RSA keys, so it's removed. The RSA-based keys are not recommended + # anymore anyway.) + KSK=$(umask 077; cd $STORAGE_ROOT/dns/dnssec; ldns-keygen -r /dev/urandom -a $algo -k _domain_); # Now create a Zone-Signing Key (ZSK) which is expected to be # rotated more often than a KSK, although we have no plans to # rotate it (and doing so would be difficult to do without - # disturbing DNS availability.) Omit `-k` and use a shorter key length. - ZSK=$(umask 077; cd $STORAGE_ROOT/dns/dnssec; ldns-keygen -r /dev/urandom -a $algo -b 1024 _domain_); + # disturbing DNS availability.) Omit `-k`. + # (This previously used -b 1024 but it's unclear if this setting makes sense + # for non-RSA keys, so it's removed.) + ZSK=$(umask 077; cd $STORAGE_ROOT/dns/dnssec; ldns-keygen -r /dev/urandom -a $algo _domain_); # These generate two sets of files like: # - # * `K_domain_.+007+08882.ds`: DS record normally provided to domain name registrar (but it's actually invalid with `_domain_`) + # * `K_domain_.+007+08882.ds`: DS record normally provided to domain name registrar (but it's actually invalid with `_domain_` so we don't use this file) # * `K_domain_.+007+08882.key`: public key # * `K_domain_.+007+08882.private`: private key (secret!) From 8cda58fb2282f53ed0bb5c3707bafe031c2ae7d7 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Mon, 12 Apr 2021 18:46:35 -0400 Subject: [PATCH 02/43] Speed up status checks a bit by removing a redundant check if the PRIMARY_HOSTNAME certificate is signed and valid --- management/dns_update.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/management/dns_update.py b/management/dns_update.py index 89777fb..c595bc3 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -127,6 +127,10 @@ def build_zones(env): from web_update import get_web_domains www_redirect_domains = set(get_web_domains(env)) - set(get_web_domains(env, include_www_redirects=False)) + # For MTA-STS, we'll need to check if the PRIMARY_HOSTNAME certificate is + # singned and valid. Check that now rather than repeatedly for each domain. + env["-primary-hostname-certificate-is-valid"] = is_domain_cert_signed_and_valid(env["PRIMARY_HOSTNAME"], env) + # Build DNS records for each zone. for domain, zonefile in zonefiles: # Build the records to put in the zone. @@ -322,24 +326,11 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en # certificate in use is not valid (e.g. because it is self-signed and a valid certificate has not # yet been provisioned). Since we cannot provision a certificate without A/AAAA records, we # always set them --- only the TXT records depend on there being valid certificates. - mta_sts_enabled = False mta_sts_records = [ ("mta-sts", "A", env["PUBLIC_IP"], "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."), ("mta-sts", "AAAA", env.get('PUBLIC_IPV6'), "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."), ] - if domain in get_mail_domains(env): - # Check that PRIMARY_HOSTNAME and the mta_sts domain both have valid certificates. - for d in (env['PRIMARY_HOSTNAME'], "mta-sts." + domain): - cert = get_ssl_certificates(env).get(d) - if not cert: - break # no certificate provisioned for this domain - cert_status = check_certificate(d, cert['certificate'], cert['private-key']) - if cert_status[0] != 'OK': - break # certificate is not valid - else: - # 'break' was not encountered above, so both domains are good - mta_sts_enabled = True - if mta_sts_enabled: + if domain in get_mail_domains(env) and env["-primary-hostname-certificate-is-valid"] and is_domain_cert_signed_and_valid("mta-sts." + domain, env): # Compute an up-to-32-character hash of the policy file. We'll take a SHA-1 hash of the policy # file (20 bytes) and encode it as base-64 (28 bytes, using alphanumeric alternate characters # instead of '+' and '/' which are not allowed in an MTA-STS policy id) but then just take its @@ -365,6 +356,13 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en return records +def is_domain_cert_signed_and_valid(domain, env): + cert = get_ssl_certificates(env).get(domain) + if not cert: return False # no certificate provisioned + cert_status = check_certificate(domain, cert['certificate'], cert['private-key']) + print(domain, cert_status) + return cert_status[0] == 'OK' + ######################################################################## def build_tlsa_record(env): From 2c295bcafd8e00a8f6a33a8b9d827a7aeacedbb5 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Fri, 23 Apr 2021 17:02:31 -0400 Subject: [PATCH 03/43] Upgrade the Roundcube persistent login cookie encryption to AES-256-CBC and increase the key length accordingly This change will force everyone to be logged out of Roundcube since the encryption key and cipher won't match anyone's already-set cookie, but this happens anyway after every Mail-in-a-Box update since we generate a new key each time already. Fixes #1968. --- CHANGELOG.md | 1 + setup/webmail.sh | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8098898..1efe77d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ In Development -------------- * Migrate to the ECDSAP256SHA256 DNSSEC algorithm. If a DS record is set for any of your domain names that have DNS hosted on your box, you will be prompted by status checks to update the DS record. +* Roundcube's login cookie is updated to use a new encryption algorithm (AES-256-CBC instead of DES-EDE-CBC). v0.53 (April 12, 2021) ---------------------- diff --git a/setup/webmail.sh b/setup/webmail.sh index 912bd5e..98e12d1 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -91,8 +91,9 @@ fi # ### Configuring Roundcube -# Generate a safe 24-character secret key of safe characters. -SECRET_KEY=$(dd if=/dev/urandom bs=1 count=18 2>/dev/null | base64 | fold -w 24 | head -n 1) +# Generate a secret key of PHP-string-safe characters appropriate +# for the cipher algorithm selected below. +SECRET_KEY=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 | sed s/=//g) # Create a configuration file. # @@ -126,7 +127,8 @@ cat > $RCM_CONFIG < ~256 bits for AES-256, see above \$config['plugins'] = array('html5_notifier', 'archive', 'zipdownload', 'password', 'managesieve', 'jqueryui', 'persistent_login', 'carddav'); \$config['skin'] = 'elastic'; \$config['login_autocomplete'] = 2; From ae3feebd80195bd467262208f2cdada1b3fc458b Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Mon, 3 May 2021 19:04:59 -0400 Subject: [PATCH 04/43] Fix warnings reported by shellcheck * SC2068: Double quote array expansions to avoid re-splitting elements. * SC2186: tempfile is deprecated. Use mktemp instead. * SC2124: Assigning an array to a string! Assign as array, or use * instead of @ to concatenate. * SC2102: Ranges can only match single chars (mentioned due to duplicates). * SC2005: Useless echo? Instead of 'echo $(cmd)', just use 'cmd'. --- setup/functions.sh | 9 ++++----- setup/mail-postfix.sh | 2 +- setup/start.sh | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/setup/functions.sh b/setup/functions.sh index 90c4c55..718a228 100644 --- a/setup/functions.sh +++ b/setup/functions.sh @@ -9,12 +9,12 @@ function hide_output { # and returns a non-zero exit code. # Get a temporary file. - OUTPUT=$(tempfile) + OUTPUT=$(mktemp) # Execute command, redirecting stderr/stdout to the temporary file. Since we # check the return code ourselves, disable 'set -e' temporarily. set +e - $@ &> $OUTPUT + "$@" &> $OUTPUT E=$? set -e @@ -22,7 +22,7 @@ function hide_output { if [ $E != 0 ]; then # Something failed. echo - echo FAILED: $@ + echo FAILED: "$@" echo ----------------------------------------- cat $OUTPUT echo ----------------------------------------- @@ -53,8 +53,7 @@ function apt_install { # install' for all of the packages. Calling `dpkg` on each package is slow, # and doesn't affect what we actually do, except in the messages, so let's # not do that anymore. - PACKAGES=$@ - apt_get_quiet install $PACKAGES + apt_get_quiet install "$@" } function get_default_hostname { diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 0a66cb0..b16fd94 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -191,7 +191,7 @@ tools/editconf.py /etc/postfix/main.cf \ # # In a basic setup we would pass mail directly to Dovecot by setting # virtual_transport to `lmtp:unix:private/dovecot-lmtp`. -tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025 +tools/editconf.py /etc/postfix/main.cf "virtual_transport=lmtp:[127.0.0.1]:10025" # Because of a spampd bug, limit the number of recipients in each connection. # See https://github.com/mail-in-a-box/mailinabox/issues/1523. tools/editconf.py /etc/postfix/main.cf lmtp_destination_recipient_limit=1 diff --git a/setup/start.sh b/setup/start.sh index cedc426..b4e38e6 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -78,7 +78,7 @@ if [ ! -d $STORAGE_ROOT ]; then mkdir -p $STORAGE_ROOT fi if [ ! -f $STORAGE_ROOT/mailinabox.version ]; then - echo $(setup/migrate.py --current) > $STORAGE_ROOT/mailinabox.version + setup/migrate.py --current > $STORAGE_ROOT/mailinabox.version chown $STORAGE_USER.$STORAGE_USER $STORAGE_ROOT/mailinabox.version fi From 9b07d86bf786bda73bc8c5ad95d2d9cb9e08be3f Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Mon, 3 May 2021 19:28:23 -0400 Subject: [PATCH 05/43] Use $(...) notation instead of legacy backtick notation for embedded shell commands shellcheck reported SC2006: Use $(...) notation instead of legacy backticked `...`. Fixed by applying shellcheck's diff output as a patch. --- setup/bootstrap.sh | 8 ++++---- setup/dns.sh | 2 +- setup/firstuser.sh | 4 ++-- setup/mail-dovecot.sh | 4 ++-- setup/management.sh | 4 ++-- setup/nextcloud.sh | 2 +- setup/preflight.sh | 2 +- setup/start.sh | 2 +- setup/webmail.sh | 2 +- setup/zpush.sh | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index d804cad..a9bc2ce 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -18,11 +18,11 @@ if [ -z "$TAG" ]; then # space, but if we put it in a comment it would confuse the status checks!) # to get the latest version, so the first such line must be the one that we # want to display in status checks. - if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then + if [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' )" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. TAG=v0.53 - elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then + elif [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' )" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. echo "You are installing the last version of Mail-in-a-Box that will" echo "support Ubuntu 14.04. If this is a new installation of Mail-in-a-Box," @@ -68,11 +68,11 @@ fi cd $HOME/mailinabox # Update it. -if [ "$TAG" != `git describe` ]; then +if [ "$TAG" != $(git describe) ]; then echo Updating Mail-in-a-Box to $TAG . . . git fetch --depth 1 --force --prune origin tag $TAG if ! git checkout -q $TAG; then - echo "Update failed. Did you modify something in `pwd`?" + echo "Update failed. Did you modify something in $(pwd)?" exit 1 fi echo diff --git a/setup/dns.sh b/setup/dns.sh index 2a1b6da..b64a658 100755 --- a/setup/dns.sh +++ b/setup/dns.sh @@ -132,7 +132,7 @@ cat > /etc/cron.daily/mailinabox-dnssec << EOF; #!/bin/bash # Mail-in-a-Box # Re-sign any DNS zones with DNSSEC because the signatures expire periodically. -`pwd`/tools/dns_update +$(pwd)/tools/dns_update EOF chmod +x /etc/cron.daily/mailinabox-dnssec diff --git a/setup/firstuser.sh b/setup/firstuser.sh index e2d6531..7caec35 100644 --- a/setup/firstuser.sh +++ b/setup/firstuser.sh @@ -1,5 +1,5 @@ # If there aren't any mail users yet, create one. -if [ -z "`management/cli.py user`" ]; then +if [ -z "$(management/cli.py user)" ]; then # The outut of "management/cli.py user" is a list of mail users. If there # aren't any yet, it'll be empty. @@ -10,7 +10,7 @@ if [ -z "`management/cli.py user`" ]; then input_box "Mail Account" \ "Let's create your first mail account. \n\nWhat email address do you want?" \ - me@`get_default_hostname` \ + me@$(get_default_hostname) \ EMAIL_ADDR if [ -z "$EMAIL_ADDR" ]; then diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index c6e6cb3..b569c40 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -45,8 +45,8 @@ apt_install \ # - https://www.dovecot.org/list/dovecot/2012-August/137569.html # - https://www.dovecot.org/list/dovecot/2011-December/132455.html tools/editconf.py /etc/dovecot/conf.d/10-master.conf \ - default_process_limit=$(echo "`nproc` * 250" | bc) \ - default_vsz_limit=$(echo "`free -tm | tail -1 | awk '{print $2}'` / 3" | bc)M \ + default_process_limit=$(echo "$(nproc) * 250" | bc) \ + default_vsz_limit=$(echo "$(free -tm | tail -1 | awk '{print $2}') / 3" | bc)M \ log_path=/var/log/mail.log # The inotify `max_user_instances` default is 128, which constrains diff --git a/setup/management.sh b/setup/management.sh index dcef089..1c57bb2 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -97,7 +97,7 @@ export LANG=en_US.UTF-8 export LC_TYPE=en_US.UTF-8 source $venv/bin/activate -exec python `pwd`/management/daemon.py +exec python $(pwd)/management/daemon.py EOF chmod +x $inst_dir/start cp --remove-destination conf/mailinabox.service /lib/systemd/system/mailinabox.service # target was previously a symlink so remove it first @@ -112,7 +112,7 @@ minute=$((RANDOM % 60)) # avoid overloading mailinabox.email cat > /etc/cron.d/mailinabox-nightly << EOF; # Mail-in-a-Box --- Do not edit / will be overwritten on update. # Run nightly tasks: backup, status checks. -$minute 3 * * * root (cd `pwd` && management/daily_tasks.sh) +$minute 3 * * * root (cd $(pwd) && management/daily_tasks.sh) EOF # Start the management server. diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 200eba9..9864598 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -128,7 +128,7 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc # Backup the existing ownCloud/Nextcloud. # Create a backup directory to store the current installation and database to - BACKUP_DIRECTORY=$STORAGE_ROOT/owncloud-backup/`date +"%Y-%m-%d-%T"` + BACKUP_DIRECTORY=$STORAGE_ROOT/owncloud-backup/$(date +"%Y-%m-%d-%T") mkdir -p "$BACKUP_DIRECTORY" if [ -d /usr/local/lib/owncloud/ ]; then echo "Upgrading Nextcloud --- backing up existing installation, configuration, and database to directory to $BACKUP_DIRECTORY..." diff --git a/setup/preflight.sh b/setup/preflight.sh index acaf80c..9d2715c 100644 --- a/setup/preflight.sh +++ b/setup/preflight.sh @@ -8,7 +8,7 @@ if [[ $EUID -ne 0 ]]; then fi # Check that we are running on Ubuntu 18.04 LTS (or 18.04.xx). -if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" != "Ubuntu 18.04 LTS" ]; then +if [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' )" != "Ubuntu 18.04 LTS" ]; then echo "Mail-in-a-Box only supports being installed on Ubuntu 18.04, sorry. You are running:" echo lsb_release -d | sed 's/.*:\s*//' diff --git a/setup/start.sh b/setup/start.sh index b4e38e6..0cca66b 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -46,7 +46,7 @@ fi # in the first dialog prompt, so we should do this before that starts. cat > /usr/local/bin/mailinabox << EOF; #!/bin/bash -cd `pwd` +cd $(pwd) source setup/start.sh EOF chmod +x /usr/local/bin/mailinabox diff --git a/setup/webmail.sh b/setup/webmail.sh index 98e12d1..55fea63 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -47,7 +47,7 @@ needs_update=0 #NODOC if [ ! -f /usr/local/lib/roundcubemail/version ]; then # not installed yet #NODOC needs_update=1 #NODOC -elif [[ "$UPDATE_KEY" != `cat /usr/local/lib/roundcubemail/version` ]]; then +elif [[ "$UPDATE_KEY" != $(cat /usr/local/lib/roundcubemail/version) ]]; then # checks if the version is what we want needs_update=1 #NODOC fi diff --git a/setup/zpush.sh b/setup/zpush.sh index 1a84e86..783f39a 100755 --- a/setup/zpush.sh +++ b/setup/zpush.sh @@ -27,7 +27,7 @@ TARGETHASH=4b312d64227ef887b24d9cc8f0ae17519586f6e2 needs_update=0 #NODOC if [ ! -f /usr/local/lib/z-push/version ]; then needs_update=1 #NODOC -elif [[ $VERSION != `cat /usr/local/lib/z-push/version` ]]; then +elif [[ $VERSION != $(cat /usr/local/lib/z-push/version) ]]; then # checks if the version needs_update=1 #NODOC fi From 69fc2fdd3aa0e1c88d7fa7434560025e1b97848c Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Mon, 3 May 2021 19:41:00 -0400 Subject: [PATCH 06/43] Hide spurrious Nextcloud setup output --- setup/nextcloud.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 9864598..3d86535 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -312,7 +312,9 @@ sudo -u www-data php /usr/local/lib/owncloud/occ upgrade if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then exit 1; fi # Disable default apps that we don't support -sudo -u www-data php /usr/local/lib/owncloud/occ app:disable photos dashboard activity +sudo -u www-data \ + php /usr/local/lib/owncloud/occ app:disable photos dashboard activity \ + | grep -v "No such app enabled" # Set PHP FPM values to support large file uploads # (semicolon is the comment character in this file, hashes produce deprecation warnings) From 16e81e14392ed70ce36c241b53c83e2751060e5f Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Sat, 8 May 2021 05:18:49 -0700 Subject: [PATCH 07/43] Fix to allow for non forced "enforce" MTA_STS_MODE (#1970) --- setup/start.sh | 2 +- setup/web.sh | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/setup/start.sh b/setup/start.sh index 0cca66b..bd743ac 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -94,7 +94,7 @@ PUBLIC_IP=$PUBLIC_IP PUBLIC_IPV6=$PUBLIC_IPV6 PRIVATE_IP=$PRIVATE_IP PRIVATE_IPV6=$PRIVATE_IPV6 -MTA_STS_MODE=${MTA_STS_MODE-} +MTA_STS_MODE=${DEFAULT_MTA_STS_MODE:-enforce} EOF # Start service configuration. diff --git a/setup/web.sh b/setup/web.sh index 42c301e..4433ff0 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -126,13 +126,13 @@ chmod a+r /var/lib/mailinabox/mozilla-autoconfig.xml # nginx configuration at /.well-known/mta-sts.txt # more documentation is available on: # https://www.uriports.com/blog/mta-sts-explained/ -# default mode is "enforce". Change to "testing" which means -# "Messages will be delivered as though there was no failure -# but a report will be sent if TLS-RPT is configured" if you -# are not sure you want this yet. Or "none". +# default mode is "enforce". In /etc/mailinabox.conf change +# "MTA_STS_MODE=testing" which means "Messages will be delivered +# as though there was no failure but a report will be sent if +# TLS-RPT is configured" if you are not sure you want this yet. Or "none". PUNY_PRIMARY_HOSTNAME=$(echo "$PRIMARY_HOSTNAME" | idn2) cat conf/mta-sts.txt \ - | sed "s/MODE/${MTA_STS_MODE:-enforce}/" \ + | sed "s/MODE/${MTA_STS_MODE}/" \ | sed "s/PRIMARY_HOSTNAME/$PUNY_PRIMARY_HOSTNAME/" \ > /var/lib/mailinabox/mta-sts.txt chmod a+r /var/lib/mailinabox/mta-sts.txt From 49813534bdaeaa82e3ac1ee70b78e91af5783dba Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Sat, 8 May 2021 05:24:04 -0700 Subject: [PATCH 08/43] Updated Nextcloud to 20.0.8, contacts to 3.5.1, calendar to 2.2.0 (#1960) --- setup/nextcloud.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 3d86535..57e5e03 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -97,12 +97,12 @@ InstallNextcloud() { } # Nextcloud Version to install. Checks are done down below to step through intermediate versions. -nextcloud_ver=20.0.1 -nextcloud_hash=f2b3faa570c541df73f209e873a1c2852e79eab8 -contacts_ver=3.4.1 -contacts_hash=aee680a75e95f26d9285efd3c1e25cf7f3bfd27e -calendar_ver=2.1.2 -calendar_hash=930c07863bb7a65652dec34793802c8d80502336 +nextcloud_ver=20.0.8 +nextcloud_hash=372b0b4bb07c7984c04917aff86b280e68fbe761 +contacts_ver=3.5.1 +contacts_hash=d2ffbccd3ed89fa41da20a1dff149504c3b33b93 +calendar_ver=2.2.0 +calendar_hash=673ad72ca28adb8d0f209015ff2dca52ffad99af user_external_ver=1.0.0 user_external_hash=3bf2609061d7214e7f0f69dd8883e55c4ec8f50a From 12aaebfc54972cab4edd990f1eec519535314a69 Mon Sep 17 00:00:00 2001 From: Jawad Seddar Date: Sat, 8 May 2021 14:25:33 +0200 Subject: [PATCH 09/43] `custom.yaml`: add support for X-Frame-Options header and proxy_redirect off (#1954) --- management/web_update.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/management/web_update.py b/management/web_update.py index 83aa91b..5048cba 100644 --- a/management/web_update.py +++ b/management/web_update.py @@ -160,17 +160,27 @@ def make_domain_config(domain, templates, ssl_certificates, env): for path, url in yaml.get("proxies", {}).items(): # Parse some flags in the fragment of the URL. pass_http_host_header = False + proxy_redirect_off = False + frame_options_header_sameorigin = False m = re.search("#(.*)$", url) if m: for flag in m.group(1).split(","): if flag == "pass-http-host": pass_http_host_header = True + elif flag == "no-proxy-redirect": + proxy_redirect_off = True + elif flag == "frame-options-sameorigin": + frame_options_header_sameorigin = True url = re.sub("#(.*)$", "", url) nginx_conf_extra += "\tlocation %s {" % path nginx_conf_extra += "\n\t\tproxy_pass %s;" % url + if proxy_redirect_off: + nginx_conf_extra += "\n\t\tproxy_redirect off;" if pass_http_host_header: nginx_conf_extra += "\n\t\tproxy_set_header Host $http_host;" + if frame_options_header_sameorigin: + nginx_conf_extra += "\n\t\tproxy_set_header X-Frame-Options SAMEORIGIN;" nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;" nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-Host $http_host;" nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-Proto $scheme;" @@ -251,3 +261,4 @@ def get_web_domains_info(env): } for domain in get_web_domains(env) ] + From bc4ae51c2d19c7753d1c2e65bc26b443dd5048c8 Mon Sep 17 00:00:00 2001 From: Hala Alajlan <36444614+halaalajlan@users.noreply.github.com> Date: Sat, 8 May 2021 15:26:40 +0300 Subject: [PATCH 10/43] Handle query dns timeout unhandled error (#1950) Co-authored-by: hala alajlan --- management/status_checks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/management/status_checks.py b/management/status_checks.py index 607fd57..1b2a16c 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -664,6 +664,8 @@ def check_mail_domain(domain, env, output): if mx is None: mxhost = None + elif mx == "[timeout]": + mxhost = None else: # query_dns returns a semicolon-delimited list # of priority-host pairs. From 3701e05d925fe780e1a43e4d54b247473136f841 Mon Sep 17 00:00:00 2001 From: Thomas Urban Date: Sat, 8 May 2021 14:30:53 +0200 Subject: [PATCH 11/43] Rewrite envelope from address in sieve forwards (#1949) Fixes #1946. --- setup/mail-dovecot.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index b569c40..26d3289 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -183,6 +183,7 @@ plugin { sieve_after = $STORAGE_ROOT/mail/sieve/global_after sieve = $STORAGE_ROOT/mail/sieve/%d/%n.sieve sieve_dir = $STORAGE_ROOT/mail/sieve/%d/%n + sieve_redirect_envelope_from = recipient } EOF From d4c5872547ee0222759be7c195a358698c5dfa66 Mon Sep 17 00:00:00 2001 From: "John @ S4" <64874788+John-S4@users.noreply.github.com> Date: Sat, 8 May 2021 05:32:58 -0700 Subject: [PATCH 12/43] Make clear that non-AWS S3 backups are supported (#1947) Just a few wording changes to show that it is possible to make S3 backups to other services than AWS - prompted by a thread on MIAB discourse. --- management/templates/system-backup.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 7cdc380..a63b38e 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -5,7 +5,7 @@

Backup Status

-

The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also have it stored on Amazon S3.

+

The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also store in on S3-compatible services like Amazon Web Services (AWS).

Configuration

@@ -17,7 +17,7 @@ - + @@ -73,8 +73,8 @@
-

Backups are stored in an Amazon Web Services S3 bucket. You must have an AWS account already.

-

You MUST manually copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Amazon S3 bucket.

+

Backups are stored in an S3-compatible bucket. You must have an AWS or other S3 service account already.

+

You MUST manually copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your S3 bucket.

@@ -84,7 +84,7 @@ {% for name, host in backup_s3_hosts %} {% endfor %} - +
@@ -343,4 +343,4 @@ function init_inputs(target_type) { set_host($('#backup-target-s3-host-select').val()); } } - \ No newline at end of file + From dbd6dae5ceda7cc0ce2c132be1f0b795f0a2c363 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 8 May 2021 09:01:40 -0400 Subject: [PATCH 13/43] Fix exit status issue cased by 69fc2fdd --- setup/nextcloud.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 57e5e03..af84834 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -314,7 +314,7 @@ if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then exit 1; fi # Disable default apps that we don't support sudo -u www-data \ php /usr/local/lib/owncloud/occ app:disable photos dashboard activity \ - | grep -v "No such app enabled" + | (grep -v "No such app enabled" || /bin/true) # Set PHP FPM values to support large file uploads # (semicolon is the comment character in this file, hashes produce deprecation warnings) From aaa81ec87979decb50a352bee30d93e3d748439d Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 8 May 2021 09:06:18 -0400 Subject: [PATCH 14/43] Fix indentation issue in bc4ae51c2d19c7753d1c2e65bc26b443dd5048c8 --- management/status_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/status_checks.py b/management/status_checks.py index 1b2a16c..67b2697 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -665,7 +665,7 @@ def check_mail_domain(domain, env, output): if mx is None: mxhost = None elif mx == "[timeout]": - mxhost = None + mxhost = None else: # query_dns returns a semicolon-delimited list # of priority-host pairs. From 354a774989b52a6084a9610dace0539d995ceead Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 9 May 2021 07:34:44 -0400 Subject: [PATCH 15/43] Remove a debug line added in 8cda58fb --- management/dns_update.py | 1 - 1 file changed, 1 deletion(-) diff --git a/management/dns_update.py b/management/dns_update.py index c595bc3..ff1e97e 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -360,7 +360,6 @@ def is_domain_cert_signed_and_valid(domain, env): cert = get_ssl_certificates(env).get(domain) if not cert: return False # no certificate provisioned cert_status = check_certificate(domain, cert['certificate'], cert['private-key']) - print(domain, cert_status) return cert_status[0] == 'OK' ######################################################################## From e421addf1c13f4ba13f09b645f6d83a1772e4483 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 9 May 2021 08:16:07 -0400 Subject: [PATCH 16/43] Pre-load domain purpopses when building DNS zonefiles rather than querying mail domains at each subdomain --- management/dns_update.py | 77 +++++++++++++++++++++++++++------------- management/web_update.py | 19 +++++----- 2 files changed, 62 insertions(+), 34 deletions(-) diff --git a/management/dns_update.py b/management/dns_update.py index ff1e97e..a9a686d 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -9,7 +9,6 @@ import ipaddress import rtyaml import dns.resolver -from mailconfig import get_mail_domains, get_mail_aliases from utils import shell, load_env_vars_from_file, safe_domain_name, sort_domains from ssl_certificates import get_ssl_certificates, check_certificate @@ -20,10 +19,14 @@ from ssl_certificates import get_ssl_certificates, check_certificate DOMAIN_RE = "^(?!\-)(?:[*][.])?(?:[a-zA-Z\d\-_]{0,62}[a-zA-Z\d_]\.){1,126}(?!\d+)[a-zA-Z\d_]{1,63}(\.?)$" def get_dns_domains(env): - # Add all domain names in use by email users and mail aliases and ensure - # PRIMARY_HOSTNAME is in the list. + # Add all domain names in use by email users and mail aliases, any + # domains we serve web for (except www redirects because that would + # lead to infinite recursion here) and ensure PRIMARY_HOSTNAME is in the list. + from mailconfig import get_mail_domains + from web_update import get_web_domains domains = set() - domains |= get_mail_domains(env) + domains |= set(get_mail_domains(env)) + domains |= set(get_web_domains(env, include_www_redirects=False)) domains.add(env['PRIMARY_HOSTNAME']) return domains @@ -97,7 +100,8 @@ def do_dns_update(env, force=False): if len(updated_domains) > 0: shell('check_call', ["/usr/sbin/service", "nsd", "restart"]) - # Write the OpenDKIM configuration tables for all of the domains. + # Write the OpenDKIM configuration tables for all of the mail domains. + from mailconfig import get_mail_domains if write_opendkim_tables(get_mail_domains(env), env): # Settings changed. Kick opendkim. shell('check_call', ["/usr/sbin/service", "opendkim", "restart"]) @@ -122,24 +126,44 @@ def build_zones(env): domains = get_dns_domains(env) zonefiles = get_dns_zones(env) - # Custom records to add to zones. - additional_records = list(get_custom_dns_config(env)) + # Create a dictionary of domains to a set of attributes for each + # domain, such as whether there are mail users at the domain. + from mailconfig import get_mail_domains from web_update import get_web_domains - www_redirect_domains = set(get_web_domains(env)) - set(get_web_domains(env, include_www_redirects=False)) + mail_domains = set(get_mail_domains(env)) + mail_user_domains = set(get_mail_domains(env, users_only=True)) # i.e. will log in for mail, Nextcloud + web_domains = set(get_web_domains(env)) + auto_domains = web_domains - set(get_web_domains(env, include_auto=False)) + domains |= auto_domains # www redirects not included in the initial list, see above + domains = { + domain: { + "user": domain in mail_user_domains, + "mail": domain in mail_domains, + "web": domain in web_domains, + "auto": domain in auto_domains, + } + for domain in domains + } # For MTA-STS, we'll need to check if the PRIMARY_HOSTNAME certificate is # singned and valid. Check that now rather than repeatedly for each domain. - env["-primary-hostname-certificate-is-valid"] = is_domain_cert_signed_and_valid(env["PRIMARY_HOSTNAME"], env) + domains[env["PRIMARY_HOSTNAME"]]["certificate-is-valid"] = is_domain_cert_signed_and_valid(env["PRIMARY_HOSTNAME"], env) + + # Load custom records to add to zones. + additional_records = list(get_custom_dns_config(env)) # Build DNS records for each zone. for domain, zonefile in zonefiles: # Build the records to put in the zone. - records = build_zone(domain, domains, additional_records, www_redirect_domains, env) + records = build_zone(domain, domains, additional_records, env) yield (domain, zonefile, records) -def build_zone(domain, all_domains, additional_records, www_redirect_domains, env, is_zone=True): +def build_zone(domain, domain_properties, additional_records, env, is_zone=True): records = [] + # Skip www redirect and autoconfiguration domains here because they are set at the zone level directly. + if domain_properties[domain]["auto"]: return records + # For top-level zones, define the authoritative name servers. # # Normally we are our own nameservers. Some TLDs require two distinct IP addresses, @@ -188,16 +212,17 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en # Add DNS records for any subdomains of this domain. We should not have a zone for # both a domain and one of its subdomains. - subdomains = [d for d in all_domains if d.endswith("." + domain)] - for subdomain in subdomains: - subdomain_qname = subdomain[0:-len("." + domain)] - subzone = build_zone(subdomain, [], additional_records, www_redirect_domains, env, is_zone=False) - for child_qname, child_rtype, child_value, child_explanation in subzone: - if child_qname == None: - child_qname = subdomain_qname - else: - child_qname += "." + subdomain_qname - records.append((child_qname, child_rtype, child_value, child_explanation)) + if is_zone: # don't recurse when we're just loading data for a subdomain + subdomains = [d for d in domain_properties if d.endswith("." + domain)] + for subdomain in subdomains: + subdomain_qname = subdomain[0:-len("." + domain)] + subzone = build_zone(subdomain, domain_properties, additional_records, env, is_zone=False) + for child_qname, child_rtype, child_value, child_explanation in subzone: + if child_qname == None: + child_qname = subdomain_qname + else: + child_qname += "." + subdomain_qname + records.append((child_qname, child_rtype, child_value, child_explanation)) has_rec_base = list(records) # clone current state def has_rec(qname, rtype, prefix=None): @@ -234,7 +259,7 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en (None, "A", env["PUBLIC_IP"], "Required. May have a different value. Sets the IP address that %s resolves to for web hosting and other services besides mail. The A record must be present but its value does not affect mail delivery." % domain), (None, "AAAA", env.get('PUBLIC_IPV6'), "Optional. Sets the IPv6 address that %s resolves to, e.g. for web hosting. (It is not necessary for receiving mail on this domain.)" % domain), ] - if "www." + domain in www_redirect_domains: + if domain_properties.get("www." + domain, {}).get("auto"): defaults += [ ("www", "A", env["PUBLIC_IP"], "Optional. Sets the IP address that www.%s resolves to so that the box can provide a redirect to the parent domain." % domain), ("www", "AAAA", env.get('PUBLIC_IPV6'), "Optional. Sets the IPv6 address that www.%s resolves to so that the box can provide a redirect to the parent domain." % domain), @@ -288,7 +313,7 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en # Add CardDAV/CalDAV SRV records on the non-primary hostname that points to the primary hostname # for autoconfiguration of mail clients (so only domains hosting user accounts need it). # The SRV record format is priority (0, whatever), weight (0, whatever), port, service provider hostname (w/ trailing dot). - if domain != env["PRIMARY_HOSTNAME"] and domain in get_mail_domains(env, users_only=True): + if domain != env["PRIMARY_HOSTNAME"] and domain_properties[domain]["user"]: for dav in ("card", "cal"): qname = "_" + dav + "davs._tcp" if not has_rec(qname, "SRV"): @@ -298,7 +323,7 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en # This allows the following clients to automatically configure email addresses in the respective applications. # autodiscover.* - Z-Push ActiveSync Autodiscover # autoconfig.* - Thunderbird Autoconfig - if domain in get_mail_domains(env, users_only=True): + if domain_properties[domain]["user"]: autodiscover_records = [ ("autodiscover", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."), ("autodiscover", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."), @@ -330,7 +355,9 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en ("mta-sts", "A", env["PUBLIC_IP"], "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."), ("mta-sts", "AAAA", env.get('PUBLIC_IPV6'), "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."), ] - if domain in get_mail_domains(env) and env["-primary-hostname-certificate-is-valid"] and is_domain_cert_signed_and_valid("mta-sts." + domain, env): + if domain_properties[domain]["mail"] \ + and domain_properties[env["PRIMARY_HOSTNAME"]]["certificate-is-valid"] \ + and is_domain_cert_signed_and_valid("mta-sts." + domain, env): # Compute an up-to-32-character hash of the policy file. We'll take a SHA-1 hash of the policy # file (20 bytes) and encode it as base-64 (28 bytes, using alphanumeric alternate characters # instead of '+' and '/' which are not allowed in an MTA-STS policy id) but then just take its diff --git a/management/web_update.py b/management/web_update.py index 5048cba..809106e 100644 --- a/management/web_update.py +++ b/management/web_update.py @@ -9,7 +9,7 @@ from dns_update import get_custom_dns_config, get_dns_zones from ssl_certificates import get_ssl_certificates, get_domain_ssl_files, check_certificate from utils import shell, safe_domain_name, sort_domains -def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True): +def get_web_domains(env, include_www_redirects=True, include_auto=True, exclude_dns_elsewhere=True): # What domains should we serve HTTP(S) for? domains = set() @@ -18,20 +18,21 @@ def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True) # if the user wants to make one. domains |= get_mail_domains(env) - if include_www_redirects: + if include_www_redirects and include_auto: # Add 'www.' subdomains that we want to provide default redirects # to the main domain for. We'll add 'www.' to any DNS zones, i.e. # the topmost of each domain we serve. domains |= set('www.' + zone for zone, zonefile in get_dns_zones(env)) - # Add Autoconfiguration domains for domains that there are user accounts at: - # 'autoconfig.' for Mozilla Thunderbird auto setup. - # 'autodiscover.' for Activesync autodiscovery. - domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env, users_only=True)) - domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env, users_only=True)) + if include_auto: + # Add Autoconfiguration domains for domains that there are user accounts at: + # 'autoconfig.' for Mozilla Thunderbird auto setup. + # 'autodiscover.' for Activesync autodiscovery. + domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env, users_only=True)) + domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env, users_only=True)) - # 'mta-sts.' for MTA-STS support for all domains that have email addresses. - domains |= set('mta-sts.' + maildomain for maildomain in get_mail_domains(env)) + # 'mta-sts.' for MTA-STS support for all domains that have email addresses. + domains |= set('mta-sts.' + maildomain for maildomain in get_mail_domains(env)) if exclude_dns_elsewhere: # ...Unless the domain has an A/AAAA record that maps it to a different From e283a1204728024c3e0cf77fdb5292fbdecde85f Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 9 May 2021 08:59:44 -0400 Subject: [PATCH 17/43] Add null SPF, DMARC, and MX records for automatically generated autoconfig, autodiscover, and mta-sts subdomains; add null MX records for custom A-record subdomains All A/AAAA-resolvable domains that don't send or receive mail should have these null records. This simplifies the handling of domains a bit by handling automatically generated subdomains more like other domains. --- CHANGELOG.md | 8 ++- management/dns_update.py | 149 ++++++++++++++++++--------------------- management/web_update.py | 2 +- 3 files changed, 78 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb4c0b6..df69bbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,15 @@ CHANGELOG In Development -------------- -* Migrate to the ECDSAP256SHA256 DNSSEC algorithm. If a DS record is set for any of your domain names that have DNS hosted on your box, you will be prompted by status checks to update the DS record. +Mail: + * Roundcube's login cookie is updated to use a new encryption algorithm (AES-256-CBC instead of DES-EDE-CBC). +DNS: + +* The ECDSAP256SHA256 DNSSEC algorithm is now available. If a DS record is set for any of your domain names that have DNS hosted on your box, you will be prompted by status checks to update the DS record at your convenience. +* Null MX records are added for domains that do not serve mail. + v0.53a (May 8, 2021) -------------------- diff --git a/management/dns_update.py b/management/dns_update.py index a9a686d..c000f34 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -135,6 +135,14 @@ def build_zones(env): web_domains = set(get_web_domains(env)) auto_domains = web_domains - set(get_web_domains(env, include_auto=False)) domains |= auto_domains # www redirects not included in the initial list, see above + + # Add ns1/ns2+PRIMARY_HOSTNAME which must also have A/AAAA records + # when the box is acting as authoritative DNS server for its domains. + for ns in ("ns1", "ns2"): + d = ns + "." + env["PRIMARY_HOSTNAME"] + domains.add(d) + auto_domains.add(d) + domains = { domain: { "user": domain in mail_user_domains, @@ -161,9 +169,6 @@ def build_zones(env): def build_zone(domain, domain_properties, additional_records, env, is_zone=True): records = [] - # Skip www redirect and autoconfiguration domains here because they are set at the zone level directly. - if domain_properties[domain]["auto"]: return records - # For top-level zones, define the authoritative name servers. # # Normally we are our own nameservers. Some TLDs require two distinct IP addresses, @@ -173,10 +178,10 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True) # 'False' in the tuple indicates these records would not be used if the zone # is managed outside of the box. if is_zone: - # Obligatory definition of ns1.PRIMARY_HOSTNAME. + # Obligatory NS record to ns1.PRIMARY_HOSTNAME. records.append((None, "NS", "ns1.%s." % env["PRIMARY_HOSTNAME"], False)) - # Define ns2.PRIMARY_HOSTNAME or whatever the user overrides. + # NS record to ns2.PRIMARY_HOSTNAME or whatever the user overrides. # User may provide one or more additional nameservers secondary_ns_list = get_secondary_dns(additional_records, mode="NS") \ or ["ns2." + env["PRIMARY_HOSTNAME"]] @@ -186,15 +191,6 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True) # In PRIMARY_HOSTNAME... if domain == env["PRIMARY_HOSTNAME"]: - # Define ns1 and ns2. - # 'False' in the tuple indicates these records would not be used if the zone - # is managed outside of the box. - records.append(("ns1", "A", env["PUBLIC_IP"], False)) - records.append(("ns2", "A", env["PUBLIC_IP"], False)) - if env.get('PUBLIC_IPV6'): - records.append(("ns1", "AAAA", env["PUBLIC_IPV6"], False)) - records.append(("ns2", "AAAA", env["PUBLIC_IPV6"], False)) - # Set the A/AAAA records. Do this early for the PRIMARY_HOSTNAME so that the user cannot override them # and we can provide different explanatory text. records.append((None, "A", env["PUBLIC_IP"], "Required. Sets the IP address of the box.")) @@ -249,21 +245,23 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True) continue records.append((qname, rtype, value, "(Set by user.)")) - # Add defaults if not overridden by the user's custom settings (and not otherwise configured). + # Add A/AAAA defaults if not overridden by the user's custom settings (and not otherwise configured). # Any CNAME or A record on the qname overrides A and AAAA. But when we set the default A record, # we should not cause the default AAAA record to be skipped because it thinks a custom A record # was set. So set has_rec_base to a clone of the current set of DNS settings, and don't update # during this process. has_rec_base = list(records) + a_expl = "Required. May have a different value. Sets the IP address that %s resolves to for web hosting and other services besides mail. The A record must be present but its value does not affect mail delivery." % domain + if domain_properties[domain]["auto"]: + if domain.startswith("ns1.") or domain.startswith("ns2."): a_expl = False # omit from 'External DNS' page since this only applies if box is its own DNS server + if domain.startswith("www."): a_expl = "Optional. Sets the IP address that %s resolves to so that the box can provide a redirect to the parent domain." % domain + if domain.startswith("mta-sts."): a_expl = "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt." + if domain.startswith("autoconfig."): a_expl = "Provides email configuration autodiscovery support for Thunderbird Autoconfig." + if domain.startswith("autodiscover."): a_expl = "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover." defaults = [ - (None, "A", env["PUBLIC_IP"], "Required. May have a different value. Sets the IP address that %s resolves to for web hosting and other services besides mail. The A record must be present but its value does not affect mail delivery." % domain), + (None, "A", env["PUBLIC_IP"], a_expl), (None, "AAAA", env.get('PUBLIC_IPV6'), "Optional. Sets the IPv6 address that %s resolves to, e.g. for web hosting. (It is not necessary for receiving mail on this domain.)" % domain), ] - if domain_properties.get("www." + domain, {}).get("auto"): - defaults += [ - ("www", "A", env["PUBLIC_IP"], "Optional. Sets the IP address that www.%s resolves to so that the box can provide a redirect to the parent domain." % domain), - ("www", "AAAA", env.get('PUBLIC_IPV6'), "Optional. Sets the IPv6 address that www.%s resolves to so that the box can provide a redirect to the parent domain." % domain), - ] for qname, rtype, value, explanation in defaults: if value is None or value.strip() == "": continue # skip IPV6 if not set if not is_zone and qname == "www": continue # don't create any default 'www' subdomains on what are themselves subdomains @@ -277,63 +275,40 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True) # Don't pin the list of records that has_rec checks against anymore. has_rec_base = records - # The MX record says where email for the domain should be delivered: Here! - if not has_rec(None, "MX", prefix="10 "): - records.append((None, "MX", "10 %s." % env["PRIMARY_HOSTNAME"], "Required. Specifies the hostname (and priority) of the machine that handles @%s mail." % domain)) + if domain_properties[domain]["mail"]: + # The MX record says where email for the domain should be delivered: Here! + if not has_rec(None, "MX", prefix="10 "): + records.append((None, "MX", "10 %s." % env["PRIMARY_HOSTNAME"], "Required. Specifies the hostname (and priority) of the machine that handles @%s mail." % domain)) - # SPF record: Permit the box ('mx', see above) to send mail on behalf of - # the domain, and no one else. - # Skip if the user has set a custom SPF record. - if not has_rec(None, "TXT", prefix="v=spf1 "): - records.append((None, "TXT", 'v=spf1 mx -all', "Recommended. Specifies that only the box is permitted to send @%s mail." % domain)) + # SPF record: Permit the box ('mx', see above) to send mail on behalf of + # the domain, and no one else. + # Skip if the user has set a custom SPF record. + if not has_rec(None, "TXT", prefix="v=spf1 "): + records.append((None, "TXT", 'v=spf1 mx -all', "Recommended. Specifies that only the box is permitted to send @%s mail." % domain)) - # Append the DKIM TXT record to the zone as generated by OpenDKIM. - # Skip if the user has set a DKIM record already. - opendkim_record_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/mail.txt') - with open(opendkim_record_file) as orf: - m = re.match(r'(\S+)\s+IN\s+TXT\s+\( ((?:"[^"]+"\s+)+)\)', orf.read(), re.S) - val = "".join(re.findall(r'"([^"]+)"', m.group(2))) - if not has_rec(m.group(1), "TXT", prefix="v=DKIM1; "): - records.append((m.group(1), "TXT", val, "Recommended. Provides a way for recipients to verify that this machine sent @%s mail." % domain)) + # Append the DKIM TXT record to the zone as generated by OpenDKIM. + # Skip if the user has set a DKIM record already. + opendkim_record_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/mail.txt') + with open(opendkim_record_file) as orf: + m = re.match(r'(\S+)\s+IN\s+TXT\s+\( ((?:"[^"]+"\s+)+)\)', orf.read(), re.S) + val = "".join(re.findall(r'"([^"]+)"', m.group(2))) + if not has_rec(m.group(1), "TXT", prefix="v=DKIM1; "): + records.append((m.group(1), "TXT", val, "Recommended. Provides a way for recipients to verify that this machine sent @%s mail." % domain)) - # Append a DMARC record. - # Skip if the user has set a DMARC record already. - if not has_rec("_dmarc", "TXT", prefix="v=DMARC1; "): - records.append(("_dmarc", "TXT", 'v=DMARC1; p=quarantine', "Recommended. Specifies that mail that does not originate from the box but claims to be from @%s or which does not have a valid DKIM signature is suspect and should be quarantined by the recipient's mail system." % domain)) + # Append a DMARC record. + # Skip if the user has set a DMARC record already. + if not has_rec("_dmarc", "TXT", prefix="v=DMARC1; "): + records.append(("_dmarc", "TXT", 'v=DMARC1; p=quarantine', "Recommended. Specifies that mail that does not originate from the box but claims to be from @%s or which does not have a valid DKIM signature is suspect and should be quarantined by the recipient's mail system." % domain)) - # For any subdomain with an A record but no SPF or DMARC record, add strict policy records. - all_resolvable_qnames = set(r[0] for r in records if r[1] in ("A", "AAAA")) - for qname in all_resolvable_qnames: - if not has_rec(qname, "TXT", prefix="v=spf1 "): - records.append((qname, "TXT", 'v=spf1 -all', "Recommended. Prevents use of this domain name for outbound mail by specifying that no servers are valid sources for mail from @%s. If you do send email from this domain name you should either override this record such that the SPF rule does allow the originating server, or, take the recommended approach and have the box handle mail for this domain (simply add any receiving alias at this domain name to make this machine treat the domain name as one of its mail domains)." % (qname + "." + domain))) - dmarc_qname = "_dmarc" + ("" if qname is None else "." + qname) - if not has_rec(dmarc_qname, "TXT", prefix="v=DMARC1; "): - records.append((dmarc_qname, "TXT", 'v=DMARC1; p=reject', "Recommended. Prevents use of this domain name for outbound mail by specifying that the SPF rule should be honoured for mail from @%s." % (qname + "." + domain))) - - # Add CardDAV/CalDAV SRV records on the non-primary hostname that points to the primary hostname - # for autoconfiguration of mail clients (so only domains hosting user accounts need it). - # The SRV record format is priority (0, whatever), weight (0, whatever), port, service provider hostname (w/ trailing dot). - if domain != env["PRIMARY_HOSTNAME"] and domain_properties[domain]["user"]: - for dav in ("card", "cal"): - qname = "_" + dav + "davs._tcp" - if not has_rec(qname, "SRV"): - records.append((qname, "SRV", "0 0 443 " + env["PRIMARY_HOSTNAME"] + ".", "Recommended. Specifies the hostname of the server that handles CardDAV/CalDAV services for email addresses on this domain.")) - - # Adds autoconfiguration A records for all domains that there are user accounts at. - # This allows the following clients to automatically configure email addresses in the respective applications. - # autodiscover.* - Z-Push ActiveSync Autodiscover - # autoconfig.* - Thunderbird Autoconfig if domain_properties[domain]["user"]: - autodiscover_records = [ - ("autodiscover", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."), - ("autodiscover", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."), - ("autoconfig", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig."), - ("autoconfig", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig.") - ] - for qname, rtype, value, explanation in autodiscover_records: - if value is None or value.strip() == "": continue # skip IPV6 if not set - if not has_rec(qname, rtype): - records.append((qname, rtype, value, explanation)) + # Add CardDAV/CalDAV SRV records on the non-primary hostname that points to the primary hostname + # for autoconfiguration of mail clients (so only domains hosting user accounts need it). + # The SRV record format is priority (0, whatever), weight (0, whatever), port, service provider hostname (w/ trailing dot). + if domain != env["PRIMARY_HOSTNAME"]: + for dav in ("card", "cal"): + qname = "_" + dav + "davs._tcp" + if not has_rec(qname, "SRV"): + records.append((qname, "SRV", "0 0 443 " + env["PRIMARY_HOSTNAME"] + ".", "Recommended. Specifies the hostname of the server that handles CardDAV/CalDAV services for email addresses on this domain.")) # If this is a domain name that there are email addresses configured for, i.e. "something@" # this domain name, then the domain name is a MTA-STS (https://tools.ietf.org/html/rfc8461) @@ -350,11 +325,9 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True) # subdomain must be valid certificate for that domain. Do not set an MTA-STS policy if either # certificate in use is not valid (e.g. because it is self-signed and a valid certificate has not # yet been provisioned). Since we cannot provision a certificate without A/AAAA records, we - # always set them --- only the TXT records depend on there being valid certificates. - mta_sts_records = [ - ("mta-sts", "A", env["PUBLIC_IP"], "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."), - ("mta-sts", "AAAA", env.get('PUBLIC_IPV6'), "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."), - ] + # always set them (by including them in the www domains) --- only the TXT records depend on there + # being valid certificates. + mta_sts_records = [ ] if domain_properties[domain]["mail"] \ and domain_properties[env["PRIMARY_HOSTNAME"]]["certificate-is-valid"] \ and is_domain_cert_signed_and_valid("mta-sts." + domain, env): @@ -374,10 +347,28 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True) if env.get("MTA_STS_TLSRPT_RUA") and not has_rec("_smtp._tls", "TXT", prefix="v=TLSRPTv1;"): mta_sts_records.append(("_smtp._tls", "TXT", "v=TLSRPTv1; rua=" + env["MTA_STS_TLSRPT_RUA"], "Optional. Enables MTA-STS reporting.")) for qname, rtype, value, explanation in mta_sts_records: - if value is None or value.strip() == "": continue # skip IPV6 if not set if not has_rec(qname, rtype): records.append((qname, rtype, value, explanation)) + # Add no-mail-here records for any qname that has an A or AAAA record + # but no MX record. This would include domain itself if domain is a + # non-mail domain and also may include qnames from custom DNS records. + # Do this once at the end of generating a zone. + if is_zone: + qnames_with_a = set(qname for (qname, rtype, value, explanation) in records if rtype in ("A", "AAAA")) + qnames_with_mx = set(qname for (qname, rtype, value, explanation) in records if rtype == "MX") + for qname in qnames_with_a - qnames_with_mx: + # Mark this domain as not sending mail with hard-fail SPF and DMARC records. + d = (qname+"." if qname else "") + domain + if not has_rec(qname, "TXT", prefix="v=spf1 "): + records.append((qname, "TXT", 'v=spf1 -all', "Recommended. Prevents use of this domain name for outbound mail by specifying that no servers are valid sources for mail from @%s. If you do send email from this domain name you should either override this record such that the SPF rule does allow the originating server, or, take the recommended approach and have the box handle mail for this domain (simply add any receiving alias at this domain name to make this machine treat the domain name as one of its mail domains)." % d)) + if not has_rec("_dmarc" + ("."+qname if qname else ""), "TXT", prefix="v=DMARC1; "): + records.append(("_dmarc" + ("."+qname if qname else ""), "TXT", 'v=DMARC1; p=reject', "Recommended. Prevents use of this domain name for outbound mail by specifying that the SPF rule should be honoured for mail from @%s." % d)) + + # And with a null MX record (https://explained-from-first-principles.com/email/#null-mx-record) + if not has_rec(qname, "MX"): + records.append((qname, "MX", '0 .', "Recommended. Prevents use of this domain name for incoming mail.")) + # Sort the records. The None records *must* go first in the nsd zone file. Otherwise it doesn't matter. records.sort(key = lambda rec : list(reversed(rec[0].split(".")) if rec[0] is not None else "")) diff --git a/management/web_update.py b/management/web_update.py index 809106e..7230182 100644 --- a/management/web_update.py +++ b/management/web_update.py @@ -27,7 +27,7 @@ def get_web_domains(env, include_www_redirects=True, include_auto=True, exclude_ if include_auto: # Add Autoconfiguration domains for domains that there are user accounts at: # 'autoconfig.' for Mozilla Thunderbird auto setup. - # 'autodiscover.' for Activesync autodiscovery. + # 'autodiscover.' for ActiveSync autodiscovery (Z-Push). domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env, users_only=True)) domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env, users_only=True)) From d510c8ae2a5b55ef1b22cc57c8ff8a2fe8597546 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 9 May 2021 10:11:40 -0400 Subject: [PATCH 18/43] Enable and recommend port 465 for mail submission instead of port 587 (fixes #1849) Port 465 with "implicit" (i.e. always-on) TLS is a more secure approach than port 587 with explicit (i.e. optional and only on with STARTTLS). Although we reject credentials on port 587 without STARTTLS, by that point credentials have already been sent. --- CHANGELOG.md | 1 + conf/fail2ban/jails.conf | 8 +++++++ conf/ios-profile.xml | 2 +- conf/mozilla-autoconfig.xml | 4 ++-- conf/zpush/backend_imap.php | 2 +- management/status_checks.py | 1 + management/templates/mail-guide.html | 4 ++-- security.md | 4 ++-- setup/mail-postfix.sh | 33 ++++++++++++++++++---------- setup/ssl.sh | 4 ++-- tests/test_mail.py | 3 +-- 11 files changed, 42 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df69bbe..1587097 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ In Development Mail: +* Sending mail is now possible on port 465 with the "SSL" or "TLS" option in mail clients, and this is now the recommended setting. Port 587 with STARTTLS remains available but should be avoided. * Roundcube's login cookie is updated to use a new encryption algorithm (AES-256-CBC instead of DES-EDE-CBC). DNS: diff --git a/conf/fail2ban/jails.conf b/conf/fail2ban/jails.conf index 5de4fd4..ce957f4 100644 --- a/conf/fail2ban/jails.conf +++ b/conf/fail2ban/jails.conf @@ -38,6 +38,14 @@ logpath = STORAGE_ROOT/owncloud/nextcloud.log maxretry = 20 findtime = 120 +[miab-postfix465] +enabled = true +port = 465 +filter = miab-postfix-submission +logpath = /var/log/mail.log +maxretry = 20 +findtime = 30 + [miab-postfix587] enabled = true port = 587 diff --git a/conf/ios-profile.xml b/conf/ios-profile.xml index f2011a4..273c0bf 100644 --- a/conf/ios-profile.xml +++ b/conf/ios-profile.xml @@ -53,7 +53,7 @@ OutgoingMailServerHostName PRIMARY_HOSTNAME OutgoingMailServerPortNumber - 587 + 465 OutgoingMailServerUseSSL OutgoingPasswordSameAsIncomingPassword diff --git a/conf/mozilla-autoconfig.xml b/conf/mozilla-autoconfig.xml index 2283462..df9cce6 100644 --- a/conf/mozilla-autoconfig.xml +++ b/conf/mozilla-autoconfig.xml @@ -16,8 +16,8 @@ PRIMARY_HOSTNAME - 587 - STARTTLS + 465 + SSL %EMAILADDRESS% password-cleartext true diff --git a/conf/zpush/backend_imap.php b/conf/zpush/backend_imap.php index a0c1233..da80c89 100644 --- a/conf/zpush/backend_imap.php +++ b/conf/zpush/backend_imap.php @@ -49,7 +49,7 @@ define('IMAP_FROM_LDAP_FULLNAME', '#givenname #sn'); define('IMAP_SMTP_METHOD', 'sendmail'); global $imap_smtp_params; -$imap_smtp_params = array('host' => 'ssl://127.0.0.1', 'port' => 587, 'auth' => true, 'username' => 'imap_username', 'password' => 'imap_password'); +$imap_smtp_params = array('host' => 'ssl://127.0.0.1', 'port' => 465, 'auth' => true, 'username' => 'imap_username', 'password' => 'imap_password'); define('MAIL_MIMEPART_CRLF', "\r\n"); define('IMAP_MEETING_USE_CALDAV', true); diff --git a/management/status_checks.py b/management/status_checks.py index 67b2697..7e766d0 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -34,6 +34,7 @@ def get_services(): { "name": "SSH Login (ssh)", "port": get_ssh_port(), "public": True, }, { "name": "Public DNS (nsd4)", "port": 53, "public": True, }, { "name": "Incoming Mail (SMTP/postfix)", "port": 25, "public": True, }, + { "name": "Outgoing Mail (SMTP 465/postfix)", "port": 465, "public": True, }, { "name": "Outgoing Mail (SMTP 587/postfix)", "port": 587, "public": True, }, #{ "name": "Postfix/master", "port": 10587, "public": True, }, { "name": "IMAPS (dovecot)", "port": 993, "public": True, }, diff --git a/management/templates/mail-guide.html b/management/templates/mail-guide.html index 0b43993..e3db9f0 100644 --- a/management/templates/mail-guide.html +++ b/management/templates/mail-guide.html @@ -30,8 +30,8 @@ Mail server {{hostname}} IMAP Port 993 IMAP Security SSL or TLS - SMTP Port 587 - SMTP Security STARTTLS (“always” or “required”, if prompted) + SMTP Port 465 + SMTP Security SSL or TLS Username: Your whole email address. Password: Your mail password. diff --git a/security.md b/security.md index ba3e384..8c39437 100644 --- a/security.md +++ b/security.md @@ -32,7 +32,7 @@ The box's administrator and its (non-administrative) mail users must sometimes c These services are protected by [TLS](https://en.wikipedia.org/wiki/Transport_Layer_Security): -* SMTP Submission (port 587). Mail users submit outbound mail through SMTP with STARTTLS on port 587. +* SMTP Submission (ports 465/587). Mail users submit outbound mail through SMTP with TLS (port 465) or STARTTLS (port 587). * IMAP/POP (ports 993, 995). Mail users check for incoming mail through IMAP or POP over TLS. * HTTPS (port 443). Webmail, the Exchange/ActiveSync protocol, the administrative control panel, and any static hosted websites are accessed over HTTPS. @@ -44,7 +44,7 @@ The services all follow these rules: Additionally: -* SMTP Submission (port 587) will not accept user credentials without STARTTLS (true also of SMTP on port 25 in case of client misconfiguration), and the submission port won't accept mail without encryption. The minimum cipher key length is 128 bits. (The box is of course configured not to be an open relay. User credentials are required to send outbound mail.) ([source](setup/mail-postfix.sh)) +* SMTP Submission on port 587 will not accept user credentials without STARTTLS (true also of SMTP on port 25 in case of client misconfiguration), and the submission port won't accept mail without encryption. The minimum cipher key length is 128 bits. (The box is of course configured not to be an open relay. User credentials are required to send outbound mail.) ([source](setup/mail-postfix.sh)) * HTTPS (port 443): The HTTPS Strict Transport Security header is set. A redirect from HTTP to HTTPS is offered. The [Qualys SSL Labs test](https://www.ssllabs.com/ssltest) should report an A+ grade. ([source 1](conf/nginx-ssl.conf), [source 2](conf/nginx.conf)) ### Password Storage diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index b16fd94..dc1fff8 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -17,7 +17,7 @@ # LMTP. Spamassassin then passes mail over to Dovecot for # storage in the user's mailbox. # -# Postfix also listens on port 587 (SMTP+STARTLS) for +# Postfix also listens on ports 465/587 (SMTPS, SMTP+STARTLS) for # connections from users who can authenticate and then sends # their email out to the outside world. Postfix queries Dovecot # to authenticate users. @@ -71,7 +71,7 @@ tools/editconf.py /etc/postfix/main.cf \ # ### Outgoing Mail -# Enable the 'submission' port 587 smtpd server and tweak its settings. +# Enable the 'submission' ports 465 and 587 and tweak their settings. # # * Enable authentication. It's disabled globally so that it is disabled on port 25, # so we need to explicitly enable it here. @@ -80,13 +80,19 @@ tools/editconf.py /etc/postfix/main.cf \ # OpenDKIM milter only. See dkim.sh. # * Even though we dont allow auth over non-TLS connections (smtpd_tls_auth_only below, and without auth the client cant # send outbound mail), don't allow non-TLS mail submission on this port anyway to prevent accidental misconfiguration. -# Setting smtpd_tls_security_level=encrypt also triggers the use of the 'mandatory' settings below. +# Setting smtpd_tls_security_level=encrypt also triggers the use of the 'mandatory' settings below (but this is ignored with smtpd_tls_wrappermode=yes.) # * Give it a different name in syslog to distinguish it from the port 25 smtpd server. # * Add a new cleanup service specific to the submission service ('authclean') # that filters out privacy-sensitive headers on mail being sent out by # authenticated users. By default Postfix also applies this to attached # emails but we turn this off by setting nested_header_checks empty. tools/editconf.py /etc/postfix/master.cf -s -w \ + "smtps=inet n - - - - smtpd + -o smtpd_tls_wrappermode=yes + -o smtpd_sasl_auth_enable=yes + -o syslog_name=postfix/submission + -o smtpd_milters=inet:127.0.0.1:8891 + -o cleanup_service_name=authclean" \ "submission=inet n - - - - smtpd -o smtpd_sasl_auth_enable=yes -o syslog_name=postfix/submission @@ -100,14 +106,14 @@ tools/editconf.py /etc/postfix/master.cf -s -w \ # Install the `outgoing_mail_header_filters` file required by the new 'authclean' service. cp conf/postfix_outgoing_mail_header_filters /etc/postfix/outgoing_mail_header_filters -# Modify the `outgoing_mail_header_filters` file to use the local machine name and ip +# Modify the `outgoing_mail_header_filters` file to use the local machine name and ip # on the first received header line. This may help reduce the spam score of email by # removing the 127.0.0.1 reference. sed -i "s/PRIMARY_HOSTNAME/$PRIMARY_HOSTNAME/" /etc/postfix/outgoing_mail_header_filters sed -i "s/PUBLIC_IP/$PUBLIC_IP/" /etc/postfix/outgoing_mail_header_filters # Enable TLS on incoming connections. It is not required on port 25, allowing for opportunistic -# encryption. On port 587 it is mandatory (see above). Shared and non-shared settings are +# encryption. On ports 465 and 587 it is mandatory (see above). Shared and non-shared settings are # given here. Shared settings include: # * Require TLS before a user is allowed to authenticate. # * Set the path to the server TLS certificate and 2048-bit DH parameters for old DH ciphers. @@ -117,9 +123,6 @@ sed -i "s/PUBLIC_IP/$PUBLIC_IP/" /etc/postfix/outgoing_mail_header_filters # won't fall back to cleartext. So we don't disable too much. smtpd_tls_exclude_ciphers applies to # both port 25 and port 587, but because we override the cipher list for both, it probably isn't used. # Use Mozilla's "Old" recommendations at https://ssl-config.mozilla.org/#server=postfix&server-version=3.3.0&config=old&openssl-version=1.1.1 -# For port 587 (via the 'mandatory' settings): -# * Use Mozilla's "Intermediate" TLS recommendations from https://ssl-config.mozilla.org/#server=postfix&server-version=3.3.0&config=intermediate&openssl-version=1.1.1 -# using and overriding the "high" cipher list so we don't conflict with the more permissive settings for port 25. tools/editconf.py /etc/postfix/main.cf \ smtpd_tls_security_level=may\ smtpd_tls_auth_only=yes \ @@ -130,18 +133,23 @@ tools/editconf.py /etc/postfix/main.cf \ smtpd_tls_ciphers=medium \ tls_medium_cipherlist=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA \ smtpd_tls_exclude_ciphers=aNULL,RC4 \ + tls_preempt_cipherlist=no \ + smtpd_tls_received_header=yes + +# For ports 465/587 (via the 'mandatory' settings): +# * Use Mozilla's "Intermediate" TLS recommendations from https://ssl-config.mozilla.org/#server=postfix&server-version=3.3.0&config=intermediate&openssl-version=1.1.1 +# using and overriding the "high" cipher list so we don't conflict with the more permissive settings for port 25. +tools/editconf.py /etc/postfix/main.cf \ smtpd_tls_mandatory_protocols="!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" \ smtpd_tls_mandatory_ciphers=high \ tls_high_cipherlist=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 \ - smtpd_tls_mandatory_exclude_ciphers=aNULL,DES,3DES,MD5,DES+MD5,RC4 \ - tls_preempt_cipherlist=no \ - smtpd_tls_received_header=yes + smtpd_tls_mandatory_exclude_ciphers=aNULL,DES,3DES,MD5,DES+MD5,RC4 # Prevent non-authenticated users from sending mail that requires being # relayed elsewhere. We don't want to be an "open relay". On outbound # mail, require one of: # -# * `permit_sasl_authenticated`: Authenticated users (i.e. on port 587). +# * `permit_sasl_authenticated`: Authenticated users (i.e. on port 465/587). # * `permit_mynetworks`: Mail that originates locally. # * `reject_unauth_destination`: No one else. (Permits mail whose destination is local and rejects other mail.) tools/editconf.py /etc/postfix/main.cf \ @@ -263,6 +271,7 @@ tools/editconf.py /etc/postfix/main.cf \ # Allow the two SMTP ports in the firewall. ufw_allow smtp +ufw_allow smtps ufw_allow submission # Restart services diff --git a/setup/ssl.sh b/setup/ssl.sh index 61b0b9e..9bd5d53 100755 --- a/setup/ssl.sh +++ b/setup/ssl.sh @@ -10,7 +10,7 @@ # # * DNSSEC DANE TLSA records # * IMAP -# * SMTP (opportunistic TLS for port 25 and submission on port 587) +# * SMTP (opportunistic TLS for port 25 and submission on ports 465/587) # * HTTPS # # The certificate is created with its CN set to the PRIMARY_HOSTNAME. It is @@ -19,7 +19,7 @@ # # The Diffie-Hellman cipher bits are used for SMTP and HTTPS, when a # Diffie-Hellman cipher is selected during TLS negotiation. Diffie-Hellman -# provides Perfect Forward Secrecy. +# provides Perfect Forward Secrecy. source setup/functions.sh # load our functions source /etc/mailinabox.conf # load global vars diff --git a/tests/test_mail.py b/tests/test_mail.py index 686d07a..8c8838a 100755 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -41,9 +41,8 @@ This is a test message. It should be automatically deleted by the test script."" ) # Connect to the server on the SMTP submission TLS port. -server = smtplib.SMTP(host, 587) +server = smtplib.SMTP_SSL(host) #server.set_debuglevel(1) -server.starttls() # Verify that the EHLO name matches the server's reverse DNS. ipaddr = socket.gethostbyname(host) # IPv4 only! From 35fa3fe891574ce43705cec39c5b5e48cd4172ea Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 15 May 2021 16:50:19 -0400 Subject: [PATCH 19/43] Changelog entries --- CHANGELOG.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1587097..620532a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ In Development Mail: -* Sending mail is now possible on port 465 with the "SSL" or "TLS" option in mail clients, and this is now the recommended setting. Port 587 with STARTTLS remains available but should be avoided. +* Forwarded mail using mail filter rules (in Roundcube; "sieve" rules) stopped re-writing the envelope address at some point, causing forwarded mail to often be marked as spam by the final recipient. These forwards will now re-write the envelope as the Mail-in-a-Box user receiving the mail to comply with SPF/DMARC rules. +* Sending mail is now possible on port 465 with the "SSL" or "TLS" option in mail clients, and this is now the recommended setting. Port 587 with STARTTLS remains available but should be avoided when configuring new mail clients. * Roundcube's login cookie is updated to use a new encryption algorithm (AES-256-CBC instead of DES-EDE-CBC). DNS: @@ -14,6 +15,19 @@ DNS: * The ECDSAP256SHA256 DNSSEC algorithm is now available. If a DS record is set for any of your domain names that have DNS hosted on your box, you will be prompted by status checks to update the DS record at your convenience. * Null MX records are added for domains that do not serve mail. +Contacts/calendar: + +* Updated Nextcloud to 20.0.8, contacts to 3.5.1, calendar to 2.2.0 (#1960). + +Control panel: + +* Fixed a crash in the status checks. +* Small wording improvements. + +Setup: + +* Minor improvements to the setup scripts. + v0.53a (May 8, 2021) -------------------- From 4cb46ea4658b91240c5676c52746e48aaaba7b3f Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 20 Jun 2021 15:50:04 -0400 Subject: [PATCH 20/43] v0.54 --- CHANGELOG.md | 4 ++-- README.md | 2 +- setup/bootstrap.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 620532a..9338b05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ CHANGELOG ========= -In Development --------------- +v0.54 (June 20, 2021) +--------------------- Mail: diff --git a/README.md b/README.md index e08312f..92fab00 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Clone this repository and checkout the tag corresponding to the most recent rele $ git clone https://github.com/mail-in-a-box/mailinabox $ cd mailinabox - $ git checkout v0.53a + $ git checkout v0.54 Begin the installation. diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index d2117df..9052105 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' )" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.53a + TAG=v0.54 elif [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' )" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From 21ad26e452efebf5cfcac951ff9c723f9da6966a Mon Sep 17 00:00:00 2001 From: NewbieOrange Date: Thu, 29 Jul 2021 04:39:40 +0800 Subject: [PATCH 21/43] Disable auto-complete for 2FA code in the control panel login form (#2013) --- management/templates/login.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/templates/login.html b/management/templates/login.html index 4c432aa..19b23d3 100644 --- a/management/templates/login.html +++ b/management/templates/login.html @@ -64,7 +64,7 @@ sudo management/cli.py user make-admin me@{{hostname}}
- +
Enter the six-digit code generated by your two factor authentication app.
From daad122236f7eca841e9a46fa19dd57f44bcb5ac Mon Sep 17 00:00:00 2001 From: lamkin <88649589+lamkin@users.noreply.github.com> Date: Mon, 16 Aug 2021 16:46:32 +0100 Subject: [PATCH 22/43] Ignore bad encoding in email addresses when parsing maillog files (#2017) local/domain parts of email address should be standard ASCII or UTF-8. Some email addresses contain extended ASCII, leading to decode failure by the UTF-8 codec (and thus failure of the Usage-Report script) This change allows maillog parsing to continue over lines containing such addresses --- management/mail_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/mail_log.py b/management/mail_log.py index 1626f82..59c32c6 100755 --- a/management/mail_log.py +++ b/management/mail_log.py @@ -586,7 +586,7 @@ def scan_postfix_submission_line(date, log, collector): def readline(filename): """ A generator that returns the lines of a file """ - with open(filename) as file: + with open(filename, errors='replace') as file: while True: line = file.readline() if not line: From 0ba841c7b66979e00e0d790b53555f05f72e9063 Mon Sep 17 00:00:00 2001 From: NewbieOrange Date: Mon, 23 Aug 2021 02:13:58 +0800 Subject: [PATCH 23/43] fail2ban now supports ipv6 (#2015) Since fail2ban 0.10.0, ipv6 support has been added. The current Ubuntu 18.04 repository has fail2ban 0.10.2, which does have ipv6 protection. --- security.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/security.md b/security.md index 8c39437..5de8c61 100644 --- a/security.md +++ b/security.md @@ -69,8 +69,6 @@ The following services are protected: SSH, IMAP (dovecot), SMTP submission (post Some other services running on the box may be missing fail2ban filters. -`fail2ban` only blocks IPv4 addresses, however. If the box has a public IPv6 address, it is not protected from these attacks. - Outbound Mail ------------- From 20ccda8710411552d69978aeff41a9281fbb2012 Mon Sep 17 00:00:00 2001 From: myfirstnameispaul Date: Mon, 28 Jun 2021 05:51:05 -0700 Subject: [PATCH 24/43] Re-order DS record algorithms by digest type and revise warning message. Note that 7, 4 is printed last in the status checks page but does not appear in the file, and I couldn't figure out why. --- management/status_checks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/management/status_checks.py b/management/status_checks.py index 7e766d0..e509ff4 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -619,7 +619,9 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False): output.print_ok("DNSSEC 'DS' record is set correctly at registrar. (Records using algorithm other than ECDSAP256SHA256 should be removed.)") return else: # no record uses alg 13 - output.print_warning("DNSSEC 'DS' record set at registrar is valid but should be updated to ECDSAP256SHA256 (see below).") + output.print_warning("""DNSSEC 'DS' record set at registrar is valid but should be updated to ECDSAP256SHA256 (see below). + IMPORTANT: Do not delete existing DNSSEC 'DS' records for this domain until confirmation that the new DNSSEC 'DS' record + for this domain is valid.""") else: if is_checking_primary: output.print_error("""The DNSSEC 'DS' record for %s is incorrect. See further details below.""" % domain) @@ -630,7 +632,8 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False): output.print_line("""Follow the instructions provided by your domain name registrar to set a DS record. Registrars support different sorts of DS records. Use the first option that works:""") - preferred_ds_order = [(7, 1), (7, 2), (8, 4), (13, 4), (8, 1), (8, 2), (13, 1), (13, 2)] # low to high + preferred_ds_order = [(7, 1), (8, 1), (13, 1), (7, 2), (8, 4), (13, 4), (8, 2), (13, 2)] # low to high + def preferred_ds_order_func(ds_suggestion): k = (int(ds_suggestion['alg']), int(ds_suggestion['digalg'])) if k in preferred_ds_order: From 67b5711c683df8f05acd48ce13c4afcb2c5d3008 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 22 Aug 2021 14:43:33 -0400 Subject: [PATCH 25/43] Recommend that DS records be updated to not use SHA1 and exclude MUST NOT methods (SHA1) and the unlikely option RSASHA1-NSEC3-SHA1 (7) + SHA-384 (4) from the DS record suggestions --- management/status_checks.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/management/status_checks.py b/management/status_checks.py index e509ff4..e2d4b1a 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -612,14 +612,14 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False): # # But it may not be preferred. Only algorithm 13 is preferred. Warn if any of the # matched zones uses a different algorithm. - if set(r[1] for r in matched_ds) == { '13' }: # all are alg 13 + if set(r[1] for r in matched_ds) == { '13' } and set(r[2] for r in matched_ds) <= { '2', '4' }: # all are alg 13 and digest type 2 or 4 output.print_ok("DNSSEC 'DS' record is set correctly at registrar.") return - elif '13' in set(r[1] for r in matched_ds): # some but not all are alg 13 - output.print_ok("DNSSEC 'DS' record is set correctly at registrar. (Records using algorithm other than ECDSAP256SHA256 should be removed.)") + elif len([r for r in matched_ds if r[1] == '13' and r[2] in ( '2', '4' )]) > 0: # some but not all are alg 13 + output.print_ok("DNSSEC 'DS' record is set correctly at registrar. (Records using algorithm other than ECDSAP256SHA256 and digest types other than SHA-256/384 should be removed.)") return else: # no record uses alg 13 - output.print_warning("""DNSSEC 'DS' record set at registrar is valid but should be updated to ECDSAP256SHA256 (see below). + output.print_warning("""DNSSEC 'DS' record set at registrar is valid but should be updated to ECDSAP256SHA256 and SHA-256 (see below). IMPORTANT: Do not delete existing DNSSEC 'DS' records for this domain until confirmation that the new DNSSEC 'DS' record for this domain is valid.""") else: @@ -632,7 +632,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False): output.print_line("""Follow the instructions provided by your domain name registrar to set a DS record. Registrars support different sorts of DS records. Use the first option that works:""") - preferred_ds_order = [(7, 1), (8, 1), (13, 1), (7, 2), (8, 4), (13, 4), (8, 2), (13, 2)] # low to high + preferred_ds_order = [(7, 2), (8, 4), (13, 4), (8, 2), (13, 2)] # low to high, see https://github.com/mail-in-a-box/mailinabox/issues/1998 def preferred_ds_order_func(ds_suggestion): k = (int(ds_suggestion['alg']), int(ds_suggestion['digalg'])) @@ -641,6 +641,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False): return -1 # index before first item output.print_line("") for i, ds_suggestion in enumerate(sorted(expected_ds_records.values(), key=preferred_ds_order_func, reverse=True)): + if preferred_ds_order_func(ds_suggestion) == -1: continue # don't offer record types that the RFC says we must not offer output.print_line("") output.print_line("Option " + str(i+1) + ":") output.print_line("----------") From ba80d9e72dd984af0ee733d55291a00e7f8685e6 Mon Sep 17 00:00:00 2001 From: David Duque Date: Mon, 23 Aug 2021 11:25:41 +0100 Subject: [PATCH 26/43] Show backup retention period form when configuring B2 backups (#2024) --- management/templates/system-backup.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index a63b38e..6cbcc4f 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -138,7 +138,7 @@ -
+
From 700188c44392aaa3a1e5cd5feaa59767db38cb53 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 22 Aug 2021 14:29:33 -0400 Subject: [PATCH 27/43] Roundcube 1.5 RC --- setup/webmail.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/webmail.sh b/setup/webmail.sh index 55fea63..d365297 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -29,8 +29,8 @@ apt_install \ # Combine the Roundcube version number with the commit hash of plugins to track # whether we have the latest version of everything. -VERSION=1.4.11 -HASH=3877f0e70f29e7d0612155632e48c3db1e626be3 +VERSION=1.5-rc +HASH=a7cb2a39702536d769c7ff93f716e27f0b93f9d9 PERSISTENT_LOGIN_VERSION=6b3fc450cae23ccb2f393d0ef67aa319e877e435 # version 5.2.0 HTML5_NOTIFIER_VERSION=68d9ca194212e15b3c7225eb6085dbcf02fd13d7 # version 0.6.4+ CARDDAV_VERSION=3.0.3 @@ -132,6 +132,7 @@ cat > $RCM_CONFIG < From 53ec0f39cb074dc43a2f8b245aa8d4d12c74914e Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 22 Aug 2021 15:02:38 -0400 Subject: [PATCH 28/43] Use 'secrets' to generate the system API key and remove some debugging-related code * Rename the 'master' API key to be called the 'system' API key * Generate the key using the Python secrets module which is meant for this * Remove some debugging helper code which will be obsoleted by the upcoming changes for session keys --- management/auth.py | 29 ++++++++++++----------------- management/daemon.py | 27 +++++---------------------- 2 files changed, 17 insertions(+), 39 deletions(-) diff --git a/management/auth.py b/management/auth.py index fd143c7..de2b61b 100644 --- a/management/auth.py +++ b/management/auth.py @@ -1,6 +1,5 @@ -import base64, os, os.path, hmac, json +import base64, os, os.path, hmac, json, secrets -from flask import make_response import utils from mailconfig import get_mail_password, get_mail_user_privileges @@ -9,7 +8,7 @@ from mfa import get_hash_mfa_state, validate_auth_mfa DEFAULT_KEY_PATH = '/var/lib/mailinabox/api.key' DEFAULT_AUTH_REALM = 'Mail-in-a-Box Management Server' -class KeyAuthService: +class AuthService: """Generate an API key for authenticating clients Clients must read the key from the key file and send the key with all HTTP @@ -18,16 +17,12 @@ class KeyAuthService: """ def __init__(self): self.auth_realm = DEFAULT_AUTH_REALM - self.key = self._generate_key() self.key_path = DEFAULT_KEY_PATH + self.init_system_api_key() - def write_key(self): - """Write key to file so authorized clients can get the key + def init_system_api_key(self): + """Write an API key to a local file so local processes can use the API""" - The key file is created with mode 0640 so that additional users can be - authorized to access the API by granting group/ACL read permissions on - the key file. - """ def create_file_with_mode(path, mode): # Based on answer by A-B-B: http://stackoverflow.com/a/15015748 old_umask = os.umask(0) @@ -36,6 +31,8 @@ class KeyAuthService: finally: os.umask(old_umask) + self.key = secrets.token_hex(24) + os.makedirs(os.path.dirname(self.key_path), exist_ok=True) with create_file_with_mode(self.key_path, 0o640) as key_file: @@ -72,8 +69,9 @@ class KeyAuthService: if username in (None, ""): raise ValueError("Authorization header invalid.") - elif username == self.key: - # The user passed the master API key which grants administrative privs. + + if username == self.key: + # The user passed the system API key which grants administrative privs. return (None, ["admin"]) else: # The user is trying to log in with a username and either a password @@ -136,8 +134,8 @@ class KeyAuthService: # email address, current hashed password, and current MFA state, so that the # key becomes invalid if any of that information changes. # - # Use an HMAC to generate the API key using our master API key as a key, - # which also means that the API key becomes invalid when our master API key + # Use an HMAC to generate the API key using our system API key as a key, + # which also means that the API key becomes invalid when our system API key # changes --- i.e. when this process is restarted. # # Raises ValueError via get_mail_password if the user doesn't exist. @@ -153,6 +151,3 @@ class KeyAuthService: hash_key = self.key.encode('ascii') return hmac.new(hash_key, msg, digestmod="sha256").hexdigest() - def _generate_key(self): - raw_key = os.urandom(32) - return base64.b64encode(raw_key).decode('ascii') diff --git a/management/daemon.py b/management/daemon.py index 8490ee4..bb723ea 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -1,5 +1,8 @@ #!/usr/local/lib/mailinabox/env/bin/python3 # +# The API can be accessed on the command line, e.g. use `curl` like so: +# curl --user $( Date: Sun, 22 Aug 2021 16:07:16 -0400 Subject: [PATCH 29/43] Replace HMAC-based session API keys with tokens stored in memory in the daemon process Since the session cache clears keys after a period of time, this fixes #1821. Based on https://github.com/mail-in-a-box/mailinabox/pull/2012, and so: Co-Authored-By: NewbieOrange Also fixes #2029 by not revealing through the login failure error message whether a user exists or not. --- api/mailinabox.yml | 34 +++++-- management/auth.py | 173 +++++++++++++++++--------------- management/daemon.py | 29 ++++-- management/templates/index.html | 6 ++ management/templates/login.html | 4 +- setup/management.sh | 4 +- tests/fail2ban.py | 2 +- 7 files changed, 149 insertions(+), 103 deletions(-) diff --git a/api/mailinabox.yml b/api/mailinabox.yml index 14cf54d..bd4b203 100644 --- a/api/mailinabox.yml +++ b/api/mailinabox.yml @@ -54,24 +54,24 @@ tags: System operations, which include system status checks, new version checks and reboot status. paths: - /me: - get: + /login: + post: tags: - User - summary: Get user information + summary: Exchange a username and password for a session API key. description: | - Returns user information. Used for user authentication. + Returns user information and a session API key. Authenticate a user by supplying the auth token as a base64 encoded string in format `email:password` using basic authentication headers. If successful, a long-lived `api_key` is returned which can be used for subsequent - requests to the API. - operationId: getMe + requests to the API in place of the password. + operationId: login x-codeSamples: - lang: curl source: | - curl -X GET "https://{host}/admin/me" \ + curl -X GET "https://{host}/admin/login" \ -u ":" responses: 200: @@ -92,6 +92,24 @@ paths: privileges: - admin status: ok + /logout: + post: + tags: + - User + summary: Invalidates a session API key. + description: | + Invalidates a session API key so that it cannot be used after this API call. + operationId: logout + x-codeSamples: + - lang: curl + source: | + curl -X GET "https://{host}/admin/logout" \ + -u ":" + responses: + 200: + description: Successful operation + content: + application/json: /system/status: post: tags: @@ -1803,7 +1821,7 @@ components: The `access-token` is comprised of the Base64 encoding of `username:password`. The `username` is the mail user's email address, and `password` can either be the mail user's - password, or the `api_key` returned from the `getMe` operation. + password, or the `api_key` returned from the `login` operation. When using `curl`, you can supply user credentials using the `-u` or `--user` parameter. requestBodies: diff --git a/management/auth.py b/management/auth.py index de2b61b..38c15e9 100644 --- a/management/auth.py +++ b/management/auth.py @@ -1,5 +1,7 @@ import base64, os, os.path, hmac, json, secrets +from datetime import timedelta +from expiringdict import ExpiringDict import utils from mailconfig import get_mail_password, get_mail_user_privileges @@ -9,16 +11,13 @@ DEFAULT_KEY_PATH = '/var/lib/mailinabox/api.key' DEFAULT_AUTH_REALM = 'Mail-in-a-Box Management Server' class AuthService: - """Generate an API key for authenticating clients - - Clients must read the key from the key file and send the key with all HTTP - requests. The key is passed as the username field in the standard HTTP - Basic Auth header. - """ def __init__(self): self.auth_realm = DEFAULT_AUTH_REALM self.key_path = DEFAULT_KEY_PATH + self.max_session_duration = timedelta(days=2) + self.init_system_api_key() + self.sessions = ExpiringDict(max_len=64, max_age_seconds=self.max_session_duration.total_seconds()) def init_system_api_key(self): """Write an API key to a local file so local processes can use the API""" @@ -31,123 +30,133 @@ class AuthService: finally: os.umask(old_umask) - self.key = secrets.token_hex(24) + self.key = secrets.token_hex(32) os.makedirs(os.path.dirname(self.key_path), exist_ok=True) with create_file_with_mode(self.key_path, 0o640) as key_file: key_file.write(self.key + '\n') - def authenticate(self, request, env): - """Test if the client key passed in HTTP Authorization header matches the service key - or if the or username/password passed in the header matches an administrator user. + def authenticate(self, request, env, login_only=False, logout=False): + """Test if the HTTP Authorization header's username matches the system key, a session key, + or if the username/password passed in the header matches a local user. Returns a tuple of the user's email address and list of user privileges (e.g. ('my@email', []) or ('my@email', ['admin']); raises a ValueError on login failure. - If the user used an API key, the user's email is returned as None.""" + If the user used the system API key, the user's email is returned as None since + this key is not associated with a user.""" - def decode(s): - return base64.b64decode(s.encode('ascii')).decode('ascii') - - def parse_basic_auth(header): + def parse_http_authorization_basic(header): + def decode(s): + return base64.b64decode(s.encode('ascii')).decode('ascii') if " " not in header: return None, None scheme, credentials = header.split(maxsplit=1) if scheme != 'Basic': return None, None - credentials = decode(credentials) if ":" not in credentials: return None, None username, password = credentials.split(':', maxsplit=1) return username, password - header = request.headers.get('Authorization') - if not header: - raise ValueError("No authorization header provided.") - - username, password = parse_basic_auth(header) - + username, password = parse_http_authorization_basic(request.headers.get('Authorization', '')) if username in (None, ""): raise ValueError("Authorization header invalid.") - if username == self.key: - # The user passed the system API key which grants administrative privs. + if username.strip() == "" and password.strip() == "": + raise ValueError("No email address, password, session key, or API key provided.") + + # If user passed the system API key, grant administrative privs. This key + # is not associated with a user. + if username == self.key and not login_only: return (None, ["admin"]) + + # If the password corresponds with a session token for the user, grant access for that user. + if password in self.sessions and self.sessions[password]["email"] == username and not login_only: + sessionid = password + session = self.sessions[sessionid] + if session["password_token"] != self.create_user_password_state_token(username, env): + # This session is invalid because the user's password/MFA state changed + # after the session was created. + del self.sessions[sessionid] + raise ValueError("Session expired.") + if logout: + # Clear the session. + del self.sessions[sessionid] + else: + # Re-up the session so that it does not expire. + self.sessions[sessionid] = session + + # If no password was given, but a username was given, we're missing some information. + elif password.strip() == "": + raise ValueError("Enter a password.") + else: - # The user is trying to log in with a username and either a password - # (and possibly a MFA token) or a user-specific API key. - return (username, self.check_user_auth(username, password, request, env)) + # The user is trying to log in with a username and a password + # (and possibly a MFA token). On failure, an exception is raised. + self.check_user_auth(username, password, request, env) + + # Get privileges for authorization. This call should never fail because by this + # point we know the email address is a valid user --- unless the user has been + # deleted after the session was granted. On error the call will return a tuple + # of an error message and an HTTP status code. + privs = get_mail_user_privileges(username, env) + if isinstance(privs, tuple): raise ValueError(privs[0]) + + # Return the authorization information. + return (username, privs) def check_user_auth(self, email, pw, request, env): # Validate a user's login email address and password. If MFA is enabled, # check the MFA token in the X-Auth-Token header. # - # On success returns a list of privileges (e.g. [] or ['admin']). On login - # failure, raises a ValueError with a login error message. + # On login failure, raises a ValueError with a login error message. On + # success, nothing is returned. - # Sanity check. - if email == "" or pw == "": - raise ValueError("Enter an email address and password.") - - # The password might be a user-specific API key. create_user_key raises - # a ValueError if the user does not exist. - if hmac.compare_digest(self.create_user_key(email, env), pw): - # OK. - pass - else: + # Authenticate. + try: # Get the hashed password of the user. Raise a ValueError if the - # email address does not correspond to a user. + # email address does not correspond to a user. But wrap it in the + # same exception as if a password fails so we don't easily reveal + # if an email address is valid. pw_hash = get_mail_password(email, env) - # Authenticate. - try: - # Use 'doveadm pw' to check credentials. doveadm will return - # a non-zero exit status if the credentials are no good, - # and check_call will raise an exception in that case. - utils.shell('check_call', [ - "/usr/bin/doveadm", "pw", - "-p", pw, - "-t", pw_hash, - ]) - except: - # Login failed. - raise ValueError("Invalid password.") + # Use 'doveadm pw' to check credentials. doveadm will return + # a non-zero exit status if the credentials are no good, + # and check_call will raise an exception in that case. + utils.shell('check_call', [ + "/usr/bin/doveadm", "pw", + "-p", pw, + "-t", pw_hash, + ]) + except: + # Login failed. + raise ValueError("Incorrect email address or password.") - # If MFA is enabled, check that MFA passes. - status, hints = validate_auth_mfa(email, request, env) - if not status: - # Login valid. Hints may have more info. - raise ValueError(",".join(hints)) + # If MFA is enabled, check that MFA passes. + status, hints = validate_auth_mfa(email, request, env) + if not status: + # Login valid. Hints may have more info. + raise ValueError(",".join(hints)) - # Get privileges for authorization. This call should never fail because by this - # point we know the email address is a valid user. But on error the call will - # return a tuple of an error message and an HTTP status code. - privs = get_mail_user_privileges(email, env) - if isinstance(privs, tuple): raise ValueError(privs[0]) - - # Return a list of privileges. - return privs - - def create_user_key(self, email, env): - # Create a user API key, which is a shared secret that we can re-generate from - # static information in our database. The shared secret contains the user's - # email address, current hashed password, and current MFA state, so that the - # key becomes invalid if any of that information changes. - # - # Use an HMAC to generate the API key using our system API key as a key, - # which also means that the API key becomes invalid when our system API key - # changes --- i.e. when this process is restarted. - # - # Raises ValueError via get_mail_password if the user doesn't exist. - - # Construct the HMAC message from the user's email address and current password. - msg = b"AUTH:" + email.encode("utf8") + b" " + get_mail_password(email, env).encode("utf8") + def create_user_password_state_token(self, email, env): + # Create a token that changes if the user's password or MFA options change + # so that sessions become invalid if any of that information changes. + msg = get_mail_password(email, env).encode("utf8") # Add to the message the current MFA state, which is a list of MFA information. # Turn it into a string stably. msg += b" " + json.dumps(get_hash_mfa_state(email, env), sort_keys=True).encode("utf8") - # Make the HMAC. + # Make a HMAC using the system API key as a hash key. hash_key = self.key.encode('ascii') return hmac.new(hash_key, msg, digestmod="sha256").hexdigest() + def create_session_key(self, username, env, type=None): + # Create a new session. + token = secrets.token_hex(32) + self.sessions[token] = { + "email": username, + "password_token": self.create_user_password_state_token(username, env), + } + return token diff --git a/management/daemon.py b/management/daemon.py index bb723ea..ca89177 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -56,8 +56,10 @@ def authorized_personnel_only(viewfunc): try: email, privs = auth_service.authenticate(request, env) except ValueError as e: - # Write a line in the log recording the failed login - log_failed_login(request) + # Write a line in the log recording the failed login, unless no authorization header + # was given which can happen on an initial request before a 403 response. + if "Authorization" in request.headers: + log_failed_login(request) # Authentication failed. error = str(e) @@ -134,11 +136,12 @@ def index(): csr_country_codes=csr_country_codes, ) -@app.route('/me') -def me(): +# Create a session key by checking the username/password in the Authorization header. +@app.route('/login', methods=["POST"]) +def login(): # Is the caller authorized? try: - email, privs = auth_service.authenticate(request, env) + email, privs = auth_service.authenticate(request, env, login_only=True) except ValueError as e: if "missing-totp-token" in str(e): return json_response({ @@ -153,19 +156,29 @@ def me(): "reason": str(e), }) + # Return a new session for the user. resp = { "status": "ok", "email": email, "privileges": privs, + "api_key": auth_service.create_session_key(email, env, type='login'), } - # Is authorized as admin? Return an API key for future use. - if "admin" in privs: - resp["api_key"] = auth_service.create_user_key(email, env) + app.logger.info("New login session created for {}".format(email)) # Return. return json_response(resp) +@app.route('/logout', methods=["POST"]) +def logout(): + try: + email, _ = auth_service.authenticate(request, env, logout=True) + app.logger.info("{} logged out".format(email)) + except ValueError as e: + pass + finally: + return json_response({ "status": "ok" }) + # MAIL @app.route('/mail/users') diff --git a/management/templates/index.html b/management/templates/index.html index 12f6ad8..267f5dd 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -367,11 +367,17 @@ var current_panel = null; var switch_back_to_panel = null; function do_logout() { + // Clear the session from the backend. + api("/logout", "POST"); + + // Forget the token. api_credentials = ["", ""]; if (typeof localStorage != 'undefined') localStorage.removeItem("miab-cp-credentials"); if (typeof sessionStorage != 'undefined') sessionStorage.removeItem("miab-cp-credentials"); + + // Return to the start. show_panel('login'); } diff --git a/management/templates/login.html b/management/templates/login.html index 19b23d3..3447d79 100644 --- a/management/templates/login.html +++ b/management/templates/login.html @@ -105,8 +105,8 @@ function do_login() { api_credentials = [$('#loginEmail').val(), $('#loginPassword').val()] api( - "/me", - "GET", + "/login", + "POST", {}, function(response) { // This API call always succeeds. It returns a JSON object indicating diff --git a/setup/management.sh b/setup/management.sh index 1c57bb2..7e31fe0 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -49,8 +49,8 @@ hide_output $venv/bin/pip install --upgrade pip # NOTE: email_validator is repeated in setup/questions.sh, so please keep the versions synced. hide_output $venv/bin/pip install --upgrade \ rtyaml "email_validator>=1.0.0" "exclusiveprocess" \ - flask dnspython python-dateutil \ - qrcode[pil] pyotp \ + flask dnspython python-dateutil expiringdict \ + qrcode[pil] pyotp \ "idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver b2sdk # CONFIGURATION diff --git a/tests/fail2ban.py b/tests/fail2ban.py index 1cb55eb..cb55c51 100644 --- a/tests/fail2ban.py +++ b/tests/fail2ban.py @@ -232,7 +232,7 @@ if __name__ == "__main__": run_test(managesieve_test, [], 20, 30, 4) # Mail-in-a-Box control panel - run_test(http_test, ["/admin/me", 200], 20, 30, 1) + run_test(http_test, ["/admin/login", 200], 20, 30, 1) # Munin via the Mail-in-a-Box control panel run_test(http_test, ["/admin/munin/", 401], 20, 30, 1) From 26932ecb103b326069f3653e4420d770189c1460 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 22 Aug 2021 16:38:49 -0400 Subject: [PATCH 30/43] Add a 'welcome' panel to the control panel and make it the default page instead of the status checks which take too long to load Fixes #2014 --- management/templates/index.html | 6 ++++++ management/templates/login.html | 2 +- management/templates/welcome.html | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 management/templates/welcome.html diff --git a/management/templates/index.html b/management/templates/index.html index 267f5dd..492a953 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -118,6 +118,10 @@
+
+ {% include "welcome.html" %} +
+
{% include "system-status.html" %}
@@ -409,6 +413,8 @@ $(function() { // Recall what the user was last looking at. if (typeof localStorage != 'undefined' && localStorage.getItem("miab-cp-lastpanel")) { show_panel(localStorage.getItem("miab-cp-lastpanel")); + } else if (api_credentials[0] != "") { + show_panel('welcome'); } else { show_panel('login'); } diff --git a/management/templates/login.html b/management/templates/login.html index 3447d79..8ae7985 100644 --- a/management/templates/login.html +++ b/management/templates/login.html @@ -163,7 +163,7 @@ function do_login() { // Open the next panel the user wants to go to. Do this after the XHR response // is over so that we don't start a new XHR request while this one is finishing, // which confuses the loading indicator. - setTimeout(function() { show_panel(!switch_back_to_panel || switch_back_to_panel == "login" ? 'system_status' : switch_back_to_panel) }, 300); + setTimeout(function() { show_panel(!switch_back_to_panel || switch_back_to_panel == "login" ? 'welcome' : switch_back_to_panel) }, 300); } }, undefined, diff --git a/management/templates/welcome.html b/management/templates/welcome.html new file mode 100644 index 0000000..124d2d2 --- /dev/null +++ b/management/templates/welcome.html @@ -0,0 +1,16 @@ + + +

{{hostname}}

+ +

Welcome to your Mail-in-a-Box control panel.

+ From e5909a62870fc3a9d39a7ffe63a5264f9666ea79 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 22 Aug 2021 16:40:07 -0400 Subject: [PATCH 31/43] Allow non-admin login to the control panel and show/hide menu items depending on the login state * When logged out, no menu items are shown. * When logged in, Log Out is shown. * When logged in as an admin, the remaining menu items are also shown. * When logged in as a non-admin, the mail and contacts/calendar instruction pages are shown. Fixes #1987 --- management/templates/index.html | 46 +++++++++++++++++++++------------ management/templates/login.html | 28 +++++++++++++++++--- management/templates/users.html | 6 ++--- 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/management/templates/index.html b/management/templates/index.html index 492a953..081d527 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -62,6 +62,9 @@ ol li { margin-bottom: 1em; } + + .if-logged-in { display: none; } + .if-logged-in-admin { display: none; } @@ -83,7 +86,7 @@
@@ -302,7 +306,7 @@ function ajax_with_indicator(options) { return false; // handy when called from onclick } -var api_credentials = ["", ""]; +var api_credentials = null; function api(url, method, data, callback, callback_error, headers) { // from http://www.webtoolkit.info/javascript-base64.html function base64encode(input) { @@ -350,9 +354,10 @@ function api(url, method, data, callback, callback_error, headers) { // We don't store user credentials in a cookie to avoid the hassle of CSRF // attacks. The Authorization header only gets set in our AJAX calls triggered // by user actions. - xhr.setRequestHeader( - 'Authorization', - 'Basic ' + base64encode(api_credentials[0] + ':' + api_credentials[1])); + if (api_credentials) + xhr.setRequestHeader( + 'Authorization', + 'Basic ' + base64encode(api_credentials.username + ':' + api_credentials.session_key)); }, success: callback, error: callback_error || default_error, @@ -375,7 +380,7 @@ function do_logout() { api("/logout", "POST"); // Forget the token. - api_credentials = ["", ""]; + api_credentials = null; if (typeof localStorage != 'undefined') localStorage.removeItem("miab-cp-credentials"); if (typeof sessionStorage != 'undefined') @@ -383,6 +388,9 @@ function do_logout() { // Return to the start. show_panel('login'); + + // Reset menus. + show_hide_menus(); } function show_panel(panelid) { @@ -405,15 +413,21 @@ function show_panel(panelid) { $(function() { // Recall saved user credentials. - if (typeof sessionStorage != 'undefined' && sessionStorage.getItem("miab-cp-credentials")) - api_credentials = sessionStorage.getItem("miab-cp-credentials").split(":"); - else if (typeof localStorage != 'undefined' && localStorage.getItem("miab-cp-credentials")) - api_credentials = localStorage.getItem("miab-cp-credentials").split(":"); + try { + if (typeof sessionStorage != 'undefined' && sessionStorage.getItem("miab-cp-credentials")) + api_credentials = JSON.parse(sessionStorage.getItem("miab-cp-credentials")); + else if (typeof localStorage != 'undefined' && localStorage.getItem("miab-cp-credentials")) + api_credentials = JSON.parse(localStorage.getItem("miab-cp-credentials")); + } catch (_) { + } + + // Toggle menu state. + show_hide_menus(); // Recall what the user was last looking at. - if (typeof localStorage != 'undefined' && localStorage.getItem("miab-cp-lastpanel")) { + if (api_credentials != null && typeof localStorage != 'undefined' && localStorage.getItem("miab-cp-lastpanel")) { show_panel(localStorage.getItem("miab-cp-lastpanel")); - } else if (api_credentials[0] != "") { + } else if (api_credentials != null) { show_panel('welcome'); } else { show_panel('login'); diff --git a/management/templates/login.html b/management/templates/login.html index 8ae7985..421c884 100644 --- a/management/templates/login.html +++ b/management/templates/login.html @@ -102,7 +102,7 @@ function do_login() { } // Exchange the email address & password for an API key. - api_credentials = [$('#loginEmail').val(), $('#loginPassword').val()] + api_credentials = { username: $('#loginEmail').val(), session_key: $('#loginPassword').val() } api( "/login", @@ -141,7 +141,9 @@ function do_login() { // Login succeeded. // Save the new credentials. - api_credentials = [response.email, response.api_key]; + api_credentials = { username: response.email, + session_key: response.api_key, + privileges: response.privileges }; // Try to wipe the username/password information. $('#loginEmail').val(''); @@ -152,14 +154,17 @@ function do_login() { // Remember the credentials. if (typeof localStorage != 'undefined' && typeof sessionStorage != 'undefined') { if ($('#loginRemember').val()) { - localStorage.setItem("miab-cp-credentials", api_credentials.join(":")); + localStorage.setItem("miab-cp-credentials", JSON.stringify(api_credentials)); sessionStorage.removeItem("miab-cp-credentials"); } else { localStorage.removeItem("miab-cp-credentials"); - sessionStorage.setItem("miab-cp-credentials", api_credentials.join(":")); + sessionStorage.setItem("miab-cp-credentials", JSON.stringify(api_credentials)); } } + // Toggle menus. + show_hide_menus(); + // Open the next panel the user wants to go to. Do this after the XHR response // is over so that we don't start a new XHR request while this one is finishing, // which confuses the loading indicator. @@ -183,4 +188,19 @@ function show_login() { } }); } + +function show_hide_menus() { + var is_logged_in = (api_credentials != null); + var privs = api_credentials ? api_credentials.privileges : []; + $('.if-logged-in').toggle(is_logged_in); + $('.if-logged-in-admin, .if-logged-in-not-admin').toggle(false); + if (is_logged_in) { + $('.if-logged-in-not-admin').toggle(true); + privs.forEach(function(priv) { + $('.if-logged-in-' + priv).toggle(true); + $('.if-logged-in-not-' + priv).toggle(false); + }); + } + $('.if-not-logged-in').toggle(!is_logged_in); +} diff --git a/management/templates/users.html b/management/templates/users.html index 24adf4a..2ad5ebd 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -203,7 +203,7 @@ function users_set_password(elem) { var email = $(elem).parents('tr').attr('data-email'); var yourpw = ""; - if (api_credentials != null && email == api_credentials[0]) + if (api_credentials != null && email == api_credentials.username) yourpw = "

If you change your own password, you will be logged out of this control panel and will need to log in again.

"; show_modal_confirm( @@ -232,7 +232,7 @@ function users_remove(elem) { var email = $(elem).parents('tr').attr('data-email'); // can't remove yourself - if (api_credentials != null && email == api_credentials[0]) { + if (api_credentials != null && email == api_credentials.username) { show_modal_error("Archive User", "You cannot archive your own account."); return; } @@ -264,7 +264,7 @@ function mod_priv(elem, add_remove) { var priv = $(elem).parents('td').find('.name').text(); // can't remove your own admin access - if (priv == "admin" && add_remove == "remove" && api_credentials != null && email == api_credentials[0]) { + if (priv == "admin" && add_remove == "remove" && api_credentials != null && email == api_credentials.username) { show_modal_error("Modify Privileges", "You cannot remove the admin privilege from yourself."); return; } From 91079ab9347b7326a11c4011ce7c6cf8cf8b1491 Mon Sep 17 00:00:00 2001 From: mailinabox-contributor <90476861+mailinabox-contributor@users.noreply.github.com> Date: Fri, 10 Sep 2021 15:12:41 -0500 Subject: [PATCH 32/43] add numeric flag value to DNSSEC DS status message (#2033) Some registrars (e.g. Porkbun) accept Key Data when creating a DS RR, but accept only a numeric flags value to indicate the key type (256 for KSK, 257 for ZSK). https://datatracker.ietf.org/doc/html/rfc5910#section-4.3 --- management/status_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/status_checks.py b/management/status_checks.py index e2d4b1a..1e7223a 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -646,7 +646,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False): output.print_line("Option " + str(i+1) + ":") output.print_line("----------") output.print_line("Key Tag: " + ds_suggestion['keytag']) - output.print_line("Key Flags: KSK") + output.print_line("Key Flags: KSK (256)") output.print_line("Algorithm: %s / %s" % (ds_suggestion['alg'], ds_suggestion['alg_name'])) output.print_line("Digest Type: %s / %s" % (ds_suggestion['digalg'], ds_suggestion['digalg_name'])) output.print_line("Digest: " + ds_suggestion['digest']) From 353084ce6726e7aaee9b9df1c87dd03df29c5f9e Mon Sep 17 00:00:00 2001 From: Elsie Hupp <9206310+elsiehupp@users.noreply.github.com> Date: Sun, 19 Sep 2021 09:53:03 -0400 Subject: [PATCH 33/43] Use "smart invert" for dark mode (#2038) * Use "smart invert" for dark mode Signed-off-by: Elsie Hupp <9206310+elsiehupp@users.noreply.github.com> * Add more contrast to form controls Co-authored-by: Joshua Tauberer --- management/templates/index.html | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/management/templates/index.html b/management/templates/index.html index 081d527..115693a 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -62,9 +62,37 @@ ol li { margin-bottom: 1em; } - + .if-logged-in { display: none; } .if-logged-in-admin { display: none; } + + /* The below only gets used if it is supported */ + @media (prefers-color-scheme: dark) { + /* Invert invert lightness but not hue */ + html { + filter: invert(100%) hue-rotate(180deg); + } + + /* Set explicit background color (necessary for Firefox) */ + html { + background-color: #111; + } + + /* Override Boostrap theme here to give more contrast. The black turns to white by the filter. */ + .form-control { + color: black !important; + } + + /* Revert the invert for the navbar */ + button, div.navbar { + filter: invert(100%) hue-rotate(180deg); + } + + /* Revert the revert for the dropdowns */ + ul.dropdown-menu { + filter: invert(100%) hue-rotate(180deg); + } + } From df46e1311b05d4f01e1908dbfaa4102c18aa9094 Mon Sep 17 00:00:00 2001 From: drpixie Date: Fri, 24 Sep 2021 22:07:40 +1000 Subject: [PATCH 34/43] Include NSD config files from /etc/nsd/nsd.conf.d/*.conf (#2035) And write MIAB dns zone config into /etc/nsd/nsd.conf.d/zones.conf. Delete lingering old zones.conf file. Co-authored-by: Joshua Tauberer --- management/dns_update.py | 2 +- setup/dns.sh | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/management/dns_update.py b/management/dns_update.py index c000f34..b79e266 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -604,7 +604,7 @@ def get_dns_zonefile(zone, env): def write_nsd_conf(zonefiles, additional_records, env): # Write the list of zones to a configuration file. - nsd_conf_file = "/etc/nsd/zones.conf" + nsd_conf_file = "/etc/nsd/nsd.conf.d/zones.conf" nsdconf = "" # Append the zones. diff --git a/setup/dns.sh b/setup/dns.sh index b64a658..c8a73a7 100755 --- a/setup/dns.sh +++ b/setup/dns.sh @@ -62,7 +62,13 @@ for ip in $PRIVATE_IP $PRIVATE_IPV6; do echo " ip-address: $ip" >> /etc/nsd/nsd.conf; done -echo "include: /etc/nsd/zones.conf" >> /etc/nsd/nsd.conf; +# Create a directory for additional configuration directives, including +# the zones.conf file written out by our management daemon. +echo "include: /etc/nsd/nsd.conf.d/*.conf" >> /etc/nsd/nsd.conf; + +# Remove the old location of zones.conf that we generate. It will +# now be stored in /etc/nsd/nsd.conf.d. +rm -f /etc/nsd/zones.conf # Create DNSSEC signing keys. From 66b15d42a505feecd1013bcb41ee2aa73ca850ee Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 19 Sep 2021 09:03:57 -0400 Subject: [PATCH 35/43] CHANGELOG entries --- CHANGELOG.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9338b05..e22592f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,33 @@ CHANGELOG ========= +In Development +-------------- + +Mail: + +* Upgraded to Roundcube 1.5 Release Candidate. + +Firewall: + +* Fail2ban's IPv6 support is enabled. + +Control panel: + +* The control panel menus are now hidden before login, but now non-admins can log in to access the mail and contacts/calendar instruction pages. +* The login form now disables browser autocomplete in the two-factor authentication code field. +* After logging in, the default page is now a fast-loading welcome page rather than the slow-loading system status checks page. +* The backup retention period option now displays for B2 backup targets. +* The DNSSEC DS record recommendations are cleaned up and now recommend changing records that use SHA1. +* Control panel logins are now tied to a session backend that allows true logouts (rather than an encrypted cookie). +* Failed logins no longer directly reveal whether the email address corresponds to a user account. +* Browser dark mode now inverts the color scheme. + +Other: + +* The mail log tool now doesn't crash if there are email addresess in log messages with invalid UTF-8 characters. +* Additional nsd.conf files can be placed in /etc/nsd.conf.d. + v0.54 (June 20, 2021) --------------------- From 79966e36e3f74a50e923c74e83faf76e16c6ef13 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 12 Sep 2021 19:51:16 -0400 Subject: [PATCH 36/43] Set a cookie for /admin/munin pages to grant access to Munin reports The /admin/munin routes used the same Authorization: header logic as the other API routes, but they are browsed directly in the browser because they are handled as static pages or as a proxy to a CGI script. This required users to enter their email username/password for HTTP basic authentication in the standard browser auth prompt, which wasn't ideal (and may leak the password in browser storage). It also stopped working when MFA was enabled for user accounts. A token is now set in a cookie when visiting /admin/munin which is then checked in the routes that proxy the Munin pages. The cookie's lifetime is kept limited to limit the opportunity for any unknown CSRF attacks via the Munin CGI script. --- CHANGELOG.md | 1 + management/auth.py | 16 +++++++++------ management/daemon.py | 36 ++++++++++++++++++++++++++++----- management/templates/index.html | 6 +++++- management/templates/munin.html | 20 ++++++++++++++++++ 5 files changed, 67 insertions(+), 12 deletions(-) create mode 100644 management/templates/munin.html diff --git a/CHANGELOG.md b/CHANGELOG.md index e22592f..7dd0b9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Control panel: * After logging in, the default page is now a fast-loading welcome page rather than the slow-loading system status checks page. * The backup retention period option now displays for B2 backup targets. * The DNSSEC DS record recommendations are cleaned up and now recommend changing records that use SHA1. +* The Munin monitoring pages no longer require a separate HTTP basic authentication login and can be used if two-factor authentication is turned on. * Control panel logins are now tied to a session backend that allows true logouts (rather than an encrypted cookie). * Failed logins no longer directly reveal whether the email address corresponds to a user account. * Browser dark mode now inverts the color scheme. diff --git a/management/auth.py b/management/auth.py index 38c15e9..0a88c45 100644 --- a/management/auth.py +++ b/management/auth.py @@ -72,14 +72,9 @@ class AuthService: return (None, ["admin"]) # If the password corresponds with a session token for the user, grant access for that user. - if password in self.sessions and self.sessions[password]["email"] == username and not login_only: + if self.get_session(username, password, "login", env) and not login_only: sessionid = password session = self.sessions[sessionid] - if session["password_token"] != self.create_user_password_state_token(username, env): - # This session is invalid because the user's password/MFA state changed - # after the session was created. - del self.sessions[sessionid] - raise ValueError("Session expired.") if logout: # Clear the session. del self.sessions[sessionid] @@ -158,5 +153,14 @@ class AuthService: self.sessions[token] = { "email": username, "password_token": self.create_user_password_state_token(username, env), + "type": type, } return token + + def get_session(self, user_email, session_key, session_type, env): + if session_key not in self.sessions: return None + session = self.sessions[session_key] + if session_type == "login" and session["email"] != user_email: return None + if session["type"] != session_type: return None + if session["password_token"] != self.create_user_password_state_token(session["email"], env): return None + return session diff --git a/management/daemon.py b/management/daemon.py index ca89177..280efec 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -654,16 +654,42 @@ def privacy_status_set(): # MUNIN @app.route('/munin/') -@app.route('/munin/') @authorized_personnel_only -def munin(filename=""): - # Checks administrative access (@authorized_personnel_only) and then just proxies - # the request to static files. +def munin_start(): + # Munin pages, static images, and dynamically generated images are served + # outside of the AJAX API. We'll start with a 'start' API that sets a cookie + # that subsequent requests will read for authorization. (We don't use cookies + # for the API to avoid CSRF vulnerabilities.) + response = make_response("OK") + response.set_cookie("session", auth_service.create_session_key(request.user_email, env, type='cookie'), + max_age=60*30, secure=True, httponly=True, samesite="Strict") # 30 minute duration + return response + +def check_request_cookie_for_admin_access(): + session = auth_service.get_session(None, request.cookies.get("session", ""), "cookie", env) + if not session: return False + privs = get_mail_user_privileges(session["email"], env) + if not isinstance(privs, list): return False + if "admin" not in privs: return False + return True + +def authorized_personnel_only_via_cookie(f): + @wraps(f) + def g(*args, **kwargs): + if not check_request_cookie_for_admin_access(): + return Response("Unauthorized", status=403, mimetype='text/plain', headers={}) + return f(*args, **kwargs) + return g + +@app.route('/munin/') +@authorized_personnel_only_via_cookie +def munin_static_file(filename=""): + # Proxy the request to static files. if filename == "": filename = "index.html" return send_from_directory("/var/cache/munin/www", filename) @app.route('/munin/cgi-graph/') -@authorized_personnel_only +@authorized_personnel_only_via_cookie def munin_cgi(filename): """ Relay munin cgi dynazoom requests /usr/lib/munin/cgi/munin-cgi-graph is a perl cgi script in the munin package diff --git a/management/templates/index.html b/management/templates/index.html index 115693a..f9c87f2 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -124,7 +124,7 @@
  • Advanced Pages
  • Custom DNS
  • External DNS
    • -
  • Munin Monitoring
    • +
  • Munin Monitoring
  • Mail
    • @@ -202,6 +202,10 @@ {% include "ssl.html" %}
    +
    + {% include "munin.html" %} +
    +