From 2a72c800f6b7c4910f5fa294d7a82ec805cdf020 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 12 May 2018 20:02:25 -0400 Subject: [PATCH] replace free_tls_certificates with certbot --- CHANGELOG.md | 6 + conf/nginx.conf | 2 +- management/daemon.py | 17 +- management/daily_tasks.sh | 2 +- management/ssl_certificates.py | 556 +++++++++++++-------------------- management/templates/ssl.html | 86 +---- management/web_update.py | 7 +- setup/management.sh | 24 +- setup/migrate.py | 11 + setup/start.sh | 16 +- setup/system.sh | 25 +- 11 files changed, 312 insertions(+), 440 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63eaf47..9e4f2e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,14 @@ CHANGELOG v0.27 (June 14, 2018) --------------------- +Mail: + * A report of box activity, including sent/received mail totals and logins by user, is now emailed to the box's administrator user each week. * Update Roundcube to version 1.3.6 and Z-Push to version 2.3.9. + +Control Panel: + +* We now use EFF's `certbot` tool to provision HTTPS certificates instead of our home-grown free_tls_certificates package. * The undocumented feature for proxying web requests to another server now sets X-Forwarded-For. v0.26c (February 13, 2018) diff --git a/conf/nginx.conf b/conf/nginx.conf index 0a08439..ce66275 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -25,7 +25,7 @@ server { # This path must be served over HTTP for ACME domain validation. # We map this to a special path where our TLS cert provisioning # tool knows to store challenge response files. - alias $STORAGE_ROOT/ssl/lets_encrypt/acme_challenges/; + alias $STORAGE_ROOT/ssl/lets_encrypt/webroot/.well-known/acme-challenge/; } } diff --git a/management/daemon.py b/management/daemon.py index f5ea215..2e23c8a 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -333,11 +333,16 @@ def ssl_get_status(): from web_update import get_web_domains_info, get_web_domains # What domains can we provision certificates for? What unexpected problems do we have? - provision, cant_provision = get_certificates_to_provision(env, show_extended_problems=False) + provision, cant_provision = get_certificates_to_provision(env, show_valid_certs=False) # What's the current status of TLS certificates on all of the domain? domains_status = get_web_domains_info(env) - domains_status = [{ "domain": d["domain"], "status": d["ssl_certificate"][0], "text": d["ssl_certificate"][1] } for d in domains_status ] + domains_status = [ + { + "domain": d["domain"], + "status": d["ssl_certificate"][0], + "text": d["ssl_certificate"][1] + ((" " + cant_provision[d["domain"]] if d["domain"] in cant_provision else "")) + } for d in domains_status ] # Warn the user about domain names not hosted here because of other settings. for domain in set(get_web_domains(env, exclude_dns_elsewhere=False)) - set(get_web_domains(env)): @@ -349,7 +354,6 @@ def ssl_get_status(): return json_response({ "can_provision": utils.sort_domains(provision, env), - "cant_provision": [{ "domain": domain, "problem": cant_provision[domain] } for domain in utils.sort_domains(cant_provision, env) ], "status": domains_status, }) @@ -376,11 +380,8 @@ def ssl_install_cert(): @authorized_personnel_only def ssl_provision_certs(): from ssl_certificates import provision_certificates - agree_to_tos_url = request.form.get('agree_to_tos_url') - status = provision_certificates(env, - agree_to_tos_url=agree_to_tos_url, - jsonable=True) - return json_response(status) + requests = provision_certificates(env, limit_domains=None) + return json_response({ "requests": requests }) # WEB diff --git a/management/daily_tasks.sh b/management/daily_tasks.sh index b5b628c..3054dd3 100755 --- a/management/daily_tasks.sh +++ b/management/daily_tasks.sh @@ -19,7 +19,7 @@ fi management/backup.py | management/email_administrator.py "Backup Status" # Provision any new certificates for new domains or domains with expiring certificates. -management/ssl_certificates.py -q --headless | management/email_administrator.py "Error Provisioning TLS Certificate" +management/ssl_certificates.py -q | management/email_administrator.py "Error Provisioning TLS Certificate" # Run status checks and email the administrator if anything changed. management/status_checks.py --show-changes | management/email_administrator.py "Status Checks Change Notice" diff --git a/management/ssl_certificates.py b/management/ssl_certificates.py index 303571b..200a346 100755 --- a/management/ssl_certificates.py +++ b/management/ssl_certificates.py @@ -1,7 +1,7 @@ #!/usr/local/lib/mailinabox/env/bin/python # Utilities for installing and selecting SSL certificates. -import os, os.path, re, shutil +import os, os.path, re, shutil, subprocess, tempfile from utils import shell, safe_domain_name, sort_domains import idna @@ -24,6 +24,16 @@ def get_ssl_certificates(env): if not os.path.exists(ssl_root): return for fn in os.listdir(ssl_root): + if fn == 'ssl_certificate.pem': + # This is always a symbolic link + # to the certificate to use for + # PRIMARY_HOSTNAME. Don't let it + # be eligible for use because we + # could end up creating a symlink + # to itself --- we want to find + # the cert that it should be a + # symlink to. + continue fn = os.path.join(ssl_root, fn) if os.path.isfile(fn): yield fn @@ -74,6 +84,12 @@ def get_ssl_certificates(env): # Add this cert to the list of certs usable for the domains. for domain in cert_domains: + # The primary hostname can only use a certificate mapped + # to the system private key. + if domain == env['PRIMARY_HOSTNAME']: + if cert._private_key._filename != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'): + continue + domains.setdefault(domain, []).append(cert) # Sort the certificates to prefer good ones. @@ -81,6 +97,7 @@ def get_ssl_certificates(env): now = datetime.datetime.utcnow() ret = { } for domain, cert_list in domains.items(): + #for c in cert_list: print(domain, c.not_valid_before, c.not_valid_after, "("+str(now)+")", c.issuer, c.subject, c._filename) cert_list.sort(key = lambda cert : ( # must be valid NOW cert.not_valid_before <= now <= cert.not_valid_after, @@ -124,21 +141,22 @@ def get_ssl_certificates(env): return ret -def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False, raw=False): - # Get the system certificate info. - ssl_private_key = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_private_key.pem')) - ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem')) - system_certificate = { - "private-key": ssl_private_key, - "certificate": ssl_certificate, - "primary-domain": env['PRIMARY_HOSTNAME'], - "certificate_object": load_pem(load_cert_chain(ssl_certificate)[0]), - } +def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False, use_main_cert=True): + if use_main_cert: + # Get the system certificate info. + ssl_private_key = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_private_key.pem')) + ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem')) + system_certificate = { + "private-key": ssl_private_key, + "certificate": ssl_certificate, + "primary-domain": env['PRIMARY_HOSTNAME'], + "certificate_object": load_pem(load_cert_chain(ssl_certificate)[0]), + } - if domain == env['PRIMARY_HOSTNAME']: - # The primary domain must use the server certificate because - # it is hard-coded in some service configuration files. - return system_certificate + if domain == env['PRIMARY_HOSTNAME']: + # The primary domain must use the server certificate because + # it is hard-coded in some service configuration files. + return system_certificate wildcard_domain = re.sub("^[^\.]+", "*", domain) if domain in ssl_certificates: @@ -155,112 +173,97 @@ def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False # PROVISIONING CERTIFICATES FROM LETSENCRYPT -def get_certificates_to_provision(env, show_extended_problems=True, force_domains=None): - # Get a set of domain names that we should now provision certificates - # for. Provision if a domain name has no valid certificate or if any - # certificate is expiring in 14 days. If provisioning anything, also - # provision certificates expiring within 30 days. The period between - # 14 and 30 days allows us to consolidate domains into multi-domain - # certificates for domains expiring around the same time. +def get_certificates_to_provision(env, limit_domains=None, show_valid_certs=True): + # Get a set of domain names that we can provision certificates for + # using certbot. We start with domains that the box is serving web + # for and subtract: + # * domains not in limit_domains if limit_domains is not empty + # * domains with custom "A" records, i.e. they are hosted elsewhere + # * domains with actual "A" records that point elsewhere + # * domains that already have certificates that will be valid for a while from web_update import get_web_domains + from status_checks import query_dns, normalize_ip - import datetime - now = datetime.datetime.utcnow() + existing_certs = get_ssl_certificates(env) - # Get domains with missing & expiring certificates. - certs = get_ssl_certificates(env) - domains = set() - domains_if_any = set() - problems = { } - for domain in get_web_domains(env): - # If the user really wants a cert for certain domains, include it. - if force_domains: - if force_domains == "ALL" or (isinstance(force_domains, list) and domain in force_domains): - domains.add(domain) + plausible_web_domains = get_web_domains(env, exclude_dns_elsewhere=False) + actual_web_domains = get_web_domains(env) + + domains_to_provision = set() + domains_cant_provision = { } + + for domain in plausible_web_domains: + # Skip domains that the user doesn't want to provision now. + if limit_domains and domain not in limit_domains: continue - # Include this domain if its certificate is missing, self-signed, or expiring soon. - try: - cert = get_domain_ssl_files(domain, certs, env, allow_missing_cert=True) - except FileNotFoundError as e: - # system certificate is not present - problems[domain] = "Error: " + str(e) - continue - if cert is None: - # No valid certificate available. - domains.add(domain) + # Check that there isn't an explicit A/AAAA record. + if domain not in actual_web_domains: + domains_cant_provision[domain] = "The domain has a custom DNS A/AAAA record that points the domain elsewhere, so there is no point to installing a TLS certificate here and we could not automatically provision one anyway because provisioning requires access to the website (which isn't here)." + + # Check that the DNS resolves to here. else: - cert = cert["certificate_object"] - if cert.issuer == cert.subject: - # This is self-signed. Get a real one. - domains.add(domain) + + # Does the domain resolve to this machine in public DNS? If not, + # we can't do domain control validation. For IPv6 is configured, + # make sure both IPv4 and IPv6 are correct because we don't know + # how Let's Encrypt will connect. + bad_dns = [] + for rtype, value in [("A", env["PUBLIC_IP"]), ("AAAA", env.get("PUBLIC_IPV6"))]: + if not value: continue # IPv6 is not configured + response = query_dns(domain, rtype) + if response != normalize_ip(value): + bad_dns.append("%s (%s)" % (response, rtype)) + + if bad_dns: + domains_cant_provision[domain] = "The domain name does not resolve to this machine: " \ + + (", ".join(bad_dns)) \ + + "." - # Valid certificate today, but is it expiring soon? - elif cert.not_valid_after-now < datetime.timedelta(days=14): - domains.add(domain) - elif cert.not_valid_after-now < datetime.timedelta(days=30): - domains_if_any.add(domain) + else: + # DNS is all good. - # It's valid. Should we report its validness? - elif show_extended_problems: - problems[domain] = "The certificate is valid for at least another 30 days --- no need to replace." + # Check for a good existing cert. + existing_cert = get_domain_ssl_files(domain, existing_certs, env, use_main_cert=False) + if existing_cert: + existing_cert_check = check_certificate(domain, existing_cert['certificate'], existing_cert['private-key'], + warn_if_expiring_soon=14) + if existing_cert_check[0] == "OK": + if show_valid_certs: + domains_cant_provision[domain] = "The domain has a valid certificate already. ({} Certificate: {}, private key {})".format( + existing_cert_check[1], + existing_cert['certificate'], + existing_cert['private-key']) + continue - # Warn the user about domains hosted elsewhere. - if not force_domains and show_extended_problems: - for domain in set(get_web_domains(env, exclude_dns_elsewhere=False)) - set(get_web_domains(env)): - problems[domain] = "The domain's DNS is pointed elsewhere, so there is no point to installing a TLS certificate here and we could not automatically provision one anyway because provisioning requires access to the website (which isn't here)." + domains_to_provision.add(domain) - # Filter out domains that we can't provision a certificate for. - def can_provision_for_domain(domain): - from status_checks import query_dns, normalize_ip - - # Does the domain resolve to this machine in public DNS? If not, - # we can't do domain control validation. For IPv6 is configured, - # make sure both IPv4 and IPv6 are correct because we don't know - # how Let's Encrypt will connect. - for rtype, value in [("A", env["PUBLIC_IP"]), ("AAAA", env.get("PUBLIC_IPV6"))]: - if not value: continue # IPv6 is not configured - response = query_dns(domain, rtype) - if response != normalize_ip(value): - problems[domain] = "The domain name does not resolve to this machine: DNS %s resolved to %s." % (rtype, response) - return False - - return True - - domains = set(filter(can_provision_for_domain, domains)) - - # If there are any domains we definitely will provision for, add in - # additional domains to do at this time. - if len(domains) > 0: - domains |= set(filter(can_provision_for_domain, domains_if_any)) - - return (domains, problems) - -def provision_certificates(env, agree_to_tos_url=None, logger=None, show_extended_problems=True, force_domains=None, jsonable=False): - import requests.exceptions - import acme.messages - - from free_tls_certificates import client + return (domains_to_provision, domains_cant_provision) +def provision_certificates(env, limit_domains): # What domains should we provision certificates for? And what # errors prevent provisioning for other domains. - domains, problems = get_certificates_to_provision(env, force_domains=force_domains, show_extended_problems=show_extended_problems) + domains, domains_cant_provision = get_certificates_to_provision(env, limit_domains=limit_domains) + + # Build a list of what happened on each domain or domain-set. + ret = [] + for domain, error in domains_cant_provision.items(): + ret.append({ + "domains": [domain], + "log": [error], + "result": "skipped", + }) - # Exit fast if there is nothing to do. - if len(domains) == 0: - return { - "requests": [], - "problems": problems, - } # Break into groups of up to 100 certificates at a time, which is Let's Encrypt's # limit for a single certificate. We'll sort to put related domains together. + max_domains_per_group = 100 domains = sort_domains(domains, env) certs = [] while len(domains) > 0: - certs.append( domains[0:100] ) - domains = domains[100:] + certs.append( domains[:max_domains_per_group] ) + domains = domains[max_domains_per_group:] # Prepare to provision. @@ -269,115 +272,82 @@ def provision_certificates(env, agree_to_tos_url=None, logger=None, show_extende if not os.path.exists(account_path): os.mkdir(account_path) - # Where should we put ACME challenge files. This is mapped to /.well-known/acme_challenge - # by the nginx configuration. - challenges_path = os.path.join(account_path, 'acme_challenges') - if not os.path.exists(challenges_path): - os.mkdir(challenges_path) - - # Read in the private key that we use for all TLS certificates. We'll need that - # to generate a CSR (done by free_tls_certificates). - with open(os.path.join(env['STORAGE_ROOT'], 'ssl/ssl_private_key.pem'), 'rb') as f: - private_key = f.read() - # Provision certificates. - - ret = [] for domain_list in certs: - # For return. - ret_item = { + ret.append({ "domains": domain_list, "log": [], - } - ret.append(ret_item) - - # Logging for free_tls_certificates. - def my_logger(message): - if logger: logger(message) - ret_item["log"].append(message) - - # Attempt to provision a certificate. + }) try: - try: - cert = client.issue_certificate( - domain_list, - account_path, - agree_to_tos_url=agree_to_tos_url, - private_key=private_key, - logger=my_logger) + # Create a CSR file for our master private key so that certbot + # uses our private key. + key_file = os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem') + with tempfile.NamedTemporaryFile() as csr_file: + # We could use openssl, but certbot requires + # that the CN domain and SAN domains match + # the domain list passed to certbot, and adding + # SAN domains openssl req is ridiculously complicated. + # subprocess.check_output([ + # "openssl", "req", "-new", + # "-key", key_file, + # "-out", csr_file.name, + # "-subj", "/CN=" + domain_list[0], + # "-sha256" ]) + from cryptography import x509 + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives.serialization import Encoding + from cryptography.hazmat.primitives import hashes + from cryptography.x509.oid import NameOID + builder = x509.CertificateSigningRequestBuilder() + builder = builder.subject_name(x509.Name([ x509.NameAttribute(NameOID.COMMON_NAME, domain_list[0]) ])) + builder = builder.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + builder = builder.add_extension(x509.SubjectAlternativeName( + [x509.DNSName(d) for d in domain_list] + ), critical=False) + request = builder.sign(load_pem(load_cert_chain(key_file)[0]), hashes.SHA256(), default_backend()) + with open(csr_file.name, "wb") as f: + f.write(request.public_bytes(Encoding.PEM)) - except client.NeedToTakeAction as e: - # Write out the ACME challenge files. - for action in e.actions: - if isinstance(action, client.NeedToInstallFile): - fn = os.path.join(challenges_path, action.file_name) - with open(fn, 'w') as f: - f.write(action.contents) - else: - raise ValueError(str(action)) + # Provision, writing to a temporary file. + webroot = os.path.join(account_path, 'webroot') + os.makedirs(webroot, exist_ok=True) + with tempfile.TemporaryDirectory() as d: + cert_file = os.path.join(d, 'cert_and_chain.pem') + print("Provisioning TLS certificates for " + ", ".join(domain_list) + ".") + certbotret = subprocess.check_output([ + "certbot", + "certonly", + #"-v", # just enough to see ACME errors + "--non-interactive", # will fail if user hasn't registered during Mail-in-a-Box setup - # Try to provision now that the challenge files are installed. + "-d", ",".join(domain_list), # first will be main domain - cert = client.issue_certificate( - domain_list, - account_path, - private_key=private_key, - logger=my_logger) + "--csr", csr_file.name, # use our private key; unfortunately this doesn't work with auto-renew so we need to save cert manually + "--cert-path", os.path.join(d, 'cert'), # we only use the full chain + "--chain-path", os.path.join(d, 'chain'), # we only use the full chain + "--fullchain-path", cert_file, - except client.NeedToAgreeToTOS as e: - # The user must agree to the Let's Encrypt terms of service agreement - # before any further action can be taken. - ret_item.update({ - "result": "agree-to-tos", - "url": e.url, - }) + "--webroot", "--webroot-path", webroot, - except client.WaitABit as e: - # We need to hold on for a bit before querying again to see if we can - # acquire a provisioned certificate. - import time, datetime - ret_item.update({ - "result": "wait", - "until": e.until_when if not jsonable else e.until_when.isoformat(), - "seconds": (e.until_when - datetime.datetime.now()).total_seconds() - }) + "--config-dir", account_path, + #"--staging", + ], stderr=subprocess.STDOUT).decode("utf8") + install_cert_copy_file(cert_file, env) - except client.AccountDataIsCorrupt as e: - # This is an extremely rare condition. - ret_item.update({ - "result": "error", - "message": "Something unexpected went wrong. It looks like your local Let's Encrypt account data is corrupted. There was a problem with the file " + e.account_file_path + ".", - }) + ret[-1]["log"].append(certbotret) + ret[-1]["result"] = "installed" + except subprocess.CalledProcessError as e: + ret[-1]["log"].append(e.output.decode("utf8")) + ret[-1]["result"] = "error" + except Exception as e: + ret[-1]["log"].append(str(e)) + ret[-1]["result"] = "error" - except (client.InvalidDomainName, client.NeedToTakeAction, client.ChallengeFailed, client.RateLimited, acme.messages.Error, requests.exceptions.RequestException) as e: - ret_item.update({ - "result": "error", - "message": "Something unexpected went wrong: " + str(e), - }) - - else: - # A certificate was issued. - - install_status = install_cert(domain_list[0], cert['cert'].decode("ascii"), b"\n".join(cert['chain']).decode("ascii"), env, raw=True) - - # str indicates the certificate was not installed. - if isinstance(install_status, str): - ret_item.update({ - "result": "error", - "message": "Something unexpected was wrong with the provisioned certificate: " + install_status, - }) - else: - # A list indicates success and what happened next. - ret_item["log"].extend(install_status) - ret_item.update({ - "result": "installed", - }) + # Run post-install steps. + ret.extend(post_install_func(env)) # Return what happened with each certificate request. - return { - "requests": ret, - "problems": problems, - } + return ret def provision_certificates_cmdline(): import sys @@ -388,151 +358,39 @@ def provision_certificates_cmdline(): Lock(die=True).forever() env = load_environment() - verbose = False - headless = False - force_domains = None - show_extended_problems = True - - args = list(sys.argv) - args.pop(0) # program name - if args and args[0] == "-v": - verbose = True - args.pop(0) - if args and args[0] == "-q": - show_extended_problems = False - args.pop(0) - if args and args[0] == "--headless": - headless = True - args.pop(0) - if args and args[0] == "--force": - force_domains = "ALL" - args.pop(0) - else: - force_domains = args + quiet = False + domains = [] - agree_to_tos_url = None - while True: - # Run the provisioning script. This installs certificates. If there are - # a very large number of domains on this box, it issues separate - # certificates for groups of domains. We have to check the result for - # each group. - def my_logger(message): - if verbose: - print(">", message) - status = provision_certificates(env, agree_to_tos_url=agree_to_tos_url, logger=my_logger, force_domains=force_domains, show_extended_problems=show_extended_problems) - agree_to_tos_url = None # reset to prevent infinite looping + for arg in sys.argv[1:]: + if arg == "-q": + quiet = True + else: + domains.append(arg) - if not status["requests"]: - # No domains need certificates. - if not headless or verbose: - if len(status["problems"]) == 0: - print("No domains hosted on this box need a new TLS certificate at this time.") - elif len(status["problems"]) > 0: - print("No TLS certificates could be provisoned at this time:") - print() - for domain in sort_domains(status["problems"], env): - print("%s: %s" % (domain, status["problems"][domain])) + # Go. + status = provision_certificates(env, limit_domains=domains) - sys.exit(0) - - # What happened? - wait_until = None - wait_domains = [] - for request in status["requests"]: - if request["result"] == "agree-to-tos": - # We may have asked already in a previous iteration. - if agree_to_tos_url is not None: - continue - - # Can't ask the user a question in this mode. Warn the user that something - # needs to be done. - if headless: - print(", ".join(request["domains"]) + " need a new or renewed TLS certificate.") - print() - print("This box can't do that automatically for you until you agree to Let's Encrypt's") - print("Terms of Service agreement. Use the Mail-in-a-Box control panel to provision") - print("certificates for these domains.") - sys.exit(1) - - print(""" -I'm going to provision a TLS certificate (formerly called a SSL certificate) -for you from Let's Encrypt (letsencrypt.org). - -TLS certificates are cryptographic keys that ensure communication between -you and this box are secure when getting and sending mail and visiting -websites hosted on this box. Let's Encrypt is a free provider of TLS -certificates. - -Please open this document in your web browser: - -%s - -It is Let's Encrypt's terms of service agreement. If you agree, I can -provision that TLS certificate. If you don't agree, you will have an -opportunity to install your own TLS certificate from the Mail-in-a-Box -control panel. - -Do you agree to the agreement? Type Y or N and press : """ - % request["url"], end='', flush=True) - - if sys.stdin.readline().strip().upper() != "Y": - print("\nYou didn't agree. Quitting.") - sys.exit(1) - - # Okay, indicate agreement on next iteration. - agree_to_tos_url = request["url"] - - if request["result"] == "wait": - # Must wait. We'll record until when. The wait occurs below. - if wait_until is None: - wait_until = request["until"] - else: - wait_until = max(wait_until, request["until"]) - wait_domains += request["domains"] - - if request["result"] == "error": - print(", ".join(request["domains"]) + ":") - print(request["message"]) - - if request["result"] == "installed": - print("A TLS certificate was successfully installed for " + ", ".join(request["domains"]) + ".") - - if wait_until: - # Wait, then loop. - import time, datetime + # Show what happened. + for request in status: + if isinstance(request, str): + print(request) + else: + if quiet and request['result'] == 'skipped': + continue + print(request['result'] + ":", ", ".join(request['domains']) + ":") + for line in request["log"]: + print(line) print() - print("A TLS certificate was requested for: " + ", ".join(wait_domains) + ".") - first = True - while wait_until > datetime.datetime.now(): - if not headless or first: - print ("We have to wait", int(round((wait_until - datetime.datetime.now()).total_seconds())), "seconds for the certificate to be issued...") - time.sleep(10) - first = False - continue # Loop! - - if agree_to_tos_url: - # The user agrees to the TOS. Loop to try again by agreeing. - continue # Loop! - - # Unless we were instructed to wait, or we just agreed to the TOS, - # we're done for now. - break - - # And finally show the domains with problems. - if len(status["problems"]) > 0: - print("TLS certificates could not be provisoned for:") - for domain in sort_domains(status["problems"], env): - print("%s: %s" % (domain, status["problems"][domain])) # INSTALLING A NEW CERTIFICATE FROM THE CONTROL PANEL def create_csr(domain, ssl_key, country_code, env): return shell("check_output", [ - "openssl", "req", "-new", - "-key", ssl_key, - "-sha256", - "-subj", "/C=%s/CN=%s" % (country_code, domain)]) + "openssl", "req", "-new", + "-key", ssl_key, + "-sha256", + "-subj", "/C=%s/CN=%s" % (country_code, domain)]) def install_cert(domain, ssl_cert, ssl_chain, env, raw=False): # Write the combined cert+chain to a temporary path and validate that it is OK. @@ -553,6 +411,16 @@ def install_cert(domain, ssl_cert, ssl_chain, env, raw=False): cert_status += " " + cert_status_details return cert_status + # Copy certifiate into ssl directory. + install_cert_copy_file(fn, env) + + # Run post-install steps. + ret = post_install_func(env) + if raw: return ret + return "\n".join(ret) + + +def install_cert_copy_file(fn, env): # Where to put it? # Make a unique path for the certificate. from cryptography.hazmat.primitives import hashes @@ -570,14 +438,26 @@ def install_cert(domain, ssl_cert, ssl_chain, env, raw=False): os.makedirs(os.path.dirname(ssl_certificate), exist_ok=True) shutil.move(fn, ssl_certificate) - ret = ["OK"] - # When updating the cert for PRIMARY_HOSTNAME, symlink it from the system +def post_install_func(env): + ret = [] + + # Get the certificate to use for PRIMARY_HOSTNAME. + ssl_certificates = get_ssl_certificates(env) + cert = get_domain_ssl_files(env['PRIMARY_HOSTNAME'], ssl_certificates, env, use_main_cert=False) + if not cert: + # Ruh-row, we don't have any certificate usable + # for the primary hostname. + ret.append("there is no valid certificate for " + env['PRIMARY_HOSTNAME']) + + # Symlink the best cert for PRIMARY_HOSTNAME to the system # certificate path, which is hard-coded for various purposes, and then # restart postfix and dovecot. - if domain == env['PRIMARY_HOSTNAME']: + system_ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem')) + if cert and os.readlink(system_ssl_certificate) != cert['certificate']: # Update symlink. - system_ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem')) + ret.append("updating primary certificate") + ssl_certificate = cert['certificate'] os.unlink(system_ssl_certificate) os.symlink(ssl_certificate, system_ssl_certificate) @@ -593,12 +473,12 @@ def install_cert(domain, ssl_cert, ssl_chain, env, raw=False): # Update the web configuration so nginx picks up the new certificate file. from web_update import do_web_update ret.append( do_web_update(env) ) - if raw: return ret - return "\n".join(ret) + + return ret # VALIDATION OF CERTIFICATES -def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring_soon=True, rounded_time=False, just_check_domain=False): +def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring_soon=10, rounded_time=False, just_check_domain=False): # Check that the ssl_certificate & ssl_private_key files are good # for the provided domain. @@ -704,7 +584,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring # We'll renew it with Lets Encrypt. expiry_info = "The certificate expires on %s." % cert_expiration_date.strftime("%x") - if ndays <= 10 and warn_if_expiring_soon: + if warn_if_expiring_soon and ndays <= warn_if_expiring_soon: # Warn on day 10 to give 4 days for us to automatically renew the # certificate, which occurs on day 14. return ("The certificate is expiring soon: " + expiry_info, None) diff --git a/management/templates/ssl.html b/management/templates/ssl.html index 54b2a94..a6b913e 100644 --- a/management/templates/ssl.html +++ b/management/templates/ssl.html @@ -8,7 +8,7 @@

You need a TLS certificate for this box’s hostname ({{hostname}}) and every other domain name and subdomain that this box is hosting a website for (see the list below).

-

Provision a certificate

+

Provision certificates

Certificate status

@@ -103,24 +88,12 @@ function show_tls(keep_provisioning_shown) { // provisioning status if (!keep_provisioning_shown) - $('#ssl_provision').toggle(res.can_provision.length + res.cant_provision.length > 0) + $('#ssl_provision').toggle(res.can_provision.length > 0) $('#ssl_provision_p').toggle(res.can_provision.length > 0); if (res.can_provision.length > 0) $('#ssl_provision_p span').text(res.can_provision.join(", ")); - $('#ssl_provision_problems_div').toggle(res.cant_provision.length > 0); - $('#ssl_provision_problems tbody').text(""); - for (var i = 0; i < res.cant_provision.length; i++) { - var domain = res.cant_provision[i]; - var row = $(""); - $('#ssl_provision_problems tbody').append(row); - row.attr('data-domain', domain.domain); - row.find('.domain a').text(domain.domain); - row.find('.domain a').attr('href', 'https://' + domain.domain); - row.find('.status').text(domain.problem); - } - // certificate status var domains = res.status; var tb = $('#ssl_domains tbody'); @@ -196,20 +169,15 @@ function install_cert() { }); } -var agree_to_tos_url_prompt = null; -var agree_to_tos_url = null; function provision_tls_cert() { // Automatically provision any certs. $('#ssl_provision_p .btn').attr('disabled', '1'); // prevent double-clicks api( "/ssl/provision", "POST", - { - agree_to_tos_url: agree_to_tos_url - }, + { }, function(status) { // Clear last attempt. - agree_to_tos_url = null; $('#ssl_provision_result').text(""); may_reenable_provision_button = true; @@ -225,52 +193,33 @@ function provision_tls_cert() { for (var i = 0; i < status.requests.length; i++) { var r = status.requests[i]; + if (r.result == "skipped") { + // not interested --- this domain wasn't in the table + // to begin with + continue; + } + // create an HTML block to display the results of this request var n = $("

"); $('#ssl_provision_result').append(n); + // plain log line + if (typeof r === "string") { + n.find("p").text(r); + continue; + } + // show a header only to disambiguate request blocks if (status.requests.length > 0) n.find("h4").text(r.domains.join(", ")); - if (r.result == "agree-to-tos") { - // user needs to agree to Let's Encrypt's TOS - agree_to_tos_url_prompt = r.url; - $('#ssl_provision_p .btn').attr('disabled', '1'); - n.find("p").html("Please open and review Let's Encrypt's terms of service agreement. You must agree to their terms for a certificate to be automatically provisioned from them."); - n.append($('')); - - // don't re-enable the Provision button -- user must use the Agree button - may_reenable_provision_button = false; - - } else if (r.result == "error") { + if (r.result == "error") { n.find("p").addClass("text-danger").text(r.message); - } else if (r.result == "wait") { - // Show a button that counts down to zero, at which point it becomes enabled. - n.find("p").text("A certificate is now in the process of being provisioned, but it takes some time. Please wait until the Finish button is enabled, and then click it to acquire the certificate."); - var b = $(''); - b.attr("disabled", "1"); - var now = new Date(); - n.append(b); - function ready_to_finish() { - var remaining = Math.round(r.seconds - (new Date() - now)/1000); - if (remaining > 0) { - setTimeout(ready_to_finish, 1000); - b.text("Finish (" + remaining + "...)") - } else { - b.text("Finish (ready)") - b.removeAttr("disabled"); - } - } - ready_to_finish(); - - // don't re-enable the Provision button -- user must use the Retry button when it becomes enabled - may_reenable_provision_button = false; - } else if (r.result == "installed") { n.find("p").addClass("text-success").text("The TLS certificate was provisioned and installed."); setTimeout("show_tls(true)", 1); // update main table of certificate statuses, call with arg keep_provisioning_shown true so that we don't clear what we just outputted + } // display the detailed log info in case of problems @@ -278,7 +227,6 @@ function provision_tls_cert() { n.append(trace); for (var j = 0; j < r.log.length; j++) trace.append($("
").text(r.log[j])); - } if (may_reenable_provision_button) diff --git a/management/web_update.py b/management/web_update.py index 1bd28e3..61b38a7 100644 --- a/management/web_update.py +++ b/management/web_update.py @@ -201,8 +201,11 @@ def get_web_domains_info(env): # for the SSL config panel, get cert status def check_cert(domain): - tls_cert = get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=True) - if tls_cert is None: return ("danger", "No Certificate Installed") + try: + tls_cert = get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=True) + except OSError: # PRIMARY_HOSTNAME cert is missing + tls_cert = None + if tls_cert is None: return ("danger", "No certificate installed.") cert_status, cert_status_details = check_certificate(domain, tls_cert["certificate"], tls_cert["private-key"]) if cert_status == "OK": return ("success", "Signed & valid. " + cert_status_details) diff --git a/setup/management.sh b/setup/management.sh index 690ccb8..064906d 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -6,6 +6,18 @@ echo "Installing Mail-in-a-Box system management daemon..." # DEPENDENCIES +# We used to install management daemon-related Python packages +# directly to /usr/local/lib. We moved to a virtualenv because +# these packages might conflict with apt-installed packages. +# We may have a lingering version of acme that conflcits with +# certbot, which we're about to install below, so remove it +# first. Once acme is installed by an apt package, this might +# break the package version and `apt-get install --reinstall python3-acme` +# might be needed in that case. +while [ -d /usr/local/lib/python3.4/dist-packages/acme ]; do + pip3 uninstall -y acme; +done + # duplicity is used to make backups of user data. It uses boto # (via Python 2) to do backups to AWS S3. boto from the Ubuntu # package manager is too out-of-date -- it doesn't support the newer @@ -14,7 +26,10 @@ echo "Installing Mail-in-a-Box system management daemon..." # # python-virtualenv is used to isolate the Python 3 packages we # install via pip from the system-installed packages. -apt_install duplicity python-pip python-virtualenv +# +# certbot installs EFF's certbot which we use to +# provision free TLS certificates. +apt_install duplicity python-pip python-virtualenv certbot hide_output pip2 install --upgrade boto # Create a virtualenv for the installation of Python 3 packages @@ -32,13 +47,10 @@ hide_output $venv/bin/pip install --upgrade pip # Install other Python 3 packages used by the management daemon. # The first line is the packages that Josh maintains himself! # NOTE: email_validator is repeated in setup/questions.sh, so please keep the versions synced. -# Force acme to be updated because it seems to need it after the -# pip/setuptools breakage (see above) and the ACME protocol may -# have changed (I got an error on one of my systems). hide_output $venv/bin/pip install --upgrade \ - rtyaml "email_validator>=1.0.0" "free_tls_certificates>=0.1.3" "exclusiveprocess" \ + rtyaml "email_validator>=1.0.0" "exclusiveprocess" \ flask dnspython python-dateutil \ - "idna>=2.0.0" "cryptography==2.2.2" "acme==0.20.0" boto psutil + "idna>=2.0.0" "cryptography==2.2.2" boto psutil # CONFIGURATION diff --git a/setup/migrate.py b/setup/migrate.py index 45f748b..1d5911a 100755 --- a/setup/migrate.py +++ b/setup/migrate.py @@ -137,6 +137,17 @@ def migration_10(env): shutil.move(sslcert, newname) os.rmdir(d) +def migration_11(env): + # Archive the old Let's Encrypt account directory managed by free_tls_certificates + # because we'll use that path now for the directory managed by certbot. + try: + old_path = os.path.join(env["STORAGE_ROOT"], 'ssl', 'lets_encrypt') + new_path = os.path.join(env["STORAGE_ROOT"], 'ssl', 'lets_encrypt-old') + shutil.move(old_path, new_path) + except: + # meh + pass + def get_current_migration(): ver = 0 while True: diff --git a/setup/start.sh b/setup/start.sh index 0409647..86b34c8 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -127,13 +127,21 @@ tools/web_update # fail2ban was first configured, but they should exist now. restart_service fail2ban -# If DNS is already working, try to provision TLS certficates from Let's Encrypt. -# Suppress extra reasons why domains aren't getting a new certificate. -management/ssl_certificates.py -q - # If there aren't any mail users yet, create one. source setup/firstuser.sh +# Register with Let's Encrypt, including agreeing to the Terms of Service. This +# is an interactive command. +if [ ! -d $STORAGE_ROOT/ssl/lets_encrypt/accounts/acme-v01.api.letsencrypt.org/ ]; then +echo +echo "-----------------------------------------------" +echo "Mail-in-a-Box uses Let's Encrypt to provision free certificates" +echo "to enable HTTPS connections to your box. You'll now be asked to agree" +echo "to Let's Encrypt's terms of service." +echo +certbot register --register-unsafely-without-email --config-dir $STORAGE_ROOT/ssl/lets_encrypt +fi + # Done. echo echo "-----------------------------------------------" diff --git a/setup/system.sh b/setup/system.sh index d26d4aa..0472805 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -68,17 +68,10 @@ then fi fi -# ### Add Mail-in-a-Box's PPA. - -# We've built several .deb packages on our own that we want to include. -# One is a replacement for Ubuntu's stock postgrey package that makes -# some enhancements. The other is dovecot-lucene, a Lucene-based full -# text search plugin for (and by) dovecot, which is not available in -# Ubuntu currently. -# -# So, first ensure add-apt-repository is installed, then use it to install -# the [mail-in-a-box ppa](https://launchpad.net/~mail-in-a-box/+archive/ubuntu/ppa). +# ### Add PPAs. +# We install some non-standard Ubuntu packages maintained by us and other +# third-party providers. First ensure add-apt-repository is installed. if [ ! -f /usr/bin/add-apt-repository ]; then echo "Installing add-apt-repository..." @@ -86,11 +79,21 @@ if [ ! -f /usr/bin/add-apt-repository ]; then apt_install software-properties-common fi +# [Main-in-a-Box's own PPA](https://launchpad.net/~mail-in-a-box/+archive/ubuntu/ppa) +# holds several .deb packages that we built on our own. +# One is a replacement for Ubuntu's stock postgrey package that makes +# some enhancements. The other is dovecot-lucene, a Lucene-based full +# text search plugin for (and by) dovecot, which is not available in +# Ubuntu currently. + hide_output add-apt-repository -y ppa:mail-in-a-box/ppa +hide_output add-apt-repository -y ppa:certbot/certbot # ### Update Packages -# Update system packages to make sure we have the latest upstream versions of things from Ubuntu. +# Update system packages to make sure we have the latest upstream versions +# of things from Ubuntu, as well as the directory of packages provide by the +# PPAs so we can install those packages later. echo Updating system packages... hide_output apt-get update