diff --git a/CHANGELOG.md b/CHANGELOG.md index d796970..4f454bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,57 @@ CHANGELOG ========= +Version 60 (October 11, 2022) +----------------------------- + +This is the first release for Ubuntu 22.04. + +**Before upgrading**, you must **first upgrade your existing Ubuntu 18.04 box to Mail-in-a-Box v0.51 or later**, if you haven't already done so. That may not be possible after Ubuntu 18.04 reaches its end of life in April 2023, so please complete the upgrade well before then. (If you are not using Nextcloud's contacts or calendar, you can migrate to the latest version of Mail-in-a-Box from any previous version.) + +For complete upgrade instructions, see: + +https://discourse.mailinabox.email/t/version-60-for-ubuntu-22-04-is-about-to-be-released/9558 + +No major features of Mail-in-a-Box have changed in this release, although some minor fixes were made. + +With the newer version of Ubuntu the following software packages we use are updated: + +* dovecot is upgraded to 2.3.16, postfix to 3.6.4, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug). +* Nextcloud is upgraded to 23.0.4. +* Roundcube is upgraded to 1.6.0. +* certbot is upgraded to 1.21 (via the Ubuntu repository instead of a PPA). +* fail2ban is upgraded to 0.11.2. +* nginx is upgraded to 1.18. +* PHP is upgraded from 7.2 to 8.0. + +Also: + +* Roundcube's login session cookie was tightened. Existing sessions may require a manual logout. +* Moved Postgrey's database under $STORAGE_ROOT. + +Version 57a (June 19, 2022) +--------------------------- + +* The Backblaze backups fix posted in Version 57 was incomplete. It's now fixed. + +Version 57 (June 12, 2022) +-------------------------- + +Setup: + +* Fixed issue upgrading from Mail-in-a-Box v0.40-v0.50 because of a changed URL that Nextcloud is downloaded from. + +Backups: + +* Fixed S3 backups which broke with duplicity 0.8.23. +* Fixed Backblaze backups which broke with latest b2sdk package by rolling back its version. + +Control panel: + +* Fixed spurious changes in system status checks messages by sorting DNSSEC DS records. +* Fixed fail2ban lockout over IPv6 from excessive loads of the system status checks. +* Fixed an incorrect IPv6 system status check message. + Version 56 (January 19, 2022) ----------------------------- diff --git a/conf/fail2ban/jails.conf b/conf/fail2ban/jails.conf index ce957f4..c1514b4 100644 --- a/conf/fail2ban/jails.conf +++ b/conf/fail2ban/jails.conf @@ -5,7 +5,7 @@ # Whitelist our own IP addresses. 127.0.0.1/8 is the default. But our status checks # ping services over the public interface so we should whitelist that address of # ours too. The string is substituted during installation. -ignoreip = 127.0.0.1/8 PUBLIC_IP +ignoreip = 127.0.0.1/8 PUBLIC_IP ::1 PUBLIC_IPV6 [dovecot] enabled = true diff --git a/conf/mailinabox.service b/conf/mailinabox.service index b4cfa6c..c1d98a0 100644 --- a/conf/mailinabox.service +++ b/conf/mailinabox.service @@ -4,6 +4,7 @@ After=multi-user.target [Service] Type=idle +IgnoreSIGPIPE=False ExecStart=/usr/local/lib/mailinabox/start [Install] diff --git a/management/auth.py b/management/auth.py index a9886b7..5fbd409 100644 --- a/management/auth.py +++ b/management/auth.py @@ -31,21 +31,8 @@ class AuthService: def init_system_api_key(self): """Write an API key to a local file so local processes can use the API""" - def create_file_with_mode(path, mode): - # Based on answer by A-B-B: http://stackoverflow.com/a/15015748 - old_umask = os.umask(0) - try: - return os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, mode), - 'w') - finally: - os.umask(old_umask) - - self.key = secrets.token_hex(32) - - os.makedirs(os.path.dirname(self.key_path), exist_ok=True) - - with create_file_with_mode(self.key_path, 0o640) as key_file: - key_file.write(self.key + '\n') + with open(self.key_path, 'r') as file: + self.key = file.read() def authenticate(self, request, env, login_only=False, logout=False): """Test if the HTTP Authorization header's username matches the system key, a session key, diff --git a/management/backup.py b/management/backup.py index dc516d6..c1861f8 100755 --- a/management/backup.py +++ b/management/backup.py @@ -20,24 +20,7 @@ import dateutil.tz import rtyaml from exclusiveprocess import Lock, CannotAcquireLock -from utils import load_environment, shell, wait_for_service, fix_boto, get_php_version, get_os_code - - -def rsync_ssh_options(port=22, direct=False): - # Just in case we pass a string - try: - port = int(port) - except Exception: - port = 22 - - if direct: - return f"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {port} -i /root/.ssh/id_rsa_miab" - else: - return [ - f"--ssh-options= -i /root/.ssh/id_rsa_miab -p {port}", - f"--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {port} -i /root/.ssh/id_rsa_miab\"", - ] - +from utils import load_environment, shell, wait_for_service, get_php_version def backup_status(env): # If backups are disabled, return no status. @@ -87,20 +70,15 @@ def backup_status(env): "volumes": int(keys[2]), } - code, collection_status = shell( - 'check_output', - [ - "/usr/bin/duplicity", - "collection-status", - "--archive-dir", - backup_cache_dir, - "--gpg-options", - "--cipher-algo=AES256", - "--log-fd", - "1", - config["target"], - ] + rsync_ssh_options(port=config["target_rsync_port"]), - get_env(env), + code, collection_status = shell('check_output', [ + "/usr/bin/duplicity", + "collection-status", + "--archive-dir", backup_cache_dir, + "--gpg-options", "--cipher-algo=AES256", + "--log-fd", "1", + get_duplicity_target_url(config), + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env), trap=True) if code != 0: # Command failed. This is likely due to an improperly configured remote @@ -249,8 +227,49 @@ def get_passphrase(env): return passphrase +def get_duplicity_target_url(config): + target = config["target"] -def get_env(env): + if get_target_type(config) == "s3": + from urllib.parse import urlsplit, urlunsplit + target = list(urlsplit(target)) + + # Although we store the S3 hostname in the target URL, + # duplicity no longer accepts it in the target URL. The hostname in + # the target URL must be the bucket name. The hostname is passed + # via get_duplicity_additional_args. Move the first part of the + # path (the bucket name) into the hostname URL component, and leave + # the rest for the path. + target[1], target[2] = target[2].lstrip('/').split('/', 1) + + target = urlunsplit(target) + + return target + +def get_duplicity_additional_args(env): + config = get_backup_config(env) + port = 0 + + try: + port = int(config["target_rsync_port"]) + except Exception: + port = 22 + + if get_target_type(config) == 'rsync': + return [ + f"--ssh-options= -i /root/.ssh/id_rsa_miab -p {port}", + f"--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {port} -i /root/.ssh/id_rsa_miab\"", + ] + elif get_target_type(config) == 's3': + # See note about hostname in get_duplicity_target_url. + from urllib.parse import urlsplit, urlunsplit + target = urlsplit(config["target"]) + endpoint_url = urlunsplit(("https", target.netloc, '', '', '')) + return ["--s3-endpoint-url", endpoint_url] + + return [] + +def get_duplicity_env_vars(env): config = get_backup_config(env) env = {"PASSPHRASE": get_passphrase(env)} @@ -319,6 +338,7 @@ def perform_backup(full_backup, user_initiated=False): service_command(php_fpm, "stop", quit=True) service_command("postfix", "stop", quit=True) service_command("dovecot", "stop", quit=True) + service_command("postgrey", "stop", quit=True) # Execute a pre-backup script that copies files outside the homedir. # Run as the STORAGE_USER user, not as root. Pass our settings in @@ -334,14 +354,21 @@ def perform_backup(full_backup, user_initiated=False): # after the first backup. See #396. try: shell('check_call', [ - "/usr/bin/duplicity", "full" if full_backup else "incr", - "--verbosity", "warning", "--no-print-statistics", "--archive-dir", - backup_cache_dir, "--exclude", backup_root, "--volsize", "250", - "--gpg-options", "--cipher-algo=AES256", env["STORAGE_ROOT"], - config["target"], "--allow-source-mismatch" - ] + rsync_ssh_options(port=config["target_rsync_port"]), get_env(env)) + "/usr/bin/duplicity", + "full" if full_backup else "incr", + "--verbosity", "warning", "--no-print-statistics", + "--archive-dir", backup_cache_dir, + "--exclude", backup_root, + "--volsize", "250", + "--gpg-options", "--cipher-algo=AES256", + env["STORAGE_ROOT"], + get_duplicity_target_url(config), + "--allow-source-mismatch" + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) finally: # Start services again. + service_command("postgrey", "start", quit=False) service_command("dovecot", "start", quit=False) service_command("postfix", "start", quit=False) service_command(php_fpm, "start", quit=False) @@ -349,10 +376,15 @@ def perform_backup(full_backup, user_initiated=False): # Remove old backups. This deletes all backup data no longer needed # from more than 3 days ago. shell('check_call', [ - "/usr/bin/duplicity", "remove-older-than", - "%dD" % config["min_age_in_days"], "--verbosity", "error", - "--archive-dir", backup_cache_dir, "--force", config["target"] - ] + rsync_ssh_options(port=config["target_rsync_port"]), get_env(env)) + "/usr/bin/duplicity", + "remove-older-than", + "%dD" % config["min_age_in_days"], + "--verbosity", "error", + "--archive-dir", backup_cache_dir, + "--force", + get_duplicity_target_url(config) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) # From duplicity's manual: # "This should only be necessary after a duplicity session fails or is @@ -360,9 +392,14 @@ def perform_backup(full_backup, user_initiated=False): # That may be unlikely here but we may as well ensure we tidy up if # that does happen - it might just have been a poorly timed reboot. shell('check_call', [ - "/usr/bin/duplicity", "cleanup", "--verbosity", "error", - "--archive-dir", backup_cache_dir, "--force", config["target"] - ] + rsync_ssh_options(port=config["target_rsync_port"]), get_env(env)) + "/usr/bin/duplicity", + "cleanup", + "--verbosity", "error", + "--archive-dir", backup_cache_dir, + "--force", + get_duplicity_target_url(config) + ] + get_duplicity_additional_args(env), + get_duplicity_env_vars(env)) # Change ownership of backups to the user-data user, so that the after-bcakup # script can access them. @@ -404,14 +441,11 @@ def run_duplicity_verification(): "info", "verify", "--compare-data", - "--archive-dir", - backup_cache_dir, - "--exclude", - backup_root, - config["target"], + "--archive-dir", backup_cache_dir, + "--exclude", backup_root, + get_duplicity_target_url(config), env["STORAGE_ROOT"], - ] + rsync_ssh_options(port=config["target_rsync_port"]), get_env(env)) - + ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) def run_duplicity_restore(args): env = load_environment() @@ -420,12 +454,10 @@ def run_duplicity_restore(args): shell('check_call', [ "/usr/bin/duplicity", "restore", - "--archive-dir", - backup_cache_dir, - config["target"], - ] + rsync_ssh_options(port=config["target_rsync_port"]) + args, - get_env(env)) - + "--archive-dir", backup_cache_dir, + get_duplicity_target_url(config), + ] + get_duplicity_additional_args(env) + args, + get_duplicity_env_vars(env)) def list_target_files(config): import urllib.parse @@ -450,7 +482,7 @@ def list_target_files(config): rsync_command = [ 'rsync', '-e', - rsync_ssh_options(config["target_rsync_port"], direct=True), + f"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {int(config.get('target_rsync_port', 22))} -i /root/.ssh/id_rsa_miab", '--list-only', '-r', rsync_target.format(host=target.netloc, path=target_path) ] @@ -486,28 +518,13 @@ def list_target_files(config): "Connection to rsync host failed: {}".format(reason)) elif target.scheme == "s3": - # match to a Region - fix_boto() # must call prior to importing boto - import boto.s3 - from boto.exception import BotoServerError - custom_region = False - for region in boto.s3.regions(): - if region.endpoint == target.hostname: - break - else: - # If region is not found this is a custom region - custom_region = True + import boto3.s3 + from botocore.exceptions import ClientError + # separate bucket from path in target bucket = target.path[1:].split('/')[0] path = '/'.join(target.path[1:].split('/')[1:]) + '/' - # Create a custom region with custom endpoint - if custom_region: - from boto.s3.connection import S3Connection - region = boto.s3.S3RegionInfo(name=bucket, - endpoint=target.hostname, - connection_cls=S3Connection) - # If no prefix is specified, set the path to '', otherwise boto won't list the files if path == '/': path = '' @@ -517,20 +534,15 @@ def list_target_files(config): # connect to the region & bucket try: - conn = region.connect(aws_access_key_id=config["target_user"], - aws_secret_access_key=config["target_pass"]) - bucket = conn.get_bucket(bucket) - except BotoServerError as e: - if e.status == 403: - raise ValueError("Invalid S3 access key or secret access key.") - elif e.status == 404: - raise ValueError("Invalid S3 bucket name.") - elif e.status == 301: - raise ValueError("Incorrect region for this bucket.") - raise ValueError(e.reason) - - return [(key.name[len(path):], key.size) - for key in bucket.list(prefix=path)] + s3 = boto3.client('s3', \ + endpoint_url=f'https://{target.hostname}', \ + aws_access_key_id=config['target_user'], \ + aws_secret_access_key=config['target_pass']) + bucket_objects = s3.list_objects_v2(Bucket=bucket, Prefix=path).get("Contents", []) + backup_list = [(key['Key'][len(path):], key['Size']) for key in bucket_objects] + except ClientError as e: + raise ValueError(e) + return backup_list elif target.scheme == 'b2': InMemoryAccountInfo = None B2Api = None @@ -562,8 +574,7 @@ def list_target_files(config): raise ValueError(config["target"]) -def backup_set_custom(env, target, target_user, target_pass, target_rsync_port, - min_age): +def backup_set_custom(env, target, target_user, target_pass, target_rsync_port, min_age): config = get_backup_config(env, for_save=True) # min_age must be an int diff --git a/management/daemon.py b/management/daemon.py index 0a49f11..936cca7 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -147,9 +147,9 @@ def index(): no_users_exist = (len(get_mail_users(env)) == 0) no_admins_exist = (len(get_admins(env)) == 0) - utils.fix_boto() # must call prior to importing boto - import boto.s3 - backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()] + import boto3.s3 + backup_s3_hosts = [(r, f"s3.{r}.amazonaws.com") for r in boto3.session.Session().get_available_regions('s3')] + return render_template( 'index.html', @@ -730,6 +730,8 @@ def system_status(): # Create a temporary pool of processes for the status checks with multiprocessing.pool.Pool(processes=5) as pool: run_checks(False, env, output, pool) + pool.close() + pool.join() return json_response(output.items) diff --git a/management/dns_update.py b/management/dns_update.py index 18bbed0..e20c21d 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -114,9 +114,9 @@ def do_dns_update(env, force=False): if len(updated_domains) == 0: updated_domains.append("DNS configuration") - # Kick nsd if anything changed. + # Tell nsd to reload changed zone files. if len(updated_domains) > 0: - shell('check_call', ["/usr/sbin/service", "nsd", "restart"]) + shell('check_call', ["/usr/sbin/nsd-control", "reload"]) # Write the OpenDKIM configuration tables for all of the mail domains. from mailconfig import get_mail_domains @@ -1299,13 +1299,9 @@ def get_secondary_dns(custom_dns, mode=None): # doesn't. if not hostname.startswith("xfr:"): if mode == "xfr": - response = dns.resolver.resolve(hostname + '.', - "A", - raise_on_no_answer=False) + response = dns.resolver.resolve(hostname+'.', "A", raise_on_no_answer=False) values.extend(map(str, response)) - response = dns.resolver.resolve(hostname + '.', - "AAAA", - raise_on_no_answer=False) + response = dns.resolver.resolve(hostname+'.', "AAAA", raise_on_no_answer=False) values.extend(map(str, response)) continue values.append(hostname) @@ -1330,14 +1326,11 @@ def set_secondary_dns(hostnames, env): # Resolve hostname. try: response = resolver.resolve(item, "A") - except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, - dns.resolver.NoAnswer): + except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): try: - response = resolver.query(item, "AAAA") - except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, - dns.resolver.NoAnswer): - raise ValueError( - "Could not resolve the IP address of %s." % item) + response = resolver.resolve(item, "AAAA") + except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + raise ValueError("Could not resolve the IP address of %s." % item) else: # Validate IP address. try: diff --git a/management/ssl_certificates.py b/management/ssl_certificates.py index 56a19bc..1f3d0ea 100755 --- a/management/ssl_certificates.py +++ b/management/ssl_certificates.py @@ -64,37 +64,33 @@ def get_ssl_certificates(env): # Not a valid PEM format for a PEM type we care about. continue - # Remember where we got this object. - pem._filename = fn - # Is it a private key? if isinstance(pem, RSAPrivateKey): - private_keys[pem.public_key().public_numbers()] = pem + private_keys[pem.public_key().public_numbers()] = { "filename": fn, "key": pem } # Is it a certificate? if isinstance(pem, Certificate): - certificates.append(pem) + certificates.append({ "filename": fn, "cert": pem }) # Process the certificates. domains = {} for cert in certificates: # What domains is this certificate good for? - cert_domains, primary_domain = get_certificate_domains(cert) - cert._primary_domain = primary_domain + cert_domains, primary_domain = get_certificate_domains(cert["cert"]) + cert["primary_domain"] = primary_domain # Is there a private key file for this certificate? - private_key = private_keys.get(cert.public_key().public_numbers()) + private_key = private_keys.get(cert["cert"].public_key().public_numbers()) if not private_key: continue - cert._private_key = private_key + cert["private_key"] = private_key # Add this cert to the list of certs usable for the domains. for domain in cert_domains: # The primary hostname can only use a certificate mapped # to the system private key. if domain == env['PRIMARY_HOSTNAME']: - if cert._private_key._filename != os.path.join( - env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'): + if cert["private_key"]["filename"] != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'): continue domains.setdefault(domain, []).append(cert) @@ -105,13 +101,12 @@ def get_ssl_certificates(env): ret = {} for domain, cert_list in domains.items(): #for c in cert_list: print(domain, c.not_valid_before, c.not_valid_after, "("+str(now)+")", c.issuer, c.subject, c._filename) - cert_list.sort( - key=lambda cert: ( - # must be valid NOW - cert.not_valid_before <= now <= cert.not_valid_after, + cert_list.sort(key = lambda cert : ( + # must be valid NOW + cert["cert"].not_valid_before <= now <= cert["cert"].not_valid_after, - # prefer one that is not self-signed - cert.issuer != cert.subject, + # prefer one that is not self-signed + cert["cert"].issuer != cert["cert"].subject, ########################################################### # The above lines ensure that valid certificates are chosen @@ -119,9 +114,9 @@ def get_ssl_certificates(env): # multiple valid certificates available for this domain. ########################################################### - # prefer one with the expiration furthest into the future so - # that we can easily rotate to new certs as we get them - cert.not_valid_after, + # prefer one with the expiration furthest into the future so + # that we can easily rotate to new certs as we get them + cert["cert"].not_valid_after, ########################################################### # We always choose the certificate that is good for the @@ -134,18 +129,18 @@ def get_ssl_certificates(env): # domain. ########################################################### - # in case a certificate is installed in multiple paths, - # prefer the... lexicographically last one? - cert._filename, - ), - reverse=True) + # in case a certificate is installed in multiple paths, + # prefer the... lexicographically last one? + cert["filename"], + + ), reverse=True) cert = cert_list.pop(0) ret[domain] = { - "private-key": cert._private_key._filename, - "certificate": cert._filename, - "primary-domain": cert._primary_domain, - "certificate_object": cert, - } + "private-key": cert["private_key"]["filename"], + "certificate": cert["filename"], + "primary-domain": cert["primary_domain"], + "certificate_object": cert["cert"], + } return ret diff --git a/management/status_checks.py b/management/status_checks.py index 892740a..bec1895 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -228,9 +228,7 @@ def check_service(i, service, env): # IPv4 ok but IPv6 failed. Try the PRIVATE_IPV6 address to see if the service is bound to the interface. elif service["port"] != 53 and try_connect(env["PRIVATE_IPV6"]): - output.print_error( - "%s is running (and available over IPv4 and the local IPv6 address), but it is not publicly accessible at %s:%d." - % (service['name'], env['PUBLIC_IP'], service['port'])) + output.print_error("%s is running (and available over IPv4 and the local IPv6 address), but it is not publicly accessible at %s:%d." % (service['name'], env['PUBLIC_IPV6'], service['port'])) else: output.print_error( "%s is running and available over IPv4 but is not accessible over IPv6 at %s port %d." @@ -383,6 +381,17 @@ def check_free_disk_space(rounded_values, env, output): disk_msg = "The disk has less than 15% free space." output.print_error(disk_msg) + # Check that there's only one duplicity cache. If there's more than one, + # it's probably no longer in use, and we can recommend clearing the cache + # to save space. The cache directory may not exist yet, which is OK. + backup_cache_path = os.path.join(env['STORAGE_ROOT'], 'backup/cache') + try: + backup_cache_count = len(os.listdir(backup_cache_path)) + except: + backup_cache_count = 0 + if backup_cache_count > 1: + output.print_warning("The backup cache directory {} has more than one backup target cache. Consider clearing this directory to save disk space." + .format(backup_cache_path)) def check_free_memory(rounded_values, env, output): # Check free memory. @@ -1113,11 +1122,8 @@ def check_dnssec(domain, if len(ds) > 0: output.print_line("") output.print_line("The DS record is currently set to:") - for rr in ds: - output.print_line( - "Key Tag: {0}, Algorithm: {1}, Digest Type: {2}, Digest: {3}". - format(*rr)) - + for rr in sorted(ds): + output.print_line("Key Tag: {0}, Algorithm: {1}, Digest Type: {2}, Digest: {3}".format(*rr)) def check_mail_domain(domain, env, output): # Check the MX record. @@ -1169,7 +1175,7 @@ def check_mail_domain(domain, env, output): output.print_ok(good_news) # Check MTA-STS policy. - loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() sts_resolver = postfix_mta_sts_resolver.resolver.STSResolver(loop=loop) valid, policy = loop.run_until_complete(sts_resolver.resolve(domain)) if valid == postfix_mta_sts_resolver.resolver.STSFetchResult.VALID: @@ -1269,8 +1275,7 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False): # Do the query. try: response = resolver.resolve(qname, rtype) - except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, - dns.resolver.NoAnswer): + except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): # Host did not have an answer for this query; not sure what the # difference is between the two exceptions. return nxdomain diff --git a/management/utils.py b/management/utils.py index 0956a09..c648730 100644 --- a/management/utils.py +++ b/management/utils.py @@ -204,15 +204,6 @@ def wait_for_service(port, public, env, timeout): return False time.sleep(min(timeout / 4, 1)) - -def fix_boto(): - # Google Compute Engine instances install some Python-2-only boto plugins that - # conflict with boto running under Python 3. Disable boto's default configuration - # file prior to importing boto so that GCE's plugin is not loaded: - import os - os.environ["BOTO_CONFIG"] = "/etc/boto3.cfg" - - def get_php_version(): # Gets the version of PHP installed in the system. return shell("check_output", ["/usr/bin/php", "-v"])[4:7] @@ -235,7 +226,6 @@ def get_os_code(): return None - if __name__ == "__main__": from web_update import get_web_domains env = load_environment() diff --git a/management/wsgi.py b/management/wsgi.py new file mode 100644 index 0000000..86cf3af --- /dev/null +++ b/management/wsgi.py @@ -0,0 +1,7 @@ +from daemon import app +import auth, utils + +app.logger.addHandler(utils.create_syslog_handler()) + +if __name__ == "__main__": + app.run(port=10222) \ No newline at end of file diff --git a/setup/dns.sh b/setup/dns.sh index 98226ff..6d42a89 100755 --- a/setup/dns.sh +++ b/setup/dns.sh @@ -10,16 +10,9 @@ source setup/functions.sh # load our functions source /etc/mailinabox.conf # load global vars -# Install the packages. -# -# * nsd: The non-recursive nameserver that publishes our DNS records. -# * ldnsutils: Helper utilities for signing DNSSEC zones. -# * openssh-client: Provides ssh-keyscan which we use to create SSHFP records. -echo "Installing nsd (DNS server)..." -apt_install ldnsutils openssh-client - # Prepare nsd's configuration. - +# We configure nsd before installation as we only want it to bind to some addresses +# and it otherwise will have port / bind conflicts with bind9 used as the local resolver mkdir -p /var/run/nsd mkdir -p /etc/nsd mkdir -p /etc/nsd/zones @@ -46,18 +39,6 @@ server: EOF -# Add log rotation -cat > /etc/logrotate.d/nsd <> /etc/nsd/nsd.conf; # now be stored in /etc/nsd/nsd.conf.d. rm -f /etc/nsd/zones.conf -# Attempting a late install of nsd (after configuration) -apt_install nsd +# Add log rotation +cat > /etc/logrotate.d/nsd < /etc/cron.daily/mailinabox-postgrey-whitelist << EOF; #!/bin/bash diff --git a/setup/management.sh b/setup/management.sh index c1b34e1..c1e33c5 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -1,23 +1,12 @@ #!/bin/bash source setup/functions.sh +source /etc/mailinabox.conf # load global vars echo "Installing Mail-in-a-Box system management daemon..." # DEPENDENCIES -# We used to install management daemon-related Python packages -# directly to /usr/local/lib. We moved to a virtualenv because -# these packages might conflict with apt-installed packages. -# We may have a lingering version of acme that conflcits with -# certbot, which we're about to install below, so remove it -# first. Once acme is installed by an apt package, this might -# break the package version and `apt-get install --reinstall python3-acme` -# might be needed in that case. -while [ -d /usr/local/lib/python3.4/dist-packages/acme ]; do - pip3 uninstall -y acme; -done - # duplicity is used to make backups of user data. # # virtualenv is used to isolate the Python 3 packages we @@ -27,9 +16,10 @@ done # provision free TLS certificates. apt_install duplicity python3-pip python3-gpg virtualenv certbot rsync -# boto is used for amazon aws backups. +# b2sdk is used for backblaze backups. +# boto3 is used for amazon aws backups. # Both are installed outside the pipenv, so they can be used by duplicity -hide_output pip3 install --upgrade boto +hide_output pip3 install --upgrade boto3 # Create a virtualenv for the installation of Python 3 packages # used by the management daemon. @@ -57,7 +47,7 @@ hide_output $venv/bin/pip install --upgrade pip # NOTE: email_validator is repeated in setup/questions.sh, so please keep the versions synced. hide_output $venv/bin/pip install --upgrade \ rtyaml "email_validator>=1.0.0" "exclusiveprocess" \ - flask dnspython python-dateutil expiringdict \ + flask dnspython python-dateutil expiringdict gunicorn \ qrcode[pil] pyotp \ "idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver @@ -129,6 +119,9 @@ rm -f /tmp/fontawesome.zip # Create an init script to start the management daemon and keep it # running after a reboot. +# Set a long timeout since some commands take a while to run, matching +# the timeout we set for PHP (fastcgi_read_timeout in the nginx confs). +# Note: Authentication currently breaks with more than 1 gunicorn worker. cat > $inst_dir/start < /var/lib/mailinabox/api.key +chmod 640 /var/lib/mailinabox/api.key + source $venv/bin/activate -exec python $(pwd)/management/daemon.py +export PYTHONPATH=$(pwd)/management +exec gunicorn -b localhost:10222 -w 1 --timeout 630 wsgi:app EOF chmod +x $inst_dir/start cp --remove-destination conf/mailinabox.service /lib/systemd/system/mailinabox.service # target was previously a symlink so remove it first diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 2e22689..06dcd7a 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -51,6 +51,11 @@ apt_install php php-fpm \ php-intl php-imagick php-gmp php-bcmath phpenmod apcu + +management/editconf.py /etc/php/$(php_version)/mods-available/apcu.ini -c ';' \ + apc.enabled=1 \ + apc.enable_cli=1 + management/editconf.py /etc/php/$(php_version)/cli/php.ini -c ';' \ apc.enable_cli=1 @@ -345,6 +350,9 @@ CONFIG_TEMP=$(/bin/mktemp) php < $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/config.php; /etc/cron.d/mailinabox-nextcloud << EOF; @@ -426,9 +434,6 @@ cat > /etc/cron.d/mailinabox-nextcloud << EOF; EOF chmod +x /etc/cron.d/mailinabox-nextcloud -# Remove previous hourly cronjob -rm -f /etc/cron.hourly/mailinabox-owncloud - # There's nothing much of interest that a user could do as an admin for Nextcloud, # and there's a lot they could mess up, so we don't make any users admins of Nextcloud. # But if we wanted to, we would do this: diff --git a/setup/start.sh b/setup/start.sh index 38ab472..cf5d041 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -72,6 +72,10 @@ fi fi # Create the STORAGE_USER and STORAGE_ROOT directory if they don't already exist. +# +# Set the directory and all of its parent directories' permissions to world +# readable since it holds files owned by different processes. +# # If the STORAGE_ROOT is missing the mailinabox.version file that lists a # migration (schema) number for the files stored there, assume this is a fresh # installation to that directory and write the file to contain the current @@ -82,6 +86,8 @@ fi if [ ! -d $STORAGE_ROOT ]; then mkdir -p $STORAGE_ROOT fi +f=$STORAGE_ROOT +while [[ $f != / ]]; do chmod a+rx "$f"; f=$(dirname "$f"); done; if [ ! -f $STORAGE_ROOT/mailinabox.version ]; then setup/migrate.py --current > $STORAGE_ROOT/mailinabox.version chown $STORAGE_USER.$STORAGE_USER $STORAGE_ROOT/mailinabox.version diff --git a/setup/system.sh b/setup/system.sh index 9d57385..584fddf 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -102,9 +102,6 @@ apt_get_quiet autoremove # Install basic utilities. # -# * haveged: Provides extra entropy to /dev/random so it doesn't stall -# when generating random numbers for private keys (e.g. during -# ldns-keygen). # * unattended-upgrades: Apt tool to install security updates automatically. # * cron: Runs background processes periodically. # * ntp: keeps the system time correct @@ -118,8 +115,8 @@ apt_get_quiet autoremove echo Installing system packages... apt_install python3 python3-dev python3-pip python3-setuptools \ - netcat-openbsd wget curl git sudo coreutils bc \ - haveged pollinate openssh-client unzip \ + netcat-openbsd wget curl git sudo coreutils bc file \ + pollinate openssh-client unzip \ unattended-upgrades cron ntp fail2ban rsyslog # ### Suppress Upgrade Prompts @@ -354,6 +351,7 @@ systemctl restart systemd-resolved rm -f /etc/fail2ban/jail.local # we used to use this file but don't anymore rm -f /etc/fail2ban/jail.d/defaults-debian.conf # removes default config so we can manage all of fail2ban rules in one config cat conf/fail2ban/jails.conf \ + | sed "s/PUBLIC_IPV6/$PUBLIC_IPV6/g" \ | sed "s/PUBLIC_IP/$PUBLIC_IP/g" \ | sed "s#STORAGE_ROOT#$STORAGE_ROOT#" \ > /etc/fail2ban/jail.d/mailinabox.conf diff --git a/setup/webmail.sh b/setup/webmail.sh index ef8f28e..3071f4b 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -83,7 +83,7 @@ if [ $needs_update == 1 ]; then # download and verify the full release of the carddav plugin wget_verify \ - https://github.com/blind-coder/rcmcarddav/releases/download/v${CARDDAV_VERSION}/carddav-v${CARDDAV_VERSION}.tar.gz \ + https://github.com/mstilkerich/rcmcarddav/releases/download/v${CARDDAV_VERSION}/carddav-v${CARDDAV_VERSION}.tar.gz \ $CARDDAV_HASH \ /tmp/carddav.tar.gz @@ -140,6 +140,11 @@ cat > $RCM_CONFIG < ${RCM_PLUGIN_DIR}/carddav/config.inc.php < 'ownCloud', 'username' => '%u', // login username 'password' => '%p', // login password - 'url' => 'https://${PRIMARY_HOSTNAME}/cloud/remote.php/carddav/addressbooks/%u/contacts', + 'url' => 'https://${PRIMARY_HOSTNAME}/cloud/remote.php/dav/addressbooks/users/%u/contacts/', 'active' => true, 'readonly' => false, 'refresh_time' => '02:00:00', @@ -230,7 +235,7 @@ chown -f -R root.www-data ${RCM_PLUGIN_DIR}/carddav chmod -R 774 ${RCM_PLUGIN_DIR}/carddav # Run Roundcube database migration script (database is created if it does not exist) -${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube +php ${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube chown www-data:www-data $STORAGE_ROOT/mail/roundcube/roundcube.sqlite chmod 664 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite diff --git a/setup/zpush.sh b/setup/zpush.sh index 860c5b7..94c7158 100755 --- a/setup/zpush.sh +++ b/setup/zpush.sh @@ -17,7 +17,7 @@ source /etc/mailinabox.conf # load global vars echo "Installing Z-Push (Exchange/ActiveSync server)..." apt_install \ - php-soap php-imap libawl-php php-xsl + php-soap php-imap libawl-php php-xml phpenmod -v php imap @@ -42,8 +42,6 @@ if [ $needs_update == 1 ]; then rm -rf /tmp/z-push.zip /tmp/z-push rm -f /usr/sbin/z-push-{admin,top} - ln -s /usr/local/lib/z-push/z-push-admin.php /usr/sbin/z-push-admin - ln -s /usr/local/lib/z-push/z-push-top.php /usr/sbin/z-push-top echo $VERSION > /usr/local/lib/z-push/version fi @@ -106,4 +104,4 @@ restart_service php$(php_version)-fpm # Fix states after upgrade -hide_output z-push-admin -a fixstates +hide_output php /usr/local/lib/z-push/z-push-admin.php -a fixstates diff --git a/tests/test_dns.py b/tests/test_dns.py index c5fe805..25c64bf 100755 --- a/tests/test_dns.py +++ b/tests/test_dns.py @@ -30,7 +30,7 @@ def test(server, description): (hostname, "TXT", "\"v=spf1 mx -all\""), ("mail._domainkey." + hostname, "TXT", "\"v=DKIM1; k=rsa; s=email; \" \"p=__KEY__\""), #("_adsp._domainkey." + hostname, "TXT", "\"dkim=all\""), - ("_dmarc." + hostname, "TXT", "\"v=DMARC1; p=quarantine\""), + ("_dmarc." + hostname, "TXT", "\"v=DMARC1; p=quarantine;\""), ] return test2(tests, server, description) @@ -48,7 +48,7 @@ def test2(tests, server, description): for qname, rtype, expected_answer in tests: # do the query and format the result as a string try: - response = dns.resolver.query(qname, rtype) + response = dns.resolver.resolve(qname, rtype) except dns.resolver.NoNameservers: # host did not have an answer for this query print("Could not connect to %s for DNS query." % server) diff --git a/tests/test_mail.py b/tests/test_mail.py index 8c8838a..312f333 100755 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -48,7 +48,7 @@ server = smtplib.SMTP_SSL(host) ipaddr = socket.gethostbyname(host) # IPv4 only! reverse_ip = dns.reversename.from_address(ipaddr) # e.g. "1.0.0.127.in-addr.arpa." try: - reverse_dns = dns.resolver.query(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname + reverse_dns = dns.resolver.resolve(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname except dns.resolver.NXDOMAIN: print("Reverse DNS lookup failed for %s. SMTP EHLO name check skipped." % ipaddr) reverse_dns = None