Import changes from upstream v60

CHANGELOG.md

@@ -1,6 +1,57 @@
 CHANGELOG
 =========
 
+Version 60 (October 11, 2022)
+-----------------------------
+
+This is the first release for Ubuntu 22.04.
+
+**Before upgrading**, you must **first upgrade your existing Ubuntu 18.04 box to Mail-in-a-Box v0.51 or later**, if you haven't already done so. That may not be possible after Ubuntu 18.04 reaches its end of life in April 2023, so please complete the upgrade well before then. (If you are not using Nextcloud's contacts or calendar, you can migrate to the latest version of Mail-in-a-Box from any previous version.)
+
+For complete upgrade instructions, see:
+
+https://discourse.mailinabox.email/t/version-60-for-ubuntu-22-04-is-about-to-be-released/9558
+
+No major features of Mail-in-a-Box have changed in this release, although some minor fixes were made.
+
+With the newer version of Ubuntu the following software packages we use are updated:
+
+* dovecot is upgraded to 2.3.16, postfix to 3.6.4, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug).
+* Nextcloud is upgraded to 23.0.4.
+* Roundcube is upgraded to 1.6.0.
+* certbot is upgraded to 1.21 (via the Ubuntu repository instead of a PPA).
+* fail2ban is upgraded to 0.11.2.
+* nginx is upgraded to 1.18.
+* PHP is upgraded from 7.2 to 8.0.
+
+Also:
+
+* Roundcube's login session cookie was tightened. Existing sessions may require a manual logout.
+* Moved Postgrey's database under $STORAGE_ROOT.
+
+Version 57a (June 19, 2022)
+---------------------------
+
+* The Backblaze backups fix posted in Version 57 was incomplete. It's now fixed.
+
+Version 57 (June 12, 2022)
+--------------------------
+
+Setup:
+
+* Fixed issue upgrading from Mail-in-a-Box v0.40-v0.50 because of a changed URL that Nextcloud is downloaded from.
+
+Backups:
+
+* Fixed S3 backups which broke with duplicity 0.8.23.
+* Fixed Backblaze backups which broke with latest b2sdk package by rolling back its version.
+
+Control panel:
+
+* Fixed spurious changes in system status checks messages by sorting DNSSEC DS records.
+* Fixed fail2ban lockout over IPv6 from excessive loads of the system status checks.
+* Fixed an incorrect IPv6 system status check message.
+
 Version 56 (January 19, 2022)
 -----------------------------
 

conf/fail2ban/jails.conf

@@ -5,7 +5,7 @@
 # Whitelist our own IP addresses. 127.0.0.1/8 is the default. But our status checks
 # ping services over the public interface so we should whitelist that address of
 # ours too. The string is substituted during installation.
-ignoreip = 127.0.0.1/8 PUBLIC_IP
+ignoreip = 127.0.0.1/8 PUBLIC_IP ::1 PUBLIC_IPV6
 
 [dovecot]
 enabled = true

conf/mailinabox.service

@@ -4,6 +4,7 @@ After=multi-user.target
 
 [Service]
 Type=idle
+IgnoreSIGPIPE=False
 ExecStart=/usr/local/lib/mailinabox/start
 
 [Install]

management/auth.py

@@ -31,21 +31,8 @@ class AuthService:
 	def init_system_api_key(self):
 		"""Write an API key to a local file so local processes can use the API"""
 
-		def create_file_with_mode(path, mode):
+		with open(self.key_path, 'r') as file:
-			# Based on answer by A-B-B: http://stackoverflow.com/a/15015748
+			self.key = file.read()
-			old_umask = os.umask(0)
-			try:
-				return os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, mode),
-								'w')
-			finally:
-				os.umask(old_umask)
-
-		self.key = secrets.token_hex(32)
-
-		os.makedirs(os.path.dirname(self.key_path), exist_ok=True)
-
-		with create_file_with_mode(self.key_path, 0o640) as key_file:
-			key_file.write(self.key + '\n')
 
 	def authenticate(self, request, env, login_only=False, logout=False):
 		"""Test if the HTTP Authorization header's username matches the system key, a session key,

management/backup.py

@@ -20,24 +20,7 @@ import dateutil.tz
 import rtyaml
 from exclusiveprocess import Lock, CannotAcquireLock
 
-from utils import load_environment, shell, wait_for_service, fix_boto, get_php_version, get_os_code
+from utils import load_environment, shell, wait_for_service, get_php_version
-
-
-def rsync_ssh_options(port=22, direct=False):
-	# Just in case we pass a string
-	try:
-		port = int(port)
-	except Exception:
-		port = 22
-
-	if direct:
-		return f"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {port} -i /root/.ssh/id_rsa_miab"
-	else:
-		return [
-			f"--ssh-options= -i /root/.ssh/id_rsa_miab -p {port}",
-			f"--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {port} -i /root/.ssh/id_rsa_miab\"",
-		]
-
 
 def backup_status(env):
 	# If backups are disabled, return no status.
@@ -87,20 +70,15 @@ def backup_status(env):
 			"volumes": int(keys[2]),
 		}
 
-	code, collection_status = shell(
+	code, collection_status = shell('check_output', [
-		'check_output',
+		"/usr/bin/duplicity",
-		[
+		"collection-status",
-			"/usr/bin/duplicity",
+		"--archive-dir", backup_cache_dir,
-			"collection-status",
+		"--gpg-options", "--cipher-algo=AES256",
-			"--archive-dir",
+		"--log-fd", "1",
-			backup_cache_dir,
+		get_duplicity_target_url(config),
-			"--gpg-options",
+		] + get_duplicity_additional_args(env),
-			"--cipher-algo=AES256",
+		get_duplicity_env_vars(env),
-			"--log-fd",
-			"1",
-			config["target"],
-		] + rsync_ssh_options(port=config["target_rsync_port"]),
-		get_env(env),
 		trap=True)
 	if code != 0:
 		# Command failed. This is likely due to an improperly configured remote
@@ -249,8 +227,49 @@ def get_passphrase(env):
 
 	return passphrase
 
+def get_duplicity_target_url(config):
+	target = config["target"]
 
-def get_env(env):
+	if get_target_type(config) == "s3":
+		from urllib.parse import urlsplit, urlunsplit
+		target = list(urlsplit(target))
+
+		# Although we store the S3 hostname in the target URL,
+		# duplicity no longer accepts it in the target URL. The hostname in
+		# the target URL must be the bucket name. The hostname is passed
+		# via get_duplicity_additional_args. Move the first part of the
+		# path (the bucket name) into the hostname URL component, and leave
+		# the rest for the path.
+		target[1], target[2] = target[2].lstrip('/').split('/', 1)
+
+		target = urlunsplit(target)
+
+	return target
+
+def get_duplicity_additional_args(env):
+	config = get_backup_config(env)
+	port = 0
+
+	try:
+		port = int(config["target_rsync_port"])
+	except Exception:
+		port = 22
+
+	if get_target_type(config) == 'rsync':
+		return [
+			f"--ssh-options= -i /root/.ssh/id_rsa_miab -p {port}",
+			f"--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {port} -i /root/.ssh/id_rsa_miab\"",
+		]
+	elif get_target_type(config) == 's3':
+		# See note about hostname in get_duplicity_target_url.
+		from urllib.parse import urlsplit, urlunsplit
+		target = urlsplit(config["target"])
+		endpoint_url = urlunsplit(("https", target.netloc, '', '', ''))
+		return ["--s3-endpoint-url",  endpoint_url]
+
+	return []
+
+def get_duplicity_env_vars(env):
 	config = get_backup_config(env)
 
 	env = {"PASSPHRASE": get_passphrase(env)}
@@ -319,6 +338,7 @@ def perform_backup(full_backup, user_initiated=False):
 	service_command(php_fpm, "stop", quit=True)
 	service_command("postfix", "stop", quit=True)
 	service_command("dovecot", "stop", quit=True)
+	service_command("postgrey", "stop", quit=True)
 
 	# Execute a pre-backup script that copies files outside the homedir.
 	# Run as the STORAGE_USER user, not as root. Pass our settings in
@@ -334,14 +354,21 @@ def perform_backup(full_backup, user_initiated=False):
 	# after the first backup. See #396.
 	try:
 		shell('check_call', [
-			"/usr/bin/duplicity", "full" if full_backup else "incr",
+			"/usr/bin/duplicity",
-			"--verbosity", "warning", "--no-print-statistics", "--archive-dir",
+			"full" if full_backup else "incr",
-			backup_cache_dir, "--exclude", backup_root, "--volsize", "250",
+			"--verbosity", "warning", "--no-print-statistics",
-			"--gpg-options", "--cipher-algo=AES256", env["STORAGE_ROOT"],
+			"--archive-dir", backup_cache_dir,
-			config["target"], "--allow-source-mismatch"
+			"--exclude", backup_root,
-		] + rsync_ssh_options(port=config["target_rsync_port"]), get_env(env))
+			"--volsize", "250",
+			"--gpg-options", "--cipher-algo=AES256",
+			env["STORAGE_ROOT"],
+			get_duplicity_target_url(config),
+			"--allow-source-mismatch"
+			] + get_duplicity_additional_args(env),
+			get_duplicity_env_vars(env))
 	finally:
 		# Start services again.
+		service_command("postgrey", "start", quit=False)
 		service_command("dovecot", "start", quit=False)
 		service_command("postfix", "start", quit=False)
 		service_command(php_fpm, "start", quit=False)
@@ -349,10 +376,15 @@ def perform_backup(full_backup, user_initiated=False):
 	# Remove old backups. This deletes all backup data no longer needed
 	# from more than 3 days ago.
 	shell('check_call', [
-		"/usr/bin/duplicity", "remove-older-than",
+		"/usr/bin/duplicity",
-		"%dD" % config["min_age_in_days"], "--verbosity", "error",
+		"remove-older-than",
-		"--archive-dir", backup_cache_dir, "--force", config["target"]
+		"%dD" % config["min_age_in_days"],
-	] + rsync_ssh_options(port=config["target_rsync_port"]), get_env(env))
+		"--verbosity", "error",
+		"--archive-dir", backup_cache_dir,
+		"--force",
+		get_duplicity_target_url(config)
+		] + get_duplicity_additional_args(env),
+		get_duplicity_env_vars(env))
 
 	# From duplicity's manual:
 	# "This should only be necessary after a duplicity session fails or is
@@ -360,9 +392,14 @@ def perform_backup(full_backup, user_initiated=False):
 	# That may be unlikely here but we may as well ensure we tidy up if
 	# that does happen - it might just have been a poorly timed reboot.
 	shell('check_call', [
-		"/usr/bin/duplicity", "cleanup", "--verbosity", "error",
+		"/usr/bin/duplicity",
-		"--archive-dir", backup_cache_dir, "--force", config["target"]
+		"cleanup",
-	] + rsync_ssh_options(port=config["target_rsync_port"]), get_env(env))
+		"--verbosity", "error",
+		"--archive-dir", backup_cache_dir,
+		"--force",
+		get_duplicity_target_url(config)
+		] + get_duplicity_additional_args(env),
+		get_duplicity_env_vars(env))
 
 	# Change ownership of backups to the user-data user, so that the after-bcakup
 	# script can access them.
@@ -404,14 +441,11 @@ def run_duplicity_verification():
 		"info",
 		"verify",
 		"--compare-data",
-		"--archive-dir",
+		"--archive-dir", backup_cache_dir,
-		backup_cache_dir,
+		"--exclude", backup_root,
-		"--exclude",
+		get_duplicity_target_url(config),
-		backup_root,
-		config["target"],
 		env["STORAGE_ROOT"],
-	] + rsync_ssh_options(port=config["target_rsync_port"]), get_env(env))
+	] + get_duplicity_additional_args(env), get_duplicity_env_vars(env))
-
 
 def run_duplicity_restore(args):
 	env = load_environment()
@@ -420,12 +454,10 @@ def run_duplicity_restore(args):
 	shell('check_call', [
 		"/usr/bin/duplicity",
 		"restore",
-		"--archive-dir",
+		"--archive-dir", backup_cache_dir,
-		backup_cache_dir,
+		get_duplicity_target_url(config),
-		config["target"],
+		] + get_duplicity_additional_args(env) + args,
-	] + rsync_ssh_options(port=config["target_rsync_port"]) + args,
+	get_duplicity_env_vars(env))
-		get_env(env))
-
 
 def list_target_files(config):
 	import urllib.parse
@@ -450,7 +482,7 @@ def list_target_files(config):
 
 		rsync_command = [
 			'rsync', '-e',
-			rsync_ssh_options(config["target_rsync_port"], direct=True),
+			f"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {int(config.get('target_rsync_port', 22))} -i /root/.ssh/id_rsa_miab",
 			'--list-only', '-r',
 			rsync_target.format(host=target.netloc, path=target_path)
 		]
@@ -486,28 +518,13 @@ def list_target_files(config):
 				"Connection to rsync host failed: {}".format(reason))
 
 	elif target.scheme == "s3":
-		# match to a Region
+		import boto3.s3
-		fix_boto()  # must call prior to importing boto
+		from botocore.exceptions import ClientError
-		import boto.s3
-		from boto.exception import BotoServerError
-		custom_region = False
-		for region in boto.s3.regions():
-			if region.endpoint == target.hostname:
-				break
-		else:
-			# If region is not found this is a custom region
-			custom_region = True
 
+		# separate bucket from path in target
 		bucket = target.path[1:].split('/')[0]
 		path = '/'.join(target.path[1:].split('/')[1:]) + '/'
 
-		# Create a custom region with custom endpoint
-		if custom_region:
-			from boto.s3.connection import S3Connection
-			region = boto.s3.S3RegionInfo(name=bucket,
-										endpoint=target.hostname,
-										connection_cls=S3Connection)
-
 		# If no prefix is specified, set the path to '', otherwise boto won't list the files
 		if path == '/':
 			path = ''
@@ -517,20 +534,15 @@ def list_target_files(config):
 
 		# connect to the region & bucket
 		try:
-			conn = region.connect(aws_access_key_id=config["target_user"],
+			s3 = boto3.client('s3', \
-								aws_secret_access_key=config["target_pass"])
+				endpoint_url=f'https://{target.hostname}', \
-			bucket = conn.get_bucket(bucket)
+				aws_access_key_id=config['target_user'], \
-		except BotoServerError as e:
+				aws_secret_access_key=config['target_pass'])
-			if e.status == 403:
+			bucket_objects = s3.list_objects_v2(Bucket=bucket, Prefix=path).get("Contents", [])
-				raise ValueError("Invalid S3 access key or secret access key.")
+			backup_list = [(key['Key'][len(path):], key['Size']) for key in bucket_objects]
-			elif e.status == 404:
+		except ClientError as e:
-				raise ValueError("Invalid S3 bucket name.")
+			raise ValueError(e)
-			elif e.status == 301:
+		return backup_list
-				raise ValueError("Incorrect region for this bucket.")
-			raise ValueError(e.reason)
-
-		return [(key.name[len(path):], key.size)
-				for key in bucket.list(prefix=path)]
 	elif target.scheme == 'b2':
 		InMemoryAccountInfo = None
 		B2Api = None
@@ -562,8 +574,7 @@ def list_target_files(config):
 		raise ValueError(config["target"])
 
 
-def backup_set_custom(env, target, target_user, target_pass, target_rsync_port,
+def backup_set_custom(env, target, target_user, target_pass, target_rsync_port, min_age):
-					min_age):
 	config = get_backup_config(env, for_save=True)
 
 	# min_age must be an int

management/daemon.py

@@ -147,9 +147,9 @@ def index():
 	no_users_exist = (len(get_mail_users(env)) == 0)
 	no_admins_exist = (len(get_admins(env)) == 0)
 
-	utils.fix_boto()  # must call prior to importing boto
+	import boto3.s3
-	import boto.s3
+	backup_s3_hosts = [(r, f"s3.{r}.amazonaws.com") for r in boto3.session.Session().get_available_regions('s3')]
-	backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()]
+
 
 	return render_template(
 		'index.html',
@@ -730,6 +730,8 @@ def system_status():
 	# Create a temporary pool of processes for the status checks
 	with multiprocessing.pool.Pool(processes=5) as pool:
 		run_checks(False, env, output, pool)
+		pool.close()
+		pool.join()
 	return json_response(output.items)
 
 

management/dns_update.py

@@ -114,9 +114,9 @@ def do_dns_update(env, force=False):
 		if len(updated_domains) == 0:
 			updated_domains.append("DNS configuration")
 
-	# Kick nsd if anything changed.
+	# Tell nsd to reload changed zone files.
 	if len(updated_domains) > 0:
-		shell('check_call', ["/usr/sbin/service", "nsd", "restart"])
+		shell('check_call', ["/usr/sbin/nsd-control", "reload"])
 
 	# Write the OpenDKIM configuration tables for all of the mail domains.
 	from mailconfig import get_mail_domains
@@ -1299,13 +1299,9 @@ def get_secondary_dns(custom_dns, mode=None):
 			# doesn't.
 			if not hostname.startswith("xfr:"):
 				if mode == "xfr":
-					response = dns.resolver.resolve(hostname + '.',
+					response = dns.resolver.resolve(hostname+'.', "A", raise_on_no_answer=False)
-													"A",
-													raise_on_no_answer=False)
 					values.extend(map(str, response))
-					response = dns.resolver.resolve(hostname + '.',
+					response = dns.resolver.resolve(hostname+'.', "AAAA", raise_on_no_answer=False)
-													"AAAA",
-													raise_on_no_answer=False)
 					values.extend(map(str, response))
 					continue
 				values.append(hostname)
@@ -1330,14 +1326,11 @@ def set_secondary_dns(hostnames, env):
 				# Resolve hostname.
 				try:
 					response = resolver.resolve(item, "A")
-				except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN,
+				except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-						dns.resolver.NoAnswer):
 					try:
-						response = resolver.query(item, "AAAA")
+						response = resolver.resolve(item, "AAAA")
-					except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN,
+					except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-							dns.resolver.NoAnswer):
+						raise ValueError("Could not resolve the IP address of %s." % item)
-						raise ValueError(
-							"Could not resolve the IP address of %s." % item)
 			else:
 				# Validate IP address.
 				try:

management/ssl_certificates.py

@@ -64,37 +64,33 @@ def get_ssl_certificates(env):
 			# Not a valid PEM format for a PEM type we care about.
 			continue
 
-		# Remember where we got this object.
-		pem._filename = fn
-
 		# Is it a private key?
 		if isinstance(pem, RSAPrivateKey):
-			private_keys[pem.public_key().public_numbers()] = pem
+			private_keys[pem.public_key().public_numbers()] = { "filename": fn, "key": pem }
 
 		# Is it a certificate?
 		if isinstance(pem, Certificate):
-			certificates.append(pem)
+			certificates.append({ "filename": fn, "cert": pem })
 
 	# Process the certificates.
 	domains = {}
 	for cert in certificates:
 		# What domains is this certificate good for?
-		cert_domains, primary_domain = get_certificate_domains(cert)
+		cert_domains, primary_domain = get_certificate_domains(cert["cert"])
-		cert._primary_domain = primary_domain
+		cert["primary_domain"] = primary_domain
 
 		# Is there a private key file for this certificate?
-		private_key = private_keys.get(cert.public_key().public_numbers())
+		private_key = private_keys.get(cert["cert"].public_key().public_numbers())
 		if not private_key:
 			continue
-		cert._private_key = private_key
+		cert["private_key"] = private_key
 
 		# Add this cert to the list of certs usable for the domains.
 		for domain in cert_domains:
 			# The primary hostname can only use a certificate mapped
 			# to the system private key.
 			if domain == env['PRIMARY_HOSTNAME']:
-				if cert._private_key._filename != os.path.join(
+				if cert["private_key"]["filename"] != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
-					env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
 					continue
 
 			domains.setdefault(domain, []).append(cert)
@@ -105,13 +101,12 @@ def get_ssl_certificates(env):
 	ret = {}
 	for domain, cert_list in domains.items():
 		#for c in cert_list: print(domain, c.not_valid_before, c.not_valid_after, "("+str(now)+")", c.issuer, c.subject, c._filename)
-		cert_list.sort(
+		cert_list.sort(key = lambda cert : (
-			key=lambda cert: (
+			# must be valid NOW
-				# must be valid NOW
+			cert["cert"].not_valid_before <= now <= cert["cert"].not_valid_after,
-				cert.not_valid_before <= now <= cert.not_valid_after,
 
-				# prefer one that is not self-signed
+			# prefer one that is not self-signed
-				cert.issuer != cert.subject,
+			cert["cert"].issuer != cert["cert"].subject,
 
 				###########################################################
 				# The above lines ensure that valid certificates are chosen
@@ -119,9 +114,9 @@ def get_ssl_certificates(env):
 				# multiple valid certificates available for this domain.
 				###########################################################
 
-				# prefer one with the expiration furthest into the future so
+			# prefer one with the expiration furthest into the future so
-				# that we can easily rotate to new certs as we get them
+			# that we can easily rotate to new certs as we get them
-				cert.not_valid_after,
+			cert["cert"].not_valid_after,
 
 				###########################################################
 				# We always choose the certificate that is good for the
@@ -134,18 +129,18 @@ def get_ssl_certificates(env):
 				# domain.
 				###########################################################
 
-				# in case a certificate is installed in multiple paths,
+			# in case a certificate is installed in multiple paths,
-				# prefer the... lexicographically last one?
+			# prefer the... lexicographically last one?
-				cert._filename,
+			cert["filename"],
-			),
+
-			reverse=True)
+		), reverse=True)
 		cert = cert_list.pop(0)
 		ret[domain] = {
-			"private-key": cert._private_key._filename,
+			"private-key": cert["private_key"]["filename"],
-			"certificate": cert._filename,
+			"certificate": cert["filename"],
-			"primary-domain": cert._primary_domain,
+			"primary-domain": cert["primary_domain"],
-			"certificate_object": cert,
+			"certificate_object": cert["cert"],
-		}
+			}
 
 	return ret
 

management/status_checks.py

@@ -228,9 +228,7 @@ def check_service(i, service, env):
 
 			# IPv4 ok but IPv6 failed. Try the PRIVATE_IPV6 address to see if the service is bound to the interface.
 			elif service["port"] != 53 and try_connect(env["PRIVATE_IPV6"]):
-				output.print_error(
+				output.print_error("%s is running (and available over IPv4 and the local IPv6 address), but it is not publicly accessible at %s:%d." % (service['name'], env['PUBLIC_IPV6'], service['port']))
-					"%s is running (and available over IPv4 and the local IPv6 address), but it is not publicly accessible at %s:%d."
-					% (service['name'], env['PUBLIC_IP'], service['port']))
 			else:
 				output.print_error(
 					"%s is running and available over IPv4 but is not accessible over IPv6 at %s port %d."
@@ -383,6 +381,17 @@ def check_free_disk_space(rounded_values, env, output):
 			disk_msg = "The disk has less than 15% free space."
 		output.print_error(disk_msg)
 
+	# Check that there's only one duplicity cache. If there's more than one,
+	# it's probably no longer in use, and we can recommend clearing the cache
+	# to save space. The cache directory may not exist yet, which is OK.
+	backup_cache_path = os.path.join(env['STORAGE_ROOT'], 'backup/cache')
+	try:
+		backup_cache_count = len(os.listdir(backup_cache_path))
+	except:
+		backup_cache_count = 0
+	if backup_cache_count > 1:
+		output.print_warning("The backup cache directory {} has more than one backup target cache. Consider clearing this directory to save disk space."
+			.format(backup_cache_path))
 
 def check_free_memory(rounded_values, env, output):
 	# Check free memory.
@@ -1113,11 +1122,8 @@ def check_dnssec(domain,
 	if len(ds) > 0:
 		output.print_line("")
 		output.print_line("The DS record is currently set to:")
-		for rr in ds:
+		for rr in sorted(ds):
-			output.print_line(
+			output.print_line("Key Tag: {0}, Algorithm: {1}, Digest Type: {2}, Digest: {3}".format(*rr))
-				"Key Tag: {0}, Algorithm: {1}, Digest Type: {2}, Digest: {3}".
-				format(*rr))
-
 
 def check_mail_domain(domain, env, output):
 	# Check the MX record.
@@ -1169,7 +1175,7 @@ def check_mail_domain(domain, env, output):
 		output.print_ok(good_news)
 
 		# Check MTA-STS policy.
-		loop = asyncio.get_event_loop()
+		loop = asyncio.new_event_loop()
 		sts_resolver = postfix_mta_sts_resolver.resolver.STSResolver(loop=loop)
 		valid, policy = loop.run_until_complete(sts_resolver.resolve(domain))
 		if valid == postfix_mta_sts_resolver.resolver.STSFetchResult.VALID:
@@ -1269,8 +1275,7 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False):
 	# Do the query.
 	try:
 		response = resolver.resolve(qname, rtype)
-	except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN,
+	except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-			dns.resolver.NoAnswer):
 		# Host did not have an answer for this query; not sure what the
 		# difference is between the two exceptions.
 		return nxdomain

management/utils.py

@@ -204,15 +204,6 @@ def wait_for_service(port, public, env, timeout):
 				return False
 		time.sleep(min(timeout / 4, 1))
 
-
-def fix_boto():
-	# Google Compute Engine instances install some Python-2-only boto plugins that
-	# conflict with boto running under Python 3. Disable boto's default configuration
-	# file prior to importing boto so that GCE's plugin is not loaded:
-	import os
-	os.environ["BOTO_CONFIG"] = "/etc/boto3.cfg"
-
-
 def get_php_version():
 	# Gets the version of PHP installed in the system.
 	return shell("check_output", ["/usr/bin/php", "-v"])[4:7]
@@ -235,7 +226,6 @@ def get_os_code():
 
 	return None
 
-
 if __name__ == "__main__":
 	from web_update import get_web_domains
 	env = load_environment()

management/wsgi.py

@@ -0,0 +1,7 @@
+from daemon import app
+import auth, utils
+
+app.logger.addHandler(utils.create_syslog_handler())
+
+if __name__ == "__main__":
+    app.run(port=10222)

setup/dns.sh

@@ -10,16 +10,9 @@
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
 
-# Install the packages.
-#
-# * nsd: The non-recursive nameserver that publishes our DNS records.
-# * ldnsutils: Helper utilities for signing DNSSEC zones.
-# * openssh-client: Provides ssh-keyscan which we use to create SSHFP records.
-echo "Installing nsd (DNS server)..."
-apt_install ldnsutils openssh-client
-
 # Prepare nsd's configuration.
-
+# We configure nsd before installation as we only want it to bind to some addresses
+# and it otherwise will have port / bind conflicts with bind9 used as the local resolver
 mkdir -p /var/run/nsd
 mkdir -p /etc/nsd
 mkdir -p /etc/nsd/zones
@@ -46,18 +39,6 @@ server:
 
 EOF
 
-# Add log rotation
-cat > /etc/logrotate.d/nsd <<EOF;
-/var/log/nsd.log {
-  weekly
-  missingok
-  rotate 12
-  compress
-  delaycompress
-  notifempty
-}
-EOF
-
 # Since we have bind9 listening on localhost for locally-generated
 # DNS queries that require a recursive nameserver, and the system
 # might have other network interfaces for e.g. tunnelling, we have
@@ -74,8 +55,25 @@ echo "include: /etc/nsd/nsd.conf.d/*.conf" >> /etc/nsd/nsd.conf;
 # now be stored in /etc/nsd/nsd.conf.d.
 rm -f /etc/nsd/zones.conf
 
-# Attempting a late install of nsd (after configuration)
+# Add log rotation
-apt_install nsd
+cat > /etc/logrotate.d/nsd <<EOF;
+/var/log/nsd.log {
+  weekly
+  missingok
+  rotate 12
+  compress
+  delaycompress
+  notifempty
+}
+EOF
+
+# Install the packages.
+#
+# * nsd: The non-recursive nameserver that publishes our DNS records.
+# * ldnsutils: Helper utilities for signing DNSSEC zones.
+# * openssh-client: Provides ssh-keyscan which we use to create SSHFP records.
+echo "Installing nsd (DNS server)..."
+apt_install nsd ldnsutils openssh-client
 
 # Create DNSSEC signing keys.
 

setup/mail-dovecot.sh

@@ -89,6 +89,8 @@ management/editconf.py /etc/dovecot/conf.d/10-ssl.conf \
 	"ssl_min_protocol=TLSv1.2" \
 	"ssl_cipher_list=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384" \
 	"ssl_prefer_server_ciphers=no" \
+	"ssl_dh_parameters_length=2048" \
+	"ssl_dh=<$STORAGE_ROOT/ssl/dh2048.pem"
 
 # Disable in-the-clear IMAP/POP because there is no reason for a user to transmit
 # login credentials outside of an encrypted connection. Only the over-TLS versions

setup/mail-postfix.sh

@@ -13,8 +13,8 @@
 # destinations according to aliases, and passses email on to
 # another service for local mail delivery.
 #
-# The first hop in local mail delivery is to Spamassassin via
+# The first hop in local mail delivery is to spampd via
-# LMTP. Spamassassin then passes mail over to Dovecot for
+# LMTP. spampd then passes mail over to Dovecot for
 # storage in the user's mailbox.
 #
 # Postfix also listens on ports 465/587 (SMTPS, SMTP+STARTLS) for
@@ -193,17 +193,17 @@ management/editconf.py /etc/postfix/main.cf \
 
 # ### Incoming Mail
 
-# Pass any incoming mail over to a local delivery agent. Spamassassin
+# Pass mail to spampd, which acts as the local delivery agent (LDA),
-# will act as the LDA agent at first. It is listening on port 10025
+# which then passes the mail over to the Dovecot LMTP server after.
-# with LMTP. Spamassassin will pass the mail over to Dovecot after.
+# spampd runs on port 10025 by default.
 #
 # In a basic setup we would pass mail directly to Dovecot by setting
 # virtual_transport to `lmtp:unix:private/dovecot-lmtp`.
 management/editconf.py /etc/postfix/main.cf "virtual_transport=lmtp:[127.0.0.1]:10025"
-# Because of a spampd bug, limit the number of recipients in each connection.
+# Clear the lmtp_destination_recipient_limit setting which in previous
+# versions of Mail-in-a-Box was set to 1 because of a spampd bug.
 # See https://github.com/mail-in-a-box/mailinabox/issues/1523.
-management/editconf.py /etc/postfix/main.cf lmtp_destination_recipient_limit=1
+management/editconf.py /etc/postfix/main.cf  -e lmtp_destination_recipient_limit=
-
 
 # Who can send mail to us? Some basic filters.
 #
@@ -232,11 +232,32 @@ management/editconf.py /etc/postfix/main.cf \
 # As a matter of fact RFC is not strict about retry timer so postfix and
 # other MTA have their own intervals. To fix the problem of receiving
 # e-mails really latter, delay of greylisting has been set to
-# 180 seconds (default is 300 seconds).
+# 180 seconds (default is 300 seconds). We will move the postgrey database
+# under $STORAGE_ROOT. This prevents a "warming up" that would have occured
+# previously with a migrated or reinstalled OS.  We will specify this new path
+# with the --dbdir=... option. Arguments within POSTGREY_OPTS can not have spaces,
+# including dbdir. This is due to the way the init script sources the
+# /etc/default/postgrey file. --dbdir=... either needs to be a path without spaces
+# (luckily $STORAGE_ROOT does not currently work with spaces), or it needs to be a
+# symlink without spaces that can point to a folder with spaces).  We'll just assume
+# $STORAGE_ROOT won't have spaces to simplify things.
 management/editconf.py /etc/default/postgrey \
-	POSTGREY_OPTS=\"'--inet=127.0.0.1:10023 --delay=180'\"
+	POSTGREY_OPTS=\""--inet=127.0.0.1:10023 --delay=180 --dbdir=$STORAGE_ROOT/mail/postgrey/db"\"
 
 
+# If the $STORAGE_ROOT/mail/postgrey is empty, copy the postgrey database over from the old location
+if [ ! -d $STORAGE_ROOT/mail/postgrey/db ]; then
+	# Stop the service
+	service postgrey stop
+	# Ensure the new paths for postgrey db exists
+	mkdir -p $STORAGE_ROOT/mail/postgrey/db
+	# Move over database files
+	mv /var/lib/postgrey/* $STORAGE_ROOT/mail/postgrey/db/ || true
+fi
+# Ensure permissions are set
+chown -R postgrey:postgrey $STORAGE_ROOT/mail/postgrey/
+chmod 700 $STORAGE_ROOT/mail/postgrey/{,db}
+
 # We are going to setup a newer whitelist for postgrey, the version included in the distribution is old
 cat > /etc/cron.daily/mailinabox-postgrey-whitelist << EOF;
 #!/bin/bash

setup/management.sh

@@ -1,23 +1,12 @@
 #!/bin/bash
 
 source setup/functions.sh
+source /etc/mailinabox.conf # load global vars
 
 echo "Installing Mail-in-a-Box system management daemon..."
 
 # DEPENDENCIES
 
-# We used to install management daemon-related Python packages
-# directly to /usr/local/lib. We moved to a virtualenv because
-# these packages might conflict with apt-installed packages.
-# We may have a lingering version of acme that conflcits with
-# certbot, which we're about to install below, so remove it
-# first. Once acme is installed by an apt package, this might
-# break the package version and `apt-get install --reinstall python3-acme`
-# might be needed in that case.
-while [ -d /usr/local/lib/python3.4/dist-packages/acme ]; do
-	pip3 uninstall -y acme;
-done
-
 # duplicity is used to make backups of user data.
 #
 # virtualenv is used to isolate the Python 3 packages we
@@ -27,9 +16,10 @@ done
 # provision free TLS certificates.
 apt_install duplicity python3-pip python3-gpg virtualenv certbot rsync
 
-# boto is used for amazon aws backups.
+# b2sdk is used for backblaze backups.
+# boto3 is used for amazon aws backups.
 # Both are installed outside the pipenv, so they can be used by duplicity
-hide_output pip3 install --upgrade boto
+hide_output pip3 install --upgrade boto3
 
 # Create a virtualenv for the installation of Python 3 packages
 # used by the management daemon.
@@ -57,7 +47,7 @@ hide_output $venv/bin/pip install --upgrade pip
 # NOTE: email_validator is repeated in setup/questions.sh, so please keep the versions synced.
 hide_output $venv/bin/pip install --upgrade \
 	rtyaml "email_validator>=1.0.0" "exclusiveprocess" \
-	flask dnspython python-dateutil expiringdict \
+	flask dnspython python-dateutil expiringdict gunicorn \
 	qrcode[pil] pyotp \
 	"idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver
 
@@ -129,6 +119,9 @@ rm -f /tmp/fontawesome.zip
 
 # Create an init script to start the management daemon and keep it
 # running after a reboot.
+# Set a long timeout since some commands take a while to run, matching
+# the timeout we set for PHP (fastcgi_read_timeout in the nginx confs).
+# Note: Authentication currently breaks with more than 1 gunicorn worker.
 cat > $inst_dir/start <<EOF;
 #!/bin/bash
 # Set character encoding flags to ensure that any non-ASCII don't cause problems.
@@ -137,8 +130,13 @@ export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
 export LC_TYPE=en_US.UTF-8
 
+mkdir -p /var/lib/mailinabox
+tr -cd '[:xdigit:]' < /dev/urandom | head -c 32 > /var/lib/mailinabox/api.key
+chmod 640 /var/lib/mailinabox/api.key
+
 source $venv/bin/activate
-exec python $(pwd)/management/daemon.py
+export PYTHONPATH=$(pwd)/management
+exec gunicorn -b localhost:10222 -w 1 --timeout 630 wsgi:app
 EOF
 chmod +x $inst_dir/start
 cp --remove-destination conf/mailinabox.service /lib/systemd/system/mailinabox.service # target was previously a symlink so remove it first

setup/nextcloud.sh

@@ -51,6 +51,11 @@ apt_install php php-fpm \
 	php-intl php-imagick php-gmp php-bcmath
 
 phpenmod apcu
+
+management/editconf.py /etc/php/$(php_version)/mods-available/apcu.ini -c ';' \
+	apc.enabled=1 \
+	apc.enable_cli=1
+
 management/editconf.py /etc/php/$(php_version)/cli/php.ini -c ';' \
 	apc.enable_cli=1
 
@@ -345,6 +350,9 @@ CONFIG_TEMP=$(/bin/mktemp)
 php <<EOF > $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/config.php;
 <?php
 include("$STORAGE_ROOT/owncloud/config.php");
+
+\$CONFIG['config_is_read_only'] = true;
+
 \$CONFIG['trusted_domains'] = array('$PRIMARY_HOSTNAME');
 
 \$CONFIG['memcache.local'] = '\OC\Memcache\APCu';
@@ -412,11 +420,11 @@ management/editconf.py /etc/php/$(php_version)/cli/conf.d/10-opcache.ini -c ';'
 	opcache.save_comments=1 \
 	opcache.revalidate_freq=1
 
-# If apc is explicitly disabled we need to enable it
+# Migrate users_external data from <0.6.0 to version 3.0.0 (see https://github.com/nextcloud/user_external).
-if grep -q apc.enabled=0 /etc/php/$(php_version)/mods-available/apcu.ini; then
+# This version was probably in use in Mail-in-a-Box v0.41 (February 26, 2019) and earlier.
-	management/editconf.py /etc/php/$(php_version)/mods-available/apcu.ini -c ';' \
+# We moved to v0.6.3 in 193763f8. Ignore errors - maybe there are duplicated users with the
-		apc.enabled=1
+# correct backend already.
-fi
+sqlite3 $STORAGE_ROOT/owncloud/owncloud.db "UPDATE oc_users_external SET backend='127.0.0.1';" || /bin/true
 
 # Set up a cron job for Nextcloud.
 cat > /etc/cron.d/mailinabox-nextcloud << EOF;
@@ -426,9 +434,6 @@ cat > /etc/cron.d/mailinabox-nextcloud << EOF;
 EOF
 chmod +x /etc/cron.d/mailinabox-nextcloud
 
-# Remove previous hourly cronjob
-rm -f /etc/cron.hourly/mailinabox-owncloud
-
 # There's nothing much of interest that a user could do as an admin for Nextcloud,
 # and there's a lot they could mess up, so we don't make any users admins of Nextcloud.
 # But if we wanted to, we would do this:

setup/start.sh

@@ -72,6 +72,10 @@ fi
 fi
 
 # Create the STORAGE_USER and STORAGE_ROOT directory if they don't already exist.
+#
+# Set the directory and all of its parent directories' permissions to world
+# readable since it holds files owned by different processes.
+#
 # If the STORAGE_ROOT is missing the mailinabox.version file that lists a
 # migration (schema) number for the files stored there, assume this is a fresh
 # installation to that directory and write the file to contain the current
@@ -82,6 +86,8 @@ fi
 if [ ! -d $STORAGE_ROOT ]; then
 	mkdir -p $STORAGE_ROOT
 fi
+f=$STORAGE_ROOT
+while [[ $f != / ]]; do chmod a+rx "$f"; f=$(dirname "$f"); done;
 if [ ! -f $STORAGE_ROOT/mailinabox.version ]; then
 	setup/migrate.py --current > $STORAGE_ROOT/mailinabox.version
 	chown $STORAGE_USER.$STORAGE_USER $STORAGE_ROOT/mailinabox.version

setup/system.sh

@@ -102,9 +102,6 @@ apt_get_quiet autoremove
 
 # Install basic utilities.
 #
-# * haveged: Provides extra entropy to /dev/random so it doesn't stall
-#	         when generating random numbers for private keys (e.g. during
-#	         ldns-keygen).
 # * unattended-upgrades: Apt tool to install security updates automatically.
 # * cron: Runs background processes periodically.
 # * ntp: keeps the system time correct
@@ -118,8 +115,8 @@ apt_get_quiet autoremove
 
 echo Installing system packages...
 apt_install python3 python3-dev python3-pip python3-setuptools \
-	netcat-openbsd wget curl git sudo coreutils bc \
+	netcat-openbsd wget curl git sudo coreutils bc file \
-	haveged pollinate openssh-client unzip \
+	pollinate openssh-client unzip \
 	unattended-upgrades cron ntp fail2ban rsyslog
 
 # ### Suppress Upgrade Prompts
@@ -354,6 +351,7 @@ systemctl restart systemd-resolved
 rm -f /etc/fail2ban/jail.local # we used to use this file but don't anymore
 rm -f /etc/fail2ban/jail.d/defaults-debian.conf # removes default config so we can manage all of fail2ban rules in one config
 cat conf/fail2ban/jails.conf \
+    | sed "s/PUBLIC_IPV6/$PUBLIC_IPV6/g" \
 	| sed "s/PUBLIC_IP/$PUBLIC_IP/g" \
 	| sed "s#STORAGE_ROOT#$STORAGE_ROOT#" \
 	> /etc/fail2ban/jail.d/mailinabox.conf

setup/webmail.sh

@@ -83,7 +83,7 @@ if [ $needs_update == 1 ]; then
 
 	# download and verify the full release of the carddav plugin
 	wget_verify \
-		https://github.com/blind-coder/rcmcarddav/releases/download/v${CARDDAV_VERSION}/carddav-v${CARDDAV_VERSION}.tar.gz \
+		https://github.com/mstilkerich/rcmcarddav/releases/download/v${CARDDAV_VERSION}/carddav-v${CARDDAV_VERSION}.tar.gz \
 		$CARDDAV_HASH \
 		/tmp/carddav.tar.gz
 
@@ -140,6 +140,11 @@ cat > $RCM_CONFIG <<EOF;
 \$config['login_username_filter'] = 'email';
 \$config['password_charset'] = 'UTF-8';
 \$config['junk_mbox'] = 'Spam';
+
+/* ensure roudcube session id's aren't leaked to other parts of the server */
+\$config['session_path'] = '/mail/';
+/* prevent CSRF, requires php 7.3+ */
+\$config['session_samesite'] = 'Strict';
 \$config['quota_zero_as_unlimited'] = true;
 EOF
 
@@ -182,7 +187,7 @@ cat > ${RCM_PLUGIN_DIR}/carddav/config.inc.php <<EOF;
 	 'name'         =>  'ownCloud',
 	 'username'     =>  '%u', // login username
 	 'password'     =>  '%p', // login password
-	 'url'          =>  'https://${PRIMARY_HOSTNAME}/cloud/remote.php/carddav/addressbooks/%u/contacts',
+	 'url'          =>  'https://${PRIMARY_HOSTNAME}/cloud/remote.php/dav/addressbooks/users/%u/contacts/',
 	 'active'       =>  true,
 	 'readonly'     =>  false,
 	 'refresh_time' => '02:00:00',
@@ -230,7 +235,7 @@ chown -f -R root.www-data ${RCM_PLUGIN_DIR}/carddav
 chmod -R 774 ${RCM_PLUGIN_DIR}/carddav
 
 # Run Roundcube database migration script (database is created if it does not exist)
-${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube
+php ${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube
 chown www-data:www-data $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 chmod 664 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 

setup/zpush.sh

@@ -17,7 +17,7 @@ source /etc/mailinabox.conf # load global vars
 
 echo "Installing Z-Push (Exchange/ActiveSync server)..."
 apt_install \
-	php-soap php-imap libawl-php php-xsl
+       php-soap php-imap libawl-php php-xml
 
 phpenmod -v php imap
 
@@ -42,8 +42,6 @@ if [ $needs_update == 1 ]; then
 	rm -rf /tmp/z-push.zip /tmp/z-push
 
 	rm -f /usr/sbin/z-push-{admin,top}
-	ln -s /usr/local/lib/z-push/z-push-admin.php /usr/sbin/z-push-admin
-	ln -s /usr/local/lib/z-push/z-push-top.php /usr/sbin/z-push-top
 	echo $VERSION > /usr/local/lib/z-push/version
 fi
 
@@ -106,4 +104,4 @@ restart_service php$(php_version)-fpm
 
 # Fix states after upgrade
 
-hide_output z-push-admin -a fixstates
+hide_output php /usr/local/lib/z-push/z-push-admin.php -a fixstates

tests/test_dns.py

@@ -30,7 +30,7 @@ def test(server, description):
 		(hostname, "TXT", "\"v=spf1 mx -all\""),
 		("mail._domainkey." + hostname, "TXT", "\"v=DKIM1; k=rsa; s=email; \" \"p=__KEY__\""),
 		#("_adsp._domainkey." + hostname, "TXT", "\"dkim=all\""),
-		("_dmarc." + hostname, "TXT", "\"v=DMARC1; p=quarantine\""),
+		("_dmarc." + hostname, "TXT", "\"v=DMARC1; p=quarantine;\""),
 	]
 	return test2(tests, server, description)
 
@@ -48,7 +48,7 @@ def test2(tests, server, description):
 	for qname, rtype, expected_answer in tests:
 		# do the query and format the result as a string
 		try:
-			response = dns.resolver.query(qname, rtype)
+			response = dns.resolver.resolve(qname, rtype)
 		except dns.resolver.NoNameservers:
 			# host did not have an answer for this query
 			print("Could not connect to %s for DNS query." % server)

tests/test_mail.py

@@ -48,7 +48,7 @@ server = smtplib.SMTP_SSL(host)
 ipaddr = socket.gethostbyname(host) # IPv4 only!
 reverse_ip = dns.reversename.from_address(ipaddr) # e.g. "1.0.0.127.in-addr.arpa."
 try:
-	reverse_dns = dns.resolver.query(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname
+	reverse_dns = dns.resolver.resolve(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname
 except dns.resolver.NXDOMAIN:
 	print("Reverse DNS lookup failed for %s. SMTP EHLO name check skipped." % ipaddr)
 	reverse_dns = None