Support Ubuntu 22.04 Jammy Jellyfish (#2083)

CHANGELOG.md

@@ -1,6 +1,34 @@
 CHANGELOG
 =========
 
+Version 60 (October 11, 2022)
+-----------------------------
+
+This is the first release for Ubuntu 22.04.
+
+**Before upgrading**, you must **first upgrade your existing Ubuntu 18.04 box to Mail-in-a-Box v0.51 or later**, if you haven't already done so. That may not be possible after Ubuntu 18.04 reaches its end of life in April 2023, so please complete the upgrade well before then. (If you are not using Nextcloud's contacts or calendar, you can migrate to the latest version of Mail-in-a-Box from any previous version.)
+
+For complete upgrade instructions, see:
+
+https://discourse.mailinabox.email/t/version-60-for-ubuntu-22-04-is-about-to-be-released/9558
+
+No major features of Mail-in-a-Box have changed in this release, although some minor fixes were made.
+
+With the newer version of Ubuntu the following software packages we use are updated:
+
+* dovecot is upgraded to 2.3.16, postfix to 3.6.4, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug).
+* Nextcloud is upgraded to 23.0.4.
+* Roundcube is upgraded to 1.6.0.
+* certbot is upgraded to 1.21 (via the Ubuntu repository instead of a PPA).
+* fail2ban is upgraded to 0.11.2.
+* nginx is upgraded to 1.18.
+* PHP is upgraded from 7.2 to 8.0.
+
+Also:
+
+* Roundcube's login session cookie was tightened. Existing sessions may require a manual logout.
+* Moved Postgrey's database under $STORAGE_ROOT.
+
 Version 57a (June 19, 2022)
 ---------------------------
 

README.md

@@ -23,7 +23,7 @@ Additionally, this project has a [Code of Conduct](CODE_OF_CONDUCT.md), which su
 In The Box
 ----------
 
-Mail-in-a-Box turns a fresh Ubuntu 18.04 LTS 64-bit machine into a working mail server by installing and configuring various components.
+Mail-in-a-Box turns a fresh Ubuntu 22.04 LTS 64-bit machine into a working mail server by installing and configuring various components.
 
 It is a one-click email appliance. There are no user-configurable setup options. It "just works."
 
@@ -54,13 +54,13 @@ Installation
 
 See the [setup guide](https://mailinabox.email/guide.html) for detailed, user-friendly instructions.
 
-For experts, start with a completely fresh (really, I mean it) Ubuntu 18.04 LTS 64-bit machine. On the machine...
+For experts, start with a completely fresh (really, I mean it) Ubuntu 22.04 LTS 64-bit machine. On the machine...
 
 Clone this repository and checkout the tag corresponding to the most recent release:
 
 	$ git clone https://github.com/mail-in-a-box/mailinabox
 	$ cd mailinabox
-	$ git checkout v57a
+	$ git checkout v60
 
 Begin the installation.
 

Vagrantfile

@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/bionic64"
+  config.vm.box = "ubuntu/jammy64"
 
   # Network config: Since it's a mail server, the machine must be connected
   # to the public web. However, we currently don't want to expose SSH since

conf/mailinabox.service

@@ -4,6 +4,7 @@ After=multi-user.target
 
 [Service]
 Type=idle
+IgnoreSIGPIPE=False
 ExecStart=/usr/local/lib/mailinabox/start
 
 [Install]

conf/nginx-top.conf

@@ -7,6 +7,6 @@
 ##       your own --- please do not ask for help from us.
 
 upstream php-fpm {
-	server unix:/var/run/php/php7.2-fpm.sock;
+	server unix:/var/run/php/php8.0-fpm.sock;
 }
 

management/auth.py

@@ -22,20 +22,8 @@ class AuthService:
 	def init_system_api_key(self):
 		"""Write an API key to a local file so local processes can use the API"""
 
-		def create_file_with_mode(path, mode):
+		with open(self.key_path, 'r') as file:
-			# Based on answer by A-B-B: http://stackoverflow.com/a/15015748
+			self.key = file.read()
-			old_umask = os.umask(0)
-			try:
-				return os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, mode), 'w')
-			finally:
-				os.umask(old_umask)
-
-		self.key = secrets.token_hex(32)
-
-		os.makedirs(os.path.dirname(self.key_path), exist_ok=True)
-
-		with create_file_with_mode(self.key_path, 0o640) as key_file:
-			key_file.write(self.key + '\n')
 
 	def authenticate(self, request, env, login_only=False, logout=False):
 		"""Test if the HTTP Authorization header's username matches the system key, a session key,

management/backup.py

@@ -12,7 +12,7 @@ import dateutil.parser, dateutil.relativedelta, dateutil.tz
 import rtyaml
 from exclusiveprocess import Lock
 
-from utils import load_environment, shell, wait_for_service, fix_boto
+from utils import load_environment, shell, wait_for_service
 
 def backup_status(env):
 	# If backups are dissbled, return no status.
@@ -197,12 +197,7 @@ def get_duplicity_target_url(config):
 		from urllib.parse import urlsplit, urlunsplit
 		target = list(urlsplit(target))
 
-		# Duplicity now defaults to boto3 as the backend for S3, but we have
+		# Although we store the S3 hostname in the target URL,
-		# legacy boto installed (boto3 doesn't support Ubuntu 18.04) so
-		# we retarget for classic boto.
-		target[0] = "boto+" + target[0]
-
-		# In addition, although we store the S3 hostname in the target URL,
 		# duplicity no longer accepts it in the target URL. The hostname in
 		# the target URL must be the bucket name. The hostname is passed
 		# via get_duplicity_additional_args. Move the first part of the
@@ -283,9 +278,10 @@ def perform_backup(full_backup):
 			if quit:
 				sys.exit(code)
 
-	service_command("php7.2-fpm", "stop", quit=True)
+	service_command("php8.0-fpm", "stop", quit=True)
 	service_command("postfix", "stop", quit=True)
 	service_command("dovecot", "stop", quit=True)
+	service_command("postgrey", "stop", quit=True)
 
 	# Execute a pre-backup script that copies files outside the homedir.
 	# Run as the STORAGE_USER user, not as root. Pass our settings in
@@ -315,9 +311,10 @@ def perform_backup(full_backup):
 			get_duplicity_env_vars(env))
 	finally:
 		# Start services again.
+		service_command("postgrey", "start", quit=False)
 		service_command("dovecot", "start", quit=False)
 		service_command("postfix", "start", quit=False)
-		service_command("php7.2-fpm", "start", quit=False)
+		service_command("php8.0-fpm", "start", quit=False)
 
 	# Remove old backups. This deletes all backup data no longer needed
 	# from more than 3 days ago.
@@ -451,26 +448,13 @@ def list_target_files(config):
 			raise ValueError("Connection to rsync host failed: {}".format(reason))
 
 	elif target.scheme == "s3":
-		# match to a Region
+		import boto3.s3
-		fix_boto() # must call prior to importing boto
+		from botocore.exceptions import ClientError
-		import boto.s3
+		
-		from boto.exception import BotoServerError
+		# separate bucket from path in target
-		custom_region = False
-		for region in boto.s3.regions():
-			if region.endpoint == target.hostname:
-				break
-		else:
-			# If region is not found this is a custom region
-			custom_region = True
-
 		bucket = target.path[1:].split('/')[0]
 		path = '/'.join(target.path[1:].split('/')[1:]) + '/'
 
-		# Create a custom region with custom endpoint
-		if custom_region:
-			from boto.s3.connection import S3Connection
-			region = boto.s3.S3RegionInfo(name=bucket, endpoint=target.hostname, connection_cls=S3Connection)
-
 		# If no prefix is specified, set the path to '', otherwise boto won't list the files
 		if path == '/':
 			path = ''
@@ -480,18 +464,15 @@ def list_target_files(config):
 
 		# connect to the region & bucket
 		try:
-			conn = region.connect(aws_access_key_id=config["target_user"], aws_secret_access_key=config["target_pass"])
+			s3 = boto3.client('s3', \
-			bucket = conn.get_bucket(bucket)
+				endpoint_url=f'https://{target.hostname}', \
-		except BotoServerError as e:
+				aws_access_key_id=config['target_user'], \
-			if e.status == 403:
+				aws_secret_access_key=config['target_pass'])
-				raise ValueError("Invalid S3 access key or secret access key.")
+			bucket_objects = s3.list_objects_v2(Bucket=bucket, Prefix=path)['Contents']
-			elif e.status == 404:
+			backup_list = [(key['Key'][len(path):], key['Size']) for key in bucket_objects]
-				raise ValueError("Invalid S3 bucket name.")
+		except ClientError as e:
-			elif e.status == 301:
+			raise ValueError(e)
-				raise ValueError("Incorrect region for this bucket.")
+		return backup_list
-			raise ValueError(e.reason)
-
-		return [(key.name[len(path):], key.size) for key in bucket.list(prefix=path)]
 	elif target.scheme == 'b2':
 		from b2sdk.v1 import InMemoryAccountInfo, B2Api
 		from b2sdk.v1.exception import NonExistentBucket

management/daemon.py

@@ -121,9 +121,9 @@ def index():
 	no_users_exist = (len(get_mail_users(env)) == 0)
 	no_admins_exist = (len(get_admins(env)) == 0)
 
-	utils.fix_boto() # must call prior to importing boto
+	import boto3.s3
-	import boto.s3
+	backup_s3_hosts = [(r, f"s3.{r}.amazonaws.com") for r in boto3.session.Session().get_available_regions('s3')]
-	backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()]
+
 
 	return render_template('index.html',
 		hostname=env['PRIMARY_HOSTNAME'],
@@ -571,6 +571,8 @@ def system_status():
 	# Create a temporary pool of processes for the status checks
 	with multiprocessing.pool.Pool(processes=5) as pool:
 		run_checks(False, env, output, pool)
+		pool.close()
+		pool.join()
 	return json_response(output.items)
 
 @app.route('/system/updates')

management/dns_update.py

@@ -96,9 +96,9 @@ def do_dns_update(env, force=False):
 		if len(updated_domains) == 0:
 			updated_domains.append("DNS configuration")
 
-	# Kick nsd if anything changed.
+	# Tell nsd to reload changed zone files.
 	if len(updated_domains) > 0:
-		shell('check_call', ["/usr/sbin/service", "nsd", "restart"])
+		shell('check_call', ["/usr/sbin/nsd-control", "reload"])
 
 	# Write the OpenDKIM configuration tables for all of the mail domains.
 	from mailconfig import get_mail_domains
@@ -1000,9 +1000,9 @@ def get_secondary_dns(custom_dns, mode=None):
 			# doesn't.
 			if not hostname.startswith("xfr:"):
 				if mode == "xfr":
-					response = dns.resolver.query(hostname+'.', "A", raise_on_no_answer=False)
+					response = dns.resolver.resolve(hostname+'.', "A", raise_on_no_answer=False)
 					values.extend(map(str, response))
-					response = dns.resolver.query(hostname+'.', "AAAA", raise_on_no_answer=False)
+					response = dns.resolver.resolve(hostname+'.', "AAAA", raise_on_no_answer=False)
 					values.extend(map(str, response))
 					continue
 				values.append(hostname)
@@ -1025,10 +1025,10 @@ def set_secondary_dns(hostnames, env):
 			if not item.startswith("xfr:"):
 				# Resolve hostname.
 				try:
-					response = resolver.query(item, "A")
+					response = resolver.resolve(item, "A")
 				except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
 					try:
-						response = resolver.query(item, "AAAA")
+						response = resolver.resolve(item, "AAAA")
 					except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
 						raise ValueError("Could not resolve the IP address of %s." % item)
 			else:

management/ssl_certificates.py

@@ -58,36 +58,33 @@ def get_ssl_certificates(env):
 			# Not a valid PEM format for a PEM type we care about.
 			continue
 
-		# Remember where we got this object.
-		pem._filename = fn
-
 		# Is it a private key?
 		if isinstance(pem, RSAPrivateKey):
-			private_keys[pem.public_key().public_numbers()] = pem
+			private_keys[pem.public_key().public_numbers()] = { "filename": fn, "key": pem }
 
 		# Is it a certificate?
 		if isinstance(pem, Certificate):
-			certificates.append(pem)
+			certificates.append({ "filename": fn, "cert": pem })
 
 	# Process the certificates.
 	domains = { }
 	for cert in certificates:
 		# What domains is this certificate good for?
-		cert_domains, primary_domain = get_certificate_domains(cert)
+		cert_domains, primary_domain = get_certificate_domains(cert["cert"])
-		cert._primary_domain = primary_domain
+		cert["primary_domain"] = primary_domain
 
 		# Is there a private key file for this certificate?
-		private_key = private_keys.get(cert.public_key().public_numbers())
+		private_key = private_keys.get(cert["cert"].public_key().public_numbers())
 		if not private_key:
 			continue
-		cert._private_key = private_key
+		cert["private_key"] = private_key
 
 		# Add this cert to the list of certs usable for the domains.
 		for domain in cert_domains:
 			# The primary hostname can only use a certificate mapped
 			# to the system private key.
 			if domain == env['PRIMARY_HOSTNAME']:
-				if cert._private_key._filename != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
+				if cert["private_key"]["filename"] != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
 					continue
 
 			domains.setdefault(domain, []).append(cert)
@@ -100,10 +97,10 @@ def get_ssl_certificates(env):
 		#for c in cert_list: print(domain, c.not_valid_before, c.not_valid_after, "("+str(now)+")", c.issuer, c.subject, c._filename)
 		cert_list.sort(key = lambda cert : (
 			# must be valid NOW
-			cert.not_valid_before <= now <= cert.not_valid_after,
+			cert["cert"].not_valid_before <= now <= cert["cert"].not_valid_after,
 
 			# prefer one that is not self-signed
-			cert.issuer != cert.subject,
+			cert["cert"].issuer != cert["cert"].subject,
 
 			###########################################################
 			# The above lines ensure that valid certificates are chosen
@@ -113,7 +110,7 @@ def get_ssl_certificates(env):
 
 			# prefer one with the expiration furthest into the future so
 			# that we can easily rotate to new certs as we get them
-			cert.not_valid_after,
+			cert["cert"].not_valid_after,
 
 			###########################################################
 			# We always choose the certificate that is good for the
@@ -128,15 +125,15 @@ def get_ssl_certificates(env):
 
 			# in case a certificate is installed in multiple paths,
 			# prefer the... lexicographically last one?
-			cert._filename,
+			cert["filename"],
 
 		), reverse=True)
 		cert = cert_list.pop(0)
 		ret[domain] = {
-			"private-key": cert._private_key._filename,
+			"private-key": cert["private_key"]["filename"],
-			"certificate": cert._filename,
+			"certificate": cert["filename"],
-			"primary-domain": cert._primary_domain,
+			"primary-domain": cert["primary_domain"],
-			"certificate_object": cert,
+			"certificate_object": cert["cert"],
 			}
 
 	return ret

management/status_checks.py

@@ -715,7 +715,7 @@ def check_mail_domain(domain, env, output):
 		output.print_ok(good_news)
 
 		# Check MTA-STS policy.
-		loop = asyncio.get_event_loop()
+		loop = asyncio.new_event_loop()
 		sts_resolver = postfix_mta_sts_resolver.resolver.STSResolver(loop=loop)
 		valid, policy = loop.run_until_complete(sts_resolver.resolve(domain))
 		if valid == postfix_mta_sts_resolver.resolver.STSFetchResult.VALID:
@@ -797,7 +797,7 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False):
 
 	# Do the query.
 	try:
-		response = resolver.query(qname, rtype)
+		response = resolver.resolve(qname, rtype)
 	except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
 		# Host did not have an answer for this query; not sure what the
 		# difference is between the two exceptions.

management/templates/system-backup.html

@@ -269,6 +269,7 @@ function show_custom_backup() {
           $("#backup-target-type").val("s3");
           var hostpath = r.target.substring(5).split('/');
           var host = hostpath.shift();
+          $("#backup-target-s3-host-select").val(host);
           $("#backup-target-s3-host").val(host);
           $("#backup-target-s3-path").val(hostpath.join('/'));
         } else if (r.target.substring(0, 5) == "b2://") {

management/utils.py

@@ -175,14 +175,6 @@ def wait_for_service(port, public, env, timeout):
 				return False
 		time.sleep(min(timeout/4, 1))
 
-def fix_boto():
-	# Google Compute Engine instances install some Python-2-only boto plugins that
-	# conflict with boto running under Python 3. Disable boto's default configuration
-	# file prior to importing boto so that GCE's plugin is not loaded:
-	import os
-	os.environ["BOTO_CONFIG"] = "/etc/boto3.cfg"
-
-
 if __name__ == "__main__":
 	from web_update import get_web_domains
 	env = load_environment()

management/wsgi.py

@@ -0,0 +1,7 @@
+from daemon import app
+import auth, utils
+
+app.logger.addHandler(utils.create_syslog_handler())
+
+if __name__ == "__main__":
+    app.run(port=10222)

setup/bootstrap.sh

@@ -9,32 +9,37 @@
 if [ -z "$TAG" ]; then
 	# If a version to install isn't explicitly given as an environment
 	# variable, then install the latest version. But the latest version
-	# depends on the operating system. Existing Ubuntu 14.04 users need
+	# depends on the machine's version of Ubuntu. Existing users need to
-	# to be able to upgrade to the latest version supporting Ubuntu 14.04,
+	# be able to upgrade to the latest version available for that version
-	# in part because an upgrade is required before jumping to Ubuntu 18.04.
+	# of Ubuntu to satisfy the migration requirements.
-	# New users on Ubuntu 18.04 need to get the latest version number too.
 	#
 	# Also, the system status checks read this script for TAG = (without the
 	# space, but if we put it in a comment it would confuse the status checks!)
 	# to get the latest version, so the first such line must be the one that we
 	# want to display in status checks.
-	if [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' )" == "Ubuntu 18.04 LTS" ]; then
+	#
-		# This machine is running Ubuntu 18.04.
+	# Allow point-release versions of the major releases, e.g. 22.04.1 is OK.
+	UBUNTU_VERSION=$( lsb_release -d | sed 's/.*:\s*//' | sed 's/\([0-9]*\.[0-9]*\)\.[0-9]/\1/' )"
+	if [ "$UBUNTU_VERSION" == "Ubuntu 22.04 LTS" ]; then
+		# This machine is running Ubuntu 22.04, which is supported by
+		# Mail-in-a-Box versions 60 and later.
+		TAG=v60
+	elif [ "$UBUNTU_VERSION" == "Ubuntu 18.04 LTS" ]; then
+		# This machine is running Ubuntu 18.04, which is supported by
+		# Mail-in-a-Box versions 0.40 through 5x.
+		echo "Support is ending for Ubuntu 18.04."
+		echo "Please immediately begin to migrate your data to"
+		echo "a new machine running Ubuntu 22.04. See:"
+		echo "https://mailinabox.email/maintenance.html#upgrade"
 		TAG=v57a
-
+	elif [ "$UBUNTU_VERSION" == "Ubuntu 14.04 LTS" ]; then
-	elif [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' )" == "Ubuntu 14.04 LTS" ]; then
+		# This machine is running Ubuntu 14.04, which is supported by
-		# This machine is running Ubuntu 14.04.
+		# Mail-in-a-Box versions 1 through v0.30.
-		echo "You are installing the last version of Mail-in-a-Box that will"
+		echo "Ubuntu 14.04 is no longer supported."
-		echo "support Ubuntu 14.04. If this is a new installation of Mail-in-a-Box,"
+		echo "The last version of Mail-in-a-Box supporting Ubuntu 14.04 will be installed."
-		echo "stop now and switch to a machine running Ubuntu 18.04. If you are"
-		echo "upgrading an existing Mail-in-a-Box --- great. After upgrading this"
-		echo "box, please visit https://mailinabox.email for notes on how to upgrade"
-		echo "to Ubuntu 18.04."
-		echo ""
 		TAG=v0.30
-
 	else
-		echo "This script must be run on a system running Ubuntu 18.04 or Ubuntu 14.04."
+		echo "This script may be used only on a machine running Ubuntu 14.04, 18.04, or 22.04."
 		exit 1
 	fi
 fi

setup/dns.sh

@@ -10,17 +10,13 @@
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
 
-# Install the packages.
-#
-# * nsd: The non-recursive nameserver that publishes our DNS records.
-# * ldnsutils: Helper utilities for signing DNSSEC zones.
-# * openssh-client: Provides ssh-keyscan which we use to create SSHFP records.
-echo "Installing nsd (DNS server)..."
-apt_install nsd ldnsutils openssh-client
-
 # Prepare nsd's configuration.
-
+# We configure nsd before installation as we only want it to bind to some addresses
+# and it otherwise will have port / bind conflicts with bind9 used as the local resolver
 mkdir -p /var/run/nsd
+mkdir -p /etc/nsd
+mkdir -p /etc/nsd/zones
+touch /etc/nsd/zones.conf
 
 cat > /etc/nsd/nsd.conf << EOF;
 # Do not edit. Overwritten by Mail-in-a-Box setup.
@@ -42,18 +38,6 @@ server:
 
 EOF
 
-# Add log rotation
-cat > /etc/logrotate.d/nsd <<EOF;
-/var/log/nsd.log {
-  weekly
-  missingok
-  rotate 12
-  compress
-  delaycompress
-  notifempty
-}
-EOF
-
 # Since we have bind9 listening on localhost for locally-generated
 # DNS queries that require a recursive nameserver, and the system
 # might have other network interfaces for e.g. tunnelling, we have
@@ -70,6 +54,26 @@ echo "include: /etc/nsd/nsd.conf.d/*.conf" >> /etc/nsd/nsd.conf;
 # now be stored in /etc/nsd/nsd.conf.d.
 rm -f /etc/nsd/zones.conf
 
+# Add log rotation
+cat > /etc/logrotate.d/nsd <<EOF;
+/var/log/nsd.log {
+  weekly
+  missingok
+  rotate 12
+  compress
+  delaycompress
+  notifempty
+}
+EOF
+
+# Install the packages.
+#
+# * nsd: The non-recursive nameserver that publishes our DNS records.
+# * ldnsutils: Helper utilities for signing DNSSEC zones.
+# * openssh-client: Provides ssh-keyscan which we use to create SSHFP records.
+echo "Installing nsd (DNS server)..."
+apt_install nsd ldnsutils openssh-client
+
 # Create DNSSEC signing keys.
 
 mkdir -p "$STORAGE_ROOT/dns/dnssec";

setup/functions.sh

@@ -4,6 +4,8 @@
 # -o pipefail: don't ignore errors in the non-last command in a pipeline
 set -euo pipefail
 
+PHP_VER=8.0
+
 function hide_output {
 	# This function hides the output of a command unless the command fails
 	# and returns a non-zero exit code.

setup/mail-dovecot.sh

@@ -84,10 +84,11 @@ tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \
 	ssl=required \
 	"ssl_cert=<$STORAGE_ROOT/ssl/ssl_certificate.pem" \
 	"ssl_key=<$STORAGE_ROOT/ssl/ssl_private_key.pem" \
-	"ssl_protocols=TLSv1.2" \
+	"ssl_min_protocol=TLSv1.2" \
 	"ssl_cipher_list=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384" \
 	"ssl_prefer_server_ciphers=no" \
-	"ssl_dh_parameters_length=2048"
+	"ssl_dh_parameters_length=2048" \
+	"ssl_dh=<$STORAGE_ROOT/ssl/dh2048.pem"
 
 # Disable in-the-clear IMAP/POP because there is no reason for a user to transmit
 # login credentials outside of an encrypted connection. Only the over-TLS versions

setup/mail-postfix.sh

@@ -13,8 +13,8 @@
 # destinations according to aliases, and passses email on to
 # another service for local mail delivery.
 #
-# The first hop in local mail delivery is to Spamassassin via
+# The first hop in local mail delivery is to spampd via
-# LMTP. Spamassassin then passes mail over to Dovecot for
+# LMTP. spampd then passes mail over to Dovecot for
 # storage in the user's mailbox.
 #
 # Postfix also listens on ports 465/587 (SMTPS, SMTP+STARTLS) for
@@ -193,16 +193,17 @@ tools/editconf.py /etc/postfix/main.cf \
 
 # ### Incoming Mail
 
-# Pass any incoming mail over to a local delivery agent. Spamassassin
+# Pass mail to spampd, which acts as the local delivery agent (LDA),
-# will act as the LDA agent at first. It is listening on port 10025
+# which then passes the mail over to the Dovecot LMTP server after.
-# with LMTP. Spamassassin will pass the mail over to Dovecot after.
+# spampd runs on port 10025 by default.
 #
 # In a basic setup we would pass mail directly to Dovecot by setting
 # virtual_transport to `lmtp:unix:private/dovecot-lmtp`.
 tools/editconf.py /etc/postfix/main.cf "virtual_transport=lmtp:[127.0.0.1]:10025"
-# Because of a spampd bug, limit the number of recipients in each connection.
+# Clear the lmtp_destination_recipient_limit setting which in previous
+# versions of Mail-in-a-Box was set to 1 because of a spampd bug.
 # See https://github.com/mail-in-a-box/mailinabox/issues/1523.
-tools/editconf.py /etc/postfix/main.cf lmtp_destination_recipient_limit=1
+tools/editconf.py /etc/postfix/main.cf  -e lmtp_destination_recipient_limit=
 
 
 # Who can send mail to us? Some basic filters.
@@ -232,11 +233,32 @@ tools/editconf.py /etc/postfix/main.cf \
 # As a matter of fact RFC is not strict about retry timer so postfix and
 # other MTA have their own intervals. To fix the problem of receiving
 # e-mails really latter, delay of greylisting has been set to
-# 180 seconds (default is 300 seconds).
+# 180 seconds (default is 300 seconds). We will move the postgrey database
+# under $STORAGE_ROOT. This prevents a "warming up" that would have occured
+# previously with a migrated or reinstalled OS.  We will specify this new path
+# with the --dbdir=... option. Arguments within POSTGREY_OPTS can not have spaces,
+# including dbdir. This is due to the way the init script sources the
+# /etc/default/postgrey file. --dbdir=... either needs to be a path without spaces
+# (luckily $STORAGE_ROOT does not currently work with spaces), or it needs to be a
+# symlink without spaces that can point to a folder with spaces).  We'll just assume
+# $STORAGE_ROOT won't have spaces to simplify things.
 tools/editconf.py /etc/default/postgrey \
-	POSTGREY_OPTS=\"'--inet=127.0.0.1:10023 --delay=180'\"
+	POSTGREY_OPTS=\""--inet=127.0.0.1:10023 --delay=180 --dbdir=$STORAGE_ROOT/mail/postgrey/db"\"
 
 
+# If the $STORAGE_ROOT/mail/postgrey is empty, copy the postgrey database over from the old location
+if [ ! -d $STORAGE_ROOT/mail/postgrey/db ]; then
+	# Stop the service
+	service postgrey stop
+	# Ensure the new paths for postgrey db exists
+	mkdir -p $STORAGE_ROOT/mail/postgrey/db
+	# Move over database files
+	mv /var/lib/postgrey/* $STORAGE_ROOT/mail/postgrey/db/ || true
+fi
+# Ensure permissions are set
+chown -R postgrey:postgrey $STORAGE_ROOT/mail/postgrey/
+chmod 700 $STORAGE_ROOT/mail/postgrey/{,db}
+
 # We are going to setup a newer whitelist for postgrey, the version included in the distribution is old
 cat > /etc/cron.daily/mailinabox-postgrey-whitelist << EOF;
 #!/bin/bash

setup/management.sh

@@ -1,23 +1,12 @@
 #!/bin/bash
 
 source setup/functions.sh
+source /etc/mailinabox.conf # load global vars
 
 echo "Installing Mail-in-a-Box system management daemon..."
 
 # DEPENDENCIES
 
-# We used to install management daemon-related Python packages
-# directly to /usr/local/lib. We moved to a virtualenv because
-# these packages might conflict with apt-installed packages.
-# We may have a lingering version of acme that conflcits with
-# certbot, which we're about to install below, so remove it
-# first. Once acme is installed by an apt package, this might
-# break the package version and `apt-get install --reinstall python3-acme`
-# might be needed in that case.
-while [ -d /usr/local/lib/python3.4/dist-packages/acme ]; do
-	pip3 uninstall -y acme;
-done
-
 # duplicity is used to make backups of user data.
 #
 # virtualenv is used to isolate the Python 3 packages we
@@ -25,12 +14,12 @@ done
 #
 # certbot installs EFF's certbot which we use to
 # provision free TLS certificates.
-apt_install duplicity python-pip virtualenv certbot rsync
+apt_install duplicity python3-pip virtualenv certbot rsync
 
 # b2sdk is used for backblaze backups.
-# boto is used for amazon aws backups.
+# boto3 is used for amazon aws backups.
 # Both are installed outside the pipenv, so they can be used by duplicity
-hide_output pip3 install --upgrade b2sdk==1.14.1 boto
+hide_output pip3 install --upgrade b2sdk boto3
 
 # Create a virtualenv for the installation of Python 3 packages
 # used by the management daemon.
@@ -49,10 +38,10 @@ hide_output $venv/bin/pip install --upgrade pip
 # NOTE: email_validator is repeated in setup/questions.sh, so please keep the versions synced.
 hide_output $venv/bin/pip install --upgrade \
 	rtyaml "email_validator>=1.0.0" "exclusiveprocess" \
-	flask dnspython python-dateutil expiringdict \
+	flask dnspython python-dateutil expiringdict gunicorn \
 	qrcode[pil] pyotp \
-	"idna>=2.0.0" "cryptography==2.2.2" psutil postfix-mta-sts-resolver \
+	"idna>=2.0.0" "cryptography==37.0.2" psutil postfix-mta-sts-resolver \
-	b2sdk==1.14.1 boto
+	b2sdk boto3
 
 # CONFIGURATION
 
@@ -89,6 +78,9 @@ rm -f /tmp/bootstrap.zip
 
 # Create an init script to start the management daemon and keep it
 # running after a reboot.
+# Set a long timeout since some commands take a while to run, matching
+# the timeout we set for PHP (fastcgi_read_timeout in the nginx confs).
+# Note: Authentication currently breaks with more than 1 gunicorn worker.
 cat > $inst_dir/start <<EOF;
 #!/bin/bash
 # Set character encoding flags to ensure that any non-ASCII don't cause problems.
@@ -97,8 +89,13 @@ export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
 export LC_TYPE=en_US.UTF-8
 
+mkdir -p /var/lib/mailinabox
+tr -cd '[:xdigit:]' < /dev/urandom | head -c 32 > /var/lib/mailinabox/api.key
+chmod 640 /var/lib/mailinabox/api.key
+
 source $venv/bin/activate
-exec python $(pwd)/management/daemon.py
+export PYTHONPATH=$(pwd)/management
+exec gunicorn -b localhost:10222 -w 1 --timeout 630 wsgi:app
 EOF
 chmod +x $inst_dir/start
 cp --remove-destination conf/mailinabox.service /lib/systemd/system/mailinabox.service # target was previously a symlink so remove it first

setup/nextcloud.sh

@@ -21,8 +21,8 @@ echo "Installing Nextcloud (contacts/calendar)..."
 #   we automatically install intermediate versions as needed.
 # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and
 #   copying it from the error message when it doesn't match what is below.
-nextcloud_ver=20.0.14
+nextcloud_ver=23.0.8
-nextcloud_hash=92cac708915f51ee2afc1787fd845476fd090c81
+nextcloud_hash=9d63416a0697eeecf243d09461f5881f8997f50a
 
 # Nextcloud apps
 # --------------
@@ -33,21 +33,26 @@ nextcloud_hash=92cac708915f51ee2afc1787fd845476fd090c81
 #   https://github.com/nextcloud/user_external/blob/master/appinfo/info.xml
 # * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and
 #   copying it from the error message when it doesn't match what is below.
-contacts_ver=4.0.7
+contacts_ver=4.2.0
-contacts_hash=45e7cf4bfe99cd8d03625cf9e5a1bb2e90549136
+contacts_hash=79b506574834db5e1b6ab47aadd4041e12ad9a9c
-calendar_ver=3.0.4
+calendar_ver=3.5.0
-calendar_hash=d0284b68135777ec9ca713c307216165b294d0fe
+calendar_hash=941381536287a015081669513f8f79f6f262508a
-user_external_ver=1.0.0
+user_external_ver=3.0.0
-user_external_hash=3bf2609061d7214e7f0f69dd8883e55c4ec8f50a
+user_external_hash=0df781b261f55bbde73d8c92da3f99397000972f
 
 # Clear prior packages and install dependencies from apt.
 
 apt-get purge -qq -y owncloud* # we used to use the package manager
 
-apt_install php php-fpm \
+apt_install curl php${PHP_VER} php${PHP_VER}-fpm \
-	php-cli php-sqlite3 php-gd php-imap php-curl php-pear curl \
+	php${PHP_VER}-cli php${PHP_VER}-sqlite3 php${PHP_VER}-gd php${PHP_VER}-imap php${PHP_VER}-curl \
-	php-dev php-gd php-xml php-mbstring php-zip php-apcu php-json \
+	php${PHP_VER}-dev php${PHP_VER}-gd php${PHP_VER}-xml php${PHP_VER}-mbstring php${PHP_VER}-zip php${PHP_VER}-apcu \
-	php-intl php-imagick php-gmp php-bcmath
+	php${PHP_VER}-intl php${PHP_VER}-imagick php${PHP_VER}-gmp php${PHP_VER}-bcmath
+
+# Enable APC before Nextcloud tools are run.
+tools/editconf.py /etc/php/$PHP_VER/mods-available/apcu.ini -c ';' \
+	apc.enabled=1 \
+	apc.enable_cli=1
 
 InstallNextcloud() {
 
@@ -90,7 +95,7 @@ InstallNextcloud() {
 	# Starting with Nextcloud 15, the app user_external is no longer included in Nextcloud core,
 	# we will install from their github repository.
 	if [ -n "$version_user_external" ]; then
-		wget_verify https://github.com/nextcloud/user_external/releases/download/v$version_user_external/user_external-$version_user_external.tar.gz $hash_user_external /tmp/user_external.tgz
+		wget_verify https://github.com/nextcloud-releases/user_external/releases/download/v$version_user_external/user_external-v$version_user_external.tar.gz $hash_user_external /tmp/user_external.tgz
 		tar -xf /tmp/user_external.tgz -C /usr/local/lib/owncloud/apps/
 		rm /tmp/user_external.tgz
 	fi
@@ -112,20 +117,20 @@ InstallNextcloud() {
 	if [ -e $STORAGE_ROOT/owncloud/owncloud.db ]; then
 		# ownCloud 8.1.1 broke upgrades. It may fail on the first attempt, but
 		# that can be OK.
-		sudo -u www-data php /usr/local/lib/owncloud/occ upgrade
+		sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ upgrade
 		if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then
 			echo "Trying ownCloud upgrade again to work around ownCloud upgrade bug..."
-			sudo -u www-data php /usr/local/lib/owncloud/occ upgrade
+			sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ upgrade
 			if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then exit 1; fi
-			sudo -u www-data php /usr/local/lib/owncloud/occ maintenance:mode --off
+			sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ maintenance:mode --off
 			echo "...which seemed to work."
 		fi
 
 		# Add missing indices. NextCloud didn't include this in the normal upgrade because it might take some time.
-		sudo -u www-data php /usr/local/lib/owncloud/occ db:add-missing-indices
+		sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ db:add-missing-indices
 
 		# Run conversion to BigInt identifiers, this process may take some time on large tables.
-		sudo -u www-data php /usr/local/lib/owncloud/occ db:convert-filecache-bigint --no-interaction
+		sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ db:convert-filecache-bigint --no-interaction
 	fi
 }
 
@@ -137,7 +142,7 @@ InstallNextcloud() {
 
 # If config.php exists, get version number, otherwise CURRENT_NEXTCLOUD_VER is empty.
 if [ -f "$STORAGE_ROOT/owncloud/config.php" ]; then
-	CURRENT_NEXTCLOUD_VER=$(php -r "include(\"$STORAGE_ROOT/owncloud/config.php\"); echo(\$CONFIG['version']);")
+	CURRENT_NEXTCLOUD_VER=$(php$PHP_VER -r "include(\"$STORAGE_ROOT/owncloud/config.php\"); echo(\$CONFIG['version']);")
 else
 	CURRENT_NEXTCLOUD_VER=""
 fi
@@ -146,8 +151,8 @@ fi
 # from the version currently installed, do the install/upgrade
 if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextcloud_ver ]]; then
 
-	# Stop php-fpm if running. If theyre not running (which happens on a previously failed install), dont bail.
+	# Stop php-fpm if running. If they are not running (which happens on a previously failed install), dont bail.
-	service php7.2-fpm stop &> /dev/null || /bin/true
+	service php$PHP_VER-fpm stop &> /dev/null || /bin/true
 
 	# Backup the existing ownCloud/Nextcloud.
 	# Create a backup directory to store the current installation and database to
@@ -174,42 +179,21 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc
 		elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^1[012] ]]; then
 			echo "Upgrades from Mail-in-a-Box prior to v0.28 (dated July 30, 2018) with Nextcloud < 13.0.6 (you have ownCloud 10, 11 or 12) are not supported. Upgrade to Mail-in-a-Box version v0.30 first. Setup will continue, but skip the Nextcloud migration."
 			return 0
-		elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^13 ]]; then
+		elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^1[3456789] ]]; then
-			# If we are running Nextcloud 13, upgrade to Nextcloud 14
+			echo "Upgrades from Mail-in-a-Box prior to v60 with Nextcloud 19 or earlier are not supported. Upgrade to the latest Mail-in-a-Box version supported on your machine first. Setup will continue, but skip the Nextcloud migration."
-			InstallNextcloud 14.0.6 4e43a57340f04c2da306c8eea98e30040399ae5a 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 9d9717b29337613b72c74e9914c69b74b346c466
+			return 0
-			CURRENT_NEXTCLOUD_VER="14.0.6"
 		fi
-		if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^14 ]]; then
+		if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^20 ]]; then
-			# During the upgrade from Nextcloud 14 to 15, user_external may cause the upgrade to fail.
+			InstallNextcloud 21.0.7 f5c7079c5b56ce1e301c6a27c0d975d608bb01c9 4.0.7 45e7cf4bfe99cd8d03625cf9e5a1bb2e90549136 3.0.4 d0284b68135777ec9ca713c307216165b294d0fe
-			# We will disable it here before the upgrade and install it again after the upgrade.
+			CURRENT_NEXTCLOUD_VER="21.0.7"
-			hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:disable user_external
+		fi
-			InstallNextcloud 15.0.8 4129d8d4021c435f2e86876225fb7f15adf764a3 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 0.7.0 555a94811daaf5bdd336c5e48a78aa8567b86437
+		if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^21 ]]; then
-			CURRENT_NEXTCLOUD_VER="15.0.8"
+			InstallNextcloud 22.2.6 9d39741f051a8da42ff7df46ceef2653a1dc70d9 4.1.0 697f6b4a664e928d72414ea2731cb2c9d1dc3077 3.2.2 ce4030ab57f523f33d5396c6a81396d440756f5f 3.0.0 0df781b261f55bbde73d8c92da3f99397000972f
+			CURRENT_NEXTCLOUD_VER="22.2.6"
 		fi
-		if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^15 ]]; then
-                        InstallNextcloud 16.0.6 0bb3098455ec89f5af77a652aad553ad40a88819 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 0.7.0 555a94811daaf5bdd336c5e48a78aa8567b86437
-                        CURRENT_NEXTCLOUD_VER="16.0.6"
-        	fi
-	        if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^16 ]]; then
-                        InstallNextcloud 17.0.6 50b98d2c2f18510b9530e558ced9ab51eb4f11b0 3.3.0 e55d0357c6785d3b1f3b5f21780cb6d41d32443a 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 0.7.0 555a94811daaf5bdd336c5e48a78aa8567b86437
-                        CURRENT_NEXTCLOUD_VER="17.0.6"
-	        fi
-	        if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^17 ]]; then
-			# Don't exit the install if this column already exists (see #2076)
-			(echo "ALTER TABLE oc_flow_operations ADD COLUMN entity VARCHAR;" | sqlite3 $STORAGE_ROOT/owncloud/owncloud.db 2>/dev/null) || true
-                        InstallNextcloud 18.0.10 39c0021a8b8477c3f1733fddefacfa5ebf921c68 3.4.1 8f685e7dc99758636d660d595e389c324e51e9d1 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a
-                        CURRENT_NEXTCLOUD_VER="18.0.10"
-	        fi
-	        if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^18 ]]; then
-                        InstallNextcloud 19.0.4 01e98791ba12f4860d3d4047b9803f97a1b55c60 3.4.1 8f685e7dc99758636d660d595e389c324e51e9d1 2.0.3 a1f3835c752929e3598eb94f22300516867ac6ab 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a
-                        CURRENT_NEXTCLOUD_VER="19.0.4"
-	        fi
 	fi
 
 	InstallNextcloud $nextcloud_ver $nextcloud_hash $contacts_ver $contacts_hash $calendar_ver $calendar_hash $user_external_ver $user_external_hash
-
-	# Nextcloud 20 needs to have some optional columns added
-	sudo -u www-data php /usr/local/lib/owncloud/occ db:add-missing-columns
 fi
 
 # ### Configuring Nextcloud
@@ -235,10 +219,10 @@ if [ ! -f $STORAGE_ROOT/owncloud/owncloud.db ]; then
   'overwrite.cli.url' => '/cloud',
   'user_backends' => array(
     array(
-      'class' => 'OC_User_IMAP',
+      'class' => '\OCA\UserExternal\IMAP',
-        'arguments' => array(
+      'arguments' => array(
-          '127.0.0.1', 143, null
+        '127.0.0.1', 143, null, null, false, false
-         ),
+       ),
     ),
   ),
   'memcache.local' => '\OC\Memcache\APCu',
@@ -280,7 +264,7 @@ EOF
 	# Execute Nextcloud's setup step, which creates the Nextcloud sqlite database.
 	# It also wipes it if it exists. And it updates config.php with database
 	# settings and deletes the autoconfig.php file.
-	(cd /usr/local/lib/owncloud; sudo -u www-data php /usr/local/lib/owncloud/index.php;)
+	(cd /usr/local/lib/owncloud; sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/index.php;)
 fi
 
 # Update config.php.
@@ -296,10 +280,12 @@ fi
 # Use PHP to read the settings file, modify it, and write out the new settings array.
 TIMEZONE=$(cat /etc/timezone)
 CONFIG_TEMP=$(/bin/mktemp)
-php <<EOF > $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/config.php;
+php$PHP_VER <<EOF > $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/config.php;
 <?php
 include("$STORAGE_ROOT/owncloud/config.php");
 
+\$CONFIG['config_is_read_only'] = true;
+
 \$CONFIG['trusted_domains'] = array('$PRIMARY_HOSTNAME');
 
 \$CONFIG['memcache.local'] = '\OC\Memcache\APCu';
@@ -311,7 +297,14 @@ include("$STORAGE_ROOT/owncloud/config.php");
 
 \$CONFIG['mail_domain'] = '$PRIMARY_HOSTNAME';
 
-\$CONFIG['user_backends'] = array(array('class' => 'OC_User_IMAP','arguments' => array('127.0.0.1', 143, null),),);
+\$CONFIG['user_backends'] = array(
+  array(
+    'class' => '\OCA\UserExternal\IMAP',
+    'arguments' => array(
+      '127.0.0.1', 143, null, null, false, false
+    ),
+  ),
+);
 
 echo "<?php\n\\\$CONFIG = ";
 var_export(\$CONFIG);
@@ -324,25 +317,25 @@ chown www-data.www-data $STORAGE_ROOT/owncloud/config.php
 # The firstrunwizard gave Josh all sorts of problems, so disabling that.
 # user_external is what allows Nextcloud to use IMAP for login. The contacts
 # and calendar apps are the extensions we really care about here.
-hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:disable firstrunwizard
+hide_output sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/console.php app:disable firstrunwizard
-hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:enable user_external
+hide_output sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/console.php app:enable user_external
-hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:enable contacts
+hide_output sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/console.php app:enable contacts
-hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:enable calendar
+hide_output sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/console.php app:enable calendar
 
 # When upgrading, run the upgrade script again now that apps are enabled. It seems like
 # the first upgrade at the top won't work because apps may be disabled during upgrade?
 # Check for success (0=ok, 3=no upgrade needed).
-sudo -u www-data php /usr/local/lib/owncloud/occ upgrade
+sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ upgrade
 if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then exit 1; fi
 
 # Disable default apps that we don't support
 sudo -u www-data \
-	php /usr/local/lib/owncloud/occ app:disable photos dashboard activity \
+	php$PHP_VER /usr/local/lib/owncloud/occ app:disable photos dashboard activity \
 	| (grep -v "No such app enabled" || /bin/true)
 
 # Set PHP FPM values to support large file uploads
 # (semicolon is the comment character in this file, hashes produce deprecation warnings)
-tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/fpm/php.ini -c ';' \
 	upload_max_filesize=16G \
 	post_max_size=16G \
 	output_buffering=16384 \
@@ -351,7 +344,7 @@ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \
 	short_open_tag=On
 
 # Set Nextcloud recommended opcache settings
-tools/editconf.py /etc/php/7.2/cli/conf.d/10-opcache.ini -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/cli/conf.d/10-opcache.ini -c ';' \
 	opcache.enable=1 \
 	opcache.enable_cli=1 \
 	opcache.interned_strings_buffer=8 \
@@ -360,23 +353,20 @@ tools/editconf.py /etc/php/7.2/cli/conf.d/10-opcache.ini -c ';' \
 	opcache.save_comments=1 \
 	opcache.revalidate_freq=1
 
-# If apc is explicitly disabled we need to enable it
+# Migrate users_external data from <0.6.0 to version 3.0.0 (see https://github.com/nextcloud/user_external).
-if grep -q apc.enabled=0 /etc/php/7.2/mods-available/apcu.ini; then
+# This version was probably in use in Mail-in-a-Box v0.41 (February 26, 2019) and earlier.
-	tools/editconf.py /etc/php/7.2/mods-available/apcu.ini -c ';' \
+# We moved to v0.6.3 in 193763f8. Ignore errors - maybe there are duplicated users with the
-		apc.enabled=1
+# correct backend already.
-fi
+sqlite3 $STORAGE_ROOT/owncloud/owncloud.db "UPDATE oc_users_external SET backend='127.0.0.1';" || /bin/true
 
 # Set up a cron job for Nextcloud.
 cat > /etc/cron.d/mailinabox-nextcloud << EOF;
 #!/bin/bash
 # Mail-in-a-Box
-*/5 * * * *	root	sudo -u www-data php -f /usr/local/lib/owncloud/cron.php
+*/5 * * * *	root	sudo -u www-data php$PHP_VER -f /usr/local/lib/owncloud/cron.php
 EOF
 chmod +x /etc/cron.d/mailinabox-nextcloud
 
-# Remove previous hourly cronjob
-rm -f /etc/cron.hourly/mailinabox-owncloud
-
 # There's nothing much of interest that a user could do as an admin for Nextcloud,
 # and there's a lot they could mess up, so we don't make any users admins of Nextcloud.
 # But if we wanted to, we would do this:
@@ -387,4 +377,4 @@ rm -f /etc/cron.hourly/mailinabox-owncloud
 # ```
 
 # Enable PHP modules and restart PHP.
-restart_service php7.2-fpm
+restart_service php$PHP_VER-fpm

setup/preflight.sh

@@ -7,11 +7,11 @@ if [[ $EUID -ne 0 ]]; then
 	exit 1
 fi
 
-# Check that we are running on Ubuntu 18.04 LTS (or 18.04.xx).
+# Check that we are running on Ubuntu 20.04 LTS (or 20.04.xx).
-if [ "$(lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' )" != "Ubuntu 18.04 LTS" ]; then
+if [ "$( lsb_release --id --short )" != "Ubuntu" ] || [ "$( lsb_release --release --short )" != "22.04" ]; then
-	echo "Mail-in-a-Box only supports being installed on Ubuntu 18.04, sorry. You are running:"
+	echo "Mail-in-a-Box only supports being installed on Ubuntu 22.04, sorry. You are running:"
 	echo
-	lsb_release -d | sed 's/.*:\s*//'
+	lsb_release --description --short
 	echo
 	echo "We can't write scripts that run on every possible setup, sorry."
 	exit 1

setup/start.sh

@@ -67,6 +67,10 @@ fi
 fi
 
 # Create the STORAGE_USER and STORAGE_ROOT directory if they don't already exist.
+#
+# Set the directory and all of its parent directories' permissions to world
+# readable since it holds files owned by different processes.
+#
 # If the STORAGE_ROOT is missing the mailinabox.version file that lists a
 # migration (schema) number for the files stored there, assume this is a fresh
 # installation to that directory and write the file to contain the current
@@ -77,6 +81,8 @@ fi
 if [ ! -d $STORAGE_ROOT ]; then
 	mkdir -p $STORAGE_ROOT
 fi
+f=$STORAGE_ROOT
+while [[ $f != / ]]; do chmod a+rx "$f"; f=$(dirname "$f"); done;
 if [ ! -f $STORAGE_ROOT/mailinabox.version ]; then
 	setup/migrate.py --current > $STORAGE_ROOT/mailinabox.version
 	chown $STORAGE_USER.$STORAGE_USER $STORAGE_ROOT/mailinabox.version

setup/system.sh

@@ -97,12 +97,13 @@ fi
 # come from there and minimal Ubuntu installs may have it turned off.
 hide_output add-apt-repository -y universe
 
-# Install the certbot PPA.
-hide_output add-apt-repository -y ppa:certbot/certbot
-
 # Install the duplicity PPA.
 hide_output add-apt-repository -y ppa:duplicity-team/duplicity-release-git
 
+# Stock PHP is now 8.1, but we're transitioning through 8.0 because
+# of Nextcloud.
+hide_output add-apt-repository --y ppa:ondrej/php
+
 # ### Update Packages
 
 # Update system packages to make sure we have the latest upstream versions
@@ -123,9 +124,6 @@ apt_get_quiet autoremove
 
 # Install basic utilities.
 #
-# * haveged: Provides extra entropy to /dev/random so it doesn't stall
-#	         when generating random numbers for private keys (e.g. during
-#	         ldns-keygen).
 # * unattended-upgrades: Apt tool to install security updates automatically.
 # * cron: Runs background processes periodically.
 # * ntp: keeps the system time correct
@@ -139,8 +137,8 @@ apt_get_quiet autoremove
 
 echo Installing system packages...
 apt_install python3 python3-dev python3-pip python3-setuptools \
-	netcat-openbsd wget curl git sudo coreutils bc \
+	netcat-openbsd wget curl git sudo coreutils bc file \
-	haveged pollinate openssh-client unzip \
+	pollinate openssh-client unzip \
 	unattended-upgrades cron ntp fail2ban rsyslog
 
 # ### Suppress Upgrade Prompts
@@ -331,7 +329,7 @@ fi #NODOC
 #  	If more queries than specified are sent, bind9 returns SERVFAIL. After flushing the cache during system checks,
 #	we ran into the limit thus we are increasing it from 75 (default value) to 100.
 apt_install bind9
-tools/editconf.py /etc/default/bind9 \
+tools/editconf.py /etc/default/named \
 	"OPTIONS=\"-u bind -4\""
 if ! grep -q "listen-on " /etc/bind/named.conf.options; then
 	# Add a listen-on directive if it doesn't exist inside the options block.

setup/web.sh

@@ -19,7 +19,7 @@ fi
 
 echo "Installing Nginx (web server)..."
 
-apt_install nginx php-cli php-fpm idn2
+apt_install nginx php${PHP_VER}-cli php${PHP_VER}-fpm idn2
 
 rm -f /etc/nginx/sites-enabled/default
 
@@ -46,15 +46,15 @@ tools/editconf.py /etc/nginx/nginx.conf -s \
 	ssl_protocols="TLSv1.2 TLSv1.3;"
 
 # Tell PHP not to expose its version number in the X-Powered-By header.
-tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/fpm/php.ini -c ';' \
 	expose_php=Off
 
 # Set PHPs default charset to UTF-8, since we use it. See #367.
-tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/fpm/php.ini -c ';' \
         default_charset="UTF-8"
 
 # Configure the path environment for php-fpm
-tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \
+tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
 	env[PATH]=/usr/local/bin:/usr/bin:/bin \
 
 # Configure php-fpm based on the amount of memory the machine has
@@ -64,7 +64,7 @@ tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \
 TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}' || /bin/true)
 if [ $TOTAL_PHYSICAL_MEM -lt 1000000 ]
 then
-        tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \
+        tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
                 pm=ondemand \
                 pm.max_children=8 \
                 pm.start_servers=2 \
@@ -72,7 +72,7 @@ then
                 pm.max_spare_servers=3
 elif [ $TOTAL_PHYSICAL_MEM -lt 2000000 ]
 then
-        tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \
+        tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
                 pm=ondemand \
                 pm.max_children=16 \
                 pm.start_servers=4 \
@@ -80,14 +80,14 @@ then
                 pm.max_spare_servers=6
 elif [ $TOTAL_PHYSICAL_MEM -lt 3000000 ]
 then
-        tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \
+        tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
                 pm=dynamic \
                 pm.max_children=60 \
                 pm.start_servers=6 \
                 pm.min_spare_servers=3 \
                 pm.max_spare_servers=9
 else
-        tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \
+        tools/editconf.py /etc/php/$PHP_VER/fpm/pool.d/www.conf -c ';' \
                 pm=dynamic \
                 pm.max_children=120 \
                 pm.start_servers=12 \
@@ -147,7 +147,7 @@ chown -R $STORAGE_USER $STORAGE_ROOT/www
 
 # Start services.
 restart_service nginx
-restart_service php7.2-fpm
+restart_service php$PHP_VER-fpm
 
 # Open ports.
 ufw_allow http

setup/webmail.sh

@@ -22,8 +22,8 @@ source /etc/mailinabox.conf # load global vars
 echo "Installing Roundcube (webmail)..."
 apt_install \
 	dbconfig-common \
-	php-cli php-sqlite3 php-intl php-json php-common php-curl php-ldap \
+	php${PHP_VER}-cli php${PHP_VER}-sqlite3 php${PHP_VER}-intl php${PHP_VER}-common php${PHP_VER}-curl php${PHP_VER}-imap \
-	php-gd php-pspell tinymce libjs-jquery libjs-jquery-mousewheel libmagic1 php-mbstring
+	php${PHP_VER}-gd php${PHP_VER}-pspell php${PHP_VER}-mbstring libjs-jquery libjs-jquery-mousewheel libmagic1
 
 # Install Roundcube from source if it is not already present or if it is out of date.
 # Combine the Roundcube version number with the commit hash of plugins to track
@@ -35,12 +35,12 @@ apt_install \
 #   https://github.com/mstilkerich/rcmcarddav/releases
 # The easiest way to get the package hashes is to run this script and get the hash from
 # the error message.
-VERSION=1.5.2
+VERSION=1.6.0
-HASH=208ce4ca0be423cc0f7070ff59bd03588b4439bf
+HASH=fd84b4fac74419bb73e7a3bcae1978d5589c52de
-PERSISTENT_LOGIN_VERSION=59ca1b0d3a02cff5fa621c1ad581d15f9d642fe8
+PERSISTENT_LOGIN_VERSION=bde7b6840c7d91de627ea14e81cf4133cbb3c07a # version 5.2
 HTML5_NOTIFIER_VERSION=68d9ca194212e15b3c7225eb6085dbcf02fd13d7 # version 0.6.4+
-CARDDAV_VERSION=4.3.0
+CARDDAV_VERSION=4.4.3
-CARDDAV_HASH=4ad7df8843951062878b1375f77c614f68bc5c61
+CARDDAV_HASH=74f8ba7aee33e78beb9de07f7f44b81f6071b644
 
 UPDATE_KEY=$VERSION:$PERSISTENT_LOGIN_VERSION:$HTML5_NOTIFIER_VERSION:$CARDDAV_VERSION
 
@@ -83,7 +83,7 @@ if [ $needs_update == 1 ]; then
 
 	# download and verify the full release of the carddav plugin
 	wget_verify \
-		https://github.com/blind-coder/rcmcarddav/releases/download/v${CARDDAV_VERSION}/carddav-v${CARDDAV_VERSION}.tar.gz \
+		https://github.com/mstilkerich/rcmcarddav/releases/download/v${CARDDAV_VERSION}/carddav-v${CARDDAV_VERSION}.tar.gz \
 		$CARDDAV_HASH \
 		/tmp/carddav.tar.gz
 
@@ -115,8 +115,7 @@ cat > $RCM_CONFIG <<EOF;
 \$config['log_dir'] = '/var/log/roundcubemail/';
 \$config['temp_dir'] = '/var/tmp/roundcubemail/';
 \$config['db_dsnw'] = 'sqlite:///$STORAGE_ROOT/mail/roundcube/roundcube.sqlite?mode=0640';
-\$config['default_host'] = 'ssl://localhost';
+\$config['imap_host'] = 'ssl://localhost:993';
-\$config['default_port'] = 993;
 \$config['imap_conn_options'] = array(
   'ssl'         => array(
      'verify_peer'  => false,
@@ -124,7 +123,7 @@ cat > $RCM_CONFIG <<EOF;
    ),
  );
 \$config['imap_timeout'] = 15;
-\$config['smtp_server'] = 'tls://127.0.0.1';
+\$config['smtp_host'] = 'tls://127.0.0.1';
 \$config['smtp_conn_options'] = array(
   'ssl'         => array(
      'verify_peer'  => false,
@@ -141,6 +140,10 @@ cat > $RCM_CONFIG <<EOF;
 \$config['login_username_filter'] = 'email';
 \$config['password_charset'] = 'UTF-8';
 \$config['junk_mbox'] = 'Spam';
+/* ensure roudcube session id's aren't leaked to other parts of the server */
+\$config['session_path'] = '/mail/';
+/* prevent CSRF, requires php 7.3+ */
+\$config['session_samesite'] = 'Strict';
 ?>
 EOF
 
@@ -154,7 +157,7 @@ cat > ${RCM_PLUGIN_DIR}/carddav/config.inc.php <<EOF;
 	 'name'         =>  'ownCloud',
 	 'username'     =>  '%u', // login username
 	 'password'     =>  '%p', // login password
-	 'url'          =>  'https://${PRIMARY_HOSTNAME}/cloud/remote.php/carddav/addressbooks/%u/contacts',
+	 'url'          =>  'https://${PRIMARY_HOSTNAME}/cloud/remote.php/dav/addressbooks/users/%u/contacts/',
 	 'active'       =>  true,
 	 'readonly'     =>  false,
 	 'refresh_time' => '02:00:00',
@@ -202,10 +205,10 @@ chown -f -R root.www-data ${RCM_PLUGIN_DIR}/carddav
 chmod -R 774 ${RCM_PLUGIN_DIR}/carddav
 
 # Run Roundcube database migration script (database is created if it does not exist)
-${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube
+php$PHP_VER ${RCM_DIR}/bin/updatedb.sh --dir ${RCM_DIR}/SQL --package roundcube
 chown www-data:www-data $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 chmod 664 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
 
 # Enable PHP modules.
-phpenmod -v php mcrypt imap
+phpenmod -v $PHP_VER imap
-restart_service php7.2-fpm
+restart_service php$PHP_VER-fpm

setup/zpush.sh

@@ -17,9 +17,9 @@ source /etc/mailinabox.conf # load global vars
 
 echo "Installing Z-Push (Exchange/ActiveSync server)..."
 apt_install \
-	php-soap php-imap libawl-php php-xsl
+       php${PHP_VER}-soap php${PHP_VER}-imap libawl-php php$PHP_VER-xml
 
-phpenmod -v php imap
+phpenmod -v $PHP_VER imap
 
 # Copy Z-Push into place.
 VERSION=2.6.2
@@ -42,8 +42,6 @@ if [ $needs_update == 1 ]; then
 	rm -rf /tmp/z-push.zip /tmp/z-push
 
 	rm -f /usr/sbin/z-push-{admin,top}
-	ln -s /usr/local/lib/z-push/z-push-admin.php /usr/sbin/z-push-admin
-	ln -s /usr/local/lib/z-push/z-push-top.php /usr/sbin/z-push-top
 	echo $VERSION > /usr/local/lib/z-push/version
 fi
 
@@ -102,8 +100,8 @@ EOF
 
 # Restart service.
 
-restart_service php7.2-fpm
+restart_service php$PHP_VER-fpm
 
 # Fix states after upgrade
 
-hide_output z-push-admin -a fixstates
+hide_output php$PHP_VER /usr/local/lib/z-push/z-push-admin.php -a fixstates

tests/test_dns.py

@@ -48,7 +48,7 @@ def test2(tests, server, description):
 	for qname, rtype, expected_answer in tests:
 		# do the query and format the result as a string
 		try:
-			response = dns.resolver.query(qname, rtype)
+			response = dns.resolver.resolve(qname, rtype)
 		except dns.resolver.NoNameservers:
 			# host did not have an answer for this query
 			print("Could not connect to %s for DNS query." % server)

tests/test_mail.py

@@ -48,7 +48,7 @@ server = smtplib.SMTP_SSL(host)
 ipaddr = socket.gethostbyname(host) # IPv4 only!
 reverse_ip = dns.reversename.from_address(ipaddr) # e.g. "1.0.0.127.in-addr.arpa."
 try:
-	reverse_dns = dns.resolver.query(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname
+	reverse_dns = dns.resolver.resolve(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname
 except dns.resolver.NXDOMAIN:
 	print("Reverse DNS lookup failed for %s. SMTP EHLO name check skipped." % ipaddr)
 	reverse_dns = None

tools/editconf.py

@@ -14,6 +14,10 @@
 #
 # NAME VALUE
 #
+# If the -e option is given and VALUE is empty, the setting is removed
+# from the configuration file if it is set (i.e. existing occurrences
+# are commented out and no new setting is added).
+#
 # If the -c option is given, then the supplied character becomes the comment character
 #
 # If the -w option is given, then setting lines continue onto following
@@ -35,6 +39,7 @@ settings = sys.argv[2:]
 
 delimiter = "="
 delimiter_re = r"\s*=\s*"
+erase_setting = False
 comment_char = "#"
 folded_lines = False
 testing = False
@@ -44,6 +49,9 @@ while settings[0][0] == "-" and settings[0] != "--":
 		# Space is the delimiter
 		delimiter = " "
 		delimiter_re = r"\s+"
+	elif opt == "-e":
+		# Erase settings that have empty values.
+		erase_setting = True
 	elif opt == "-w":
 		# Line folding is possible in this file.
 		folded_lines = True
@@ -81,7 +89,7 @@ while len(input_lines) > 0:
 
 	# See if this line is for any settings passed on the command line.
 	for i in range(len(settings)):
-		# Check that this line contain this setting from the command-line arguments.
+		# Check if this line contain this setting from the command-line arguments.
 		name, val = settings[i].split("=", 1)
 		m = re.match(
 			   "(\s*)"
@@ -91,8 +99,10 @@ while len(input_lines) > 0:
 		if not m: continue
 		indent, is_comment, existing_val = m.groups()
 
-		# If this is already the setting, do nothing.
+		# If this is already the setting, keep it in the file, except:
-		if is_comment is None and existing_val == val:
+		# * If we've already seen it before, then remove this duplicate line.
+		# * If val is empty and erase_setting is on, then comment it out.
+		if is_comment is None and existing_val == val and not (not val and erase_setting):
 			# It may be that we've already inserted this setting higher
 			# in the file so check for that first.
 			if i in found: break
@@ -107,8 +117,9 @@ while len(input_lines) > 0:
 			# the line is already commented, pass it through
 			buf += line
 		
-		# if this option oddly appears more than once, don't add the setting again
+		# if this option already is set don't add the setting again,
-		if i in found:
+		# or if we're clearing the setting with -e, don't add it
+		if (i in found) or (not val and erase_setting):
 			break
 		
 		# add the new setting
@@ -122,11 +133,13 @@ while len(input_lines) > 0:
 		# If did not match any setting names, pass this line through.
 		buf += line
 		
-# Put any settings we didn't see at the end of the file.
+# Put any settings we didn't see at the end of the file,
+# except settings being cleared.
 for i in range(len(settings)):
 	if i not in found:
 		name, val = settings[i].split("=", 1)
-		buf += name + delimiter + val + "\n"
+		if not (not val and erase_setting):
+			buf += name + delimiter + val + "\n"
 
 if not testing:
 	# Write out the new file.

tools/owncloud-restore.sh

@@ -26,7 +26,7 @@ if [ ! -f $1/config.php ]; then
 fi
 
 echo "Restoring backup from $1"
-service php7.2-fpm stop
+service php8.0-fpm stop
 
 # remove the current ownCloud/Nextcloud installation
 rm -rf /usr/local/lib/owncloud/
@@ -43,7 +43,7 @@ ln -sf $STORAGE_ROOT/owncloud/config.php /usr/local/lib/owncloud/config/config.p
 chown -f -R www-data.www-data $STORAGE_ROOT/owncloud /usr/local/lib/owncloud
 chown www-data.www-data $STORAGE_ROOT/owncloud/config.php
 
-sudo -u www-data php /usr/local/lib/owncloud/occ maintenance:mode --off
+sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ maintenance:mode --off
 
-service php7.2-fpm start
+service php8.0-fpm start
 echo "Done"

tools/owncloud-unlockadmin.sh

@@ -20,4 +20,4 @@ echo
 echo Press enter to continue.
 read
 
-sudo -u www-data php /usr/local/lib/owncloud/occ group:adduser admin $ADMIN && echo Done.
+sudo -u www-data php$PHP_VER /usr/local/lib/owncloud/occ group:adduser admin $ADMIN && echo Done.