Import changes from upstream (v56)

CHANGELOG.md

@@ -1,6 +1,29 @@
 CHANGELOG
 =========
 
+Version 56 (January 19, 2022)
+-----------------------------
+
+Software updates:
+
+* Roundcube updated to 1.5.2 (from 1.5.0), and the persistent_login and CardDAV (to 4.3.0 from 3.0.3) plugins are updated.
+* Nextcloud updated to 20.0.14 (from 20.0.8), contacts to 4.0.7 (from 3.5.1), and calendar to 3.0.4 (from 2.2.0).
+
+Setup:
+
+* Fixed failed setup if a previous attempt failed while updating Nextcloud.
+
+Control panel:
+
+* Fixed a crash if a custom DNS entry is not under a zone managed by the box.
+* Fix DNSSEC instructions typo.
+
+Other:
+
+* Set systemd journald log retention to 10 days (from no limit) to reduce disk usage.
+* Fixed log processing for submission lines that have a sasl_sender or other extra information.
+* Fix DNS secondary nameserver refesh failure retry period.
+
 Version 55 (October 18, 2021)
 -----------------------------
 

CONTRIBUTING.md

@@ -20,9 +20,9 @@ _If you're seeing an error message about your *IP address being listed in the Sp
 
 ### Modifying your `hosts` file
 
-After a while, Mail-in-a-Box will be available at `192.168.50.4` (unless you changed that in your `Vagrantfile`). To be able to use the web-based bits, we recommend to add a hostname to your `hosts` file:
+After a while, Mail-in-a-Box will be available at `192.168.56.4` (unless you changed that in your `Vagrantfile`). To be able to use the web-based bits, we recommend to add a hostname to your `hosts` file:
 
-    $ echo "192.168.50.4 mailinabox.lan" | sudo tee -a /etc/hosts
+    $ echo "192.168.56.4 mailinabox.lan" | sudo tee -a /etc/hosts
 
 You should now be able to navigate to https://mailinabox.lan/admin using your browser. There should be an initial admin user with the name `me@mailinabox.lan` and the password `12345678`.
 

api/mailinabox.yml

@@ -77,7 +77,7 @@ paths:
       x-codeSamples:
         - lang: curl
           source: |
-            curl -X GET "https://{host}/admin/login" \
+            curl -X POST "https://{host}/admin/login" \
               -u "<email>:<password>"
       responses:
         200:
@@ -109,13 +109,15 @@ paths:
       x-codeSamples:
         - lang: curl
           source: |
-            curl -X GET "https://{host}/admin/logout" \
+            curl -X POST "https://{host}/admin/logout" \
               -u "<email>:<session_key>"
       responses:
         200:
           description: Successful operation
           content:
             application/json:
+              schema:
+                $ref: '#/components/schemas/LogoutResponse'
   /system/status:
     post:
       tags:
@@ -3415,3 +3417,8 @@ components:
           nullable: true
     MfaDisableSuccessResponse:
       type: string
+    LogoutResponse:
+      type: object
+      properties:
+        status:
+          type: string

management/daemon.py

@@ -356,7 +356,7 @@ def dns_get_records(qname=None, rtype=None):
 		r["sort-order"]["created"] = i
 	domain_sort_order = utils.sort_domains([r["qname"] for r in records], env)
 	for i, r in enumerate(sorted(records, key = lambda r : (
-			zones.index(r["zone"]),
+			zones.index(r["zone"]) if r.get("zone") else 0, # record is not within a zone managed by the box
 			domain_sort_order.index(r["qname"]),
 			r["rtype"]))):
 		r["sort-order"]["qname"] = i

management/dns_update.py

@@ -570,7 +570,7 @@ def write_nsd_zone(domain, zonefile, records, env, force):
 	# @ the PRIMARY_HOSTNAME. Hopefully that's legit.
 	#
 	# For the refresh through TTL fields, a good reference is:
-	# http://www.peerwisdom.org/2013/05/15/dns-understanding-the-soa-record/
+	# https://www.ripe.net/publications/docs/ripe-203
 	#
 	# A hash of the available DNSSEC keys are added in a comment so that when
 	# the keys change we force a re-generation of the zone which triggers
@@ -583,7 +583,7 @@ $TTL {ttl}          ; default time to live
 @ IN SOA ns1.{primary_domain}. hostmaster.{primary_domain}. (
            __SERIAL__     ; serial number
            7200     ; Refresh (secondary nameserver update interval)
-           {ttl}    ; Retry (when refresh fails, how often to try again)
+           3600     ; Retry (when refresh fails, how often to try again, should be lower than the refresh)
            1209600  ; Expire (when refresh fails, how long secondary nameserver will keep records around anyway)
            {ttl}    ; Negative TTL (how long negative responses are cached)
            )

management/mail_log.py

@@ -549,8 +549,9 @@ def scan_postfix_submission_line(date, log, collector):
     """
 
     # Match both the 'plain' and 'login' sasl methods, since both authentication methods are
-    # allowed by Dovecot
+    # allowed by Dovecot. Exclude trailing comma after the username when additional fields 
-    m = re.match("([A-Z0-9]+): client=(\S+), sasl_method=(PLAIN|LOGIN), sasl_username=(\S+)", log)
+	# follow after.
+    m = re.match("([A-Z0-9]+): client=(\S+), sasl_method=(PLAIN|LOGIN), sasl_username=(\S+)(?<!,)", log)
 
     if m:
         _, client, method, user = m.groups()

management/status_checks.py

@@ -803,7 +803,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
 		output.print_line("Option " + str(i+1) + ":")
 		output.print_line("----------")
 		output.print_line("Key Tag: " + ds_suggestion['keytag'])
-		output.print_line("Key Flags: KSK (256)")
+		output.print_line("Key Flags: KSK / 257")
 		output.print_line("Algorithm: %s / %s" % (ds_suggestion['alg'], ds_suggestion['alg_name']))
 		output.print_line("Digest Type: %s / %s" % (ds_suggestion['digalg'], ds_suggestion['digalg_name']))
 		output.print_line("Digest: " + ds_suggestion['digest'])

management/templates/system-backup.html

@@ -5,7 +5,7 @@
 
 <h2>Backup Status</h2>
 
-<p>The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also store in on S3-compatible services like Amazon Web Services (AWS).</p>
+<p>The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also store it on S3-compatible services like Amazon Web Services (AWS).</p>
 
 <h3>Configuration</h3>
 

security.md

@@ -3,7 +3,12 @@ Mail-in-a-Box Security Guide
 
 Mail-in-a-Box turns a fresh Ubuntu 18.04 LTS 64-bit machine into a mail server appliance by installing and configuring various components.
 
-This page documents the security features of Mail-in-a-Box. The term “box” is used below to mean a configured Mail-in-a-Box.
+This page documents the security posture of Mail-in-a-Box. The term “box” is used below to mean a configured Mail-in-a-Box.
+
+Reporting Security Vulnerabilities
+----------------------------------
+
+Security vulnerabilities should be reported to the [project's maintainer](https://joshdata.me) via email.
 
 Threat Model
 ------------
@@ -49,9 +54,7 @@ Additionally:
 
 ### Password Storage
 
-The passwords for mail users are stored on disk using the [SHA512-CRYPT](http://man7.org/linux/man-pages/man3/crypt.3.html) hashing scheme. ([source](management/mailconfig.py))
+The passwords for mail users are stored on disk using the [SHA512-CRYPT](http://man7.org/linux/man-pages/man3/crypt.3.html) hashing scheme. ([source](management/mailconfig.py)) Password changes (as well as changes to control panel two-factor authentication settings) expire any control panel login sessions.
-
-When using the web-based administrative control panel, after logging in an API key is placed in the browser's local storage (rather than, say, the user's actual password). The API key is an HMAC based on the user's email address and current password, and it is keyed by a secret known only to the control panel service. By resetting an administrator's password, any HMACs previously generated for that user will expire.
 
 ### Console access
 
@@ -65,7 +68,7 @@ If DNSSEC is enabled at the box's domain name's registrar, the SSHFP record that
 
 `fail2ban` provides some protection from brute-force login attacks (repeated logins that guess account passwords) by blocking offending IP addresses at the network level.
 
-The following services are protected: SSH, IMAP (dovecot), SMTP submission (postfix), webmail (roundcube), Nextcloud/CalDAV/CardDAV (over HTTP), and the Mail-in-a-Box control panel & munin (over HTTP).
+The following services are protected: SSH, IMAP (dovecot), SMTP submission (postfix), webmail (roundcube), Nextcloud/CalDAV/CardDAV (over HTTP), and the Mail-in-a-Box control panel (over HTTP).
 
 Some other services running on the box may be missing fail2ban filters.
 

setup/management.sh

@@ -25,7 +25,7 @@ done
 #
 # certbot installs EFF's certbot which we use to
 # provision free TLS certificates.
-apt_install duplicity python3-pip python3-gpg virtualenv certbot
+apt_install duplicity python3-pip python3-gpg virtualenv certbot rsync
 
 # boto is used for amazon aws backups.
 # Both are installed outside the pipenv, so they can be used by duplicity

setup/nextcloud.sh

@@ -9,6 +9,41 @@ source /etc/mailinabox.conf # load global vars
 
 echo "Installing Nextcloud (contacts/calendar)..."
 
+# Nextcloud core and app (plugin) versions to install.
+# With each version we store a hash to ensure we install what we expect.
+
+# Nextcloud core
+# --------------
+# * See https://nextcloud.com/changelog for the latest version.
+# * Check https://docs.nextcloud.com/server/latest/admin_manual/installation/system_requirements.html
+#   for whether it supports the version of PHP available on this machine.
+# * Since Nextcloud only supports upgrades from consecutive major versions,
+#   we automatically install intermediate versions as needed.
+# * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and
+#   copying it from the error message when it doesn't match what is below.
+nextcloud_ver=20.0.14
+nextcloud_hash=92cac708915f51ee2afc1787fd845476fd090c81
+
+# Nextcloud apps
+# --------------
+# * Find the most recent tag that is compatible with the Nextcloud version above by
+#   consulting the <dependencies>...<nextcloud> node at:
+#   https://github.com/nextcloud-releases/contacts/blob/maaster/appinfo/info.xml
+#   https://github.com/nextcloud-releases/calendar/blob/master/appinfo/info.xml
+#   https://github.com/nextcloud/user_external/blob/master/appinfo/info.xml
+# * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and
+#   copying it from the error message when it doesn't match what is below.
+contacts_ver=4.0.7
+contacts_hash=8ab31d205408e4f12067d8a4daa3595d46b513e3
+calendar_ver=3.0.4
+calendar_hash=6fb1e998d307c53245faf1c37a96eb982bbee8ba
+user_external_ver=1.0.0
+user_external_hash=3bf2609061d7214e7f0f69dd8883e55c4ec8f50a
+
+# Clear prior packages and install dependencies from apt.
+
+apt-get purge -qq -y owncloud* # we used to use the package manager
+
 apt_install php php-fpm \
 	php-cli php-sqlite3 php-gd php-imap php-curl php-pear curl \
 	php-dev php-gd php-xml php-mbstring php-zip php-apcu php-json \
@@ -44,11 +79,11 @@ InstallNextcloud() {
 	# their github repositories.
 	mkdir -p /usr/local/lib/owncloud/apps
 
-	wget_verify https://github.com/nextcloud-releases/contacts/releases/download/v$version_contacts/contacts.tar.gz $hash_contacts /tmp/contacts.tgz
+	wget_verify https://github.com/nextcloud-releases/contacts/releases/download/v$version_contacts/contacts-v$version_contacts.tar.gz $hash_contacts /tmp/contacts.tgz
 	tar xf /tmp/contacts.tgz -C /usr/local/lib/owncloud/apps/
 	rm /tmp/contacts.tgz
 
-	wget_verify https://github.com/nextcloud-releases/calendar/releases/download/v$version_calendar/calendar.tar.gz $hash_calendar /tmp/calendar.tgz
+	wget_verify https://github.com/nextcloud-releases/calendar/releases/download/v$version_calendar/calendar-v$version_calendar.tar.gz $hash_calendar /tmp/calendar.tgz
 	tar xf /tmp/calendar.tgz -C /usr/local/lib/owncloud/apps/
 	rm /tmp/calendar.tgz
 
@@ -94,16 +129,6 @@ InstallNextcloud() {
 	fi
 }
 
-# Nextcloud Version to install. Checks are done down below to step through intermediate versions.
-nextcloud_ver=20.0.13
-nextcloud_hash=a1fe460e3e65753552fb4beb1a47ab09ff277d95
-contacts_ver=4.0.3
-contacts_hash=5ddd629e72fdb80c5ff80186954f32a464b9dcd1
-calendar_ver=2.3.4
-calendar_hash=bf4a8e5d40731013a6f971eea3149e852e9fb47c
-user_external_ver=1.0.0
-user_external_hash=3bf2609061d7214e7f0f69dd8883e55c4ec8f50a
-
 # Current Nextcloud Version, #1623
 # Checking /usr/local/lib/owncloud/version.php shows version of the Nextcloud application, not the DB
 # $STORAGE_ROOT/owncloud is kept together even during a backup.  It is better to rely on config.php than
@@ -170,7 +195,8 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc
                         CURRENT_NEXTCLOUD_VER="17.0.6"
 	        fi
 	        if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^17 ]]; then
-			echo "ALTER TABLE oc_flow_operations ADD COLUMN entity VARCHAR;" | sqlite3 $STORAGE_ROOT/owncloud/owncloud.db
+			# Don't exit the install if this column already exists (see #2076)
+			(echo "ALTER TABLE oc_flow_operations ADD COLUMN entity VARCHAR;" | sqlite3 $STORAGE_ROOT/owncloud/owncloud.db 2>/dev/null) || true
                         InstallNextcloud 18.0.10 39c0021a8b8477c3f1733fddefacfa5ebf921c68 3.4.1 aee680a75e95f26d9285efd3c1e25cf7f3bfd27e 2.0.3 9d9717b29337613b72c74e9914c69b74b346c466 1.0.0 3bf2609061d7214e7f0f69dd8883e55c4ec8f50a
                         CURRENT_NEXTCLOUD_VER="18.0.10"
 	        fi

setup/system.sh

@@ -75,7 +75,12 @@ then
 	fi
 fi
 
-# Certbot doesn't require a PPA in Debian
+# ### Set log retention policy.
+
+# Set the systemd journal log retention from infinite to 10 days,
+# since over time the logs take up a large amount of space.
+# (See https://discourse.mailinabox.email/t/journalctl-reclaim-space-on-small-mailinabox/6728/11.)
+management/editconf.py /etc/systemd/journald.conf MaxRetentionSec=10day
 
 # ### Update Packages
 

setup/webmail.sh

@@ -28,13 +28,19 @@ apt_install \
 # Install Roundcube from source if it is not already present or if it is out of date.
 # Combine the Roundcube version number with the commit hash of plugins to track
 # whether we have the latest version of everything.
-
+# For the latest versions, see:
-VERSION=1.5.0
+#   https://github.com/roundcube/roundcubemail/releases
-HASH=2a9d11d9c10c8e8756120606c47eef702f00fe6d
+#   https://github.com/mfreiholz/persistent_login/commits/master
-PERSISTENT_LOGIN_VERSION=327362e7615e9192ea7a8871e5a0230b8ea6930a # version 5.2.0 + chinese localization
+#   https://github.com/stremlau/html5_notifier/commits/master
+#   https://github.com/mstilkerich/rcmcarddav/releases
+# The easiest way to get the package hashes is to run this script and get the hash from
+# the error message.
+VERSION=1.5.2
+HASH=208ce4ca0be423cc0f7070ff59bd03588b4439bf
+PERSISTENT_LOGIN_VERSION=59ca1b0d3a02cff5fa621c1ad581d15f9d642fe8
 HTML5_NOTIFIER_VERSION=68d9ca194212e15b3c7225eb6085dbcf02fd13d7 # version 0.6.4+
-CARDDAV_VERSION=4.2.0
+CARDDAV_VERSION=4.3.0
-CARDDAV_HASH=d412a038bb698eeee631ec20e8b1118baeb2e554
+CARDDAV_HASH=4ad7df8843951062878b1375f77c614f68bc5c61
 
 UPDATE_KEY=$VERSION:$PERSISTENT_LOGIN_VERSION:$HTML5_NOTIFIER_VERSION:$CARDDAV_VERSION
 
@@ -79,11 +85,11 @@ if [ $needs_update == 1 ]; then
 	wget_verify \
 		https://github.com/blind-coder/rcmcarddav/releases/download/v${CARDDAV_VERSION}/carddav-v${CARDDAV_VERSION}.tar.gz \
 		$CARDDAV_HASH \
-		/tmp/carddav.tgz
+		/tmp/carddav.tar.gz
 
 	# unzip and cleanup
-	tar xf /tmp/carddav.tgz -C ${RCM_PLUGIN_DIR}
+	tar -C ${RCM_PLUGIN_DIR} -zxf /tmp/carddav.tar.gz
-	rm -f /tmp/carddav.tgz
+	rm -f /tmp/carddav.tar.gz
 
 	# record the version we've installed
 	echo $UPDATE_KEY > ${RCM_DIR}/version