Add missing sudo to upgrade instruction (#1816)

CHANGELOG

@@ -6,6 +6,23 @@ The version corresponds to SimpleLogin Docker `image tag`.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [4.6.2] - 2022-06-15
+- support dot in alias prefix
+- Set the "X-SimpleLogin-Envelope-To" to the alias during forward
+- Use a different format for VERP
+- use same footer as landing page
+- remove obsolete endpoints /alias/custom/new, /alias/options, /v2/alias/options, /v3/alias/options
+- support search on contact page
+- Better search using Postgres fulltext search
+- Use AGPL license instead of MIT
+- auto-create alias rule for custom domain
+- use re2 instead of re to avoid ReDOS attack
+- Use alembic instead of flask migrate which depends on flask-sqlalchemy
+- Remove flask-sqlalchemy
+- Able to block sender
+- subdomain
+- GPDR data export
+
 ## [3.4.0] - 2021-04-06
 Support ARM arch
 Remove unused config like DEBUG, CLOUDWATCH, DKIM_PUBLIC_KEY_PATH, DKIM_DNS_VALUE

Dockerfile

@@ -5,7 +5,7 @@ COPY ./static/package*.json /code/static/
 RUN cd /code/static && npm install
 
 # Main image
-FROM python:3.7
+FROM python:3.10
 
 # Keeps Python from generating .pyc files in the container
 ENV PYTHONDONTWRITEBYTECODE 1
@@ -13,7 +13,7 @@ ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
 # Add poetry to PATH
-ENV PATH="${PATH}:/root/.poetry/bin"
+ENV PATH="${PATH}:/root/.local/bin"
 
 WORKDIR /code
 
@@ -24,7 +24,7 @@ COPY poetry.lock pyproject.toml ./
 RUN pip install -U pip \
     && apt-get update \
     && apt install -y curl netcat gcc python3-dev gnupg git libre2-dev \
-    && curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - \
+    && curl -sSL https://install.python-poetry.org | python3 - \
     # Remove curl and netcat from the image
     && apt-get purge -y curl netcat \
     # Run poetry

README.md

@@ -329,8 +329,8 @@ smtpd_recipient_restrictions =
    reject_unknown_recipient_domain,
    permit_mynetworks,
    reject_unauth_destination,
-   reject_rbl_client zen.spamhaus.org,
-   reject_rbl_client bl.spamcop.net,
+   reject_rbl_client zen.spamhaus.org=127.0.0.[2..11],
+   reject_rbl_client bl.spamcop.net=127.0.0.2,
    permit
 ```
 
@@ -345,6 +345,7 @@ password = mypassword
 dbname = simplelogin
 
 query = SELECT domain FROM custom_domain WHERE domain='%s' AND verified=true
+    UNION SELECT domain FROM public_domain WHERE domain='%s'
     UNION SELECT '%s' WHERE '%s' = 'mydomain.com' LIMIT 1;
 ```
 
@@ -360,6 +361,7 @@ dbname = simplelogin
 
 # forward to smtp:127.0.0.1:20381 for custom domain AND email domain
 query = SELECT 'smtp:127.0.0.1:20381' FROM custom_domain WHERE domain = '%s' AND verified=true
+    UNION SELECT 'smtp:127.0.0.1:20381' FROM public_domain WHERE domain = '%s'
     UNION SELECT 'smtp:127.0.0.1:20381' WHERE '%s' = 'mydomain.com' LIMIT 1;
 ```
 
@@ -424,7 +426,7 @@ docker run --rm \
     -v $(pwd)/dkim.pub.key:/dkim.pub.key \
     -v $(pwd)/simplelogin.env:/code/.env \
     --network="sl-network" \
-    simplelogin/app:3.4.0 flask db upgrade
+    simplelogin/app:4.6.5-beta alembic upgrade head
 ```
 
 This command could take a while to download the `simplelogin/app` docker image.
@@ -439,7 +441,7 @@ docker run --rm \
     -v $(pwd)/dkim.key:/dkim.key \
     -v $(pwd)/dkim.pub.key:/dkim.pub.key \
     --network="sl-network" \
-    simplelogin/app:3.4.0 python init_app.py
+    simplelogin/app:4.6.5-beta python init_app.py
 ```
 
 Now, it's time to run the `webapp` container!
@@ -455,7 +457,7 @@ docker run -d \
     -p 127.0.0.1:7777:7777 \
     --restart always \
     --network="sl-network" \
-    simplelogin/app:3.4.0
+    simplelogin/app:4.6.5-beta
 ```
 
 Next run the `email handler`
@@ -471,7 +473,7 @@ docker run -d \
     -p 127.0.0.1:20381:20381 \
     --restart always \
     --network="sl-network" \
-    simplelogin/app:3.4.0 python email_handler.py
+    simplelogin/app:4.6.5-beta python email_handler.py
 ```
 
 And finally the `job runner`
@@ -486,7 +488,7 @@ docker run -d \
     -v $(pwd)/dkim.pub.key:/dkim.pub.key \
     --restart always \
     --network="sl-network" \
-    simplelogin/app:3.4.0 python job_runner.py
+    simplelogin/app:4.6.5-beta python job_runner.py
 ```
 
 ### Nginx

app/config.py

@@ -480,3 +480,5 @@ DISABLE_CREATE_CONTACTS_FOR_FREE_USERS = False
 PARTNER_API_TOKEN_SECRET = os.environ.get("PARTNER_API_TOKEN_SECRET") or (
     FLASK_SECRET + "partnerapitoken"
 )
+
+SQLALCHEMY_POOL_PRE_PING = "SQLALCHEMY_POOL_PRE_PING" in os.environ

app/dashboard/views/custom_domain.py

@@ -3,12 +3,12 @@ from flask_login import login_required, current_user
 from flask_wtf import FlaskForm
 from wtforms import StringField, validators
 
-from app.config import EMAIL_SERVERS_WITH_PRIORITY
+from app import config
 from app.dashboard.base import dashboard_bp
 from app.db import Session
 from app.email_utils import get_email_domain_part
 from app.log import LOG
-from app.models import CustomDomain, Mailbox, DomainMailbox, SLDomain
+from app.models import CustomDomain, Mailbox, DomainMailbox
 
 
 class NewCustomDomainForm(FlaskForm):
@@ -43,9 +43,7 @@ def custom_domain():
                 if new_domain.startswith("https://"):
                     new_domain = new_domain[len("https://") :]
 
-                if SLDomain.get_by(domain=new_domain):
-                    flash("A custom domain cannot be a built-in domain.", "error")
-                elif CustomDomain.get_by(domain=new_domain):
+                if CustomDomain.get_by(domain=new_domain):
                     flash(f"{new_domain} already used", "error")
                 elif get_email_domain_part(current_user.email) == new_domain:
                     flash(
@@ -60,7 +58,7 @@ def custom_domain():
                         f"{new_domain} already used in a SimpleLogin mailbox", "error"
                     )
                 else:
-                    new_custom_domain = CustomDomain.create(
+                    new_custom_domain: CustomDomain = CustomDomain.create(
                         domain=new_domain, user_id=current_user.id
                     )
                     # new domain has ownership verified if its parent has the ownership verified
@@ -76,6 +74,13 @@ def custom_domain():
                             )
                             new_custom_domain.ownership_verified = True
 
+                    # in self-hosting, a custom domain can be the same as the root EMAIL_DOMAIN
+                    if new_domain == config.EMAIL_DOMAIN:
+                        new_custom_domain.ownership_verified = True
+                        new_custom_domain.verified = True
+                        new_custom_domain.spf_verified = True
+                        new_custom_domain.dkim_verified = True
+
                     Session.commit()
 
                     mailbox_ids = request.form.getlist("mailbox_ids")
@@ -115,7 +120,7 @@ def custom_domain():
         "dashboard/custom_domain.html",
         custom_domains=custom_domains,
         new_custom_domain_form=new_custom_domain_form,
-        EMAIL_SERVERS_WITH_PRIORITY=EMAIL_SERVERS_WITH_PRIORITY,
+        EMAIL_SERVERS_WITH_PRIORITY=config.EMAIL_SERVERS_WITH_PRIORITY,
         errors=errors,
         mailboxes=mailboxes,
     )

app/models.py

@@ -1350,13 +1350,25 @@ class Alias(Base, ModelMixin):
         return False
 
     @staticmethod
-    def get_custom_domain(alias_address) -> Optional["CustomDomain"]:
+    def get_custom_domain(
+        alias_address, user: Optional[User]
+    ) -> Optional["CustomDomain"]:
         alias_domain = validate_email(
             alias_address, check_deliverability=False, allow_smtputf8=False
         ).domain
 
         # handle the case a SLDomain is also a CustomDomain
-        if SLDomain.get_by(domain=alias_domain) is None:
+        sl_domain = SLDomain.get_by(domain=alias_domain)
+        if sl_domain:
+            # "root" user can have a custom domain that's also a SL domain
+            # in this case prefer this custom domain over the SL one
+            if user:
+                custom_domain = CustomDomain.get_by(
+                    domain=alias_domain, user_id=user.id
+                )
+                if custom_domain:
+                    return custom_domain
+        else:
             custom_domain = CustomDomain.get_by(domain=alias_domain)
             if custom_domain:
                 return custom_domain
@@ -1379,9 +1391,15 @@ class Alias(Base, ModelMixin):
         if DomainDeletedAlias.get_by(email=email):
             raise AliasInTrashError
 
+        user_id = kw.get("user_id")
+        if user_id:
+            user = User.get(user_id)
+        else:
+            user = None
+
         # detect whether alias should belong to a custom domain
         if "custom_domain_id" not in kw:
-            custom_domain = Alias.get_custom_domain(email)
+            custom_domain = Alias.get_custom_domain(email, user)
             if custom_domain:
                 new_alias.custom_domain_id = custom_domain.id
 

docs/upgrade.md

@@ -4,10 +4,54 @@ No emails or any data is lost in the upgrade process. The same process is by the
 
 Sometimes upgrading to a major version might require running a manual migration. This is for example the case when upgrading to 2.0.0. In this case please follow the corresponding migration first before running these scripts.
 
-If you are running versions prior to 3x, please:
+If you are running versions prior to 4.x.x, please:
 
-1. first upgrade to 2.1.2 then
-2. upgrade to the latest version which is 3.4.0
+1. first upgrade to 3.4.0 then
+2. upgrade to the latest version which is 4.6.5-beta
+
+<details>
+<summary>After upgrade to 4.x.x from 3.4.0</summary>
+<p>
+
+Please update `/etc/postfix/pgsql-relay-domains.cf` to the following. Make sure to replace `mydomain.com` by your actual domain.
+
+```
+# postgres config
+hosts = localhost
+user = myuser
+password = mypassword
+dbname = simplelogin
+
+query = SELECT domain FROM custom_domain WHERE domain='%s' AND verified=true
+    UNION SELECT domain FROM public_domain WHERE domain='%s'
+    UNION SELECT '%s' WHERE '%s' = 'mydomain.com' LIMIT 1;
+```
+
+and `/etc/postfix/pgsql-transport-maps.cf` to
+
+```
+# postgres config
+hosts = localhost
+user = myuser
+password = mypassword
+dbname = simplelogin
+
+# forward to smtp:127.0.0.1:20381 for custom domain AND email domain
+query = SELECT 'smtp:127.0.0.1:20381' FROM custom_domain WHERE domain = '%s' AND verified=true
+    UNION SELECT 'smtp:127.0.0.1:20381' FROM public_domain WHERE domain = '%s'
+    UNION SELECT 'smtp:127.0.0.1:20381' WHERE '%s' = 'mydomain.com' LIMIT 1;
+```
+
+Please run the following command to update the `email_log` table:
+
+```bash
+docker exec -it sl-db psql -U myuser simplelogin
+update email_log set alias_id=(select alias_id from contact where contact.id = email_log.contact_id);
+exit
+```
+
+</p>
+</details>
 
 <details>
 <summary>After upgrade to 3x from 2x</summary>
@@ -119,23 +163,23 @@ for user in User.query.all():
 </p>
 </details>
 
-## Upgrade to the latest version 3.4.0
+## Upgrade to the latest version 4.6.5-beta
 
 ```bash
 # Pull the latest version
-sudo docker pull simplelogin/app:3.4.0
+sudo docker pull simplelogin/app:4.6.5-beta
 
 # Stop SimpleLogin containers
 sudo docker stop sl-email sl-migration sl-app sl-db sl-job-runner
 
 # Make sure to remove these containers to avoid conflict
-sudo docker rm -f sl-email sl-migration sl-app sl-db
+sudo docker rm -f sl-email sl-migration sl-app sl-db sl-job-runner
 
 # create ./sl/upload/ if not exist
 mkdir -p ./sl/upload/
 
 # Run the database container. Make sure to replace `myuser` and `mypassword`
-docker run -d \
+sudo docker run -d \
     --name sl-db \
     -e POSTGRES_PASSWORD=mypassword \
     -e POSTGRES_USER=myuser \
@@ -155,7 +199,7 @@ sudo docker run --rm \
     -v $(pwd)/dkim.pub.key:/dkim.pub.key \
     -v $(pwd)/simplelogin.env:/code/.env \
     --network="sl-network" \
-    simplelogin/app:3.4.0 flask db upgrade
+    simplelogin/app:4.6.5-beta alembic upgrade head
 
 # Run init data
 sudo docker run --rm \
@@ -166,7 +210,7 @@ sudo docker run --rm \
     -v $(pwd)/dkim.key:/dkim.key \
     -v $(pwd)/dkim.pub.key:/dkim.pub.key \
     --network="sl-network" \
-    simplelogin/app:3.4.0 python init_app.py
+    simplelogin/app:4.6.5-beta python init_app.py
 
 # Run the webapp container
 sudo docker run -d \
@@ -179,7 +223,7 @@ sudo docker run -d \
     -p 127.0.0.1:7777:7777 \
     --restart always \
     --network="sl-network" \
-    simplelogin/app:3.4.0
+    simplelogin/app:4.6.5-beta
 
 # Run the email handler container
 sudo docker run -d \
@@ -192,10 +236,10 @@ sudo docker run -d \
     -p 127.0.0.1:20381:20381 \
     --restart always \
     --network="sl-network" \
-    simplelogin/app:3.4.0 python email_handler.py
+    simplelogin/app:4.6.5-beta python email_handler.py
     
 # Run the job runner
-docker run -d \
+sudo docker run -d \
     --name sl-job-runner \
     -v $(pwd)/sl:/sl \
     -v $(pwd)/sl/upload:/code/static/upload \
@@ -204,7 +248,7 @@ docker run -d \
     -v $(pwd)/dkim.pub.key:/dkim.pub.key \
     --restart always \
     --network="sl-network" \
-    simplelogin/app:3.4.0 python job_runner.py
+    simplelogin/app:4.6.5-beta python job_runner.py
     
 ```
 

server.py

@@ -140,6 +140,10 @@ def create_app() -> Flask:
     # enable to print all queries generated by sqlalchemy
     # app.config["SQLALCHEMY_ECHO"] = True
 
+    if config.SQLALCHEMY_POOL_PRE_PING:
+        # to handle the "server closed the connection unexpectedly" error
+        app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {"pool_pre_ping": True}
+
     app.secret_key = FLASK_SECRET
 
     app.config["TEMPLATES_AUTO_RELOAD"] = True