From 01fa8cf72c2123e42e83a615f3c8c90c74a2c55c Mon Sep 17 00:00:00 2001 From: Michael Kroes Date: Wed, 13 Apr 2016 17:52:13 -0400 Subject: [PATCH 1/4] add fail2ban jails for ownCloud, postfix submission, roundcube, and the Mail-in-a-Box management daemon (tests squashed into this commit by josh) --- CHANGELOG.md | 4 + conf/fail2ban/{ => filter.d}/dovecotimap.conf | 0 .../filter.d/miab-management-daemon.conf | 12 ++ conf/fail2ban/filter.d/miab-munin.conf | 7 + conf/fail2ban/filter.d/miab-owncloud.conf | 7 + .../filter.d/miab-postfix-submission.conf | 7 + conf/fail2ban/filter.d/miab-roundcube.conf | 9 + conf/fail2ban/{jail.local => jails.conf} | 65 ++++-- management/daemon.py | 25 ++- setup/owncloud.sh | 5 + setup/system.sh | 10 +- tests/fail2ban.py | 193 ++++++++++++++++++ 12 files changed, 326 insertions(+), 18 deletions(-) rename conf/fail2ban/{ => filter.d}/dovecotimap.conf (100%) create mode 100644 conf/fail2ban/filter.d/miab-management-daemon.conf create mode 100644 conf/fail2ban/filter.d/miab-munin.conf create mode 100644 conf/fail2ban/filter.d/miab-owncloud.conf create mode 100644 conf/fail2ban/filter.d/miab-postfix-submission.conf create mode 100644 conf/fail2ban/filter.d/miab-roundcube.conf rename conf/fail2ban/{jail.local => jails.conf} (58%) create mode 100644 tests/fail2ban.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e3568cd..5e15794 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ Mail: * Roundcube is updated to version 1.2.0. +System: + +* fail2ban jails added for SMTP submission, Roundcube, ownCloud, the control panel, and munin. + v0.18c (June 2, 2016) --------------------- diff --git a/conf/fail2ban/dovecotimap.conf b/conf/fail2ban/filter.d/dovecotimap.conf similarity index 100% rename from conf/fail2ban/dovecotimap.conf rename to conf/fail2ban/filter.d/dovecotimap.conf diff --git a/conf/fail2ban/filter.d/miab-management-daemon.conf b/conf/fail2ban/filter.d/miab-management-daemon.conf new file mode 100644 index 0000000..0b0489c --- /dev/null +++ b/conf/fail2ban/filter.d/miab-management-daemon.conf @@ -0,0 +1,12 @@ +# Fail2Ban filter Mail-in-a-Box management daemon + +[INCLUDES] + +before = common.conf + +[Definition] + +_daemon = mailinabox + +failregex = Mail-in-a-Box Management Daemon: Failed login attempt from ip - timestamp .* +ignoreregex = diff --git a/conf/fail2ban/filter.d/miab-munin.conf b/conf/fail2ban/filter.d/miab-munin.conf new file mode 100644 index 0000000..b254cc6 --- /dev/null +++ b/conf/fail2ban/filter.d/miab-munin.conf @@ -0,0 +1,7 @@ +[INCLUDES] + +before = common.conf + +[Definition] +failregex= - .*GET /admin/munin/.* HTTP/1.1\" 401.* +ignoreregex = diff --git a/conf/fail2ban/filter.d/miab-owncloud.conf b/conf/fail2ban/filter.d/miab-owncloud.conf new file mode 100644 index 0000000..a9a13f2 --- /dev/null +++ b/conf/fail2ban/filter.d/miab-owncloud.conf @@ -0,0 +1,7 @@ +[INCLUDES] + +before = common.conf + +[Definition] +failregex=Login failed: .*Remote IP: '[\)'] +ignoreregex = diff --git a/conf/fail2ban/filter.d/miab-postfix-submission.conf b/conf/fail2ban/filter.d/miab-postfix-submission.conf new file mode 100644 index 0000000..236e133 --- /dev/null +++ b/conf/fail2ban/filter.d/miab-postfix-submission.conf @@ -0,0 +1,7 @@ +[INCLUDES] + +before = common.conf + +[Definition] +failregex=postfix/submission/smtpd.*warning.*\[\]: .* authentication (failed|aborted) +ignoreregex = diff --git a/conf/fail2ban/filter.d/miab-roundcube.conf b/conf/fail2ban/filter.d/miab-roundcube.conf new file mode 100644 index 0000000..c6979c8 --- /dev/null +++ b/conf/fail2ban/filter.d/miab-roundcube.conf @@ -0,0 +1,9 @@ +[INCLUDES] + +before = common.conf + +[Definition] + +failregex = IMAP Error: Login failed for .*? from \. AUTHENTICATE.* + +ignoreregex = diff --git a/conf/fail2ban/jail.local b/conf/fail2ban/jails.conf similarity index 58% rename from conf/fail2ban/jail.local rename to conf/fail2ban/jails.conf index dc33880..7cc9097 100644 --- a/conf/fail2ban/jail.local +++ b/conf/fail2ban/jails.conf @@ -1,4 +1,5 @@ -# Fail2Ban configuration file for Mail-in-a-Box +# Fail2Ban configuration file for Mail-in-a-Box. Do not edit. +# This file is re-generated on updates. [DEFAULT] # Whitelist our own IP addresses. 127.0.0.1/8 is the default. But our status checks @@ -6,24 +7,52 @@ # ours too. The string is substituted during installation. ignoreip = 127.0.0.1/8 PUBLIC_IP -# JAILS - -[ssh] -maxretry = 7 -bantime = 3600 - -[ssh-ddos] -enabled = true - -[sasl] -enabled = true - [dovecot] enabled = true filter = dovecotimap +logpath = /var/log/mail.log findtime = 30 maxretry = 20 + +[miab-management] +enabled = true +filter = miab-management-daemon +port = http,https +logpath = /var/log/syslog +maxretry = 20 +findtime = 30 + +[miab-munin] +enabled = true +port = http,https +filter = miab-munin +logpath = /var/log/nginx/access.log +maxretry = 20 +findtime = 30 + +[miab-owncloud] +enabled = true +port = http,https +filter = miab-owncloud +logpath = STORAGE_ROOT/owncloud/owncloud.log +maxretry = 20 +findtime = 30 + +[miab-postfix587] +enabled = true +port = 587 +filter = miab-postfix-submission logpath = /var/log/mail.log +maxretry = 20 +findtime = 30 + +[miab-roundcube] +enabled = true +port = http,https +filter = miab-roundcube +logpath = /var/log/roundcubemail/errors +maxretry = 20 +findtime = 30 [recidive] enabled = true @@ -39,3 +68,13 @@ action = iptables-allports[name=recidive] # By default we don't configure this address and no action is required from the admin anyway. # So the notification is ommited. This will prevent message appearing in the mail.log that mail # can't be delivered to fail2ban@$HOSTNAME. + +[sasl] +enabled = true + +[ssh] +maxretry = 7 +bantime = 3600 + +[ssh-ddos] +enabled = true diff --git a/management/daemon.py b/management/daemon.py index 5400925..9bc6429 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -1,7 +1,8 @@ #!/usr/bin/python3 -import os, os.path, re, json +import os, os.path, re, json, time import subprocess + from functools import wraps from flask import Flask, request, render_template, abort, Response, send_from_directory, make_response @@ -45,6 +46,9 @@ def authorized_personnel_only(viewfunc): privs = [] error = "Incorrect username or password" + # Write a line in the log recording the failed login + log_failed_login(request) + # Authorized to access an API view? if "admin" in privs: # Call view func. @@ -117,6 +121,9 @@ def me(): try: email, privs = auth_service.authenticate(request, env) except ValueError as e: + # Log the failed login + log_failed_login(request) + return json_response({ "status": "invalid", "reason": "Incorrect username or password", @@ -583,6 +590,22 @@ def munin_cgi(filename): app.logger.warning("munin_cgi: munin-cgi-graph returned 404 status code. PATH_INFO=%s", env['PATH_INFO']) return response +def log_failed_login(request): + # We need to figure out the ip to list in the message, all our calls are routed + # through nginx who will put the original ip in X-Forwarded-For. + # During setup we call the management interface directly to determine the user + # status. So we can't always use X-Forwarded-For because during setup that header + # will not be present. + if request.headers.getlist("X-Forwarded-For"): + ip = request.headers.getlist("X-Forwarded-For")[0] + else: + ip = request.remote_addr + + # We need to add a timestamp to the log message, otherwise /dev/log will eat the "duplicate" + # message. + app.logger.warning( "Mail-in-a-Box Management Daemon: Failed login attempt from ip %s - timestamp %s" % (ip, time.time())) + + # APP if __name__ == '__main__': diff --git a/setup/owncloud.sh b/setup/owncloud.sh index cc58a5c..eb12d5c 100755 --- a/setup/owncloud.sh +++ b/setup/owncloud.sh @@ -163,7 +163,10 @@ fi # so set it here. It also can change if the box's PRIMARY_HOSTNAME changes, so # this will make sure it has the right value. # * Some settings weren't included in previous versions of Mail-in-a-Box. +# * We need to set the timezone to the system timezone to allow fail2ban to ban +# users within the proper timeframe # Use PHP to read the settings file, modify it, and write out the new settings array. +TIMEZONE=$(cat /etc/timezone) CONFIG_TEMP=$(/bin/mktemp) php < $CONFIG_TEMP && mv $CONFIG_TEMP $STORAGE_ROOT/owncloud/config.php; /etc/fail2ban/jail.local -cp conf/fail2ban/dovecotimap.conf /etc/fail2ban/filter.d/dovecotimap.conf + | sed "s#STORAGE_ROOT#$STORAGE_ROOT#" \ + > /etc/fail2ban/jail.d/mailinabox.conf +cp -f conf/fail2ban/filter.d/* /etc/fail2ban/filter.d/ restart_service fail2ban diff --git a/tests/fail2ban.py b/tests/fail2ban.py new file mode 100644 index 0000000..f4b4829 --- /dev/null +++ b/tests/fail2ban.py @@ -0,0 +1,193 @@ +# Test that a box's fail2ban setting are working +# correctly by attempting a bunch of failed logins. +# Specify SSH login information the command line - +# we use that to reset fail2ban after each test, +# and we extract the hostname from that to open +# connections to. +###################################################################### + +import sys, os, time, functools + +# parse command line + +if len(sys.argv) < 2: + print("Usage: tests/fail2ban.py user@hostname") + sys.exit(1) + +ssh_user, hostname = sys.argv[1].split("@", 1) + +# define some test types + +import socket +socket.setdefaulttimeout(10) + +class IsBlocked(Exception): + """Tests raise this exception when it appears that a fail2ban + jail is in effect, i.e. on a connection refused error.""" + pass + +def smtp_test(): + import smtplib + + try: + server = smtplib.SMTP(hostname, 587) + except ConnectionRefusedError: + # looks like fail2ban worked + raise IsBlocked() + server.starttls() + server.ehlo_or_helo_if_needed() + + try: + server.login("fakeuser", "fakepassword") + raise Exception("authentication didn't fail") + except smtplib.SMTPAuthenticationError: + # athentication should fail + pass + + try: + server.quit() + except: + # ignore errors here + pass + +def imap_test(): + import imaplib + + try: + M = imaplib.IMAP4_SSL(hostname) + except ConnectionRefusedError: + # looks like fail2ban worked + raise IsBlocked() + + try: + M.login("fakeuser", "fakepassword") + raise Exception("authentication didn't fail") + except imaplib.IMAP4.error: + # authentication should fail + pass + finally: + M.logout() # shuts down connection, has nothing to do with login() + +def http_test(url, expected_status, postdata=None, qsargs=None, auth=None): + import urllib.parse + import requests + from requests.auth import HTTPBasicAuth + + # form request + url = urllib.parse.urljoin("https://" + hostname, url) + if qsargs: url += "?" + urllib.parse.urlencode(qsargs) + urlopen = requests.get if not postdata else requests.post + + try: + # issue request + r = urlopen( + url, + auth=HTTPBasicAuth(*auth) if auth else None, + data=postdata, + headers={'User-Agent': 'Mail-in-a-Box fail2ban tester'}, + timeout=4) + except requests.exceptions.ConnectionError as e: + if "Connection refused" in str(e): + raise IsBlocked() + raise # some other unexpected condition + + # return response status code + if r.status_code != expected_status: + r.raise_for_status() # anything but 200 + raise IOError("Got unexpected status code %s." % r.status_code) + +# define how to run a test + +def restart_fail2ban_service(final=False): + # Log in over SSH to restart fail2ban. + command = "sudo fail2ban-client reload" + if not final: + # Stop recidive jails during testing. + command += " && sudo fail2ban-client stop recidive" + os.system("ssh %s@%s \"%s\"" % (ssh_user, hostname, command)) + +def testfunc_runner(i, testfunc, *args): + print(i+1, end=" ", flush=True) + testfunc(*args) + +def run_test(testfunc, args, count, within_seconds, parallel): + # Run testfunc count times in within_seconds seconds (and actually + # within a little less time so we're sure we're under the limit). + # + # Because some services are slow, like IMAP, we can't necessarily + # run testfunc sequentially and still get to count requests within + # the required time. So we split the requests across threads. + + import requests.exceptions + from multiprocessing import Pool + + restart_fail2ban_service() + + # Log. + print(testfunc.__name__, " ".join(str(a) for a in args), "...") + + # Record the start time so we can know how to evenly space our + # calls to testfunc. + start_time = time.time() + + with Pool(parallel) as p: + # Distribute the requests across the pool. + asyncresults = [] + for i in range(count): + ar = p.apply_async(testfunc_runner, [i, testfunc] + list(args)) + asyncresults.append(ar) + + # Wait for all runs to finish. + p.close() + p.join() + + # Check for errors. + for ar in asyncresults: + try: + ar.get() + except IsBlocked: + print("Test machine prematurely blocked!") + return False + + # Did we make enough requests within the limit? + if (time.time()-start_time) > within_seconds: + raise Exception("Test failed to make %s requests in %d seconds." % (count, within_seconds)) + + # Wait a moment for the block to be put into place. + time.sleep(4) + + # The next call should fail. + print("*", end=" ", flush=True) + try: + testfunc(*args) + except IsBlocked: + # Success -- this one is supposed to be refused. + print("blocked [OK]") + return True # OK + + print("not blocked!") + return False + +###################################################################### + +if __name__ == "__main__": + # run tests + + # SMTP bans at 10 even though we say 20 in the config because we get + # doubled-up warnings in the logs, we'll let that be for now + run_test(smtp_test, [], 10, 30, 8) + + # IMAP + run_test(imap_test, [], 20, 30, 4) + + # Mail-in-a-Box contorl panel + run_test(http_test, ["/admin/me", 200], 20, 30, 1) + + # Munin via the Mail-in-a-Box contorl panel + run_test(http_test, ["/admin/munin/", 401], 20, 30, 1) + + # ownCloud + run_test(http_test, ["/cloud/remote.php/caldav/calendars/user@domain/personal", 401], 20, 30, 1) + + # restart fail2ban so that this client machine is no longer blocked + restart_fail2ban_service(final=True) From bf5e9200f8b6f4f9790b9e30df429b1467580ef1 Mon Sep 17 00:00:00 2001 From: Michael Kroes Date: Sun, 26 Jun 2016 13:26:42 +0200 Subject: [PATCH 2/4] Update owncloud url to use webdav and increase http timeout --- tests/fail2ban.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fail2ban.py b/tests/fail2ban.py index f4b4829..feda279 100644 --- a/tests/fail2ban.py +++ b/tests/fail2ban.py @@ -85,7 +85,7 @@ def http_test(url, expected_status, postdata=None, qsargs=None, auth=None): auth=HTTPBasicAuth(*auth) if auth else None, data=postdata, headers={'User-Agent': 'Mail-in-a-Box fail2ban tester'}, - timeout=4) + timeout=8) except requests.exceptions.ConnectionError as e: if "Connection refused" in str(e): raise IsBlocked() @@ -187,7 +187,7 @@ if __name__ == "__main__": run_test(http_test, ["/admin/munin/", 401], 20, 30, 1) # ownCloud - run_test(http_test, ["/cloud/remote.php/caldav/calendars/user@domain/personal", 401], 20, 30, 1) + run_test(http_test, ["/cloud/remote.php/webdav", 401, None, None, ["aa", "aa"]], 20, 30, 1) # restart fail2ban so that this client machine is no longer blocked restart_fail2ban_service(final=True) From d9ac321f257dd052a36b4905b876a37c7fb90550 Mon Sep 17 00:00:00 2001 From: Michael Kroes Date: Sun, 26 Jun 2016 14:17:12 +0200 Subject: [PATCH 3/4] Owncloud needs more time to detect blocks. It doesn't respond as fast as the other services. Also owncloud logs UTC (since latest update) even though the timezone is not UTC. Also to detect a block, we get a timeout instead of a refused) --- conf/fail2ban/jails.conf | 2 +- setup/owncloud.sh | 2 ++ tests/fail2ban.py | 8 +++++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/conf/fail2ban/jails.conf b/conf/fail2ban/jails.conf index 7cc9097..0146b64 100644 --- a/conf/fail2ban/jails.conf +++ b/conf/fail2ban/jails.conf @@ -36,7 +36,7 @@ port = http,https filter = miab-owncloud logpath = STORAGE_ROOT/owncloud/owncloud.log maxretry = 20 -findtime = 30 +findtime = 120 [miab-postfix587] enabled = true diff --git a/setup/owncloud.sh b/setup/owncloud.sh index eb12d5c..e8d1df1 100755 --- a/setup/owncloud.sh +++ b/setup/owncloud.sh @@ -126,6 +126,7 @@ if [ ! -f $STORAGE_ROOT/owncloud/owncloud.db ]; then 'mail_from_address' => 'owncloud', 'mail_domain' => '$PRIMARY_HOSTNAME', 'logtimezone' => '$TIMEZONE', + 'logdateformat' => 'Y-m-d H:i:s', ); ?> EOF @@ -179,6 +180,7 @@ include("$STORAGE_ROOT/owncloud/config.php"); \$CONFIG['mail_from_address'] = 'administrator'; # just the local part, matches our master administrator address \$CONFIG['logtimezone'] = '$TIMEZONE'; +\$CONFIG['logdateformat'] = 'Y-m-d H:i:s'; echo " Date: Mon, 27 Jun 2016 05:19:12 +0200 Subject: [PATCH 4/4] Remove owncloud log configuration from initial setup and only apply it during the configuration updates. This applies to both the timezone and the log format --- setup/owncloud.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup/owncloud.sh b/setup/owncloud.sh index e8d1df1..7904524 100755 --- a/setup/owncloud.sh +++ b/setup/owncloud.sh @@ -92,7 +92,6 @@ if [ ! -f $STORAGE_ROOT/owncloud/owncloud.db ]; then mkdir -p $STORAGE_ROOT/owncloud # Create an initial configuration file. - TIMEZONE=$(cat /etc/timezone) instanceid=oc$(echo $PRIMARY_HOSTNAME | sha1sum | fold -w 10 | head -n 1) cat > $STORAGE_ROOT/owncloud/config.php < '', 'mail_from_address' => 'owncloud', 'mail_domain' => '$PRIMARY_HOSTNAME', - 'logtimezone' => '$TIMEZONE', - 'logdateformat' => 'Y-m-d H:i:s', ); ?> EOF @@ -166,6 +163,7 @@ fi # * Some settings weren't included in previous versions of Mail-in-a-Box. # * We need to set the timezone to the system timezone to allow fail2ban to ban # users within the proper timeframe +# * We need to set the logdateformat to something that will work correctly with fail2ban # Use PHP to read the settings file, modify it, and write out the new settings array. TIMEZONE=$(cat /etc/timezone) CONFIG_TEMP=$(/bin/mktemp)