initial cluster svc impl

src/main/java/password/pwm/AppProperty.java

@@ -69,6 +69,10 @@ public enum     AppProperty {
     CONFIG_GUIDE_IDLE_TIMEOUT                       ("configGuide.idleTimeoutSeconds"),
     CONFIG_MANAGER_ZIPDEBUG_MAXLOGLINES             ("configManager.zipDebug.maxLogLines"),
     CONFIG_MANAGER_ZIPDEBUG_MAXLOGSECONDS           ("configManager.zipDebug.maxLogSeconds"),
+    CLUSTER_DB_ENABLE                               ("cluster.db.enable"),
+    CLUSTER_DB_HEARTBEAT_SECONDS                    ("cluster.db.heartbeatSeconds"),
+    CLUSTER_DB_NODE_TIMEOUT_SECONDS                 ("cluster.db.nodeTimeoutSeconds"),
+    CLUSTER_DB_NODE_PURGE_SECONDS                   ("cluster.db.nodePurgeSeconds"),
     DB_JDBC_LOAD_STRATEGY                           ("db.jdbcLoadStrategy"),
     DB_CONNECTIONS_MAX                              ("db.connections.max"),
     DB_CONNECTIONS_TIMEOUT_MS                       ("db.connections.timeoutMs"),

src/main/java/password/pwm/PwmApplication.java

@@ -42,6 +42,7 @@ import password.pwm.ldap.search.UserSearchEngine;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmServiceManager;
 import password.pwm.svc.cache.CacheService;
+import password.pwm.svc.cluster.ClusterService;
 import password.pwm.svc.event.AuditEvent;
 import password.pwm.svc.event.AuditRecordFactory;
 import password.pwm.svc.event.AuditService;
@@ -503,6 +504,10 @@ public class PwmApplication {
         return (VersionChecker)pwmServiceManager.getService(VersionChecker.class);
     }
 
+    public ClusterService getClusterService() {
+        return (ClusterService) pwmServiceManager.getService(ClusterService.class);
+    }
+
     public ErrorInformation getLastLocalDBFailure() {
         return lastLocalDBFailure;
     }

src/main/java/password/pwm/config/Configuration.java

@@ -96,6 +96,9 @@ public class Configuration implements Serializable, SettingReader {
 
     private DataCache dataCache = new DataCache();
 
+    private String cachshedConfigurationHash;
+
+
     // --------------------------- CONSTRUCTORS ---------------------------
 
     public Configuration(final StoredConfigurationImpl storedConfiguration) {
@@ -882,11 +885,14 @@ public class Configuration implements Serializable, SettingReader {
     public boolean isDevDebugMode() {
         return Boolean.parseBoolean(readAppProperty(AppProperty.LOGGING_DEV_OUTPUT));
     }
-    
-    public String configurationHash() 
+
+    public String configurationHash()
             throws PwmUnrecoverableException 
     {
-        return storedConfiguration.settingChecksum();
+        if (this.cachshedConfigurationHash == null) {
+            this.cachshedConfigurationHash = storedConfiguration.settingChecksum();
+        }
+        return cachshedConfigurationHash;
     }
 
     public Set<PwmSetting> nonDefaultSettings() {

src/main/java/password/pwm/svc/PwmServiceManager.java

@@ -34,6 +34,7 @@ import password.pwm.http.state.SessionStateService;
 import password.pwm.ldap.LdapConnectionService;
 import password.pwm.ldap.search.UserSearchEngine;
 import password.pwm.svc.cache.CacheService;
+import password.pwm.svc.cluster.ClusterService;
 import password.pwm.svc.event.AuditService;
 import password.pwm.svc.intruder.IntruderManager;
 import password.pwm.svc.report.ReportService;
@@ -45,6 +46,7 @@ import password.pwm.svc.wordlist.SeedlistManager;
 import password.pwm.svc.wordlist.SharedHistoryManager;
 import password.pwm.svc.wordlist.WordlistManager;
 import password.pwm.util.VersionChecker;
+import password.pwm.util.db.DatabaseService;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.operations.CrService;
@@ -72,7 +74,7 @@ public class PwmServiceManager {
     public enum PwmServiceClassEnum {
         SecureService(          SecureService.class,             true),
         LdapConnectionService(  LdapConnectionService.class,     true),
-        DatabaseService(        password.pwm.util.db.DatabaseService.class,           true),
+        DatabaseService(        DatabaseService.class,           true),
         SharedHistoryManager(   SharedHistoryManager.class,      false),
         AuditService(           AuditService.class,              false),
         StatisticsManager(      StatisticsManager.class,         false),
@@ -93,6 +95,7 @@ public class PwmServiceManager {
         SessionTrackService(    SessionTrackService.class,       false),
         SessionStateSvc(        SessionStateService.class,       false),
         UserSearchEngine(       UserSearchEngine.class,          true),
+        ClusterService(         ClusterService.class,            false),
 
         ;
 

src/main/java/password/pwm/svc/cluster/ClusterProvider.java

@@ -0,0 +1,35 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.cluster;
+
+import password.pwm.error.PwmUnrecoverableException;
+
+import java.util.List;
+
+public interface ClusterProvider {
+    void close();
+
+    boolean isMaster();
+
+    List<NodeInfo> nodes() throws PwmUnrecoverableException;
+}

src/main/java/password/pwm/svc/cluster/ClusterService.java

@@ -0,0 +1,106 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.cluster;
+
+import password.pwm.PwmApplication;
+import password.pwm.error.PwmException;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.health.HealthRecord;
+import password.pwm.svc.PwmService;
+import password.pwm.util.logging.PwmLogger;
+
+import java.util.Collections;
+import java.util.List;
+
+public class ClusterService implements PwmService {
+
+    private static final PwmLogger LOGGER = PwmLogger.forClass(ClusterService.class);
+
+    private PwmApplication pwmApplication;
+    private STATUS status = STATUS.NEW;
+    private ClusterProvider clusterProvider;
+
+    @Override
+    public STATUS status()
+    {
+        return status;
+    }
+
+    @Override
+    public void init(final PwmApplication pwmApplication) throws PwmException
+    {
+        status = STATUS.OPENING;
+        this.pwmApplication = pwmApplication;
+
+        try {
+            if (this.pwmApplication.getConfig().hasDbConfigured()) {
+                clusterProvider = new DatabaseClusterProvider(pwmApplication);
+            }
+        } catch (PwmException e) {
+            LOGGER.error("error starting up cluster provider service: " + e.getMessage());
+            status = STATUS.CLOSED;
+            return;
+        }
+
+        status = STATUS.OPEN;
+    }
+
+    @Override
+    public void close()
+    {
+        if (clusterProvider != null) {
+            clusterProvider.close();
+            clusterProvider = null;
+        }
+        clusterProvider = null;
+        status = STATUS.CLOSED;
+    }
+
+    @Override
+    public List<HealthRecord> healthCheck()
+    {
+        return null;
+    }
+
+    @Override
+    public ServiceInfoBean serviceInfo()
+    {
+        return null;
+    }
+
+    public boolean isMaster() {
+        if (clusterProvider != null) {
+            return clusterProvider.isMaster();
+        }
+
+        return false;
+    }
+
+    public List<NodeInfo> nodes() throws PwmUnrecoverableException
+    {
+        if (status == STATUS.OPEN && clusterProvider != null) {
+            return clusterProvider.nodes();
+        }
+        return Collections.emptyList();
+    }
+}

src/main/java/password/pwm/svc/cluster/DatabaseClusterProvider.java

@@ -0,0 +1,240 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.cluster;
+
+import password.pwm.PwmApplication;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmException;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.db.DatabaseService;
+import password.pwm.util.db.DatabaseTable;
+import password.pwm.util.java.ClosableIterator;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogger;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+class DatabaseClusterProvider implements ClusterProvider {
+
+    private static final PwmLogger LOGGER = PwmLogger.forClass(DatabaseClusterProvider.class);
+
+    private static final DatabaseTable TABLE = DatabaseTable.CLUSTER_STATE;
+
+
+    private static final String KEY_PREFIX_NODE = "node-";
+
+    private final PwmApplication pwmApplication;
+    private final DatabaseService databaseService;
+    private final ScheduledExecutorService executorService;
+
+    private ErrorInformation lastError;
+
+    private final Map<String,DatabaseStoredNodeData> nodeDatas = new ConcurrentHashMap<>();
+
+    private final DatabaseClusterSettings settings;
+
+    DatabaseClusterProvider(final PwmApplication pwmApplication) throws PwmUnrecoverableException
+    {
+        this.pwmApplication = pwmApplication;
+        this.settings = DatabaseClusterSettings.fromConfig(pwmApplication.getConfig());
+
+        if (!settings.isEnable()) {
+            throw new PwmUnrecoverableException(new ErrorInformation(PwmError.CONFIG_FORMAT_ERROR,"database clustering is not enabled via app property"));
+        }
+
+        this.databaseService = pwmApplication.getDatabaseService();
+        this.executorService = JavaHelper.makeSingleThreadExecutorService(pwmApplication, DatabaseClusterProvider.class);
+
+        final long intervalSeconds = settings.getHeartbeatInterval().getTotalSeconds();
+
+        this.executorService.scheduleAtFixedRate(
+                new HeartbeatProcess(),
+                1,
+                intervalSeconds,
+                TimeUnit.SECONDS
+        );
+    }
+
+    @Override
+    public void close() {
+        JavaHelper.closeAndWaitExecutor(executorService, new TimeDuration(1, TimeUnit.SECONDS));
+    }
+
+
+    @Override
+    public List<NodeInfo> nodes() throws PwmUnrecoverableException
+    {
+        final Map<String,NodeInfo> returnObj = new TreeMap<>();
+        final String configHash = pwmApplication.getConfig().configurationHash();
+        for (final DatabaseStoredNodeData storedNodeData : nodeDatas.values()) {
+            final boolean configMatch = configHash.equals(storedNodeData.getConfigHash());
+            final boolean timedOut = isTimedOut(storedNodeData);
+
+            final NodeInfo.NodeState nodeState = isMaster(storedNodeData)
+                    ? NodeInfo.NodeState.master
+                    : timedOut
+                    ? NodeInfo.NodeState.offline
+                    : NodeInfo.NodeState.online;
+
+            final Instant startupTime = nodeState == NodeInfo.NodeState.offline
+                    ? null
+                    : storedNodeData.getStartupTimestamp();
+
+
+            final NodeInfo nodeInfo = new NodeInfo(
+                    storedNodeData.getInstanceID(),
+                    storedNodeData.getTimestamp(),
+                    startupTime,
+                    nodeState,
+                    configMatch
+            );
+            returnObj.put(nodeInfo.getInstanceID(), nodeInfo);
+        }
+
+        return Collections.unmodifiableList(new ArrayList<>(returnObj.values()));
+    }
+
+
+    private String masterInstanceId() {
+        final List<DatabaseStoredNodeData> copiedDatas = new ArrayList<>(nodeDatas.values());
+        if (copiedDatas.isEmpty()) {
+            return null;
+        }
+
+        String masterID = null;
+        Instant eldestRecord = Instant.now();
+
+        for (final DatabaseStoredNodeData nodeData : copiedDatas) {
+            if (!isTimedOut(nodeData)) {
+                if (nodeData.getStartupTimestamp().isBefore(eldestRecord)) {
+                    eldestRecord = nodeData.getStartupTimestamp();
+                    masterID = nodeData.getInstanceID();
+                }
+            }
+        }
+        return masterID;
+    }
+
+    @Override
+    public boolean isMaster() {
+        final String myID = pwmApplication.getInstanceID();
+        final String masterID = masterInstanceId();
+        return myID.equals(masterID);
+    }
+
+    private boolean isMaster(final DatabaseStoredNodeData databaseStoredNodeData) {
+        final String masterID = masterInstanceId();
+        return databaseStoredNodeData.getInstanceID().equals(masterID);
+    }
+
+    private String dbKeyForStoredNode(final DatabaseStoredNodeData storedNodeData) throws PwmUnrecoverableException
+    {
+        final String instanceID = storedNodeData.getInstanceID();
+        final String hash = pwmApplication.getSecureService().hash(instanceID);
+        final String truncatedHash = hash.length() > 64
+                ? hash.substring(0, 64)
+                : hash;
+
+        return KEY_PREFIX_NODE + truncatedHash;
+    }
+
+    private boolean isTimedOut(final DatabaseStoredNodeData storedNodeData) {
+        final TimeDuration age = TimeDuration.fromCurrent(storedNodeData.getTimestamp());
+        return age.isLongerThan(settings.getNodeTimeout());
+    }
+
+    private class HeartbeatProcess implements Runnable {
+        public void run() {
+            writeNodeStatus();
+            readNodeStatuses();
+            purgeOutdatedNodes();
+        }
+
+        void writeNodeStatus()
+        {
+            try {
+                final DatabaseStoredNodeData storedNodeData = DatabaseStoredNodeData.makeNew(pwmApplication);
+                final String key = dbKeyForStoredNode(storedNodeData);
+                final String value = JsonUtil.serialize(storedNodeData);
+                databaseService.getAccessor().put(TABLE, key, value);
+            } catch (PwmException e) {
+                final String errorMsg = "error writing database cluster heartbeat: " + e.getMessage();
+                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE, errorMsg);
+                lastError = errorInformation;
+                LOGGER.error(lastError);
+            }
+        }
+
+        void readNodeStatuses()
+        {
+            try (ClosableIterator<String> tableIterator = databaseService.getAccessor().iterator(TABLE)) {
+                while (tableIterator.hasNext()) {
+                    final String dbKey = tableIterator.next();
+                    if (dbKey.startsWith(KEY_PREFIX_NODE)) {
+                        final String rawValueInDb = databaseService.getAccessor().get(TABLE, dbKey);
+                        final DatabaseStoredNodeData nodeDataInDb = JsonUtil.deserialize(rawValueInDb, DatabaseStoredNodeData.class);
+                        nodeDatas.put(nodeDataInDb.getInstanceID(), nodeDataInDb);
+                    }
+                }
+            } catch (PwmException e) {
+                final String errorMsg = "error reading database node statuses: " + e.getMessage();
+                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE, errorMsg);
+                lastError = errorInformation;
+                LOGGER.error(lastError);
+            }
+        }
+
+        void purgeOutdatedNodes() {
+            for (final DatabaseStoredNodeData storedNodeData : nodeDatas.values()) {
+                final TimeDuration recordAge = TimeDuration.fromCurrent(storedNodeData.getTimestamp());
+                final String instanceID = storedNodeData.getInstanceID();
+
+                if (recordAge.isLongerThan(settings.getNodePurgeInterval())) {
+                    // purge outdated records
+                    LOGGER.debug("purging outdated node reference to instanceID '" + instanceID + "'");
+
+                    try {
+                        databaseService.getAccessor().remove(TABLE, dbKeyForStoredNode(storedNodeData));
+                    } catch (PwmException e) {
+                        final String errorMsg = "error purging outdated node reference: " + e.getMessage();
+                        final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE, errorMsg);
+                        lastError = errorInformation;
+                        LOGGER.error(lastError);
+                    }
+                    nodeDatas.remove(instanceID);
+                }
+            }
+        }
+    }
+}

src/main/java/password/pwm/svc/cluster/DatabaseClusterSettings.java

@@ -0,0 +1,50 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.cluster;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import password.pwm.AppProperty;
+import password.pwm.config.Configuration;
+import password.pwm.util.java.TimeDuration;
+
+import java.util.concurrent.TimeUnit;
+
+@Getter
+@AllArgsConstructor(access = AccessLevel.PRIVATE)
+class DatabaseClusterSettings {
+    private final boolean enable;
+    private final TimeDuration heartbeatInterval;
+    private final TimeDuration nodeTimeout;
+    private final TimeDuration nodePurgeInterval;
+
+    static DatabaseClusterSettings fromConfig(final Configuration configuration) {
+        return new DatabaseClusterSettings(
+                Boolean.parseBoolean(configuration.readAppProperty(AppProperty.CLUSTER_DB_ENABLE)),
+                new TimeDuration( Integer.parseInt(configuration.readAppProperty(AppProperty.CLUSTER_DB_HEARTBEAT_SECONDS)), TimeUnit.SECONDS),
+                new TimeDuration( Integer.parseInt(configuration.readAppProperty(AppProperty.CLUSTER_DB_NODE_TIMEOUT_SECONDS)), TimeUnit.SECONDS),
+                new TimeDuration( Integer.parseInt(configuration.readAppProperty(AppProperty.CLUSTER_DB_NODE_PURGE_SECONDS)), TimeUnit.SECONDS)
+        );
+    }
+}

src/main/java/password/pwm/svc/cluster/DatabaseStoredNodeData.java

@@ -0,0 +1,54 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.cluster;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import password.pwm.PwmApplication;
+import password.pwm.error.PwmUnrecoverableException;
+
+import java.io.Serializable;
+import java.time.Instant;
+
+@Getter
+@AllArgsConstructor(access = AccessLevel.PRIVATE)
+class DatabaseStoredNodeData implements Serializable {
+    private Instant timestamp;
+    private Instant startupTimestamp;
+    private String instanceID;
+    private String guid;
+    private String configHash;
+
+    static DatabaseStoredNodeData makeNew(final PwmApplication pwmApplication)
+            throws PwmUnrecoverableException
+    {
+        return new DatabaseStoredNodeData(
+                Instant.now(),
+                pwmApplication.getStartupTime(),
+                pwmApplication.getInstanceID(),
+                pwmApplication.getInstanceNonce(),
+                pwmApplication.getConfig().configurationHash()
+        );
+    }
+}

src/main/java/password/pwm/util/db/DatabaseClusterService.java → src/main/java/password/pwm/svc/cluster/NodeInfo.java

@@ -20,22 +20,27 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.db;
+package password.pwm.svc.cluster;
 
-public class DatabaseClusterService {
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
 
+import java.io.Serializable;
+import java.time.Instant;
 
+@Getter
+@AllArgsConstructor(access = AccessLevel.PACKAGE)
+public class NodeInfo implements Serializable {
+    private String instanceID;
+    private Instant lastSeen;
+    private Instant startupTime;
+    private NodeState nodeState;
+    private boolean configMatch;
 
-    private static final String KEY_ENGINE_START_PREFIX = "engine-start-";
-
-    private void heartbeat() {
-        /*
-        try {
-            put(DatabaseTable.PWM_META, KEY_ENGINE_START_PREFIX + instanceID, JavaHelper.toIsoDate(new java.util.Date()));
-        } catch (DatabaseException e) {
-            final String errorMsg = "error writing engine start time value: " + e.getMessage();
-            throw new DatabaseException(new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,errorMsg));
-        }
-        */
+    enum NodeState {
+        master,
+        online,
+        offline
     }
 }

src/main/java/password/pwm/util/db/DatabaseTable.java

@@ -30,4 +30,5 @@ public enum DatabaseTable {
     TOKENS,
     OTP,
     PW_NOTIFY,
+    CLUSTER_STATE,
 }

src/main/java/password/pwm/util/java/JavaHelper.java

@@ -57,6 +57,7 @@ import java.util.TimeZone;
 import java.util.TreeSet;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Predicate;
@@ -348,7 +349,7 @@ public class JavaHelper {
         return new CSVPrinter(new OutputStreamWriter(outputStream,PwmConstants.DEFAULT_CHARSET), PwmConstants.DEFAULT_CSV_FORMAT);
     }
 
-    public static ExecutorService makeSingleThreadExecutorService(
+    public static ScheduledExecutorService makeSingleThreadExecutorService(
             final PwmApplication pwmApplication,
             final Class clazz
     )

src/main/java/password/pwm/ws/server/RestResultBean.java

@@ -106,7 +106,7 @@ public class RestResultBean implements Serializable {
         final RestResultBean restResultBean = new RestResultBean();
         restResultBean.setError(true);
         restResultBean.setErrorMessage(errorInformation.toUserStr(locale, config));
-        if (forceDetail || pwmApplication.determineIfDetailErrorMsgShown()) {
+        if (forceDetail || (pwmApplication != null && pwmApplication.determineIfDetailErrorMsgShown())) {
             restResultBean.setErrorDetail(errorInformation.toDebugStr());
         }
         restResultBean.setErrorCode(errorInformation.getError().getErrorCode());

src/main/resources/password/pwm/AppProperty.properties

@@ -48,6 +48,10 @@ client.warningHeader.show=true
 client.pwShowRevertTimeout=45000
 client.js.enableHtml5Dialog=true
 client.jsp.showIcons=true
+cluster.db.enable=true
+cluster.db.heartbeatSeconds=60
+cluster.db.nodeTimeoutSeconds=600
+cluster.db.nodePurgeSeconds=86400
 config.reloadOnChange=true
 config.maxJdbcJarSize=10240000
 config.maxPersistentLoginSeconds=3600

src/main/webapp/WEB-INF/jsp/admin-dashboard.jsp

@@ -28,6 +28,7 @@
 <%@ page import="password.pwm.i18n.Admin" %>
 <%@ page import="password.pwm.i18n.Display" %>
 <%@ page import="password.pwm.svc.PwmService" %>
+<%@ page import="password.pwm.svc.cluster.NodeInfo" %>
 <%@ page import="password.pwm.svc.sessiontrack.SessionTrackService" %>
 <%@ page import="password.pwm.svc.stats.Statistic" %>
 <%@ page import="password.pwm.util.java.FileSystemUtility" %>
@@ -35,7 +36,8 @@
 <%@ page import="password.pwm.util.java.StringUtil" %>
 <%@ page import="password.pwm.util.java.TimeDuration" %>
 <%@ page import="password.pwm.util.localdb.LocalDB" %>
-<%@ page import="java.text.DateFormat" %>
+<%@ page import="java.lang.management.ManagementFactory" %>
+<%@ page import="java.lang.management.ThreadInfo" %>
 <%@ page import="java.text.NumberFormat" %>
 <%@ page import="java.time.Instant" %>
 <%@ page import="java.util.Collection" %>
@@ -43,9 +45,6 @@
 <%@ page import="java.util.List" %>
 <%@ page import="java.util.Locale" %>
 <%@ page import="java.util.Map" %>
-<%@ page import="java.util.TreeMap" %>
-<%@ page import="java.lang.management.ThreadInfo" %>
-<%@ page import="java.lang.management.ManagementFactory" %>
 <!DOCTYPE html>
 <%@ page language="java" session="true" isThreadSafe="true"
          contentType="text/html" %>
@@ -733,6 +732,64 @@
                 </div>
                 <% } %>
             </div>
+            <% if (dashboard_pwmApplication.getClusterService().status() == PwmService.STATUS.OPEN) { %>
+            <div id="Status" data-dojo-type="dijit.layout.ContentPane" title="Nodes" class="tabContent">
+                <div style="max-height: 400px; overflow: auto;">
+                    <table class="nomargin">
+                        <tr>
+                            <td style="font-weight:bold;">
+                                Instance ID
+                            </td>
+                            <td style="font-weight:bold;">
+                                Uptime
+                            </td>
+                            <td style="font-weight:bold;">
+                                Last Seen
+                            </td>
+                            <td style="font-weight:bold;">
+                                Master
+                            </td>
+                            <td style="font-weight:bold;">
+                                Config Match
+                            </td>
+                        </tr>
+                        <% for (final NodeInfo nodeInfo : dashboard_pwmApplication.getClusterService().nodes()) { %>
+                        <tr>
+                            <td>
+                                <%= nodeInfo.getInstanceID()  %>
+                            </td>
+                            <td>
+                                <% if (nodeInfo.getStartupTime() == null) { %>
+                                <pwm:display key="Value_NotApplicable"/>
+                                <% } else { %>
+                                <%= TimeDuration.fromCurrent(nodeInfo.getStartupTime()).asLongString(dashboard_pwmRequest.getLocale()) %>
+                                <% } %>
+                            </td>
+                            <td>
+                                <span class="timestamp">
+                                    <%= JspUtility.freindlyWrite(pageContext, nodeInfo.getLastSeen()) %>
+                                </span>
+                            </td>
+                            <td>
+                                <%= nodeInfo.getNodeState() %>
+                            </td>
+                            <td>
+                                <%= JspUtility.freindlyWrite(pageContext, nodeInfo.isConfigMatch())%>
+                            </td>
+                        </tr>
+                        <% } %>
+                    </table>
+                    <br/>
+                    <div class="footnote">
+                    <% if (dashboard_pwmApplication.getClusterService().isMaster()) { %>
+                    This node is the current master.
+                    <% } else { %>
+                    This node is not the current master.
+                    <% } %>
+                    </div>
+                </div>
+            </div>
+            <% } %>
         </div>
     </div>
     <div class="push"></div>