ldap-backed clustering mode

server/pom.xml

@@ -136,23 +136,31 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>
                 <version>3.0.1</version>
-                <configuration>
-                    <archive>
-                        <manifestEntries>
-                            <Implementation-Archive-Name>pwm.javadoc</Implementation-Archive-Name>
-                            <Implementation-Title>${project.name}</Implementation-Title>
-                            <Implementation-Version>${project.version}</Implementation-Version>
-                            <Implementation-Vendor>${project.organization.name}</Implementation-Vendor>
-                            <Implementation-URL>${project.organization.url}</Implementation-URL>
-                            <Implementation-Build-Java-Vendor>${java.vendor}</Implementation-Build-Java-Vendor>
-                            <Implementation-Build-Java-Version>${java.version}</Implementation-Build-Java-Version>
-                            <Implementation-Build>${build.number}</Implementation-Build>
-                            <Implementation-Build-Timestamp>${timestamp.iso}</Implementation-Build-Timestamp>
-                            <Implementation-Revision>${build.revision}</Implementation-Revision>
-                            <Implementation-Version-Display>v${project.version} b${build.number} r${build.revision}</Implementation-Version-Display>
-                        </manifestEntries>
-                    </archive>
-                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                        <configuration>
+                            <additionalOptions>-Xdoclint:none</additionalOptions>
+                            <archive>
+                                <manifestEntries>
+                                    <Implementation-Archive-Name>pwm.javadoc</Implementation-Archive-Name>
+                                    <Implementation-Title>${project.name}</Implementation-Title>
+                                    <Implementation-Version>${project.version}</Implementation-Version>
+                                    <Implementation-Vendor>${project.organization.name}</Implementation-Vendor>
+                                    <Implementation-URL>${project.organization.url}</Implementation-URL>
+                                    <Implementation-Build-Java-Vendor>${java.vendor}</Implementation-Build-Java-Vendor>
+                                    <Implementation-Build-Java-Version>${java.version}</Implementation-Build-Java-Version>
+                                    <Implementation-Build>${build.number}</Implementation-Build>
+                                    <Implementation-Build-Timestamp>${timestamp.iso}</Implementation-Build-Timestamp>
+                                    <Implementation-Revision>${build.revision}</Implementation-Revision>
+                                    <Implementation-Version-Display>v${project.version} b${build.number} r${build.revision}</Implementation-Version-Display>
+                                </manifestEntries>
+                            </archive>
+                        </configuration>
+                    </execution>
+                </executions>
             </plugin>
             <plugin>
                 <artifactId>maven-resources-plugin</artifactId>
@@ -547,7 +555,7 @@
         <dependency>
             <groupId>com.blueconic</groupId>
             <artifactId>browscap-java</artifactId>
-            <version>1.2.3</version>
+            <version>1.2.4</version>
         </dependency>
         <dependency>
             <groupId>org.jetbrains.xodus</groupId>

server/src/main/java/password/pwm/AppProperty.java

@@ -81,6 +81,9 @@ public enum AppProperty
     CLUSTER_DB_HEARTBEAT_SECONDS                    ( "cluster.db.heartbeatSeconds" ),
     CLUSTER_DB_NODE_TIMEOUT_SECONDS                 ( "cluster.db.nodeTimeoutSeconds" ),
     CLUSTER_DB_NODE_PURGE_SECONDS                   ( "cluster.db.nodePurgeSeconds" ),
+    CLUSTER_LDAP_HEARTBEAT_SECONDS                  ( "cluster.ldap.heartbeatSeconds" ),
+    CLUSTER_LDAP_NODE_TIMEOUT_SECONDS               ( "cluster.ldap.nodeTimeoutSeconds" ),
+    CLUSTER_LDAP_NODE_PURGE_SECONDS                 ( "cluster.ldap.nodePurgeSeconds" ),
     DB_JDBC_LOAD_STRATEGY                           ( "db.jdbcLoadStrategy" ),
     DB_CONNECTIONS_MAX                              ( "db.connections.max" ),
     DB_CONNECTIONS_TIMEOUT_MS                       ( "db.connections.timeoutMs" ),

server/src/main/java/password/pwm/PwmApplication.java

@@ -660,7 +660,7 @@ public class PwmApplication
 
     private String fetchInstanceID( final LocalDB localDB, final PwmApplication pwmApplication )
     {
-        String newInstanceID = pwmApplication.getConfig().readSettingAsString( PwmSetting.PWM_INSTANCE_NAME );
+        String newInstanceID = pwmApplication.getPwmEnvironment().getParameters().get( PwmEnvironment.ApplicationParameter.InstanceID );
 
         if ( newInstanceID != null && newInstanceID.trim().length() > 0 )
         {

server/src/main/java/password/pwm/PwmEnvironment.java

@@ -76,7 +76,8 @@ public class PwmEnvironment
         AutoWriteTomcatConfOutputFile,
         AppliancePort,
         ApplianceHostnameFile,
-        ApplianceTokenFile,;
+        ApplianceTokenFile,
+        InstanceID,;
 
         public static ApplicationParameter forString( final String input )
         {

server/src/main/java/password/pwm/config/PwmSetting.java

@@ -81,8 +81,6 @@ public enum PwmSetting
             "pwm.homeURL", PwmSettingSyntax.STRING, PwmSettingCategory.GENERAL ),
     URL_INTRO(
             "pwm.introURL", PwmSettingSyntax.SELECT, PwmSettingCategory.GENERAL ),
-    PWM_INSTANCE_NAME(
-            "pwmInstanceName", PwmSettingSyntax.STRING, PwmSettingCategory.GENERAL ),
     IDLE_TIMEOUT_SECONDS(
             "idleTimeoutSeconds", PwmSettingSyntax.DURATION, PwmSettingCategory.GENERAL ),
     HIDE_CONFIGURATION_HEALTH_WARNINGS(
@@ -95,11 +93,12 @@ public enum PwmSetting
             "http.proxy.url", PwmSettingSyntax.STRING, PwmSettingCategory.GENERAL ),
     HTTP_PROXY_EXCEPTIONS(
             "http.proxy.exceptions", PwmSettingSyntax.STRING_ARRAY, PwmSettingCategory.GENERAL ),
-
     APP_PROPERTY_OVERRIDES(
             "pwm.appProperty.overrides", PwmSettingSyntax.STRING_ARRAY, PwmSettingCategory.GENERAL ),
 
     // clustering
+    CLUSTER_STORAGE_MODE(
+            "cluster.storageMode", PwmSettingSyntax.SELECT, PwmSettingCategory.CLUSTERING ),
     SECURITY_LOGIN_SESSION_MODE(
             "security.loginSession.mode", PwmSettingSyntax.SELECT, PwmSettingCategory.CLUSTERING ),
     SECURITY_MODULE_SESSION_MODE(

server/src/main/java/password/pwm/error/PwmError.java

@@ -303,6 +303,8 @@ public enum PwmError
             5091, "Error_FileTypeIncorrect", null ),
     ERROR_FILE_TOO_LARGE(
             5092, "Error_FileTooLarge", null ),
+    ERROR_CLUSTER_SERVICE_ERROR(
+            5093, "Error_ClusterServiceError", null ),
 
     ERROR_REMOTE_ERROR_VALUE(
             6000, "Error_RemoteErrorValue", null, ErrorFlag.Permanent ),

server/src/main/java/password/pwm/health/HealthMessage.java

@@ -50,6 +50,7 @@ public enum HealthMessage
     Appliance_PendingUpdates( HealthStatus.CAUTION, HealthTopic.Appliance ),
     Appliance_UpdatesNotEnabled( HealthStatus.CAUTION, HealthTopic.Appliance ),
     Appliance_UpdateServiceNotConfigured( HealthStatus.WARN, HealthTopic.Appliance ),
+    Cluster_Error( HealthStatus.CAUTION, HealthTopic.Application ),
     Config_MissingProxyDN( HealthStatus.CONFIG, HealthTopic.Configuration ),
     Config_MissingProxyPassword( HealthStatus.CONFIG, HealthTopic.Configuration ),
     Config_NoSiteURL( HealthStatus.WARN, HealthTopic.Configuration ),

server/src/main/java/password/pwm/svc/cluster/ClusterProvider.java → server/src/main/java/password/pwm/svc/cluster/ClusterDataServiceProvider.java

@@ -23,14 +23,16 @@
 package password.pwm.svc.cluster;
 
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.java.TimeDuration;
 
-import java.util.List;
+import java.util.Map;
 
-public interface ClusterProvider
+public interface ClusterDataServiceProvider
 {
-    void close( );
+    Map<String, StoredNodeData> readStoredData( ) throws PwmUnrecoverableException;
 
-    boolean isMaster( );
+    void writeNodeStatus( StoredNodeData storedNodeData ) throws PwmUnrecoverableException;
 
-    List<NodeInfo> nodes( ) throws PwmUnrecoverableException;
+    int purgeOutdatedNodes( TimeDuration maxNodeAge )
+            throws PwmUnrecoverableException;
 }

server/src/main/java/password/pwm/svc/cluster/DatabaseClusterProvider.java → server/src/main/java/password/pwm/svc/cluster/ClusterMachine.java

@@ -27,11 +27,7 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.util.db.DatabaseService;
-import password.pwm.util.db.DatabaseTable;
-import password.pwm.util.java.ClosableIterator;
 import password.pwm.util.java.JavaHelper;
-import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
@@ -45,38 +41,32 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
-class DatabaseClusterProvider implements ClusterProvider
+class ClusterMachine
 {
-
-    private static final PwmLogger LOGGER = PwmLogger.forClass( DatabaseClusterProvider.class );
-
-    private static final DatabaseTable TABLE = DatabaseTable.CLUSTER_STATE;
-
-
-    private static final String KEY_PREFIX_NODE = "node-";
+    private static final PwmLogger LOGGER = PwmLogger.forClass( ClusterMachine.class );
 
     private final PwmApplication pwmApplication;
-    private final DatabaseService databaseService;
     private final ScheduledExecutorService executorService;
+    private final ClusterDataServiceProvider clusterDataServiceProvider;
 
     private ErrorInformation lastError;
 
-    private final Map<String, DatabaseStoredNodeData> nodeDatas = new ConcurrentHashMap<>();
+    private final Map<String, StoredNodeData> knownNodes = new ConcurrentHashMap<>();
 
-    private final DatabaseClusterSettings settings;
+    private final ClusterSettings settings;
+    private final ClusterStatistics clusterStatistics = new ClusterStatistics();
 
-    DatabaseClusterProvider( final PwmApplication pwmApplication ) throws PwmUnrecoverableException
+    ClusterMachine(
+            final PwmApplication pwmApplication,
+            final ClusterDataServiceProvider clusterDataServiceProvider,
+            final ClusterSettings clusterSettings
+    )
     {
         this.pwmApplication = pwmApplication;
-        this.settings = DatabaseClusterSettings.fromConfig( pwmApplication.getConfig() );
-
-        if ( !settings.isEnable() )
-        {
-            throw new PwmUnrecoverableException( new ErrorInformation( PwmError.CONFIG_FORMAT_ERROR, "database clustering is not enabled via app property" ) );
-        }
+        this.clusterDataServiceProvider = clusterDataServiceProvider;
+        this.settings = clusterSettings;
 
-        this.databaseService = pwmApplication.getDatabaseService();
-        this.executorService = JavaHelper.makeSingleThreadExecutorService( pwmApplication, DatabaseClusterProvider.class );
+        this.executorService = JavaHelper.makeSingleThreadExecutorService( pwmApplication, ClusterMachine.class );
 
         final long intervalSeconds = settings.getHeartbeatInterval().getTotalSeconds();
 
@@ -88,19 +78,17 @@ class DatabaseClusterProvider implements ClusterProvider
         );
     }
 
-    @Override
     public void close( )
     {
         JavaHelper.closeAndWaitExecutor( executorService, new TimeDuration( 1, TimeUnit.SECONDS ) );
     }
 
 
-    @Override
     public List<NodeInfo> nodes( ) throws PwmUnrecoverableException
     {
         final Map<String, NodeInfo> returnObj = new TreeMap<>();
         final String configHash = pwmApplication.getConfig().configurationHash();
-        for ( final DatabaseStoredNodeData storedNodeData : nodeDatas.values() )
+        for ( final StoredNodeData storedNodeData : knownNodes.values() )
         {
             final boolean configMatch = configHash.equals( storedNodeData.getConfigHash() );
             final boolean timedOut = isTimedOut( storedNodeData );
@@ -132,7 +120,7 @@ class DatabaseClusterProvider implements ClusterProvider
 
     private String masterInstanceId( )
     {
-        final List<DatabaseStoredNodeData> copiedDatas = new ArrayList<>( nodeDatas.values() );
+        final List<StoredNodeData> copiedDatas = new ArrayList<>( knownNodes.values() );
         if ( copiedDatas.isEmpty() )
         {
             return null;
@@ -141,7 +129,7 @@ class DatabaseClusterProvider implements ClusterProvider
         String masterID = null;
         Instant eldestRecord = Instant.now();
 
-        for ( final DatabaseStoredNodeData nodeData : copiedDatas )
+        for ( final StoredNodeData nodeData : copiedDatas )
         {
             if ( !isTimedOut( nodeData ) )
             {
@@ -155,7 +143,6 @@ class DatabaseClusterProvider implements ClusterProvider
         return masterID;
     }
 
-    @Override
     public boolean isMaster( )
     {
         final String myID = pwmApplication.getInstanceID();
@@ -163,27 +150,21 @@ class DatabaseClusterProvider implements ClusterProvider
         return myID.equals( masterID );
     }
 
-    private boolean isMaster( final DatabaseStoredNodeData databaseStoredNodeData )
+    private boolean isMaster( final StoredNodeData storedNodeData )
     {
         final String masterID = masterInstanceId();
-        return databaseStoredNodeData.getInstanceID().equals( masterID );
+        return storedNodeData.getInstanceID().equals( masterID );
     }
 
-    private String dbKeyForStoredNode( final DatabaseStoredNodeData storedNodeData ) throws PwmUnrecoverableException
+    private boolean isTimedOut( final StoredNodeData storedNodeData )
     {
-        final String instanceID = storedNodeData.getInstanceID();
-        final String hash = pwmApplication.getSecureService().hash( instanceID );
-        final String truncatedHash = hash.length() > 64
-                ? hash.substring( 0, 64 )
-                : hash;
-
-        return KEY_PREFIX_NODE + truncatedHash;
+        final TimeDuration age = TimeDuration.fromCurrent( storedNodeData.getTimestamp() );
+        return age.isLongerThan( settings.getNodeTimeout() );
     }
 
-    private boolean isTimedOut( final DatabaseStoredNodeData storedNodeData )
+    public ErrorInformation getLastError( )
     {
-        final TimeDuration age = TimeDuration.fromCurrent( storedNodeData.getTimestamp() );
-        return age.isLongerThan( settings.getNodeTimeout() );
+        return lastError;
     }
 
     private class HeartbeatProcess implements Runnable
@@ -199,15 +180,14 @@ class DatabaseClusterProvider implements ClusterProvider
         {
             try
             {
-                final DatabaseStoredNodeData storedNodeData = DatabaseStoredNodeData.makeNew( pwmApplication );
-                final String key = dbKeyForStoredNode( storedNodeData );
-                final String value = JsonUtil.serialize( storedNodeData );
-                databaseService.getAccessor().put( TABLE, key, value );
+                final StoredNodeData storedNodeData = StoredNodeData.makeNew( pwmApplication );
+                clusterDataServiceProvider.writeNodeStatus( storedNodeData );
+                clusterStatistics.getClusterWrites().incrementAndGet();
             }
             catch ( PwmException e )
             {
                 final String errorMsg = "error writing database cluster heartbeat: " + e.getMessage();
-                final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, errorMsg );
+                final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_CLUSTER_SERVICE_ERROR, errorMsg );
                 lastError = errorInformation;
                 LOGGER.error( lastError );
             }
@@ -215,23 +195,16 @@ class DatabaseClusterProvider implements ClusterProvider
 
         void readNodeStatuses( )
         {
-            try ( ClosableIterator<String> tableIterator = databaseService.getAccessor().iterator( TABLE ) )
+            try
             {
-                while ( tableIterator.hasNext() )
-                {
-                    final String dbKey = tableIterator.next();
-                    if ( dbKey.startsWith( KEY_PREFIX_NODE ) )
-                    {
-                        final String rawValueInDb = databaseService.getAccessor().get( TABLE, dbKey );
-                        final DatabaseStoredNodeData nodeDataInDb = JsonUtil.deserialize( rawValueInDb, DatabaseStoredNodeData.class );
-                        nodeDatas.put( nodeDataInDb.getInstanceID(), nodeDataInDb );
-                    }
-                }
+                final Map<String, StoredNodeData> readNodeData = clusterDataServiceProvider.readStoredData();
+                knownNodes.putAll( readNodeData );
+                clusterStatistics.getClusterReads().incrementAndGet();
             }
             catch ( PwmException e )
             {
-                final String errorMsg = "error reading database node statuses: " + e.getMessage();
-                final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, errorMsg );
+                final String errorMsg = "error reading node statuses: " + e.getMessage();
+                final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_CLUSTER_SERVICE_ERROR, errorMsg );
                 lastError = errorInformation;
                 LOGGER.error( lastError );
             }
@@ -239,30 +212,23 @@ class DatabaseClusterProvider implements ClusterProvider
 
         void purgeOutdatedNodes( )
         {
-            for ( final DatabaseStoredNodeData storedNodeData : nodeDatas.values() )
+            try
             {
-                final TimeDuration recordAge = TimeDuration.fromCurrent( storedNodeData.getTimestamp() );
-                final String instanceID = storedNodeData.getInstanceID();
-
-                if ( recordAge.isLongerThan( settings.getNodePurgeInterval() ) )
-                {
-                    // purge outdated records
-                    LOGGER.debug( "purging outdated node reference to instanceID '" + instanceID + "'" );
-
-                    try
-                    {
-                        databaseService.getAccessor().remove( TABLE, dbKeyForStoredNode( storedNodeData ) );
-                    }
-                    catch ( PwmException e )
-                    {
-                        final String errorMsg = "error purging outdated node reference: " + e.getMessage();
-                        final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, errorMsg );
-                        lastError = errorInformation;
-                        LOGGER.error( lastError );
-                    }
-                    nodeDatas.remove( instanceID );
-                }
+                final int purges = clusterDataServiceProvider.purgeOutdatedNodes( settings.getNodePurgeInterval() );
+                clusterStatistics.getNodePurges().addAndGet( purges );
+            }
+            catch ( PwmException e )
+            {
+                final String errorMsg = "error purging outdated node reference: " + e.getMessage();
+                final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_CLUSTER_SERVICE_ERROR, errorMsg );
+                lastError = errorInformation;
+                LOGGER.error( lastError );
             }
         }
     }
+
+    public ClusterStatistics getClusterStatistics( )
+    {
+        return clusterStatistics;
+    }
 }

server/src/main/java/password/pwm/svc/cluster/ClusterService.java

@@ -23,14 +23,24 @@
 package password.pwm.svc.cluster;
 
 import password.pwm.PwmApplication;
+import password.pwm.bean.UserIdentity;
+import password.pwm.config.PwmSetting;
+import password.pwm.config.option.DataStorageMethod;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.PwmService;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.JsonUtil;
 import password.pwm.util.logging.PwmLogger;
 
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 public class ClusterService implements PwmService
 {
@@ -39,7 +49,10 @@ public class ClusterService implements PwmService
 
     private PwmApplication pwmApplication;
     private STATUS status = STATUS.NEW;
-    private ClusterProvider clusterProvider;
+    private ClusterMachine clusterMachine;
+    private DataStorageMethod dataStore;
+    private ErrorInformation startupError;
+
 
     @Override
     public STATUS status( )
@@ -53,52 +66,103 @@ public class ClusterService implements PwmService
         status = STATUS.OPENING;
         this.pwmApplication = pwmApplication;
 
+
         try
         {
-            if ( this.pwmApplication.getConfig().hasDbConfigured() )
+            final ClusterSettings clusterSettings;
+            final ClusterDataServiceProvider clusterDataServiceProvider;
+            dataStore = figureDataStorageMethod( pwmApplication );
+
+            if ( dataStore != null )
             {
-                clusterProvider = new DatabaseClusterProvider( pwmApplication );
+                switch ( dataStore )
+                {
+                    case DB:
+                    {
+                        LOGGER.trace( "starting database-backed cluster provider" );
+                        clusterSettings = ClusterSettings.fromConfigForDB( pwmApplication.getConfig() );
+                        clusterDataServiceProvider = new DatabaseClusterDataService( pwmApplication );
+                    }
+                    break;
+
+                    case LDAP:
+                    {
+                        LOGGER.trace( "starting ldap-backed cluster provider" );
+                        clusterSettings = ClusterSettings.fromConfigForLDAP( pwmApplication.getConfig() );
+                        clusterDataServiceProvider = new LDAPClusterDataService( pwmApplication );
+                    }
+                    break;
+
+                    default:
+                        LOGGER.debug( "no suitable storage method configured " );
+                        JavaHelper.unhandledSwitchStatement( dataStore );
+                        return;
+
+                }
+
+                clusterMachine = new ClusterMachine( pwmApplication, clusterDataServiceProvider, clusterSettings );
+                status = STATUS.OPEN;
+                return;
             }
         }
-        catch ( PwmException e )
+        catch ( Exception e )
         {
-            LOGGER.error( "error starting up cluster provider service: " + e.getMessage() );
-            status = STATUS.CLOSED;
-            return;
+            LOGGER.error( "error starting up cluster service: " + e.getMessage() );
         }
 
-        status = STATUS.OPEN;
+        status = STATUS.CLOSED;
     }
 
     @Override
     public void close( )
     {
-        if ( clusterProvider != null )
+        if ( clusterMachine != null )
         {
-            clusterProvider.close();
-            clusterProvider = null;
+            clusterMachine.close();
+            clusterMachine = null;
         }
-        clusterProvider = null;
         status = STATUS.CLOSED;
     }
 
     @Override
     public List<HealthRecord> healthCheck( )
     {
+        if ( clusterMachine != null )
+        {
+            final ErrorInformation errorInformation = clusterMachine.getLastError();
+            if ( errorInformation != null )
+            {
+                final HealthRecord healthRecord = HealthRecord.forMessage( HealthMessage.Cluster_Error, errorInformation.getDetailedErrorMsg() );
+                return Collections.singletonList( healthRecord );
+            }
+        }
+
+        if ( startupError != null )
+        {
+            final HealthRecord healthRecord = HealthRecord.forMessage( HealthMessage.Cluster_Error, startupError.getDetailedErrorMsg() );
+            return Collections.singletonList( healthRecord );
+        }
+
         return null;
     }
 
     @Override
     public ServiceInfoBean serviceInfo( )
     {
-        return null;
+        final Map<String, String> props = new HashMap<>();
+
+        if ( clusterMachine != null )
+        {
+            props.putAll( JsonUtil.deserializeStringMap( JsonUtil.serialize( clusterMachine.getClusterStatistics() ) ) );
+        }
+        return new ServiceInfoBean( Collections.singleton( dataStore ), props );
     }
 
     public boolean isMaster( )
     {
-        if ( clusterProvider != null )
+        if ( status == STATUS.OPEN && clusterMachine != null )
         {
-            return clusterProvider.isMaster();
+            return clusterMachine.isMaster();
         }
 
         return false;
@@ -106,10 +170,40 @@ public class ClusterService implements PwmService
 
     public List<NodeInfo> nodes( ) throws PwmUnrecoverableException
     {
-        if ( status == STATUS.OPEN && clusterProvider != null )
+        if ( status == STATUS.OPEN && clusterMachine != null )
         {
-            return clusterProvider.nodes();
+            return clusterMachine.nodes();
         }
         return Collections.emptyList();
     }
+
+    private DataStorageMethod figureDataStorageMethod( final PwmApplication pwmApplication )
+            throws PwmUnrecoverableException
+    {
+        final DataStorageMethod method = pwmApplication.getConfig().readSettingAsEnum( PwmSetting.CLUSTER_STORAGE_MODE, DataStorageMethod.class );
+        if ( method == DataStorageMethod.LDAP )
+        {
+            final UserIdentity userIdentity = pwmApplication.getConfig().getDefaultLdapProfile().getTestUser( pwmApplication );
+            if ( userIdentity == null )
+            {
+                final String msg = "LDAP storage type selected, but LDAP test user not defined.";
+                LOGGER.debug( msg );
+                startupError = new ErrorInformation( PwmError.ERROR_CLUSTER_SERVICE_ERROR, msg );
+                return null;
+            }
+        }
+
+        if ( method == DataStorageMethod.DB )
+        {
+            if ( !pwmApplication.getConfig().hasDbConfigured() )
+            {
+                final String msg = "DB storage type selected, but remote DB is not configured.";
+                LOGGER.debug( msg );
+                startupError = new ErrorInformation( PwmError.ERROR_CLUSTER_SERVICE_ERROR, msg );
+                return null;
+            }
+        }
+
+        return method;
+    }
 }

server/src/main/java/password/pwm/svc/cluster/DatabaseClusterSettings.java → server/src/main/java/password/pwm/svc/cluster/ClusterSettings.java

@@ -24,29 +24,36 @@ package password.pwm.svc.cluster;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Getter;
+import lombok.Value;
 import password.pwm.AppProperty;
 import password.pwm.config.Configuration;
 import password.pwm.util.java.TimeDuration;
 
 import java.util.concurrent.TimeUnit;
 
-@Getter
+@Value
 @AllArgsConstructor( access = AccessLevel.PRIVATE )
-class DatabaseClusterSettings
+class ClusterSettings
 {
-    private final boolean enable;
     private final TimeDuration heartbeatInterval;
     private final TimeDuration nodeTimeout;
     private final TimeDuration nodePurgeInterval;
 
-    static DatabaseClusterSettings fromConfig( final Configuration configuration )
+    static ClusterSettings fromConfigForDB( final Configuration configuration )
     {
-        return new DatabaseClusterSettings(
-                Boolean.parseBoolean( configuration.readAppProperty( AppProperty.CLUSTER_DB_ENABLE ) ),
+        return new ClusterSettings(
                 new TimeDuration( Integer.parseInt( configuration.readAppProperty( AppProperty.CLUSTER_DB_HEARTBEAT_SECONDS ) ), TimeUnit.SECONDS ),
                 new TimeDuration( Integer.parseInt( configuration.readAppProperty( AppProperty.CLUSTER_DB_NODE_TIMEOUT_SECONDS ) ), TimeUnit.SECONDS ),
                 new TimeDuration( Integer.parseInt( configuration.readAppProperty( AppProperty.CLUSTER_DB_NODE_PURGE_SECONDS ) ), TimeUnit.SECONDS )
         );
     }
+
+    static ClusterSettings fromConfigForLDAP( final Configuration configuration )
+    {
+        return new ClusterSettings(
+                new TimeDuration( Integer.parseInt( configuration.readAppProperty( AppProperty.CLUSTER_LDAP_HEARTBEAT_SECONDS ) ), TimeUnit.SECONDS ),
+                new TimeDuration( Integer.parseInt( configuration.readAppProperty( AppProperty.CLUSTER_LDAP_NODE_TIMEOUT_SECONDS ) ), TimeUnit.SECONDS ),
+                new TimeDuration( Integer.parseInt( configuration.readAppProperty( AppProperty.CLUSTER_LDAP_NODE_PURGE_SECONDS ) ), TimeUnit.SECONDS )
+        );
+    }
 }

server/src/main/java/password/pwm/svc/cluster/ClusterStatistics.java

@@ -0,0 +1,36 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.cluster;
+
+import lombok.Value;
+
+import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@Value
+public class ClusterStatistics implements Serializable
+{
+    private final AtomicInteger clusterWrites = new AtomicInteger( 0 );
+    private final AtomicInteger clusterReads = new AtomicInteger( 0 );
+    private final AtomicInteger nodePurges = new AtomicInteger( 0 );
+}

server/src/main/java/password/pwm/svc/cluster/DatabaseClusterDataService.java

@@ -0,0 +1,147 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.cluster;
+
+import password.pwm.PwmApplication;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.db.DatabaseAccessor;
+import password.pwm.util.db.DatabaseException;
+import password.pwm.util.db.DatabaseTable;
+import password.pwm.util.java.ClosableIterator;
+import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogger;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class DatabaseClusterDataService implements ClusterDataServiceProvider
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( DatabaseClusterDataService.class );
+
+    private static final DatabaseTable TABLE = DatabaseTable.CLUSTER_STATE;
+    private static final String KEY_PREFIX_NODE = "node-";
+
+    private final PwmApplication pwmApplication;
+
+    public DatabaseClusterDataService( final PwmApplication pwmApplication )
+    {
+        this.pwmApplication = pwmApplication;
+    }
+
+    private DatabaseAccessor getDatabaseAccessor()
+            throws PwmUnrecoverableException
+    {
+        return pwmApplication.getDatabaseService().getAccessor();
+    }
+
+    private String localKeyForStoredNode( final StoredNodeData storedNodeData )
+            throws PwmUnrecoverableException
+    {
+        final String instanceID = storedNodeData.getInstanceID();
+        final String hash = pwmApplication.getSecureService().hash( instanceID );
+        final String truncatedHash = hash.length() > 64
+                ? hash.substring( 0, 64 )
+                : hash;
+
+        return KEY_PREFIX_NODE + truncatedHash;
+    }
+
+
+    @Override
+    public Map<String, StoredNodeData> readStoredData( )
+            throws PwmUnrecoverableException
+    {
+        final Map<String, StoredNodeData> returnList = new LinkedHashMap<>();
+        final DatabaseAccessor databaseAccessor = getDatabaseAccessor();
+        try ( ClosableIterator<String> tableIterator = databaseAccessor.iterator( TABLE ) )
+        {
+            while ( tableIterator.hasNext() )
+            {
+                final String dbKey = tableIterator.next();
+                if ( dbKey.startsWith( KEY_PREFIX_NODE ) )
+                {
+                    final String rawValueInDb = databaseAccessor.get( TABLE, dbKey );
+                    final StoredNodeData nodeDataInDb = JsonUtil.deserialize( rawValueInDb, StoredNodeData.class );
+                    returnList.put( nodeDataInDb.getInstanceID(), nodeDataInDb );
+                }
+            }
+        }
+        catch ( DatabaseException e )
+        {
+            throw new PwmUnrecoverableException( PwmError.ERROR_DB_UNAVAILABLE, "unexpected database error reading cluster node status: " + e.getMessage() );
+        }
+        return returnList;
+    }
+
+    @Override
+    public void writeNodeStatus( final StoredNodeData storedNodeData ) throws PwmUnrecoverableException
+    {
+        try
+        {
+            final DatabaseAccessor databaseAccessor = getDatabaseAccessor();
+            final String key = localKeyForStoredNode( storedNodeData );
+            final String value = JsonUtil.serialize( storedNodeData );
+            databaseAccessor.put( TABLE, key, value );
+        }
+        catch ( DatabaseException e )
+        {
+            throw new PwmUnrecoverableException( PwmError.ERROR_DB_UNAVAILABLE, "unexpected database error writing cluster node status: " + e.getMessage() );
+        }
+    }
+
+    @Override
+    public int purgeOutdatedNodes( final TimeDuration maxNodeAge )
+            throws PwmUnrecoverableException
+    {
+        int nodesPurged = 0;
+
+        try
+        {
+            final Map<String, StoredNodeData> nodeDatas = readStoredData();
+            final DatabaseAccessor databaseAccessor = getDatabaseAccessor();
+            for ( final StoredNodeData storedNodeData : nodeDatas.values() )
+            {
+                final TimeDuration recordAge = TimeDuration.fromCurrent( storedNodeData.getTimestamp() );
+                final String instanceID = storedNodeData.getInstanceID();
+
+
+                if ( recordAge.isLongerThan( maxNodeAge ) )
+                {
+                    // purge outdated records
+                    LOGGER.debug( "purging outdated node reference to instanceID '" + instanceID + "'" );
+
+                    databaseAccessor.remove( TABLE, localKeyForStoredNode( storedNodeData ) );
+                    nodesPurged++;
+                }
+            }
+        }
+        catch ( DatabaseException e )
+        {
+            throw new PwmUnrecoverableException( PwmError.ERROR_DB_UNAVAILABLE, "unexpected database error writing cluster node status: " + e.getMessage() );
+        }
+
+        return nodesPurged;
+    }
+}

server/src/main/java/password/pwm/svc/cluster/LDAPClusterDataService.java

@@ -0,0 +1,178 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.cluster;
+
+import com.novell.ldapchai.ChaiUser;
+import com.novell.ldapchai.exception.ChaiException;
+import lombok.Value;
+import password.pwm.PwmApplication;
+import password.pwm.bean.UserIdentity;
+import password.pwm.config.PwmSetting;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogger;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+public class LDAPClusterDataService implements ClusterDataServiceProvider
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( LDAPClusterDataService.class );
+
+    private final PwmApplication pwmApplication;
+    private static final String VALUE_PREFIX = "0006#.#.#";
+
+    public LDAPClusterDataService( final PwmApplication pwmApplication )
+    {
+        this.pwmApplication = pwmApplication;
+    }
+
+    @Override
+    public Map<String, StoredNodeData> readStoredData( ) throws PwmUnrecoverableException
+    {
+        final Map<String, StoredNodeData> returnData = new LinkedHashMap<>(  );
+
+        final LDAPHelper ldapHelper = LDAPHelper.createLDAPHelper( pwmApplication );
+
+        try
+        {
+            final Set<String> values = ldapHelper.getChaiUser().readMultiStringAttribute( ldapHelper.getAttr() );
+            for ( final String value : values )
+            {
+                if ( value.startsWith( VALUE_PREFIX ) )
+                {
+                    final String rawValue = value.substring( VALUE_PREFIX.length() );
+                    final StoredNodeData storedNodeData = JsonUtil.deserialize( rawValue, StoredNodeData.class );
+                    returnData.put( storedNodeData.getInstanceID(),  storedNodeData );
+                }
+            }
+        }
+        catch ( ChaiException e )
+        {
+            throw new PwmUnrecoverableException( PwmError.ERROR_LDAP_DATA_ERROR, "error reading cluster data: " + e.getMessage() );
+        }
+
+        return returnData;
+    }
+
+    @Override
+    public void writeNodeStatus( final StoredNodeData storedNodeData ) throws PwmUnrecoverableException
+    {
+        final Map<String, StoredNodeData> currentServerData = readStoredData();
+        final StoredNodeData removeNode = currentServerData.get( storedNodeData.getInstanceID() );
+
+        final LDAPHelper ldapHelper = LDAPHelper.createLDAPHelper( pwmApplication );
+
+        final String newRawValue = VALUE_PREFIX + JsonUtil.serialize( storedNodeData );
+
+        try
+        {
+            if ( removeNode != null )
+            {
+                final String oldRawValue = VALUE_PREFIX + JsonUtil.serialize( removeNode );
+                ldapHelper.getChaiUser().replaceAttribute( ldapHelper.getAttr(), oldRawValue, newRawValue );
+            }
+            else
+            {
+                ldapHelper.getChaiUser().addAttribute( ldapHelper.getAttr(), newRawValue );
+            }
+        }
+        catch ( ChaiException e )
+        {
+            throw new PwmUnrecoverableException( PwmError.ERROR_LDAP_DATA_ERROR, "error writing cluster data: " + e.getMessage() );
+        }
+
+    }
+
+    @Override
+    public int purgeOutdatedNodes( final TimeDuration maxNodeAge ) throws PwmUnrecoverableException
+    {
+        final LDAPHelper ldapHelper = LDAPHelper.createLDAPHelper( pwmApplication );
+
+        int nodesPurged = 0;
+
+        final Map<String, StoredNodeData> nodeDatas = readStoredData();
+
+        for ( final StoredNodeData storedNodeData : nodeDatas.values() )
+        {
+            final TimeDuration recordAge = TimeDuration.fromCurrent( storedNodeData.getTimestamp() );
+            final String instanceID = storedNodeData.getInstanceID();
+
+            if ( recordAge.isLongerThan( maxNodeAge ) )
+            {
+                // purge outdated records
+                LOGGER.debug( "purging outdated node reference to instanceID '" + instanceID + "'" );
+
+                try
+                {
+                    final String oldRawValue = VALUE_PREFIX + JsonUtil.serialize( storedNodeData );
+                    ldapHelper.getChaiUser().deleteAttribute( ldapHelper.getAttr(), oldRawValue );
+                    nodesPurged++;
+                }
+                catch ( ChaiException e )
+                {
+                    throw new PwmUnrecoverableException( PwmError.ERROR_LDAP_DATA_ERROR, "error purging cluster data: " + e.getMessage() );
+                }
+            }
+        }
+
+        return nodesPurged;
+
+    }
+
+    @Value
+    private static class LDAPHelper
+    {
+        private final PwmApplication pwmApplication;
+        private final UserIdentity userIdentity;
+        private final ChaiUser chaiUser;
+        private final String attr;
+
+        private LDAPHelper( final PwmApplication pwmApplication )
+                throws PwmUnrecoverableException
+        {
+            this.pwmApplication = pwmApplication;
+
+            userIdentity = pwmApplication.getConfig().getDefaultLdapProfile().getTestUser( pwmApplication );
+            if ( userIdentity == null )
+            {
+                final String ldapProfileID = pwmApplication.getConfig().getDefaultLdapProfile().getIdentifier();
+                final String errorMsg = "a test user is not configured for ldap profile '" + ldapProfileID + "'";
+                final ErrorInformation errorInformation = new ErrorInformation( PwmError.CONFIG_FORMAT_ERROR, errorMsg );
+                throw new PwmUnrecoverableException( errorInformation );
+            }
+            chaiUser = pwmApplication.getProxiedChaiUser( userIdentity );
+            attr = userIdentity.getLdapProfile( pwmApplication.getConfig() ).readSettingAsString( PwmSetting.CHALLENGE_USER_ATTRIBUTE );
+
+        }
+
+        static LDAPHelper createLDAPHelper( final PwmApplication pwmApplication ) throws PwmUnrecoverableException
+        {
+            return new LDAPHelper( pwmApplication );
+        }
+    }
+}

server/src/main/java/password/pwm/svc/cluster/DatabaseStoredNodeData.java → server/src/main/java/password/pwm/svc/cluster/StoredNodeData.java

@@ -33,7 +33,7 @@ import java.time.Instant;
 
 @Getter
 @AllArgsConstructor( access = AccessLevel.PRIVATE )
-class DatabaseStoredNodeData implements Serializable
+class StoredNodeData implements Serializable
 {
     private Instant timestamp;
     private Instant startupTimestamp;
@@ -41,10 +41,10 @@ class DatabaseStoredNodeData implements Serializable
     private String guid;
     private String configHash;
 
-    static DatabaseStoredNodeData makeNew( final PwmApplication pwmApplication )
+    static StoredNodeData makeNew( final PwmApplication pwmApplication )
             throws PwmUnrecoverableException
     {
-        return new DatabaseStoredNodeData(
+        return new StoredNodeData(
                 Instant.now(),
                 pwmApplication.getStartupTime(),
                 pwmApplication.getInstanceID(),

server/src/main/resources/password/pwm/AppProperty.properties

@@ -58,6 +58,9 @@ cluster.db.enable=true
 cluster.db.heartbeatSeconds=60
 cluster.db.nodeTimeoutSeconds=600
 cluster.db.nodePurgeSeconds=86400
+cluster.ldap.heartbeatSeconds=60
+cluster.ldap.nodeTimeoutSeconds=600
+cluster.ldap.nodePurgeSeconds=86400
 config.reloadOnChange=true
 config.maxJdbcJarSize=10240000
 config.maxPersistentLoginSeconds=3600

server/src/main/resources/password/pwm/config/PwmSetting.xml

@@ -101,11 +101,6 @@
             <option value="/public">/public</option>
         </options>
     </setting>
-    <setting hidden="false" key="pwmInstanceName" level="2">
-        <default>
-            <value />
-        </default>
-    </setting>
     <setting hidden="false" key="idleTimeoutSeconds" level="1" required="true">
         <properties>
             <property key="Minimum">60</property>
@@ -985,9 +980,7 @@
     </setting>
     <setting hidden="false" key="sms.httpRequestHeaders" level="2">
         <regex>^[A-Za-z0-9_\.-]+:.*</regex>
-        <default>
-            <value />
-        </default>
+        <default/>
     </setting>
     <setting hidden="false" key="sms.maxTextLength" level="2" required="true">
         <default>
@@ -1567,6 +1560,18 @@
             <value>28800</value>
         </default>
     </setting>
+    <setting hidden="false" key="cluster.storageMode" level="2">
+        <default>
+            <value>LDAP</value>
+        </default>
+        <default template="DB">
+            <value>DB</value>
+        </default>
+        <options>
+            <option value="LDAP">LDAP Directory</option>
+            <option value="DB">Remote Database</option>
+        </options>
+    </setting>
     <setting hidden="false" key="security.loginSession.mode" level="2">
         <default>
             <value>CRYPTCOOKIE</value>

server/src/main/resources/password/pwm/i18n/Error.properties

@@ -162,7 +162,7 @@ Error_PasswordOnlyBad=Password incorrect.  Please try again.
 Error_RecoverySequenceIncomplete=A problem occurred during the forgotten password sequence, please try again.
 Error_FileTypeIncorrect=The file type is not correct.
 Error_FileTooLarge=The file is too large.
-
+Error_ClusterServiceError=An error occurred with the cluster service: %1%.   Check the log files for more information.
 Error_RemoteErrorValue=Remote Error: %1%
 
 Error_ConfigUploadSuccess=File uploaded successfully

server/src/main/resources/password/pwm/i18n/Health.properties

@@ -40,6 +40,7 @@ HealthMessage_BrokenMethod=broken method invocation for '%1%', error: %2%
 HealthMessage_Appliance_PendingUpdates=Appliance updates are available.
 HealthMessage_Appliance_UpdatesNotEnabled=Appliance auto-update service is not enabled.
 HealthMessage_Appliance_UpdateServiceNotConfigured=Appliance update service has not been configured.
+HealthMessage_Cluster_Error=The cluster system can not operate normally: %1%
 HealthMessage_Config_NoSiteURL=The site URL is not configured, please configure %1%
 HealthMessage_Config_LDAPWireTrace=The %1% setting is enabled and should be disabled for proper security
 HealthMessage_Config_PromiscuousLDAP=%1% setting should be set to false for proper security

server/src/main/resources/password/pwm/i18n/PwmSetting.properties

@@ -266,6 +266,7 @@ Setting_Description_challenge.showConfirmation=Enable this option to show the re
 Setting_Description_challenge.token.sendMethod=Select the methods you want to use for sending the token code or new password to the user.
 Setting_Description_challenge.userAttribute=Specify the attribute to use for response storage when storing responses in an LDAP directory.
 Setting_Description_changePassword.writeAttributes=Add actions to take after a user change password event occurs.  @PwmAppName@ invokes these actions just after writing the password.  You can use macros within the action and are expanded based on the logged in user.
+Setting_Description_cluster.storageMode=Data storage system used for cluster module.  <p>If <b>LDAP</b> is selected, a test user (<a data-gotoSettingLink\="ldap.testuser.username">@PwmSettingReference\:ldap.testuser.username@</a>) must be configured and the response storage attribute (<a data-gotoSettingLink\="ldap.testuser.username">@PwmSettingReference\:ldap.testuser.username@</a>) must be writable by the proxy user.</p><p>If <b>DATABASE</b> is selected then a database must be configured and available for @PwmAppName@ to operate.</p>  
 Setting_Description_command.checkResponses.queryMatch=Controls which users are forced to setup responses.  Users that match this permission will be forced to setup responses. 
 Setting_Description_db.classname=Add the remote database JDBC driver class name.  Consult the database vendor to determine the correct class name for your database.<br/><br/><table><tr><td class\="key">Database Type</td><td class\="key">Example Class Name</td></tr><tr><td>MS-SQL</td><td>com.microsoft.sqlserver.jdbc.SQLServerDriver</td></tr><tr><td>MS-SQL using jTDS</td><td>net.sourceforge.jtds.jdbc.Driver</td></tr><tr><td>Oracle</td><td>oracle.jdbc.OracleDriver</td></tr></table><div class="footnote">The above are examples only, consult your database documentation for the proper setting value.</div>
 Setting_Description_db.columnType.key=Specify the database column type for key columns.  @PwmAppName@ uses the column type only during schema creation.  All tables are two columns: a key and a value column.  For most databases the standard VARCHAR column format is appropriate for the key column.  Data stored in the key column generally is US-ASCII keys.
@@ -599,7 +600,6 @@ Setting_Description_pwm.appProperty.overrides=(Troubleshooting only) Specify an
 Setting_Description_pwm.forwardURL=Specify a URL that @PwmAppName@ forwards users to after the users complete any activity which does not require a log out.<br/><br/>You can override this setting for any given user session by adding a <i>forwardURL</i> parameter to any HTTP request. If blank, the system forwards the user to the @PwmAppName@ menu.
 Setting_Description_pwm.homeURL=Specify the URL to redirect the user to upon clicking the home button. If blank, the home button returns the user to the application context URL.
 Setting_Description_pwm.introURL=URL to redirect user to upon accessing the base context of this server (<code>@PwmContextPath@</code>).  The value must start with a slash (<code>/</code>) character, and it will be prepended by the base application path.
-Setting_Description_pwmInstanceName=Specify the name of this application instance. If blank, @PwmAppName@ uses a persistent, randomly generated value. The recommended value is blank.
 Setting_Description_pwm.logoutURL=Specify the URL to redirect user to upon logout. If users access the site through a web authentication gateway, set the Logout URL to the gateway's Logout URL. If you are using a gateway and do not include the proper logout URL here, then users encounter authentication errors, intruder lockouts, and other problems. If things are working properly then the users see the gateway log out screen when logging out.<br/><br/>You can set the Logout URL to any appropriate relative or absolute URL.  At the time the user's browser requests this URL, the local session has already been invalidated.<br/><br/>You can always override this setting for any given user session by adding a <b>logoutURL</b> parameter to any HTTP request during the session.
 Setting_Description_pwm.publishStats.enable=Enable this option to periodically share anonymous statistics of @PwmAppName@. The published statistics are\:<ul><li>Version/Build Information</li><li>Cumulative Statistics</li><li>Which settings are non-default (but not the actual setting values)</li><li>Operating system name and version</li></ul>Enabling this setting helps @PwmAppName@ developers know which features are used most often.
 Setting_Description_pwm.publishStats.siteDescription=This optional value can be included if you want to identify your site when the anonymous statistics are published.   You could use your organizations name or other descriptive value.
@@ -767,6 +767,7 @@ Setting_Label_challenge.showConfirmation=Show Response Confirmation
 Setting_Label_challenge.token.sendMethod=Token Send Method
 Setting_Label_challenge.userAttribute=Response Storage Attribute
 Setting_Label_changePassword.writeAttributes=Post Password Change Actions
+Setting_Label_cluster.storageMode=Cluster Mode
 Setting_Label_command.checkResponses.queryMatch=Check Responses Match
 Setting_Label_db.classname=Database Class
 Setting_Label_db.columnType.key=Database Key Column Type
@@ -1100,7 +1101,6 @@ Setting_Label_pwm.appProperty.overrides=App Property Overrides
 Setting_Label_pwm.forwardURL=Forward URL
 Setting_Label_pwm.homeURL=Home URL
 Setting_Label_pwm.introURL=Intro URL
-Setting_Label_pwmInstanceName=Instance Name
 Setting_Label_pwm.logoutURL=Logout URL
 Setting_Label_pwm.publishStats.enable=Enable Anonymous Statistics Publishing
 Setting_Label_pwm.publishStats.siteDescription=Site Description

webapp/src/main/webapp/WEB-INF/jsp/admin-dashboard.jsp

@@ -272,7 +272,7 @@
                             </td>
                             <td>
                                 <% for (final DataStorageMethod loopMethod : loopService.getStorageMethod()) { %>
-                                <%=loopMethod.toString()%>
+                                <%=loopMethod == null ? "" : loopMethod.toString()%>
                                 <br/>
                                 <% } %>
                             </td>