database thread correctness and pooling

pom.xml

@@ -532,7 +532,7 @@
         <dependency>
             <groupId>org.projectlombok</groupId>
             <artifactId>lombok</artifactId>
-            <version>1.16.14</version>
+            <version>1.16.16</version>
             <scope>provided</scope>
         </dependency>
 
@@ -609,7 +609,7 @@
         <dependency>
             <groupId>com.github.ldapchai</groupId>
             <artifactId>ldapchai</artifactId>
-            <version>0.6.8</version>
+            <version>0.6.9</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
@@ -639,7 +639,7 @@
         <dependency>
             <groupId>org.graylog2</groupId>
             <artifactId>syslog4j</artifactId>
-            <version>0.9.58</version>
+            <version>0.9.60</version>
         </dependency>
         <dependency>
             <groupId>log4j</groupId>
@@ -674,12 +674,12 @@
         <dependency>
             <groupId>org.bouncycastle</groupId>
             <artifactId>bcprov-jdk15on</artifactId>
-            <version>1.56</version>
+            <version>1.57</version>
         </dependency>
         <dependency>
             <groupId>org.bouncycastle</groupId>
             <artifactId>bcpkix-jdk15on</artifactId>
-            <version>1.56</version>
+            <version>1.57</version>
         </dependency>
         <dependency>
             <groupId>javax.xml</groupId>
@@ -739,7 +739,7 @@
         <dependency>
             <groupId>com.github.ben-manes.caffeine</groupId>
             <artifactId>caffeine</artifactId>
-            <version>2.4.0</version>
+            <version>2.5.2</version>
         </dependency>
 
 
@@ -749,17 +749,17 @@
         <dependency>
             <groupId>org.webjars.bower</groupId>
             <artifactId>dojo</artifactId>
-            <version>1.12.1</version>
+            <version>1.12.2</version>
         </dependency>
         <dependency>
             <groupId>org.webjars.bower</groupId>
             <artifactId>dijit</artifactId>
-            <version>1.12.1</version>
+            <version>1.12.2</version>
         </dependency>
         <dependency>
             <groupId>org.webjars.bower</groupId>
             <artifactId>dojox</artifactId>
-            <version>1.12.1</version>
+            <version>1.12.2</version>
         </dependency>
         <dependency>
             <groupId>org.webjars.bower</groupId>

src/main/java/password/pwm/AppProperty.java

@@ -70,6 +70,9 @@ public enum     AppProperty {
     CONFIG_MANAGER_ZIPDEBUG_MAXLOGLINES             ("configManager.zipDebug.maxLogLines"),
     CONFIG_MANAGER_ZIPDEBUG_MAXLOGSECONDS           ("configManager.zipDebug.maxLogSeconds"),
     DB_JDBC_LOAD_STRATEGY                           ("db.jdbcLoadStrategy"),
+    DB_CONNECTIONS_MAX                              ("db.connections.max"),
+    DB_CONNECTIONS_TIMEOUT_MS                       ("db.connections.timeoutMs"),
+    DB_CONNECTIONS_WATCHDOG_FREQUENCY_SECONDS       ("db.connections.watchdogFrequencySeconds"),
     DOWNLOAD_FILENAME_STATISTICS_CSV                ("download.filename.statistics.csv"),
     DOWNLOAD_FILENAME_USER_REPORT_SUMMARY_CSV       ("download.filename.reportSummary.csv"),
     DOWNLOAD_FILENAME_USER_REPORT_RECORDS_CSV       ("download.filename.reportRecords.csv"),

src/main/java/password/pwm/PwmAboutProperty.java

@@ -25,7 +25,7 @@ package password.pwm;
 import password.pwm.config.PwmSetting;
 import password.pwm.i18n.Display;
 import password.pwm.util.LocaleHelper;
-import password.pwm.util.db.DatabaseAccessor;
+import password.pwm.util.db.DatabaseService;
 import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.logging.PwmLogger;
@@ -200,7 +200,7 @@ public enum PwmAboutProperty {
 
         { // database info
             try {
-                final DatabaseAccessor databaseAccessor = pwmApplication.getDatabaseAccessor();
+                final DatabaseService databaseAccessor = pwmApplication.getDatabaseService();
                 if (databaseAccessor != null) {
                     final Map<PwmAboutProperty,String> debugData = databaseAccessor.getConnectionDebugProperties();
                     aboutMap.putAll(debugData);

src/main/java/password/pwm/PwmApplication.java

@@ -61,7 +61,7 @@ import password.pwm.util.PasswordData;
 import password.pwm.util.VersionChecker;
 import password.pwm.util.cli.commands.ExportHttpsTomcatConfigCommand;
 import password.pwm.util.db.DatabaseAccessor;
-import password.pwm.util.db.DatabaseAccessorImpl;
+import password.pwm.util.db.DatabaseService;
 import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
@@ -244,11 +244,11 @@ public class PwmApplication {
         this.localDBLogger = PwmLogManager.initializeLocalDBLogger(this);
 
         // log the loaded configuration
-        LOGGER.info("configuration load completed");
+        LOGGER.debug("configuration load completed");
 
         // read the pwm servlet instance id
         instanceID = fetchInstanceID(localDB, this);
-        LOGGER.info("using '" + getInstanceID() + "' for instance's ID (instanceID)");
+        LOGGER.debug("using '" + getInstanceID() + "' for instance's ID (instanceID)");
 
         // read the pwm installation date
         installTime = fetchInstallDate(startupTime);
@@ -531,11 +531,19 @@ public class PwmApplication {
         return pwmEnvironment.getApplicationMode();
     }
 
-    public synchronized DatabaseAccessor getDatabaseAccessor()
+    public DatabaseAccessor getDatabaseAccessor()
+
+            throws PwmUnrecoverableException
     {
-        return (DatabaseAccessorImpl)pwmServiceManager.getService(DatabaseAccessorImpl.class);
+        return getDatabaseService().getAccessor();
+    }
+
+    public DatabaseService getDatabaseService() {
+        return (DatabaseService)pwmServiceManager.getService(DatabaseService.class);
     }
 
+
+
     private Instant fetchInstallDate(final Instant startupTime) {
         if (localDB != null) {
             try {

src/main/java/password/pwm/bean/PasswordStatus.java

@@ -31,11 +31,10 @@ import java.io.Serializable;
 @Getter
 @Builder
 public class PasswordStatus implements Serializable {
-
-    private boolean expired = false;
-    private boolean preExpired = false;
-    private boolean violatesPolicy = false;
-    private boolean warnPeriod = false;
+    private final boolean expired;
+    private final boolean preExpired;
+    private final boolean violatesPolicy;
+    private final boolean warnPeriod;
 
     @Override
     public String toString() {

src/main/java/password/pwm/health/DatabaseStatusChecker.java

@@ -26,7 +26,7 @@ import password.pwm.PwmApplication;
 import password.pwm.PwmEnvironment;
 import password.pwm.config.Configuration;
 import password.pwm.error.PwmException;
-import password.pwm.util.db.DatabaseAccessorImpl;
+import password.pwm.util.db.DatabaseAccessor;
 import password.pwm.util.db.DatabaseTable;
 import password.pwm.util.logging.PwmLogger;
 
@@ -51,19 +51,22 @@ public class DatabaseStatusChecker implements HealthChecker {
         if (!config.hasDbConfigured()) {
             return Collections.singletonList(new HealthRecord(HealthStatus.INFO,HealthTopic.Database,"Database not configured"));
         }
-        final DatabaseAccessorImpl impl = new DatabaseAccessorImpl();
-        try {
 
+        PwmApplication runtimeInstance = null;
+        try {
             final PwmEnvironment runtimeEnvironment = pwmApplication.getPwmEnvironment().makeRuntimeInstance(config);
-            final PwmApplication runtimeInstance = new PwmApplication(runtimeEnvironment);
-            impl.init(runtimeInstance);
-            impl.get(DatabaseTable.PWM_META, "test");
-            return impl.healthCheck();
+            runtimeInstance = new PwmApplication(runtimeEnvironment);
+            final DatabaseAccessor accessor = runtimeInstance.getDatabaseService().getAccessor();
+            accessor.get(DatabaseTable.PWM_META, "test");
+            return runtimeInstance.getDatabaseService().healthCheck();
         } catch (PwmException e) {
             LOGGER.error("error during healthcheck: " + e.getMessage());
-            return impl.healthCheck();
+            e.printStackTrace();
+            return runtimeInstance.getDatabaseService().healthCheck();
         } finally {
-            impl.close();
+            if (runtimeInstance != null) {
+                runtimeInstance.shutdown();
+            }
         }
     }
 }

src/main/java/password/pwm/http/filter/AuthenticationFilter.java

@@ -121,8 +121,8 @@ public class AuthenticationFilter extends AbstractPwmFilter {
                 this.processUnAuthenticatedSession(pwmRequest, chain);
             }
         } catch (PwmUnrecoverableException e) {
-            LOGGER.error(e.toString());
-            throw new ServletException(e.toString());
+            LOGGER.error(e.getErrorInformation());
+            pwmRequest.respondWithError(e.getErrorInformation(), true);
         }
     }
 

src/main/java/password/pwm/ldap/LdapConnectionService.java

@@ -34,6 +34,7 @@ import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.PwmService;
+import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.logging.PwmLogger;
 
@@ -43,7 +44,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
 
 public class LdapConnectionService implements PwmService {
     private static final PwmLogger LOGGER = PwmLogger.forClass(LdapConnectionService.class);
@@ -52,8 +52,7 @@ public class LdapConnectionService implements PwmService {
     private final Map<LdapProfile, ErrorInformation> lastLdapErrors = new ConcurrentHashMap<>();
     private PwmApplication pwmApplication;
     private STATUS status = STATUS.NEW;
-    private int connectionsPerProfile = 1;
-    private final AtomicInteger slotIncrementer = new AtomicInteger(0);
+    private AtomicLoopIntIncrementer slotIncrementer;
     private final ThreadLocal<ChaiProvider> threadLocalProvider = new ThreadLocal<>();
 
     public STATUS status()
@@ -69,8 +68,9 @@ public class LdapConnectionService implements PwmService {
         // read the lastLoginTime
         this.lastLdapErrors.putAll(readLastLdapFailure(pwmApplication));
 
-        connectionsPerProfile = maxSlotsPerProfile(pwmApplication);
+        final int connectionsPerProfile = maxSlotsPerProfile(pwmApplication);
         LOGGER.trace("allocating " + connectionsPerProfile + " ldap proxy connections per profile");
+        slotIncrementer = new AtomicLoopIntIncrementer(connectionsPerProfile);
 
         for (final LdapProfile ldapProfile: pwmApplication.getConfig().getLdapProfiles().values()) {
             proxyChaiProviders.put(ldapProfile, new ConcurrentHashMap<>());
@@ -122,7 +122,7 @@ public class LdapConnectionService implements PwmService {
             return threadLocalProvider.get();
         }
 
-        final int slot = nextSlot();
+        final int slot = slotIncrementer.next();
 
         final ChaiProvider proxyChaiProvider = proxyChaiProviders.get(identifier).get(slot);
 
@@ -205,16 +205,6 @@ public class LdapConnectionService implements PwmService {
         return Collections.emptyMap();
     }
 
-    private int nextSlot() {
-        return slotIncrementer.getAndUpdate(operand -> {
-            operand++;
-            if (operand >= connectionsPerProfile) {
-                operand = 0;
-            }
-            return operand;
-        });
-    }
-
     private int maxSlotsPerProfile(final PwmApplication pwmApplication) {
         final int maxConnections = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.LDAP_PROXY_MAX_CONNECTIONS));
         final int perProfile = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.LDAP_PROXY_CONNECTION_PER_PROFILE));

src/main/java/password/pwm/ldap/UserInfoBean.java

@@ -46,39 +46,46 @@ import java.util.Map;
 public class UserInfoBean implements UserInfo {
 // ------------------------------ FIELDS ------------------------------
 
-    private UserIdentity userIdentity;
-    private String username;
-    private String userEmailAddress;
-    private String userSmsNumber;
-    private String userGuid;
+    private final UserIdentity userIdentity;
+    private final String username;
+    private final String userEmailAddress;
+    private final String userSmsNumber;
+    private final String userGuid;
 
     /**
      * A listing of all readable attributes on the ldap user object
      */
-    private Map<String,String> cachedPasswordRuleAttributes = Collections.emptyMap();
+    @Builder.Default
+    private final Map<String,String> cachedPasswordRuleAttributes = Collections.emptyMap();
 
-    private Map<String,String> cachedAttributeValues = Collections.emptyMap();
-    
-    private Map<ProfileType,String> profileIDs = new HashMap<>();
+    @Builder.Default
+    private final Map<String,String> cachedAttributeValues = Collections.emptyMap();
 
-    private PasswordStatus passwordStatus = PasswordStatus.builder().build();
+    @Builder.Default
+    private final Map<ProfileType,String> profileIDs = new HashMap<>();
 
-    private PwmPasswordPolicy passwordPolicy = PwmPasswordPolicy.defaultPolicy();
-    private ChallengeProfile challengeProfile = null;
-    private ResponseInfoBean responseInfoBean = null;
-    private OTPUserRecord otpUserRecord = null;
+    @Builder.Default
+    private final PasswordStatus passwordStatus = PasswordStatus.builder().build();
 
-    private Instant passwordExpirationTime;
-    private Instant passwordLastModifiedTime;
-    private Instant lastLdapLoginTime;
-    private Instant accountExpirationTime;
+    @Builder.Default
+    private final PwmPasswordPolicy passwordPolicy = PwmPasswordPolicy.defaultPolicy();
 
-    private boolean requiresNewPassword;
-    private boolean requiresResponseConfig;
-    private boolean requiresOtpConfig;
-    private boolean requiresUpdateProfile;
+    private final ChallengeProfile challengeProfile;
+    private final ResponseInfoBean responseInfoBean;
+    private final OTPUserRecord otpUserRecord;
 
-    private Map<String,String> attributes;
+    private final Instant passwordExpirationTime;
+    private final Instant passwordLastModifiedTime;
+    private final Instant lastLdapLoginTime;
+    private final Instant accountExpirationTime;
+
+    private final boolean requiresNewPassword;
+    private final boolean requiresResponseConfig;
+    private final boolean requiresOtpConfig;
+    private final boolean requiresUpdateProfile;
+
+    @Builder.Default
+    private Map<String,String> attributes = Collections.emptyMap();
 
     @Override
     public String readStringAttribute(final String attribute) throws PwmUnrecoverableException

src/main/java/password/pwm/ldap/search/SearchConfiguration.java

@@ -43,8 +43,14 @@ public class SearchConfiguration implements Serializable {
     private Map<FormConfiguration, String> formValues;
     private transient ChaiProvider chaiProvider;
     private long searchTimeout;
+
+    @Builder.Default
     private boolean enableValueEscaping = true;
+
+    @Builder.Default
     private boolean enableContextValidation = true;
+
+    @Builder.Default
     private boolean enableSplitWhitespace = false;
 
     void validate() {

src/main/java/password/pwm/svc/PwmServiceManager.java

@@ -45,7 +45,6 @@ import password.pwm.svc.wordlist.SeedlistManager;
 import password.pwm.svc.wordlist.SharedHistoryManager;
 import password.pwm.svc.wordlist.WordlistManager;
 import password.pwm.util.VersionChecker;
-import password.pwm.util.db.DatabaseAccessorImpl;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.operations.CrService;
@@ -73,7 +72,7 @@ public class PwmServiceManager {
     public enum PwmServiceClassEnum {
         SecureService(          SecureService.class,             true),
         LdapConnectionService(  LdapConnectionService.class,     true),
-        DatabaseAccessorImpl(   DatabaseAccessorImpl.class,      true),
+        DatabaseService(        password.pwm.util.db.DatabaseService.class,           true),
         SharedHistoryManager(   SharedHistoryManager.class,      false),
         AuditService(           AuditService.class,              false),
         StatisticsManager(      StatisticsManager.class,         false),

src/main/java/password/pwm/svc/event/DatabaseUserHistory.java

@@ -25,13 +25,13 @@ package password.pwm.svc.event;
 import com.novell.ldapchai.exception.ChaiUnavailableException;
 import password.pwm.PwmApplication;
 import password.pwm.bean.UserIdentity;
-import password.pwm.ldap.UserInfo;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.ldap.LdapOperationsHelper;
-import password.pwm.util.db.DatabaseAccessor;
+import password.pwm.ldap.UserInfo;
 import password.pwm.util.db.DatabaseException;
+import password.pwm.util.db.DatabaseService;
 import password.pwm.util.db.DatabaseTable;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.logging.PwmLogger;
@@ -46,11 +46,11 @@ class DatabaseUserHistory implements UserHistoryStore {
     private static final DatabaseTable TABLE = DatabaseTable.USER_AUDIT;
 
     final PwmApplication pwmApplication;
-    final DatabaseAccessor databaseAccessor;
+    final DatabaseService databaseService;
 
     DatabaseUserHistory(final PwmApplication pwmApplication) {
         this.pwmApplication = pwmApplication;
-        this.databaseAccessor = pwmApplication.getDatabaseAccessor();
+        this.databaseService = pwmApplication.getDatabaseService();
     }
 
     @Override
@@ -92,20 +92,22 @@ class DatabaseUserHistory implements UserHistoryStore {
         }
     }
 
-    private StoredHistory readStoredHistory(final String guid) throws DatabaseException {
-        final String str = this.databaseAccessor.get(TABLE, guid);
+    private StoredHistory readStoredHistory(final String guid) throws DatabaseException, PwmUnrecoverableException
+    {
+        final String str = this.databaseService.getAccessor().get(TABLE, guid);
         if (str == null || str.length() < 1) {
             return new StoredHistory();
         }
         return JsonUtil.deserialize(str,StoredHistory.class);
     }
 
-    private void writeStoredHistory(final String guid, final StoredHistory storedHistory) throws DatabaseException {
+    private void writeStoredHistory(final String guid, final StoredHistory storedHistory) throws DatabaseException, PwmUnrecoverableException
+    {
         if (storedHistory == null) {
             return;
         }
         final String str = JsonUtil.serialize(storedHistory);
-        databaseAccessor.put(TABLE,guid,str);
+        databaseService.getAccessor().put(TABLE,guid,str);
     }
 
     static class StoredHistory implements Serializable {

src/main/java/password/pwm/svc/intruder/DataStoreRecordStore.java

@@ -25,6 +25,7 @@ package password.pwm.svc.intruder;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmDataStoreException;
 import password.pwm.error.PwmError;
+import password.pwm.error.PwmException;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.svc.PwmService;
@@ -86,7 +87,8 @@ class DataStoreRecordStore implements RecordStore {
     }
 
     @Override
-    public void write(final String key, final IntruderRecord record) throws PwmOperationalException {
+    public void write(final String key, final IntruderRecord record) throws PwmOperationalException, PwmUnrecoverableException
+    {
         final String jsonRecord = JsonUtil.serialize(record);
         try {
             dataStore.put(key, jsonRecord);
@@ -96,7 +98,8 @@ class DataStoreRecordStore implements RecordStore {
     }
 
     @Override
-    public ClosableIterator<IntruderRecord> iterator() throws PwmOperationalException {
+    public ClosableIterator<IntruderRecord> iterator() throws PwmOperationalException, PwmUnrecoverableException
+    {
         try {
             return new RecordIterator(dataStore.iterator());
         } catch (PwmDataStoreException e) {
@@ -160,7 +163,7 @@ class DataStoreRecordStore implements RecordStore {
                 for (final String key : recordsToRemove) {
                     dataStore.remove(key);
                 }
-            } catch (PwmDataStoreException e) {
+            } catch (PwmException e) {
                 LOGGER.error("unable to perform removal of identified stale records: " + e.getMessage());
             }
             recordsRemoved += recordsToRemove.size();

src/main/java/password/pwm/svc/intruder/IntruderManager.java

@@ -27,7 +27,6 @@ import password.pwm.PwmApplication;
 import password.pwm.bean.EmailItemBean;
 import password.pwm.bean.SessionLabel;
 import password.pwm.bean.UserIdentity;
-import password.pwm.ldap.UserInfo;
 import password.pwm.config.Configuration;
 import password.pwm.config.FormConfiguration;
 import password.pwm.config.PwmSetting;
@@ -36,13 +35,13 @@ import password.pwm.config.option.IntruderStorageMethod;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
-import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
 import password.pwm.health.HealthStatus;
 import password.pwm.health.HealthTopic;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmSession;
+import password.pwm.ldap.UserInfo;
 import password.pwm.ldap.UserInfoFactory;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.event.AuditEvent;
@@ -119,7 +118,7 @@ public class IntruderManager implements Serializable, PwmService {
         }
         if (!pwmApplication.getConfig().readSettingAsBoolean(PwmSetting.INTRUDER_ENABLE)) {
             final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,"intruder module not enabled");
-            LOGGER.error(errorInformation.toDebugStr());
+            LOGGER.debug(errorInformation.toDebugStr());
             status = STATUS.CLOSED;
             return;
         }
@@ -141,7 +140,7 @@ public class IntruderManager implements Serializable, PwmService {
                     break;
 
                 case DATABASE:
-                    dataStore = new DatabaseDataStore(pwmApplication.getDatabaseAccessor(), DatabaseTable.INTRUDER);
+                    dataStore = new DatabaseDataStore(pwmApplication.getDatabaseService(), DatabaseTable.INTRUDER);
                     debugMsg = "starting using Remote Database data store";
                     storageMethodUsed = DataStorageMethod.DB;
                     break;
@@ -415,7 +414,7 @@ public class IntruderManager implements Serializable, PwmService {
     }
 
     public List<Map<String,Object>> getRecords(final RecordType recordType, final int maximum)
-            throws PwmOperationalException
+            throws PwmException
     {
         final RecordManager manager = recordManagers.get(recordType);
         final ArrayList<Map<String,Object>> returnList = new ArrayList<>();

src/main/java/password/pwm/svc/intruder/RecordManager.java

@@ -22,7 +22,7 @@
 
 package password.pwm.svc.intruder;
 
-import password.pwm.error.PwmOperationalException;
+import password.pwm.error.PwmException;
 import password.pwm.util.java.ClosableIterator;
 
 public interface RecordManager {
@@ -38,5 +38,5 @@ public interface RecordManager {
 
     IntruderRecord readIntruderRecord( String subject);
 
-    ClosableIterator<IntruderRecord> iterator() throws PwmOperationalException;
+    ClosableIterator<IntruderRecord> iterator() throws PwmException;
 }

src/main/java/password/pwm/svc/intruder/RecordManagerImpl.java

@@ -130,7 +130,7 @@ class RecordManagerImpl implements RecordManager {
     private void writeIntruderRecord(final IntruderRecord intruderRecord) {
         try {
             recordStore.write(makeKey(intruderRecord.getSubject()),intruderRecord);
-        } catch (PwmOperationalException e) {
+        } catch (PwmException e) {
             LOGGER.warn("unexpected error attempting to write intruder record " + JsonUtil.serialize(intruderRecord) + ", error: " + e.getMessage());
         }
     }
@@ -148,7 +148,7 @@ class RecordManagerImpl implements RecordManager {
 
 
     @Override
-    public ClosableIterator<IntruderRecord> iterator() throws PwmOperationalException {
+    public ClosableIterator<IntruderRecord> iterator() throws PwmException {
         return new RecordIterator<>(recordStore.iterator());
     }
 

src/main/java/password/pwm/svc/intruder/RecordStore.java

@@ -31,9 +31,9 @@ import password.pwm.util.localdb.LocalDBException;
 interface RecordStore {
     IntruderRecord read(String key) throws PwmUnrecoverableException;
 
-    void write(String key, IntruderRecord record) throws PwmOperationalException;
+    void write(String key, IntruderRecord record) throws PwmOperationalException, PwmUnrecoverableException;
 
-    ClosableIterator<IntruderRecord> iterator() throws PwmOperationalException;
+    ClosableIterator<IntruderRecord> iterator() throws PwmOperationalException, PwmUnrecoverableException;
 
     void cleanup(TimeDuration maxRecordAge) throws LocalDBException;
 }

src/main/java/password/pwm/svc/token/DataStoreTokenMachine.java

@@ -139,7 +139,8 @@ public class DataStoreTokenMachine implements TokenMachine {
         dataStore.remove(storedHash);
     }
 
-    public int size() throws PwmOperationalException {
+    public int size() throws PwmOperationalException, PwmUnrecoverableException
+    {
         return dataStore.size();
     }
 

src/main/java/password/pwm/svc/token/TokenService.java

@@ -169,7 +169,7 @@ public class TokenService implements PwmService {
                 }
 
                 case STORE_DB: {
-                    final DataStore dataStore = new DatabaseDataStore(pwmApplication.getDatabaseAccessor(), DatabaseTable.TOKENS);
+                    final DataStore dataStore = new DatabaseDataStore(pwmApplication.getDatabaseService(), DatabaseTable.TOKENS);
                     tokenMachine = new DataStoreTokenMachine(pwmApplication, this, dataStore);
                     usedStorageMethod = DataStorageMethod.DB;
                     break;

src/main/java/password/pwm/util/DataStore.java

@@ -23,6 +23,7 @@
 package password.pwm.util;
 
 import password.pwm.error.PwmDataStoreException;
+import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.java.ClosableIterator;
 
 public interface DataStore {
@@ -34,22 +35,25 @@ public interface DataStore {
             throws PwmDataStoreException;
 
     boolean contains(String key)
-            throws PwmDataStoreException;
+            throws PwmDataStoreException, PwmUnrecoverableException;
 
     String get(String key)
-            throws PwmDataStoreException;
+            throws PwmDataStoreException, PwmUnrecoverableException;
 
     ClosableIterator<String> iterator()
-            throws PwmDataStoreException;
+            throws PwmDataStoreException, PwmUnrecoverableException;
 
     Status status();
 
     boolean put(String key, String value)
-            throws PwmDataStoreException;
+            throws PwmDataStoreException, PwmUnrecoverableException;
 
-    boolean remove(String key)
-            throws PwmDataStoreException;
+    boolean putIfAbsent(String key, String value)
+            throws PwmDataStoreException, PwmUnrecoverableException;
+
+    void remove(String key)
+            throws PwmDataStoreException, PwmUnrecoverableException;
 
     int size()
-            throws PwmDataStoreException;
+            throws PwmDataStoreException, PwmUnrecoverableException;
 }

src/main/java/password/pwm/util/DataStoreFactory.java

@@ -23,15 +23,17 @@
 package password.pwm.util;
 
 import password.pwm.PwmApplication;
+import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.db.DatabaseDataStore;
 import password.pwm.util.db.DatabaseTable;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBDataStore;
 
 public abstract class DataStoreFactory {
-    public static DataStore autoDbOrLocalDBstore(final PwmApplication pwmApplication, final DatabaseTable table, final LocalDB.DB db) {
+    public static DataStore autoDbOrLocalDBstore(final PwmApplication pwmApplication, final DatabaseTable table, final LocalDB.DB db) throws PwmUnrecoverableException
+    {
         if (pwmApplication.getConfig().hasDbConfigured()) {
-            return new DatabaseDataStore(pwmApplication.getDatabaseAccessor(), table);
+            return new DatabaseDataStore(pwmApplication.getDatabaseService(), table);
         }
 
         return new LocalDBDataStore(pwmApplication.getLocalDB(), db);

src/main/java/password/pwm/util/db/DBConfiguration.java

@@ -22,6 +22,9 @@
 
 package password.pwm.util.db;
 
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
 import password.pwm.AppProperty;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
@@ -35,6 +38,8 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
+@Getter
+@AllArgsConstructor(access = AccessLevel.PRIVATE)
 public class DBConfiguration implements Serializable {
     private final String driverClassname;
     private final String connectionString;
@@ -44,60 +49,8 @@ public class DBConfiguration implements Serializable {
     private final String columnTypeValue;
     private final byte[] jdbcDriver;
     private final List<JDBCDriverLoader.ClassLoaderStrategy> classLoaderStrategies;
-
-    private DBConfiguration(
-            final String driverClassname,
-            final String connectionString,
-            final String username,
-            final PasswordData password,
-            final String columnTypeKey,
-            final String columnTypeValue,
-            final byte[] jdbcDriver,
-            final List<JDBCDriverLoader.ClassLoaderStrategy> classLoaderStrategies
-    ) {
-        this.driverClassname = driverClassname;
-        this.connectionString = connectionString;
-        this.username = username;
-        this.password = password;
-        this.columnTypeKey = columnTypeKey;
-        this.columnTypeValue = columnTypeValue;
-        this.jdbcDriver = jdbcDriver;
-        this.classLoaderStrategies = classLoaderStrategies;
-    }
-
-    public String getDriverClassname() {
-        return driverClassname;
-    }
-
-    public String getConnectionString() {
-        return connectionString;
-    }
-
-    public String getUsername() {
-        return username;
-    }
-
-    public PasswordData getPassword() {
-        return password;
-    }
-
-    public String getColumnTypeKey() {
-        return columnTypeKey;
-    }
-
-    public String getColumnTypeValue() {
-        return columnTypeValue;
-    }
-
-
-    public byte[] getJdbcDriver()
-    {
-        return jdbcDriver;
-    }
-
-    public List<JDBCDriverLoader.ClassLoaderStrategy> getClassLoaderStrategies() {
-        return classLoaderStrategies;
-    }
+    private final int maxConnections;
+    private final int connectionTimeout;
 
     public boolean isEnabled() {
         return
@@ -125,6 +78,8 @@ public class DBConfiguration implements Serializable {
                  Arrays.asList(strategyList.split(","))
          );
 
+         final int maxConnections = Integer.parseInt(config.readAppProperty(AppProperty.DB_CONNECTIONS_MAX));
+         final int connectionTimeout = Integer.parseInt(config.readAppProperty(AppProperty.DB_CONNECTIONS_TIMEOUT_MS));
 
          return new DBConfiguration(
                  config.readSettingAsString(PwmSetting.DATABASE_CLASS),
@@ -134,8 +89,9 @@ public class DBConfiguration implements Serializable {
                  config.readSettingAsString(PwmSetting.DATABASE_COLUMN_TYPE_KEY),
                  config.readSettingAsString(PwmSetting.DATABASE_COLUMN_TYPE_VALUE),
                  jdbcDriverBytes,
-                 strategies
+                 strategies,
+                 maxConnections,
+                 connectionTimeout
          );
-
      }
 }

src/main/java/password/pwm/util/db/DatabaseAccessor.java

@@ -22,21 +22,16 @@
 
 package password.pwm.util.db;
 
-import password.pwm.PwmAboutProperty;
 import password.pwm.util.java.ClosableIterator;
 
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
-import java.util.Map;
 
 public interface DatabaseAccessor {
-    Map<PwmAboutProperty,String> getConnectionDebugProperties();
-
     /**
      * Indicates if the method is actually performing an DB operation.
      */
     @Retention(RetentionPolicy.RUNTIME)
-    public
     @interface DbOperation {
     }
 
@@ -44,7 +39,6 @@ public interface DatabaseAccessor {
      * Indicates if the method may cause a modification of the database.
      */
     @Retention(RetentionPolicy.RUNTIME)
-    public
     @interface DbModifyOperation {
     }
 
@@ -58,6 +52,15 @@ public interface DatabaseAccessor {
     )
             throws DatabaseException;
 
+    @DbOperation
+    @DbModifyOperation
+    boolean putIfAbsent(
+            DatabaseTable table,
+            String key,
+            String value
+    )
+            throws DatabaseException;
+
     @DbOperation
     boolean contains(
             DatabaseTable table,
@@ -77,7 +80,7 @@ public interface DatabaseAccessor {
 
     @DbOperation
     @DbModifyOperation
-    boolean remove(
+    void remove(
             DatabaseTable table,
             String key
     )
@@ -86,6 +89,4 @@ public interface DatabaseAccessor {
     @DbOperation
     int size(DatabaseTable table) throws
             DatabaseException;
-
-    boolean isMasterServer();
 }

src/main/java/password/pwm/util/db/DatabaseAccessorImpl.java

@@ -22,452 +22,118 @@
 
 package password.pwm.util.db;
 
-import password.pwm.PwmAboutProperty;
-import password.pwm.PwmApplication;
-import password.pwm.PwmApplicationMode;
-import password.pwm.PwmConstants;
-import password.pwm.config.Configuration;
-import password.pwm.config.PwmSetting;
-import password.pwm.config.option.DataStorageMethod;
-import password.pwm.error.ErrorInformation;
-import password.pwm.error.PwmError;
-import password.pwm.error.PwmException;
-import password.pwm.health.HealthRecord;
-import password.pwm.health.HealthStatus;
-import password.pwm.health.HealthTopic;
-import password.pwm.svc.PwmService;
-import password.pwm.svc.stats.Statistic;
-import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.java.ClosableIterator;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
-import java.io.File;
-import java.lang.reflect.Method;
 import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.Driver;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.sql.Statement;
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
 import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantLock;
 
 /**
  * @author Jason D. Rivard
  */
-public class DatabaseAccessorImpl implements PwmService, DatabaseAccessor {
-// ------------------------------ FIELDS ------------------------------
+class DatabaseAccessorImpl implements DatabaseAccessor {
 
     private static final PwmLogger LOGGER = PwmLogger.forClass(DatabaseAccessorImpl.class, true);
-    private static final String KEY_COLUMN = "id";
-    private static final String VALUE_COLUMN = "value";
 
-    private static final int KEY_COLUMN_LENGTH = PwmConstants.DATABASE_ACCESSOR_KEY_LENGTH;
+    private final Connection connection;
+    private final DatabaseService databaseService;
+    private final DBConfiguration dbConfiguration;
 
-    private static final String KEY_TEST = "write-test-key";
-    private static final String KEY_ENGINE_START_PREFIX = "engine-start-";
+    private final boolean traceLogEnabled;
 
-    private DBConfiguration dbConfiguration;
-    private Driver driver;
-    private String instanceID;
-    private boolean traceLogging;
-    private volatile Connection connection;
-    private volatile PwmService.STATUS status = PwmService.STATUS.NEW;
-    private ErrorInformation lastError;
-    private PwmApplication pwmApplication;
+    private final ReentrantLock LOCK = new ReentrantLock();
 
-    private JDBCDriverLoader.DriverLoader jdbcDriverLoader;
-
-    private ExecutorService masterStatusService;
-    private final AtomicBoolean masterStatus = new AtomicBoolean(false);
-
-// --------------------------- CONSTRUCTORS ---------------------------
-
-    public DatabaseAccessorImpl()
+    DatabaseAccessorImpl(
+            final DatabaseService databaseService,
+            final DBConfiguration dbConfiguration,
+            final Connection connection,
+            final boolean traceLogEnabled
+    )
     {
+        this.connection = connection;
+        this.dbConfiguration = dbConfiguration;
+        this.traceLogEnabled = traceLogEnabled;
+        this.databaseService = databaseService;
     }
 
-// ------------------------ INTERFACE METHODS ------------------------
-
-
-// --------------------- Interface PwmService ---------------------
-
-    public STATUS status() {
-        return status;
-    }
 
-    public void init(final PwmApplication pwmApplication) throws PwmException {
-        this.pwmApplication = pwmApplication;
-        final Configuration config = pwmApplication.getConfig();
-        init(config);
-    }
-
-    public void close()
+    private void processSqlException(
+            final DatabaseUtil.DebugInfo debugInfo,
+            final SQLException e
+    )
+            throws DatabaseException
     {
-        status = PwmService.STATUS.CLOSED;
-        if (connection != null) {
-            try {
-                connection.close();
-            } catch (Exception e) {
-                LOGGER.debug("error while closing DB: " + e.getMessage());
-            }
-        }
-
-        try {
-            driver = null;
-        } catch (Exception e) {
-            LOGGER.debug("error while de-registering driver: " + e.getMessage());
-        }
-
-        connection = null;
-
-        if (jdbcDriverLoader != null) {
-            jdbcDriverLoader.unloadDriver();
-            jdbcDriverLoader = null;
-        }
-    }
-
-    private void init(final Configuration config) throws PwmException {
-
-        this.dbConfiguration = DBConfiguration.fromConfiguration(config);
-        this.instanceID = pwmApplication == null ? null : pwmApplication.getInstanceID();
-        this.traceLogging = config.readSettingAsBoolean(PwmSetting.DATABASE_DEBUG_TRACE);
-
-        if (!dbConfiguration.isEnabled()) {
-            status = PwmService.STATUS.CLOSED;
-            LOGGER.debug("skipping database connection open, no connection parameters configured");
-        }
-
-        masterStatusService = JavaHelper.makeSingleThreadExecutorService(pwmApplication, DatabaseAccessorImpl.class);
-    }
-
-    public List<HealthRecord> healthCheck() {
-        if (status == PwmService.STATUS.CLOSED) {
-            return Collections.emptyList();
-        }
-
-        final List<HealthRecord> returnRecords = new ArrayList<>();
-
-        try {
-            preOperationCheck();
-        } catch (DatabaseException e) {
-            lastError = e.getErrorInformation();
-            returnRecords.add(new HealthRecord(HealthStatus.WARN, HealthTopic.Database, "Database server is not available: " + e.getErrorInformation().toDebugStr()));
-            return returnRecords;
-        }
-
-        try {
-            final Map<String,String> tempMap = new HashMap<>();
-            tempMap.put("instance",instanceID);
-            tempMap.put("date",(new java.util.Date()).toString());
-            this.put(DatabaseTable.PWM_META, DatabaseAccessorImpl.KEY_TEST, JsonUtil.serializeMap(tempMap));
-        } catch (PwmException e) {
-            returnRecords.add(new HealthRecord(HealthStatus.WARN, HealthTopic.Database, "Error writing to database: " + e.getErrorInformation().toDebugStr()));
-            return returnRecords;
-        }
-
-        if (lastError != null) {
-            final TimeDuration errorAge = TimeDuration.fromCurrent(lastError.getDate());
-
-            if (errorAge.isShorterThan(TimeDuration.HOUR)) {
-                final String msg = "Database server was recently unavailable ("
-                        + errorAge.asLongString(PwmConstants.DEFAULT_LOCALE)
-                        + " ago at " + lastError.getDate().toString()+ "): " + lastError.toDebugStr();
-                returnRecords.add(new HealthRecord(HealthStatus.CAUTION, HealthTopic.Database, msg));
-            }
-        }
-
-        if (returnRecords.isEmpty()) {
-            returnRecords.add(new HealthRecord(HealthStatus.GOOD, HealthTopic.Database, "Database connection to " + this.dbConfiguration.getConnectionString() + " okay"));
-        }
-
-        return returnRecords;
+        final DatabaseException databaseException = DatabaseUtil.convertSqlException(debugInfo, e);
+        databaseService.setLastError(databaseException.getErrorInformation());
+        throw databaseException;
     }
 
-// -------------------------- OTHER METHODS --------------------------
 
-    private synchronized void init()
+    @Override
+    public boolean put(
+            final DatabaseTable table,
+            final String key,
+            final String value
+    )
             throws DatabaseException
     {
-        status = PwmService.STATUS.OPENING;
-        final Instant startTime = Instant.now();
-        LOGGER.debug("opening connection to database " + this.dbConfiguration.getConnectionString());
+        final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create("put", table, key, value);
 
-        connection = openDB(dbConfiguration);
-        for (final DatabaseTable table : DatabaseTable.values()) {
-            initTable(connection, table, dbConfiguration);
-        }
-
-        status = PwmService.STATUS.OPEN;
-
-        try {
-            put(DatabaseTable.PWM_META, KEY_ENGINE_START_PREFIX + instanceID, JavaHelper.toIsoDate(new java.util.Date()));
-        } catch (DatabaseException e) {
-            final String errorMsg = "error writing engine start time value: " + e.getMessage();
-            throw new DatabaseException(new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,errorMsg));
-        }
-
-        LOGGER.debug("successfully connected to remote database (" + TimeDuration.fromCurrent(startTime).asCompactString() + ")");
-    }
-
-    private Connection openDB(final DBConfiguration dbConfiguration) throws DatabaseException {
-        final String connectionURL = dbConfiguration.getConnectionString();
-
-        final JDBCDriverLoader.DriverWrapper wrapper = JDBCDriverLoader.loadDriver(pwmApplication, dbConfiguration);
-        driver = wrapper.getDriver();
-        jdbcDriverLoader = wrapper.getDriverLoader();
-
-        try {
-            LOGGER.debug("initiating connecting to database " + connectionURL);
-            final Properties connectionProperties = new Properties();
-            if (dbConfiguration.getUsername() != null && !dbConfiguration.getUsername().isEmpty()) {
-                connectionProperties.setProperty("user", dbConfiguration.getUsername());
-            }
-            if (dbConfiguration.getPassword() != null) {
-                connectionProperties.setProperty("password", dbConfiguration.getPassword().getStringValue());
-            }
-            final Connection connection = driver.connect(connectionURL, connectionProperties);
-
-
-            final Map<PwmAboutProperty,String> debugProps = getConnectionDebugProperties(connection);
-            LOGGER.debug("connected to database " + connectionURL + ", properties: " + JsonUtil.serializeMap(debugProps));
-
-            connection.setAutoCommit(true);
-            return connection;
-        } catch (Throwable e) {
-            final String errorMsg = "error connecting to database: " + JavaHelper.readHostileExceptionMessage(e);
-            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,errorMsg);
-            LOGGER.error(errorInformation);
-            throw new DatabaseException(errorInformation);
-        }
-    }
-
-    private static void initTable(final Connection connection, final DatabaseTable table, final DBConfiguration dbConfiguration) throws DatabaseException {
-        boolean tableExists = false;
-        try {
-            checkIfTableExists(connection, table);
-            LOGGER.trace("table " + table + " appears to exist");
-            tableExists = true;
-        } catch (SQLException e) { // assume error was due to table missing;
-            LOGGER.trace("error while checking for table: " + e.getMessage() + ", assuming due to table non-existence");
-        }
-
-        if (!tableExists) {
-            createTable(connection, table, dbConfiguration);
-        }
-    }
-
-    private static void createTable(final Connection connection, final DatabaseTable table, final DBConfiguration dbConfiguration) throws DatabaseException {
-        {
-            final StringBuilder sqlString = new StringBuilder();
-            sqlString.append("CREATE table ").append(table.toString()).append(" (").append("\n");
-            sqlString.append("  " + KEY_COLUMN + " ").append(dbConfiguration.getColumnTypeKey()).append("(").append(
-                    KEY_COLUMN_LENGTH).append(") NOT NULL PRIMARY KEY,").append("\n");
-            sqlString.append("  " + VALUE_COLUMN + " ").append(dbConfiguration.getColumnTypeValue()).append(" ");
-            sqlString.append("\n");
-            sqlString.append(")").append("\n");
-
-            LOGGER.trace("attempting to execute the following sql statement:\n " + sqlString.toString());
-
-            Statement statement = null;
+        return execute(debugInfo, () -> {
+            boolean exists = false;
             try {
-                statement = connection.createStatement();
-                statement.execute(sqlString.toString());
-                LOGGER.debug("created table " + table.toString());
-            } catch (SQLException ex) {
-                final String errorMsg = "error creating new table " + table.toString() + ": " + ex.getMessage();
-                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE, errorMsg);
-                throw new DatabaseException(errorInformation);
-            } finally {
-                close(statement);
+                exists = containsImpl(table, key);
+            } catch (SQLException e) {
+                processSqlException(debugInfo, e);
             }
-        }
-
-        {
-            final String indexName = table.toString() + "_IDX";
-            final StringBuilder sqlString = new StringBuilder();
-            sqlString.append("CREATE index ").append(indexName);
-            sqlString.append(" ON ").append(table.toString());
-            sqlString.append(" (").append(KEY_COLUMN).append(")");
-            Statement statement = null;
 
-            LOGGER.trace("attempting to execute the following sql statement:\n " + sqlString.toString());
-
-            try {
-                statement = connection.createStatement();
-                statement.execute(sqlString.toString());
-                LOGGER.debug("created index " + indexName);
-            } catch (SQLException ex) {
-                final String errorMsg = "error creating new index " + indexName + ": " + ex.getMessage();
-                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE, errorMsg);
-                throw new DatabaseException(errorInformation);
-            } finally {
-                close(statement);
+            final String sqlText;
+            if (!exists) {
+                sqlText = "INSERT INTO " + table.toString() + "(" + DatabaseService.KEY_COLUMN + ", " + DatabaseService.VALUE_COLUMN + ") VALUES(?,?)";
+            } else {
+                sqlText = "UPDATE " + table.toString() + " SET " + DatabaseService.VALUE_COLUMN + "=? WHERE " + DatabaseService.KEY_COLUMN + "=?";
             }
-        }
-    }
 
-    private static void checkIfTableExists(final Connection connection, final DatabaseTable table) throws SQLException {
-        final StringBuilder sb = new StringBuilder();
-        sb.append("SELECT * FROM  ").append(table.toString()).append(" WHERE " + KEY_COLUMN + " = '0'");
-        Statement statement = null;
-        ResultSet resultSet = null;
-        try {
-            statement = connection.createStatement();
-            resultSet = statement.executeQuery(sb.toString());
-        } finally {
-            close(statement);
-            close(resultSet);
-        }
+            executeUpdate(sqlText, debugInfo, key, value);
+            return !exists;
+        });
     }
 
+
     @Override
-    public boolean put(
+    public boolean putIfAbsent(
             final DatabaseTable table,
             final String key,
             final String value
     )
-            throws DatabaseException {
-
-        preOperationCheck();
-        if (traceLogging) {
-            LOGGER.trace("attempting put operation for table=" + table + ", key=" + key);
-        }
-        if (!contains(table, key)) {
-            final String sqlText = "INSERT INTO " + table.toString() + "(" + KEY_COLUMN + ", " + VALUE_COLUMN + ") VALUES(?,?)";
-            PreparedStatement statement = null;
+            throws DatabaseException
+    {
+        final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create("putIfAbsent", table, key, value);
 
+        return execute(debugInfo, () -> {
+            boolean valueExists = false;
             try {
-                statement = connection.prepareStatement(sqlText);
-                statement.setString(1, key);
-                statement.setString(2, value);
-                statement.executeUpdate();
-            } catch (SQLException e) {
-                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,"put operation failed: " + e.getMessage());
-                lastError = errorInformation;
-                throw new DatabaseException(errorInformation);
-            } finally {
-                close(statement);
+                valueExists = containsImpl(table, key);
+            } catch (final SQLException e) {
+                processSqlException(debugInfo, e);
             }
-            return false;
-        }
-
-        final String sqlText = "UPDATE " + table.toString() + " SET " + VALUE_COLUMN + "=? WHERE " + KEY_COLUMN + "=?";
-        PreparedStatement statement = null;
-
-        try {
-            statement = connection.prepareStatement(sqlText);
-            statement.setString(1, value);
-            statement.setString(2, key);
-            statement.executeUpdate();
-        } catch (SQLException e) {
-            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,"put operation failed: " + e.getMessage());
-            lastError = errorInformation;
-            throw new DatabaseException(errorInformation);
-        } finally {
-            close(statement);
-        }
-
-        if (traceLogging) {
-            final Map<String,Object> debugOutput = new LinkedHashMap<>();
-            debugOutput.put("table",table);
-            debugOutput.put("key",key);
-            debugOutput.put("value",value);
-            LOGGER.trace("put operation result: " + JsonUtil.serializeMap(debugOutput, JsonUtil.Flag.PrettyPrint));
-        }
-
-        updateStats(false,true);
-        return true;
-    }
 
-    private synchronized void preOperationCheck() throws DatabaseException {
-        if (status == PwmService.STATUS.CLOSED) {
-            throw new DatabaseException(new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,"database connection is not open"));
-        }
-
-        if (status == PwmService.STATUS.NEW) {
-            init();
-        }
-
-        if (!isValid(connection)) {
-            init();
-        }
-    }
-
-    private boolean isValid(final Connection connection) {
-        if (connection == null) {
-            return false;
-        }
-
-        if (status != PwmService.STATUS.OPEN) {
-            return false;
-        }
-
-        try {
-            final Method getFreeSpaceMethod = File.class.getMethod("isValid");
-            final Object rawResult = getFreeSpaceMethod.invoke(connection,10);
-            return (Boolean) rawResult;
-        } catch (NoSuchMethodException e) {
-            /* no error, pre java 1.6 doesn't have this method */
-        } catch (Exception e) {
-            LOGGER.debug("error checking for isValid for " + connection.toString() + ",: " + e.getMessage());
-        }
-
-        final StringBuilder sb = new StringBuilder();
-        sb.append("SELECT * FROM ").append(DatabaseTable.PWM_META.toString()).append(" WHERE " + KEY_COLUMN + " = ?");
-        PreparedStatement statement = null;
-        ResultSet resultSet = null;
-        try {
-            statement = connection.prepareStatement(sb.toString());
-            statement.setString(1, KEY_ENGINE_START_PREFIX + instanceID);
-            statement.setMaxRows(1);
-            resultSet = statement.executeQuery();
-            if (resultSet.next()) {
-                resultSet.getString(VALUE_COLUMN);
+            if (!valueExists) {
+                final String insertSql = "INSERT INTO " + table.name() + "(" + DatabaseService.KEY_COLUMN + ", " + DatabaseService.VALUE_COLUMN + ") VALUES(?,?)";
+                executeUpdate(insertSql, debugInfo, key, value);
             }
-        } catch (SQLException e) {
-            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,"isValid operation failed: " + e.getMessage());
-            lastError = errorInformation;
-            LOGGER.error(errorInformation.toDebugStr());
-            return false;
-        } finally {
-            close(statement);
-            close(resultSet);
-        }
-        return true;
-    }
 
-    private static void close(final Statement statement) {
-        if (statement != null) {
-            try {
-                statement.close();
-            } catch (SQLException e) {
-                LOGGER.error("unexpected error during close statement object " + e.getMessage(), e);
-            }
-        }
+            return !valueExists;
+        });
     }
 
-    private static void close(final ResultSet resultSet) {
-        if (resultSet != null) {
-            try {
-                resultSet.close();
-            } catch (SQLException e) {
-                LOGGER.error("unexpected error during close resultSet object " + e.getMessage(), e);
-            }
-        }
-    }
 
     @Override
     public boolean contains(
@@ -476,16 +142,17 @@ public class DatabaseAccessorImpl implements PwmService, DatabaseAccessor {
     )
             throws DatabaseException
     {
-        final boolean result = get(table, key) != null;
-        if (traceLogging) {
-            final Map<String,Object> debugOutput = new LinkedHashMap<>();
-            debugOutput.put("table",table);
-            debugOutput.put("key",key);
-            debugOutput.put("result",result);
-            LOGGER.trace("contains operation result: " + JsonUtil.serializeMap(debugOutput, JsonUtil.Flag.PrettyPrint));
-        }
-        updateStats(true,false);
-        return result;
+        final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create("contains", table, key, null);
+
+        return execute(debugInfo, () -> {
+            boolean valueExists = false;
+            try {
+                valueExists = containsImpl(table, key);
+            } catch (final SQLException e) {
+                processSqlException(debugInfo, e);
+            }
+            return valueExists;
+        });
     }
 
     @Override
@@ -495,137 +162,114 @@ public class DatabaseAccessorImpl implements PwmService, DatabaseAccessor {
     )
             throws DatabaseException
     {
-        if (traceLogging) {
-            LOGGER.trace("attempting get operation for table=" + table + ", key=" + key);
-        }
-        preOperationCheck();
-        final StringBuilder sb = new StringBuilder();
-        sb.append("SELECT * FROM ").append(table.toString()).append(" WHERE " + KEY_COLUMN + " = ?");
-
-        PreparedStatement statement = null;
-        ResultSet resultSet = null;
-        String returnValue = null;
-        try {
-            statement = connection.prepareStatement(sb.toString());
-            statement.setString(1, key);
-            statement.setMaxRows(1);
-            resultSet = statement.executeQuery();
+        final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create("get", table, key, null);
 
-            if (resultSet.next()) {
-                returnValue = resultSet.getString(VALUE_COLUMN);
-            }
-        } catch (SQLException e) {
-            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,"get operation failed: " + e.getMessage());
-            lastError = errorInformation;
-            throw new DatabaseException(errorInformation);
-        } finally {
-            close(statement);
-            close(resultSet);
-        }
+        return execute(debugInfo, () -> {
+            final String sqlStatement = "SELECT * FROM " + table.name() + " WHERE " + DatabaseService.KEY_COLUMN + " = ?";
 
-        if (traceLogging) {
-            final LinkedHashMap<String,Object> debugOutput = new LinkedHashMap<>();
-            debugOutput.put("table",table);
-            debugOutput.put("key",key);
-            debugOutput.put("result",returnValue);
-            LOGGER.trace("get operation result: " + JsonUtil.serializeMap(debugOutput, JsonUtil.Flag.PrettyPrint));
-        }
+            try (PreparedStatement statement = connection.prepareStatement(sqlStatement)) {
+                statement.setString(1, key);
+                statement.setMaxRows(1);
 
-        updateStats(true,false);
-        return returnValue;
+                try (ResultSet resultSet= statement.executeQuery()) {
+                    if (resultSet.next()) {
+                        return resultSet.getString(DatabaseService.VALUE_COLUMN);
+                    }
+                }
+            } catch (SQLException e) {
+                processSqlException(debugInfo, e);
+            }
+            return null;
+        });
     }
 
     @Override
     public ClosableIterator<String> iterator(final DatabaseTable table)
             throws DatabaseException
     {
-        preOperationCheck();
-        return new DBIterator(table);
+        try {
+            LOCK.lock();
+            return new DBIterator(table);
+        } finally {
+            LOCK.unlock();
+        }
     }
 
     @Override
-    public boolean remove(
+    public void remove(
             final DatabaseTable table,
             final String key
     )
             throws DatabaseException
     {
-        if (traceLogging) {
-            LOGGER.trace("attempting remove operation for table=" + table + ", key=" + key);
-        }
+        final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create("remove", table, key, null);
 
-        final boolean result = contains(table, key);
-        if (result) {
-            final StringBuilder sqlText = new StringBuilder();
-            sqlText.append("DELETE FROM ").append(table.toString()).append(" WHERE " + KEY_COLUMN + "=?");
+        execute(debugInfo, () -> {
 
-            PreparedStatement statement = null;
-            try {
-                statement = connection.prepareStatement(sqlText.toString());
-                statement.setString(1, key);
-                statement.executeUpdate();
-                LOGGER.trace("remove operation succeeded for table=" + table + ", key=" + key);
-            } catch (SQLException e) {
-                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,"remove operation failed: " + e.getMessage());
-                lastError = errorInformation;
-                throw new DatabaseException(errorInformation);
-            } finally {
-                close(statement);
-            }
-        }
 
-        if (traceLogging) {
-            final Map<String,Object> debugOutput = new LinkedHashMap<>();
-            debugOutput.put("table",table);
-            debugOutput.put("key",key);
-            debugOutput.put("result",result);
-            LOGGER.trace("remove operation result: " + JsonUtil.serializeMap(debugOutput, JsonUtil.Flag.PrettyPrint));
-        }
+            final String sqlText = "DELETE FROM " + table.name() + " WHERE " + DatabaseService.KEY_COLUMN + "=?";
+            executeUpdate(sqlText, debugInfo, key);
 
-        updateStats(true, false);
-        return result;
+            return null;
+        });
     }
 
     @Override
     public int size(final DatabaseTable table) throws
-            DatabaseException {
-        preOperationCheck();
+            DatabaseException
+    {
+        final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create("size", table, null, null);
 
-        final StringBuilder sb = new StringBuilder();
-        sb.append("SELECT COUNT(" + KEY_COLUMN + ") FROM ").append(table.toString());
+        return execute(debugInfo, () -> {
+            final String sqlStatement = "SELECT COUNT(" + DatabaseService.KEY_COLUMN + ") FROM " + table.name();
+
+
+            try (PreparedStatement statement = connection.prepareStatement(sqlStatement)) {
+                try (ResultSet resultSet = statement.executeQuery()) {
+                    if (resultSet.next()) {
+                        return resultSet.getInt(1);
+                    }
+                }
+            } catch (SQLException e) {
+                processSqlException(debugInfo, e);
+            }
+
+            return 0;
+        });
+    }
+
+    boolean isValid() {
+        if (connection == null) {
+            return false;
+        }
 
-        PreparedStatement statement = null;
-        ResultSet resultSet = null;
         try {
-            statement = connection.prepareStatement(sb.toString());
-            resultSet = statement.executeQuery();
-            if (resultSet.next()) {
-                return resultSet.getInt(1);
+            if (connection.isClosed()) {
+                return false;
             }
+
+            final int connectionTimeout = dbConfiguration.getConnectionTimeout();
+
+            if (!connection.isValid(connectionTimeout)) {
+                return false;
+            }
+
         } catch (SQLException e) {
-            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,"size operation failed: " + e.getMessage());
-            lastError = errorInformation;
-            throw new DatabaseException(errorInformation);
-        } finally {
-            close(statement);
-            close(resultSet);
+            LOGGER.debug("error while checking connection validity: " + e.getMessage());
         }
 
-        updateStats(true,false);
-        return 0;
+        return true;
     }
 
-// -------------------------- ENUMERATIONS --------------------------
 
-    // -------------------------- INNER CLASSES --------------------------
 
     public class DBIterator implements ClosableIterator<String> {
         private final DatabaseTable table;
         private final ResultSet resultSet;
-        private java.lang.String nextValue;
+        private String nextValue;
         private boolean finished;
 
-        public DBIterator(final DatabaseTable table)
+        DBIterator(final DatabaseTable table)
                 throws DatabaseException
         {
             this.table = table;
@@ -634,24 +278,24 @@ public class DatabaseAccessorImpl implements PwmService, DatabaseAccessor {
         }
 
         private ResultSet init() throws DatabaseException {
-            final StringBuilder sb = new StringBuilder();
-            sb.append("SELECT " + KEY_COLUMN + " FROM ").append(table.toString());
+            final String sqlText = "SELECT " + DatabaseService.KEY_COLUMN + " FROM " + table.name();
 
             try {
-                final PreparedStatement statement = connection.prepareStatement(sb.toString());
-                return statement.executeQuery();
+                final PreparedStatement statement = connection.prepareStatement(sqlText);
+                final ResultSet resultSet = statement.executeQuery();
+                connection.commit();
+                return resultSet;
             } catch (SQLException e) {
-                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,"get iterator failed: " + e.getMessage());
-                lastError = errorInformation;
-                throw new DatabaseException(errorInformation);
+                processSqlException(null, e);
             }
+            return null; // unreachable
         }
 
         public boolean hasNext() {
             return !finished;
         }
 
-        public java.lang.String next() {
+        public String next() {
             if (finished) {
                 throw new IllegalStateException("iterator completed");
             }
@@ -667,7 +311,7 @@ public class DatabaseAccessorImpl implements PwmService, DatabaseAccessor {
         private void getNextItem() {
             try {
                 if (resultSet.next()) {
-                    nextValue = resultSet.getString(KEY_COLUMN);
+                    nextValue = resultSet.getString(DatabaseService.KEY_COLUMN);
                 } else {
                     close();
                 }
@@ -675,7 +319,7 @@ public class DatabaseAccessorImpl implements PwmService, DatabaseAccessor {
                 finished = true;
                 LOGGER.warn("unexpected error during result set iteration: " + e.getMessage());
             }
-            updateStats(true,false);
+            databaseService.updateStats(DatabaseService.OperationType.READ);
         }
 
         public void close() {
@@ -690,63 +334,96 @@ public class DatabaseAccessorImpl implements PwmService, DatabaseAccessor {
         }
     }
 
-    public ServiceInfo serviceInfo()
-    {
-        if (status() == STATUS.OPEN) {
-            return new ServiceInfo(Collections.singletonList(DataStorageMethod.DB));
-        } else {
-            return new ServiceInfo(Collections.emptyList());
+    private void traceBegin(final DatabaseUtil.DebugInfo debugInfo) {
+        if (!traceLogEnabled) {
+            return;
         }
+
+        LOGGER.trace("begin operation: " + StringUtil.mapToString(JsonUtil.deserializeStringMap(JsonUtil.serialize(debugInfo))));
     }
 
-    private void updateStats(final boolean readOperation, final boolean writeOperation) {
-        if (pwmApplication != null && pwmApplication.getApplicationMode() == PwmApplicationMode.RUNNING) {
-            final StatisticsManager statisticsManager = pwmApplication.getStatisticsManager();
-            if (statisticsManager != null && statisticsManager.status() == STATUS.OPEN) {
-                if (readOperation) {
-                    statisticsManager.updateEps(Statistic.EpsType.DB_READS,1);
-                }
-                if (writeOperation) {
-                    statisticsManager.updateEps(Statistic.EpsType.DB_WRITES,1);
-                }
-            }
+    private void traceResult(
+            final DatabaseUtil.DebugInfo debugInfo,
+            final Object result
+    ) {
+        if (!traceLogEnabled) {
+            return;
         }
+
+        final Map<String,String> map = JsonUtil.deserializeStringMap(JsonUtil.serialize(debugInfo));
+        map.put("duration", TimeDuration.fromCurrent(debugInfo.getStartTime()).asCompactString());
+        if (result != null) {
+            map.put("result", String.valueOf(result));
+        }
+        LOGGER.trace("operation result: " + StringUtil.mapToString(map));
     }
 
+    private interface SqlFunction<T>  {
+        T execute() throws DatabaseException;
+    }
 
-    @Override
-    public Map<PwmAboutProperty,String> getConnectionDebugProperties() {
-        return getConnectionDebugProperties(connection);
+    private <T> T execute(final DatabaseUtil.DebugInfo debugInfo, final SqlFunction<T> sqlFunction) throws DatabaseException
+    {
+        traceBegin(debugInfo);
+
+        try {
+            LOCK.lock();
+
+            try {
+                final T result = sqlFunction.execute();
+                traceResult(debugInfo, result);
+                databaseService.updateStats(DatabaseService.OperationType.WRITE);
+                return result;
+            } finally {
+                DatabaseUtil.commit(connection);
+            }
+
+        } finally {
+            LOCK.unlock();
+        }
+
+    }
+
+    Connection getConnection()
+    {
+        return connection;
     }
 
-    private static Map<PwmAboutProperty,String> getConnectionDebugProperties(final Connection connection) {
-        if (connection != null) {
+    void close() {
+        try {
+            LOCK.lock();
             try {
-                final Map<PwmAboutProperty,String> returnObj = new LinkedHashMap<>();
-                final DatabaseMetaData databaseMetaData = connection.getMetaData();
-                returnObj.put(PwmAboutProperty.database_driverName, databaseMetaData.getDriverName());
-                returnObj.put(PwmAboutProperty.database_driverVersion, databaseMetaData.getDriverVersion());
-                returnObj.put(PwmAboutProperty.database_databaseProductName, databaseMetaData.getDatabaseProductName());
-                returnObj.put(PwmAboutProperty.database_databaseProductVersion, databaseMetaData.getDatabaseProductVersion());
-                return Collections.unmodifiableMap(returnObj);
+                connection.close();
             } catch (SQLException e) {
-                LOGGER.error("error reading jdbc meta data: " + e.getMessage());
+                LOGGER.warn("error while closing connection: " + e.getMessage());
             }
+        } finally {
+            LOCK.unlock();
         }
-        return Collections.emptyMap();
     }
 
-    @Override
-    public boolean isMasterServer()
+    private boolean containsImpl(final DatabaseTable table, final String key) throws SQLException
     {
-        return false;
-    }
+        final String selectSql = "SELECT * FROM " + table.name() + " WHERE " + DatabaseService.KEY_COLUMN + " = ?";
+        try (PreparedStatement selectStatement = connection.prepareStatement(selectSql);) {
+            selectStatement.setString(1, key);
+            selectStatement.setMaxRows(1);
 
-    private class MasterCheckTask implements Runnable {
-        @Override
-        public void run()
-        {
+            try (ResultSet resultSet = selectStatement.executeQuery()) {
+                return resultSet.next();
+            }
+        }
+    }
 
+    private void executeUpdate(final String sqlStatement, final DatabaseUtil.DebugInfo debugInfo, final String... params) throws DatabaseException
+    {
+        try (PreparedStatement statement = connection.prepareStatement(sqlStatement)){
+            for (int i = 0; i < params.length; i++) {
+                statement.setString(i + 1, params[i]);
+            }
+            statement.executeUpdate();
+        } catch (SQLException e) {
+            processSqlException(debugInfo, e);
         }
     }
 }

src/main/java/password/pwm/util/db/DatabaseClusterService.java

@@ -0,0 +1,41 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.util.db;
+
+public class DatabaseClusterService {
+
+
+
+    private static final String KEY_ENGINE_START_PREFIX = "engine-start-";
+
+    private void heartbeat() {
+        /*
+        try {
+            put(DatabaseTable.PWM_META, KEY_ENGINE_START_PREFIX + instanceID, JavaHelper.toIsoDate(new java.util.Date()));
+        } catch (DatabaseException e) {
+            final String errorMsg = "error writing engine start time value: " + e.getMessage();
+            throw new DatabaseException(new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,errorMsg));
+        }
+        */
+    }
+}

src/main/java/password/pwm/util/db/DatabaseDataStore.java

@@ -23,50 +23,62 @@
 package password.pwm.util.db;
 
 import password.pwm.error.PwmDataStoreException;
+import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.DataStore;
 import password.pwm.util.java.ClosableIterator;
 
 public class DatabaseDataStore implements DataStore {
-    private final DatabaseAccessor databaseAccessor;
+    private final DatabaseService databaseService;
     private final DatabaseTable table;
 
-    public DatabaseDataStore(final DatabaseAccessor databaseAccessor, final DatabaseTable table) {
-        this.databaseAccessor = databaseAccessor;
+    public DatabaseDataStore(final DatabaseService databaseService, final DatabaseTable table) {
+        this.databaseService = databaseService;
         this.table = table;
     }
 
     public void close() throws PwmDataStoreException {
     }
 
-    public boolean contains(final String key) throws PwmDataStoreException {
-        return databaseAccessor.contains(table, key);
+    public boolean contains(final String key) throws PwmDataStoreException, PwmUnrecoverableException
+    {
+        return databaseService.getAccessor().contains(table, key);
     }
 
-    public String get(final String key) throws PwmDataStoreException {
-        return databaseAccessor.get(table,key);
+    public String get(final String key) throws PwmDataStoreException, PwmUnrecoverableException
+    {
+        return databaseService.getAccessor().get(table,key);
     }
 
-    public ClosableIterator<String> iterator() throws PwmDataStoreException {
-        return databaseAccessor.iterator(table);
+    public ClosableIterator<String> iterator() throws PwmDataStoreException, PwmUnrecoverableException
+    {
+        return databaseService.getAccessor().iterator(table);
     }
 
     public Status status() {
-        if (databaseAccessor == null) {
+        if (databaseService == null) {
             return null;
         }
 
         return Status.OPEN;
     }
 
-    public boolean put(final String key, final String value) throws PwmDataStoreException {
-        return databaseAccessor.put(table, key, value);
+    public boolean put(final String key, final String value) throws PwmDataStoreException, PwmUnrecoverableException
+    {
+        return databaseService.getAccessor().put(table, key, value);
     }
 
-    public boolean remove(final String key) throws PwmDataStoreException {
-        return databaseAccessor.remove(table, key);
+    public boolean putIfAbsent(final String key, final String value) throws PwmDataStoreException, PwmUnrecoverableException
+    {
+        return databaseService.getAccessor().putIfAbsent(table, key, value);
     }
 
-    public int size() throws PwmDataStoreException {
-        return databaseAccessor.size(table);
+    public void remove(final String key) throws PwmDataStoreException, PwmUnrecoverableException
+    {
+        databaseService.getAccessor().remove(table, key);
+    }
+
+    public int size() throws PwmDataStoreException, PwmUnrecoverableException
+    {
+        return databaseService.getAccessor().size(table);
     }
 }

src/main/java/password/pwm/util/db/DatabaseService.java

@@ -0,0 +1,392 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.util.db;
+
+import password.pwm.AppProperty;
+import password.pwm.PwmAboutProperty;
+import password.pwm.PwmApplication;
+import password.pwm.PwmApplicationMode;
+import password.pwm.PwmConstants;
+import password.pwm.config.Configuration;
+import password.pwm.config.PwmSetting;
+import password.pwm.config.option.DataStorageMethod;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmException;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.health.HealthRecord;
+import password.pwm.health.HealthStatus;
+import password.pwm.health.HealthTopic;
+import password.pwm.svc.PwmService;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
+import password.pwm.util.java.AtomicLoopIntIncrementer;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogger;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.Driver;
+import java.sql.SQLException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class DatabaseService implements PwmService
+{
+    private static final String KEY_TEST = "write-test-key";
+
+    static final String KEY_COLUMN = "id";
+    static final String VALUE_COLUMN = "value";
+
+    private static final PwmLogger LOGGER = PwmLogger.forClass(DatabaseService.class);
+    static final int KEY_COLUMN_LENGTH = PwmConstants.DATABASE_ACCESSOR_KEY_LENGTH;
+
+    private DBConfiguration dbConfiguration;
+
+    private Driver driver;
+    private JDBCDriverLoader.DriverLoader jdbcDriverLoader;
+
+    private ErrorInformation lastError;
+    private PwmApplication pwmApplication;
+
+    private STATUS status = STATUS.NEW;
+
+    private AtomicLoopIntIncrementer slotIncrementer;
+    private Map<Integer,DatabaseAccessorImpl> accessors = new ConcurrentHashMap<>();
+
+    private ScheduledExecutorService executorService;
+
+    private volatile boolean initialized = false;
+
+
+    @Override
+    public STATUS status()
+    {
+        return status;
+    }
+
+    @Override
+    public void init(final PwmApplication pwmApplication) throws PwmException
+    {
+        this.pwmApplication = pwmApplication;
+        init();
+
+        executorService = Executors.newSingleThreadScheduledExecutor(
+                JavaHelper.makePwmThreadFactory(
+                        JavaHelper.makeThreadName(pwmApplication, this.getClass()) + "-",
+                        true
+                ));
+
+        final int watchdogFrequencySeconds = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.DB_CONNECTIONS_WATCHDOG_FREQUENCY_SECONDS));
+        executorService.scheduleWithFixedDelay(new ConnectionMonitor(),watchdogFrequencySeconds, watchdogFrequencySeconds, TimeUnit.SECONDS);
+    }
+
+    private synchronized void init()
+    {
+        if (initialized) {
+            return;
+        }
+
+        final Instant startTime = Instant.now();
+        status = STATUS.OPENING;
+
+        try {
+            final Configuration config = pwmApplication.getConfig();
+            this.dbConfiguration = DBConfiguration.fromConfiguration(config);
+
+            if (!dbConfiguration.isEnabled()) {
+                status = PwmService.STATUS.CLOSED;
+                LOGGER.debug("skipping database connection open, no connection parameters configured");
+                initialized = true;
+                return;
+            }
+
+            LOGGER.debug("opening connection to database " + this.dbConfiguration.getConnectionString());
+            slotIncrementer = new AtomicLoopIntIncrementer(dbConfiguration.getMaxConnections());
+
+            { // make initial connection and establish schema
+                clearCurrentAccessors();
+
+                final Connection connection = openConnection(dbConfiguration);
+                for (final DatabaseTable table : DatabaseTable.values()) {
+                    DatabaseUtil.initTable(connection, table, dbConfiguration);
+                }
+
+                connection.close();
+            }
+
+            accessors.clear();
+            { // set up connection pool
+                final boolean traceLogging = config.readSettingAsBoolean(PwmSetting.DATABASE_DEBUG_TRACE);
+                for (int i = 0; i < dbConfiguration.getMaxConnections(); i++) {
+                    final Connection connection = openConnection(dbConfiguration);
+                    final DatabaseAccessorImpl accessor = new DatabaseAccessorImpl(this, this.dbConfiguration, connection, traceLogging);
+                    accessors.put(i, accessor);
+                }
+            }
+
+            LOGGER.debug("successfully connected to remote database (" + TimeDuration.fromCurrent(startTime).asCompactString() + ")");
+
+            status = STATUS.OPEN;
+            initialized = true;
+        } catch (Throwable t) {
+            final String errorMsg = "exception initializing database service: " + t.getMessage();
+            LOGGER.warn(errorMsg);
+            initialized = false;
+            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE, errorMsg);
+            lastError = errorInformation;
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        status = PwmService.STATUS.CLOSED;
+
+        if (executorService != null) {
+            executorService.shutdown();
+        }
+
+        clearCurrentAccessors();
+
+        try {
+            driver = null;
+        } catch (Exception e) {
+            LOGGER.debug("error while de-registering driver: " + e.getMessage());
+        }
+
+        if (jdbcDriverLoader != null) {
+            jdbcDriverLoader.unloadDriver();
+            jdbcDriverLoader = null;
+        }
+    }
+
+    private void clearCurrentAccessors() {
+        for (DatabaseAccessorImpl accessor : accessors.values()) {
+            accessor.close();
+        }
+        accessors.clear();
+    }
+
+    public List<HealthRecord> healthCheck() {
+        if (status == PwmService.STATUS.CLOSED) {
+            return Collections.emptyList();
+        }
+
+        final List<HealthRecord> returnRecords = new ArrayList<>();
+
+        if (!initialized) {
+            returnRecords.add(new HealthRecord(HealthStatus.WARN, HealthTopic.Database, makeUninitializedError().getDetailedErrorMsg()));
+            return returnRecords;
+        }
+
+        try {
+            final Map<String,String> tempMap = new HashMap<>();
+            tempMap.put("date", JavaHelper.toIsoDate(Instant.now()));
+            final DatabaseAccessor accessor = getAccessor();
+            accessor.put(DatabaseTable.PWM_META, KEY_TEST, JsonUtil.serializeMap(tempMap));
+        } catch (PwmException e) {
+            returnRecords.add(new HealthRecord(HealthStatus.WARN, HealthTopic.Database, "Error writing to database: " + e.getErrorInformation().toDebugStr()));
+            return returnRecords;
+        }
+
+        if (lastError != null) {
+            final TimeDuration errorAge = TimeDuration.fromCurrent(lastError.getDate());
+
+            if (errorAge.isShorterThan(TimeDuration.HOUR)) {
+                final String msg = "Database server was recently unavailable ("
+                        + errorAge.asLongString(PwmConstants.DEFAULT_LOCALE)
+                        + " ago at " + lastError.getDate().toString()+ "): " + lastError.toDebugStr();
+                returnRecords.add(new HealthRecord(HealthStatus.CAUTION, HealthTopic.Database, msg));
+            }
+        }
+
+        if (returnRecords.isEmpty()) {
+            returnRecords.add(new HealthRecord(HealthStatus.GOOD, HealthTopic.Database, "Database connection to " + this.dbConfiguration.getConnectionString() + " okay"));
+        }
+
+        return returnRecords;
+    }
+
+    private ErrorInformation makeUninitializedError() {
+
+        final String errorMsg;
+        if (dbConfiguration != null && !dbConfiguration.isEnabled()) {
+            errorMsg = "database is not configured";
+        } else {
+            if (lastError != null) {
+                errorMsg = "unable to initialize database: " + lastError.getDetailedErrorMsg();
+            } else {
+                errorMsg = "database is not yet initialized";
+            }
+        }
+        return new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE, errorMsg);
+    }
+
+
+    @Override
+    public ServiceInfo serviceInfo()
+    {
+        if (status() == STATUS.OPEN) {
+            return new ServiceInfo(Collections.singletonList(DataStorageMethod.DB));
+        } else {
+            return new ServiceInfo(Collections.emptyList());
+        }
+    }
+
+    public DatabaseAccessor getAccessor()
+            throws PwmUnrecoverableException
+    {
+        if (status == PwmService.STATUS.CLOSED) {
+            throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,"database connection is not open"));
+        }
+
+        if (!initialized) {
+            throw new PwmUnrecoverableException(makeUninitializedError());
+        }
+
+        return accessors.get(slotIncrementer.next());
+    }
+
+    private Connection openConnection(final DBConfiguration dbConfiguration)
+            throws DatabaseException
+    {
+        final String connectionURL = dbConfiguration.getConnectionString();
+
+        final JDBCDriverLoader.DriverWrapper wrapper = JDBCDriverLoader.loadDriver(pwmApplication, dbConfiguration);
+        driver = wrapper.getDriver();
+        jdbcDriverLoader = wrapper.getDriverLoader();
+
+        try {
+            LOGGER.debug("initiating connecting to database " + connectionURL);
+            final Properties connectionProperties = new Properties();
+            if (dbConfiguration.getUsername() != null && !dbConfiguration.getUsername().isEmpty()) {
+                connectionProperties.setProperty("user", dbConfiguration.getUsername());
+            }
+            if (dbConfiguration.getPassword() != null) {
+                connectionProperties.setProperty("password", dbConfiguration.getPassword().getStringValue());
+            }
+
+            final Connection connection = driver.connect(connectionURL, connectionProperties);
+            final Map<PwmAboutProperty,String> debugProps = getConnectionDebugProperties(connection);
+            LOGGER.debug("connected to database " + connectionURL + ", properties: " + JsonUtil.serializeMap(debugProps));
+
+            connection.setAutoCommit(false);
+            return connection;
+        } catch (Throwable e) {
+            final String errorMsg = "error connecting to database: " + JavaHelper.readHostileExceptionMessage(e);
+            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE,errorMsg);
+            LOGGER.error(errorInformation);
+            throw new DatabaseException(errorInformation);
+        }
+    }
+
+
+    enum OperationType {
+        WRITE,
+        READ,
+    }
+
+    void updateStats(final OperationType operationType) {
+        if (pwmApplication != null && pwmApplication.getApplicationMode() == PwmApplicationMode.RUNNING) {
+            final StatisticsManager statisticsManager = pwmApplication.getStatisticsManager();
+            if (statisticsManager != null && statisticsManager.status() == PwmService.STATUS.OPEN) {
+                if (operationType == OperationType.READ) {
+                    statisticsManager.updateEps(Statistic.EpsType.DB_READS,1);
+                }
+                if (operationType == OperationType.WRITE) {
+                    statisticsManager.updateEps(Statistic.EpsType.DB_WRITES,1);
+                }
+            }
+        }
+    }
+
+    public Map<PwmAboutProperty,String> getConnectionDebugProperties() {
+        if (!initialized) {
+            return Collections.emptyMap();
+        }
+        final Connection connection = accessors.get(0).getConnection();
+        return getConnectionDebugProperties(connection);
+    }
+
+    private static Map<PwmAboutProperty,String> getConnectionDebugProperties(final Connection connection) {
+        if (connection != null) {
+            try {
+                final Map<PwmAboutProperty,String> returnObj = new LinkedHashMap<>();
+                final DatabaseMetaData databaseMetaData = connection.getMetaData();
+                returnObj.put(PwmAboutProperty.database_driverName, databaseMetaData.getDriverName());
+                returnObj.put(PwmAboutProperty.database_driverVersion, databaseMetaData.getDriverVersion());
+                returnObj.put(PwmAboutProperty.database_databaseProductName, databaseMetaData.getDatabaseProductName());
+                returnObj.put(PwmAboutProperty.database_databaseProductVersion, databaseMetaData.getDatabaseProductVersion());
+                return Collections.unmodifiableMap(returnObj);
+            } catch (SQLException e) {
+                LOGGER.error("error reading jdbc meta data: " + e.getMessage());
+            }
+        }
+        return Collections.emptyMap();
+    }
+
+    void setLastError(final ErrorInformation lastError)
+    {
+        this.lastError = lastError;
+    }
+
+
+
+    private class ConnectionMonitor implements Runnable {
+        @Override
+        public void run()
+        {
+            if (initialized) {
+                boolean valid = true;
+                for (final DatabaseAccessorImpl databaseAccessor : accessors.values()) {
+                    if (!databaseAccessor.isValid()) {
+                        valid = false;
+                        break;
+                    }
+                }
+                if (!valid) {
+                    LOGGER.warn("database connection lost; will retry connect periodically");
+                    initialized = false;
+                }
+
+            }
+
+            if (!initialized) {
+                init();
+            }
+        }
+    }
+}

src/main/java/password/pwm/util/db/DatabaseUtil.java

@@ -0,0 +1,220 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.util.db;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.util.logging.PwmLogger;
+
+import java.io.Serializable;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.time.Instant;
+import java.util.concurrent.atomic.AtomicInteger;
+
+class DatabaseUtil {
+    private static final AtomicInteger OP_COUNTER = new AtomicInteger();
+
+    private static final PwmLogger LOGGER = PwmLogger.forClass(DatabaseUtil.class);
+
+    private DatabaseUtil()
+    {
+    }
+
+
+    static void close(final Statement statement) throws DatabaseException
+    {
+        if (statement != null) {
+            try {
+                statement.close();
+            } catch (SQLException e) {
+                LOGGER.error("unexpected error during close statement object " + e.getMessage(), e);
+                throw new DatabaseException(new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE, "statement close failure: " + e.getMessage()));
+            }
+        }
+    }
+
+    static void close(final ResultSet resultSet) throws DatabaseException
+    {
+        if (resultSet != null) {
+            try {
+                resultSet.close();
+            } catch (SQLException e) {
+                LOGGER.error("unexpected error during close resultSet object " + e.getMessage(), e);
+                throw new DatabaseException(new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE, "resultset close failure: " + e.getMessage()));
+            }
+        }
+    }
+
+    static void commit(final Connection connection)
+            throws DatabaseException
+    {
+        try {
+            connection.commit();
+        } catch (SQLException e) {
+            LOGGER.warn("database commit failed: " + e.getMessage());
+            throw new DatabaseException(new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE, "commit failure: " + e.getMessage()));
+        }
+    }
+
+    static DatabaseException convertSqlException(
+            final DebugInfo debugInfo,
+            final SQLException e
+    )
+    {
+        final String errorMsg = debugInfo.getOpName() + " operation opId=" + debugInfo.getOpId() + " failed, error: " + e.getMessage();
+        final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE, errorMsg);
+        return new DatabaseException(errorInformation);
+    }
+
+    static void initTable(
+            final Connection connection,
+            final DatabaseTable table,
+            final DBConfiguration dbConfiguration
+    )
+            throws DatabaseException
+    {
+        boolean tableExists = false;
+        try {
+            checkIfTableExists(connection, table);
+            LOGGER.trace("table " + table + " appears to exist");
+            tableExists = true;
+        } catch (DatabaseException e) { // assume error was due to table missing;
+            LOGGER.trace("error while checking for table: " + e.getMessage() + ", assuming due to table non-existence");
+        }
+
+        if (!tableExists) {
+            createTable(connection, table, dbConfiguration);
+        }
+    }
+
+    private static void createTable(
+            final Connection connection,
+            final DatabaseTable table,
+            final DBConfiguration dbConfiguration
+    )
+            throws DatabaseException
+    {
+        {
+            final StringBuilder sqlString = new StringBuilder();
+            sqlString.append("CREATE table ").append(table.toString()).append(" (").append("\n");
+            sqlString.append("  " + DatabaseService.KEY_COLUMN + " ").append(dbConfiguration.getColumnTypeKey()).append("(").append(
+                    DatabaseService.KEY_COLUMN_LENGTH).append(") NOT NULL PRIMARY KEY,").append("\n");
+            sqlString.append("  " + DatabaseService.VALUE_COLUMN + " ").append(dbConfiguration.getColumnTypeValue()).append(" ");
+            sqlString.append("\n");
+            sqlString.append(")").append("\n");
+
+            LOGGER.trace("attempting to execute the following sql statement:\n " + sqlString.toString());
+
+            Statement statement = null;
+            try {
+                statement = connection.createStatement();
+                statement.execute(sqlString.toString());
+                connection.commit();
+                LOGGER.debug("created table " + table.toString());
+            } catch (SQLException ex) {
+                final String errorMsg = "error creating new table " + table.toString() + ": " + ex.getMessage();
+                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE, errorMsg);
+                throw new DatabaseException(errorInformation);
+            } finally {
+                DatabaseUtil.close(statement);
+            }
+        }
+
+        {
+            final String indexName = table.toString() + "_IDX";
+            final StringBuilder sqlString = new StringBuilder();
+            sqlString.append("CREATE index ").append(indexName);
+            sqlString.append(" ON ").append(table.toString());
+            sqlString.append(" (").append(DatabaseService.KEY_COLUMN).append(")");
+            Statement statement = null;
+
+            LOGGER.trace("attempting to execute the following sql statement:\n " + sqlString.toString());
+
+            try {
+                statement = connection.createStatement();
+                statement.execute(sqlString.toString());
+                connection.commit();
+                LOGGER.debug("created index " + indexName);
+            } catch (SQLException ex) {
+                final String errorMsg = "error creating new index " + indexName + ": " + ex.getMessage();
+                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_DB_UNAVAILABLE, errorMsg);
+                throw new DatabaseException(errorInformation);
+            } finally {
+                DatabaseUtil.close(statement);
+            }
+        }
+    }
+
+    private static void checkIfTableExists(
+            final Connection connection,
+            final DatabaseTable table
+    )
+            throws DatabaseException
+    {
+        final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create("checkIfTableExists",null,null,null);
+        final StringBuilder sb = new StringBuilder();
+        sb.append("SELECT * FROM  ").append(table.toString()).append(" WHERE " + DatabaseService.KEY_COLUMN + " = '0'");
+        Statement statement = null;
+        ResultSet resultSet = null;
+        try {
+            statement = connection.createStatement();
+            resultSet = statement.executeQuery(sb.toString());
+        } catch (SQLException e) {
+            throw DatabaseUtil.convertSqlException(debugInfo, e);
+        } finally {
+            DatabaseUtil.close(statement);
+            DatabaseUtil.close(resultSet);
+        }
+    }
+
+
+    @Getter
+    @AllArgsConstructor
+    static class DebugInfo implements Serializable {
+        private final Instant startTime = Instant.now();
+        private final int opId = OP_COUNTER.incrementAndGet();
+        private final String opName;
+        private final DatabaseTable table;
+        private final String key;
+        private final String value;
+
+        static DebugInfo create(
+                final String opName,
+                final DatabaseTable table,
+                final String key,
+                final String value
+        ) {
+            return new DebugInfo(
+                    opName,
+                    table,
+                    key,
+                    value
+            );
+        }
+    }
+}

src/main/java/password/pwm/util/java/AtomicLoopIntIncrementer.java

@@ -0,0 +1,50 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.util.java;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Thread safe rotating int incrementer with configurable floor and ceiling values.
+ */
+public class AtomicLoopIntIncrementer {
+    private final AtomicInteger incrementer = new AtomicInteger(0);
+    private final int ceiling;
+    private final int floor;
+
+    public AtomicLoopIntIncrementer(final int ceiling)
+    {
+        this.ceiling = ceiling;
+        this.floor = 0;
+    }
+
+    public int next() {
+        return incrementer.getAndUpdate(operand -> {
+            operand++;
+            if (operand >= ceiling) {
+                operand = floor;
+            }
+            return operand;
+        });
+    }
+}

src/main/java/password/pwm/util/java/JavaHelper.java

@@ -41,6 +41,7 @@ import java.lang.reflect.Method;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
@@ -256,7 +257,7 @@ public class JavaHelper {
     }
 
     public static String toIsoDate(final Instant instant) {
-        return instant == null ? "" : instant.toString();
+        return instant == null ? "" : instant.truncatedTo(ChronoUnit.SECONDS).toString();
     }
 
     public static String toIsoDate(final Date date) {

src/main/java/password/pwm/util/java/JsonUtil.java

@@ -197,27 +197,16 @@ public class JsonUtil {
      * GsonSerializer that stores instants in ISO 8601 format, with a deserialier that also reads local-platform format reading.
      */
     private static class InstantTypeAdapter implements JsonSerializer<Instant>, JsonDeserializer<Instant> {
-        private static final DateFormat ISO_DATE_FORMAT;
-        private static final DateFormat GSON_DATE_FORMAT;
-
-        static {
-            ISO_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
-            ISO_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("Zulu"));
-
-            GSON_DATE_FORMAT = DateFormat.getDateTimeInstance(DateFormat.DEFAULT, DateFormat.DEFAULT);
-            GSON_DATE_FORMAT.setTimeZone(TimeZone.getDefault());
-        }
-
         private InstantTypeAdapter() {
         }
 
         public synchronized JsonElement serialize(final Instant instant, final Type type, final JsonSerializationContext jsonSerializationContext) {
-            return new JsonPrimitive(instant.toString());
+            return new JsonPrimitive(JavaHelper.toIsoDate(instant));
         }
 
         public synchronized Instant deserialize(final JsonElement jsonElement, final Type type, final JsonDeserializationContext jsonDeserializationContext) {
             try {
-                return Instant.parse(jsonElement.getAsString());
+                return JavaHelper.parseIsoToInstant(jsonElement.getAsString());
             } catch (Exception e) {
                 LOGGER.debug("unable to parse stored json Instant.class timestamp '" + jsonElement.getAsString() + "' error: " + e.getMessage());
                 throw new JsonParseException(e);

src/main/java/password/pwm/util/localdb/AbstractJDBC_LocalDB.java

@@ -342,6 +342,45 @@ public abstract class AbstractJDBC_LocalDB implements LocalDBProvider {
         return true;
     }
 
+    public boolean putIfAbsent(final LocalDB.DB db, final String key, final String value)
+            throws LocalDBException
+    {
+        preCheck(true);
+        final String selectSql ="SELECT * FROM " + db.toString() + " WHERE " + KEY_COLUMN + " = ?";
+
+        PreparedStatement selectStatement = null;
+        ResultSet resultSet = null;
+        PreparedStatement insertStatement = null;
+        try {
+            LOCK.writeLock().lock();
+            selectStatement = dbConnection.prepareStatement(selectSql);
+            selectStatement.setString(1, key);
+            selectStatement.setMaxRows(1);
+            resultSet = selectStatement.executeQuery();
+
+            final boolean valueExists = resultSet.next();
+
+            if (!valueExists) {
+                final String insertSql = "INSERT INTO " + db.toString() + "(" + KEY_COLUMN + ", " + VALUE_COLUMN + ") VALUES(?,?)";
+                insertStatement = dbConnection.prepareStatement(insertSql);
+                insertStatement.setString(1, key);
+                insertStatement.setString(2, value);
+                insertStatement.executeUpdate();
+            }
+
+            dbConnection.commit();
+
+            return !valueExists;
+        } catch (final SQLException ex) {
+            throw new LocalDBException(new ErrorInformation(PwmError.ERROR_LOCALDB_UNAVAILABLE,ex.getMessage()));
+        } finally {
+            close(selectStatement);
+            close(resultSet);
+            close(insertStatement);
+            LOCK.writeLock().unlock();
+        }
+    }
+
     public boolean remove(final LocalDB.DB db, final String key)
             throws LocalDBException {
         preCheck(true);

src/main/java/password/pwm/util/localdb/LocalDB.java

@@ -89,6 +89,10 @@ public interface LocalDB {
     boolean put(DB db, String key, String value)
             throws LocalDBException;
 
+    @WriteOperation
+    boolean putIfAbsent(DB db, String key, String value)
+            throws LocalDBException;
+
     @WriteOperation
     boolean remove(DB db, String key)
             throws LocalDBException;
@@ -152,8 +156,6 @@ public interface LocalDB {
     }
 
 
-// -------------------------- INNER CLASSES --------------------------
-
     @Retention(RetentionPolicy.RUNTIME)
     @interface
     ReadOperation {
@@ -167,81 +169,4 @@ public interface LocalDB {
 
     interface LocalDBIterator<K> extends ClosableIterator<String> {
     }
-
-    class TransactionItem implements Serializable, Comparable {
-        private final DB db;
-        private final String key;
-        private final String value;
-
-        public TransactionItem(final DB db, final String key, final String value) {
-            if (key == null || value == null || db == null) {
-                throw new IllegalArgumentException("db, key or value can not be null");
-            }
-
-            this.db = db;
-            this.key = key;
-            this.value = value;
-        }
-
-        public DB getDb() {
-            return db;
-        }
-
-        public String getKey() {
-            return key;
-        }
-
-        public String getValue() {
-            return value;
-        }
-
-        @Override
-        public String toString() {
-            return "db=" + db + ", key=" + key + ", value=" + value;
-        }
-
-        @Override
-        public boolean equals(final Object o) {
-            if (this == o) {
-                return true;
-            }
-            if (o == null || getClass() != o.getClass()) {
-                return false;
-            }
-
-            final TransactionItem that = (TransactionItem) o;
-
-            return db == that.db && key.equals(that.key) && value.equals(that.value);
-        }
-
-        @Override
-        public int hashCode() {
-            int result;
-            result = db.hashCode();
-            result = 31 * result + key.hashCode();
-            result = 31 * result + value.hashCode();
-            return result;
-        }
-
-        @Override
-        public int compareTo(final Object o) {
-            if (!(o instanceof TransactionItem)) {
-                throw new IllegalArgumentException("can only compare same object type");
-            }
-
-            int result = db.compareTo(db);
-
-            if (result == 0) {
-                result = getKey().compareTo(((TransactionItem) o).getKey());
-
-                if (result == 0) {
-                    result = getValue().compareTo(((TransactionItem) o).getValue());
-                }
-            }
-
-            return result;
-        }
-
-
-    }
 }

src/main/java/password/pwm/util/localdb/LocalDBAdaptor.java

@@ -181,6 +181,23 @@ public class LocalDBAdaptor implements LocalDB {
         return preExisting;
     }
 
+    @WriteOperation
+    public boolean putIfAbsent(final DB db, final String key, final String value) throws LocalDBException {
+        ParameterValidator.validateDBValue(db);
+        ParameterValidator.validateKeyValue(key);
+        ParameterValidator.validateValueValue(value);
+
+        final boolean success = innerDB.putIfAbsent(db, key, value);
+        if (success) {
+            if (SIZE_CACHE_MANAGER != null) {
+                SIZE_CACHE_MANAGER.incrementSize(db);
+            }
+        }
+
+        markWrite(1);
+        return success;
+    }
+
     @WriteOperation
     public boolean remove(final DB db, final String key) throws LocalDBException {
         ParameterValidator.validateDBValue(db);

src/main/java/password/pwm/util/localdb/LocalDBDataStore.java

@@ -81,8 +81,12 @@ public class LocalDBDataStore implements DataStore {
         return localDB.put(db, key, value);
     }
 
-    public boolean remove(final String key) throws PwmDataStoreException {
-        return localDB.remove(db, key);
+    public boolean putIfAbsent(final String key, final String value) throws PwmDataStoreException {
+        return localDB.putIfAbsent(db, key, value);
+    }
+
+    public void remove(final String key) throws PwmDataStoreException {
+        localDB.remove(db, key);
     }
 
     public int size() throws PwmDataStoreException {

src/main/java/password/pwm/util/localdb/LocalDBFactory.java

@@ -99,11 +99,13 @@ public class LocalDBFactory {
 
         final StringBuilder debugText = new StringBuilder();
         debugText.append("LocalDB open in ").append(openTime.asCompactString());
-        debugText.append(", db size: ").append(StringUtil.formatDiskSize(FileSystemUtility.getFileDirectorySize(localDB.getFileLocation())));
-        debugText.append(" at ").append(dbDirectory.toString());
-        final long freeSpace = FileSystemUtility.diskSpaceRemaining(localDB.getFileLocation());
-        if (freeSpace >= 0) {
-            debugText.append(", ").append(StringUtil.formatDiskSize(freeSpace)).append(" free");
+        if (localDB.getFileLocation() != null) {
+            debugText.append(", db size: ").append(StringUtil.formatDiskSize(FileSystemUtility.getFileDirectorySize(localDB.getFileLocation())));
+            debugText.append(" at ").append(dbDirectory.toString());
+            final long freeSpace = FileSystemUtility.diskSpaceRemaining(localDB.getFileLocation());
+            if (freeSpace >= 0) {
+                debugText.append(", ").append(StringUtil.formatDiskSize(freeSpace)).append(" free");
+            }
         }
         LOGGER.info(debugText);
 

src/main/java/password/pwm/util/localdb/LocalDBProvider.java

@@ -66,6 +66,10 @@ public interface LocalDBProvider {
     boolean put(LocalDB.DB db, String key, String value)
             throws LocalDBException;
 
+    @LocalDB.WriteOperation
+    boolean putIfAbsent(LocalDB.DB db, String key, String value)
+            throws LocalDBException;
+
     @LocalDB.WriteOperation
     boolean remove(LocalDB.DB db, String key)
             throws LocalDBException;

src/main/java/password/pwm/util/localdb/Memory_LocalDB.java

@@ -30,7 +30,6 @@ import java.io.File;
 import java.io.Serializable;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
@@ -45,9 +44,8 @@ public class Memory_LocalDB implements LocalDBProvider {
 
     private static final long MIN_FREE_MEMORY = 1024 * 1024;  // 1mb
     private LocalDB.Status state = LocalDB.Status.NEW;
-    private Map<LocalDB.DB, Map<String, String>> maps = new HashMap<>();
 
-// -------------------------- STATIC METHODS --------------------------
+    private Map<LocalDB.DB, Map<String, String>> maps = new ConcurrentHashMap<>();
 
     private static void checkFreeMem() throws LocalDBException {
         final long currentFreeMem = Runtime.getRuntime().freeMemory();
@@ -68,8 +66,6 @@ public class Memory_LocalDB implements LocalDBProvider {
         checkFreeMem();
     }
 
-// --------------------------- CONSTRUCTORS ---------------------------
-
     public Memory_LocalDB() {
         for (final LocalDB.DB db : LocalDB.DB.values()) {
             final Map<String, String> newMap = new ConcurrentHashMap<>();
@@ -77,11 +73,6 @@ public class Memory_LocalDB implements LocalDBProvider {
         }
     }
 
-// ------------------------ INTERFACE METHODS ------------------------
-
-
-// --------------------- Interface PwmLocalDB.DB ---------------------
-
     @LocalDB.WriteOperation
     public void close()
             throws LocalDBException {
@@ -106,8 +97,13 @@ public class Memory_LocalDB implements LocalDBProvider {
     }
 
     @LocalDB.WriteOperation
-    public void init(final File dbDirectory, final Map<String, String> initParameters, final Map<LocalDBProvider.Parameter,String> parameters)
-            throws LocalDBException {
+    public void init(
+            final File dbDirectory,
+            final Map<String, String> initParameters,
+            final Map<LocalDBProvider.Parameter,String> parameters
+    )
+            throws LocalDBException
+    {
         final boolean readOnly = LocalDBUtility.hasBooleanParameter(Parameter.readOnly, parameters);
         if (readOnly) {
             maps = Collections.unmodifiableMap(maps);
@@ -146,15 +142,22 @@ public class Memory_LocalDB implements LocalDBProvider {
     }
 
     @LocalDB.WriteOperation
-    public boolean remove(final LocalDB.DB db, final String key)
+    public boolean putIfAbsent(final LocalDB.DB db, final String key, final String value)
             throws LocalDBException {
         opertationPreCheck();
 
         final Map<String, String> map = maps.get(db);
-        return null != map.remove(key);
+        final String oldValue = map.putIfAbsent(key, value);
+        return oldValue == null;
     }
 
-    public void returnIterator(final LocalDB.DB db) throws LocalDBException {
+    @LocalDB.WriteOperation
+    public boolean remove(final LocalDB.DB db, final String key)
+            throws LocalDBException {
+        opertationPreCheck();
+
+        final Map<String, String> map = maps.get(db);
+        return null != map.remove(key);
     }
 
     public int size(final LocalDB.DB db)
@@ -190,7 +193,7 @@ public class Memory_LocalDB implements LocalDBProvider {
     }
 
 
-    private class DbIterator<K> implements LocalDB.LocalDBIterator<String> {
+    private class DbIterator implements LocalDB.LocalDBIterator<String> {
         private final Iterator<String> iterator;
 
         private DbIterator(final LocalDB.DB db) {

src/main/java/password/pwm/util/localdb/WorkQueueProcessor.java

@@ -456,11 +456,23 @@ public class WorkQueueProcessor<W extends Serializable> {
     @Getter
     @Builder
     public static class Settings implements Serializable {
+        @Builder.Default
         private int maxEvents = 1000;
+
+        @Builder.Default
         private int preThreads = 0;
+
+        @Builder.Default
         private TimeDuration maxSubmitWaitTime = new TimeDuration(5, TimeUnit.SECONDS);
+
+        @Builder.Default
+
         private TimeDuration retryInterval = new TimeDuration(30, TimeUnit.SECONDS);
+        @Builder.Default
+
         private TimeDuration retryDiscardAge = new TimeDuration(1, TimeUnit.HOURS);
+
+        @Builder.Default
         private TimeDuration maxShutdownWaitTime = new TimeDuration(30, TimeUnit.SECONDS);
     }
 

src/main/java/password/pwm/util/localdb/Xodus_LocalDB.java

@@ -311,6 +311,21 @@ public class Xodus_LocalDB implements LocalDBProvider {
         });
     }
 
+    @LocalDB.WriteOperation
+    public boolean putIfAbsent(final LocalDB.DB db, final String key, final String value) throws LocalDBException {
+        checkStatus(true);
+        return environment.computeInTransaction(transaction -> {
+            final ByteIterable k = bindMachine.keyToEntry(key);
+            final ByteIterable v = bindMachine.valueToEntry(value);
+            final Store store = getStore(db);
+            final ByteIterable existingValue = store.get(transaction, k);
+            if (existingValue != null) {
+                return false;
+            }
+            return store.put(transaction,k,v);
+        });
+    }
+
     @Override
     public boolean remove(final LocalDB.DB db, final String key) throws LocalDBException {
         checkStatus(true);

src/main/java/password/pwm/util/operations/cr/DbCrOperator.java

@@ -69,7 +69,7 @@ public class DbCrOperator implements CrOperator {
         }
 
         try {
-            final DatabaseAccessor databaseAccessor = pwmApplication.getDatabaseAccessor();
+            final DatabaseAccessor databaseAccessor = pwmApplication.getDatabaseService().getAccessor();
             final String responseStringBlob = databaseAccessor.get(DatabaseTable.PWM_RESPONSES, userGUID);
             if (responseStringBlob != null && responseStringBlob.length() > 0) {
                 final ResponseSet userResponseSet = ChaiResponseSet.parseChaiResponseSetXML(responseStringBlob, theUser);
@@ -109,7 +109,7 @@ public class DbCrOperator implements CrOperator {
         }
 
         try {
-            final DatabaseAccessor databaseAccessor = pwmApplication.getDatabaseAccessor();
+            final DatabaseAccessor databaseAccessor = pwmApplication.getDatabaseService().getAccessor();
             databaseAccessor.remove(DatabaseTable.PWM_RESPONSES, userGUID);
             LOGGER.info("cleared responses for user " + theUser.getEntryDN() + " in remote database");
         } catch (DatabaseException e) {
@@ -140,15 +140,11 @@ public class DbCrOperator implements CrOperator {
                     responseInfoBean.getCsIdentifier()
             );
 
-            final DatabaseAccessor databaseAccessor = pwmApplication.getDatabaseAccessor();
+            final DatabaseAccessor databaseAccessor = pwmApplication.getDatabaseService().getAccessor();
             databaseAccessor.put(DatabaseTable.PWM_RESPONSES, userGUID, responseSet.stringValue());
             LOGGER.info("saved responses for " + theUser.getEntryDN() + " in remote database (key=" + userGUID + ")");
         } catch (ChaiException e) {
-            final ErrorInformation errorInfo = new ErrorInformation(PwmError.ERROR_WRITING_RESPONSES, "unexpected error saving responses for " + theUser.getEntryDN() + " in remote database: " + e.getMessage());
-            final PwmUnrecoverableException pwmOE = new PwmUnrecoverableException(errorInfo);
-            LOGGER.error(errorInfo.toDebugStr());
-            pwmOE.initCause(e);
-            throw pwmOE;
+            throw PwmUnrecoverableException.fromChaiException(e);
         } catch (DatabaseException e) {
             final ErrorInformation errorInfo = new ErrorInformation(PwmError.ERROR_WRITING_RESPONSES, "unexpected error saving responses for " + theUser.getEntryDN() + " in remote database: " + e.getMessage());
             final PwmUnrecoverableException pwmOE = new PwmUnrecoverableException(errorInfo);

src/main/java/password/pwm/ws/server/rest/RestAppDataServer.java

@@ -35,7 +35,7 @@ import password.pwm.config.PwmSetting;
 import password.pwm.config.option.SelectableContextMode;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
-import password.pwm.error.PwmOperationalException;
+import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.http.IdleTimeoutCalculator;
 import password.pwm.http.PwmRequest;
@@ -225,7 +225,7 @@ public class RestAppDataServer extends AbstractRestServer {
             for (final RecordType recordType : RecordType.values()) {
                 returnData.put(recordType.toString(),restRequestBean.getPwmApplication().getIntruderManager().getRecords(recordType, max));
             }
-        } catch (PwmOperationalException e) {
+        } catch (PwmException e) {
             final ErrorInformation errorInfo = new ErrorInformation(PwmError.ERROR_UNKNOWN,e.getMessage());
             return RestResultBean.fromError(errorInfo, restRequestBean).asJsonResponse();
         }

src/main/resources/password/pwm/AppProperty.properties

@@ -62,6 +62,9 @@ configGuide.idleTimeoutSeconds=3600
 configManager.zipDebug.maxLogLines=100000
 configManager.zipDebug.maxLogSeconds=30
 db.jdbcLoadStrategy=AppPathFileLoader,Classpath
+db.connections.max=5
+db.connections.timeoutMs=30000
+db.connections.watchdogFrequencySeconds=30
 download.filename.statistics.csv=Statistics.csv
 download.filename.reportSummary.csv=UserReportSummary.csv
 download.filename.reportRecords.csv=UserReportRecords.csv
@@ -172,7 +175,7 @@ ldap.search.parallel.threadMax=50
 ldap.oracle.postTempPasswordUseCurrentTime=false
 localdb.aggressiveCompact.enabled=false
 localdb.implementation=password.pwm.util.localdb.Xodus_LocalDB
-localdb.initParameters=00
+localdb.initParameters=
 localdb.logWriter.bufferSize=500
 localdb.logWriter.maxBufferWaitMs=60000
 localdb.logWriter.maxTrimSize=5001

src/main/resources/password/pwm/i18n/PwmSetting.properties

@@ -459,7 +459,7 @@ Setting_Description_locale.cookie.age=Specify the duration of time to remember a
 Setting_Description_logoutAfterPasswordChange=Enable this option to force users to log out (and send them to the logoutURL) after a password change.<br/><br/>In most cases, leave this option enabled (default), especially if you are using some type of single sign-on service.
 Setting_Description_network.allowMultiIPSession=Enable this option to allow @PwmAppName@ to access a single HTTP session from different source IP addresses.  Some load balancing or proxy network infrastructures might require this, but in most cases disable it.  Especially since typical sessions are very short, there is not a practical reason for a user to access the same session from multiple client addresses.
 Setting_Description_network.ip.permittedRange=Enable this option to have @PwmAppName@ only permit connections originating from the specified IP address ranges.  If disabled (default), @PwmAppName@ permits any source IP address. <p>Supported range specifications are\:<p><ul><li>Full IPv4 address, such as <b>12.34.56.78</b></li><li>Full IPv6 address, such as <b>2001\:18e8\:3\:171\:218\:8bff\:fe2a\:56a4</b></li><li>Partial IPv4 address, such as <b>12.34</b> (which matches any IP addres starting <b>12.34</b></li><li>IPv4 network/netmask, such as <b>18.25.0.0/255.255.0.0</b></li><li>IPv4 or IPv6 CIDR slash notation, such as <b>18.25.0.0/16</b> or <b>2001\:18e8\:3\:171\:\:/64</b></li></ul>
-Setting_Description_network.requiredHttpHeaders=<p>Add any required HTTP header name and value pairs.  If specified, any HTTP request sent to the server must honor these headers.  This feature is useful if you have a security gateway and wish to only allow sessions from the gateway.</p><p>The settings must be in "name\=value" format.</p>
+Setting_Description_network.requiredHttpHeaders=<p>If specified, any HTTP/S request sent to this @PwmAppName@ application server must include these headers.  This feature is useful if you have an upstream security gateway, proxy or web server and wish to only allow sessions from the gateway, and deny direct access to this @PwmAppName@ application server from clients.</p><p>The settings must be in <code>name\=value</code> format.  If the upstream security gateway, proxy or web server is not setting these name/value headers, you will no longer be able to access this @PwmAppName@ application server.</p><p><b>WARNING:</b>If the client you are using to access this server is not setting the headers configured here, this @PwmAppName@ server will become inaccessible.</p>
 Setting_Description_network.reverseDNS.enable=Enable this option to have @PwmAppName@ use its reverse DNS system to record the hostname of the client.  In some cases this can cause performance issues so you can disable it if you do not requrie it.
 Setting_Description_newUser.createContext=Specify the LDAP context where @PwmAppName@ creates new users.  You can use macros in this setting.  @PwmAppName@ uses the default LDAP profile for new user creation.
 Setting_Description_newUser.deleteOnFail=Enable this option to have @PwmAppName@ delete the new user account if the creation fails for some reason.  It deletes the (potentially partially-created) "broken" account in LDAP.