storage library updates

pom.xml

@@ -353,7 +353,12 @@
         <dependency>
             <groupId>org.mapdb</groupId>
             <artifactId>mapdb</artifactId>
-            <version>1.0.5</version>
+            <version>3.0.0-M2</version>
+        </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <version>1.3.176</version>
         </dependency>
         <dependency>
             <groupId>net.glxn</groupId>

src/main/java/password/pwm/PwmConstants.java

@@ -95,8 +95,6 @@ public abstract class PwmConstants {
 
     public static final String APPLICATION_PATH_INFO_FILE = readPwmConstantsBundle("applicationPathInfoFile");
 
-    public static final int LOCALDB_LOGGER_MAX_QUEUE_SIZE = Integer.parseInt(readPwmConstantsBundle("pwmDBLoggerMaxQueueSize"));
-    public static final int LOCALDB_LOGGER_MAX_DIRTY_BUFFER_MS = Integer.parseInt(readPwmConstantsBundle("pwmDBLoggerMaxDirtyBufferMS"));
     public static final boolean ENABLE_EULA_DISPLAY = Boolean.parseBoolean(readPwmConstantsBundle("enableEulaDisplay"));
     public static final boolean TRIAL_MODE = Boolean.parseBoolean(readPwmConstantsBundle("trial"));
     public static final int TRIAL_MAX_AUTHENTICATIONS = 100;

src/main/java/password/pwm/PwmEnvironment.java

@@ -32,10 +32,7 @@ import password.pwm.util.JsonUtil;
 import password.pwm.util.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.io.Serializable;
+import java.io.*;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.util.*;
@@ -460,6 +457,7 @@ public class PwmEnvironment implements Serializable {
                     lock = f.tryLock();
                     if (lock != null) {
                         LOGGER.debug("obtained file lock on file " + lockfile.getAbsolutePath());
+                        writeLockFileContents(lockfile);
                     } else {
                         LOGGER.debug("unable to obtain file lock on file " + lockfile.getAbsolutePath());
                     }
@@ -469,6 +467,29 @@ public class PwmEnvironment implements Serializable {
             }
         }
 
+        void writeLockFileContents(final File file) {
+            FileWriter fileWriter = null;
+            try {
+                final Properties props = new Properties();
+                props.put("timestamp", PwmConstants.DEFAULT_DATETIME_FORMAT.format(new Date()));
+                props.put("applicationPath",PwmEnvironment.this.getApplicationPath() == null ? "n/a" : PwmEnvironment.this.getApplicationPath().getAbsolutePath());
+                props.put("configurationFile", PwmEnvironment.this.getConfigurationFile() == null ? "n/a" : PwmEnvironment.this.getConfigurationFile().getAbsolutePath());
+                final String comment = PwmConstants.PWM_APP_NAME + " file lock";
+                fileWriter = new FileWriter(lockfile);
+                props.store(new FileWriter(file, false), comment);
+            } catch (IOException e) {
+                LOGGER.error("unable to write contents of application lock file: " + e.getMessage());
+            } finally {
+                if (fileWriter != null) {
+                    try {
+                        fileWriter.close();
+                    } catch (IOException e) {
+                        LOGGER.error("unable to close contents of application lock file: " + e.getMessage());
+                    }
+                }
+            }
+        }
+
         public void releaseFileLock() {
             if (lock != null && lock.isValid()) {
                 try {

src/main/java/password/pwm/health/HealthMessage.java

@@ -76,6 +76,9 @@ public enum HealthMessage {
     LocalDB_BAD                             (HealthStatus.WARN,     HealthTopic.LocalDB),
     LocalDB_NEW                             (HealthStatus.WARN,     HealthTopic.LocalDB),
     LocalDB_CLOSED                          (HealthStatus.WARN,     HealthTopic.LocalDB),
+    LocalDBLogger_NOTOPEN                   (HealthStatus.CAUTION,  HealthTopic.LocalDB),
+    LocalDBLogger_HighRecordCount           (HealthStatus.CAUTION,  HealthTopic.LocalDB),
+    LocalDBLogger_OldRecordPresent          (HealthStatus.CAUTION,  HealthTopic.LocalDB),
     ServiceClosed_LocalDBUnavail            (HealthStatus.CAUTION,  HealthTopic.Application),
     ServiceClosed_AppReadOnly               (HealthStatus.CAUTION,  HealthTopic.Application),
     SMS_SendFailure                         (HealthStatus.WARN,     HealthTopic.SMS),

src/main/java/password/pwm/http/HttpEventManager.java

@@ -33,6 +33,8 @@ import javax.servlet.http.HttpSession;
 import javax.servlet.http.HttpSessionActivationListener;
 import javax.servlet.http.HttpSessionEvent;
 import javax.servlet.http.HttpSessionListener;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 /**
  * Servlet event listener, defined in web.xml
@@ -87,6 +89,8 @@ public class HttpEventManager implements
 
     public void contextInitialized(final ServletContextEvent servletContextEvent)
     {
+        Logger.getLogger("org.glassfish.jersey").setLevel(Level.SEVERE);
+
         if (null != servletContextEvent.getServletContext().getAttribute(PwmConstants.CONTEXT_ATTR_CONTEXT_MANAGER)) {
             LOGGER.warn("notice, previous servlet ContextManager exists");
         }

src/main/java/password/pwm/http/filter/GZIPFilter.java

@@ -88,7 +88,7 @@ public class GZIPFilter implements Filter {
             pwmApplication = ContextManager.getPwmApplication((HttpServletRequest) servletRequest);
             return Boolean.parseBoolean(pwmApplication.getConfig().readAppProperty(AppProperty.HTTP_ENABLE_GZIP));
         } catch (PwmUnrecoverableException e) {
-            LOGGER.trace("unable to read http-gzip app-property, defaulting to non-gzip: " + e.getMessage());
+            //LOGGER.trace("unable to read http-gzip app-property, defaulting to non-gzip: " + e.getMessage());
         }
         return false;
     }

src/main/java/password/pwm/http/servlet/configmanager/ConfigManagerLocalDBServlet.java

@@ -125,7 +125,7 @@ public class ConfigManagerLocalDBServlet extends AbstractPwmServlet {
         try {
             final int bufferSize = Integer.parseInt(pwmRequest.getConfig().readAppProperty(AppProperty.HTTP_DOWNLOAD_BUFFER_SIZE));
             final OutputStream bos = new BufferedOutputStream(resp.getOutputStream(),bufferSize);
-            localDBUtility.exportLocalDB(bos, LOGGER.asAppendable(PwmLogLevel.DEBUG, pwmRequest.getSessionLabel()), false);
+            localDBUtility.exportLocalDB(bos, LOGGER.asAppendable(PwmLogLevel.DEBUG, pwmRequest.getSessionLabel()), true);
             LOGGER.debug(pwmRequest, "completed localDBExport process in " + TimeDuration.fromCurrent(startTime).asCompactString());
         } catch (Exception e) {
             LOGGER.error(pwmRequest, "error downloading export localdb: " + e.getMessage());

src/main/java/password/pwm/http/servlet/configmanager/DebugItemGenerator.java

@@ -365,12 +365,11 @@ public class DebugItemGenerator {
             }
 
             {
-                final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
-                final CSVPrinter csvPrinter = Helper.makeCsvPrinter(byteArrayOutputStream);
+                final CSVPrinter csvPrinter = Helper.makeCsvPrinter(outputStream);
                 {
                     final List<String> headerRow = new ArrayList<>();
-                    headerRow.add("Filename");
                     headerRow.add("Filepath");
+                    headerRow.add("Filename");
                     headerRow.add("Last Modified");
                     headerRow.add("Size");
                     headerRow.add("sha1sum");
@@ -378,15 +377,14 @@ public class DebugItemGenerator {
                 }
                 for (final FileSystemUtility.FileSummaryInformation fileSummaryInformation : fileSummaryInformations) {
                     final List<String> dataRow = new ArrayList<>();
-                    dataRow.add(fileSummaryInformation.getFilename());
                     dataRow.add(fileSummaryInformation.getFilepath());
+                    dataRow.add(fileSummaryInformation.getFilename());
                     dataRow.add(PwmConstants.DEFAULT_DATETIME_FORMAT.format(fileSummaryInformation.getModified()));
                     dataRow.add(String.valueOf(fileSummaryInformation.getSize()));
                     dataRow.add(fileSummaryInformation.getSha1sum());
                     csvPrinter.printRecord(dataRow);
                 }
                 csvPrinter.flush();
-                outputStream.write(byteArrayOutputStream.toByteArray());
             }
         }
     }
@@ -446,8 +444,7 @@ public class DebugItemGenerator {
             final StoredConfigurationImpl storedConfiguration = ConfigManagerServlet.readCurrentConfiguration(pwmRequest);
             final LDAPPermissionCalculator ldapPermissionCalculator = new LDAPPermissionCalculator(storedConfiguration);
 
-            final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
-            final CSVPrinter csvPrinter = Helper.makeCsvPrinter(byteArrayOutputStream);
+            final CSVPrinter csvPrinter = Helper.makeCsvPrinter(outputStream);
             {
                 final List<String> headerRow = new ArrayList<>();
                 headerRow.add("Attribute");
@@ -468,7 +465,6 @@ public class DebugItemGenerator {
                 csvPrinter.printRecord(dataRow);
             }
             csvPrinter.flush();
-            outputStream.write(byteArrayOutputStream.toByteArray());
         }
     }
 

src/main/java/password/pwm/util/FileSystemUtility.java

@@ -71,7 +71,7 @@ public class FileSystemUtility {
         }
         return new FileSummaryInformation(
                 file.getName(),
-                file.getAbsolutePath(),
+                file.getParentFile().getAbsolutePath(),
                 new Date(file.lastModified()),
                 file.length(),
                 SecureEngine.hash(file, PwmHashAlgorithm.SHA1)

src/main/java/password/pwm/util/PwmServletContextListener.java

@@ -1,21 +0,0 @@
-package password.pwm.util;
-
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.servlet.ServletContextEvent;
-import javax.servlet.ServletContextListener;
-
-public class PwmServletContextListener implements ServletContextListener {
-
-    @Override
-    public void contextInitialized(ServletContextEvent sce) {
-        // Doing this here so the level will be set before any calls are made to LogManager or Logger.
-        Logger.getLogger("org.glassfish.jersey").setLevel(Level.SEVERE);
-    }
-
-    @Override
-    public void contextDestroyed(ServletContextEvent sce) {
-    }
-
-}

src/main/java/password/pwm/util/StringUtil.java

@@ -119,7 +119,7 @@ public abstract class StringUtil {
     }
 
     public static Map<String, String> convertStringListToNameValuePair(final Collection<String> input, final String separator) {
-        if (input == null) {
+        if (input == null || input.isEmpty()) {
             return Collections.emptyMap();
         }
 
@@ -128,10 +128,14 @@ public abstract class StringUtil {
             if (loopStr != null && separator != null && loopStr.contains(separator)) {
                 final int separatorLocation = loopStr.indexOf(separator);
                 final String key = loopStr.substring(0, separatorLocation);
-                final String value = loopStr.substring(separatorLocation + separator.length(), loopStr.length());
-                returnMap.put(key, value);
+                if (!key.trim().isEmpty()) {
+                    final String value = loopStr.substring(separatorLocation + separator.length(), loopStr.length());
+                    returnMap.put(key, value);
+                }
             } else {
-                returnMap.put(loopStr, "");
+                if (!loopStr.trim().isEmpty()) {
+                    returnMap.put(loopStr, "");
+                }
             }
         }
 
@@ -282,4 +286,21 @@ public abstract class StringUtil {
 
         return chunks.toArray(new String[numOfChunks]);
     }
+
+    public static String mapToString(final Map<String,String> map, final String keyValueSeparator, final String recordSeparator) {
+        final StringBuilder sb = new StringBuilder();
+        for (final Iterator<String> iterator = map.keySet().iterator(); iterator.hasNext(); ) {
+            final String key = iterator.next();
+            final String value = map.get(key);
+            if (key != null && value != null && !key.trim().isEmpty() && !value.trim().isEmpty()) {
+                sb.append(key.trim());
+                sb.append(keyValueSeparator);
+                sb.append(value.trim());
+                if (iterator.hasNext()) {
+                    sb.append(recordSeparator);
+                }
+            }
+        }
+        return sb.toString();
+    }
 }

src/main/java/password/pwm/util/localdb/AbstractJDBC_LocalDB.java

@@ -171,16 +171,6 @@ public abstract class AbstractJDBC_LocalDB implements LocalDBProvider {
             LOCK.writeLock().unlock();
         }
 
-        try {
-            LOCK.writeLock().lock();
-            DriverManager.deregisterDriver(driver);
-            driver = null;
-        } catch (SQLException e) {
-            LOGGER.error("unable to de-register sql driver: " + e.getMessage());
-        } finally {
-            LOCK.writeLock().unlock();
-        }
-
         LOGGER.debug("closed");
     }
 

src/main/java/password/pwm/util/localdb/Berkeley_LocalDB.java

@@ -53,6 +53,15 @@ public class Berkeley_LocalDB implements LocalDBProvider {
     private final static int CLOSE_RETRY_SECONDS = 120;
     private final static int ITERATOR_LIMIT = 100;
 
+    private static final Map<String,String> DEFAULT_INIT_PARAMS;
+    static {
+        final Map<String,String> defaultInitParams = new HashMap<>();
+        defaultInitParams.put("je.maxMemory","50000000");
+        defaultInitParams.put("je.log.fileMax","10000000");
+        defaultInitParams.put("je.cleaner.minUtilization","60");
+        DEFAULT_INIT_PARAMS = Collections.unmodifiableMap(defaultInitParams);
+    }
+
     private final static TupleBinding<String> STRING_TUPLE = TupleBinding.getPrimitiveBinding(String.class);
 
     private Environment environment;
@@ -101,9 +110,14 @@ public class Berkeley_LocalDB implements LocalDBProvider {
         environmentConfig.setTransactional(IS_TRANSACTIONAL);
         environmentConfig.setReadOnly(readonly);
 
+        final Map<String,String> effectiveProperties = new HashMap<>(DEFAULT_INIT_PARAMS);
+        if (initProps != null) {
+            effectiveProperties.putAll(initProps);
+        }
+
         if (initProps != null) {
-            for (final String key : initProps.keySet()) {
-                environmentConfig.setConfigParam(key, initProps.get(key));
+            for (final String key : effectiveProperties.keySet()) {
+                environmentConfig.setConfigParam(key, effectiveProperties.get(key));
             }
         }
 

src/main/java/password/pwm/util/localdb/Derby_LocalDB.java

@@ -30,11 +30,9 @@ import password.pwm.util.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
 import java.io.File;
-import java.sql.CallableStatement;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
+import java.sql.*;
 import java.util.Map;
+import java.util.Properties;
 
 /**
  * Apache Derby Wrapper for {@link LocalDB} interface.   Uses a single table per DB, with
@@ -50,6 +48,8 @@ public class Derby_LocalDB extends AbstractJDBC_LocalDB {
 
     private static final String OPTION_KEY_RECLAIM_SPACE = "reclaimAllSpace";
 
+    private Driver driver;
+
     Derby_LocalDB()
             throws Exception
     {
@@ -68,7 +68,10 @@ public class Derby_LocalDB extends AbstractJDBC_LocalDB {
             throws SQLException
     {
         try {
-            DriverManager.getConnection("jdbc:derby:;shutdown=true");
+            if (driver != null) {
+                driver.connect("jdbc:derby:;shutdown=true", new Properties());
+            }
+
         } catch (SQLException e) {
             if ("XJ015".equals(e.getSQLState())) {
                 LOGGER.trace("Derby shutdown succeeded. SQLState=" + e.getSQLState() + ", message=" + e.getMessage());
@@ -76,7 +79,22 @@ public class Derby_LocalDB extends AbstractJDBC_LocalDB {
                 throw e;
             }
         }
-        connection.close();
+
+        try {
+            if (driver != null) {
+                DriverManager.deregisterDriver(driver);
+            }
+        } catch (Exception e) {
+            LOGGER.error("error while de-registering derby driver: " + e.getMessage());
+        }
+
+        driver = null;
+
+        try {
+            connection.close();
+        } catch (Exception e) {
+            LOGGER.error("error while closing derby connection: " + e.getMessage());
+        }
     }
 
     @Override
@@ -90,8 +108,8 @@ public class Derby_LocalDB extends AbstractJDBC_LocalDB {
         final String connectionURL = baseConnectionURL + ";create=true";
 
         try {
-            Class.forName(driverClasspath).newInstance(); //load driver.
-            final Connection connection = DriverManager.getConnection(connectionURL);
+            driver = (Driver)Class.forName(driverClasspath).newInstance(); //load driver.
+            final Connection connection = driver.connect(connectionURL, new Properties());
             connection.setAutoCommit(false);
 
             if (initOptions != null && initOptions.containsKey(OPTION_KEY_RECLAIM_SPACE) && Boolean.parseBoolean(initOptions.get(OPTION_KEY_RECLAIM_SPACE))) {
@@ -146,7 +164,7 @@ public class Derby_LocalDB extends AbstractJDBC_LocalDB {
 
     private void reclaimSpace(final Connection dbConnection, final LocalDB.DB db)
     {
-        if (getStatus() != LocalDB.Status.OPEN || readOnly) {
+        if (readOnly) {
             return;
         }
 

src/main/java/password/pwm/util/localdb/H2_LocalDB.java

@@ -24,20 +24,31 @@ package password.pwm.util.localdb;
 
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
+import password.pwm.util.StringUtil;
 import password.pwm.util.logging.PwmLogger;
 
 import java.io.File;
-import java.sql.Connection;
-import java.sql.Driver;
-import java.sql.DriverManager;
-import java.sql.SQLException;
+import java.sql.*;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.Properties;
 
 public class H2_LocalDB extends AbstractJDBC_LocalDB {
 
     private static final PwmLogger LOGGER = PwmLogger.forClass(H2_LocalDB.class, true);
 
     private static final String H2_CLASSPATH = "org.h2.Driver";
+    private static final Map<String,String> DEFAULT_INIT_PARAMS;
+    static {
+        final Map<String,String> defaultInitParams = new HashMap<>();
+        defaultInitParams.put("DB_CLOSE_ON_EXIT","FALSE");
+        defaultInitParams.put("COMPRESS","TRUE");
+        //defaultInitParams.put("TRACE_LEVEL_FILE","2");
+        DEFAULT_INIT_PARAMS = Collections.unmodifiableMap(defaultInitParams);
+    }
+
+    private Driver driver;
 
     H2_LocalDB()
             throws Exception
@@ -56,8 +67,15 @@ public class H2_LocalDB extends AbstractJDBC_LocalDB {
     closeConnection(final Connection connection)
             throws SQLException
     {
-        connection.close();
-        DriverManager.getConnection("jdbc:h2:;shutdown=true");
+        try {
+            connection.close();
+            if (driver != null) {
+                DriverManager.deregisterDriver(driver);
+                driver = null;
+            }
+        } catch (Exception e) {
+            LOGGER.error("error during H2 shutdown: " + e.getMessage());
+        }
     }
 
     @Override
@@ -66,14 +84,14 @@ public class H2_LocalDB extends AbstractJDBC_LocalDB {
             final String driverClasspath,
             final Map<String,String> initParams
     ) throws LocalDBException {
-        final String filePath = databaseDirectory.getAbsolutePath() + File.separator + "h2-db";
-        final String baseConnectionURL = "jdbc:h2:" + filePath;
+        final String filePath = databaseDirectory.getAbsolutePath() + File.separator + "localdb-h2";
+        final String baseConnectionURL = "jdbc:h2:" + filePath + ";" + makeInitStringParams(initParams);
 
         try {
-            final Driver driver = (Driver)Class.forName(H2_CLASSPATH).newInstance();
-            DriverManager.registerDriver(driver);
-            final Connection connection = DriverManager.getConnection(baseConnectionURL);
-            connection.setAutoCommit(false);
+            driver = (Driver)Class.forName(H2_CLASSPATH).newInstance();
+            final Properties connectionProps = new Properties();
+            final Connection connection = driver.connect(baseConnectionURL, connectionProps);
+            connection.setAutoCommit(true);
             return connection;
         } catch (Throwable e) {
             final String errorMsg = "error opening DB: " + e.getMessage();
@@ -81,4 +99,11 @@ public class H2_LocalDB extends AbstractJDBC_LocalDB {
             throw new LocalDBException(new ErrorInformation(PwmError.ERROR_LOCALDB_UNAVAILABLE,errorMsg));
         }
     }
+
+    private static String makeInitStringParams(final Map<String,String> initParams) {
+        final Map<String,String> params = new HashMap<>();
+        params.putAll(DEFAULT_INIT_PARAMS);
+        params.putAll(initParams);
+        return StringUtil.mapToString(params,"=",";");
+    }
 }

src/main/java/password/pwm/util/localdb/LocalDBFactory.java

@@ -71,6 +71,7 @@ public class LocalDBFactory {
             initParameters = StringUtil.convertStringListToNameValuePair(Arrays.asList(initStrings.split(";;;")), "=");
         }
 
+
         final LocalDBProvider dbProvider = createInstance(className);
         LOGGER.debug("initializing " + className + " localDBProvider instance");
 

src/main/java/password/pwm/util/localdb/LocalDBStoredQueue.java

@@ -328,7 +328,7 @@ public class
 
     public Iterator<String> descendingIterator() {
         try {
-            return new InnerIterator<String>(internalQueue, false);
+            return new InnerIterator(internalQueue, false);
         } catch (LocalDBException e) {
             throw new IllegalStateException(e);
         }
@@ -338,7 +338,7 @@ public class
 
     public Iterator<String> iterator() {
         try {
-            return new InnerIterator<String>(internalQueue, true);
+            return new InnerIterator(internalQueue, true);
         } catch (LocalDBException e) {
             throw new IllegalStateException(e);
         }
@@ -378,7 +378,7 @@ public class
 
 // -------------------------- INNER CLASSES --------------------------
 
-    private class InnerIterator<K> implements Iterator {
+    private class InnerIterator implements Iterator<String> {
         private Position position;
         private final InternalQueue internalQueue;
         private final boolean first;

src/main/java/password/pwm/util/localdb/LocalDBUtility.java

@@ -60,9 +60,10 @@ public class LocalDBUtility {
             throw new PwmOperationalException(PwmError.ERROR_UNKNOWN,"outputFileStream for exportLocalDB cannot be null");
         }
 
-        writeStringToOut(debugOutput,"counting records in LocalDB...");
+
         final int totalLines;
         if (showLineCount) {
+            writeStringToOut(debugOutput,"counting records in LocalDB...");
             exportLineCounter = 0;
             for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
                 if (loopDB.isBackup()) {
@@ -85,9 +86,9 @@ public class LocalDBUtility {
                 if (showLineCount) {
                     final float percentComplete = (float) exportLineCounter / (float) totalLines;
                     final String percentStr = DecimalFormat.getPercentInstance().format(percentComplete);
-                    writeStringToOut(debugOutput," exported " + exportLineCounter + " records, " + percentStr + " complete");
+                    writeStringToOut(debugOutput,"exported " + exportLineCounter + " records, " + percentStr + " complete");
                 } else {
-                    writeStringToOut(debugOutput," exported " + exportLineCounter + " records");
+                    writeStringToOut(debugOutput,"exported " + exportLineCounter + " records");
                 }
             }
         },30 * 1000, 30 * 1000);
@@ -110,6 +111,7 @@ public class LocalDBUtility {
                     } finally {
                         localDBIterator.close();
                     }
+                    csvPrinter.flush();
                 }
             }
         } catch (IOException e) {
@@ -182,12 +184,14 @@ public class LocalDBUtility {
         this.prepareForImport();
 
         importLineCounter = 0;
-        if (totalLines > 0) writeStringToOut(out, " total lines: " + totalLines);
+        if (totalLines > 0) {
+            writeStringToOut(out, "total lines: " + totalLines);
+        }
 
         writeStringToOut(out, "beginning restore...");
 
         final Date startTime = new Date();
-        final TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(900, 50, 50 * 1000);
+        final TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(100, 50, 5 * 1000);
 
         final Map<LocalDB.DB,Map<String,String>> transactionMap = new HashMap<>();
         for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
@@ -202,9 +206,9 @@ public class LocalDBUtility {
                 if (totalLines > 0) {
                     final ProgressInfo progressInfo = new ProgressInfo(startTime, totalLines, importLineCounter);
                     writeStringToOut(out,
-                            " " + progressInfo.debugOutput() + ", transactionSize=" + transactionCalculator.getTransactionSize());
+                            progressInfo.debugOutput() + ", transactionSize=" + transactionCalculator.getTransactionSize());
                 } else {
-                    writeStringToOut(out, " linesImported=" + importLineCounter);
+                    writeStringToOut(out, "linesImported=" + importLineCounter + ", transactionSize=" + transactionCalculator.getTransactionSize());
                 }
             }
         }, 0, 30 * 1000);

src/main/java/password/pwm/util/localdb/MapDB_LocalDB.java

@@ -23,6 +23,7 @@
 package password.pwm.util.localdb;
 
 import org.mapdb.DBMaker;
+import org.mapdb.Serializer;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.util.TimeDuration;
@@ -134,7 +135,10 @@ public class MapDB_LocalDB implements LocalDBProvider {
             LOCK.writeLock().lock();
             this.dbDirectory = dbDirectory;
             final File dbFile = new File(dbDirectory.getAbsolutePath() + File.separator + FILE_NAME);
-            recman = DBMaker.newFileDB(dbFile).make();
+            recman = DBMaker.fileDB(dbFile.getAbsolutePath()).make();
+
+            LOGGER.debug("beginning DB compact");
+            recman.getStore().compact();
 
             LOGGER.info("LocalDB opened in " + TimeDuration.fromCurrent(startTime).asCompactString());
             status = LocalDB.Status.OPEN;
@@ -270,9 +274,11 @@ public class MapDB_LocalDB implements LocalDBProvider {
     private static Map<String, String> openHTree(
             final String name,
             final org.mapdb.DB recman
-    )
-            throws IOException {
-        return recman.getTreeMap(name);
+    ) {
+        return recman.hashMap(name)
+                .keySerializer(Serializer.STRING)
+                .valueSerializer(Serializer.STRING)
+                .open();
     }
 
 // -------------------------- INNER CLASSES --------------------------

src/main/java/password/pwm/util/logging/LocalDBLogger.java

@@ -23,14 +23,14 @@
 package password.pwm.util.logging;
 
 import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
 import password.pwm.config.option.DataStorageMethod;
 import password.pwm.error.PwmException;
+import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
-import password.pwm.health.HealthStatus;
-import password.pwm.health.HealthTopic;
 import password.pwm.svc.PwmService;
-import password.pwm.util.*;
+import password.pwm.util.FileSystemUtility;
+import password.pwm.util.Helper;
+import password.pwm.util.TimeDuration;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBException;
 import password.pwm.util.localdb.LocalDBStoredQueue;
@@ -38,7 +38,9 @@ import password.pwm.util.localdb.LocalDBStoredQueue;
 import java.io.Serializable;
 import java.text.NumberFormat;
 import java.util.*;
-import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
@@ -53,92 +55,83 @@ public class LocalDBLogger implements PwmService {
 
     private final static PwmLogger LOGGER = PwmLogger.forClass(LocalDBLogger.class);
 
-    private final static int MINIMUM_MAXIMUM_EVENTS = 100;
 
-    private volatile long tailTimestampMs = -1L;
-    private long lastQueueFlushTimestamp = System.currentTimeMillis();
 
     private final LocalDB localDB;
-    private final Settings settings;
-    private final Queue<PwmLogEvent> eventQueue = new LinkedBlockingQueue<>(PwmConstants.LOCALDB_LOGGER_MAX_QUEUE_SIZE);
+    private final LocalDBLoggerSettings settings;
     private final LocalDBStoredQueue localDBListQueue;
+    private ScheduledExecutorService executorService;
 
     private volatile STATUS status = STATUS.NEW;
-    private volatile boolean writerThreadActive = false;
+    private volatile boolean cleanOnWrite = false;
     private boolean hasShownReadError = false;
 
-    private final TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(
-            2049, // transaction time goal
-            5,
-            PwmConstants.LOCALDB_LOGGER_MAX_QUEUE_SIZE
-    );
 
 // --------------------------- CONSTRUCTORS ---------------------------
 
-    public LocalDBLogger(final PwmApplication pwmApplication, final LocalDB localDB, final Settings settings)
+    public LocalDBLogger(final PwmApplication pwmApplication, final LocalDB localDB, final LocalDBLoggerSettings settings)
             throws LocalDBException
     {
         final long startTime = System.currentTimeMillis();
         status = STATUS.OPENING;
-        this.settings = settings.copy();
+        this.settings = settings;
         this.localDB = localDB;
         this.localDBListQueue = LocalDBStoredQueue.createLocalDBStoredQueue(pwmApplication,
                 this.localDB, LocalDB.DB.EVENTLOG_EVENTS);
 
-        if (settings.maxEvents == 0) {
+        if (settings.getMaxEvents() == 0) {
             LOGGER.info("maxEvents set to zero, clearing LocalDBLogger history and LocalDBLogger will remain closed");
             localDBListQueue.clear();
             throw new IllegalArgumentException("maxEvents=0, will remain closed");
         }
 
-        if (settings.getMaxEvents() < MINIMUM_MAXIMUM_EVENTS) {
-            LOGGER.warn("maxEvents less than required minimum of " + MINIMUM_MAXIMUM_EVENTS + ", resetting maxEvents=" + MINIMUM_MAXIMUM_EVENTS);
-            settings.setMaxEvents(MINIMUM_MAXIMUM_EVENTS);
-        }
-
         if (localDB == null) {
             throw new IllegalArgumentException("LocalDB is not available");
         }
 
-        this.tailTimestampMs = readTailTimestamp();
         status = STATUS.OPEN;
 
-        { // start the writer thread
-            final Thread writerThread = new Thread(new WriterThread());
-            writerThread.setName(Helper.makeThreadName(pwmApplication, LocalDBLogger.class));
-            writerThread.setDaemon(true);
-            writerThread.start();
-        }
+        executorService = Executors.newSingleThreadScheduledExecutor(
+                Helper.makePwmThreadFactory(
+                        Helper.makeThreadName(pwmApplication, this.getClass()) + "-",
+                        true
+                ));
+
+        executorService.scheduleAtFixedRate(new CleanupTask(),0,1, TimeUnit.MINUTES);
 
         final TimeDuration timeDuration = TimeDuration.fromCurrent(startTime);
         LOGGER.info("open in " + timeDuration.asCompactString() + ", " + debugStats());
     }
 
 
-    private long readTailTimestamp() {
+    public Date getTailDate() {
         final PwmLogEvent loopEvent;
+        if (localDBListQueue.isEmpty()) {
+            return null;
+        }
         try {
             loopEvent = readEvent(localDBListQueue.getLast());
             if (loopEvent != null) {
                 final Date tailDate = loopEvent.getDate();
                 if (tailDate != null) {
-                    return tailDate.getTime();
+                    return tailDate;
                 }
             }
         } catch (Exception e) {
             LOGGER.error("unexpected error attempting to determine tail event timestamp: " + e.getMessage());
         }
 
-        return -1;
+        return null;
     }
 
 
     private String debugStats() {
         final StringBuilder sb = new StringBuilder();
         sb.append("events=").append(localDBListQueue.size());
-        sb.append(", tailAge=").append(TimeDuration.fromCurrent(tailTimestampMs).asCompactString());
+        final Date tailAge = getTailDate();
+        sb.append(", tailAge=").append(tailAge == null ? "n/a" : TimeDuration.fromCurrent(tailAge).asCompactString());
         sb.append(", maxEvents=").append(settings.getMaxEvents());
-        sb.append(", maxAge=").append(settings.getMaxAgeMs() > 1 ? new TimeDuration(settings.getMaxAgeMs()).asCompactString() : "none");
+        sb.append(", maxAge=").append(settings.getMaxAge().asCompactString());
         sb.append(", localDBSize=").append(Helper.formatDiskSize(FileSystemUtility.getFileDirectorySize(localDB.getFileLocation())));
         return sb.toString();
     }
@@ -149,108 +142,47 @@ public class LocalDBLogger implements PwmService {
         LOGGER.debug("LocalDBLogger closing... (" + debugStats() + ")");
         status = STATUS.CLOSED;
 
-        { // wait for the writer to die.
-            final long startTime = System.currentTimeMillis();
-            while (writerThreadActive && TimeDuration.fromCurrent(startTime).isShorterThan(60 * 1000)) {
-                Helper.pause(1000);
-                if (writerThreadActive) {
-                    LOGGER.debug("waiting for writer thread to close...");
-                }
-            }
-
-            if (writerThreadActive) {
-                LOGGER.warn("logger thread still open");
-            }
-        }
-
-        if (!writerThreadActive) { // try to close the queue
-            final long startTime = System.currentTimeMillis();
-            while (!eventQueue.isEmpty() && TimeDuration.fromCurrent(startTime).isShorterThan(30 * 1000)) {
-                flushQueue();
-            }
-        }
-
-        if (!eventQueue.isEmpty()) {
-            LOGGER.warn("abandoning " + eventQueue.size() + " events waiting to be written to LocalDB log");
+        if (executorService != null) {
+            executorService.shutdown();
+            executorService = null;
         }
 
         LOGGER.debug("LocalDBLogger close completed (" + debugStats() + ")");
     }
 
-    private int flushQueue() {
-        final List<PwmLogEvent> tempList = new ArrayList<>();
-        final int desiredTransactionSize = transactionCalculator.getTransactionSize();
-        PwmLogEvent nextEvent = eventQueue.poll();
-        while (nextEvent != null && tempList.size() < desiredTransactionSize) {
-            tempList.add(nextEvent);
-            nextEvent = eventQueue.poll();
-        }
-
-        if (!tempList.isEmpty()) {
-            doWrite(tempList);
-            lastQueueFlushTimestamp = System.currentTimeMillis();
-            //System.out.println("flush size: " + tempList.size());
-        }
-
-        return tempList.size();
-    }
-
-    private void doWrite(final Collection<PwmLogEvent> events) {
-        final List<String> transactions = new ArrayList<>();
-        try {
-            for (final PwmLogEvent event : events) {
-                final String encodedString = event.toEncodedString();
-                if (encodedString.length() < LocalDB.MAX_VALUE_LENGTH) {
-                    transactions.add(encodedString);
-                }
-            }
-
-            localDBListQueue.addAll(transactions);
-        } catch (Exception e) {
-            LOGGER.error("error writing to localDBLogger: " + e.getMessage(), e);
-        }
-    }
-
-    public Date getTailDate() {
-        return new Date(tailTimestampMs);
-    }
-
     public int getStoredEventCount() {
         return localDBListQueue.size();
     }
 
-    public int getPendingEventCount() {
-        return eventQueue.size();
-    }
-
     private int determineTailRemovalCount() {
+        final int MAX_TAIL_REMOVAL = 501;
+
         final int currentItemCount = localDBListQueue.size();
 
         // must keep at least one position populated
-        if (currentItemCount <= 1) {
+        if (currentItemCount <= LocalDBLoggerSettings.MINIMUM_MAXIMUM_EVENTS) {
             return 0;
         }
 
         // purge excess events by count
         if (currentItemCount > settings.getMaxEvents()) {
-            return currentItemCount - settings.getMaxEvents();
+            return Math.min(MAX_TAIL_REMOVAL, currentItemCount - settings.getMaxEvents());
         }
 
         // purge the tail if it is missing or has invalid timestamp
-        if (tailTimestampMs == -1) {
+        final Date tailTimestamp = getTailDate();
+        if (tailTimestamp == null) {
             return 1;
         }
 
         // purge excess events by age;
-        if (settings.getMaxAgeMs() > 0 && tailTimestampMs > 0) {
-            final TimeDuration tailAge = TimeDuration.fromCurrent(tailTimestampMs);
-            if (tailAge.isLongerThan(settings.getMaxAgeMs())) {
-                final long maxRemovalPercentageOfSize = getStoredEventCount() / 500;
-                if (maxRemovalPercentageOfSize > 100) {
-                    return 500;
-                } else {
-                    return 1;
-                }
+        final TimeDuration tailAge = TimeDuration.fromCurrent(tailTimestamp);
+        if (tailAge.isLongerThan(settings.getMaxAge())) {
+            final long maxRemovalPercentageOfSize = getStoredEventCount() / MAX_TAIL_REMOVAL;
+            if (maxRemovalPercentageOfSize > 100) {
+                return MAX_TAIL_REMOVAL;
+            } else {
+                return 1;
             }
         }
         return 0;
@@ -322,14 +254,6 @@ public class LocalDBLogger implements PwmService {
         return new SearchResults(localDBListQueue.iterator(), searchParameters);
     }
 
-    public TimeDuration getDirtyQueueTime() {
-        if (eventQueue.isEmpty()) {
-            return TimeDuration.ZERO;
-        }
-
-        return TimeDuration.fromCurrent(lastQueueFlushTimestamp);
-    }
-
     private PwmLogEvent readEvent(final String value) {
         try {
             return PwmLogEvent.fromEncodedString(value);
@@ -411,75 +335,39 @@ public class LocalDBLogger implements PwmService {
         return eventMatchesParams;
     }
 
-
     public void writeEvent(final PwmLogEvent event) {
         if (status == STATUS.OPEN) {
             if (settings.getMaxEvents() > 0) {
-                boolean success = eventQueue.offer(event);
-                if (!success) { // if event queue isn't overflowed, simply add event to write queue
-                    final long startEventTime = System.currentTimeMillis();
-                    while (TimeDuration.fromCurrent(startEventTime).isShorterThan(30 * 1000) && !success) {
-                        Helper.pause(100);
-                        success = eventQueue.offer(event);
-                    }
-                    if (!success) {
-                        LOGGER.warn("discarding event due to full write queue: " + event.toString());
+                try {
+                    if (cleanOnWrite) {
+                        localDBListQueue.removeLast();
                     }
+                    localDBListQueue.add(event.toEncodedString());
+                } catch (Exception e) {
+                    LOGGER.error("error writing to localDBLogger: " + e.getMessage(), e);
                 }
             }
         }
     }
 
-// -------------------------- INNER CLASSES --------------------------
-
-    private class WriterThread implements Runnable {
+    private class CleanupTask implements Runnable {
         public void run() {
-            LOGGER.debug("writer thread open");
-            writerThreadActive = true;
             try {
-                loop();
-            } catch (Exception e) {
-                LOGGER.fatal("unexpected fatal error during LocalDBLogger log event writing; logging to localDB will be suspended.", e);
-            }
-            writerThreadActive = false;
-        }
-
-        private void loop() throws LocalDBException {
-            LOGGER.debug("starting writer thread loop");
-
-            while (status == STATUS.OPEN) {
-                long startLoopTime = System.currentTimeMillis();
-                final int writesDone = flushQueue();
-
-                final int purgeCount = determineTailRemovalCount();
-                int purgesDone = 0;
-                if (purgeCount > 0) {
-                    int removalCount = purgeCount > transactionCalculator.getTransactionSize() + 1 ? transactionCalculator.getTransactionSize() + 1 : purgeCount;
-                    localDBListQueue.removeLast(removalCount);
-                    tailTimestampMs = readTailTimestamp();
-                    purgesDone = removalCount;
-                }
-
-                final int totalWork = writesDone + purgesDone;
-                if (totalWork == 0) {
-                    Helper.pause(settings.getMaxDirtyQueueAgeMs());
-                    if (settings.isDevDebug()) {
-                        LOGGER.trace("no work on last cycle, sleeping for " + new TimeDuration(settings.getMaxDirtyQueueAgeMs()).asCompactString() + " queue size=" + getPendingEventCount());
-                    }
-                } else if (totalWork < 5) {
-                    Helper.pause(settings.getMaxDirtyQueueAgeMs());
-                    if (settings.isDevDebug()) {
-                        LOGGER.trace("minor work on last cycle, sleeping for " + new TimeDuration(settings.getMaxDirtyQueueAgeMs()).asCompactString() + " queue size=" + getPendingEventCount());
-                    }
-                } else {
-                    final TimeDuration txnDuration = TimeDuration.fromCurrent(startLoopTime);
-                    transactionCalculator.recordLastTransactionDuration(txnDuration);
-                    if (settings.isDevDebug()) {
-                        LOGGER.trace("tick writes=" + writesDone+ ", purges=" + purgesDone + ", queue=" + getPendingEventCount() + ", txnCalcSize=" + transactionCalculator.getTransactionSize() + ", txnDuration=" + txnDuration.getTotalMilliseconds());
+                int cleanupCount = 1;
+                while (cleanupCount > 0 && (status == STATUS.OPEN  && localDBListQueue.getPwmDB().status() == LocalDB.Status.OPEN)) {
+                    cleanupCount = determineTailRemovalCount();
+                    if (cleanupCount > 0) {
+                        cleanOnWrite = true;
+                        final Date startTime = new Date();
+                        localDBListQueue.removeLast(cleanupCount);
+                        final TimeDuration purgeTime = TimeDuration.fromCurrent(startTime);
+                        Helper.pause(Math.max(Math.min(purgeTime.getMilliseconds(),100),1000));
                     }
                 }
+            } catch (Exception e) {
+                LOGGER.fatal("unexpected error during LocalDBLogger log event cleanup: " + e.getMessage(), e);
             }
-            LOGGER.debug("writer thread exiting");
+            cleanOnWrite = false;
         }
     }
 
@@ -575,19 +463,21 @@ public class LocalDBLogger implements PwmService {
         final List<HealthRecord> healthRecords = new ArrayList<>();
 
         if (status != STATUS.OPEN) {
-            healthRecords.add(new HealthRecord(HealthStatus.WARN, HealthTopic.Application, "LocalDBLogger is not open, status is " + status.toString()));
+            healthRecords.add(HealthRecord.forMessage(HealthMessage.LocalDBLogger_NOTOPEN, status.toString()));
             return healthRecords;
         }
 
         final int eventCount = getStoredEventCount();
         if (eventCount > settings.getMaxEvents() + 5000) {
-            healthRecords.add(new HealthRecord(HealthStatus.CAUTION, HealthTopic.Application, "Record count of " + NumberFormat.getInstance().format(eventCount) + " records, is more than the configured maximum of " + NumberFormat.getInstance().format(settings.getMaxEvents())));
+            final NumberFormat numberFormat = NumberFormat.getInstance();
+            healthRecords.add(HealthRecord.forMessage(HealthMessage.LocalDBLogger_HighRecordCount,numberFormat.format(eventCount),numberFormat.format(settings.getMaxEvents())));
         }
 
         final Date tailDate = getTailDate();
         final TimeDuration timeDuration = TimeDuration.fromCurrent(tailDate);
-        if (timeDuration.isLongerThan(settings.getMaxAgeMs())) { // older than max age
-            healthRecords.add(new HealthRecord(HealthStatus.CAUTION, HealthTopic.Application, "Oldest record is " + timeDuration.asCompactString() + ", configured maximum is " + new TimeDuration(settings.getMaxAgeMs()).asCompactString()));
+        final TimeDuration maxTimeDuration = settings.getMaxAge().add(TimeDuration.HOUR);
+        if (timeDuration.isLongerThan(maxTimeDuration)) { // older than max age + 1h
+            healthRecords.add(HealthRecord.forMessage(HealthMessage.LocalDBLogger_OldRecordPresent, timeDuration.asCompactString(), maxTimeDuration.asCompactString()));
         }
 
         return healthRecords;
@@ -605,10 +495,7 @@ public class LocalDBLogger implements PwmService {
         numberFormat.setMaximumFractionDigits(3);
         numberFormat.setMinimumFractionDigits(3);
 
-        final StringBuilder sb = new StringBuilder();
-        sb.append(this.getStoredEventCount()).append(" / ").append(maxEvents);
-        sb.append(" (").append(numberFormat.format(percentFull)).append("%)");
-        return sb.toString();
+        return String.valueOf(this.getStoredEventCount()) + " / " + maxEvents + " (" + numberFormat.format(percentFull) + "%)";
     }
 
     public ServiceInfo serviceInfo()
@@ -616,55 +503,5 @@ public class LocalDBLogger implements PwmService {
         return new ServiceInfo(Collections.singletonList(DataStorageMethod.LOCALDB));
     }
 
-    public static class Settings implements Serializable {
-        private int maxEvents = 100 * 1000;
-        private long maxAgeMs = (long)4 * (long)7 * (long)24 * (long)60 * (long)60 * (long)1000; // 4 weeks
-        private long maxDirtyQueueAgeMs = PwmConstants.LOCALDB_LOGGER_MAX_DIRTY_BUFFER_MS;
-        private boolean devDebug = false;
-
-        public int getMaxEvents()
-        {
-            return maxEvents;
-        }
-
-        public void setMaxEvents(int maxEvents)
-        {
-            this.maxEvents = maxEvents;
-        }
-
-        public long getMaxAgeMs()
-        {
-            return maxAgeMs;
-        }
-
-        public void setMaxAgeMs(long maxAgeMs)
-        {
-            this.maxAgeMs = maxAgeMs;
-        }
-
-        public long getMaxDirtyQueueAgeMs()
-        {
-            return maxDirtyQueueAgeMs;
-        }
-
-        public void setMaxDirtyQueueAgeMs(long maxDirtyQueueAgeMs)
-        {
-            this.maxDirtyQueueAgeMs = maxDirtyQueueAgeMs;
-        }
-
-        public boolean isDevDebug()
-        {
-            return devDebug;
-        }
-
-        public void setDevDebug(boolean devDebug)
-        {
-            this.devDebug = devDebug;
-        }
-
-        private Settings copy() {
-            return JsonUtil.deserialize(JsonUtil.serialize(this),this.getClass());
-        }
-    }
 }
 

src/main/java/password/pwm/util/logging/LocalDBLoggerSettings.java

@@ -0,0 +1,74 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2016 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.util.logging;
+
+import password.pwm.config.Configuration;
+import password.pwm.config.PwmSetting;
+import password.pwm.util.TimeDuration;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+public class LocalDBLoggerSettings implements Serializable {
+    final static int MINIMUM_MAXIMUM_EVENTS = 100;
+    final static TimeDuration MINIMUM_MAX_AGE = TimeDuration.HOUR;
+
+    private final int maxEvents;
+    private final TimeDuration maxAge;
+    private final Set<Flag> flags;
+
+    public enum Flag {
+        DevDebug,
+    }
+
+    public LocalDBLoggerSettings(int maxEvents, TimeDuration maxAge, Set<Flag> flags) {
+        this.maxEvents = maxEvents < 1 ? 0 : Math.max(MINIMUM_MAXIMUM_EVENTS, maxEvents);
+        this.maxAge = maxAge == null || maxAge.isShorterThan(MINIMUM_MAX_AGE) ? MINIMUM_MAX_AGE : maxAge;
+        this.flags = flags == null ? Collections.<Flag>emptySet() : Collections.unmodifiableSet(flags);
+    }
+
+    public int getMaxEvents() {
+        return maxEvents;
+    }
+
+    public TimeDuration getMaxAge() {
+        return maxAge;
+    }
+
+    public Set<Flag> getFlags() {
+        return flags;
+    }
+
+    public static LocalDBLoggerSettings fromConfiguration(final Configuration configuration) {
+        final Set<Flag> flags = new HashSet<>();
+        if (configuration.isDevDebugMode()) {
+            flags.add(Flag.DevDebug);
+        }
+        final int maxEvents = (int) configuration.readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_EVENTS);
+        final long maxAgeMS = 1000 * configuration.readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_AGE);
+        final TimeDuration maxAge = new TimeDuration(maxAgeMS);
+        return new LocalDBLoggerSettings(maxEvents, maxAge, flags);
+    }
+}

src/main/java/password/pwm/util/logging/PwmLogManager.java

@@ -32,7 +32,6 @@ import password.pwm.PwmApplication;
 import password.pwm.PwmApplicationMode;
 import password.pwm.PwmConstants;
 import password.pwm.config.Configuration;
-import password.pwm.config.PwmSetting;
 import password.pwm.util.FileSystemUtility;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBException;
@@ -185,9 +184,7 @@ public class PwmLogManager {
         final LocalDBLogger localDBLogger;
         final PwmLogLevel localDBLogLevel = pwmApplication.getConfig().getEventLogLocalDBLevel();
         try {
-            final int maxEvents = (int) pwmApplication.getConfig().readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_EVENTS);
-            final long maxAgeMS = 1000 * pwmApplication.getConfig().readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_AGE);
-            localDBLogger = initLocalDBLogger(localDB, maxEvents, maxAgeMS, pwmApplication);
+            localDBLogger = initLocalDBLogger(localDB, pwmApplication);
             if (localDBLogger != null) {
                 PwmLogger.setLocalDBLogger(localDBLogLevel, localDBLogger);
             }
@@ -216,16 +213,10 @@ public class PwmLogManager {
 
     static LocalDBLogger initLocalDBLogger(
             final LocalDB pwmDB,
-            final int maxEvents,
-            final long maxAgeMS,
             final PwmApplication pwmApplication
     ) {
-        final boolean devDebugMode = pwmApplication.getConfig().isDevDebugMode();
         try {
-            final LocalDBLogger.Settings settings = new LocalDBLogger.Settings();
-            settings.setMaxEvents(maxEvents);
-            settings.setMaxAgeMs(maxAgeMS);
-            settings.setDevDebug(devDebugMode);
+            final LocalDBLoggerSettings settings = LocalDBLoggerSettings.fromConfiguration(pwmApplication.getConfig());
             return new LocalDBLogger(pwmApplication, pwmDB, settings);
         } catch (LocalDBException e) {
             //nothing to do;

src/main/java/password/pwm/util/secure/SecureEngine.java

@@ -34,6 +34,8 @@ import javax.crypto.Cipher;
 import javax.crypto.Mac;
 import javax.crypto.SecretKey;
 import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
 import java.security.GeneralSecurityException;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
@@ -198,14 +200,27 @@ public class SecureEngine {
             final File file,
             final PwmHashAlgorithm hashAlgorithm
     )
-            throws IOException, PwmUnrecoverableException {
-        if (file == null || !file.exists()) {
-            return null;
-        }
+            throws IOException, PwmUnrecoverableException
+    {
         FileInputStream fileInputStream = null;
         try {
+            final MessageDigest messageDigest = MessageDigest.getInstance(hashAlgorithm.getAlgName());
             fileInputStream = new FileInputStream(file);
-            return hash(fileInputStream, hashAlgorithm);
+            final FileChannel fileChannel = fileInputStream.getChannel();
+            final ByteBuffer byteBuffer = ByteBuffer.allocateDirect(1024 * 16); // allocation in bytes - 1024, 2048, 4096, 8192
+
+            while (fileChannel.read(byteBuffer) > 0) {
+                byteBuffer.flip();
+                messageDigest.update(byteBuffer);
+                byteBuffer.clear();
+            }
+
+            return Helper.byteArrayToHexString(messageDigest.digest());
+
+        } catch (NoSuchAlgorithmException | IOException e) {
+            final String errorMsg = "unexpected error during hash operation: " + e.getMessage();
+            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_CRYPT_ERROR, errorMsg);
+            throw new PwmUnrecoverableException(errorInformation);
         } finally {
             if (fileInputStream != null) {
                 fileInputStream.close();

src/main/resources/password/pwm/AppProperty.properties

@@ -138,11 +138,11 @@ ldap.guid.pattern=@UUID@
 ldap.browser.maxEntries=1000
 ldap.search.paging.enable=auto
 ldap.search.paging.size=500
-localdb.compression.enabled=true
+localdb.compression.enabled=false
 localdb.decompression.enabled=true
 localdb.compression.minSize=1024
 localdb.implementation=password.pwm.util.localdb.Berkeley_LocalDB
-localdb.initParameters=je.maxMemory=10000000;;;je.log.fileMax=10000000;;;je.cleaner.minUtilization=70
+localdb.initParameters=
 macro.randomChar.maxLength=100
 macro.ldapAttr.maxLength=100
 naaf.id=41414141414141414141414141414141

src/main/resources/password/pwm/PwmConstants.properties

@@ -34,15 +34,13 @@ defaultBadPasswordAttempt=BADPASSWORDATTEMPT
 log.removedValue=*hidden*
 pwm.appName=PWM
 trial=false
-url.pwm-home=http://code.google.com/p/pwm
+url.pwm-home=https://github.com/pwm-project/pwm
 url.pwm-cloud=https://pwm-cloud.appspot.com
 versionCheckFrequencyMs=129600000
 versionCheckFailRetryMs=1800000
 statisticsPublishFrequencyMs=259200000
 paramName.token=token
 defaultConfigFilename=PwmConfiguration.xml
-pwmDBLoggerMaxQueueSize=50000
-pwmDBLoggerMaxDirtyBufferMS=500
 enableEulaDisplay=false
 databaseAccessor.keyLength=128
 missingVersionString=[Version Missing]

src/main/resources/password/pwm/i18n/Health.properties

@@ -67,6 +67,9 @@ HealthMessage_LocalDB_OK=LocalDB and related services are operating correctly
 HealthMessage_LocalDB_BAD=LocalDB is not online. Startup error: %1%
 HealthMessage_LocalDB_NEW=LocalDB status is NEW (loading) state, until LocalDB loads, statistics, online logging, wordlists and other features are disabled
 HealthMessage_LocalDB_CLOSED=LocalDB is CLOSED, statistics, online logging, wordlists and other features are disabled.  Check logs to troubleshoot
+HealthMessage_LocalDBLogger_NOTOPEN=LocalDBLogger is not open, status is %1%
+HealthMessage_LocalDBLogger_HighRecordCount=LocalDBLogger event log record count of %1% records, is more than the configured maximum of %2%.  Excess records are being purged.
+HealthMessage_LocalDBLogger_OldRecordPresent=Oldest LocalDBLogger event log record is %1%, configured maximum is %2%.  Excess records are being purged.
 HealthMessage_ServiceClosed_LocalDBUnavail=unable to start %1% service, LocalDB is not available
 HealthMessage_ServiceClosed_AppReadOnly=unable to start %1% service, application is in read-only mode
 Message_SMS_SendFailure=Unable to send sms due to error: %1%

src/main/webapp/WEB-INF/jsp/admin-dashboard.jsp

@@ -425,16 +425,6 @@
                                 <%= numberFormat.format(dashboard_pwmApplication.getAuditManager().vaultSize()) %>
                             </td>
                         </tr>
-                        <tr>
-                            <td class="key">
-                                Log Events in Write Queue
-                            </td>
-                            <td>
-                                <%= dashboard_pwmApplication.getLocalDBLogger() != null
-                                        ? numberFormat.format(dashboard_pwmApplication.getLocalDBLogger().getPendingEventCount())
-                                        : JspUtility.getMessage(pageContext, Display.Value_NotApplicable) %>
-                            </td>
-                        </tr>
                         <tr>
                             <td class="key">
                                 Log Events in LocalDB
@@ -443,17 +433,6 @@
                                 <%= dashboard_pwmApplication.getLocalDBLogger().sizeToDebugString() %>
                             </td>
                         </tr>
-                        <tr>
-                            <td class="key">
-                                Oldest Log Event in Write Queue
-                            </td>
-                            <td>
-                                <%= dashboard_pwmApplication.getLocalDBLogger() != null
-                                        ? dashboard_pwmApplication.getLocalDBLogger().getDirtyQueueTime().asCompactString()
-                                        : JspUtility.getMessage(pageContext, Display.Value_NotApplicable)
-                                %>
-                            </td>
-                        </tr>
                         <tr>
                             <td class="key">
                                 Oldest Log Event in LocalDB

src/main/webapp/WEB-INF/web.xml

@@ -28,9 +28,6 @@
     <display-name>PWM Password Management</display-name>
     <!-- <distributable/> Clustering/Session replication is not supported -->
     <description>Password Management Servlet</description>
-    <listener>
-        <listener-class>password.pwm.util.PwmServletContextListener</listener-class>
-    </listener>
     <context-param>
         <description>
             Explicit location of application working directory. If a relative path is specified, it is relative to the

src/main/webapp/public/resources/js/uilibrary.js

@@ -388,7 +388,7 @@ UILibrary.uploadFileDialog = function(options) {
         xhr.upload.addEventListener('progress',progressFunction,false);
         xhr.upload.onprogress = progressFunction;
         xhr.open("POST", uploadUrl, true);
-        fd.append("uploadFile", files[0]);
+        fd.append("fileUpload", files[0]);
         xhr.send(fd);
         PWM_GLOBAL['inhibitHealthUpdate'] = true;
         PWM_MAIN.IdleTimeoutHandler.cancelCountDownTimer();

src/test/java/password/pwm/tests/LocalDBLoggerTest.java

@@ -22,13 +22,12 @@
 
 package password.pwm.tests;
 
-import junit.framework.Assert;
 import junit.framework.TestCase;
-import password.pwm.AppProperty;
 import password.pwm.PwmConstants;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.svc.stats.EventRateMeter;
 import password.pwm.util.FileSystemUtility;
 import password.pwm.util.Helper;
 import password.pwm.util.Percent;
@@ -36,15 +35,16 @@ import password.pwm.util.TimeDuration;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBFactory;
 import password.pwm.util.logging.LocalDBLogger;
+import password.pwm.util.logging.LocalDBLoggerSettings;
 import password.pwm.util.logging.PwmLogEvent;
 import password.pwm.util.logging.PwmLogLevel;
 import password.pwm.util.secure.PwmRandom;
-import password.pwm.svc.stats.EventRateMeter;
 
 import java.io.File;
 import java.math.RoundingMode;
-import java.text.SimpleDateFormat;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 public class LocalDBLoggerTest extends TestCase {
 
@@ -53,11 +53,10 @@ public class LocalDBLoggerTest extends TestCase {
     private LocalDBLogger localDBLogger;
     private LocalDB localDB;
     private Configuration config;
-    private int maxSize;
 
 
-    private int eventsAdded;
-    private int eventsRemaining;
+    private final AtomicInteger eventsAdded = new AtomicInteger(0);
+    private final AtomicInteger eventsRemaining = new AtomicInteger(0);
     final StringBuffer randomValue = new StringBuffer();
     final Random random = new Random();
 
@@ -82,10 +81,9 @@ public class LocalDBLoggerTest extends TestCase {
         );
 
 
-        final LocalDBLogger.Settings settings = new LocalDBLogger.Settings();
-        settings.setMaxEvents((int)reader.getConfiguration().readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_EVENTS));
-        settings.setMaxAgeMs(reader.getConfiguration().readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_AGE) * (long)1000);
-        settings.setDevDebug(Boolean.parseBoolean(reader.getConfiguration().readAppProperty(AppProperty.LOGGING_DEV_OUTPUT)));
+        final int maxEvents = (int)reader.getConfiguration().readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_EVENTS);
+        final long maxAgeMs = reader.getConfiguration().readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_AGE) * (long)1000;
+        final LocalDBLoggerSettings settings = new LocalDBLoggerSettings(maxEvents, new TimeDuration(maxAgeMs), Collections.<LocalDBLoggerSettings.Flag>emptySet());
 
         localDBLogger = new LocalDBLogger(null, localDB, settings);
 
@@ -99,54 +97,46 @@ public class LocalDBLoggerTest extends TestCase {
 
     public void testBulkAddEvents() {
         final int startingSize = localDBLogger.getStoredEventCount();
-        eventsRemaining = BULK_EVENT_COUNT;
-        eventsAdded = 0;
+        eventsRemaining.addAndGet(BULK_EVENT_COUNT);
         final Timer timer = new Timer();
         timer.scheduleAtFixedRate(new DebugOutputTimerTask(),5 * 1000, 5 * 1000);
 
-        for (int loopCount = 0; loopCount < 1; loopCount++) {
+        for (int loopCount = 0; loopCount < 5; loopCount++) {
             final Thread populatorThread = new PopulatorThread();
             populatorThread.start();
         }
 
-        while (eventsRemaining > 0) {
+        while (eventsRemaining.get() > 0) {
             Helper.pause(5);
         }
 
         final long startWaitTime = System.currentTimeMillis();
-        while (System.currentTimeMillis() - startWaitTime < 30 * 1000 && localDBLogger.getPendingEventCount() > 0) {
+        while (TimeDuration.fromCurrent(startWaitTime).isShorterThan(new TimeDuration(10, TimeUnit.HOURS)) && eventsRemaining.get() > 0) {
             Helper.pause(500);
         }
         Helper.pause(5000);
 
-        if (startingSize + BULK_EVENT_COUNT >= maxSize) {
-            Assert.assertEquals(localDBLogger.getStoredEventCount(), maxSize);
-        } else {
-            Assert.assertEquals(localDBLogger.getStoredEventCount(), startingSize + BULK_EVENT_COUNT);
-        }
         timer.cancel();
     }
 
+
     private class PopulatorThread extends Thread {
         public void run() {
             int loopCount = 3;
-            while (eventsRemaining > 0) {
-                while (localDBLogger.getPendingEventCount() >= PwmConstants.LOCALDB_LOGGER_MAX_QUEUE_SIZE - (loopCount + 1)) {
-                    Helper.pause(11);
-                }
+            while (eventsRemaining.get() > 0) {
                 final Collection<PwmLogEvent> events = makeEvents(loopCount);
                 for (final PwmLogEvent logEvent : events) {
                     localDBLogger.writeEvent(logEvent);
                     eventRateMeter.markEvents(1);
-                    eventsRemaining--;
-                    eventsAdded++;
+                    eventsRemaining.decrementAndGet();
+                    eventsAdded.incrementAndGet();
                 }
             }
         }
     }
 
     private Collection<PwmLogEvent> makeEvents(final int count) {
-        final Collection<PwmLogEvent> events = new ArrayList<PwmLogEvent>();
+        final Collection<PwmLogEvent> events = new ArrayList<>();
 
         for (int i = 0; i < count; i++) {
             events.add(makeEvent());
@@ -176,14 +166,13 @@ public class LocalDBLoggerTest extends TestCase {
             final long maxEvents = config.readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_EVENTS);
             final long eventCount = localDBLogger.getStoredEventCount();
             final Percent percent = new Percent(eventCount,maxEvents);
-            sb.append(new SimpleDateFormat("HH:mm:ss").format(new Date()));
+            sb.append(PwmConstants.DEFAULT_DATETIME_FORMAT.format(new Date()));
             sb.append(", added ").append(eventsAdded);
             sb.append(", db size: ").append(Helper.formatDiskSize(FileSystemUtility.getFileDirectorySize(localDB.getFileLocation())));
             sb.append(", events: ").append(localDBLogger.getStoredEventCount()).append("/").append(maxEvents);
             sb.append(" (").append(percent.pretty(3)).append(")");
             sb.append(", free space: ").append(Helper.formatDiskSize(
                     FileSystemUtility.diskSpaceRemaining(localDB.getFileLocation())));
-            sb.append(", pending: ").append(localDBLogger.getPendingEventCount());
             sb.append(", eps: ").append(eventRateMeter.readEventRate().setScale(0, RoundingMode.UP));
             System.out.println(sb);
         }

src/test/java/password/pwm/tests/TestHelper.properties

@@ -1,25 +0,0 @@
-#
-# Password Management Servlets (PWM)
-# http://code.google.com/p/pwm/
-#
-# Copyright (c) 2006-2009 Novell, Inc.
-# Copyright (c) 2009-2012 The PWM Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-
-applicationPath=/home/amb/t/appPath
-localDBPath=/home/amb/t/appPath/LocalDB
-configurationFile=/home/amb/t/appPath/PwmConfiguration.xml