add xodus and h2mv localdb implementations

pom.xml

@@ -193,13 +193,13 @@
                 <groupId>org.jasig.mojo.jspc</groupId>
                 <artifactId>jspc-maven-plugin</artifactId>
                 <version>2.0.0</version>
-				<dependencies>
-				    <dependency>
+                <dependencies>
+                    <dependency>
                         <groupId>org.jasig.mojo.jspc</groupId>
                         <artifactId>jspc-compiler-tomcat7</artifactId>
                         <version>2.0.0</version>
                     </dependency>
-				</dependencies>
+                </dependencies>
                 <configuration>
                     <includeInProject>false</includeInProject>
                 </configuration>
@@ -228,7 +228,7 @@
                                 <fileset>
                                     <directory>target/classes/jsp</directory>
                                 </fileset>
-                              </filesets>
+                            </filesets>
                         </configuration>
                     </execution>
                 </executions>
@@ -401,8 +401,8 @@
 
         <dependency>
             <groupId>com.github.ldapchai</groupId>
-            <version>0.6.8</version>
             <artifactId>ldapchai</artifactId>
+            <version>0.6.8</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
@@ -534,6 +534,11 @@
             <artifactId>UserAgentUtils</artifactId>
             <version>1.18</version>
         </dependency>
+        <dependency>
+            <groupId>org.jetbrains.xodus</groupId>
+            <artifactId>xodus-environment</artifactId>
+            <version>1.0-SNAPSHOT</version>
+        </dependency>
 
 
 
@@ -590,6 +595,13 @@
             <name>project</name>
             <url>file:///${project.basedir}/local-maven-repo</url>
         </repository>
+        <repository>
+            <id>maven-public</id> <!-- used by xodus -->
+            <url>https://oss.sonatype.org/content/groups/public</url>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
         <repository>
             <id>oracleReleases</id>
             <name>Oracle Released Java Packages</name>

src/main/java/password/pwm/AppProperty.java

@@ -131,6 +131,9 @@ public enum AppProperty {
     LOCALDB_COMPRESSION_MINSIZE                     ("localdb.compression.minSize"),
     LOCALDB_IMPLEMENTATION                          ("localdb.implementation"),
     LOCALDB_INIT_STRING                             ("localdb.initParameters"),
+    LOCALDB_LOGWRITER_BUFFER_SIZE                   ("localdb.logWriter.bufferSize"),
+    LOCALDB_LOGWRITER_MAX_BUFFER_WAIT_MS            ("localdb.logWriter.maxBufferWaitMs"),
+    LOCALDB_LOGWRITER_MAX_TRIM_SIZE                 ("localdb.logWriter.maxTrimSize"),
     MACRO_RANDOM_CHAR_MAX_LENGTH                    ("macro.randomChar.maxLength"),
     MACRO_LDAP_ATTR_CHAR_MAX_LENGTH                 ("macro.ldapAttr.maxLength"),
     NAAF_ID                                         ("naaf.id"),

src/main/java/password/pwm/util/localdb/H2MV_LocalDB.java

@@ -0,0 +1,180 @@
+package password.pwm.util.localdb;
+
+import org.h2.mvstore.MVStore;
+import password.pwm.util.Helper;
+import password.pwm.util.TimeDuration;
+import password.pwm.util.logging.PwmLogger;
+
+import java.io.File;
+import java.util.*;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class H2MV_LocalDB implements LocalDBProvider {
+    private static final PwmLogger LOGGER = PwmLogger.forClass(H2MV_LocalDB.class);
+
+    private MVStore mvStore;
+    private Map<LocalDB.DB,Map<String,String>> dbMaps = new HashMap<>();
+    private File fileLocation;
+    private volatile int writeTicks = 0;
+    private ScheduledExecutorService executorService;
+
+
+    @Override
+    public void init(File dbDirectory, Map<String, String> initParameters, Map<Parameter,String> parameters) throws LocalDBException {
+        LOGGER.trace("begin db open");
+        mvStore = new MVStore.Builder()
+                .fileName(dbDirectory + File.separator + "h2mv-localdb")
+                .open();
+        LOGGER.trace("open");
+
+        for (final LocalDB.DB db : LocalDB.DB.values()) {
+            LOGGER.trace("opening db " + db);
+            final Map<String,String> dbMap = mvStore.openMap(db.toString());
+            dbMaps.put(db,dbMap);
+        }
+
+        executorService = Executors.newSingleThreadScheduledExecutor(
+                Helper.makePwmThreadFactory(
+                        Helper.makeThreadName(null, this.getClass()) + "-",
+                        true
+                ));
+
+        compact(true);
+
+        this.fileLocation = dbDirectory;
+    }
+
+    @Override
+    public void close() throws LocalDBException {
+        mvStore.close();
+    }
+
+    @Override
+    public int size(LocalDB.DB db) throws LocalDBException {
+        return dbMaps.get(db).size();
+    }
+    @Override
+    public boolean contains(LocalDB.DB db, String key) throws LocalDBException {
+        return dbMaps.get(db).containsKey(key);
+    }
+
+    @Override
+    public String get(LocalDB.DB db, String key) throws LocalDBException {
+        return dbMaps.get(db).get(key);
+    }
+
+
+    @Override
+    public LocalDB.LocalDBIterator<String> iterator(final LocalDB.DB db) throws LocalDBException {
+        return new LocalDB.LocalDBIterator<String>() {
+
+            private Iterator<String> stringIterator = dbMaps.get(db).keySet().iterator();
+            @Override
+            public void close() {
+
+            }
+
+            @Override
+            public boolean hasNext() {
+                return stringIterator.hasNext();
+            }
+
+            @Override
+            public String next() {
+                return stringIterator.next();
+            }
+
+            @Override
+            public void remove() {
+                stringIterator.remove();
+            }
+        };
+    }
+
+    @Override
+    public void putAll(LocalDB.DB db, Map<String, String> keyValueMap) throws LocalDBException {
+        dbMaps.get(db).putAll(keyValueMap);
+        writeTicks++;
+        postWriteActivities();
+        mvStore.commit();
+    }
+
+    @Override
+    public boolean put(LocalDB.DB db, String key, String value) throws LocalDBException {
+        final String oldValue = dbMaps.get(db).put(key,value);
+        writeTicks++;
+        postWriteActivities();
+        return oldValue != null;
+    }
+
+    @Override
+    public boolean remove(LocalDB.DB db, String key) throws LocalDBException {
+        final String oldValue = dbMaps.get(db).remove(key);
+        writeTicks++;
+        postWriteActivities();
+        return oldValue != null;
+    }
+
+    @Override
+    public void removeAll(LocalDB.DB db, Collection<String> keys) throws LocalDBException {
+        dbMaps.get(db).keySet().removeAll(keys);
+        writeTicks++;
+        postWriteActivities();
+    }
+
+
+    @Override
+    public void truncate(LocalDB.DB db) throws LocalDBException {
+        dbMaps.get(db).clear();
+        writeTicks++;
+        compact(true);
+        postWriteActivities();
+    }
+
+    @Override
+    public File getFileLocation() {
+        return fileLocation;
+    }
+
+    @Override
+    public LocalDB.Status getStatus() {
+        return LocalDB.Status.OPEN;
+    }
+
+    private void postWriteActivities() {
+        mvStore.commit();
+        if (writeTicks > 10000) {
+            writeTicks = 0;
+            compact(false);
+        }
+    }
+
+    private void compact(boolean full) {
+        if (full) {
+            executorService.schedule(new Runnable() {
+                @Override
+                public void run() {
+                    LOGGER.trace("begin full compact");
+                    final Date startDate = new Date();
+                    mvStore.compactMoveChunks();
+                    LOGGER.trace("end full compact " + TimeDuration.fromCurrent(startDate).asCompactString());
+                }
+            }, 0, TimeUnit.MILLISECONDS);
+
+        } else {
+            executorService.schedule(new Runnable() {
+                @Override
+                public void run() {
+                    final int compactSize = 1024 * 1024 * 1024;
+                    final int compactPercent = 70;
+                    LOGGER.trace("begin compact");
+                    final Date startDate = new Date();
+                    mvStore.compact(compactPercent, compactSize);
+                    LOGGER.trace("end compact " + TimeDuration.fromCurrent(startDate).asCompactString());
+                }
+            }, 0, TimeUnit.MILLISECONDS);
+        }
+    }
+}

src/main/java/password/pwm/util/localdb/LocalDBFactory.java

@@ -75,7 +75,9 @@ public class LocalDBFactory {
             initParameters = StringUtil.convertStringListToNameValuePair(Arrays.asList(initStrings.split(";;;")), "=");
         }
 
-        final Map<LocalDBProvider.Parameter,String> parameters = makeParameterMap(pwmApplication.getConfig(), readonly);
+        final Map<LocalDBProvider.Parameter,String> parameters = pwmApplication == null
+                ? Collections.<LocalDBProvider.Parameter, String>emptyMap()
+                : makeParameterMap(pwmApplication.getConfig(), readonly);
         final LocalDBProvider dbProvider = createInstance(className);
         LOGGER.debug("initializing " + className + " localDBProvider instance");
 

src/main/java/password/pwm/util/localdb/Xodus_LocalDB.java

@@ -0,0 +1,338 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2016 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.util.localdb;
+
+import jetbrains.exodus.ByteIterable;
+import jetbrains.exodus.bindings.StringBinding;
+import jetbrains.exodus.env.*;
+import jetbrains.exodus.management.Statistics;
+import org.jetbrains.annotations.NotNull;
+import password.pwm.util.JsonUtil;
+import password.pwm.util.TimeDuration;
+import password.pwm.util.logging.PwmLogger;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+
+public class Xodus_LocalDB implements LocalDBProvider {
+    private static final PwmLogger LOGGER = PwmLogger.forClass(Xodus_LocalDB.class);
+
+    private Environment environment;
+    private Map<LocalDB.DB,Store> stores = new HashMap<>();
+    private File fileLocation;
+
+    private LocalDB.Status status = LocalDB.Status.NEW;
+
+    private final ReadWriteLock LOCK = new ReentrantReadWriteLock();
+
+    @Override
+    public void init(File dbDirectory, Map<String, String> initParameters, Map<Parameter,String> parameters) throws LocalDBException {
+        this.fileLocation = dbDirectory;
+
+        LOGGER.trace("begin environment open");
+        final Date startTime = new Date();
+
+        final EnvironmentConfig environmentConfig = new EnvironmentConfig();
+        environmentConfig.setLogDurableWrite(true);
+        environmentConfig.setGcEnabled(true);
+
+        for (final String key : initParameters.keySet()) {
+            final String value = initParameters.get(key);
+            LOGGER.trace("setting environment config key=" + key + ", value=" + value);
+            environmentConfig.setSetting(key,value);
+        }
+        environment = Environments.newInstance(dbDirectory.getAbsolutePath() + File.separator + "xodus", environmentConfig);
+        LOGGER.trace("environment open (" + TimeDuration.fromCurrent(startTime).asCompactString() + ")");
+
+        for (final LocalDB.DB db : LocalDB.DB.values()) {
+            stores.put(db, openStore(db));
+        }
+
+        status = LocalDB.Status.OPEN;
+
+        for (final LocalDB.DB db : LocalDB.DB.values()) {
+            LOGGER.trace("opened " + db + " with " + this.size(db) + " records");
+        }
+    }
+
+    @Override
+    public void close() throws LocalDBException {
+        LOCK.writeLock().lock();
+        try {
+            environment.close();
+            status = LocalDB.Status.CLOSED;
+            LOGGER.debug("closed");
+        } finally {
+            LOCK.writeLock().unlock();
+        }
+    }
+
+    @Override
+    public int size(final LocalDB.DB db) throws LocalDBException {
+        LOCK.readLock().lock();
+        try {
+            return environment.computeInReadonlyTransaction(new TransactionalComputable<Integer>() {
+                @Override
+                public Integer compute(@NotNull Transaction transaction) {
+                    return (int) stores.get(db).count(transaction);
+                }
+            });
+        } finally {
+            LOCK.readLock().unlock();
+        }
+    }
+    @Override
+    public boolean contains(LocalDB.DB db, String key) throws LocalDBException {
+        return get(db, key) != null;
+    }
+
+    @Override
+    public String get(final LocalDB.DB db, final String key) throws LocalDBException {
+        LOCK.readLock().lock();
+        try {
+            final Store store = stores.get(db);
+            return environment.computeInTransaction(new TransactionalComputable<String>() {
+                @Override
+                public String compute(@NotNull Transaction transaction) {
+                    final ByteIterable returnValue = store.get(transaction,StringBinding.stringToEntry(key));
+                    if (returnValue != null) {
+                        return StringBinding.entryToString(returnValue);
+                    }
+                    return null;
+                }
+            });
+        } finally {
+            LOCK.readLock().unlock();
+        }
+    }
+
+    @Override
+    public LocalDB.LocalDBIterator<String> iterator(final LocalDB.DB db) throws LocalDBException {
+        return new InnerIterator(db);
+    }
+
+    private class InnerIterator implements LocalDB.LocalDBIterator<String> {
+        final private Transaction transaction;
+        final private Cursor cursor;
+
+        private boolean closed;
+        private String nextValue = "";
+
+        InnerIterator(final LocalDB.DB db) {
+            this.transaction = environment.beginReadonlyTransaction();
+            this.cursor = stores.get(db).openCursor(transaction);
+            doNext();
+        }
+
+        private void doNext() {
+            try {
+                if (closed) {
+                    return;
+                }
+
+                if (!cursor.getNext()) {
+                    close();
+                    return;
+                }
+                final ByteIterable value = cursor.getKey();
+                if (value == null || value.getLength() == 0) {
+                    close();
+                    return;
+                }
+                final String decodedValue = StringBinding.entryToString(value);
+                if (decodedValue == null) {
+                    close();
+                    return;
+                }
+                nextValue = decodedValue;
+            } catch (Exception e) {
+                e.printStackTrace();
+                throw e;
+            }
+        }
+
+        @Override
+        public void close() {
+            if (closed) {
+                return;
+            }
+            cursor.close();
+            transaction.abort();
+            nextValue = null;
+            closed = true;
+        }
+
+        @Override
+        public boolean hasNext() {
+            return !closed && nextValue != null;
+        }
+
+        @Override
+        public String next() {
+            if (closed) {
+                return null;
+            }
+            final String value = nextValue;
+            doNext();
+            return value;
+        }
+
+        @Override
+        public void remove() {
+            throw new UnsupportedOperationException("remove not supported");
+        }
+    }
+
+    @Override
+    public void putAll(final LocalDB.DB db, final Map<String, String> keyValueMap) throws LocalDBException {
+        LOCK.readLock().lock();
+        try {
+            final Store store = stores.get(db);
+            environment.executeInTransaction(new TransactionalExecutable() {
+                @Override
+                public void execute(@NotNull Transaction transaction) {
+                    for (final String key : keyValueMap.keySet()) {
+                        final String value = keyValueMap.get(key);
+                        final ByteIterable k = StringBinding.stringToEntry(key);
+                        final ByteIterable v = StringBinding.stringToEntry(value);
+                        store.put(transaction,k,v);
+                    }
+                }
+            });
+        } finally {
+            LOCK.readLock().unlock();
+        }
+    }
+
+
+    @Override
+    public boolean put(final LocalDB.DB db, final String key, final String value) throws LocalDBException {
+        LOCK.readLock().lock();
+        try {
+            final Store store = stores.get(db);
+            return environment.computeInTransaction(new TransactionalComputable<Boolean>() {
+                @Override
+                public Boolean compute(@NotNull Transaction transaction) {
+                    final ByteIterable k = StringBinding.stringToEntry(key);
+                    final ByteIterable v = StringBinding.stringToEntry(value);
+                    return store.put(transaction,k,v);
+                }
+            });
+        } finally {
+            LOCK.readLock().unlock();
+        }
+    }
+
+    @Override
+    public boolean remove(final LocalDB.DB db, final String key) throws LocalDBException {
+        LOCK.readLock().lock();
+        try {
+            final Store store = stores.get(db);
+            return environment.computeInTransaction(new TransactionalComputable<Boolean>() {
+                @Override
+                public Boolean compute(@NotNull Transaction transaction) {
+                    return store.delete(transaction,StringBinding.stringToEntry(key));
+                }
+            });
+        } finally {
+            LOCK.readLock().unlock();
+        }
+    }
+
+    @Override
+    public void removeAll(final LocalDB.DB db, final Collection<String> keys) throws LocalDBException {
+        LOCK.readLock().lock();
+        try {
+            final Store store = stores.get(db);
+            environment.executeInTransaction(new TransactionalExecutable() {
+                @Override
+                public void execute(@NotNull Transaction transaction) {
+                    for (final String key : keys) {
+                        store.delete(transaction, StringBinding.stringToEntry(key));
+                    }
+                }
+            });
+        } finally {
+            LOCK.readLock().unlock();
+        }
+    }
+
+
+    @Override
+    public void truncate(final LocalDB.DB db) throws LocalDBException {
+        LOGGER.trace("being truncate of " + db.toString() + ", size=" + this.size(db));
+        final Date startDate = new Date();
+        LOCK.writeLock().lock();
+        try {
+            stores.remove(db);
+
+            environment.executeInTransaction(new TransactionalExecutable() {
+                @Override
+                public void execute(@NotNull Transaction transaction) {
+                    environment.removeStore(db.toString(), transaction);
+                }
+            });
+
+            stores.put(db, openStore(db));
+        } finally {
+            LOCK.writeLock().unlock();
+        }
+        LOGGER.trace("completed truncate of " + db.toString()
+                + " (" + TimeDuration.fromCurrent(startDate).asCompactString() + ")"
+                + ", size=" + this.size(db));
+    }
+
+    private Store openStore(final LocalDB.DB db) {
+        return environment.computeInTransaction(new TransactionalComputable<Store>() {
+            @Override
+            public Store compute(@NotNull Transaction txn) {
+                return environment.openStore(db.toString(), StoreConfig.WITHOUT_DUPLICATES, txn);
+            }
+        });
+    }
+
+    @Override
+    public File getFileLocation() {
+        return fileLocation;
+    }
+
+    @Override
+    public LocalDB.Status getStatus() {
+        return status;
+    }
+
+    private void outputStats() {
+        Statistics statistics = environment.getStatistics();
+        final Map<String,String> outputStats = new HashMap<>();
+        outputStats.put(EnvironmentStatistics.BYTES_READ, Long.toString(statistics.getStatisticsItem(EnvironmentStatistics.BYTES_READ).getTotal()));
+        outputStats.put(EnvironmentStatistics.BYTES_WRITTEN, Long.toString(statistics.getStatisticsItem(EnvironmentStatistics.BYTES_WRITTEN).getTotal()));
+        outputStats.put(EnvironmentStatistics.TRANSACTIONS, Long.toString(statistics.getStatisticsItem(EnvironmentStatistics.TRANSACTIONS).getTotal()));
+        LOGGER.trace("stats: " + JsonUtil.serializeMap(outputStats));
+    }
+}

src/main/java/password/pwm/util/logging/LocalDBLogger.java

@@ -35,12 +35,15 @@ import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBException;
 import password.pwm.util.localdb.LocalDBStoredQueue;
 
+import java.io.IOException;
 import java.io.Serializable;
 import java.text.NumberFormat;
 import java.util.*;
+import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
@@ -55,18 +58,17 @@ public class LocalDBLogger implements PwmService {
 
     private final static PwmLogger LOGGER = PwmLogger.forClass(LocalDBLogger.class);
 
-
-
     private final LocalDB localDB;
     private final LocalDBLoggerSettings settings;
     private final LocalDBStoredQueue localDBListQueue;
-    private ScheduledExecutorService executorService;
+    private final Queue<PwmLogEvent> eventQueue;
+    private final ScheduledExecutorService cleanerService;
+    private final ScheduledExecutorService writerService;
+    private final AtomicBoolean cleanOnWriteFlag = new AtomicBoolean(false);
 
     private volatile STATUS status = STATUS.NEW;
-    private volatile boolean cleanOnWrite = false;
     private boolean hasShownReadError = false;
 
-
 // --------------------------- CONSTRUCTORS ---------------------------
 
     public LocalDBLogger(final PwmApplication pwmApplication, final LocalDB localDB, final LocalDBLoggerSettings settings)
@@ -89,15 +91,24 @@ public class LocalDBLogger implements PwmService {
             throw new IllegalArgumentException("LocalDB is not available");
         }
 
+        eventQueue = new ArrayBlockingQueue<>(settings.getMaxBufferSize(), true);
+
         status = STATUS.OPEN;
 
-        executorService = Executors.newSingleThreadScheduledExecutor(
+        cleanerService = Executors.newSingleThreadScheduledExecutor(
                 Helper.makePwmThreadFactory(
-                        Helper.makeThreadName(pwmApplication, this.getClass()) + "-",
+                        Helper.makeThreadName(pwmApplication, this.getClass()) + "-cleaner-",
                         true
                 ));
 
-        executorService.scheduleAtFixedRate(new CleanupTask(),0,1, TimeUnit.MINUTES);
+        writerService = Executors.newSingleThreadScheduledExecutor(
+                Helper.makePwmThreadFactory(
+                        Helper.makeThreadName(pwmApplication, this.getClass()) + "-writer-",
+                        true
+                ));
+
+        cleanerService.scheduleAtFixedRate(new CleanupTask(), 0, 1, TimeUnit.MINUTES);
+        writerService.scheduleWithFixedDelay(new FlushTask(), 0, 103, TimeUnit.MILLISECONDS);
 
         final TimeDuration timeDuration = TimeDuration.fromCurrent(startTime);
         LOGGER.info("open in " + timeDuration.asCompactString() + ", " + debugStats());
@@ -139,13 +150,22 @@ public class LocalDBLogger implements PwmService {
 // -------------------------- OTHER METHODS --------------------------
 
     public void close() {
-        LOGGER.debug("LocalDBLogger closing... (" + debugStats() + ")");
-        status = STATUS.CLOSED;
-
-        if (executorService != null) {
-            executorService.shutdown();
-            executorService = null;
+        if (status != STATUS.CLOSED) {
+            LOGGER.debug("LocalDBLogger closing... (" + debugStats() + ")");
+            if (cleanerService != null) {
+                cleanerService.shutdown();
+            }
+            if (writerService != null) {
+                writerService.shutdown();
+                try {
+                    writerService.awaitTermination(30, TimeUnit.SECONDS);
+                } catch (InterruptedException e) {
+                    LOGGER.warn("timed out waiting for writer thread to finish");
+                }
+            }
+            flushEvents();
         }
+        status = STATUS.CLOSED;
 
         LOGGER.debug("LocalDBLogger close completed (" + debugStats() + ")");
     }
@@ -155,7 +175,7 @@ public class LocalDBLogger implements PwmService {
     }
 
     private int determineTailRemovalCount() {
-        final int MAX_TAIL_REMOVAL = 501;
+        final int maxTrailSize = settings.getMaxTrimSize();
 
         final int currentItemCount = localDBListQueue.size();
 
@@ -166,7 +186,7 @@ public class LocalDBLogger implements PwmService {
 
         // purge excess events by count
         if (currentItemCount > settings.getMaxEvents()) {
-            return Math.min(MAX_TAIL_REMOVAL, currentItemCount - settings.getMaxEvents());
+            return Math.min(maxTrailSize, currentItemCount - settings.getMaxEvents());
         }
 
         // purge the tail if it is missing or has invalid timestamp
@@ -178,9 +198,9 @@ public class LocalDBLogger implements PwmService {
         // purge excess events by age;
         final TimeDuration tailAge = TimeDuration.fromCurrent(tailTimestamp);
         if (tailAge.isLongerThan(settings.getMaxAge())) {
-            final long maxRemovalPercentageOfSize = getStoredEventCount() / MAX_TAIL_REMOVAL;
+            final long maxRemovalPercentageOfSize = getStoredEventCount() / maxTrailSize;
             if (maxRemovalPercentageOfSize > 100) {
-                return MAX_TAIL_REMOVAL;
+                return maxTrailSize;
             } else {
                 return 1;
             }
@@ -338,14 +358,48 @@ public class LocalDBLogger implements PwmService {
     public void writeEvent(final PwmLogEvent event) {
         if (status == STATUS.OPEN) {
             if (settings.getMaxEvents() > 0) {
-                try {
-                    if (cleanOnWrite) {
-                        localDBListQueue.removeLast();
+                final Date startTime = new Date();
+                while (!eventQueue.offer(event)) {
+                    if (TimeDuration.fromCurrent(startTime).isLongerThan(settings.getMaxBufferWaitTime())) {
+                        LOGGER.warn("discarded event after waiting max buffer wait time of " + settings.getMaxBufferWaitTime().asCompactString());
+                        return;
                     }
-                    localDBListQueue.add(event.toEncodedString());
-                } catch (Exception e) {
-                    LOGGER.error("error writing to localDBLogger: " + e.getMessage(), e);
+                    Helper.pause(100);
+                }
+            }
+        }
+    }
+
+    private void flushEvents() {
+        final List<String> localBuffer = new ArrayList<>();
+        while (localBuffer.size() < (settings.getMaxBufferSize()) - 1 & !eventQueue.isEmpty()) {
+            final PwmLogEvent pwmLogEvent = eventQueue.poll();
+            try {
+                localBuffer.add(pwmLogEvent.toEncodedString());
+            } catch (IOException e) {
+                LOGGER.warn("error flushing events to localDB: " + e.getMessage(), e);
+            }
+        }
+
+        try {
+            if (cleanOnWriteFlag.get()) {
+                localDBListQueue.removeLast(localBuffer.size());
+            }
+            localDBListQueue.addAll(localBuffer);
+        } catch (Exception e) {
+            LOGGER.error("error writing to localDBLogger: " + e.getMessage(), e);
+        }
+    }
+
+    private class FlushTask implements Runnable {
+        @Override
+        public void run() {
+            try {
+                while (!eventQueue.isEmpty() && status == STATUS.OPEN) {
+                    flushEvents();
                 }
+            } catch(Throwable t) {
+                LOGGER.fatal("localDBLogger flush thread has failed: " + t.getMessage(),t);
             }
         }
     }
@@ -357,7 +411,7 @@ public class LocalDBLogger implements PwmService {
                 while (cleanupCount > 0 && (status == STATUS.OPEN  && localDBListQueue.getPwmDB().status() == LocalDB.Status.OPEN)) {
                     cleanupCount = determineTailRemovalCount();
                     if (cleanupCount > 0) {
-                        cleanOnWrite = true;
+                        cleanOnWriteFlag.set(true);
                         final Date startTime = new Date();
                         localDBListQueue.removeLast(cleanupCount);
                         final TimeDuration purgeTime = TimeDuration.fromCurrent(startTime);
@@ -367,7 +421,7 @@ public class LocalDBLogger implements PwmService {
             } catch (Exception e) {
                 LOGGER.fatal("unexpected error during LocalDBLogger log event cleanup: " + e.getMessage(), e);
             }
-            cleanOnWrite = false;
+            cleanOnWriteFlag.set(localDBListQueue.size() >= settings.getMaxEvents());
         }
     }
 

src/main/java/password/pwm/util/logging/LocalDBLoggerSettings.java

@@ -22,6 +22,7 @@
 
 package password.pwm.util.logging;
 
+import password.pwm.AppProperty;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
 import password.pwm.util.TimeDuration;
@@ -30,6 +31,7 @@ import java.io.Serializable;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
 public class LocalDBLoggerSettings implements Serializable {
     final static int MINIMUM_MAXIMUM_EVENTS = 100;
@@ -38,21 +40,38 @@ public class LocalDBLoggerSettings implements Serializable {
     private final int maxEvents;
     private final TimeDuration maxAge;
     private final Set<Flag> flags;
+    private final int maxBufferSize;
+    private final TimeDuration maxBufferWaitTime;
+    private final int maxTrimSize;
 
     public enum Flag {
         DevDebug,
     }
 
-    public LocalDBLoggerSettings(int maxEvents, TimeDuration maxAge, Set<Flag> flags) {
+    private LocalDBLoggerSettings(
+            int maxEvents,
+            TimeDuration maxAge,
+            Set<Flag> flags,
+            int maxBufferSize,
+            TimeDuration maxBufferWaitTime,
+            int maxTrimSize
+    ) {
         this.maxEvents = maxEvents < 1 ? 0 : Math.max(MINIMUM_MAXIMUM_EVENTS, maxEvents);
         this.maxAge = maxAge == null || maxAge.isShorterThan(MINIMUM_MAX_AGE) ? MINIMUM_MAX_AGE : maxAge;
         this.flags = flags == null ? Collections.<Flag>emptySet() : Collections.unmodifiableSet(flags);
+        this.maxBufferSize = maxBufferSize;
+        this.maxBufferWaitTime = maxBufferWaitTime;
+        this.maxTrimSize = maxTrimSize;
     }
 
     public int getMaxEvents() {
         return maxEvents;
     }
 
+    public TimeDuration getMaxBufferWaitTime() {
+        return maxBufferWaitTime;
+    }
+
     public TimeDuration getMaxAge() {
         return maxAge;
     }
@@ -61,6 +80,14 @@ public class LocalDBLoggerSettings implements Serializable {
         return flags;
     }
 
+    public int getMaxBufferSize() {
+        return maxBufferSize;
+    }
+
+    public int getMaxTrimSize() {
+        return maxTrimSize;
+    }
+
     public static LocalDBLoggerSettings fromConfiguration(final Configuration configuration) {
         final Set<Flag> flags = new HashSet<>();
         if (configuration.isDevDebugMode()) {
@@ -69,6 +96,60 @@ public class LocalDBLoggerSettings implements Serializable {
         final int maxEvents = (int) configuration.readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_EVENTS);
         final long maxAgeMS = 1000 * configuration.readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_AGE);
         final TimeDuration maxAge = new TimeDuration(maxAgeMS);
-        return new LocalDBLoggerSettings(maxEvents, maxAge, flags);
+        final int maxBufferSize = Integer.parseInt(configuration.readAppProperty(AppProperty.LOCALDB_LOGWRITER_BUFFER_SIZE));
+        final TimeDuration maxBufferWaitTime = new TimeDuration(Long.parseLong(configuration.readAppProperty(AppProperty.LOCALDB_LOGWRITER_MAX_BUFFER_WAIT_MS)));
+        final int maxTrimSize = Integer.parseInt(configuration.readAppProperty(AppProperty.LOCALDB_LOGWRITER_MAX_TRIM_SIZE));
+
+        return new Builder()
+                .setMaxEvents(maxEvents)
+                .setMaxAge(maxAge)
+                .setFlags(flags)
+                .setMaxBufferSize(maxBufferSize)
+                .setMaxBufferWaitTime(maxBufferWaitTime)
+                .setMaxTrimSize(maxTrimSize)
+                .createLocalDBLoggerSettings();
+    }
+
+    public static class Builder {
+        private int maxEvents = 1 * 1000 * 1000;
+        private TimeDuration maxAge = new TimeDuration(7, TimeUnit.DAYS);
+        private Set<Flag> flags = Collections.emptySet();
+        private int maxBufferSize = 1000;
+        private TimeDuration maxBufferWaitTime = new TimeDuration(1, TimeUnit.MINUTES);
+        private int maxTrimSize = 501;
+
+        public Builder setMaxEvents(int maxEvents) {
+            this.maxEvents = maxEvents;
+            return this;
+        }
+
+        public Builder setMaxAge(TimeDuration maxAge) {
+            this.maxAge = maxAge;
+            return this;
+        }
+
+        public Builder setFlags(Set<Flag> flags) {
+            this.flags = flags;
+            return this;
+        }
+
+        public Builder setMaxTrimSize(int maxTrimSize) {
+            this.maxTrimSize = maxTrimSize;
+            return this;
+        }
+
+        public Builder setMaxBufferSize(int maxBufferSize) {
+            this.maxBufferSize = maxBufferSize;
+            return this;
+        }
+
+        public Builder setMaxBufferWaitTime(TimeDuration maxBufferWaitTime) {
+            this.maxBufferWaitTime = maxBufferWaitTime;
+            return this;
+        }
+
+        public LocalDBLoggerSettings createLocalDBLoggerSettings() {
+            return new LocalDBLoggerSettings(maxEvents, maxAge, flags, maxBufferSize, maxBufferWaitTime, maxTrimSize);
+        }
     }
 }

src/main/resources/password/pwm/AppProperty.properties

@@ -152,6 +152,9 @@ localdb.decompression.enabled=true
 localdb.compression.minSize=1024
 localdb.implementation=password.pwm.util.localdb.Berkeley_LocalDB
 localdb.initParameters=
+localdb.logWriter.bufferSize=500
+localdb.logWriter.maxBufferWaitMs=60000
+localdb.logWriter.maxTrimSize=501
 macro.randomChar.maxLength=100
 macro.ldapAttr.maxLength=100
 naaf.id=41414141414141414141414141414141
@@ -224,7 +227,7 @@ security.sharedHistory.saltLength=64
 security.certs.validateTimestamps=false
 security.ldap.resolveCanonicalDN=true
 security.ldap.canonicalCacheSeconds=30
-security.defaultEphemeralBlockAlg=AES128_GCM
+security.defaultEphemeralBlockAlg=AES128_HMAC256
 security.defaultEphemeralHashAlg=SHA512
 security.config.minSecurityKeyLength=32
 seedlist.builtin.path=/WEB-INF/seedlist.zip

src/main/resources/password/pwm/config/PwmSetting.xml

@@ -1688,7 +1688,7 @@
             <value>1000000</value>
         </default>
         <properties>
-            <property key="Minimum">1</property>
+            <property key="Minimum">0</property>
             <property key="Maximum">1000000000</property>
         </properties>
     </setting>

src/test/java/password/pwm/tests/LocalDBLoggerTest.java

@@ -23,45 +23,42 @@
 package password.pwm.tests;
 
 import junit.framework.TestCase;
+import password.pwm.AppProperty;
 import password.pwm.PwmConstants;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.stored.ConfigurationReader;
 import password.pwm.svc.stats.EventRateMeter;
-import password.pwm.util.FileSystemUtility;
-import password.pwm.util.Helper;
-import password.pwm.util.Percent;
-import password.pwm.util.TimeDuration;
+import password.pwm.util.*;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBFactory;
-import password.pwm.util.logging.LocalDBLogger;
-import password.pwm.util.logging.LocalDBLoggerSettings;
-import password.pwm.util.logging.PwmLogEvent;
-import password.pwm.util.logging.PwmLogLevel;
+import password.pwm.util.logging.*;
 import password.pwm.util.secure.PwmRandom;
 
 import java.io.File;
+import java.io.Serializable;
 import java.math.RoundingMode;
+import java.text.NumberFormat;
 import java.util.*;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class LocalDBLoggerTest extends TestCase {
 
-    private static final int BULK_EVENT_COUNT = 150 * 1000 * 1000;
+    private final NumberFormat numberFormat = NumberFormat.getNumberInstance();
 
     private LocalDBLogger localDBLogger;
     private LocalDB localDB;
     private Configuration config;
 
-
     private final AtomicInteger eventsAdded = new AtomicInteger(0);
-    private final AtomicInteger eventsRemaining = new AtomicInteger(0);
-    final StringBuffer randomValue = new StringBuffer();
-    final Random random = new Random();
 
     private EventRateMeter eventRateMeter = new EventRateMeter(new TimeDuration(60 * 1000));
 
+    private Settings settings;
+    private Date startTime;
 
 
     @Override
@@ -80,101 +77,155 @@ public class LocalDBLoggerTest extends TestCase {
                 config
         );
 
+        { // open localDBLogger based on configuration settings;
+            final int maxEvents = (int) reader.getConfiguration().readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_EVENTS);
+            final long maxAgeMs = reader.getConfiguration().readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_AGE) * (long) 1000;
+            final LocalDBLoggerSettings settings = new LocalDBLoggerSettings.Builder().setMaxEvents(maxEvents).setMaxAge(new TimeDuration(maxAgeMs)).setFlags(Collections.<LocalDBLoggerSettings.Flag>emptySet()).createLocalDBLoggerSettings();
+            localDBLogger = new LocalDBLogger(null, localDB, settings);
+        }
 
-        final int maxEvents = (int)reader.getConfiguration().readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_EVENTS);
-        final long maxAgeMs = reader.getConfiguration().readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_AGE) * (long)1000;
-        final LocalDBLoggerSettings settings = new LocalDBLoggerSettings(maxEvents, new TimeDuration(maxAgeMs), Collections.<LocalDBLoggerSettings.Flag>emptySet());
-
-        localDBLogger = new LocalDBLogger(null, localDB, settings);
+        settings = new Settings();
+        settings.threads = 10;
+        settings.testDuration = new TimeDuration(1, TimeUnit.HOURS);
+        settings.valueLength = 5000;
+        settings.batchSize = 100;
+    }
 
-        {
-            final int randomLength = 84000;
-            while (randomValue.length() < randomLength) {
-                randomValue.append(PwmRandom.getInstance().nextChar());
-            }
-        }
+    private void out(String output) {
+        System.out.println(PwmConstants.DEFAULT_DATETIME_FORMAT.format(new Date())+ " " + output);
     }
 
-    public void testBulkAddEvents() {
-        final int startingSize = localDBLogger.getStoredEventCount();
-        eventsRemaining.addAndGet(BULK_EVENT_COUNT);
+    public void testBulkAddEvents() throws InterruptedException {
+        out("starting bulk add...  ");
+        out("settings=" + JsonUtil.serialize(settings));
+        startTime = new Date();
         final Timer timer = new Timer();
-        timer.scheduleAtFixedRate(new DebugOutputTimerTask(),5 * 1000, 5 * 1000);
 
-        for (int loopCount = 0; loopCount < 5; loopCount++) {
-            final Thread populatorThread = new PopulatorThread();
-            populatorThread.start();
-        }
+        final int threadCount = settings.threads;
+        final ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(
+                threadCount,
+                threadCount,
+                1,
+                TimeUnit.SECONDS,
+                new ArrayBlockingQueue<Runnable>(threadCount + 1)
+        );
 
-        while (eventsRemaining.get() > 0) {
-            Helper.pause(5);
-        }
+        timer.scheduleAtFixedRate(new DebugOutputTimerTask(),5 * 1000, 30 * 1000);
 
-        final long startWaitTime = System.currentTimeMillis();
-        while (TimeDuration.fromCurrent(startWaitTime).isShorterThan(new TimeDuration(10, TimeUnit.HOURS)) && eventsRemaining.get() > 0) {
-            Helper.pause(500);
+        for (int loopCount = 0; loopCount < threadCount; loopCount++) {
+            threadPoolExecutor.execute(new PopulatorThread());
         }
-        Helper.pause(5000);
 
+        threadPoolExecutor.shutdown();
+        threadPoolExecutor.awaitTermination(1, TimeUnit.DAYS);
         timer.cancel();
+        out("bulk operations completed");
+        out("settings=" + JsonUtil.serialize(settings));
+        out(" results=" + JsonUtil.serialize(makeResults()));
+        outputDebugInfo();
     }
 
 
     private class PopulatorThread extends Thread {
         public void run() {
-            int loopCount = 3;
-            while (eventsRemaining.get() > 0) {
-                final Collection<PwmLogEvent> events = makeEvents(loopCount);
+            final RandomValueMaker randomValueMaker = new RandomValueMaker(settings.valueLength);
+            while (TimeDuration.fromCurrent(startTime).isShorterThan(settings.testDuration)) {
+                final Collection<PwmLogEvent> events = makeEvents(randomValueMaker);
                 for (final PwmLogEvent logEvent : events) {
                     localDBLogger.writeEvent(logEvent);
                     eventRateMeter.markEvents(1);
-                    eventsRemaining.decrementAndGet();
                     eventsAdded.incrementAndGet();
                 }
             }
         }
     }
 
-    private Collection<PwmLogEvent> makeEvents(final int count) {
+    private Collection<PwmLogEvent> makeEvents(final RandomValueMaker randomValueMaker) {
+        final int count = settings.batchSize;
         final Collection<PwmLogEvent> events = new ArrayList<>();
-
         for (int i = 0; i < count; i++) {
-            events.add(makeEvent());
+            final String description = randomValueMaker.next();
+            PwmLogEvent event = PwmLogEvent.createPwmLogEvent(
+                    new Date(),
+                    LocalDBLogger.class.getName(),
+                    description, "", "", null, null, PwmLogLevel.TRACE);
+            events.add(event);
         }
 
-        //System.out.println("made "  + size + " events in " + TimeDuration.fromCurrent(startTime).asCompactString());
         return events;
     }
 
-    private PwmLogEvent makeEvent() {
-        final int randomPos = random.nextInt(randomValue.length() - 1);
-        randomValue.replace(randomPos, randomPos + 1,String.valueOf(random.nextInt(9)));
+    private void outputDebugInfo() {
+        final StringBuilder sb = new StringBuilder();
+        sb.append("added ").append(numberFormat.format(eventsAdded.get()));
+        sb.append(", size: ").append(Helper.formatDiskSize(FileSystemUtility.getFileDirectorySize(localDB.getFileLocation())));
+        sb.append(", eventsInDb: ").append(figureEventsInDbStat());
+        sb.append(", free: ").append(Helper.formatDiskSize(
+                FileSystemUtility.diskSpaceRemaining(localDB.getFileLocation())));
+        sb.append(", eps: ").append(eventRateMeter.readEventRate().setScale(0, RoundingMode.UP));
+        sb.append(", remain: ").append(settings.testDuration.subtract(TimeDuration.fromCurrent(startTime)).asCompactString());
+        sb.append(", tail: ").append(TimeDuration.fromCurrent(localDBLogger.getTailDate()).asCompactString());
+        out(sb.toString());
+    }
 
-        final int startPos = random.nextInt(randomValue.length() - 100);
-        final int endPos = startPos + random.nextInt(randomValue.length() - startPos);
+    private String figureEventsInDbStat() {
+        final long maxEvents = config.readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_EVENTS);
+        final long eventCount = localDBLogger.getStoredEventCount();
+        final Percent percent = new Percent(eventCount,maxEvents);
+        return numberFormat.format(localDBLogger.getStoredEventCount()) + "/" + numberFormat.format(maxEvents)
+                +  " (" + percent.pretty(2) + ")";
+    }
 
-        final String description = randomValue.substring(startPos,endPos);
-        return PwmLogEvent.createPwmLogEvent(
-                new Date(System.currentTimeMillis()),
-                LocalDBLogger.class.getName(),
-                description, "", "", null, null, PwmLogLevel.TRACE);
+    private Results makeResults() {
+        Results results = new Results();
+        results.dbClass = config.readAppProperty(AppProperty.LOCALDB_IMPLEMENTATION);
+        results.duration = TimeDuration.fromCurrent(startTime).asCompactString();
+        results.recordsAdded = eventsAdded.get();
+        results.dbSize = Helper.formatDiskSize(FileSystemUtility.getFileDirectorySize(localDB.getFileLocation()));
+        results.eventsInDb = figureEventsInDbStat();
+        return results;
     }
 
     private class DebugOutputTimerTask extends TimerTask {
         public void run() {
-            final StringBuilder sb = new StringBuilder();
-            final long maxEvents = config.readSettingAsLong(PwmSetting.EVENTS_PWMDB_MAX_EVENTS);
-            final long eventCount = localDBLogger.getStoredEventCount();
-            final Percent percent = new Percent(eventCount,maxEvents);
-            sb.append(PwmConstants.DEFAULT_DATETIME_FORMAT.format(new Date()));
-            sb.append(", added ").append(eventsAdded);
-            sb.append(", db size: ").append(Helper.formatDiskSize(FileSystemUtility.getFileDirectorySize(localDB.getFileLocation())));
-            sb.append(", events: ").append(localDBLogger.getStoredEventCount()).append("/").append(maxEvents);
-            sb.append(" (").append(percent.pretty(3)).append(")");
-            sb.append(", free space: ").append(Helper.formatDiskSize(
-                    FileSystemUtility.diskSpaceRemaining(localDB.getFileLocation())));
-            sb.append(", eps: ").append(eventRateMeter.readEventRate().setScale(0, RoundingMode.UP));
-            System.out.println(sb);
+            outputDebugInfo();
+        }
+    }
+
+    private static class Settings implements Serializable {
+        private TimeDuration testDuration;
+        private int threads;
+        private int valueLength;
+        private int batchSize;
+    }
+
+    private static class Results implements Serializable {
+        private String dbClass;
+        private String duration;
+        private int recordsAdded;
+        private String dbSize;
+        private String eventsInDb;
+    }
+
+    private static class RandomValueMaker {
+        private int outputLength;
+        final StringBuffer randomValue = new StringBuffer();
+        final Random random = new Random();
+
+        public RandomValueMaker(final int outputLength) {
+            this.outputLength = outputLength;
+            randomValue.append(PwmRandom.getInstance().alphaNumericString(outputLength * 50));
+        }
+
+        public String next() {
+            final int randomPos = random.nextInt(randomValue.length() - 1);
+            randomValue.replace(randomPos, randomPos + 1,String.valueOf(random.nextInt(9)));
+
+            final int startPos = random.nextInt(randomValue.length() - outputLength);
+            final int endPos = startPos + outputLength;
+
+
+            return randomValue.substring(startPos,endPos);
         }
     }
 }

src/test/resources/password/pwm/tests/TestHelper.properties

@@ -0,0 +1,25 @@
+#
+# Password Management Servlets (PWM)
+# http://code.google.com/p/pwm/
+#
+# Copyright (c) 2006-2009 Novell, Inc.
+# Copyright (c) 2009-2012 The PWM Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+applicationPath=
+localDBPath=
+configurationFile=