bugfix omnibus

pom.xml

@@ -447,11 +447,6 @@
             <artifactId>commons-lang3</artifactId>
             <version>3.4</version>
         </dependency>
-        <dependency>
-            <groupId>commons-validator</groupId>
-            <artifactId>commons-validator</artifactId>
-            <version>1.5.1</version>
-        </dependency>
         <dependency>
             <groupId>com.sun.mail</groupId>
             <artifactId>javax.mail</artifactId>

src/main/java/password/pwm/AppProperty.java

@@ -36,6 +36,7 @@ public enum AppProperty {
     APPLICATION_WORDLIST_RETRY_SECONDS              ("application.wordlistRetryImportSeconds"),
     AUDIT_EVENTS_EMAILFROM                          ("audit.events.emailFrom"),
     AUDIT_EVENTS_EMAILSUBJECT                       ("audit.events.emailSubject"),
+    AUDIT_EVENTS_LOCALDB_MAX_BULK_REMOVALS          ("audit.events.localdb.maxBulkRemovals"),
     AUDIT_SYSLOG_MAX_MESSAGE_LENGTH                 ("audit.syslog.message.length"),
     AUDIT_SYSLOG_TRUNCATE_MESSAGE                   ("audit.syslog.message.truncateMsg"),
     BACKUP_LOCATION                                 ("backup.path"),
@@ -226,7 +227,6 @@ public enum AppProperty {
     QUEUE_SYSLOG_RETRY_TIMEOUT_MS                   ("queue.syslog.retryTimeoutMs"),
     QUEUE_SYSLOG_MAX_AGE_MS                         ("queue.syslog.maxAgeMs"),
     QUEUE_SYSLOG_MAX_COUNT                          ("queue.syslog.maxCount"),
-    QUEUE_MAX_CLOSE_TIMEOUT_MS                      ("queue.maxCloseTimeoutMs"),
     RECAPTCHA_CLIENT_JS_URL                         ("recaptcha.clientJsUrl"),
     RECAPTCHA_CLIENT_IFRAME_URL                     ("recaptcha.clientIframeUrl"),
     RECAPTCHA_VALIDATE_URL                          ("recaptcha.validateUrl"),

src/main/java/password/pwm/PwmAboutProperty.java

@@ -47,6 +47,8 @@ public enum PwmAboutProperty {
     app_currentPublishedVersionCheckTime,
     app_siteUrl,
     app_instanceID,
+    app_trialMode,
+    app_applianceMode,
     app_wordlistSize,
     app_seedlistSize,
     app_sharedHistorySize,
@@ -107,6 +109,8 @@ public enum PwmAboutProperty {
         aboutMap.put(app_installTime,              dateFormatForInfoBean(pwmApplication.getInstallTime()));
         aboutMap.put(app_siteUrl,                  pwmApplication.getConfig().readSettingAsString(PwmSetting.PWM_SITE_URL));
         aboutMap.put(app_instanceID,               pwmApplication.getInstanceID());
+        aboutMap.put(app_trialMode,                Boolean.toString(PwmConstants.TRIAL_MODE));
+        aboutMap.put(app_applianceMode,            Boolean.toString(pwmApplication.getPwmEnvironment() != null && pwmApplication.getPwmEnvironment().getFlags().contains(PwmEnvironment.ApplicationFlag.Appliance)));
         aboutMap.put(app_chaiApiVersion,           PwmConstants.CHAI_API_VERSION);
 
         if (pwmApplication.getConfig().readSettingAsBoolean(PwmSetting.VERSION_CHECK_ENABLE)) {

src/main/java/password/pwm/PwmEnvironment.java

@@ -43,11 +43,11 @@ public class PwmEnvironment implements Serializable {
 
     // data elements
     private final PwmApplicationMode applicationMode;
-    private final Configuration config;
+    private transient final Configuration config;
     private final File applicationPath;
     private final boolean internalRuntimeInstance;
     private final File configurationFile;
-    private final ContextManager contextManager;
+    private transient final ContextManager contextManager;
     private final Collection<ApplicationFlag> flags;
     private final Map<ApplicationParameter,String> parameters;
 

src/main/java/password/pwm/config/FormConfiguration.java

@@ -22,7 +22,6 @@
 
 package password.pwm.config;
 
-import org.apache.commons.validator.routines.EmailValidator;
 import password.pwm.AppProperty;
 import password.pwm.PwmConstants;
 import password.pwm.error.*;
@@ -322,15 +321,15 @@ public class FormConfiguration implements Serializable {
      * @return
      */
     public static boolean testEmailAddress(final Configuration config, final String address) {
+        final String patternStr;
         if (config != null) {
-            final String patternStr = config.readAppProperty(AppProperty.FORM_EMAIL_REGEX);
-            if (patternStr != null && !patternStr.isEmpty()) {
-                final Pattern pattern = Pattern.compile(patternStr);
-                final Matcher matcher = pattern.matcher(address);
-                return matcher.matches();
-            }
+            patternStr = config.readAppProperty(AppProperty.FORM_EMAIL_REGEX);
+        } else {
+            patternStr = AppProperty.FORM_EMAIL_REGEX.getDefaultValue();
         }
 
-        return EmailValidator.getInstance(true, true).isValid(address);
+        final Pattern pattern = Pattern.compile(patternStr);
+        final Matcher matcher = pattern.matcher(address);
+        return matcher.matches();
     }
 }

src/main/java/password/pwm/config/PwmSetting.java

@@ -83,8 +83,6 @@ public enum PwmSetting {
             "knownLocales", PwmSettingSyntax.STRING_ARRAY, PwmSettingCategory.LOCALIZATION),
     LOCALE_COOKIE_MAX_AGE(
             "locale.cookie.age", PwmSettingSyntax.DURATION, PwmSettingCategory.LOCALIZATION),
-    PWMDB_LOCATION(
-            "pwmDb.location", PwmSettingSyntax.STRING, PwmSettingCategory.GENERAL),
     HTTP_PROXY_URL(
             "http.proxy.url", PwmSettingSyntax.STRING, PwmSettingCategory.GENERAL),
     APP_PROPERTY_OVERRIDES(
@@ -1095,6 +1093,8 @@ public enum PwmSetting {
             "recovery.require.otp", PwmSettingSyntax.BOOLEAN, PwmSettingCategory.RECOVERY_SETTINGS),
     HELPDESK_ENABLE_OTP_VERIFY(
             "helpdesk.otp.verify", PwmSettingSyntax.BOOLEAN, PwmSettingCategory.HELPDESK_PROFILE),
+    PWMDB_LOCATION(
+            "pwmDb.location", PwmSettingSyntax.STRING, PwmSettingCategory.GENERAL),
 
 
 

src/main/java/password/pwm/health/ConfigurationChecker.java

@@ -102,10 +102,6 @@ public class ConfigurationChecker implements HealthChecker {
             LOGGER.error(PwmConstants.HEALTH_SESSION_LABEL,"error while inspecting site URL setting: " + e.getMessage());
         }
 
-        if (!config.readSettingAsBoolean(PwmSetting.REQUIRE_HTTPS)) {
-            records.add(HealthRecord.forMessage(HealthMessage.Config_RequireHttps,settingToOutputText(PwmSetting.REQUIRE_HTTPS)));
-        }
-
         if (config.readSettingAsBoolean(PwmSetting.LDAP_ENABLE_WIRE_TRACE)) {
             records.add(HealthRecord.forMessage(HealthMessage.Config_LDAPWireTrace,settingToOutputText(PwmSetting.LDAP_ENABLE_WIRE_TRACE)));
         }

src/main/java/password/pwm/health/HealthMessage.java

@@ -47,7 +47,6 @@ public enum HealthMessage {
     Config_MissingProxyDN                   (HealthStatus.CONFIG,   HealthTopic.Configuration),
     Config_MissingProxyPassword             (HealthStatus.CONFIG,   HealthTopic.Configuration),
     Config_NoSiteURL                        (HealthStatus.WARN,     HealthTopic.Configuration),
-    Config_RequireHttps                     (HealthStatus.CONFIG,   HealthTopic.Configuration),
     Config_LDAPWireTrace                    (HealthStatus.WARN,     HealthTopic.Configuration),
     Config_PromiscuousLDAP                  (HealthStatus.CONFIG,   HealthTopic.Configuration),
     Config_ShowDetailedErrors               (HealthStatus.CONFIG,   HealthTopic.Configuration),

src/main/java/password/pwm/http/servlet/configmanager/DebugItemGenerator.java

@@ -23,10 +23,7 @@
 package password.pwm.http.servlet.configmanager;
 
 import org.apache.commons.csv.CSVPrinter;
-import password.pwm.AppProperty;
-import password.pwm.PwmAboutProperty;
-import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
+import password.pwm.*;
 import password.pwm.bean.pub.SessionStateInfoBean;
 import password.pwm.config.Configuration;
 import password.pwm.config.stored.StoredConfigurationImpl;
@@ -58,7 +55,8 @@ public class DebugItemGenerator {
             ConfigurationDebugJsonItemGenerator.class,
             ConfigurationDebugTextItemGenerator.class,
             AboutItemGenerator.class,
-            EnvironmentItemGenerator.class,
+            SystemEnvironmentItemGenerator.class,
+            AppEnvironmentGenerator.class,
             AppPropertiesItemGenerator.class,
             InfoDebugItemGenerator.class,
             HealthDebugItemGenerator.class,
@@ -212,10 +210,10 @@ public class DebugItemGenerator {
         }
     }
 
-    static class EnvironmentItemGenerator implements Generator {
+    static class SystemEnvironmentItemGenerator implements Generator {
         @Override
         public String getFilename() {
-            return "environment.properties";
+            return "system-environment.properties";
         }
 
         @Override
@@ -234,6 +232,23 @@ public class DebugItemGenerator {
         }
     }
 
+    static class AppEnvironmentGenerator implements Generator {
+        @Override
+        public String getFilename() {
+            return "app-environment.json";
+        }
+
+        @Override
+        public void outputItem(PwmApplication pwmApplication, PwmRequest pwmRequest, OutputStream outputStream) throws Exception
+        {
+            final PwmEnvironment pwmEnvironment = pwmApplication.getPwmEnvironment();
+
+            // java threads
+            final String output = JsonUtil.serialize(pwmEnvironment);
+            outputStream.write(output.getBytes(PwmConstants.DEFAULT_CHARSET));
+        }
+    }
+
     static class AppPropertiesItemGenerator implements Generator {
         @Override
         public String getFilename() {

src/main/java/password/pwm/http/servlet/oauth/OAuthConsumerServlet.java

@@ -421,6 +421,7 @@ public class OAuthConsumerServlet extends AbstractPwmServlet {
         }
 
         debugOutput.append(" body:\n ").append(bodyResponse);
+        LOGGER.trace(pwmRequest, debugOutput.toString());
 
         if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
             throw new PwmUnrecoverableException(new ErrorInformation(
@@ -429,41 +430,42 @@ public class OAuthConsumerServlet extends AbstractPwmServlet {
             ));
         }
 
-        LOGGER.trace(pwmRequest, debugOutput.toString());
         return new RestResults(httpResponse, bodyResponse);
     }
 
     public static String figureOauthSelfEndPointUrl(final PwmRequest pwmRequest) {
-        final String inputURI, debugSource;
+        final String debugSource, redirect_uri;
 
         {
             final String returnUrlOverride = pwmRequest.getConfig().readAppProperty(AppProperty.OAUTH_RETURN_URL_OVERRIDE);
             final String siteURL = pwmRequest.getConfig().readSettingAsString(PwmSetting.PWM_SITE_URL);
             if (returnUrlOverride != null && !returnUrlOverride.trim().isEmpty()) {
-                inputURI = returnUrlOverride;
                 debugSource = "AppProperty(\"" + AppProperty.OAUTH_RETURN_URL_OVERRIDE.getKey() + "\")";
+                redirect_uri = returnUrlOverride
+                        + PwmServletDefinition.OAuthConsumer.servletUrl();
             } else if (siteURL != null && !siteURL.trim().isEmpty()) {
-                inputURI = siteURL;
                 debugSource = "SiteURL Setting";
+                redirect_uri = siteURL
+                        + PwmServletDefinition.OAuthConsumer.servletUrl();
             } else {
                 debugSource = "Input Request URL";
-                inputURI = pwmRequest.getHttpServletRequest().getRequestURL().toString();
+                final String inputURI = pwmRequest.getHttpServletRequest().getRequestURL().toString();
+                try {
+                    final URI requestUri = new URI(inputURI);
+                    final int port = requestUri.getPort();
+                    redirect_uri = requestUri.getScheme() + "://" + requestUri.getHost()
+                            + (port > 0 && port != 80 && port != 443 ? ":" + requestUri.getPort() : "")
+                            + pwmRequest.getContextPath()
+                            + PwmServletDefinition.OAuthConsumer.servletUrl();
+                } catch (URISyntaxException e) {
+                    throw new IllegalStateException("unable to parse inbound request uri while generating oauth redirect: " + e.getMessage());
+                }
             }
         }
 
-        final String redirect_uri;
-        try {
-            final URI requestUri = new URI(inputURI);
-            redirect_uri = requestUri.getScheme() + "://" + requestUri.getHost()
-                    + (requestUri.getPort() > 0 ? ":" + requestUri.getPort() : "")
-                    + PwmServletDefinition.OAuthConsumer.servletUrl();
-        } catch (URISyntaxException e) {
-            throw new IllegalStateException("unable to parse inbound request uri while generating oauth redirect: " + e.getMessage());
-        }
         LOGGER.trace("calculated oauth self end point URI as '" + redirect_uri + "' using method " + debugSource);
         return redirect_uri;
     }
-
     static class RestResults {
         final HttpResponse httpResponse;
         final String responseBody;

src/main/java/password/pwm/svc/event/AuditEvent.java

@@ -132,9 +132,21 @@ public enum AuditEvent {
     }
 
     public enum Type {
-        USER,
-        SYSTEM,
-        HELPDESK,
+        USER(UserAuditRecord.class),
+        SYSTEM(SyslogAuditService.class),
+        HELPDESK(HelpdeskAuditRecord.class),
+
+        ;
+
+        private final Class clazz;
+
+        Type(Class clazz) {
+            this.clazz = clazz;
+        }
+
+        public Class getDataClass() {
+            return clazz;
+        }
     }
 
     public String getXdasTaxonomy() {

src/main/java/password/pwm/svc/event/AuditService.java

@@ -147,8 +147,8 @@ public class AuditService implements PwmService {
                     LOGGER.debug("localDB audit vault will remain closed due to max records setting");
                     pwmApplication.getLocalDB().truncate(LocalDB.DB.AUDIT_EVENTS);
                 } else {
-                    auditVault = new LocalDbAuditVault(pwmApplication, pwmApplication.getLocalDB());
-                    auditVault.init(settings);
+                    auditVault = new LocalDbAuditVault();
+                    auditVault.init(pwmApplication, pwmApplication.getLocalDB(), settings);
                 }
             } else {
                 LOGGER.debug("localDB audit vault will remain closed due to application mode");
@@ -268,6 +268,10 @@ public class AuditService implements PwmService {
         return auditVault.oldestRecord();
     }
 
+    public String sizeToDebugString() {
+        return auditVault.sizeToDebugString();
+    }
+
     public void submit(final AuditEvent auditEvent, final UserInfoBean userInfoBean, final PwmSession pwmSession)
             throws PwmUnrecoverableException
     {
@@ -302,7 +306,11 @@ public class AuditService implements PwmService {
 
         // add to audit db
         if (auditVault != null) {
-            auditVault.add(auditRecord);
+            try {
+                auditVault.add(auditRecord);
+            } catch (PwmOperationalException e) {
+                LOGGER.warn("discarding audit event due to storage error: " + e.getMessage());
+            }
         }
 
         // email alert

src/main/java/password/pwm/svc/event/AuditVault.java

@@ -22,14 +22,21 @@
 
 package password.pwm.svc.event;
 
+import password.pwm.PwmApplication;
+import password.pwm.error.PwmException;
+import password.pwm.error.PwmOperationalException;
 import password.pwm.util.TimeDuration;
+import password.pwm.util.localdb.LocalDB;
+import password.pwm.util.localdb.LocalDBException;
 
 import java.util.Date;
 import java.util.Iterator;
 
 public interface AuditVault {
 
-    void init(Settings settings);
+    void init(final PwmApplication pwmApplication, final LocalDB localDB, final Settings settings) throws LocalDBException, PwmException;
+
+    void close();
 
     int size();
 
@@ -37,7 +44,8 @@ public interface AuditVault {
 
     Iterator<AuditRecord> readVault();
 
-    void add(AuditRecord record);
+    String sizeToDebugString();
+    void add(AuditRecord record) throws PwmOperationalException;
 
     class Settings {
         private long maxRecordCount;

src/main/java/password/pwm/svc/event/LocalDbAuditVault.java

@@ -22,9 +22,11 @@
 
 package password.pwm.svc.event;
 
+import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
-import password.pwm.util.JsonUtil;
-import password.pwm.util.TimeDuration;
+import password.pwm.error.PwmException;
+import password.pwm.svc.PwmService;
+import password.pwm.util.*;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBException;
 import password.pwm.util.localdb.LocalDBStoredQueue;
@@ -33,27 +35,59 @@ import password.pwm.util.logging.PwmLogger;
 import java.util.Date;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 
 public class LocalDbAuditVault implements AuditVault {
     private static final PwmLogger LOGGER = PwmLogger.forClass(LocalDbAuditVault.class);
 
-    private static final int MAX_REMOVALS_PER_ADD = 100;
-
     private LocalDBStoredQueue auditDB;
     private Settings settings;
     private Date oldestRecord;
 
+    private int maxBulkRemovals = 105;
+
+    private ScheduledExecutorService executorService;
+    private volatile PwmService.STATUS status = PwmService.STATUS.NEW;
+
+
     public LocalDbAuditVault(
-            final PwmApplication pwmApplication,
-            final LocalDB localDB
     )
             throws LocalDBException
     {
-        this.auditDB = LocalDBStoredQueue.createLocalDBStoredQueue(pwmApplication, localDB, LocalDB.DB.AUDIT_EVENTS);
     }
 
-    public void init(final Settings settings) {
+    public void init(
+            final PwmApplication pwmApplication,
+            final LocalDB localDB,
+            final Settings settings
+    )
+            throws PwmException
+    {
         this.settings = settings;
+        this.auditDB = LocalDBStoredQueue.createLocalDBStoredQueue(pwmApplication, localDB, LocalDB.DB.AUDIT_EVENTS);
+        this.maxBulkRemovals = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.AUDIT_EVENTS_LOCALDB_MAX_BULK_REMOVALS));
+
+        readOldestRecord();
+
+        executorService = Executors.newSingleThreadScheduledExecutor(
+                Helper.makePwmThreadFactory(
+                        Helper.makeThreadName(pwmApplication,this.getClass()) + "-",
+                        true
+                ));
+
+        status = PwmService.STATUS.OPEN;
+        executorService.scheduleWithFixedDelay(new TrimmerThread(), 0, 10, TimeUnit.MINUTES);
+    }
+
+    public void close() {
+        executorService.shutdown();
+        status = PwmService.STATUS.CLOSED;
+    }
+
+    public PwmService.STATUS getStatus() {
+        return status;
     }
 
     @Override
@@ -94,6 +128,15 @@ public class LocalDbAuditVault implements AuditVault {
         }
     }
 
+    @Override
+    public String sizeToDebugString() {
+        final long storedEvents = this.size();
+        final long maxEvents = settings.getMaxRecordCount();
+        final Percent percent = new Percent(storedEvents, maxEvents);
+
+        return storedEvents + " / " + maxEvents + " (" + percent.pretty(2) + ")";
+    }
+
     private static AuditRecord deSerializeRecord(final String input) {
         final Map<String,String> tempMap = JsonUtil.deserializeStringMap(input);
         String errorMsg = "";
@@ -101,19 +144,17 @@ public class LocalDbAuditVault implements AuditVault {
             if (tempMap != null) {
                 final String eventCode = tempMap.get("eventCode");
                 if (eventCode != null && eventCode.length() > 0) {
-                    final AuditEvent event = AuditEvent.valueOf(eventCode);
-                    if (event != null) {
-                        switch (event.getType()) {
-                            case USER:
-                                return JsonUtil.deserialize(input, UserAuditRecord.class);
-                            case SYSTEM:
-                                return JsonUtil.deserialize(input, SystemAuditRecord.class);
-                            case HELPDESK:
-                                return JsonUtil.deserialize(input, HelpdeskAuditRecord.class);
-                            default:
-                                throw new IllegalArgumentException("unknown audit record type: " + event.getType());
-                        }
+                    final AuditEvent event;
+                    try {
+                        event = AuditEvent.valueOf(eventCode);
+                    } catch (IllegalArgumentException e) {
+                        errorMsg = "error de-serializing audit record: " + e.getMessage();
+                        LOGGER.error(errorMsg);
+                        return null;
                     }
+                    final Class clazz = event.getType().getDataClass();
+                    final com.google.gson.reflect.TypeToken typeToken = com.google.gson.reflect.TypeToken.get(clazz);
+                    return JsonUtil.deserialize(input, typeToken);
                 }
             }
         } catch (Exception e) {
@@ -130,34 +171,74 @@ public class LocalDbAuditVault implements AuditVault {
 
         final String jsonRecord = JsonUtil.serialize(record);
         auditDB.addLast(jsonRecord);
-        trim();
-    }
-
-    private void trim() {
-        if (oldestRecord != null && TimeDuration.fromCurrent(oldestRecord).isLongerThan(settings.getMaxRecordAge())) {
-            return;
-        }
 
-        if (auditDB.isEmpty()) {
-            return;
+        if (auditDB.size() > settings.getMaxRecordCount()) {
+            removeRecords(1);
         }
+    }
 
-        int workActions = 0;
-        while (workActions < MAX_REMOVALS_PER_ADD && !auditDB.isEmpty()) {
+    private void readOldestRecord() {
+        if (auditDB != null && !auditDB.isEmpty()) {
             final String stringFirstRecord = auditDB.getFirst();
             final UserAuditRecord firstRecord = JsonUtil.deserialize(stringFirstRecord, UserAuditRecord.class);
             oldestRecord = firstRecord.getTimestamp();
-            if (TimeDuration.fromCurrent(oldestRecord).isLongerThan(settings.getMaxRecordAge())) {
-                auditDB.removeFirst();
-                workActions++;
-            } else {
-                break;
+        }
+    }
+
+    private void removeRecords(int count) {
+        auditDB.removeFirst(count);
+        readOldestRecord();
+    }
+
+    private class TrimmerThread implements Runnable {
+
+        // keep transaction duration around 100ms if possible.
+        final TransactionSizeCalculator transactionSizeCalculator = new TransactionSizeCalculator(
+                new TransactionSizeCalculator.SettingsBuilder()
+                        .setDurationGoal(new TimeDuration(101, TimeUnit.MILLISECONDS))
+                        .setMaxTransactions(5003)
+                        .setMinTransactions(3)
+                        .createSettings()
+        );
+
+        @Override
+        public void run() {
+            long startTime = System.currentTimeMillis();
+            while (trim(transactionSizeCalculator.getTransactionSize())
+                    && status == PwmService.STATUS.OPEN
+                    ) {
+                final long executeTime = System.currentTimeMillis() - startTime;
+                transactionSizeCalculator.recordLastTransactionDuration(executeTime);
+                transactionSizeCalculator.pause();
+                startTime = System.currentTimeMillis();
             }
         }
 
-        while (auditDB.size() > settings.getMaxRecordCount() && workActions < MAX_REMOVALS_PER_ADD) {
-            auditDB.removeFirst();
-            workActions++;
+        private boolean trim(final int maxRemovals) {
+            if (auditDB.isEmpty()) {
+                return false;
+            }
+
+            if (auditDB.size() > settings.getMaxRecordCount() + maxRemovals) {
+                removeRecords(maxRemovals);
+                return true;
+            }
+
+            int workActions = 0;
+            while (oldestRecord != null
+                    && workActions < maxRemovals
+                    && !auditDB.isEmpty()
+                    && status == PwmService.STATUS.OPEN
+                    ) {
+                if (TimeDuration.fromCurrent(oldestRecord).isLongerThan(settings.getMaxRecordAge())) {
+                    removeRecords(1);
+                    workActions++;
+                } else {
+                    break;
+                }
+            }
+
+            return workActions > 0;
         }
     }
 }

src/main/java/password/pwm/svc/event/SyslogAuditService.java

@@ -71,6 +71,8 @@ public class SyslogAuditService {
 
     private static final int WARNING_WINDOW_MS = 30 * 60 * 1000;
     private static final String SYSLOG_INSTANCE_NAME = "syslog-audit";
+    private static final int LENGTH_OVERSIZE = 1024;
+
 
     private SyslogIF syslogInstance = null;
     private ErrorInformation lastError = null;
@@ -154,7 +156,7 @@ public class SyslogAuditService {
 
         syslogConfigIF.setThreaded(false);
         syslogConfigIF.setMaxQueueSize(0);
-        syslogConfigIF.setMaxMessageLength(maxLength);
+        syslogConfigIF.setMaxMessageLength(maxLength + LENGTH_OVERSIZE);
         syslogConfigIF.setThrowExceptionOnWrite(true);
         syslogConfigIF.setHost(syslogConfig.getHost());
         syslogConfigIF.setPort(syslogConfig.getPort());

src/main/java/password/pwm/svc/wordlist/Populator.java

@@ -42,6 +42,7 @@ import java.text.NumberFormat;
 import java.util.Date;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @author Jason D. Rivard
@@ -67,7 +68,14 @@ class Populator {
 
     private final PopulationStats overallStats = new PopulationStats();
     private PopulationStats perReportStats = new PopulationStats();
-    private TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(600, 10, 50 * 1000);
+    private TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(
+            new TransactionSizeCalculator.SettingsBuilder()
+                    .setDurationGoal(new TimeDuration(600, TimeUnit.MILLISECONDS))
+                    .setMinTransactions(10)
+                    .setMaxTransactions(50 * 1000)
+                    .createSettings()
+    );
+
     private int loopLines;
 
     private final Map<String,String> bufferedWords = new TreeMap<>();

src/main/java/password/pwm/util/LDAPPermissionCalculator.java

@@ -183,13 +183,16 @@ public class LDAPPermissionCalculator implements Serializable {
                 if (!(Boolean)storedConfiguration.readSetting(PwmSetting.PEOPLE_SEARCH_ENABLE).toNativeObject()) {
                     return Collections.emptyList();
                 }
-                final boolean proxyOverride = (Boolean)storedConfiguration.readSetting(PwmSetting.PEOPLE_SEARCH_USE_PROXY, profile).toNativeObject()
-                        || (Boolean)storedConfiguration.readSetting(PwmSetting.PEOPLE_SEARCH_ENABLE_PUBLIC, profile).toNativeObject();
+                final boolean proxyOverride = (Boolean)storedConfiguration.readSetting(PwmSetting.PEOPLE_SEARCH_USE_PROXY, profile).toNativeObject();
+                final boolean publicOverride = (Boolean)storedConfiguration.readSetting(PwmSetting.PEOPLE_SEARCH_ENABLE_PUBLIC, profile).toNativeObject();
 
-                if (proxyOverride) {
+                if (proxyOverride || publicOverride) {
                     final Collection<LDAPPermissionInfo> configuredRecords = pwmSetting.getLDAPPermissionInfo();
                     final Collection<LDAPPermissionInfo> returnRecords = new ArrayList<>();
                     for (final LDAPPermissionInfo ldapPermissionInfo : configuredRecords) {
+                        if (!(proxyOverride && publicOverride)) {
+                            returnRecords.add(ldapPermissionInfo);  // include regular self-other permission
+                        }
                         returnRecords.add(new LDAPPermissionInfo(ldapPermissionInfo.getAccess(), LDAPPermissionInfo.Actor.proxy));
                     }
                     return returnRecords;

src/main/java/password/pwm/util/TransactionSizeCalculator.java

@@ -22,103 +22,138 @@
 
 package password.pwm.util;
 
-public class TransactionSizeCalculator {
+import java.util.concurrent.TimeUnit;
 
-    private final TimeDuration setting_Goal;
-    private final TimeDuration setting_OutOfRange;
-    private final int setting_MaxTransactions;
-    private final int setting_MinTransactions;
-    private final int setting_MaxOutOfRangeCount = 3;
+public class TransactionSizeCalculator {
 
-    private final int initialTransactionSize;
+    private final Settings settings;
 
     private volatile int transactionSize;
-    private volatile int outOfRangeCount;
+    private volatile long lastDuration = 1;
 
-    public TransactionSizeCalculator(
-            final long goalTimeMS,
-            final int minTransactions,
-            final int maxTransactions
-    )
-    {
-        this(goalTimeMS, minTransactions, maxTransactions, minTransactions + 1);
-    }
-
-    public TransactionSizeCalculator(
-            final long goalTimeMS,
-            final int minTransactions,
-            final int maxTransactions,
-            final int initialTransactionSize
-    )
-    {
-        this.setting_Goal = new TimeDuration(goalTimeMS);
-        this.setting_OutOfRange = new TimeDuration(setting_Goal.getTotalMilliseconds() * 5);
-        this.setting_MaxTransactions = maxTransactions;
-        this.setting_MinTransactions = minTransactions;
-        {
-            int initialSize = initialTransactionSize;
-            if (initialSize > maxTransactions) {
-                initialSize = maxTransactions;
-            }
-            if (initialSize < minTransactions) {
-                initialSize = minTransactions;
-            }
-            this.initialTransactionSize = initialSize;
-        }
-        transactionSize = initialTransactionSize;
+    public TransactionSizeCalculator(final Settings settings) {
+        this.settings = settings;
+        reset();
     }
 
     public void reset() {
-        transactionSize = initialTransactionSize;
+        transactionSize = settings.getMinTransactions();
+        lastDuration = settings.getDurationGoal().getTotalMilliseconds();
     }
 
     public void recordLastTransactionDuration(final long duration) {
         recordLastTransactionDuration(new TimeDuration(duration));
     }
 
+    public void pause() {
+        Helper.pause(Math.min(lastDuration,settings.getDurationGoal().getTotalMilliseconds() * 2));
+    }
+
     public void recordLastTransactionDuration(final TimeDuration duration)
     {
-        final long difference = Math.abs(duration.getTotalMilliseconds() - setting_Goal.getTotalMilliseconds());
+        lastDuration = duration.getTotalMilliseconds();
+        final long durationGoalMs = settings.getDurationGoal().getTotalMilliseconds();
+        final long difference = Math.abs(duration.getTotalMilliseconds() - durationGoalMs);
+        final int closeThreshold = (int)(durationGoalMs * .15f);
 
         int newTransactionSize;
-        boolean outOfRange = false;
-        if (duration.isShorterThan(setting_Goal)) {
-            if (difference > 100) {
+        if (duration.isShorterThan(settings.getDurationGoal())) {
+            if (difference > closeThreshold) {
                 newTransactionSize = ((int) (transactionSize + (transactionSize * 0.1)) + 1);
             } else {
                 newTransactionSize = transactionSize + 1;
             }
-
-        } else if (duration.isLongerThan(setting_Goal) && duration.isShorterThan(setting_OutOfRange)) {
-            if (difference > 100) {
-            newTransactionSize = ((int) (transactionSize - (transactionSize * 0.1)) - 1);
+        } else if (duration.isLongerThan(settings.getDurationGoal())) {
+            if (difference > (10 * durationGoalMs)) {
+                newTransactionSize = settings.getMinTransactions();
+            } else if (difference > (2 * durationGoalMs)) {
+                newTransactionSize = transactionSize / 2;
+            } else if (difference > closeThreshold) {
+                newTransactionSize = ((int) (transactionSize - (transactionSize * 0.1)) - 1);
             } else {
-            newTransactionSize = transactionSize - 1;
-            }
-
-            if (duration.isLongerThan(setting_OutOfRange)) {
-                outOfRange = true;
+                newTransactionSize = transactionSize - 1;
             }
         } else {
             newTransactionSize = transactionSize;
         }
 
-        if (outOfRange) {
-            outOfRangeCount++;
-            if (outOfRangeCount > setting_MaxOutOfRangeCount) {
-                newTransactionSize = initialTransactionSize;
-                outOfRangeCount = 0;
-            }
-        } else {
-            outOfRangeCount = 0;
-        }
-
-        newTransactionSize = newTransactionSize > setting_MaxTransactions ? setting_MaxTransactions : newTransactionSize;
-        newTransactionSize = newTransactionSize < setting_MinTransactions ? setting_MinTransactions : newTransactionSize;
+        newTransactionSize = Math.min(newTransactionSize, settings.getMaxTransactions());
+        newTransactionSize = Math.max(newTransactionSize, settings.getMinTransactions());
         this.transactionSize = newTransactionSize;
     }
 
     public int getTransactionSize() {
         return transactionSize;
     }
+
+    public static class Settings {
+        private final TimeDuration durationGoal;
+        private final int maxTransactions;
+        private final int minTransactions;
+
+        private Settings(TimeDuration durationGoal, int maxTransactions, int minTransactions) {
+            this.durationGoal = durationGoal;
+            this.maxTransactions = maxTransactions;
+            this.minTransactions = minTransactions;
+
+            if (minTransactions < 1) {
+                throw new IllegalArgumentException("minTransactions must be a positive integer");
+            }
+
+            if (maxTransactions < 1) {
+                throw new IllegalArgumentException("maxTransactions must be a positive integer");
+            }
+
+            if (minTransactions > maxTransactions) {
+                throw new IllegalArgumentException("minTransactions must be less than maxTransactions");
+            }
+
+            if (durationGoal == null) {
+                throw new IllegalArgumentException("durationGoal must not be null");
+            }
+
+            if (durationGoal.getTotalMilliseconds() < 1) {
+                throw new IllegalArgumentException("durationGoal must be greater than 0ms");
+            }
+        }
+
+
+
+        public TimeDuration getDurationGoal() {
+            return durationGoal;
+        }
+
+        public int getMaxTransactions() {
+            return maxTransactions;
+        }
+
+        public int getMinTransactions() {
+            return minTransactions;
+        }
+    }
+
+    public static class SettingsBuilder {
+        private TimeDuration durationGoal = new TimeDuration(100, TimeUnit.MILLISECONDS);
+        private int maxTransactions = 5003;
+        private int minTransactions = 3;
+
+        public SettingsBuilder setDurationGoal(TimeDuration durationGoal) {
+            this.durationGoal = durationGoal;
+            return this;
+        }
+
+        public SettingsBuilder setMaxTransactions(int maxTransactions) {
+            this.maxTransactions = maxTransactions;
+            return this;
+        }
+
+        public SettingsBuilder setMinTransactions(int minTransactions) {
+            this.minTransactions = minTransactions;
+            return this;
+        }
+
+        public Settings createSettings() {
+            return new Settings(durationGoal, maxTransactions, minTransactions);
+        }
+    }
 }

src/main/java/password/pwm/util/WorkQueueProcessor.java

@@ -22,6 +22,7 @@
 
 package password.pwm.util;
 
+import com.google.gson.annotations.SerializedName;
 import password.pwm.PwmApplication;
 import password.pwm.PwmConstants;
 import password.pwm.error.ErrorInformation;
@@ -63,6 +64,7 @@ public class WorkQueueProcessor<W extends Serializable> {
         SUCCESS,
         FAILED,
         RETRY,
+        NOOP,
     }
 
     private static class IDGenerator {
@@ -254,7 +256,7 @@ public class WorkQueueProcessor<W extends Serializable> {
                     return;
                 }
             } catch (Throwable e) {
-                LOGGER.error("error reading queued item: " + e.getMessage(), e);
+                LOGGER.warn("discarding stored record due to parsing error: " + e.getMessage() + ", record=" + nextStrValue);
                 removeQueueTop();
                 return;
             }
@@ -285,6 +287,10 @@ public class WorkQueueProcessor<W extends Serializable> {
                         }
                         break;
 
+                        case NOOP:
+                            break;
+
+
                         default:
                             throw new IllegalStateException("unexpected processResult type " + processResult);
                     }
@@ -303,20 +309,27 @@ public class WorkQueueProcessor<W extends Serializable> {
     }
 
     private static class ItemWrapper<W extends Serializable> implements Serializable {
-        private final Date date;
+        @SerializedName("t")
+        private final Date timestamp;
+
+        @SerializedName("m")
         private final String item;
+
+        @SerializedName("c")
         private final String className;
+
+        @SerializedName("i")
         private final String id;
 
         ItemWrapper(final Date submitDate, final W workItem, final String itemId) {
-            this.date = submitDate;
+            this.timestamp = submitDate;
             this.item = JsonUtil.serialize(workItem);
             this.className = workItem.getClass().getName();
             this.id = itemId;
         }
 
         Date getDate() {
-            return date;
+            return timestamp;
         }
 
         W getWorkItem() throws PwmOperationalException {

src/main/java/password/pwm/util/cli/MainClass.java

@@ -23,9 +23,9 @@
 package password.pwm.util.cli;
 
 import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.EnhancedPatternLayout;
 import org.apache.log4j.Layout;
 import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
 import org.apache.log4j.varia.NullAppender;
 import password.pwm.PwmApplication;
 import password.pwm.PwmApplicationMode;
@@ -53,7 +53,7 @@ import java.util.*;
 public class MainClass {
     private static final PwmLogger LOGGER = PwmLogger.forClass(MainClass.class);
 
-    private static final String LOGGING_PATTERN = "%d{yyyy-MM-dd HH:mm:ss}, %-5p, %c{2}, %m%n";
+    private static final String LOGGING_PATTERN = "%d{yyyy-MM-dd'T'HH:mm:ssX}{GMT}, %-5p, %c{2}, %m%n";
 
     private static MainOptions MAIN_OPTIONS = new MainOptions();
 
@@ -364,7 +364,7 @@ public class MainClass {
             return;
         }
 
-        final Layout patternLayout = new PatternLayout(LOGGING_PATTERN);
+        final Layout patternLayout = new EnhancedPatternLayout(LOGGING_PATTERN);
         final ConsoleAppender consoleAppender = new ConsoleAppender(patternLayout);
         for (final Package logPackage : PwmLogManager.LOGGING_PACKAGES) {
             if (logPackage != null) {

src/main/java/password/pwm/util/localdb/LocalDBStoredQueue.java

@@ -101,6 +101,14 @@ LocalDBStoredQueue implements Queue<String>, Deque<String>
         }
     }
 
+    public void removeFirst(final int removalCount) {
+        try {
+            internalQueue.removeFirst(removalCount, false);
+        } catch (LocalDBException e) {
+            throw new IllegalStateException("unexpected localDB error while modifying queue: " + e.getMessage(), e);
+        }
+    }
+
 // ------------------------ INTERFACE METHODS ------------------------
 
 
@@ -247,7 +255,7 @@ LocalDBStoredQueue implements Queue<String>, Deque<String>
 
     public String pollFirst() {
         try {
-            final List<String> values = internalQueue.removeFirst(1);
+            final List<String> values = internalQueue.removeFirst(1, true);
             if (values == null || values.isEmpty()) {
                 return null;
             }
@@ -375,7 +383,7 @@ LocalDBStoredQueue implements Queue<String>, Deque<String>
         return this.peekFirst();
     }
 
-    public LocalDB getPwmDB() {
+    public LocalDB getLocalDB() {
         return internalQueue.localDB;
     }
 
@@ -606,7 +614,7 @@ LocalDBStoredQueue implements Queue<String>, Deque<String>
             return tailPosition.distanceToHead(headPosition).intValue() + 1;
         }
 
-        List<String> removeFirst(final int removalCount) throws LocalDBException {
+        List<String> removeFirst(final int removalCount, final boolean returnValues) throws LocalDBException {
             try {
                 LOCK.writeLock().lock();
 
@@ -622,9 +630,11 @@ LocalDBStoredQueue implements Queue<String>, Deque<String>
                 int removedPositions = 0;
                 while (removedPositions < removalCount) {
                     removalKeys.add(previousHead.toString());
-                    final String loopValue = localDB.get(DB, previousHead.toString());
-                    if (loopValue != null) {
-                        removedValues.add(loopValue);
+                    if (returnValues) {
+                        final String loopValue = localDB.get(DB, previousHead.toString());
+                        if (loopValue != null) {
+                            removedValues.add(loopValue);
+                        }
                     }
                     previousHead = previousHead.equals(tailPosition) ? previousHead : previousHead.previous();
                     removedPositions++;

src/main/java/password/pwm/util/localdb/LocalDBUtility.java

@@ -41,6 +41,7 @@ import java.io.*;
 import java.math.BigDecimal;
 import java.text.DecimalFormat;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
@@ -185,7 +186,13 @@ public class LocalDBUtility {
         writeStringToOut(out, "beginning localdb import...");
 
         final Date startTime = new Date();
-        final TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(100, 50, 5 * 1000);
+        final TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(
+                new TransactionSizeCalculator.SettingsBuilder()
+                        .setDurationGoal(new TimeDuration(100, TimeUnit.MILLISECONDS))
+                        .setMinTransactions(50)
+                        .setMaxTransactions(5 * 1000)
+                        .createSettings()
+        );
 
         final Map<LocalDB.DB,Map<String,String>> transactionMap = new HashMap<>();
         for (final LocalDB.DB loopDB : LocalDB.DB.values()) {

src/main/java/password/pwm/util/logging/LocalDBLogger.java

@@ -408,7 +408,7 @@ public class LocalDBLogger implements PwmService {
         public void run() {
             try {
                 int cleanupCount = 1;
-                while (cleanupCount > 0 && (status == STATUS.OPEN  && localDBListQueue.getPwmDB().status() == LocalDB.Status.OPEN)) {
+                while (cleanupCount > 0 && (status == STATUS.OPEN  && localDBListQueue.getLocalDB().status() == LocalDB.Status.OPEN)) {
                     cleanupCount = determineTailRemovalCount();
                     if (cleanupCount > 0) {
                         cleanOnWriteFlag.set(true);

src/main/resources/password/pwm/AppProperty.properties

@@ -28,7 +28,8 @@ application.fileLock.waitSeconds=120
 application.wordlistRetryImportSeconds=600
 audit.events.emailFrom=Audit Event Notification <@DefaultEmailFromAddress@>
 audit.events.emailSubject=@PwmAppName@ - Audit Event - %EVENT%
-audit.syslog.message.length=1024
+audit.events.localdb.maxBulkRemovals=301
+audit.syslog.message.length=900
 audit.syslog.message.truncateMsg=[truncated]
 backup.path=backup
 backup.config.count=20
@@ -59,7 +60,7 @@ configEditor.idleTimeoutSeconds=900
 configGuide.idleTimeoutSeconds=3600
 configManager.zipDebug.maxLogLines=100000
 configManager.zipDebug.maxLogSeconds=30
-form.email.regexTest=
+form.email.regexTest=^[_+a-zA-Z0-9-]+(\\.[_a-zA-Z0-9-]+)*@[a-zA-Z0-9-]+(\\.[a-zA-Z0-9-]+)*$
 healthCheck.nominalCheckIntervalSeconds=60
 healthCheck.minimumCheckIntervalSeconds=10
 healthCheck.maximumRecordAgeSeconds=300
@@ -209,7 +210,6 @@ queue.sms.maxCount=100000
 queue.syslog.retryTimeoutMs=30000
 queue.syslog.maxAgeMs=86400000
 queue.syslog.maxCount=100000
-queue.maxCloseTimeoutMs=5000
 reporting.ldap.searchTimeoutMs=1800000
 recaptcha.clientJsUrl=//www.google.com/recaptcha/api.js
 recaptcha.clientIframeUrl=//www.google.com/recaptcha/api/noscript

src/main/resources/password/pwm/config/PwmSetting.xml

@@ -2853,7 +2853,7 @@
         </default>
     </setting>
     <setting hidden="false" key="peopleSearch.orgChart.parentAttribute" level="1">
-        <ldapPermission actor="proxy" access="read"/>
+        <ldapPermission actor="self_other" access="read"/>
         <default>
             <value>manager</value>
         </default>
@@ -3237,11 +3237,6 @@
             <value>{"name":"postalCode","minimumLength":1,"maximumLength":64,"type":"text","required":true,"confirmationRequired":false,"readonly":false,"labels":{"":"Postal Code"},"regexErrors":{"":""},"description":{"":""},"selectOptions":{}}</value>
         </default>
     </setting>
-    <setting hidden="false" key="pwmDb.location" level="2" required="true">
-        <default>
-            <value><![CDATA[LocalDB]]></value>
-        </default>
-    </setting>
     <setting hidden="false" key="db.classname" level="1">
         <default>
             <value />
@@ -3536,6 +3531,11 @@
             <value>false</value>
         </default>
     </setting>
+    <setting hidden="true" key="pwmDb.location" level="2" required="true">
+        <default>
+            <value><![CDATA[LocalDB]]></value>
+        </default>
+    </setting>
     <!-- DEPRECATED SETTINGS -->
     <category hidden="false" key="TEMPLATES">
     </category>

src/main/resources/password/pwm/i18n/Health.properties

@@ -36,7 +36,6 @@ HealthMessage_Email_SendFailure=Unable to send email due to error: %1%
 HealthMessage_MissingResource=missing resource: bundle=%1%, locale=%2%, key=%3%
 HealthMessage_BrokenMethod=broken method invocation for '%1%', error: %2%
 HealthMessage_Config_NoSiteURL=The site URL is not configured, please configure %1%
-HealthMessage_Config_RequireHttps=%1% setting should be set to true for proper security
 HealthMessage_Config_LDAPWireTrace=The %1% setting is enabled and should be disabled for proper security
 HealthMessage_Config_PromiscuousLDAP=%1% setting should be set to false for proper security
 HealthMessage_Config_ShowDetailedErrors=%1% setting should be set to false for proper security

src/main/resources/password/pwm/i18n/Health_nl.properties

@@ -32,7 +32,6 @@ HealthMessage_LDAP_TestUserNoTempPass=het wachtwoord van de testgebruiker kan ni
 HealthMessage_LDAP_TestUserOK=De LDAP-testgebruiker functioneert normaal for profile %1%
 HealthMessage_Health_Config_ConfigMode=De applicatie is momenteel in <b>configuratiemodus</b>.   Iedereen die deze applicatie kan benaderen kan wijzigingen in de configuratie aanbrengen zonder zicht te hoeven aanmelden. Vergrendel de configuratie wanneer u klaar bent om de installatie te beveiligen.
 HealthMessage_Config_NoSiteURL=De URL van de site is niet geconfigureerd en kan niet automatisch gevonden worden; configureer %1%
-HealthMessage_Config_RequireHttps=Optie %1% dient te worden ingeschakeld voor afdoende beveiliging
 HealthMessage_Config_PromiscuousLDAP=Optie %1% dient te worden uitgeschakeld voor afdoende beveiliging
 HealthMessage_Config_AddTestUser=Optie %1% moet worden ingesteld om de correcte werking te controleren
 HealthMessage_Config_ShowDetailedErrors=Optie %1% dient te worden uitgeschakeld voor afdoende beveiliging

src/main/webapp/WEB-INF/jsp/admin-dashboard.jsp

@@ -424,7 +424,7 @@
                                 Local Audit Records
                             </td>
                             <td>
-                                <%= numberFormat.format(dashboard_pwmApplication.getAuditManager().vaultSize()) %>
+                                <%= dashboard_pwmApplication.getAuditManager().sizeToDebugString() %>
                             </td>
                         </tr>
                         <tr>