svc package refactoring

.idea/codeStyleSettings.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectCodeStyleSettingsManager">
+    <option name="PER_PROJECT_SETTINGS">
+      <value>
+        <XML>
+          <option name="XML_LEGACY_SETTINGS_IMPORTED" value="true" />
+        </XML>
+      </value>
+    </option>
+    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default (1)" />
+  </component>
+</project>

.idea/uiDesigner.xml

@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Palette2">
+    <group name="Swing">
+      <item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
+      </item>
+      <item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
+      </item>
+      <item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
+      </item>
+      <item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
+        <default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
+      </item>
+      <item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
+        <initial-values>
+          <property name="text" value="Button" />
+        </initial-values>
+      </item>
+      <item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
+        <initial-values>
+          <property name="text" value="RadioButton" />
+        </initial-values>
+      </item>
+      <item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
+        <initial-values>
+          <property name="text" value="CheckBox" />
+        </initial-values>
+      </item>
+      <item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
+        <initial-values>
+          <property name="text" value="Label" />
+        </initial-values>
+      </item>
+      <item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
+          <preferred-size width="150" height="-1" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
+          <preferred-size width="150" height="-1" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
+          <preferred-size width="150" height="-1" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
+          <preferred-size width="150" height="50" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
+          <preferred-size width="150" height="50" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
+          <preferred-size width="150" height="50" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
+      </item>
+      <item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
+          <preferred-size width="150" height="50" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
+          <preferred-size width="150" height="50" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
+          <preferred-size width="150" height="50" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
+          <preferred-size width="200" height="200" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
+          <preferred-size width="200" height="200" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
+      </item>
+      <item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
+      </item>
+      <item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
+      </item>
+      <item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
+      </item>
+      <item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
+          <preferred-size width="-1" height="20" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
+      </item>
+      <item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
+      </item>
+    </group>
+  </component>
+</project>

pwm/servlet/src/password/pwm/PwmApplication.java

@@ -34,19 +34,30 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.event.AuditEvent;
-import password.pwm.event.AuditManager;
-import password.pwm.event.SystemAuditRecord;
 import password.pwm.health.HealthMonitor;
 import password.pwm.http.servlet.resource.ResourceServletService;
 import password.pwm.ldap.LdapConnectionService;
+import password.pwm.svc.PwmService;
+import password.pwm.svc.cache.CacheService;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.event.AuditService;
+import password.pwm.svc.event.SystemAuditRecord;
+import password.pwm.svc.intruder.IntruderManager;
+import password.pwm.svc.intruder.RecordType;
+import password.pwm.svc.report.ReportService;
 import password.pwm.svc.sessiontrack.SessionTrackService;
-import password.pwm.token.TokenService;
-import password.pwm.util.*;
-import password.pwm.util.cache.CacheService;
+import password.pwm.svc.shorturl.UrlShortenerService;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
+import password.pwm.svc.token.TokenService;
+import password.pwm.svc.wordlist.SeedlistManager;
+import password.pwm.svc.wordlist.SharedHistoryManager;
+import password.pwm.svc.wordlist.WordlistManager;
+import password.pwm.util.FileSystemUtility;
+import password.pwm.util.Helper;
+import password.pwm.util.JsonUtil;
+import password.pwm.util.TimeDuration;
 import password.pwm.util.db.DatabaseAccessorImpl;
-import password.pwm.util.intruder.IntruderManager;
-import password.pwm.util.intruder.RecordType;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBFactory;
 import password.pwm.util.logging.LocalDBLogger;
@@ -58,14 +69,8 @@ import password.pwm.util.operations.CrService;
 import password.pwm.util.operations.OtpService;
 import password.pwm.util.queue.EmailQueueManager;
 import password.pwm.util.queue.SmsQueueManager;
-import password.pwm.util.report.ReportService;
 import password.pwm.util.secure.PwmRandom;
 import password.pwm.util.secure.SecureService;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
-import password.pwm.wordlist.SeedlistManager;
-import password.pwm.wordlist.SharedHistoryManager;
-import password.pwm.wordlist.WordlistManager;
 
 import java.io.File;
 import java.util.*;
@@ -131,7 +136,7 @@ public class PwmApplication {
             DatabaseAccessorImpl.class,
             SharedHistoryManager.class,
             HealthMonitor.class,
-            AuditManager.class,
+            AuditService.class,
             StatisticsManager.class,
             WordlistManager.class,
             SeedlistManager.class,
@@ -410,8 +415,8 @@ public class PwmApplication {
         return (EmailQueueManager)pwmServices.get(EmailQueueManager.class);
     }
 
-    public AuditManager getAuditManager() {
-        return (AuditManager)pwmServices.get(AuditManager.class);
+    public AuditService getAuditManager() {
+        return (AuditService)pwmServices.get(AuditService.class);
     }
 
     public SmsQueueManager getSmsQueue() {

pwm/servlet/src/password/pwm/VersionChecker.java

@@ -36,6 +36,7 @@ import password.pwm.health.HealthStatus;
 import password.pwm.health.HealthTopic;
 import password.pwm.http.client.PwmHttpClient;
 import password.pwm.i18n.Display;
+import password.pwm.svc.PwmService;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.TimeDuration;
 import password.pwm.util.localdb.LocalDB;

pwm/servlet/src/password/pwm/config/FormUtility.java

@@ -36,11 +36,11 @@ import password.pwm.error.*;
 import password.pwm.http.PwmRequest;
 import password.pwm.ldap.UserDataReader;
 import password.pwm.ldap.UserSearchEngine;
+import password.pwm.svc.cache.CacheKey;
+import password.pwm.svc.cache.CachePolicy;
+import password.pwm.svc.cache.CacheService;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.StringUtil;
-import password.pwm.util.cache.CacheKey;
-import password.pwm.util.cache.CachePolicy;
-import password.pwm.util.cache.CacheService;
 import password.pwm.util.logging.PwmLogger;
 
 import java.util.*;

pwm/servlet/src/password/pwm/config/function/SyslogCertImportFunction.java

@@ -29,10 +29,10 @@ import password.pwm.config.SettingUIFunction;
 import password.pwm.config.stored.StoredConfigurationImpl;
 import password.pwm.config.value.X509CertificateValue;
 import password.pwm.error.*;
-import password.pwm.event.SyslogAuditService;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmSession;
 import password.pwm.i18n.Message;
+import password.pwm.svc.event.SyslogAuditService;
 import password.pwm.util.X509Utils;
 
 import java.security.cert.X509Certificate;

pwm/servlet/src/password/pwm/config/stored/ConfigurationReader.java

@@ -31,8 +31,8 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.event.AuditEvent;
-import password.pwm.event.SystemAuditRecord;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.event.SystemAuditRecord;
 import password.pwm.util.FileSystemUtility;
 import password.pwm.util.Helper;
 import password.pwm.util.JsonUtil;

pwm/servlet/src/password/pwm/health/HealthMonitor.java

@@ -24,9 +24,9 @@ package password.pwm.health;
 
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
-import password.pwm.PwmService;
 import password.pwm.config.option.DataStorageMethod;
 import password.pwm.error.PwmException;
+import password.pwm.svc.PwmService;
 import password.pwm.util.logging.PwmLogger;
 
 import java.io.Serializable;

pwm/servlet/src/password/pwm/http/PwmSession.java

@@ -35,14 +35,14 @@ import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.http.bean.*;
 import password.pwm.ldap.UserStatusReader;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.LocaleHelper;
 import password.pwm.util.LoginCookieManager;
 import password.pwm.util.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.secure.PwmRandom;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
 
 import javax.servlet.http.HttpSession;
 import java.io.Serializable;

pwm/servlet/src/password/pwm/http/ServletHelper.java

@@ -35,14 +35,14 @@ import password.pwm.config.PwmSetting;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.IPMatcher;
 import password.pwm.util.LocaleHelper;
 import password.pwm.util.StringUtil;
 import password.pwm.util.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.secure.PwmRandom;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.Cookie;

pwm/servlet/src/password/pwm/http/filter/AuthenticationFilter.java

@@ -43,13 +43,13 @@ import password.pwm.ldap.UserSearchEngine;
 import password.pwm.ldap.auth.AuthenticationType;
 import password.pwm.ldap.auth.PwmAuthenticationSource;
 import password.pwm.ldap.auth.SessionAuthenticator;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.BasicAuthInfo;
 import password.pwm.util.CASAuthenticationHelper;
 import password.pwm.util.LocaleHelper;
 import password.pwm.util.LoginCookieManager;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;

pwm/servlet/src/password/pwm/http/filter/ConfigAccessFilter.java

@@ -17,9 +17,9 @@ import password.pwm.http.PwmSession;
 import password.pwm.http.ServletHelper;
 import password.pwm.http.bean.ConfigManagerBean;
 import password.pwm.ldap.auth.AuthenticationType;
+import password.pwm.svc.intruder.RecordType;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.TimeDuration;
-import password.pwm.util.intruder.RecordType;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.secure.PwmHashAlgorithm;
 import password.pwm.util.secure.PwmSecurityKey;

pwm/servlet/src/password/pwm/http/filter/SessionFilter.java

@@ -34,11 +34,11 @@ import password.pwm.error.PwmError;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.http.*;
+import password.pwm.svc.stats.Statistic;
 import password.pwm.util.Helper;
 import password.pwm.util.StringUtil;
 import password.pwm.util.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.stats.Statistic;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.Cookie;

pwm/servlet/src/password/pwm/http/servlet/AbstractPwmServlet.java

@@ -27,8 +27,8 @@ import password.pwm.PwmApplication;
 import password.pwm.Validator;
 import password.pwm.error.*;
 import password.pwm.http.*;
+import password.pwm.svc.stats.Statistic;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.stats.Statistic;
 import password.pwm.ws.server.RestResultBean;
 
 import javax.servlet.ServletException;

pwm/servlet/src/password/pwm/http/servlet/ActivateUserServlet.java

@@ -33,8 +33,6 @@ import password.pwm.bean.*;
 import password.pwm.config.*;
 import password.pwm.config.option.MessageSendMethod;
 import password.pwm.error.*;
-import password.pwm.event.AuditEvent;
-import password.pwm.event.AuditRecord;
 import password.pwm.http.HttpMethod;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmSession;
@@ -47,15 +45,17 @@ import password.pwm.ldap.UserSearchEngine;
 import password.pwm.ldap.auth.AuthenticationType;
 import password.pwm.ldap.auth.PwmAuthenticationSource;
 import password.pwm.ldap.auth.SessionAuthenticator;
-import password.pwm.token.TokenPayload;
-import password.pwm.token.TokenService;
-import password.pwm.token.TokenType;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.event.AuditRecord;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.token.TokenPayload;
+import password.pwm.svc.token.TokenService;
+import password.pwm.svc.token.TokenType;
 import password.pwm.util.PostChangePasswordAction;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
 import password.pwm.util.operations.ActionExecutor;
 import password.pwm.util.operations.PasswordUtility;
-import password.pwm.util.stats.Statistic;
 import password.pwm.ws.client.rest.RestTokenDataClient;
 
 import javax.servlet.ServletException;

pwm/servlet/src/password/pwm/http/servlet/AdminServlet.java

@@ -32,9 +32,9 @@ import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.http.HttpMethod;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmURL;
+import password.pwm.svc.report.ReportService;
+import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.report.ReportService;
-import password.pwm.util.stats.StatisticsManager;
 
 import javax.servlet.ServletException;
 import javax.servlet.annotation.WebServlet;

pwm/servlet/src/password/pwm/http/servlet/CaptchaServlet.java

@@ -41,12 +41,12 @@ import password.pwm.http.ServletHelper;
 import password.pwm.http.client.PwmHttpClient;
 import password.pwm.http.client.PwmHttpClientRequest;
 import password.pwm.http.client.PwmHttpClientResponse;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.PasswordData;
 import password.pwm.util.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
 
 import javax.servlet.ServletException;
 import javax.servlet.annotation.WebServlet;

pwm/servlet/src/password/pwm/http/servlet/ChangePasswordServlet.java

@@ -41,8 +41,6 @@ import password.pwm.config.PwmSetting;
 import password.pwm.config.option.RequireCurrentPasswordMode;
 import password.pwm.config.profile.PwmPasswordRule;
 import password.pwm.error.*;
-import password.pwm.event.AuditEvent;
-import password.pwm.event.AuditRecord;
 import password.pwm.http.HttpMethod;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmSession;
@@ -51,6 +49,9 @@ import password.pwm.http.bean.LoginInfoBean;
 import password.pwm.i18n.Message;
 import password.pwm.ldap.PasswordChangeProgressChecker;
 import password.pwm.ldap.auth.AuthenticationType;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.event.AuditRecord;
+import password.pwm.svc.stats.Statistic;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.PasswordData;
 import password.pwm.util.PwmPasswordRuleValidator;
@@ -58,7 +59,6 @@ import password.pwm.util.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
 import password.pwm.util.operations.PasswordUtility;
-import password.pwm.util.stats.Statistic;
 import password.pwm.ws.server.RestResultBean;
 
 import javax.servlet.ServletException;

pwm/servlet/src/password/pwm/http/servlet/ForgottenUsernameServlet.java

@@ -42,9 +42,9 @@ import password.pwm.ldap.LdapUserDataReader;
 import password.pwm.ldap.UserDataReader;
 import password.pwm.ldap.UserSearchEngine;
 import password.pwm.ldap.UserStatusReader;
+import password.pwm.svc.stats.Statistic;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.stats.Statistic;
 
 import javax.servlet.ServletException;
 import javax.servlet.annotation.WebServlet;

pwm/servlet/src/password/pwm/http/servlet/GuestRegistrationServlet.java

@@ -49,6 +49,7 @@ import password.pwm.ldap.LdapUserDataReader;
 import password.pwm.ldap.UserDataReader;
 import password.pwm.ldap.UserSearchEngine;
 import password.pwm.ldap.UserStatusReader;
+import password.pwm.svc.stats.Statistic;
 import password.pwm.util.FormMap;
 import password.pwm.util.Helper;
 import password.pwm.util.PasswordData;
@@ -57,7 +58,6 @@ import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
 import password.pwm.util.operations.ActionExecutor;
 import password.pwm.util.operations.PasswordUtility;
-import password.pwm.util.stats.Statistic;
 
 import javax.servlet.ServletException;
 import javax.servlet.annotation.WebServlet;

pwm/servlet/src/password/pwm/http/servlet/NewUserServlet.java

@@ -42,7 +42,6 @@ import password.pwm.config.option.TokenStorageMethod;
 import password.pwm.config.profile.LdapProfile;
 import password.pwm.config.profile.NewUserProfile;
 import password.pwm.error.*;
-import password.pwm.event.AuditEvent;
 import password.pwm.http.HttpMethod;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmSession;
@@ -54,15 +53,16 @@ import password.pwm.ldap.UserDataReader;
 import password.pwm.ldap.UserSearchEngine;
 import password.pwm.ldap.auth.PwmAuthenticationSource;
 import password.pwm.ldap.auth.SessionAuthenticator;
-import password.pwm.token.TokenPayload;
-import password.pwm.token.TokenService;
-import password.pwm.token.TokenType;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.token.TokenPayload;
+import password.pwm.svc.token.TokenService;
+import password.pwm.svc.token.TokenType;
 import password.pwm.util.*;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
 import password.pwm.util.operations.ActionExecutor;
 import password.pwm.util.operations.PasswordUtility;
-import password.pwm.util.stats.Statistic;
 import password.pwm.ws.client.rest.RestTokenDataClient;
 import password.pwm.ws.server.RestResultBean;
 import password.pwm.ws.server.rest.RestCheckPasswordServer;

pwm/servlet/src/password/pwm/http/servlet/SetupOtpServlet.java

@@ -31,19 +31,20 @@ import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.option.ForceSetupPolicy;
 import password.pwm.error.*;
-import password.pwm.event.AuditEvent;
-import password.pwm.event.UserAuditRecord;
 import password.pwm.http.HttpMethod;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmSession;
 import password.pwm.http.bean.SetupOtpBean;
 import password.pwm.ldap.auth.AuthenticationType;
+import password.pwm.svc.PwmService;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.event.UserAuditRecord;
+import password.pwm.svc.stats.Statistic;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.StringUtil;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.operations.OtpService;
 import password.pwm.util.otp.OTPUserRecord;
-import password.pwm.util.stats.Statistic;
 import password.pwm.ws.server.RestResultBean;
 
 import javax.servlet.ServletException;

pwm/servlet/src/password/pwm/http/servlet/SetupResponsesServlet.java

@@ -40,8 +40,6 @@ import password.pwm.bean.UserInfoBean;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.profile.ChallengeProfile;
 import password.pwm.error.*;
-import password.pwm.event.AuditEvent;
-import password.pwm.event.UserAuditRecord;
 import password.pwm.http.HttpMethod;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmSession;
@@ -49,10 +47,12 @@ import password.pwm.http.bean.SetupResponsesBean;
 import password.pwm.i18n.Message;
 import password.pwm.ldap.UserStatusReader;
 import password.pwm.ldap.auth.AuthenticationType;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.event.UserAuditRecord;
+import password.pwm.svc.stats.Statistic;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.stats.Statistic;
 import password.pwm.ws.server.RestResultBean;
 
 import javax.servlet.ServletException;

pwm/servlet/src/password/pwm/http/servlet/ShortcutServlet.java

@@ -34,8 +34,8 @@ import password.pwm.http.HttpMethod;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmSession;
 import password.pwm.ldap.LdapPermissionTester;
+import password.pwm.svc.stats.Statistic;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.stats.Statistic;
 
 import javax.servlet.ServletException;
 import javax.servlet.annotation.WebServlet;

pwm/servlet/src/password/pwm/http/servlet/UpdateProfileServlet.java

@@ -33,19 +33,19 @@ import password.pwm.bean.UserIdentity;
 import password.pwm.bean.UserInfoBean;
 import password.pwm.config.*;
 import password.pwm.error.*;
-import password.pwm.event.AuditEvent;
-import password.pwm.event.AuditRecord;
 import password.pwm.http.HttpMethod;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmSession;
 import password.pwm.http.bean.UpdateProfileBean;
 import password.pwm.i18n.Message;
 import password.pwm.ldap.UserStatusReader;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.event.AuditRecord;
+import password.pwm.svc.stats.Statistic;
 import password.pwm.util.Helper;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
 import password.pwm.util.operations.ActionExecutor;
-import password.pwm.util.stats.Statistic;
 import password.pwm.ws.server.RestResultBean;
 
 import javax.servlet.ServletException;

pwm/servlet/src/password/pwm/http/servlet/configmanager/ConfigManagerServlet.java

@@ -48,6 +48,7 @@ import password.pwm.i18n.Config;
 import password.pwm.i18n.Display;
 import password.pwm.i18n.Message;
 import password.pwm.ldap.LdapOperationsHelper;
+import password.pwm.svc.PwmService;
 import password.pwm.util.FileSystemUtility;
 import password.pwm.util.Helper;
 import password.pwm.util.JsonUtil;

pwm/servlet/src/password/pwm/http/servlet/configmanager/ConfigManagerWordlistServlet.java

@@ -34,10 +34,10 @@ import password.pwm.http.PwmRequest;
 import password.pwm.http.ServletHelper;
 import password.pwm.http.servlet.AbstractPwmServlet;
 import password.pwm.i18n.Message;
+import password.pwm.svc.wordlist.StoredWordlistDataBean;
+import password.pwm.svc.wordlist.WordlistType;
 import password.pwm.util.Helper;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.wordlist.StoredWordlistDataBean;
-import password.pwm.wordlist.WordlistType;
 import password.pwm.ws.server.RestResultBean;
 
 import javax.servlet.ServletException;

pwm/servlet/src/password/pwm/http/servlet/forgottenpw/ForgottenPasswordServlet.java

@@ -43,7 +43,6 @@ import password.pwm.config.profile.ForgottenPasswordProfile;
 import password.pwm.config.profile.ProfileType;
 import password.pwm.config.profile.ProfileUtility;
 import password.pwm.error.*;
-import password.pwm.event.AuditEvent;
 import password.pwm.http.HttpMethod;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmSession;
@@ -60,14 +59,17 @@ import password.pwm.ldap.auth.AuthenticationType;
 import password.pwm.ldap.auth.AuthenticationUtility;
 import password.pwm.ldap.auth.PwmAuthenticationSource;
 import password.pwm.ldap.auth.SessionAuthenticator;
-import password.pwm.token.TokenPayload;
-import password.pwm.token.TokenService;
-import password.pwm.token.TokenType;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.intruder.RecordType;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
+import password.pwm.svc.token.TokenPayload;
+import password.pwm.svc.token.TokenService;
+import password.pwm.svc.token.TokenType;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.PasswordData;
 import password.pwm.util.PostChangePasswordAction;
 import password.pwm.util.RandomPasswordGenerator;
-import password.pwm.util.intruder.RecordType;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
 import password.pwm.util.operations.ActionExecutor;
@@ -75,8 +77,6 @@ import password.pwm.util.operations.OtpService;
 import password.pwm.util.operations.PasswordUtility;
 import password.pwm.util.operations.cr.NMASCrOperator;
 import password.pwm.util.otp.OTPUserRecord;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
 import password.pwm.ws.client.rest.RestTokenDataClient;
 import password.pwm.ws.client.rest.naaf.PwmNAAFVerificationMethod;
 

pwm/servlet/src/password/pwm/http/servlet/helpdesk/HelpdeskDetailInfoBean.java

@@ -24,7 +24,7 @@ package password.pwm.http.servlet.helpdesk;
 
 import password.pwm.bean.UserInfoBean;
 import password.pwm.config.FormConfiguration;
-import password.pwm.event.UserAuditRecord;
+import password.pwm.svc.event.UserAuditRecord;
 
 import java.io.Serializable;
 import java.util.Date;

pwm/servlet/src/password/pwm/http/servlet/helpdesk/HelpdeskServlet.java

@@ -41,8 +41,6 @@ import password.pwm.config.option.HelpdeskUIMode;
 import password.pwm.config.option.MessageSendMethod;
 import password.pwm.config.profile.HelpdeskProfile;
 import password.pwm.error.*;
-import password.pwm.event.AuditEvent;
-import password.pwm.event.HelpdeskAuditRecord;
 import password.pwm.http.HttpMethod;
 import password.pwm.http.PwmHttpRequestWrapper;
 import password.pwm.http.PwmRequest;
@@ -51,19 +49,21 @@ import password.pwm.http.servlet.AbstractPwmServlet;
 import password.pwm.i18n.Display;
 import password.pwm.i18n.Message;
 import password.pwm.ldap.*;
-import password.pwm.token.TokenService;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.event.HelpdeskAuditRecord;
+import password.pwm.svc.intruder.IntruderManager;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
+import password.pwm.svc.token.TokenService;
 import password.pwm.util.Helper;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.LocaleHelper;
 import password.pwm.util.TimeDuration;
-import password.pwm.util.intruder.IntruderManager;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
 import password.pwm.util.operations.ActionExecutor;
 import password.pwm.util.operations.OtpService;
 import password.pwm.util.otp.OTPUserRecord;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
 import password.pwm.ws.server.RestResultBean;
 
 import javax.servlet.ServletException;
@@ -778,15 +778,15 @@ public class HelpdeskServlet extends AbstractPwmServlet {
                     tokenKey
             );
         } catch (PwmException e) {
-            LOGGER.error(pwmRequest,e.getErrorInformation());
+            LOGGER.error(pwmRequest, e.getErrorInformation());
             pwmRequest.outputJsonResult(RestResultBean.fromError(e.getErrorInformation(),pwmRequest));
             return;
         }
 
         StatisticsManager.incrementStat(pwmRequest,Statistic.HELPDESK_TOKENS_SENT);
         final HashMap<String,String> output = new HashMap<>();
-        output.put("destination",destDisplayString.toString());
-        output.put("token",tokenKey);
+        output.put("destination", destDisplayString.toString());
+        output.put("token", tokenKey);
         final RestResultBean restResultBean = new RestResultBean();
         restResultBean.setData(output);
         pwmRequest.outputJsonResult(restResultBean);

pwm/servlet/src/password/pwm/http/servlet/peoplesearch/PeopleSearchServlet.java

@@ -41,15 +41,15 @@ import password.pwm.http.PwmRequest;
 import password.pwm.http.servlet.AbstractPwmServlet;
 import password.pwm.i18n.Display;
 import password.pwm.ldap.*;
+import password.pwm.svc.cache.CacheKey;
+import password.pwm.svc.cache.CachePolicy;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.LocaleHelper;
 import password.pwm.util.TimeDuration;
-import password.pwm.util.cache.CacheKey;
-import password.pwm.util.cache.CachePolicy;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
 import password.pwm.ws.server.RestResultBean;
 
 import javax.servlet.ServletException;

pwm/servlet/src/password/pwm/http/servlet/resource/ResourceFileServlet.java

@@ -34,11 +34,11 @@ import password.pwm.http.HttpMethod;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.ServletHelper;
 import password.pwm.http.servlet.AbstractPwmServlet;
+import password.pwm.svc.stats.EventRateMeter;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.StringUtil;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.stats.EventRateMeter;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
 
 import javax.servlet.ServletContext;
 import javax.servlet.ServletException;

pwm/servlet/src/password/pwm/http/servlet/resource/ResourceServletService.java

@@ -25,11 +25,11 @@ package password.pwm.http.servlet.resource;
 import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
-import password.pwm.PwmService;
 import password.pwm.error.PwmException;
 import password.pwm.health.HealthRecord;
+import password.pwm.svc.PwmService;
+import password.pwm.svc.stats.EventRateMeter;
 import password.pwm.util.Percent;
-import password.pwm.util.stats.EventRateMeter;
 
 import java.math.BigDecimal;
 import java.util.Collections;

pwm/servlet/src/password/pwm/ldap/LdapConnectionService.java

@@ -25,7 +25,6 @@ package password.pwm.ldap;
 import com.google.gson.reflect.TypeToken;
 import com.novell.ldapchai.provider.ChaiProvider;
 import password.pwm.PwmApplication;
-import password.pwm.PwmService;
 import password.pwm.config.option.DataStorageMethod;
 import password.pwm.config.profile.LdapProfile;
 import password.pwm.error.ErrorInformation;
@@ -33,6 +32,7 @@ import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
+import password.pwm.svc.PwmService;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.logging.PwmLogger;
 

pwm/servlet/src/password/pwm/ldap/LdapOperationsHelper.java

@@ -45,13 +45,13 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.PasswordData;
 import password.pwm.util.StringUtil;
 import password.pwm.util.X509Utils;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
 
 import javax.net.ssl.X509TrustManager;
 import java.security.cert.X509Certificate;

pwm/servlet/src/password/pwm/ldap/LdapPermissionTester.java

@@ -41,8 +41,8 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.util.cache.CacheKey;
-import password.pwm.util.cache.CachePolicy;
+import password.pwm.svc.cache.CacheKey;
+import password.pwm.svc.cache.CachePolicy;
 import password.pwm.util.logging.PwmLogger;
 
 import java.util.Collections;

pwm/servlet/src/password/pwm/ldap/UserSearchEngine.java

@@ -31,7 +31,6 @@ import com.novell.ldapchai.util.SearchHelper;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.PwmConstants;
-import password.pwm.PwmService;
 import password.pwm.bean.SessionLabel;
 import password.pwm.bean.UserIdentity;
 import password.pwm.config.FormConfiguration;
@@ -40,11 +39,12 @@ import password.pwm.config.option.DuplicateMode;
 import password.pwm.config.profile.LdapProfile;
 import password.pwm.error.*;
 import password.pwm.http.PwmRequest;
+import password.pwm.svc.PwmService;
+import password.pwm.svc.stats.Statistic;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.StringUtil;
 import password.pwm.util.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.stats.Statistic;
 
 import java.io.Serializable;
 import java.util.*;

pwm/servlet/src/password/pwm/ldap/auth/LDAPAuthenticationRequest.java

@@ -40,18 +40,18 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.event.AuditEvent;
 import password.pwm.ldap.LdapOperationsHelper;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.intruder.IntruderManager;
+import password.pwm.svc.intruder.RecordType;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.PasswordData;
 import password.pwm.util.RandomPasswordGenerator;
 import password.pwm.util.TimeDuration;
-import password.pwm.util.intruder.IntruderManager;
-import password.pwm.util.intruder.RecordType;
 import password.pwm.util.logging.PwmLogLevel;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.operations.PasswordUtility;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
 
 import java.util.Collections;
 import java.util.Date;

pwm/servlet/src/password/pwm/ldap/auth/SessionAuthenticator.java

@@ -41,12 +41,12 @@ import password.pwm.http.PwmSession;
 import password.pwm.ldap.LdapOperationsHelper;
 import password.pwm.ldap.UserSearchEngine;
 import password.pwm.ldap.UserStatusReader;
+import password.pwm.svc.intruder.IntruderManager;
+import password.pwm.svc.intruder.RecordType;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.PasswordData;
-import password.pwm.util.intruder.IntruderManager;
-import password.pwm.util.intruder.RecordType;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
 
 import java.util.Date;
 

pwm/servlet/src/password/pwm/PwmService.java → pwm/servlet/src/password/pwm/svc/PwmService.java

@@ -1,69 +1,70 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm;
-
-import password.pwm.config.option.DataStorageMethod;
-import password.pwm.error.PwmException;
-import password.pwm.health.HealthRecord;
-
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.List;
-
-/**
- * An interface for daemon/background services.  Services are initialized, shutdown and accessed via {@link PwmApplication}.  Some services
- * will have associated background threads, so implementations will generally be thread safe.
- */
-public interface PwmService {
-
-    enum STATUS {
-        NEW,
-        OPENING,
-        OPEN,
-        CLOSED
-    }
-
-    STATUS status();
-
-    void init(PwmApplication pwmApplication) throws PwmException;
-    
-    void close();
-
-    List<HealthRecord> healthCheck();
-
-    ServiceInfo serviceInfo();
-
-    class ServiceInfo implements Serializable {
-        public Collection<DataStorageMethod> usedStorageMethods;
-
-        public ServiceInfo(Collection<DataStorageMethod> usedStorageMethods)
-        {
-            this.usedStorageMethods = usedStorageMethods;
-        }
-
-        public Collection<DataStorageMethod> getUsedStorageMethods()
-        {
-            return usedStorageMethods;
-        }
-    }
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc;
+
+import password.pwm.PwmApplication;
+import password.pwm.config.option.DataStorageMethod;
+import password.pwm.error.PwmException;
+import password.pwm.health.HealthRecord;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * An interface for daemon/background services.  Services are initialized, shutdown and accessed via {@link PwmApplication}.  Some services
+ * will have associated background threads, so implementations will generally be thread safe.
+ */
+public interface PwmService {
+
+    enum STATUS {
+        NEW,
+        OPENING,
+        OPEN,
+        CLOSED
+    }
+
+    STATUS status();
+
+    void init(PwmApplication pwmApplication) throws PwmException;
+    
+    void close();
+
+    List<HealthRecord> healthCheck();
+
+    ServiceInfo serviceInfo();
+
+    class ServiceInfo implements Serializable {
+        public Collection<DataStorageMethod> usedStorageMethods;
+
+        public ServiceInfo(Collection<DataStorageMethod> usedStorageMethods)
+        {
+            this.usedStorageMethods = usedStorageMethods;
+        }
+
+        public Collection<DataStorageMethod> getUsedStorageMethods()
+        {
+            return usedStorageMethods;
+        }
+    }
+}

pwm/servlet/src/password/pwm/util/cache/CacheKey.java → pwm/servlet/src/password/pwm/svc/cache/CacheKey.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.cache;
+package password.pwm.svc.cache;
 
 import password.pwm.bean.UserIdentity;
 import password.pwm.error.PwmUnrecoverableException;

pwm/servlet/src/password/pwm/util/cache/CachePolicy.java → pwm/servlet/src/password/pwm/svc/cache/CachePolicy.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.cache;
+package password.pwm.svc.cache;
 
 import java.io.Serializable;
 import java.util.Date;

pwm/servlet/src/password/pwm/util/cache/CacheService.java → pwm/servlet/src/password/pwm/svc/cache/CacheService.java

@@ -20,15 +20,15 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.cache;
+package password.pwm.svc.cache;
 
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
-import password.pwm.PwmService;
 import password.pwm.config.option.DataStorageMethod;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
+import password.pwm.svc.PwmService;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.TimeDuration;
 import password.pwm.util.localdb.LocalDB;

pwm/servlet/src/password/pwm/util/cache/CacheStore.java → pwm/servlet/src/password/pwm/svc/cache/CacheStore.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.cache;
+package password.pwm.svc.cache;
 
 import password.pwm.error.PwmUnrecoverableException;
 

pwm/servlet/src/password/pwm/util/cache/CacheStoreInfo.java → pwm/servlet/src/password/pwm/svc/cache/CacheStoreInfo.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.cache;
+package password.pwm.svc.cache;
 
 import java.io.Serializable;
 

pwm/servlet/src/password/pwm/util/cache/LocalDBCacheStore.java → pwm/servlet/src/password/pwm/svc/cache/LocalDBCacheStore.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.cache;
+package password.pwm.svc.cache;
 
 import password.pwm.PwmApplication;
 import password.pwm.error.PwmUnrecoverableException;

pwm/servlet/src/password/pwm/util/cache/MemoryCacheStore.java → pwm/servlet/src/password/pwm/svc/cache/MemoryCacheStore.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.cache;
+package password.pwm.svc.cache;
 
 import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
 import password.pwm.error.PwmUnrecoverableException;

pwm/servlet/src/password/pwm/event/AuditEvent.java → pwm/servlet/src/password/pwm/svc/event/AuditEvent.java

@@ -1,113 +1,113 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.event;
-
-import password.pwm.config.Configuration;
-import password.pwm.i18n.Message;
-
-import java.util.Locale;
-
-public enum AuditEvent {
-
-    // system events
-    STARTUP(Message.EventLog_Startup, Type.SYSTEM),
-    SHUTDOWN(Message.EventLog_Shutdown, Type.SYSTEM),
-    FATAL_EVENT(Message.EventLog_FatalEvent, Type.SYSTEM),
-    MODIFY_CONFIGURATION(Message.EventLog_ModifyConfiguration, Type.SYSTEM),
-    INTRUDER_LOCK(Message.EventLog_IntruderLockout, Type.SYSTEM),
-    INTRUDER_ATTEMPT(Message.EventLog_IntruderAttempt, Type.SYSTEM),
-
-    // user events not stored in user event history
-    AUTHENTICATE(Message.EventLog_Authenticate, Type.USER),
-    AGREEMENT_PASSED(Message.EventLog_AgreementPassed, Type.USER),
-    TOKEN_ISSUED(Message.EventLog_TokenIssued, Type.USER),
-    TOKEN_CLAIMED(Message.EventLog_TokenClaimed, Type.USER),
-    CLEAR_RESPONSES(Message.EventLog_ClearResponses, Type.USER),
-
-    // user events stored in user event history
-    CHANGE_PASSWORD(Message.EventLog_ChangePassword, Type.USER),
-    UNLOCK_PASSWORD(Message.EventLog_UnlockPassword, Type.USER),
-    RECOVER_PASSWORD(Message.EventLog_RecoverPassword, Type.USER),
-    SET_RESPONSES(Message.EventLog_SetupResponses, Type.USER),
-    SET_OTP_SECRET(Message.Eventlog_SetupOtpSecret, Type.USER),
-    ACTIVATE_USER(Message.EventLog_ActivateUser, Type.USER),
-    CREATE_USER(Message.EventLog_CreateUser, Type.USER),
-    UPDATE_PROFILE(Message.EventLog_UpdateProfile, Type.USER),
-    INTRUDER_USER(Message.EventLog_IntruderUser, Type.USER),
-
-    // helpdesk events
-    HELPDESK_SET_PASSWORD(Message.EventLog_HelpdeskSetPassword, Type.HELPDESK),
-    HELPDESK_UNLOCK_PASSWORD(Message.EventLog_HelpdeskUnlockPassword, Type.HELPDESK),
-    HELPDESK_CLEAR_RESPONSES(Message.EventLog_HelpdeskClearResponses, Type.HELPDESK),
-    HELPDESK_CLEAR_OTP_SECRET(Message.EventLog_HelpdeskClearOtpSecret, Type.HELPDESK),
-    HELPDESK_ACTION(Message.EventLog_HelpdeskAction, Type.HELPDESK),
-    HELPDESK_DELETE_USER(Message.EventLog_HelpdeskDeleteUser, Type.HELPDESK),
-    HELPDESK_VIEW_DETAIL(Message.EventLog_HelpdeskViewDetail, Type.HELPDESK),
-    HELPDESK_VERIFY_OTP(Message.EventLog_HelpdeskViewDetail, Type.HELPDESK),
-
-
-    ;
-
-    final private Message message;
-    private Type type;
-
-    AuditEvent(final Message message, final Type type) {
-        this.message = message;
-        this.type = type;
-    }
-
-    public Message getMessage() {
-        return message;
-    }
-
-    public static AuditEvent forKey(final String key) {
-        for (final AuditEvent loopEvent : AuditEvent.values()) {
-            final Message message = loopEvent.getMessage();
-            if (message != null) {
-                final String resourceKey = message.getKey();
-                if (resourceKey.equals(key)) {
-                    return loopEvent;
-                }
-            }
-        }
-
-        return null;
-    }
-
-    public String getLocalizedString(final Configuration config, final Locale locale) {
-        if (this.getMessage() == null) {
-            return "[unknown event]";
-        }
-        return Message.getLocalizedMessage(locale,this.getMessage(),config);
-    }
-
-    public Type getType() {
-        return type;
-    }
-
-    public enum Type {
-        USER,
-        SYSTEM,
-        HELPDESK,
-    }
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.event;
+
+import password.pwm.config.Configuration;
+import password.pwm.i18n.Message;
+
+import java.util.Locale;
+
+public enum AuditEvent {
+
+    // system events
+    STARTUP(Message.EventLog_Startup, Type.SYSTEM),
+    SHUTDOWN(Message.EventLog_Shutdown, Type.SYSTEM),
+    FATAL_EVENT(Message.EventLog_FatalEvent, Type.SYSTEM),
+    MODIFY_CONFIGURATION(Message.EventLog_ModifyConfiguration, Type.SYSTEM),
+    INTRUDER_LOCK(Message.EventLog_IntruderLockout, Type.SYSTEM),
+    INTRUDER_ATTEMPT(Message.EventLog_IntruderAttempt, Type.SYSTEM),
+
+    // user events not stored in user event history
+    AUTHENTICATE(Message.EventLog_Authenticate, Type.USER),
+    AGREEMENT_PASSED(Message.EventLog_AgreementPassed, Type.USER),
+    TOKEN_ISSUED(Message.EventLog_TokenIssued, Type.USER),
+    TOKEN_CLAIMED(Message.EventLog_TokenClaimed, Type.USER),
+    CLEAR_RESPONSES(Message.EventLog_ClearResponses, Type.USER),
+
+    // user events stored in user event history
+    CHANGE_PASSWORD(Message.EventLog_ChangePassword, Type.USER),
+    UNLOCK_PASSWORD(Message.EventLog_UnlockPassword, Type.USER),
+    RECOVER_PASSWORD(Message.EventLog_RecoverPassword, Type.USER),
+    SET_RESPONSES(Message.EventLog_SetupResponses, Type.USER),
+    SET_OTP_SECRET(Message.Eventlog_SetupOtpSecret, Type.USER),
+    ACTIVATE_USER(Message.EventLog_ActivateUser, Type.USER),
+    CREATE_USER(Message.EventLog_CreateUser, Type.USER),
+    UPDATE_PROFILE(Message.EventLog_UpdateProfile, Type.USER),
+    INTRUDER_USER(Message.EventLog_IntruderUser, Type.USER),
+
+    // helpdesk events
+    HELPDESK_SET_PASSWORD(Message.EventLog_HelpdeskSetPassword, Type.HELPDESK),
+    HELPDESK_UNLOCK_PASSWORD(Message.EventLog_HelpdeskUnlockPassword, Type.HELPDESK),
+    HELPDESK_CLEAR_RESPONSES(Message.EventLog_HelpdeskClearResponses, Type.HELPDESK),
+    HELPDESK_CLEAR_OTP_SECRET(Message.EventLog_HelpdeskClearOtpSecret, Type.HELPDESK),
+    HELPDESK_ACTION(Message.EventLog_HelpdeskAction, Type.HELPDESK),
+    HELPDESK_DELETE_USER(Message.EventLog_HelpdeskDeleteUser, Type.HELPDESK),
+    HELPDESK_VIEW_DETAIL(Message.EventLog_HelpdeskViewDetail, Type.HELPDESK),
+    HELPDESK_VERIFY_OTP(Message.EventLog_HelpdeskViewDetail, Type.HELPDESK),
+
+
+    ;
+
+    final private Message message;
+    private Type type;
+
+    AuditEvent(final Message message, final Type type) {
+        this.message = message;
+        this.type = type;
+    }
+
+    public Message getMessage() {
+        return message;
+    }
+
+    public static AuditEvent forKey(final String key) {
+        for (final AuditEvent loopEvent : AuditEvent.values()) {
+            final Message message = loopEvent.getMessage();
+            if (message != null) {
+                final String resourceKey = message.getKey();
+                if (resourceKey.equals(key)) {
+                    return loopEvent;
+                }
+            }
+        }
+
+        return null;
+    }
+
+    public String getLocalizedString(final Configuration config, final Locale locale) {
+        if (this.getMessage() == null) {
+            return "[unknown event]";
+        }
+        return Message.getLocalizedMessage(locale,this.getMessage(),config);
+    }
+
+    public Type getType() {
+        return type;
+    }
+
+    public enum Type {
+        USER,
+        SYSTEM,
+        HELPDESK,
+    }
+}

pwm/servlet/src/password/pwm/event/AuditRecord.java → pwm/servlet/src/password/pwm/svc/event/AuditRecord.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.event;
+package password.pwm.svc.event;
 
 import java.io.Serializable;
 import java.util.Date;

pwm/servlet/src/password/pwm/event/AuditManager.java → pwm/servlet/src/password/pwm/svc/event/AuditService.java

@@ -1,500 +1,500 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.event;
-
-import org.apache.commons.csv.CSVPrinter;
-import password.pwm.AppProperty;
-import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
-import password.pwm.PwmService;
-import password.pwm.bean.EmailItemBean;
-import password.pwm.bean.SessionLabel;
-import password.pwm.bean.UserIdentity;
-import password.pwm.bean.UserInfoBean;
-import password.pwm.config.Configuration;
-import password.pwm.config.PwmSetting;
-import password.pwm.config.option.DataStorageMethod;
-import password.pwm.config.option.UserEventStorageMethod;
-import password.pwm.error.*;
-import password.pwm.health.HealthRecord;
-import password.pwm.health.HealthStatus;
-import password.pwm.health.HealthTopic;
-import password.pwm.http.PwmSession;
-import password.pwm.ldap.LdapOperationsHelper;
-import password.pwm.util.Helper;
-import password.pwm.util.JsonUtil;
-import password.pwm.util.LocaleHelper;
-import password.pwm.util.TimeDuration;
-import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.macro.MacroMachine;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.*;
-
-public class AuditManager implements PwmService {
-    private static final PwmLogger LOGGER = PwmLogger.forClass(AuditManager.class);
-
-    private STATUS status = STATUS.NEW;
-    private AuditSettings settings;
-    private ServiceInfo serviceInfo = new ServiceInfo(Collections.<DataStorageMethod>emptyList());
-
-    private SyslogAuditService syslogManager;
-    private ErrorInformation lastError;
-    private UserHistoryStore userHistoryStore;
-    private AuditVault auditVault;
-
-    private PwmApplication pwmApplication;
-
-    public AuditManager() {
-    }
-
-    public HelpdeskAuditRecord createHelpdeskAuditRecord(
-            final AuditEvent eventCode,
-            final UserIdentity perpetrator,
-            final String message,
-            final UserIdentity target,
-            final String sourceAddress,
-            final String sourceHost
-    )
-    {
-        String perpUserDN = null, perpUserID = null, perpLdapProfile = null, targetUserDN = null, targetUserID = null, targetLdapProfile = null;
-        if (perpetrator != null) {
-            perpUserDN = perpetrator.getUserDN();
-            perpLdapProfile = perpetrator.getLdapProfileID();
-            try {
-                perpUserID = LdapOperationsHelper.readLdapUsernameValue(pwmApplication,perpetrator);
-            } catch (Exception e) {
-                LOGGER.error("unable to read userID for " + perpetrator + ", error: " + e.getMessage());
-            }
-        }
-        if (target != null) {
-            targetUserDN = target.getUserDN();
-            targetLdapProfile = target.getLdapProfileID();
-            try {
-                targetUserID = LdapOperationsHelper.readLdapUsernameValue(pwmApplication,target);
-            } catch (Exception e) {
-                LOGGER.error("unable to read userID for " + perpetrator + ", error: " + e.getMessage());
-            }
-        }
-
-        return HelpdeskAuditRecord.create(eventCode, perpUserID, perpUserDN, perpLdapProfile, message, targetUserID, targetUserDN,
-                targetLdapProfile, sourceAddress, sourceHost);
-    }
-
-    public UserAuditRecord createUserAuditRecord(
-            final AuditEvent eventCode,
-            final UserIdentity perpetrator,
-            final String message,
-            final String sourceAddress,
-            final String sourceHost
-    )
-    {
-        String perpUserDN = null, perpUserID = null, perpLdapProfile = null, targetUserDN = null, targetUserID = null, targetLdapProfile = null;
-        if (perpetrator != null) {
-            perpUserDN = perpetrator.getUserDN();
-            perpLdapProfile = perpetrator.getLdapProfileID();
-            try {
-                perpUserID = LdapOperationsHelper.readLdapUsernameValue(pwmApplication,perpetrator);
-            } catch (Exception e) {
-                LOGGER.error("unable to read userID for " + perpetrator + ", error: " + e.getMessage());
-            }
-        }
-
-        return HelpdeskAuditRecord.create(eventCode, perpUserID, perpUserDN, perpLdapProfile, message, targetUserID, targetUserDN,
-                targetLdapProfile, sourceAddress, sourceHost);
-    }
-
-    public SystemAuditRecord createSystemAuditRecord(
-            final AuditEvent eventCode,
-            final String message
-    )
-    {
-        return SystemAuditRecord.create(eventCode, message, pwmApplication.getInstanceID());
-    }
-
-    public UserAuditRecord createUserAuditRecord(
-            final AuditEvent eventCode,
-            final UserIdentity perpetrator,
-            final SessionLabel sessionLabel
-    )
-    {
-        return createUserAuditRecord(
-                eventCode,
-                perpetrator,
-                sessionLabel,
-                null
-        );
-    }
-
-    public UserAuditRecord createUserAuditRecord(
-            final AuditEvent eventCode,
-            final UserIdentity perpetrator,
-            final SessionLabel sessionLabel,
-            final String message
-    )
-    {
-        return createUserAuditRecord(
-                eventCode,
-                perpetrator,
-                message,
-                sessionLabel != null ? sessionLabel.getSrcAddress() : null,
-                sessionLabel != null ? sessionLabel.getSrcHostname() : null
-        );
-    }
-
-    public UserAuditRecord createUserAuditRecord(
-            final AuditEvent eventCode,
-            final UserInfoBean userInfoBean,
-            final PwmSession pwmSession
-    )
-    {
-        return createUserAuditRecord(
-                eventCode,
-                userInfoBean.getUserIdentity(),
-                null,
-                pwmSession.getSessionStateBean().getSrcAddress(),
-                pwmSession.getSessionStateBean().getSrcHostname()
-        );
-    }
-
-
-    public STATUS status() {
-        return status;
-    }
-
-    public void init(PwmApplication pwmApplication) throws PwmException {
-        this.status = STATUS.OPENING;
-        this.pwmApplication = pwmApplication;
-
-        settings = new AuditSettings(pwmApplication.getConfig());
-
-        if (pwmApplication.getApplicationMode() == null || pwmApplication.getApplicationMode() == PwmApplication.MODE.READ_ONLY) {
-            this.status = STATUS.CLOSED;
-            LOGGER.warn("unable to start - Application is in read-only mode");
-            return;
-        }
-
-        if (pwmApplication.getLocalDB() == null || pwmApplication.getLocalDB().status() != LocalDB.Status.OPEN) {
-            this.status = STATUS.CLOSED;
-            LOGGER.warn("unable to start - LocalDB is not available");
-            return;
-        }
-
-        final String syslogConfigString = pwmApplication.getConfig().readSettingAsString(PwmSetting.AUDIT_SYSLOG_SERVERS);
-        if (syslogConfigString != null && !syslogConfigString.isEmpty()) {
-            try {
-                syslogManager = new SyslogAuditService(pwmApplication);
-            } catch (Exception e) {
-                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_SYSLOG_WRITE_ERROR, "startup error: " + e.getMessage());
-                LOGGER.error(errorInformation.toDebugStr());
-            }
-        }
-        {
-            final UserEventStorageMethod userEventStorageMethod = pwmApplication.getConfig().readSettingAsEnum(PwmSetting.EVENTS_USER_STORAGE_METHOD, UserEventStorageMethod.class);
-            final String debugMsg;
-            final DataStorageMethod storageMethodUsed;
-            switch (userEventStorageMethod) {
-                case AUTO:
-                    if (pwmApplication.getConfig().hasDbConfigured()) {
-                        debugMsg = "starting using auto-configured data store, Remote Database selected";
-                        this.userHistoryStore = new DatabaseUserHistory(pwmApplication);
-                        storageMethodUsed = DataStorageMethod.DB;
-                    } else {
-                        debugMsg = "starting using auto-configured data store, LDAP selected";
-                        this.userHistoryStore = new LdapXmlUserHistory(pwmApplication);
-                        storageMethodUsed = DataStorageMethod.LDAP;
-                    }
-                    break;
-
-                case DATABASE:
-                    this.userHistoryStore = new DatabaseUserHistory(pwmApplication);
-                    debugMsg = "starting using Remote Database data store";
-                    storageMethodUsed = DataStorageMethod.DB;
-                    break;
-
-                case LDAP:
-                    this.userHistoryStore = new LdapXmlUserHistory(pwmApplication);
-                    debugMsg = "starting using LocalDB data store";
-                    storageMethodUsed = DataStorageMethod.LDAP;
-                    break;
-
-                default:
-                    lastError = new ErrorInformation(PwmError.ERROR_UNKNOWN,"unknown storageMethod selected: " + userEventStorageMethod);
-                    status = STATUS.CLOSED;
-                    return;
-            }
-            LOGGER.info(debugMsg);
-            serviceInfo = new ServiceInfo(Collections.singletonList(storageMethodUsed));
-        }
-        {
-            final TimeDuration maxRecordAge = new TimeDuration(pwmApplication.getConfig().readSettingAsLong(PwmSetting.EVENTS_AUDIT_MAX_AGE) * 1000);
-            final int maxRecords = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.AUDIT_VAULT_MAX_RECORDS));
-            final AuditVault.Settings settings = new AuditVault.Settings(
-                    maxRecords,
-                    maxRecordAge
-            );
-
-            if (pwmApplication.getLocalDB() != null && pwmApplication.getApplicationMode() != PwmApplication.MODE.READ_ONLY) {
-                auditVault = new LocalDbAuditVault(pwmApplication, pwmApplication.getLocalDB());
-                auditVault.init(settings);
-            }
-        }
-
-        this.status = STATUS.OPEN;
-    }
-
-    @Override
-    public void close() {
-        if (syslogManager != null) {
-            syslogManager.close();
-        }
-        this.status = STATUS.CLOSED;
-    }
-
-    @Override
-    public List<HealthRecord> healthCheck() {
-        if (status != STATUS.OPEN) {
-            return Collections.emptyList();
-        }
-
-        final List<HealthRecord> healthRecords = new ArrayList<>();
-        if (syslogManager != null) {
-            healthRecords.addAll(syslogManager.healthCheck());
-        }
-
-        if (lastError != null) {
-            healthRecords.add(new HealthRecord(HealthStatus.WARN, HealthTopic.Audit, lastError.toDebugStr()));
-        }
-
-        return healthRecords;
-    }
-
-    public Iterator<AuditRecord> readVault() {
-        return auditVault.readVault();
-    }
-
-    public List<UserAuditRecord> readUserHistory(final PwmSession pwmSession)
-            throws PwmUnrecoverableException
-    {
-        return readUserHistory(pwmSession.getUserInfoBean());
-    }
-
-    public List<UserAuditRecord> readUserHistory(final UserInfoBean userInfoBean)
-            throws PwmUnrecoverableException
-    {
-        return userHistoryStore.readUserHistory(userInfoBean);
-    }
-
-    protected void sendAsEmail(final AuditRecord record)
-            throws PwmUnrecoverableException
-    {
-        if (record == null || record.getEventCode() == null) {
-            return;
-        }
-        if (settings.getAlertFromAddress() == null || settings.getAlertFromAddress().length() < 1) {
-            return;
-        }
-
-        switch (record.getEventCode().getType()) {
-            case SYSTEM:
-                for (final String toAddress : settings.getSystemEmailAddresses()) {
-                    sendAsEmail(pwmApplication, null, record, toAddress, settings.getAlertFromAddress());
-                }
-                break;
-
-            case USER:
-                for (final String toAddress : settings.getUserEmailAddresses()) {
-                    sendAsEmail(pwmApplication, null, record, toAddress, settings.getAlertFromAddress());
-                }
-                break;
-        }
-    }
-
-    private static void sendAsEmail(
-            final PwmApplication pwmApplication,
-            final SessionLabel sessionLabel,
-            final AuditRecord record,
-            final String toAddress,
-            final String fromAddress
-
-    )
-            throws PwmUnrecoverableException
-    {
-        final String subject = PwmConstants.PWM_APP_NAME + " - Audit Event - " + record.getEventCode().toString();
-
-        final StringBuilder body = new StringBuilder();
-        final String jsonRecord = JsonUtil.serialize(record);
-        final Map<String,Object> mapRecord = JsonUtil.deserializeMap(jsonRecord);
-
-        for (final String key : mapRecord.keySet()) {
-            body.append(key);
-            body.append("=");
-            body.append(mapRecord.get(key));
-            body.append("\n");
-        }
-
-        final EmailItemBean emailItem = new EmailItemBean(toAddress, fromAddress, subject, body.toString(), null);
-        final MacroMachine macroMachine = MacroMachine.forNonUserSpecific(pwmApplication, sessionLabel);
-        pwmApplication.getEmailQueue().submitEmail(emailItem, null, macroMachine);
-    }
-
-    public int vaultSize() {
-        if (status != STATUS.OPEN || auditVault == null) {
-            return -1;
-        }
-
-        return auditVault.size();
-    }
-
-    public void submit(final AuditEvent auditEvent, final UserInfoBean userInfoBean, final PwmSession pwmSession)
-            throws PwmUnrecoverableException
-    {
-        final UserAuditRecord auditRecord = createUserAuditRecord(auditEvent, userInfoBean, pwmSession);
-        submit(auditRecord);
-    }
-
-    public void submit(final AuditRecord auditRecord)
-            throws PwmUnrecoverableException
-    {
-
-        final String jsonRecord = JsonUtil.serialize(auditRecord);
-
-        if (status != STATUS.OPEN) {
-            LOGGER.debug("discarding audit event (AuditManager is not open); event=" + jsonRecord);
-            return;
-        }
-
-        if (auditRecord.getEventCode() == null) {
-            LOGGER.error("discarding audit event, missing event type; event=" + jsonRecord);
-            return;
-        }
-
-        if (!settings.getPermittedEvents().contains(auditRecord.getEventCode())) {
-            LOGGER.debug("discarding event, " + auditRecord.getEventCode() + " are being ignored; event=" + jsonRecord);
-            return;
-        }
-
-        // add to debug log
-        LOGGER.info("audit event: " + jsonRecord);
-
-        // add to audit db
-        if (auditVault != null) {
-            auditVault.add(auditRecord);
-        }
-
-        // email alert
-        sendAsEmail(auditRecord);
-
-        // add to user ldap record
-        if (auditRecord instanceof UserAuditRecord) {
-            if (settings.getUserStoredEvents().contains(auditRecord.getEventCode())) {
-                userHistoryStore.updateUserHistory((UserAuditRecord) auditRecord);
-        }
-        }
-
-        // send to syslog
-        if (syslogManager != null) {
-            try {
-                syslogManager.add(auditRecord);
-            } catch (PwmOperationalException e) {
-                lastError = e.getErrorInformation();
-            }
-        }
-    }
-
-
-    public int outputVaultToCsv(OutputStream outputStream, final Locale locale, final boolean includeHeader)
-            throws IOException
-    {
-        final Configuration config = null;
-
-        final CSVPrinter csvPrinter = Helper.makeCsvPrinter(outputStream);
-
-        csvPrinter.printComment(" " + PwmConstants.PWM_APP_NAME + " audit record output ");
-        csvPrinter.printComment(" " + PwmConstants.DEFAULT_DATETIME_FORMAT.format(new Date()));
-
-        if (includeHeader) {
-            final List<String> headers = new ArrayList<>();
-            headers.add("Type");
-            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_EventCode",config,password.pwm.i18n.Admin.class));
-            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_Timestamp",config,password.pwm.i18n.Admin.class));
-            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_GUID",config,password.pwm.i18n.Admin.class));
-            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_Message",config,password.pwm.i18n.Admin.class));
-            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_Instance",config,password.pwm.i18n.Admin.class));
-            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_PerpetratorID",config,password.pwm.i18n.Admin.class));
-            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_PerpetratorDN",config,password.pwm.i18n.Admin.class));
-            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_TargetID",config,password.pwm.i18n.Admin.class));
-            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_TargetDN",config,password.pwm.i18n.Admin.class));
-            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_SourceAddress",config,password.pwm.i18n.Admin.class));
-            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_SourceHost",config,password.pwm.i18n.Admin.class));
-            csvPrinter.printRecord(headers);
-        }
-
-        int counter = 0;
-        for (final Iterator<AuditRecord> recordIterator = readVault(); recordIterator.hasNext();) {
-            final AuditRecord loopRecord = recordIterator.next();
-            counter++;
-
-            final List<String> lineOutput = new ArrayList<>();
-            lineOutput.add(loopRecord.getEventCode().getType().toString());
-            lineOutput.add(loopRecord.getEventCode().toString());
-            lineOutput.add(PwmConstants.DEFAULT_DATETIME_FORMAT.format(loopRecord.getTimestamp()));
-            lineOutput.add(loopRecord.getGuid());
-            lineOutput.add(loopRecord.getMessage() == null ? "" : loopRecord.getMessage());
-            if (loopRecord instanceof SystemAuditRecord) {
-                lineOutput.add(((SystemAuditRecord)loopRecord).getInstance());
-            }
-            if (loopRecord instanceof UserAuditRecord) {
-                lineOutput.add(((UserAuditRecord)loopRecord).getPerpetratorID());
-                lineOutput.add(((UserAuditRecord)loopRecord).getPerpetratorDN());
-                lineOutput.add("");
-                lineOutput.add("");
-                lineOutput.add(((UserAuditRecord)loopRecord).getSourceAddress());
-                lineOutput.add(((UserAuditRecord)loopRecord).getSourceHost());
-            }
-            if (loopRecord instanceof HelpdeskAuditRecord) {
-                lineOutput.add(((HelpdeskAuditRecord)loopRecord).getPerpetratorID());
-                lineOutput.add(((HelpdeskAuditRecord)loopRecord).getPerpetratorDN());
-                lineOutput.add(((HelpdeskAuditRecord)loopRecord).getTargetID());
-                lineOutput.add(((HelpdeskAuditRecord)loopRecord).getTargetDN());
-                lineOutput.add(((HelpdeskAuditRecord)loopRecord).getSourceAddress());
-                lineOutput.add(((HelpdeskAuditRecord)loopRecord).getSourceHost());
-            }
-            csvPrinter.printRecord(lineOutput);
-        }
-        csvPrinter.flush();
-
-        return counter;
-    }
-
-    public ServiceInfo serviceInfo()
-    {
-        return serviceInfo;
-    }
-
-    public int syslogQueueSize() {
-        return syslogManager != null ? syslogManager.queueSize() : 0;
-    }
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.event;
+
+import org.apache.commons.csv.CSVPrinter;
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.PwmConstants;
+import password.pwm.bean.EmailItemBean;
+import password.pwm.bean.SessionLabel;
+import password.pwm.bean.UserIdentity;
+import password.pwm.bean.UserInfoBean;
+import password.pwm.config.Configuration;
+import password.pwm.config.PwmSetting;
+import password.pwm.config.option.DataStorageMethod;
+import password.pwm.config.option.UserEventStorageMethod;
+import password.pwm.error.*;
+import password.pwm.health.HealthRecord;
+import password.pwm.health.HealthStatus;
+import password.pwm.health.HealthTopic;
+import password.pwm.http.PwmSession;
+import password.pwm.ldap.LdapOperationsHelper;
+import password.pwm.svc.PwmService;
+import password.pwm.util.Helper;
+import password.pwm.util.JsonUtil;
+import password.pwm.util.LocaleHelper;
+import password.pwm.util.TimeDuration;
+import password.pwm.util.localdb.LocalDB;
+import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.macro.MacroMachine;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.*;
+
+public class AuditService implements PwmService {
+    private static final PwmLogger LOGGER = PwmLogger.forClass(AuditService.class);
+
+    private STATUS status = STATUS.NEW;
+    private AuditSettings settings;
+    private ServiceInfo serviceInfo = new ServiceInfo(Collections.<DataStorageMethod>emptyList());
+
+    private SyslogAuditService syslogManager;
+    private ErrorInformation lastError;
+    private UserHistoryStore userHistoryStore;
+    private AuditVault auditVault;
+
+    private PwmApplication pwmApplication;
+
+    public AuditService() {
+    }
+
+    public HelpdeskAuditRecord createHelpdeskAuditRecord(
+            final AuditEvent eventCode,
+            final UserIdentity perpetrator,
+            final String message,
+            final UserIdentity target,
+            final String sourceAddress,
+            final String sourceHost
+    )
+    {
+        String perpUserDN = null, perpUserID = null, perpLdapProfile = null, targetUserDN = null, targetUserID = null, targetLdapProfile = null;
+        if (perpetrator != null) {
+            perpUserDN = perpetrator.getUserDN();
+            perpLdapProfile = perpetrator.getLdapProfileID();
+            try {
+                perpUserID = LdapOperationsHelper.readLdapUsernameValue(pwmApplication,perpetrator);
+            } catch (Exception e) {
+                LOGGER.error("unable to read userID for " + perpetrator + ", error: " + e.getMessage());
+            }
+        }
+        if (target != null) {
+            targetUserDN = target.getUserDN();
+            targetLdapProfile = target.getLdapProfileID();
+            try {
+                targetUserID = LdapOperationsHelper.readLdapUsernameValue(pwmApplication,target);
+            } catch (Exception e) {
+                LOGGER.error("unable to read userID for " + perpetrator + ", error: " + e.getMessage());
+            }
+        }
+
+        return HelpdeskAuditRecord.create(eventCode, perpUserID, perpUserDN, perpLdapProfile, message, targetUserID, targetUserDN,
+                targetLdapProfile, sourceAddress, sourceHost);
+    }
+
+    public UserAuditRecord createUserAuditRecord(
+            final AuditEvent eventCode,
+            final UserIdentity perpetrator,
+            final String message,
+            final String sourceAddress,
+            final String sourceHost
+    )
+    {
+        String perpUserDN = null, perpUserID = null, perpLdapProfile = null, targetUserDN = null, targetUserID = null, targetLdapProfile = null;
+        if (perpetrator != null) {
+            perpUserDN = perpetrator.getUserDN();
+            perpLdapProfile = perpetrator.getLdapProfileID();
+            try {
+                perpUserID = LdapOperationsHelper.readLdapUsernameValue(pwmApplication,perpetrator);
+            } catch (Exception e) {
+                LOGGER.error("unable to read userID for " + perpetrator + ", error: " + e.getMessage());
+            }
+        }
+
+        return HelpdeskAuditRecord.create(eventCode, perpUserID, perpUserDN, perpLdapProfile, message, targetUserID, targetUserDN,
+                targetLdapProfile, sourceAddress, sourceHost);
+    }
+
+    public SystemAuditRecord createSystemAuditRecord(
+            final AuditEvent eventCode,
+            final String message
+    )
+    {
+        return SystemAuditRecord.create(eventCode, message, pwmApplication.getInstanceID());
+    }
+
+    public UserAuditRecord createUserAuditRecord(
+            final AuditEvent eventCode,
+            final UserIdentity perpetrator,
+            final SessionLabel sessionLabel
+    )
+    {
+        return createUserAuditRecord(
+                eventCode,
+                perpetrator,
+                sessionLabel,
+                null
+        );
+    }
+
+    public UserAuditRecord createUserAuditRecord(
+            final AuditEvent eventCode,
+            final UserIdentity perpetrator,
+            final SessionLabel sessionLabel,
+            final String message
+    )
+    {
+        return createUserAuditRecord(
+                eventCode,
+                perpetrator,
+                message,
+                sessionLabel != null ? sessionLabel.getSrcAddress() : null,
+                sessionLabel != null ? sessionLabel.getSrcHostname() : null
+        );
+    }
+
+    public UserAuditRecord createUserAuditRecord(
+            final AuditEvent eventCode,
+            final UserInfoBean userInfoBean,
+            final PwmSession pwmSession
+    )
+    {
+        return createUserAuditRecord(
+                eventCode,
+                userInfoBean.getUserIdentity(),
+                null,
+                pwmSession.getSessionStateBean().getSrcAddress(),
+                pwmSession.getSessionStateBean().getSrcHostname()
+        );
+    }
+
+
+    public STATUS status() {
+        return status;
+    }
+
+    public void init(PwmApplication pwmApplication) throws PwmException {
+        this.status = STATUS.OPENING;
+        this.pwmApplication = pwmApplication;
+
+        settings = new AuditSettings(pwmApplication.getConfig());
+
+        if (pwmApplication.getApplicationMode() == null || pwmApplication.getApplicationMode() == PwmApplication.MODE.READ_ONLY) {
+            this.status = STATUS.CLOSED;
+            LOGGER.warn("unable to start - Application is in read-only mode");
+            return;
+        }
+
+        if (pwmApplication.getLocalDB() == null || pwmApplication.getLocalDB().status() != LocalDB.Status.OPEN) {
+            this.status = STATUS.CLOSED;
+            LOGGER.warn("unable to start - LocalDB is not available");
+            return;
+        }
+
+        final String syslogConfigString = pwmApplication.getConfig().readSettingAsString(PwmSetting.AUDIT_SYSLOG_SERVERS);
+        if (syslogConfigString != null && !syslogConfigString.isEmpty()) {
+            try {
+                syslogManager = new SyslogAuditService(pwmApplication);
+            } catch (Exception e) {
+                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_SYSLOG_WRITE_ERROR, "startup error: " + e.getMessage());
+                LOGGER.error(errorInformation.toDebugStr());
+            }
+        }
+        {
+            final UserEventStorageMethod userEventStorageMethod = pwmApplication.getConfig().readSettingAsEnum(PwmSetting.EVENTS_USER_STORAGE_METHOD, UserEventStorageMethod.class);
+            final String debugMsg;
+            final DataStorageMethod storageMethodUsed;
+            switch (userEventStorageMethod) {
+                case AUTO:
+                    if (pwmApplication.getConfig().hasDbConfigured()) {
+                        debugMsg = "starting using auto-configured data store, Remote Database selected";
+                        this.userHistoryStore = new DatabaseUserHistory(pwmApplication);
+                        storageMethodUsed = DataStorageMethod.DB;
+                    } else {
+                        debugMsg = "starting using auto-configured data store, LDAP selected";
+                        this.userHistoryStore = new LdapXmlUserHistory(pwmApplication);
+                        storageMethodUsed = DataStorageMethod.LDAP;
+                    }
+                    break;
+
+                case DATABASE:
+                    this.userHistoryStore = new DatabaseUserHistory(pwmApplication);
+                    debugMsg = "starting using Remote Database data store";
+                    storageMethodUsed = DataStorageMethod.DB;
+                    break;
+
+                case LDAP:
+                    this.userHistoryStore = new LdapXmlUserHistory(pwmApplication);
+                    debugMsg = "starting using LocalDB data store";
+                    storageMethodUsed = DataStorageMethod.LDAP;
+                    break;
+
+                default:
+                    lastError = new ErrorInformation(PwmError.ERROR_UNKNOWN,"unknown storageMethod selected: " + userEventStorageMethod);
+                    status = STATUS.CLOSED;
+                    return;
+            }
+            LOGGER.info(debugMsg);
+            serviceInfo = new ServiceInfo(Collections.singletonList(storageMethodUsed));
+        }
+        {
+            final TimeDuration maxRecordAge = new TimeDuration(pwmApplication.getConfig().readSettingAsLong(PwmSetting.EVENTS_AUDIT_MAX_AGE) * 1000);
+            final int maxRecords = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.AUDIT_VAULT_MAX_RECORDS));
+            final AuditVault.Settings settings = new AuditVault.Settings(
+                    maxRecords,
+                    maxRecordAge
+            );
+
+            if (pwmApplication.getLocalDB() != null && pwmApplication.getApplicationMode() != PwmApplication.MODE.READ_ONLY) {
+                auditVault = new LocalDbAuditVault(pwmApplication, pwmApplication.getLocalDB());
+                auditVault.init(settings);
+            }
+        }
+
+        this.status = STATUS.OPEN;
+    }
+
+    @Override
+    public void close() {
+        if (syslogManager != null) {
+            syslogManager.close();
+        }
+        this.status = STATUS.CLOSED;
+    }
+
+    @Override
+    public List<HealthRecord> healthCheck() {
+        if (status != STATUS.OPEN) {
+            return Collections.emptyList();
+        }
+
+        final List<HealthRecord> healthRecords = new ArrayList<>();
+        if (syslogManager != null) {
+            healthRecords.addAll(syslogManager.healthCheck());
+        }
+
+        if (lastError != null) {
+            healthRecords.add(new HealthRecord(HealthStatus.WARN, HealthTopic.Audit, lastError.toDebugStr()));
+        }
+
+        return healthRecords;
+    }
+
+    public Iterator<AuditRecord> readVault() {
+        return auditVault.readVault();
+    }
+
+    public List<UserAuditRecord> readUserHistory(final PwmSession pwmSession)
+            throws PwmUnrecoverableException
+    {
+        return readUserHistory(pwmSession.getUserInfoBean());
+    }
+
+    public List<UserAuditRecord> readUserHistory(final UserInfoBean userInfoBean)
+            throws PwmUnrecoverableException
+    {
+        return userHistoryStore.readUserHistory(userInfoBean);
+    }
+
+    protected void sendAsEmail(final AuditRecord record)
+            throws PwmUnrecoverableException
+    {
+        if (record == null || record.getEventCode() == null) {
+            return;
+        }
+        if (settings.getAlertFromAddress() == null || settings.getAlertFromAddress().length() < 1) {
+            return;
+        }
+
+        switch (record.getEventCode().getType()) {
+            case SYSTEM:
+                for (final String toAddress : settings.getSystemEmailAddresses()) {
+                    sendAsEmail(pwmApplication, null, record, toAddress, settings.getAlertFromAddress());
+                }
+                break;
+
+            case USER:
+                for (final String toAddress : settings.getUserEmailAddresses()) {
+                    sendAsEmail(pwmApplication, null, record, toAddress, settings.getAlertFromAddress());
+                }
+                break;
+        }
+    }
+
+    private static void sendAsEmail(
+            final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
+            final AuditRecord record,
+            final String toAddress,
+            final String fromAddress
+
+    )
+            throws PwmUnrecoverableException
+    {
+        final String subject = PwmConstants.PWM_APP_NAME + " - Audit Event - " + record.getEventCode().toString();
+
+        final StringBuilder body = new StringBuilder();
+        final String jsonRecord = JsonUtil.serialize(record);
+        final Map<String,Object> mapRecord = JsonUtil.deserializeMap(jsonRecord);
+
+        for (final String key : mapRecord.keySet()) {
+            body.append(key);
+            body.append("=");
+            body.append(mapRecord.get(key));
+            body.append("\n");
+        }
+
+        final EmailItemBean emailItem = new EmailItemBean(toAddress, fromAddress, subject, body.toString(), null);
+        final MacroMachine macroMachine = MacroMachine.forNonUserSpecific(pwmApplication, sessionLabel);
+        pwmApplication.getEmailQueue().submitEmail(emailItem, null, macroMachine);
+    }
+
+    public int vaultSize() {
+        if (status != STATUS.OPEN || auditVault == null) {
+            return -1;
+        }
+
+        return auditVault.size();
+    }
+
+    public void submit(final AuditEvent auditEvent, final UserInfoBean userInfoBean, final PwmSession pwmSession)
+            throws PwmUnrecoverableException
+    {
+        final UserAuditRecord auditRecord = createUserAuditRecord(auditEvent, userInfoBean, pwmSession);
+        submit(auditRecord);
+    }
+
+    public void submit(final AuditRecord auditRecord)
+            throws PwmUnrecoverableException
+    {
+
+        final String jsonRecord = JsonUtil.serialize(auditRecord);
+
+        if (status != STATUS.OPEN) {
+            LOGGER.debug("discarding audit event (AuditManager is not open); event=" + jsonRecord);
+            return;
+        }
+
+        if (auditRecord.getEventCode() == null) {
+            LOGGER.error("discarding audit event, missing event type; event=" + jsonRecord);
+            return;
+        }
+
+        if (!settings.getPermittedEvents().contains(auditRecord.getEventCode())) {
+            LOGGER.debug("discarding event, " + auditRecord.getEventCode() + " are being ignored; event=" + jsonRecord);
+            return;
+        }
+
+        // add to debug log
+        LOGGER.info("audit event: " + jsonRecord);
+
+        // add to audit db
+        if (auditVault != null) {
+            auditVault.add(auditRecord);
+        }
+
+        // email alert
+        sendAsEmail(auditRecord);
+
+        // add to user ldap record
+        if (auditRecord instanceof UserAuditRecord) {
+            if (settings.getUserStoredEvents().contains(auditRecord.getEventCode())) {
+                userHistoryStore.updateUserHistory((UserAuditRecord) auditRecord);
+        }
+        }
+
+        // send to syslog
+        if (syslogManager != null) {
+            try {
+                syslogManager.add(auditRecord);
+            } catch (PwmOperationalException e) {
+                lastError = e.getErrorInformation();
+            }
+        }
+    }
+
+
+    public int outputVaultToCsv(OutputStream outputStream, final Locale locale, final boolean includeHeader)
+            throws IOException
+    {
+        final Configuration config = null;
+
+        final CSVPrinter csvPrinter = Helper.makeCsvPrinter(outputStream);
+
+        csvPrinter.printComment(" " + PwmConstants.PWM_APP_NAME + " audit record output ");
+        csvPrinter.printComment(" " + PwmConstants.DEFAULT_DATETIME_FORMAT.format(new Date()));
+
+        if (includeHeader) {
+            final List<String> headers = new ArrayList<>();
+            headers.add("Type");
+            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_EventCode",config,password.pwm.i18n.Admin.class));
+            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_Timestamp",config,password.pwm.i18n.Admin.class));
+            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_GUID",config,password.pwm.i18n.Admin.class));
+            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_Message",config,password.pwm.i18n.Admin.class));
+            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_Instance",config,password.pwm.i18n.Admin.class));
+            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_PerpetratorID",config,password.pwm.i18n.Admin.class));
+            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_PerpetratorDN",config,password.pwm.i18n.Admin.class));
+            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_TargetID",config,password.pwm.i18n.Admin.class));
+            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_TargetDN",config,password.pwm.i18n.Admin.class));
+            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_SourceAddress",config,password.pwm.i18n.Admin.class));
+            headers.add(LocaleHelper.getLocalizedMessage(locale,"Field_Audit_SourceHost",config,password.pwm.i18n.Admin.class));
+            csvPrinter.printRecord(headers);
+        }
+
+        int counter = 0;
+        for (final Iterator<AuditRecord> recordIterator = readVault(); recordIterator.hasNext();) {
+            final AuditRecord loopRecord = recordIterator.next();
+            counter++;
+
+            final List<String> lineOutput = new ArrayList<>();
+            lineOutput.add(loopRecord.getEventCode().getType().toString());
+            lineOutput.add(loopRecord.getEventCode().toString());
+            lineOutput.add(PwmConstants.DEFAULT_DATETIME_FORMAT.format(loopRecord.getTimestamp()));
+            lineOutput.add(loopRecord.getGuid());
+            lineOutput.add(loopRecord.getMessage() == null ? "" : loopRecord.getMessage());
+            if (loopRecord instanceof SystemAuditRecord) {
+                lineOutput.add(((SystemAuditRecord)loopRecord).getInstance());
+            }
+            if (loopRecord instanceof UserAuditRecord) {
+                lineOutput.add(((UserAuditRecord)loopRecord).getPerpetratorID());
+                lineOutput.add(((UserAuditRecord)loopRecord).getPerpetratorDN());
+                lineOutput.add("");
+                lineOutput.add("");
+                lineOutput.add(((UserAuditRecord)loopRecord).getSourceAddress());
+                lineOutput.add(((UserAuditRecord)loopRecord).getSourceHost());
+            }
+            if (loopRecord instanceof HelpdeskAuditRecord) {
+                lineOutput.add(((HelpdeskAuditRecord)loopRecord).getPerpetratorID());
+                lineOutput.add(((HelpdeskAuditRecord)loopRecord).getPerpetratorDN());
+                lineOutput.add(((HelpdeskAuditRecord)loopRecord).getTargetID());
+                lineOutput.add(((HelpdeskAuditRecord)loopRecord).getTargetDN());
+                lineOutput.add(((HelpdeskAuditRecord)loopRecord).getSourceAddress());
+                lineOutput.add(((HelpdeskAuditRecord)loopRecord).getSourceHost());
+            }
+            csvPrinter.printRecord(lineOutput);
+        }
+        csvPrinter.flush();
+
+        return counter;
+    }
+
+    public ServiceInfo serviceInfo()
+    {
+        return serviceInfo;
+    }
+
+    public int syslogQueueSize() {
+        return syslogManager != null ? syslogManager.queueSize() : 0;
+    }
+}

pwm/servlet/src/password/pwm/event/AuditSettings.java → pwm/servlet/src/password/pwm/svc/event/AuditSettings.java

@@ -1,4 +1,4 @@
-package password.pwm.event;
+package password.pwm.svc.event;
 
 import password.pwm.AppProperty;
 import password.pwm.config.Configuration;

pwm/servlet/src/password/pwm/event/AuditVault.java → pwm/servlet/src/password/pwm/svc/event/AuditVault.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.event;
+package password.pwm.svc.event;
 
 import password.pwm.util.TimeDuration;
 

pwm/servlet/src/password/pwm/event/DatabaseUserHistory.java → pwm/servlet/src/password/pwm/svc/event/DatabaseUserHistory.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.event;
+package password.pwm.svc.event;
 
 import com.novell.ldapchai.exception.ChaiUnavailableException;
 import password.pwm.PwmApplication;

pwm/servlet/src/password/pwm/event/HelpdeskAuditRecord.java → pwm/servlet/src/password/pwm/svc/event/HelpdeskAuditRecord.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.event;
+package password.pwm.svc.event;
 
 import java.util.Date;
 

pwm/servlet/src/password/pwm/event/LdapXmlUserHistory.java → pwm/servlet/src/password/pwm/svc/event/LdapXmlUserHistory.java

@@ -1,318 +1,318 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.event;
-
-import com.novell.ldapchai.ChaiUser;
-import com.novell.ldapchai.exception.ChaiOperationException;
-import com.novell.ldapchai.exception.ChaiUnavailableException;
-import com.novell.ldapchai.util.ConfigObjectRecord;
-import org.jdom2.CDATA;
-import org.jdom2.Document;
-import org.jdom2.Element;
-import org.jdom2.JDOMException;
-import org.jdom2.input.SAXBuilder;
-import org.jdom2.output.Format;
-import org.jdom2.output.XMLOutputter;
-import password.pwm.PwmApplication;
-import password.pwm.bean.UserIdentity;
-import password.pwm.bean.UserInfoBean;
-import password.pwm.config.PwmSetting;
-import password.pwm.error.PwmError;
-import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.util.logging.PwmLogger;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.io.StringReader;
-import java.util.Collections;
-import java.util.Date;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * Wrapper class to handle user event history.
- *
- * @author Jason D. Rivard
- */
-class LdapXmlUserHistory implements UserHistoryStore, Serializable {
-// ------------------------------ FIELDS ------------------------------
-
-    private static final PwmLogger LOGGER = PwmLogger.forClass(LdapXmlUserHistory.class);
-
-    private static final String XML_ATTR_TIMESTAMP = "timestamp";
-    private static final String XML_ATTR_TRANSACTION = "eventCode";
-    private static final String XML_ATTR_SRC_IP = "srcIP";
-    private static final String XML_ATTR_SRC_HOST = "srcHost";
-    private static final String XML_NODE_ROOT = "history";
-    private static final String XML_NODE_RECORD = "record";
-
-    private static final String COR_RECORD_ID = "0001";
-
-    // -------------------------- STATIC METHODS --------------------------
-    private final PwmApplication pwmApplication;
-
-    LdapXmlUserHistory(PwmApplication pwmApplication) {
-        this.pwmApplication = pwmApplication;
-    }
-
-    public void updateUserHistory(final UserAuditRecord auditRecord)
-            throws PwmUnrecoverableException
-    {
-        try {
-            updateUserHistoryImpl(auditRecord);
-        } catch (ChaiUnavailableException e) {
-            throw new PwmUnrecoverableException(PwmError.forChaiError(e.getErrorCode()));
-        }
-    }
-
-    void updateUserHistoryImpl(final UserAuditRecord auditRecord)
-            throws PwmUnrecoverableException, ChaiUnavailableException
-    {
-        // user info
-        final UserIdentity userIdentity;
-        if (auditRecord instanceof HelpdeskAuditRecord && auditRecord.getType() == AuditEvent.Type.HELPDESK) {
-            final HelpdeskAuditRecord helpdeskAuditRecord = (HelpdeskAuditRecord)auditRecord;
-            userIdentity = new UserIdentity(helpdeskAuditRecord.getTargetDN(),helpdeskAuditRecord.getTargetLdapProfile());
-        } else {
-            userIdentity = new UserIdentity(auditRecord.getPerpetratorDN(),auditRecord.getPerpetratorLdapProfile());
-        }
-        final ChaiUser theUser = pwmApplication.getProxiedChaiUser(userIdentity);
-
-        // settings
-        final String corRecordIdentifer = COR_RECORD_ID;
-        final String corAttribute = pwmApplication.getConfig().readSettingAsString(PwmSetting.EVENTS_LDAP_ATTRIBUTE);
-
-        // quit if settings no good;
-        if (corAttribute == null || corAttribute.length() < 1) {
-            LOGGER.debug("no user event log attribute configured, skipping write of log data");
-            return;
-        }
-
-        // read current value;
-        final StoredHistory storedHistory;
-        final ConfigObjectRecord theCor;
-        try {
-            final List corList = ConfigObjectRecord.readRecordFromLDAP(theUser, corAttribute, corRecordIdentifer, null, null);
-            if (!corList.isEmpty()) {
-                theCor = (ConfigObjectRecord) corList.get(0);
-            } else {
-                theCor = ConfigObjectRecord.createNew(theUser, corAttribute, corRecordIdentifer, null, null);
-            }
-
-            storedHistory = StoredHistory.fromXml(theCor.getPayload());
-        } catch (ChaiOperationException e) {
-            LOGGER.error("ldap error writing user event log: " + e.getMessage());
-            return;
-        }
-
-        // add next record to blob
-        final StoredEvent storedEvent = StoredEvent.fromAuditRecord(auditRecord);
-        storedHistory.addEvent(storedEvent);
-
-        // trim the blob.
-        final int maxUserEvents = (int) pwmApplication.getConfig().readSettingAsLong(PwmSetting.EVENTS_LDAP_MAX_EVENTS);
-        storedHistory.trim(maxUserEvents);
-
-        // write the blob.
-        try {
-            theCor.updatePayload(storedHistory.toXml());
-        } catch (ChaiOperationException e) {
-            LOGGER.error("ldap error writing user event log: " + e.getMessage());
-        }
-    }
-
-    public List<UserAuditRecord> readUserHistory(final UserInfoBean userInfoBean)
-            throws PwmUnrecoverableException
-    {
-        try {
-            final ChaiUser theUser = pwmApplication.getProxiedChaiUser(userInfoBean.getUserIdentity());
-            final StoredHistory storedHistory = readUserHistory(pwmApplication, theUser);
-            return storedHistory.asAuditRecords(userInfoBean);
-        } catch (ChaiUnavailableException e) {
-            throw new PwmUnrecoverableException(PwmError.forChaiError(e.getErrorCode()));
-        }
-    }
-
-    private StoredHistory readUserHistory(
-            final PwmApplication pwmApplication,
-            final ChaiUser chaiUser
-    )
-            throws ChaiUnavailableException, PwmUnrecoverableException {
-        final String corRecordIdentifer = COR_RECORD_ID;
-        final String corAttribute = pwmApplication.getConfig().readSettingAsString(PwmSetting.EVENTS_LDAP_ATTRIBUTE);
-
-        if (corAttribute == null || corAttribute.length() < 1) {
-            LOGGER.trace("no user event log attribute configured, skipping read of log data");
-            return new StoredHistory();
-        }
-
-        try {
-            final List corList = ConfigObjectRecord.readRecordFromLDAP(chaiUser, corAttribute, corRecordIdentifer, null, null);
-
-            if (!corList.isEmpty()) {
-                final ConfigObjectRecord theCor = (ConfigObjectRecord) corList.get(0);
-                return StoredHistory.fromXml(theCor.getPayload());
-            }
-        } catch (ChaiOperationException e) {
-            LOGGER.error("ldap error reading user event log: " + e.getMessage());
-        }
-        return new StoredHistory();
-    }
-
-    private static class StoredHistory {
-        private final LinkedList<StoredEvent> records = new LinkedList<>();
-
-        public void addEvent(final StoredEvent storedEvent) {
-            records.add(storedEvent);
-        }
-
-        public void trim(final int size) {
-            while (records.size() > size) {
-                records.removeFirst();
-            }
-        }
-
-        public List<UserAuditRecord> asAuditRecords(final UserInfoBean userInfoBean) {
-            final List<UserAuditRecord> returnList = new LinkedList<>();
-            for (final StoredEvent loopEvent : records) {
-                returnList.add(loopEvent.asAuditRecord(userInfoBean));
-            }
-            return Collections.unmodifiableList(returnList);
-        }
-
-        public String toXml() {
-            final Element rootElement = new Element(XML_NODE_ROOT);
-
-            for (final StoredEvent loopEvent : records) {
-                if (loopEvent.getAuditEvent() != null) {
-                    final Element hrElement = new Element(XML_NODE_RECORD);
-                    hrElement.setAttribute(XML_ATTR_TIMESTAMP, String.valueOf(loopEvent.getTimestamp()));
-                    hrElement.setAttribute(XML_ATTR_TRANSACTION, loopEvent.getAuditEvent().getMessage().getKey());
-                    if (loopEvent.getSourceAddress() != null && loopEvent.getSourceAddress().length() > 0) {
-                        hrElement.setAttribute(XML_ATTR_SRC_IP,loopEvent.getSourceAddress());
-                    }
-                    if (loopEvent.getSourceHost() != null && loopEvent.getSourceHost().length() > 0) {
-                        hrElement.setAttribute(XML_ATTR_SRC_HOST,loopEvent.getSourceHost());
-                    }
-                    if (loopEvent.getMessage() != null) {
-                        hrElement.setContent(new CDATA(loopEvent.getMessage()));
-                    }
-                    rootElement.addContent(hrElement);
-                }
-            }
-
-            final Document doc = new Document(rootElement);
-            final XMLOutputter outputter = new XMLOutputter();
-            outputter.setFormat(Format.getCompactFormat());
-            return outputter.outputString(doc);
-        }
-
-        public static StoredHistory fromXml(final String input) {
-            final StoredHistory returnHistory = new StoredHistory();
-
-            if (input == null || input.length() < 1) {
-                return returnHistory;
-            }
-
-            try {
-                final SAXBuilder builder = new SAXBuilder();
-                final Document doc = builder.build(new StringReader(input));
-                final Element rootElement = doc.getRootElement();
-
-                for (final Element hrElement : rootElement.getChildren(XML_NODE_RECORD)) {
-                    final long timeStamp = hrElement.getAttribute(XML_ATTR_TIMESTAMP).getLongValue();
-                    final String transactionCode = hrElement.getAttribute(XML_ATTR_TRANSACTION).getValue();
-                    final AuditEvent eventCode = AuditEvent.forKey(transactionCode);
-                    final String srcAddr = hrElement.getAttribute(XML_ATTR_SRC_IP) != null ? hrElement.getAttribute(XML_ATTR_SRC_IP).toString() : "";
-                    final String srcHost = hrElement.getAttribute(XML_ATTR_SRC_HOST) != null ? hrElement.getAttribute(XML_ATTR_SRC_HOST).toString() : "";
-                    final String message = hrElement.getText();
-                    final StoredEvent storedEvent = new StoredEvent(eventCode,timeStamp,message,srcAddr,srcHost);
-                    returnHistory.addEvent(storedEvent);
-                }
-            } catch (JDOMException e) {
-                LOGGER.error("error parsing user event history record: " + e.getMessage());
-            } catch (IOException e) {
-                LOGGER.error("error parsing user event history record: " + e.getMessage());
-            }
-            return returnHistory;
-        }
-    }
-
-    private static class StoredEvent implements Serializable {
-        private AuditEvent auditEvent;
-        private long timestamp;
-        private String message;
-        private String sourceAddress;
-        private String sourceHost;
-
-
-        private StoredEvent(AuditEvent auditEvent, long timestamp, String message, String sourceAddress, String sourceHost) {
-            this.auditEvent = auditEvent;
-            this.timestamp = timestamp;
-            this.message = message;
-            this.sourceAddress = sourceAddress;
-            this.sourceHost = sourceHost;
-        }
-
-        public AuditEvent getAuditEvent() {
-            return auditEvent;
-        }
-
-        public long getTimestamp() {
-            return timestamp;
-        }
-
-        public String getMessage() {
-            return message;
-        }
-
-        public String getSourceAddress() {
-            return sourceAddress;
-        }
-
-        public String getSourceHost() {
-            return sourceHost;
-        }
-
-        public static StoredEvent fromAuditRecord(final UserAuditRecord auditRecord) {
-            return new StoredEvent(auditRecord.getEventCode(),auditRecord.getTimestamp().getTime(),auditRecord.getMessage(),auditRecord.getSourceAddress(),auditRecord.getSourceHost());
-        }
-
-        public UserAuditRecord asAuditRecord(final UserInfoBean userInfoBean) {
-            return UserAuditRecord.create(
-                    new Date(this.getTimestamp()),
-                    this.getAuditEvent(),
-                    null,
-                    null,
-                    userInfoBean.getUserIdentity().getUserDN(),
-                    this.getMessage(),
-                    this.getSourceAddress(),
-                    this.getSourceHost()
-            );
-        }
-    }
-
-// -------------------------- INNER CLASSES --------------------------
-
-}
-
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.event;
+
+import com.novell.ldapchai.ChaiUser;
+import com.novell.ldapchai.exception.ChaiOperationException;
+import com.novell.ldapchai.exception.ChaiUnavailableException;
+import com.novell.ldapchai.util.ConfigObjectRecord;
+import org.jdom2.CDATA;
+import org.jdom2.Document;
+import org.jdom2.Element;
+import org.jdom2.JDOMException;
+import org.jdom2.input.SAXBuilder;
+import org.jdom2.output.Format;
+import org.jdom2.output.XMLOutputter;
+import password.pwm.PwmApplication;
+import password.pwm.bean.UserIdentity;
+import password.pwm.bean.UserInfoBean;
+import password.pwm.config.PwmSetting;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.logging.PwmLogger;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.io.StringReader;
+import java.util.Collections;
+import java.util.Date;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Wrapper class to handle user event history.
+ *
+ * @author Jason D. Rivard
+ */
+class LdapXmlUserHistory implements UserHistoryStore, Serializable {
+// ------------------------------ FIELDS ------------------------------
+
+    private static final PwmLogger LOGGER = PwmLogger.forClass(LdapXmlUserHistory.class);
+
+    private static final String XML_ATTR_TIMESTAMP = "timestamp";
+    private static final String XML_ATTR_TRANSACTION = "eventCode";
+    private static final String XML_ATTR_SRC_IP = "srcIP";
+    private static final String XML_ATTR_SRC_HOST = "srcHost";
+    private static final String XML_NODE_ROOT = "history";
+    private static final String XML_NODE_RECORD = "record";
+
+    private static final String COR_RECORD_ID = "0001";
+
+    // -------------------------- STATIC METHODS --------------------------
+    private final PwmApplication pwmApplication;
+
+    LdapXmlUserHistory(PwmApplication pwmApplication) {
+        this.pwmApplication = pwmApplication;
+    }
+
+    public void updateUserHistory(final UserAuditRecord auditRecord)
+            throws PwmUnrecoverableException
+    {
+        try {
+            updateUserHistoryImpl(auditRecord);
+        } catch (ChaiUnavailableException e) {
+            throw new PwmUnrecoverableException(PwmError.forChaiError(e.getErrorCode()));
+        }
+    }
+
+    void updateUserHistoryImpl(final UserAuditRecord auditRecord)
+            throws PwmUnrecoverableException, ChaiUnavailableException
+    {
+        // user info
+        final UserIdentity userIdentity;
+        if (auditRecord instanceof HelpdeskAuditRecord && auditRecord.getType() == AuditEvent.Type.HELPDESK) {
+            final HelpdeskAuditRecord helpdeskAuditRecord = (HelpdeskAuditRecord)auditRecord;
+            userIdentity = new UserIdentity(helpdeskAuditRecord.getTargetDN(),helpdeskAuditRecord.getTargetLdapProfile());
+        } else {
+            userIdentity = new UserIdentity(auditRecord.getPerpetratorDN(),auditRecord.getPerpetratorLdapProfile());
+        }
+        final ChaiUser theUser = pwmApplication.getProxiedChaiUser(userIdentity);
+
+        // settings
+        final String corRecordIdentifer = COR_RECORD_ID;
+        final String corAttribute = pwmApplication.getConfig().readSettingAsString(PwmSetting.EVENTS_LDAP_ATTRIBUTE);
+
+        // quit if settings no good;
+        if (corAttribute == null || corAttribute.length() < 1) {
+            LOGGER.debug("no user event log attribute configured, skipping write of log data");
+            return;
+        }
+
+        // read current value;
+        final StoredHistory storedHistory;
+        final ConfigObjectRecord theCor;
+        try {
+            final List corList = ConfigObjectRecord.readRecordFromLDAP(theUser, corAttribute, corRecordIdentifer, null, null);
+            if (!corList.isEmpty()) {
+                theCor = (ConfigObjectRecord) corList.get(0);
+            } else {
+                theCor = ConfigObjectRecord.createNew(theUser, corAttribute, corRecordIdentifer, null, null);
+            }
+
+            storedHistory = StoredHistory.fromXml(theCor.getPayload());
+        } catch (ChaiOperationException e) {
+            LOGGER.error("ldap error writing user event log: " + e.getMessage());
+            return;
+        }
+
+        // add next record to blob
+        final StoredEvent storedEvent = StoredEvent.fromAuditRecord(auditRecord);
+        storedHistory.addEvent(storedEvent);
+
+        // trim the blob.
+        final int maxUserEvents = (int) pwmApplication.getConfig().readSettingAsLong(PwmSetting.EVENTS_LDAP_MAX_EVENTS);
+        storedHistory.trim(maxUserEvents);
+
+        // write the blob.
+        try {
+            theCor.updatePayload(storedHistory.toXml());
+        } catch (ChaiOperationException e) {
+            LOGGER.error("ldap error writing user event log: " + e.getMessage());
+        }
+    }
+
+    public List<UserAuditRecord> readUserHistory(final UserInfoBean userInfoBean)
+            throws PwmUnrecoverableException
+    {
+        try {
+            final ChaiUser theUser = pwmApplication.getProxiedChaiUser(userInfoBean.getUserIdentity());
+            final StoredHistory storedHistory = readUserHistory(pwmApplication, theUser);
+            return storedHistory.asAuditRecords(userInfoBean);
+        } catch (ChaiUnavailableException e) {
+            throw new PwmUnrecoverableException(PwmError.forChaiError(e.getErrorCode()));
+        }
+    }
+
+    private StoredHistory readUserHistory(
+            final PwmApplication pwmApplication,
+            final ChaiUser chaiUser
+    )
+            throws ChaiUnavailableException, PwmUnrecoverableException {
+        final String corRecordIdentifer = COR_RECORD_ID;
+        final String corAttribute = pwmApplication.getConfig().readSettingAsString(PwmSetting.EVENTS_LDAP_ATTRIBUTE);
+
+        if (corAttribute == null || corAttribute.length() < 1) {
+            LOGGER.trace("no user event log attribute configured, skipping read of log data");
+            return new StoredHistory();
+        }
+
+        try {
+            final List corList = ConfigObjectRecord.readRecordFromLDAP(chaiUser, corAttribute, corRecordIdentifer, null, null);
+
+            if (!corList.isEmpty()) {
+                final ConfigObjectRecord theCor = (ConfigObjectRecord) corList.get(0);
+                return StoredHistory.fromXml(theCor.getPayload());
+            }
+        } catch (ChaiOperationException e) {
+            LOGGER.error("ldap error reading user event log: " + e.getMessage());
+        }
+        return new StoredHistory();
+    }
+
+    private static class StoredHistory {
+        private final LinkedList<StoredEvent> records = new LinkedList<>();
+
+        public void addEvent(final StoredEvent storedEvent) {
+            records.add(storedEvent);
+        }
+
+        public void trim(final int size) {
+            while (records.size() > size) {
+                records.removeFirst();
+            }
+        }
+
+        public List<UserAuditRecord> asAuditRecords(final UserInfoBean userInfoBean) {
+            final List<UserAuditRecord> returnList = new LinkedList<>();
+            for (final StoredEvent loopEvent : records) {
+                returnList.add(loopEvent.asAuditRecord(userInfoBean));
+            }
+            return Collections.unmodifiableList(returnList);
+        }
+
+        public String toXml() {
+            final Element rootElement = new Element(XML_NODE_ROOT);
+
+            for (final StoredEvent loopEvent : records) {
+                if (loopEvent.getAuditEvent() != null) {
+                    final Element hrElement = new Element(XML_NODE_RECORD);
+                    hrElement.setAttribute(XML_ATTR_TIMESTAMP, String.valueOf(loopEvent.getTimestamp()));
+                    hrElement.setAttribute(XML_ATTR_TRANSACTION, loopEvent.getAuditEvent().getMessage().getKey());
+                    if (loopEvent.getSourceAddress() != null && loopEvent.getSourceAddress().length() > 0) {
+                        hrElement.setAttribute(XML_ATTR_SRC_IP,loopEvent.getSourceAddress());
+                    }
+                    if (loopEvent.getSourceHost() != null && loopEvent.getSourceHost().length() > 0) {
+                        hrElement.setAttribute(XML_ATTR_SRC_HOST,loopEvent.getSourceHost());
+                    }
+                    if (loopEvent.getMessage() != null) {
+                        hrElement.setContent(new CDATA(loopEvent.getMessage()));
+                    }
+                    rootElement.addContent(hrElement);
+                }
+            }
+
+            final Document doc = new Document(rootElement);
+            final XMLOutputter outputter = new XMLOutputter();
+            outputter.setFormat(Format.getCompactFormat());
+            return outputter.outputString(doc);
+        }
+
+        public static StoredHistory fromXml(final String input) {
+            final StoredHistory returnHistory = new StoredHistory();
+
+            if (input == null || input.length() < 1) {
+                return returnHistory;
+            }
+
+            try {
+                final SAXBuilder builder = new SAXBuilder();
+                final Document doc = builder.build(new StringReader(input));
+                final Element rootElement = doc.getRootElement();
+
+                for (final Element hrElement : rootElement.getChildren(XML_NODE_RECORD)) {
+                    final long timeStamp = hrElement.getAttribute(XML_ATTR_TIMESTAMP).getLongValue();
+                    final String transactionCode = hrElement.getAttribute(XML_ATTR_TRANSACTION).getValue();
+                    final AuditEvent eventCode = AuditEvent.forKey(transactionCode);
+                    final String srcAddr = hrElement.getAttribute(XML_ATTR_SRC_IP) != null ? hrElement.getAttribute(XML_ATTR_SRC_IP).toString() : "";
+                    final String srcHost = hrElement.getAttribute(XML_ATTR_SRC_HOST) != null ? hrElement.getAttribute(XML_ATTR_SRC_HOST).toString() : "";
+                    final String message = hrElement.getText();
+                    final StoredEvent storedEvent = new StoredEvent(eventCode,timeStamp,message,srcAddr,srcHost);
+                    returnHistory.addEvent(storedEvent);
+                }
+            } catch (JDOMException e) {
+                LOGGER.error("error parsing user event history record: " + e.getMessage());
+            } catch (IOException e) {
+                LOGGER.error("error parsing user event history record: " + e.getMessage());
+            }
+            return returnHistory;
+        }
+    }
+
+    private static class StoredEvent implements Serializable {
+        private AuditEvent auditEvent;
+        private long timestamp;
+        private String message;
+        private String sourceAddress;
+        private String sourceHost;
+
+
+        private StoredEvent(AuditEvent auditEvent, long timestamp, String message, String sourceAddress, String sourceHost) {
+            this.auditEvent = auditEvent;
+            this.timestamp = timestamp;
+            this.message = message;
+            this.sourceAddress = sourceAddress;
+            this.sourceHost = sourceHost;
+        }
+
+        public AuditEvent getAuditEvent() {
+            return auditEvent;
+        }
+
+        public long getTimestamp() {
+            return timestamp;
+        }
+
+        public String getMessage() {
+            return message;
+        }
+
+        public String getSourceAddress() {
+            return sourceAddress;
+        }
+
+        public String getSourceHost() {
+            return sourceHost;
+        }
+
+        public static StoredEvent fromAuditRecord(final UserAuditRecord auditRecord) {
+            return new StoredEvent(auditRecord.getEventCode(),auditRecord.getTimestamp().getTime(),auditRecord.getMessage(),auditRecord.getSourceAddress(),auditRecord.getSourceHost());
+        }
+
+        public UserAuditRecord asAuditRecord(final UserInfoBean userInfoBean) {
+            return UserAuditRecord.create(
+                    new Date(this.getTimestamp()),
+                    this.getAuditEvent(),
+                    null,
+                    null,
+                    userInfoBean.getUserIdentity().getUserDN(),
+                    this.getMessage(),
+                    this.getSourceAddress(),
+                    this.getSourceHost()
+            );
+        }
+    }
+
+// -------------------------- INNER CLASSES --------------------------
+
+}
+

pwm/servlet/src/password/pwm/event/LocalDbAuditVault.java → pwm/servlet/src/password/pwm/svc/event/LocalDbAuditVault.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.event;
+package password.pwm.svc.event;
 
 import password.pwm.PwmApplication;
 import password.pwm.util.JsonUtil;

pwm/servlet/src/password/pwm/event/SyslogAuditService.java → pwm/servlet/src/password/pwm/svc/event/SyslogAuditService.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.event;
+package password.pwm.svc.event;
 
 import org.productivity.java.syslog4j.SyslogIF;
 import org.productivity.java.syslog4j.impl.AbstractSyslogConfigIF;

pwm/servlet/src/password/pwm/event/SystemAuditRecord.java → pwm/servlet/src/password/pwm/svc/event/SystemAuditRecord.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.event;
+package password.pwm.svc.event;
 
 import java.io.Serializable;
 

pwm/servlet/src/password/pwm/event/UserAuditRecord.java → pwm/servlet/src/password/pwm/svc/event/UserAuditRecord.java

@@ -1,103 +1,103 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.event;
-
-import java.io.Serializable;
-import java.util.Date;
-
-/**
- * UserAuditRecord data
- */
-public class UserAuditRecord extends AuditRecord implements Serializable {
-    protected String perpetratorID;
-    protected String perpetratorDN;
-    protected String perpetratorLdapProfile;
-    protected String sourceAddress;
-    protected String sourceHost;
-
-    protected UserAuditRecord(
-            final Date timestamp,
-            final AuditEvent eventCode,
-            final String perpetratorID,
-            final String perpetratorDN,
-            final String perpetratorLdapProfile,
-            final String message,
-            final String sourceAddress,
-            final String sourceHost
-    ) {
-        super(timestamp, eventCode, message);
-        this.perpetratorID = perpetratorID;
-        this.perpetratorDN = perpetratorDN;
-        this.perpetratorLdapProfile = perpetratorLdapProfile;
-        this.sourceAddress = sourceAddress;
-        this.sourceHost = sourceHost;
-    }
-
-    static UserAuditRecord create(
-            final AuditEvent eventCode,
-            final String perpetratorID,
-            final String perpetratorDN,
-            final String perpetratorLdapProfile,
-            final String message,
-            final String sourceAddress,
-            final String sourceHost
-    )
-    {
-        return new UserAuditRecord(new Date(), eventCode, perpetratorID, perpetratorDN, perpetratorLdapProfile, message, sourceAddress, sourceHost);
-    }
-
-    static UserAuditRecord create(
-            final Date timestamp,
-            final AuditEvent eventCode,
-            final String perpetratorID,
-            final String perpetratorDN,
-            final String perpetratorLdapProfile,
-            final String message,
-            final String sourceAddress,
-            final String sourceHost
-    )
-    {
-        return new UserAuditRecord(timestamp, eventCode, perpetratorID, perpetratorDN, perpetratorLdapProfile, message, sourceAddress, sourceHost);
-    }
-
-    public String getPerpetratorID() {
-        return perpetratorID;
-    }
-
-    public String getPerpetratorDN() {
-        return perpetratorDN;
-    }
-
-    public String getSourceAddress() {
-        return sourceAddress;
-    }
-
-    public String getSourceHost() {
-        return sourceHost;
-    }
-
-    public String getPerpetratorLdapProfile()
-    {
-        return perpetratorLdapProfile;
-    }
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.event;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * UserAuditRecord data
+ */
+public class UserAuditRecord extends AuditRecord implements Serializable {
+    protected String perpetratorID;
+    protected String perpetratorDN;
+    protected String perpetratorLdapProfile;
+    protected String sourceAddress;
+    protected String sourceHost;
+
+    protected UserAuditRecord(
+            final Date timestamp,
+            final AuditEvent eventCode,
+            final String perpetratorID,
+            final String perpetratorDN,
+            final String perpetratorLdapProfile,
+            final String message,
+            final String sourceAddress,
+            final String sourceHost
+    ) {
+        super(timestamp, eventCode, message);
+        this.perpetratorID = perpetratorID;
+        this.perpetratorDN = perpetratorDN;
+        this.perpetratorLdapProfile = perpetratorLdapProfile;
+        this.sourceAddress = sourceAddress;
+        this.sourceHost = sourceHost;
+    }
+
+    static UserAuditRecord create(
+            final AuditEvent eventCode,
+            final String perpetratorID,
+            final String perpetratorDN,
+            final String perpetratorLdapProfile,
+            final String message,
+            final String sourceAddress,
+            final String sourceHost
+    )
+    {
+        return new UserAuditRecord(new Date(), eventCode, perpetratorID, perpetratorDN, perpetratorLdapProfile, message, sourceAddress, sourceHost);
+    }
+
+    static UserAuditRecord create(
+            final Date timestamp,
+            final AuditEvent eventCode,
+            final String perpetratorID,
+            final String perpetratorDN,
+            final String perpetratorLdapProfile,
+            final String message,
+            final String sourceAddress,
+            final String sourceHost
+    )
+    {
+        return new UserAuditRecord(timestamp, eventCode, perpetratorID, perpetratorDN, perpetratorLdapProfile, message, sourceAddress, sourceHost);
+    }
+
+    public String getPerpetratorID() {
+        return perpetratorID;
+    }
+
+    public String getPerpetratorDN() {
+        return perpetratorDN;
+    }
+
+    public String getSourceAddress() {
+        return sourceAddress;
+    }
+
+    public String getSourceHost() {
+        return sourceHost;
+    }
+
+    public String getPerpetratorLdapProfile()
+    {
+        return perpetratorLdapProfile;
+    }
+}

pwm/servlet/src/password/pwm/event/UserHistoryStore.java → pwm/servlet/src/password/pwm/svc/event/UserHistoryStore.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.event;
+package password.pwm.svc.event;
 
 import password.pwm.bean.UserInfoBean;
 import password.pwm.error.PwmUnrecoverableException;

pwm/servlet/src/password/pwm/util/intruder/DataStoreRecordStore.java → pwm/servlet/src/password/pwm/svc/intruder/DataStoreRecordStore.java

@@ -20,10 +20,10 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.intruder;
+package password.pwm.svc.intruder;
 
-import password.pwm.PwmService;
 import password.pwm.error.*;
+import password.pwm.svc.PwmService;
 import password.pwm.util.ClosableIterator;
 import password.pwm.util.DataStore;
 import password.pwm.util.JsonUtil;

pwm/servlet/src/password/pwm/util/intruder/IntruderManager.java → pwm/servlet/src/password/pwm/svc/intruder/IntruderManager.java

@@ -1,576 +1,576 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.util.intruder;
-
-import password.pwm.AppProperty;
-import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
-import password.pwm.PwmService;
-import password.pwm.bean.EmailItemBean;
-import password.pwm.bean.SessionLabel;
-import password.pwm.bean.UserIdentity;
-import password.pwm.bean.UserInfoBean;
-import password.pwm.config.Configuration;
-import password.pwm.config.FormConfiguration;
-import password.pwm.config.PwmSetting;
-import password.pwm.config.option.DataStorageMethod;
-import password.pwm.config.option.IntruderStorageMethod;
-import password.pwm.error.*;
-import password.pwm.event.AuditEvent;
-import password.pwm.event.SystemAuditRecord;
-import password.pwm.event.UserAuditRecord;
-import password.pwm.health.HealthRecord;
-import password.pwm.health.HealthStatus;
-import password.pwm.health.HealthTopic;
-import password.pwm.http.PwmSession;
-import password.pwm.ldap.LdapUserDataReader;
-import password.pwm.ldap.UserStatusReader;
-import password.pwm.util.*;
-import password.pwm.util.db.DatabaseDataStore;
-import password.pwm.util.db.DatabaseTable;
-import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.localdb.LocalDBDataStore;
-import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.secure.PwmRandom;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
-
-import java.io.Serializable;
-import java.net.InetAddress;
-import java.util.*;
-
-// ------------------------------ FIELDS ------------------------------
-
-public class IntruderManager implements Serializable, PwmService {
-    private static final PwmLogger LOGGER = PwmLogger.forClass(IntruderManager.class);
-
-    private PwmApplication pwmApplication;
-    private STATUS status = STATUS.NEW;
-    private ErrorInformation startupError;
-    private Timer timer;
-
-    private final Map<RecordType, RecordManager> recordManagers = new HashMap<>();
-
-    private ServiceInfo serviceInfo = new ServiceInfo(Collections.<DataStorageMethod>emptyList());
-
-    public IntruderManager() {
-        for (RecordType recordType : RecordType.values()) {
-            recordManagers.put(recordType, new StubRecordManager());
-        }
-    }
-
-    @Override
-    public STATUS status() {
-        return status;
-    }
-
-    @Override
-    public void init(PwmApplication pwmApplication)
-            throws PwmException
-    {
-        this.pwmApplication = pwmApplication;
-        final Configuration config = pwmApplication.getConfig();
-        status = STATUS.OPENING;
-        if (pwmApplication.getLocalDB() == null || pwmApplication.getLocalDB().status() != LocalDB.Status.OPEN) {
-            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,"unable to start IntruderManager, LocalDB unavailable");
-            LOGGER.error(errorInformation.toDebugStr());
-            startupError = errorInformation;
-            status = STATUS.CLOSED;
-            return;
-        }
-        if (!pwmApplication.getConfig().readSettingAsBoolean(PwmSetting.INTRUDER_ENABLE)) {
-            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,"intruder module not enabled");
-            LOGGER.error(errorInformation.toDebugStr());
-            status = STATUS.CLOSED;
-            return;
-        }
-        final DataStore dataStore;
-        {
-            final IntruderStorageMethod intruderStorageMethod = pwmApplication.getConfig().readSettingAsEnum(PwmSetting.INTRUDER_STORAGE_METHOD, IntruderStorageMethod.class);
-            final String debugMsg;
-            final DataStorageMethod storageMethodUsed;
-            switch (intruderStorageMethod) {
-                case AUTO:
-                    dataStore = DataStoreFactory.autoDbOrLocalDBstore(pwmApplication, DatabaseTable.INTRUDER, LocalDB.DB.INTRUDER);
-                    if (dataStore instanceof DatabaseDataStore) {
-                        debugMsg = "starting using auto-configured data store, Remote Database selected";
-                        storageMethodUsed = DataStorageMethod.DB;
-                    } else {
-                        debugMsg = "starting using auto-configured data store, LocalDB selected";
-                        storageMethodUsed = DataStorageMethod.LOCALDB;
-                    }
-                    break;
-
-                case DATABASE:
-                    dataStore = new DatabaseDataStore(pwmApplication.getDatabaseAccessor(), DatabaseTable.INTRUDER);
-                    debugMsg = "starting using Remote Database data store";
-                    storageMethodUsed = DataStorageMethod.DB;
-                    break;
-
-                case LOCALDB:
-                    dataStore = new LocalDBDataStore(pwmApplication.getLocalDB(), LocalDB.DB.INTRUDER);
-                    debugMsg = "starting using LocalDB data store";
-                    storageMethodUsed = DataStorageMethod.LOCALDB;
-                    break;
-
-                default:
-                    startupError = new ErrorInformation(PwmError.ERROR_UNKNOWN,"unknown storageMethod selected: " + intruderStorageMethod);
-                    status = STATUS.CLOSED;
-                    return;
-            }
-            LOGGER.info(debugMsg);
-            serviceInfo = new ServiceInfo(Collections.singletonList(storageMethodUsed));
-        }
-        final RecordStore recordStore;
-        {
-            recordStore = new DataStoreRecordStore(dataStore, this);
-            final String threadName = Helper.makeThreadName(pwmApplication, this.getClass()) + " timer";
-            timer = new Timer(threadName, true);
-            final long maxRecordAge = Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_RETENTION_TIME_MS));
-            final long cleanerRunFrequency = Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_CLEANUP_FREQUENCY_MS));
-            timer.schedule(new TimerTask() {
-                @Override
-                public void run() {
-                    try {
-                        recordStore.cleanup(new TimeDuration(maxRecordAge));
-                    } catch (Exception e) {
-                        LOGGER.error("error cleaning recordStore: " + e.getMessage(),e);
-                    }
-                }
-            },1000,cleanerRunFrequency);
-        }
-
-        try {
-            {
-                final IntruderSettings settings = new IntruderSettings();
-                settings.setCheckCount((int)config.readSettingAsLong(PwmSetting.INTRUDER_USER_MAX_ATTEMPTS));
-                settings.setResetDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_USER_RESET_TIME)));
-                settings.setCheckDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_USER_CHECK_TIME)));
-                if (settings.getCheckCount() == 0 || settings.getCheckDuration().getTotalMilliseconds() == 0 || settings.getResetDuration().getTotalMilliseconds() == 0) {
-                    LOGGER.info("intruder user checking will remain disabled due to configuration settings");
-                } else {
-                    recordManagers.put(RecordType.USERNAME, new RecordManagerImpl(RecordType.USERNAME, recordStore, settings));
-                    recordManagers.put(RecordType.USER_ID, new RecordManagerImpl(RecordType.USER_ID, recordStore, settings));
-                }
-            }
-            {
-                final IntruderSettings settings = new IntruderSettings();
-                settings.setCheckCount((int)config.readSettingAsLong(PwmSetting.INTRUDER_ATTRIBUTE_MAX_ATTEMPTS));
-                settings.setResetDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_ATTRIBUTE_RESET_TIME)));
-                settings.setCheckDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_ATTRIBUTE_CHECK_TIME)));
-                if (settings.getCheckCount() == 0 || settings.getCheckDuration().getTotalMilliseconds() == 0 || settings.getResetDuration().getTotalMilliseconds() == 0) {
-                    LOGGER.info("intruder user checking will remain disabled due to configuration settings");
-                } else {
-                    recordManagers.put(RecordType.ATTRIBUTE, new RecordManagerImpl(RecordType.ATTRIBUTE, recordStore, settings));
-                }
-            }
-            {
-                final IntruderSettings settings = new IntruderSettings();
-                settings.setCheckCount((int)config.readSettingAsLong(PwmSetting.INTRUDER_TOKEN_DEST_MAX_ATTEMPTS));
-                settings.setResetDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_TOKEN_DEST_RESET_TIME)));
-                settings.setCheckDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_TOKEN_DEST_CHECK_TIME)));
-                if (settings.getCheckCount() == 0 || settings.getCheckDuration().getTotalMilliseconds() == 0 || settings.getResetDuration().getTotalMilliseconds() == 0) {
-                    LOGGER.info("intruder user checking will remain disabled due to configuration settings");
-                } else {
-                    recordManagers.put(RecordType.TOKEN_DEST, new RecordManagerImpl(RecordType.TOKEN_DEST, recordStore, settings));
-                }
-            }
-            {
-                final IntruderSettings settings = new IntruderSettings();
-                settings.setCheckCount((int)config.readSettingAsLong(PwmSetting.INTRUDER_ADDRESS_MAX_ATTEMPTS));
-                settings.setResetDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_ADDRESS_RESET_TIME)));
-                settings.setCheckDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_ADDRESS_CHECK_TIME)));
-                if (settings.getCheckCount() == 0 || settings.getCheckDuration().getTotalMilliseconds() == 0 || settings.getResetDuration().getTotalMilliseconds() == 0) {
-                    LOGGER.info("intruder address checking will remain disabled due to configuration settings");
-                } else {
-                    recordManagers.put(RecordType.ADDRESS, new RecordManagerImpl(RecordType.ADDRESS, recordStore, settings));
-                }
-            }
-            status = STATUS.OPEN;
-        } catch (Exception e) {
-            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,"unexpected error starting intruder manager: " + e.getMessage());
-            LOGGER.error(errorInformation.toDebugStr());
-            startupError = errorInformation;
-            close();
-        }
-    }
-
-    public void clear() {
-
-    }
-
-    @Override
-    public void close() {
-        status = STATUS.CLOSED;
-        if (timer != null) {
-            timer.cancel();
-            timer = null;
-        }
-    }
-
-    @Override
-    public List<HealthRecord> healthCheck() {
-        if (startupError != null && status != STATUS.OPEN) {
-            return Collections.singletonList(new HealthRecord(HealthStatus.WARN, HealthTopic.Application,"unable to start: " + startupError.toDebugStr()));
-        }
-        return Collections.emptyList();
-    }
-
-    public void check(final RecordType recordType, final String subject)
-            throws PwmUnrecoverableException
-    {
-        if (recordType == null) {
-            throw new IllegalArgumentException("recordType is required");
-        }
-
-        if (subject == null || subject.length() < 1) {
-            return;
-        }
-
-        final RecordManager manager = recordManagers.get(recordType);
-        final boolean locked = manager.checkSubject(subject);
-
-        if (locked) {
-            switch (recordType) {
-                case ADDRESS:
-                    throw new PwmUnrecoverableException(PwmError.ERROR_INTRUDER_ADDRESS);
-
-                case ATTRIBUTE:
-                    throw new PwmUnrecoverableException(PwmError.ERROR_INTRUDER_ATTR_SEARCH);
-
-                case TOKEN_DEST:
-                    throw new PwmUnrecoverableException(PwmError.ERROR_INTRUDER_TOKEN_DEST);
-            }
-            throw new PwmUnrecoverableException(PwmError.ERROR_INTRUDER_USER);
-        }
-    }
-
-    public void clear(final RecordType recordType, final String subject)
-            throws PwmUnrecoverableException
-    {
-        if (recordType == null) {
-            throw new IllegalArgumentException("recordType is required");
-        }
-
-        if (subject == null || subject.length() < 1) {
-            return;
-        }
-
-        final RecordManager manager = recordManagers.get(recordType);
-        manager.clearSubject(subject);
-    }
-
-    public void mark(final RecordType recordType, final String subject, final SessionLabel sessionLabel)
-            throws PwmUnrecoverableException
-    {
-        if (recordType == null) {
-            throw new IllegalArgumentException("recordType is required");
-        }
-
-        if (subject == null || subject.length() < 1) {
-            return;
-        }
-
-        if (recordType == RecordType.ADDRESS) {
-            try {
-                final InetAddress inetAddress = InetAddress.getByName(subject);
-                if (inetAddress.isAnyLocalAddress() || inetAddress.isLoopbackAddress() || inetAddress.isLinkLocalAddress()) {
-                    LOGGER.debug("disregarding local address intruder attempt from: " + subject);
-                    return;
-                }
-            } catch(Exception e) {
-                LOGGER.error("error examining address: " + subject);
-            }
-        }
-
-        final RecordManager manager = recordManagers.get(recordType);
-        manager.markSubject(subject);
-
-        { // send intruder attempt audit event
-            final Map<String,Object> messageObj = new LinkedHashMap<>();
-            messageObj.put("type", recordType);
-            messageObj.put("subject", subject);
-            final String message = JsonUtil.serializeMap(messageObj);
-            final SystemAuditRecord auditRecord = pwmApplication.getAuditManager().createSystemAuditRecord(AuditEvent.INTRUDER_ATTEMPT,message);
-            pwmApplication.getAuditManager().submit(auditRecord);
-        }
-
-        try {
-            check(recordType, subject);
-        } catch (PwmUnrecoverableException e) {
-            if (!manager.isAlerted(subject) ) {
-                { // send intruder attempt lock event
-                    final Map<String,Object> messageObj = new LinkedHashMap<>();
-                    messageObj.put("type", recordType);
-                    messageObj.put("subject", subject);
-                    final String message = JsonUtil.serializeMap(messageObj);
-                    final SystemAuditRecord auditRecord = pwmApplication.getAuditManager().createSystemAuditRecord(AuditEvent.INTRUDER_LOCK,message);
-                    pwmApplication.getAuditManager().submit(auditRecord);
-                }
-
-                if (recordType == RecordType.USER_ID) {
-                    final UserIdentity userIdentity = UserIdentity.fromKey(subject, pwmApplication);
-                    final UserAuditRecord auditRecord = pwmApplication.getAuditManager().createUserAuditRecord(
-                            AuditEvent.INTRUDER_USER,
-                            userIdentity,
-                            sessionLabel
-                    );
-                    pwmApplication.getAuditManager().submit(auditRecord);
-                    sendAlert(manager.readIntruderRecord(subject), sessionLabel);
-                }
-
-                manager.markAlerted(subject);
-                final StatisticsManager statisticsManager = pwmApplication.getStatisticsManager();
-                if (statisticsManager != null && statisticsManager.status() == STATUS.OPEN) {
-                    statisticsManager.incrementValue(Statistic.INTRUDER_ATTEMPTS);
-                    statisticsManager.updateEps(Statistic.EpsType.INTRUDER_ATTEMPTS,1);
-                    statisticsManager.incrementValue(recordType.getLockStatistic());
-                }
-            }
-            throw e;
-        }
-
-        delayPenalty(manager.readIntruderRecord(subject), sessionLabel == null ? null : sessionLabel);
-    }
-
-
-    private void delayPenalty(final IntruderRecord intruderRecord, final SessionLabel sessionLabel) {
-        int points = 0;
-        if (intruderRecord != null) {
-            points += intruderRecord.getAttemptCount();
-            long delayPenalty = Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_MIN_DELAY_PENALTY_MS)); // minimum
-            delayPenalty += points * Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_DELAY_PER_COUNT_MS));
-            delayPenalty += PwmRandom.getInstance().nextInt((int)Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_DELAY_MAX_JITTER_MS))); // add some randomness;
-            delayPenalty = delayPenalty > Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_MAX_DELAY_PENALTY_MS)) ? Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_MAX_DELAY_PENALTY_MS)) : delayPenalty;
-            LOGGER.trace(sessionLabel, "delaying response " + delayPenalty + "ms due to intruder record: " + JsonUtil.serialize(intruderRecord));
-            Helper.pause(delayPenalty);
-        }
-    }
-
-    private void sendAlert(final IntruderRecord intruderRecord, final SessionLabel sessionLabel) {
-        if (intruderRecord == null) {
-            return;
-        }
-
-        if (intruderRecord.getType() == RecordType.USER_ID) {
-            try {
-                final UserIdentity identity = UserIdentity.fromDelimitedKey(intruderRecord.getSubject());
-                sendIntruderNoticeEmail(pwmApplication, sessionLabel, identity);
-            } catch (PwmUnrecoverableException e) {
-                LOGGER.error("unable to send intruder mail, can't read userDN/ldapProfile from stored record: " + e.getMessage());
-            }
-        }
-    }
-
-    public List<Map<String,Object>> getRecords(final RecordType recordType, int maximum)
-            throws PwmOperationalException
-    {
-        final RecordManager manager = recordManagers.get(recordType);
-        final ArrayList<Map<String,Object>> returnList = new ArrayList<>();
-
-        ClosableIterator<IntruderRecord> theIterator = null;
-        try {
-            theIterator = manager.iterator();
-            while (theIterator.hasNext() && returnList.size() < maximum) {
-                final IntruderRecord intruderRecord = theIterator.next();
-                if (intruderRecord != null && intruderRecord.getType() == recordType) {
-                    final Map<String, Object> rowData = new HashMap<>();
-                    rowData.put("subject", intruderRecord.getSubject());
-                    rowData.put("timestamp", intruderRecord.getTimeStamp());
-                    rowData.put("count", String.valueOf(intruderRecord.getAttemptCount()));
-                    try {
-                        check(recordType, intruderRecord.getSubject());
-                        rowData.put("status", "watching");
-                    } catch (PwmException e) {
-                        rowData.put("status", "locked");
-                    }
-                    returnList.add(rowData);
-                }
-            }
-        } finally {
-            if (theIterator != null) {
-                theIterator.close();
-            }
-        }
-        return returnList;
-    }
-
-    public Convenience convenience() {
-        return new Convenience();
-    }
-
-    public class Convenience {
-        protected Convenience() {
-        }
-
-        public void markAddressAndSession(final PwmSession pwmSession)
-                throws PwmUnrecoverableException
-        {
-            if (pwmSession != null) {
-                final String subject = pwmSession.getSessionStateBean().getSrcAddress();
-                pwmSession.getSessionStateBean().incrementIntruderAttempts();
-                mark(RecordType.ADDRESS, subject, pwmSession.getLabel());
-            }
-        }
-
-        public void checkAddressAndSession(final PwmSession pwmSession)
-                throws PwmUnrecoverableException
-        {
-            if (pwmSession != null) {
-                final String subject = pwmSession.getSessionStateBean().getSrcAddress();
-                check(RecordType.ADDRESS, subject);
-                final int maxAllowedAttempts = (int)pwmApplication.getConfig().readSettingAsLong(PwmSetting.INTRUDER_SESSION_MAX_ATTEMPTS);
-                if (maxAllowedAttempts != 0 && pwmSession.getSessionStateBean().getIntruderAttempts() > maxAllowedAttempts) {
-                    throw new PwmUnrecoverableException(PwmError.ERROR_INTRUDER_SESSION);
-                }
-            }
-        }
-
-        public void clearAddressAndSession(final PwmSession pwmSession)
-                throws PwmUnrecoverableException
-        {
-            if (pwmSession != null) {
-                final String subject = pwmSession.getSessionStateBean().getSrcAddress();
-                clear(RecordType.ADDRESS, subject);
-                pwmSession.getSessionStateBean().clearIntruderAttempts();
-            }
-        }
-
-        public void markUserIdentity(final UserIdentity userIdentity, final SessionLabel sessionLabel)
-                throws PwmUnrecoverableException
-        {
-            if (userIdentity != null) {
-                final String subject = userIdentity.toDelimitedKey();
-                mark(RecordType.USER_ID, subject, sessionLabel);
-            }
-        }
-
-        public void markUserIdentity(final UserIdentity userIdentity, final PwmSession pwmSession)
-                throws PwmUnrecoverableException
-        {
-            if (userIdentity != null) {
-                final String subject = userIdentity.toDelimitedKey();
-                mark(RecordType.USER_ID, subject, pwmSession.getLabel());
-            }
-        }
-
-        public void checkUserIdentity(final UserIdentity userIdentity)
-                throws PwmUnrecoverableException
-        {
-            if (userIdentity != null) {
-                final String subject = userIdentity.toDelimitedKey();
-                check(RecordType.USER_ID, subject);
-            }
-        }
-
-        public void clearUserIdentity(final UserIdentity userIdentity)
-                throws PwmUnrecoverableException
-        {
-            if (userIdentity != null) {
-                final String subject = userIdentity.toDelimitedKey();
-                clear(RecordType.USER_ID, subject);
-            }
-        }
-
-        public void markAttributes(final Map<FormConfiguration, String> formValues, final PwmSession pwmSession)
-                throws PwmUnrecoverableException
-        {
-            final List<String> subjects = attributeFormToList(formValues);
-            for (String subject : subjects) {
-                mark(RecordType.ATTRIBUTE, subject, pwmSession.getLabel());
-            }
-        }
-
-        public void clearAttributes(final Map<FormConfiguration, String> formValues)
-                throws PwmUnrecoverableException
-        {
-            final List<String> subjects = attributeFormToList(formValues);
-            for (String subject : subjects) {
-                clear(RecordType.ATTRIBUTE, subject);
-            }
-        }
-
-        public void checkAttributes(final Map<FormConfiguration, String> formValues)
-                throws PwmUnrecoverableException
-        {
-            final List<String> subjects = attributeFormToList(formValues);
-            for (String subject : subjects) {
-                check(RecordType.ATTRIBUTE, subject);
-            }
-        }
-
-        private List<String> attributeFormToList(final Map<FormConfiguration, String> formValues) {
-            final List<String> returnList = new ArrayList<>();
-            if (formValues != null) {
-                for (final FormConfiguration formConfiguration : formValues.keySet()) {
-                    final String value = formValues.get(formConfiguration);
-                    if (value != null && value.length() > 0) {
-                        returnList.add(formConfiguration.getName() + ":" + value);
-                    }
-                }
-            }
-            return returnList;
-        }
-
-    }
-
-    private static void sendIntruderNoticeEmail(
-            final PwmApplication pwmApplication,
-            final SessionLabel sessionLabel,
-            final UserIdentity userIdentity
-    )
-    {
-        final Configuration config = pwmApplication.getConfig();
-        final EmailItemBean configuredEmailSetting = config.readSettingAsEmail(PwmSetting.EMAIL_INTRUDERNOTICE, PwmConstants.DEFAULT_LOCALE);
-
-        if (configuredEmailSetting == null) {
-            return;
-        }
-
-        try {
-            final UserStatusReader userStatusReader = new UserStatusReader(pwmApplication, null);
-            final UserInfoBean userInfoBean = userStatusReader.populateUserInfoBean(
-                    PwmConstants.DEFAULT_LOCALE,
-                    userIdentity
-            );
-
-            final MacroMachine macroMachine = new MacroMachine(
-                    pwmApplication,
-                    sessionLabel,
-                    userInfoBean,
-                    null,
-                    LdapUserDataReader.appProxiedReader(pwmApplication, userIdentity));
-
-            pwmApplication.getEmailQueue().submitEmail(configuredEmailSetting, userInfoBean, macroMachine);
-        } catch (PwmUnrecoverableException e) {
-            LOGGER.error("error reading user info while sending intruder notice for user " + userIdentity + ", error: " + e.getMessage());
-        }
-
-    }
-
-    public ServiceInfo serviceInfo()
-    {
-        return serviceInfo;
-    }
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.intruder;
+
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.PwmConstants;
+import password.pwm.bean.EmailItemBean;
+import password.pwm.bean.SessionLabel;
+import password.pwm.bean.UserIdentity;
+import password.pwm.bean.UserInfoBean;
+import password.pwm.config.Configuration;
+import password.pwm.config.FormConfiguration;
+import password.pwm.config.PwmSetting;
+import password.pwm.config.option.DataStorageMethod;
+import password.pwm.config.option.IntruderStorageMethod;
+import password.pwm.error.*;
+import password.pwm.health.HealthRecord;
+import password.pwm.health.HealthStatus;
+import password.pwm.health.HealthTopic;
+import password.pwm.http.PwmSession;
+import password.pwm.ldap.LdapUserDataReader;
+import password.pwm.ldap.UserStatusReader;
+import password.pwm.svc.PwmService;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.event.SystemAuditRecord;
+import password.pwm.svc.event.UserAuditRecord;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
+import password.pwm.util.*;
+import password.pwm.util.db.DatabaseDataStore;
+import password.pwm.util.db.DatabaseTable;
+import password.pwm.util.localdb.LocalDB;
+import password.pwm.util.localdb.LocalDBDataStore;
+import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.macro.MacroMachine;
+import password.pwm.util.secure.PwmRandom;
+
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.util.*;
+
+// ------------------------------ FIELDS ------------------------------
+
+public class IntruderManager implements Serializable, PwmService {
+    private static final PwmLogger LOGGER = PwmLogger.forClass(IntruderManager.class);
+
+    private PwmApplication pwmApplication;
+    private STATUS status = STATUS.NEW;
+    private ErrorInformation startupError;
+    private Timer timer;
+
+    private final Map<RecordType, RecordManager> recordManagers = new HashMap<>();
+
+    private ServiceInfo serviceInfo = new ServiceInfo(Collections.<DataStorageMethod>emptyList());
+
+    public IntruderManager() {
+        for (RecordType recordType : RecordType.values()) {
+            recordManagers.put(recordType, new StubRecordManager());
+        }
+    }
+
+    @Override
+    public STATUS status() {
+        return status;
+    }
+
+    @Override
+    public void init(PwmApplication pwmApplication)
+            throws PwmException
+    {
+        this.pwmApplication = pwmApplication;
+        final Configuration config = pwmApplication.getConfig();
+        status = STATUS.OPENING;
+        if (pwmApplication.getLocalDB() == null || pwmApplication.getLocalDB().status() != LocalDB.Status.OPEN) {
+            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,"unable to start IntruderManager, LocalDB unavailable");
+            LOGGER.error(errorInformation.toDebugStr());
+            startupError = errorInformation;
+            status = STATUS.CLOSED;
+            return;
+        }
+        if (!pwmApplication.getConfig().readSettingAsBoolean(PwmSetting.INTRUDER_ENABLE)) {
+            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,"intruder module not enabled");
+            LOGGER.error(errorInformation.toDebugStr());
+            status = STATUS.CLOSED;
+            return;
+        }
+        final DataStore dataStore;
+        {
+            final IntruderStorageMethod intruderStorageMethod = pwmApplication.getConfig().readSettingAsEnum(PwmSetting.INTRUDER_STORAGE_METHOD, IntruderStorageMethod.class);
+            final String debugMsg;
+            final DataStorageMethod storageMethodUsed;
+            switch (intruderStorageMethod) {
+                case AUTO:
+                    dataStore = DataStoreFactory.autoDbOrLocalDBstore(pwmApplication, DatabaseTable.INTRUDER, LocalDB.DB.INTRUDER);
+                    if (dataStore instanceof DatabaseDataStore) {
+                        debugMsg = "starting using auto-configured data store, Remote Database selected";
+                        storageMethodUsed = DataStorageMethod.DB;
+                    } else {
+                        debugMsg = "starting using auto-configured data store, LocalDB selected";
+                        storageMethodUsed = DataStorageMethod.LOCALDB;
+                    }
+                    break;
+
+                case DATABASE:
+                    dataStore = new DatabaseDataStore(pwmApplication.getDatabaseAccessor(), DatabaseTable.INTRUDER);
+                    debugMsg = "starting using Remote Database data store";
+                    storageMethodUsed = DataStorageMethod.DB;
+                    break;
+
+                case LOCALDB:
+                    dataStore = new LocalDBDataStore(pwmApplication.getLocalDB(), LocalDB.DB.INTRUDER);
+                    debugMsg = "starting using LocalDB data store";
+                    storageMethodUsed = DataStorageMethod.LOCALDB;
+                    break;
+
+                default:
+                    startupError = new ErrorInformation(PwmError.ERROR_UNKNOWN,"unknown storageMethod selected: " + intruderStorageMethod);
+                    status = STATUS.CLOSED;
+                    return;
+            }
+            LOGGER.info(debugMsg);
+            serviceInfo = new ServiceInfo(Collections.singletonList(storageMethodUsed));
+        }
+        final RecordStore recordStore;
+        {
+            recordStore = new DataStoreRecordStore(dataStore, this);
+            final String threadName = Helper.makeThreadName(pwmApplication, this.getClass()) + " timer";
+            timer = new Timer(threadName, true);
+            final long maxRecordAge = Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_RETENTION_TIME_MS));
+            final long cleanerRunFrequency = Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_CLEANUP_FREQUENCY_MS));
+            timer.schedule(new TimerTask() {
+                @Override
+                public void run() {
+                    try {
+                        recordStore.cleanup(new TimeDuration(maxRecordAge));
+                    } catch (Exception e) {
+                        LOGGER.error("error cleaning recordStore: " + e.getMessage(),e);
+                    }
+                }
+            },1000,cleanerRunFrequency);
+        }
+
+        try {
+            {
+                final IntruderSettings settings = new IntruderSettings();
+                settings.setCheckCount((int)config.readSettingAsLong(PwmSetting.INTRUDER_USER_MAX_ATTEMPTS));
+                settings.setResetDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_USER_RESET_TIME)));
+                settings.setCheckDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_USER_CHECK_TIME)));
+                if (settings.getCheckCount() == 0 || settings.getCheckDuration().getTotalMilliseconds() == 0 || settings.getResetDuration().getTotalMilliseconds() == 0) {
+                    LOGGER.info("intruder user checking will remain disabled due to configuration settings");
+                } else {
+                    recordManagers.put(RecordType.USERNAME, new RecordManagerImpl(RecordType.USERNAME, recordStore, settings));
+                    recordManagers.put(RecordType.USER_ID, new RecordManagerImpl(RecordType.USER_ID, recordStore, settings));
+                }
+            }
+            {
+                final IntruderSettings settings = new IntruderSettings();
+                settings.setCheckCount((int)config.readSettingAsLong(PwmSetting.INTRUDER_ATTRIBUTE_MAX_ATTEMPTS));
+                settings.setResetDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_ATTRIBUTE_RESET_TIME)));
+                settings.setCheckDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_ATTRIBUTE_CHECK_TIME)));
+                if (settings.getCheckCount() == 0 || settings.getCheckDuration().getTotalMilliseconds() == 0 || settings.getResetDuration().getTotalMilliseconds() == 0) {
+                    LOGGER.info("intruder user checking will remain disabled due to configuration settings");
+                } else {
+                    recordManagers.put(RecordType.ATTRIBUTE, new RecordManagerImpl(RecordType.ATTRIBUTE, recordStore, settings));
+                }
+            }
+            {
+                final IntruderSettings settings = new IntruderSettings();
+                settings.setCheckCount((int)config.readSettingAsLong(PwmSetting.INTRUDER_TOKEN_DEST_MAX_ATTEMPTS));
+                settings.setResetDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_TOKEN_DEST_RESET_TIME)));
+                settings.setCheckDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_TOKEN_DEST_CHECK_TIME)));
+                if (settings.getCheckCount() == 0 || settings.getCheckDuration().getTotalMilliseconds() == 0 || settings.getResetDuration().getTotalMilliseconds() == 0) {
+                    LOGGER.info("intruder user checking will remain disabled due to configuration settings");
+                } else {
+                    recordManagers.put(RecordType.TOKEN_DEST, new RecordManagerImpl(RecordType.TOKEN_DEST, recordStore, settings));
+                }
+            }
+            {
+                final IntruderSettings settings = new IntruderSettings();
+                settings.setCheckCount((int)config.readSettingAsLong(PwmSetting.INTRUDER_ADDRESS_MAX_ATTEMPTS));
+                settings.setResetDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_ADDRESS_RESET_TIME)));
+                settings.setCheckDuration(new TimeDuration(1000 * config.readSettingAsLong(PwmSetting.INTRUDER_ADDRESS_CHECK_TIME)));
+                if (settings.getCheckCount() == 0 || settings.getCheckDuration().getTotalMilliseconds() == 0 || settings.getResetDuration().getTotalMilliseconds() == 0) {
+                    LOGGER.info("intruder address checking will remain disabled due to configuration settings");
+                } else {
+                    recordManagers.put(RecordType.ADDRESS, new RecordManagerImpl(RecordType.ADDRESS, recordStore, settings));
+                }
+            }
+            status = STATUS.OPEN;
+        } catch (Exception e) {
+            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,"unexpected error starting intruder manager: " + e.getMessage());
+            LOGGER.error(errorInformation.toDebugStr());
+            startupError = errorInformation;
+            close();
+        }
+    }
+
+    public void clear() {
+
+    }
+
+    @Override
+    public void close() {
+        status = STATUS.CLOSED;
+        if (timer != null) {
+            timer.cancel();
+            timer = null;
+        }
+    }
+
+    @Override
+    public List<HealthRecord> healthCheck() {
+        if (startupError != null && status != STATUS.OPEN) {
+            return Collections.singletonList(new HealthRecord(HealthStatus.WARN, HealthTopic.Application,"unable to start: " + startupError.toDebugStr()));
+        }
+        return Collections.emptyList();
+    }
+
+    public void check(final RecordType recordType, final String subject)
+            throws PwmUnrecoverableException
+    {
+        if (recordType == null) {
+            throw new IllegalArgumentException("recordType is required");
+        }
+
+        if (subject == null || subject.length() < 1) {
+            return;
+        }
+
+        final RecordManager manager = recordManagers.get(recordType);
+        final boolean locked = manager.checkSubject(subject);
+
+        if (locked) {
+            switch (recordType) {
+                case ADDRESS:
+                    throw new PwmUnrecoverableException(PwmError.ERROR_INTRUDER_ADDRESS);
+
+                case ATTRIBUTE:
+                    throw new PwmUnrecoverableException(PwmError.ERROR_INTRUDER_ATTR_SEARCH);
+
+                case TOKEN_DEST:
+                    throw new PwmUnrecoverableException(PwmError.ERROR_INTRUDER_TOKEN_DEST);
+            }
+            throw new PwmUnrecoverableException(PwmError.ERROR_INTRUDER_USER);
+        }
+    }
+
+    public void clear(final RecordType recordType, final String subject)
+            throws PwmUnrecoverableException
+    {
+        if (recordType == null) {
+            throw new IllegalArgumentException("recordType is required");
+        }
+
+        if (subject == null || subject.length() < 1) {
+            return;
+        }
+
+        final RecordManager manager = recordManagers.get(recordType);
+        manager.clearSubject(subject);
+    }
+
+    public void mark(final RecordType recordType, final String subject, final SessionLabel sessionLabel)
+            throws PwmUnrecoverableException
+    {
+        if (recordType == null) {
+            throw new IllegalArgumentException("recordType is required");
+        }
+
+        if (subject == null || subject.length() < 1) {
+            return;
+        }
+
+        if (recordType == RecordType.ADDRESS) {
+            try {
+                final InetAddress inetAddress = InetAddress.getByName(subject);
+                if (inetAddress.isAnyLocalAddress() || inetAddress.isLoopbackAddress() || inetAddress.isLinkLocalAddress()) {
+                    LOGGER.debug("disregarding local address intruder attempt from: " + subject);
+                    return;
+                }
+            } catch(Exception e) {
+                LOGGER.error("error examining address: " + subject);
+            }
+        }
+
+        final RecordManager manager = recordManagers.get(recordType);
+        manager.markSubject(subject);
+
+        { // send intruder attempt audit event
+            final Map<String,Object> messageObj = new LinkedHashMap<>();
+            messageObj.put("type", recordType);
+            messageObj.put("subject", subject);
+            final String message = JsonUtil.serializeMap(messageObj);
+            final SystemAuditRecord auditRecord = pwmApplication.getAuditManager().createSystemAuditRecord(AuditEvent.INTRUDER_ATTEMPT,message);
+            pwmApplication.getAuditManager().submit(auditRecord);
+        }
+
+        try {
+            check(recordType, subject);
+        } catch (PwmUnrecoverableException e) {
+            if (!manager.isAlerted(subject) ) {
+                { // send intruder attempt lock event
+                    final Map<String,Object> messageObj = new LinkedHashMap<>();
+                    messageObj.put("type", recordType);
+                    messageObj.put("subject", subject);
+                    final String message = JsonUtil.serializeMap(messageObj);
+                    final SystemAuditRecord auditRecord = pwmApplication.getAuditManager().createSystemAuditRecord(AuditEvent.INTRUDER_LOCK,message);
+                    pwmApplication.getAuditManager().submit(auditRecord);
+                }
+
+                if (recordType == RecordType.USER_ID) {
+                    final UserIdentity userIdentity = UserIdentity.fromKey(subject, pwmApplication);
+                    final UserAuditRecord auditRecord = pwmApplication.getAuditManager().createUserAuditRecord(
+                            AuditEvent.INTRUDER_USER,
+                            userIdentity,
+                            sessionLabel
+                    );
+                    pwmApplication.getAuditManager().submit(auditRecord);
+                    sendAlert(manager.readIntruderRecord(subject), sessionLabel);
+                }
+
+                manager.markAlerted(subject);
+                final StatisticsManager statisticsManager = pwmApplication.getStatisticsManager();
+                if (statisticsManager != null && statisticsManager.status() == STATUS.OPEN) {
+                    statisticsManager.incrementValue(Statistic.INTRUDER_ATTEMPTS);
+                    statisticsManager.updateEps(Statistic.EpsType.INTRUDER_ATTEMPTS,1);
+                    statisticsManager.incrementValue(recordType.getLockStatistic());
+                }
+            }
+            throw e;
+        }
+
+        delayPenalty(manager.readIntruderRecord(subject), sessionLabel == null ? null : sessionLabel);
+    }
+
+
+    private void delayPenalty(final IntruderRecord intruderRecord, final SessionLabel sessionLabel) {
+        int points = 0;
+        if (intruderRecord != null) {
+            points += intruderRecord.getAttemptCount();
+            long delayPenalty = Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_MIN_DELAY_PENALTY_MS)); // minimum
+            delayPenalty += points * Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_DELAY_PER_COUNT_MS));
+            delayPenalty += PwmRandom.getInstance().nextInt((int)Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_DELAY_MAX_JITTER_MS))); // add some randomness;
+            delayPenalty = delayPenalty > Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_MAX_DELAY_PENALTY_MS)) ? Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.INTRUDER_MAX_DELAY_PENALTY_MS)) : delayPenalty;
+            LOGGER.trace(sessionLabel, "delaying response " + delayPenalty + "ms due to intruder record: " + JsonUtil.serialize(intruderRecord));
+            Helper.pause(delayPenalty);
+        }
+    }
+
+    private void sendAlert(final IntruderRecord intruderRecord, final SessionLabel sessionLabel) {
+        if (intruderRecord == null) {
+            return;
+        }
+
+        if (intruderRecord.getType() == RecordType.USER_ID) {
+            try {
+                final UserIdentity identity = UserIdentity.fromDelimitedKey(intruderRecord.getSubject());
+                sendIntruderNoticeEmail(pwmApplication, sessionLabel, identity);
+            } catch (PwmUnrecoverableException e) {
+                LOGGER.error("unable to send intruder mail, can't read userDN/ldapProfile from stored record: " + e.getMessage());
+            }
+        }
+    }
+
+    public List<Map<String,Object>> getRecords(final RecordType recordType, int maximum)
+            throws PwmOperationalException
+    {
+        final RecordManager manager = recordManagers.get(recordType);
+        final ArrayList<Map<String,Object>> returnList = new ArrayList<>();
+
+        ClosableIterator<IntruderRecord> theIterator = null;
+        try {
+            theIterator = manager.iterator();
+            while (theIterator.hasNext() && returnList.size() < maximum) {
+                final IntruderRecord intruderRecord = theIterator.next();
+                if (intruderRecord != null && intruderRecord.getType() == recordType) {
+                    final Map<String, Object> rowData = new HashMap<>();
+                    rowData.put("subject", intruderRecord.getSubject());
+                    rowData.put("timestamp", intruderRecord.getTimeStamp());
+                    rowData.put("count", String.valueOf(intruderRecord.getAttemptCount()));
+                    try {
+                        check(recordType, intruderRecord.getSubject());
+                        rowData.put("status", "watching");
+                    } catch (PwmException e) {
+                        rowData.put("status", "locked");
+                    }
+                    returnList.add(rowData);
+                }
+            }
+        } finally {
+            if (theIterator != null) {
+                theIterator.close();
+            }
+        }
+        return returnList;
+    }
+
+    public Convenience convenience() {
+        return new Convenience();
+    }
+
+    public class Convenience {
+        protected Convenience() {
+        }
+
+        public void markAddressAndSession(final PwmSession pwmSession)
+                throws PwmUnrecoverableException
+        {
+            if (pwmSession != null) {
+                final String subject = pwmSession.getSessionStateBean().getSrcAddress();
+                pwmSession.getSessionStateBean().incrementIntruderAttempts();
+                mark(RecordType.ADDRESS, subject, pwmSession.getLabel());
+            }
+        }
+
+        public void checkAddressAndSession(final PwmSession pwmSession)
+                throws PwmUnrecoverableException
+        {
+            if (pwmSession != null) {
+                final String subject = pwmSession.getSessionStateBean().getSrcAddress();
+                check(RecordType.ADDRESS, subject);
+                final int maxAllowedAttempts = (int)pwmApplication.getConfig().readSettingAsLong(PwmSetting.INTRUDER_SESSION_MAX_ATTEMPTS);
+                if (maxAllowedAttempts != 0 && pwmSession.getSessionStateBean().getIntruderAttempts() > maxAllowedAttempts) {
+                    throw new PwmUnrecoverableException(PwmError.ERROR_INTRUDER_SESSION);
+                }
+            }
+        }
+
+        public void clearAddressAndSession(final PwmSession pwmSession)
+                throws PwmUnrecoverableException
+        {
+            if (pwmSession != null) {
+                final String subject = pwmSession.getSessionStateBean().getSrcAddress();
+                clear(RecordType.ADDRESS, subject);
+                pwmSession.getSessionStateBean().clearIntruderAttempts();
+            }
+        }
+
+        public void markUserIdentity(final UserIdentity userIdentity, final SessionLabel sessionLabel)
+                throws PwmUnrecoverableException
+        {
+            if (userIdentity != null) {
+                final String subject = userIdentity.toDelimitedKey();
+                mark(RecordType.USER_ID, subject, sessionLabel);
+            }
+        }
+
+        public void markUserIdentity(final UserIdentity userIdentity, final PwmSession pwmSession)
+                throws PwmUnrecoverableException
+        {
+            if (userIdentity != null) {
+                final String subject = userIdentity.toDelimitedKey();
+                mark(RecordType.USER_ID, subject, pwmSession.getLabel());
+            }
+        }
+
+        public void checkUserIdentity(final UserIdentity userIdentity)
+                throws PwmUnrecoverableException
+        {
+            if (userIdentity != null) {
+                final String subject = userIdentity.toDelimitedKey();
+                check(RecordType.USER_ID, subject);
+            }
+        }
+
+        public void clearUserIdentity(final UserIdentity userIdentity)
+                throws PwmUnrecoverableException
+        {
+            if (userIdentity != null) {
+                final String subject = userIdentity.toDelimitedKey();
+                clear(RecordType.USER_ID, subject);
+            }
+        }
+
+        public void markAttributes(final Map<FormConfiguration, String> formValues, final PwmSession pwmSession)
+                throws PwmUnrecoverableException
+        {
+            final List<String> subjects = attributeFormToList(formValues);
+            for (String subject : subjects) {
+                mark(RecordType.ATTRIBUTE, subject, pwmSession.getLabel());
+            }
+        }
+
+        public void clearAttributes(final Map<FormConfiguration, String> formValues)
+                throws PwmUnrecoverableException
+        {
+            final List<String> subjects = attributeFormToList(formValues);
+            for (String subject : subjects) {
+                clear(RecordType.ATTRIBUTE, subject);
+            }
+        }
+
+        public void checkAttributes(final Map<FormConfiguration, String> formValues)
+                throws PwmUnrecoverableException
+        {
+            final List<String> subjects = attributeFormToList(formValues);
+            for (String subject : subjects) {
+                check(RecordType.ATTRIBUTE, subject);
+            }
+        }
+
+        private List<String> attributeFormToList(final Map<FormConfiguration, String> formValues) {
+            final List<String> returnList = new ArrayList<>();
+            if (formValues != null) {
+                for (final FormConfiguration formConfiguration : formValues.keySet()) {
+                    final String value = formValues.get(formConfiguration);
+                    if (value != null && value.length() > 0) {
+                        returnList.add(formConfiguration.getName() + ":" + value);
+                    }
+                }
+            }
+            return returnList;
+        }
+
+    }
+
+    private static void sendIntruderNoticeEmail(
+            final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
+            final UserIdentity userIdentity
+    )
+    {
+        final Configuration config = pwmApplication.getConfig();
+        final EmailItemBean configuredEmailSetting = config.readSettingAsEmail(PwmSetting.EMAIL_INTRUDERNOTICE, PwmConstants.DEFAULT_LOCALE);
+
+        if (configuredEmailSetting == null) {
+            return;
+        }
+
+        try {
+            final UserStatusReader userStatusReader = new UserStatusReader(pwmApplication, null);
+            final UserInfoBean userInfoBean = userStatusReader.populateUserInfoBean(
+                    PwmConstants.DEFAULT_LOCALE,
+                    userIdentity
+            );
+
+            final MacroMachine macroMachine = new MacroMachine(
+                    pwmApplication,
+                    sessionLabel,
+                    userInfoBean,
+                    null,
+                    LdapUserDataReader.appProxiedReader(pwmApplication, userIdentity));
+
+            pwmApplication.getEmailQueue().submitEmail(configuredEmailSetting, userInfoBean, macroMachine);
+        } catch (PwmUnrecoverableException e) {
+            LOGGER.error("error reading user info while sending intruder notice for user " + userIdentity + ", error: " + e.getMessage());
+        }
+
+    }
+
+    public ServiceInfo serviceInfo()
+    {
+        return serviceInfo;
+    }
 }

pwm/servlet/src/password/pwm/util/intruder/IntruderRecord.java → pwm/servlet/src/password/pwm/svc/intruder/IntruderRecord.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.intruder;
+package password.pwm.svc.intruder;
 
 import java.io.Serializable;
 import java.util.Date;

pwm/servlet/src/password/pwm/util/intruder/IntruderSettings.java → pwm/servlet/src/password/pwm/svc/intruder/IntruderSettings.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.intruder;
+package password.pwm.svc.intruder;
 
 import password.pwm.util.TimeDuration;
 

pwm/servlet/src/password/pwm/util/intruder/RecordManager.java → pwm/servlet/src/password/pwm/svc/intruder/RecordManager.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.intruder;
+package password.pwm.svc.intruder;
 
 import password.pwm.error.PwmOperationalException;
 import password.pwm.util.ClosableIterator;

pwm/servlet/src/password/pwm/util/intruder/RecordManagerImpl.java → pwm/servlet/src/password/pwm/svc/intruder/RecordManagerImpl.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.intruder;
+package password.pwm.svc.intruder;
 
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;

pwm/servlet/src/password/pwm/util/intruder/RecordStore.java → pwm/servlet/src/password/pwm/svc/intruder/RecordStore.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.intruder;
+package password.pwm.svc.intruder;
 
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;

pwm/servlet/src/password/pwm/util/intruder/RecordType.java → pwm/servlet/src/password/pwm/svc/intruder/RecordType.java

@@ -20,9 +20,9 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.intruder;
+package password.pwm.svc.intruder;
 
-import password.pwm.util.stats.Statistic;
+import password.pwm.svc.stats.Statistic;
 
 public enum RecordType {
     ADDRESS(Statistic.LOCKED_ADDRESSES),

pwm/servlet/src/password/pwm/util/intruder/StubRecordManager.java → pwm/servlet/src/password/pwm/svc/intruder/StubRecordManager.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.intruder;
+package password.pwm.svc.intruder;
 
 import password.pwm.error.PwmOperationalException;
 import password.pwm.util.ClosableIterator;

pwm/servlet/src/password/pwm/util/report/ReportService.java → pwm/servlet/src/password/pwm/svc/report/ReportService.java

@@ -1,686 +1,686 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.util.report;
-
-import com.novell.ldapchai.exception.ChaiOperationException;
-import com.novell.ldapchai.exception.ChaiUnavailableException;
-import com.novell.ldapchai.provider.ChaiProvider;
-import org.apache.commons.csv.CSVPrinter;
-import password.pwm.AppProperty;
-import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
-import password.pwm.PwmService;
-import password.pwm.bean.UserIdentity;
-import password.pwm.bean.UserInfoBean;
-import password.pwm.config.Configuration;
-import password.pwm.config.PwmSetting;
-import password.pwm.config.option.DataStorageMethod;
-import password.pwm.error.*;
-import password.pwm.health.HealthRecord;
-import password.pwm.i18n.Display;
-import password.pwm.ldap.UserSearchEngine;
-import password.pwm.ldap.UserStatusReader;
-import password.pwm.util.*;
-import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.localdb.LocalDBException;
-import password.pwm.util.logging.PwmLogger;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.math.MathContext;
-import java.util.*;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-public class ReportService implements PwmService {
-    private static final PwmLogger LOGGER = PwmLogger.forClass(ReportService.class);
-
-    private final AvgTracker avgTracker = new AvgTracker(100);
-
-    private PwmApplication pwmApplication;
-    private STATUS status = STATUS.NEW;
-    private boolean cancelFlag = false;
-    private ReportStatusInfo reportStatus = new ReportStatusInfo("");
-    private ReportSummaryData summaryData = ReportSummaryData.newSummaryData(null);
-    private ScheduledExecutorService executorService;
-
-    private UserCacheService userCacheService;
-    private ReportSettings settings = new ReportSettings();
-
-    public ReportService() {
-    }
-
-    public STATUS status()
-    {
-        return status;
-    }
-
-    public void clear()
-            throws LocalDBException, PwmUnrecoverableException
-    {
-        final Date startTime = new Date();
-        LOGGER.info(PwmConstants.REPORTING_SESSION_LABEL,"clearing cached report data");
-        if (userCacheService != null) {
-            userCacheService.clear();
-        }
-        summaryData = ReportSummaryData.newSummaryData(settings.getTrackDays());
-        reportStatus = new ReportStatusInfo(settings.getSettingsHash());
-        saveTempData();
-        LOGGER.info(PwmConstants.REPORTING_SESSION_LABEL,"finished clearing report " + TimeDuration.fromCurrent(startTime).asCompactString());
-    }
-
-    @Override
-    public void init(PwmApplication pwmApplication)
-            throws PwmException
-    {
-        status = STATUS.OPENING;
-        this.pwmApplication = pwmApplication;
-
-        if (pwmApplication.getApplicationMode() == PwmApplication.MODE.READ_ONLY) {
-            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"application mode is read-only, will remain closed");
-            status = STATUS.CLOSED;
-            return;
-        }
-
-        if (pwmApplication.getLocalDB() == null || LocalDB.Status.OPEN != pwmApplication.getLocalDB().status()) {
-            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"LocalDB is not open, will remain closed");
-            status = STATUS.CLOSED;
-            return;
-        }
-
-        if (!pwmApplication.getConfig().readSettingAsBoolean(PwmSetting.REPORTING_ENABLE)) {
-            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"reporting module is not enabled, will remain closed");
-            status = STATUS.CLOSED;
-            clear();
-            return;
-        }
-
-        try {
-            userCacheService = new UserCacheService();
-            userCacheService.init(pwmApplication);
-        } catch (Exception e) {
-            LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL,"unable to init cache service");
-            status = STATUS.CLOSED;
-            return;
-        }
-
-        settings = ReportSettings.readSettingsFromConfig(pwmApplication.getConfig());
-        summaryData = ReportSummaryData.newSummaryData(settings.getTrackDays());
-
-        executorService = Executors.newSingleThreadScheduledExecutor(
-                Helper.makePwmThreadFactory(
-                        Helper.makeThreadName(pwmApplication,this.getClass()) + "-",
-                        true
-                ));
-
-
-        String startupMsg = "report service started";
-        LOGGER.debug(startupMsg);
-
-        executorService.submit(new InitializationTask());
-
-        status = STATUS.OPEN;
-    }
-
-    @Override
-    public void close()
-    {
-        status = STATUS.CLOSED;
-        saveTempData();
-        pwmApplication.writeAppAttribute(PwmApplication.AppAttribute.REPORT_CLEAN_FLAG, "true");
-        if (userCacheService != null) {
-            userCacheService.close();
-        }
-        if (executorService != null) {
-            executorService.shutdown();
-        }
-        executorService = null;
-    }
-
-    private void saveTempData() {
-        try {
-            final String jsonInfo = JsonUtil.serialize(reportStatus);
-            pwmApplication.writeAppAttribute(PwmApplication.AppAttribute.REPORT_STATUS,jsonInfo);
-        } catch (Exception e) {
-            LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL,"error writing cached report dredge info into memory: " + e.getMessage());
-        }
-    }
-
-    private void initTempData()
-            throws LocalDBException, PwmUnrecoverableException
-    {
-        final String cleanFlag = pwmApplication.readAppAttribute(PwmApplication.AppAttribute.REPORT_CLEAN_FLAG);
-        if (!"true".equals(cleanFlag)) {
-            LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL, "did not shut down cleanly");
-            reportStatus = new ReportStatusInfo(settings.getSettingsHash());
-            reportStatus.setTotal(userCacheService.size());
-        } else {
-            try {
-                final String jsonInfo = pwmApplication.readAppAttribute(PwmApplication.AppAttribute.REPORT_STATUS);
-                if (jsonInfo != null && !jsonInfo.isEmpty()) {
-                    reportStatus = JsonUtil.deserialize(jsonInfo,ReportStatusInfo.class);
-                }
-            } catch (Exception e) {
-                LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL,"error loading cached report status info into memory: " + e.getMessage());
-            }
-        }
-
-        reportStatus = reportStatus == null ? new ReportStatusInfo(settings.getSettingsHash()) : reportStatus; //safety
-
-        final String currentSettingCache = settings.getSettingsHash();
-        if (reportStatus.getSettingsHash() != null && !reportStatus.getSettingsHash().equals(currentSettingCache)) {
-            LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL,"configuration has changed, will clear cached report data");
-            clear();
-        }
-
-        reportStatus.setInProgress(false);
-
-        pwmApplication.writeAppAttribute(PwmApplication.AppAttribute.REPORT_CLEAN_FLAG, "false");
-    }
-
-    @Override
-    public List<HealthRecord> healthCheck()
-    {
-        return null;
-    }
-
-    @Override
-    public ServiceInfo serviceInfo()
-    {
-        return new ServiceInfo(Collections.singletonList(DataStorageMethod.LDAP));
-    }
-
-    public void scheduleImmediateUpdate() {
-        if (!reportStatus.isInProgress()) {
-            executorService.submit(new DredgeTask());
-            LOGGER.trace(PwmConstants.REPORTING_SESSION_LABEL,"submitted new ldap dredge task to executorService");
-        }
-    }
-
-    public void cancelUpdate() {
-        cancelFlag = true;
-    }
-
-    private void updateCacheFromLdap()
-            throws ChaiUnavailableException, ChaiOperationException, PwmOperationalException, PwmUnrecoverableException
-    {
-        LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL, "beginning process to updating user cache records from ldap");
-        if (status != STATUS.OPEN) {
-            return;
-        }
-        cancelFlag = false;
-        reportStatus = new ReportStatusInfo(settings.getSettingsHash());
-        reportStatus.setInProgress(true);
-        reportStatus.setStartDate(new Date());
-        try {
-            final Queue<UserIdentity> allUsers = new LinkedList<>(getListOfUsers());
-            reportStatus.setTotal(allUsers.size());
-            while (status == STATUS.OPEN && !allUsers.isEmpty() && !cancelFlag) {
-                final Date startUpdateTime = new Date();
-                final UserIdentity userIdentity = allUsers.poll();
-                try {
-                    if (updateCachedRecordFromLdap(userIdentity)) {
-                        reportStatus.setUpdated(reportStatus.getUpdated() + 1);
-                    }
-                } catch (Exception e) {
-                    String errorMsg = "error while updating report cache for " + userIdentity.toString() + ", cause: ";
-                    errorMsg += e instanceof PwmException ? ((PwmException) e).getErrorInformation().toDebugStr() : e.getMessage();
-                    final ErrorInformation errorInformation;
-                    errorInformation = new ErrorInformation(PwmError.ERROR_REPORTING_ERROR,errorMsg);
-                    LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL,errorInformation.toDebugStr());
-                    reportStatus.setLastError(errorInformation);
-                    reportStatus.setErrors(reportStatus.getErrors() + 1);
-                }
-                reportStatus.setCount(reportStatus.getCount() + 1);
-                reportStatus.getEventRateMeter().markEvents(1);
-                final TimeDuration totalUpdateTime = TimeDuration.fromCurrent(startUpdateTime);
-                if (settings.isAutoCalcRest()) {
-                    avgTracker.addSample(totalUpdateTime.getTotalMilliseconds());
-                    Helper.pause(avgTracker.avgAsLong());
-                } else {
-                    Helper.pause(settings.getRestTime().getTotalMilliseconds());
-                }
-            }
-            if (cancelFlag) {
-                reportStatus.setLastError(new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,"report cancelled by operator"));
-            }
-        } finally {
-            reportStatus.setFinishDate(new Date());
-            reportStatus.setInProgress(false);
-        }
-        LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"update user cache process completed: " + JsonUtil.serialize(reportStatus));
-    }
-
-    private void updateRestingCacheData() {
-        final long startTime = System.currentTimeMillis();
-        int examinedRecords = 0;
-        ClosableIterator<UserCacheRecord> iterator = null;
-        try {
-            LOGGER.trace(PwmConstants.REPORTING_SESSION_LABEL, "checking size of stored cache records");
-            final int totalRecords = userCacheService.size();
-            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL, "beginning cache review process of " + totalRecords + " records");
-            iterator = iterator();
-            Date lastLogOutputTime = new Date();
-            while (iterator.hasNext() && status == STATUS.OPEN) {
-                final UserCacheRecord record = iterator.next(); // (purge routine is embedded in next();
-
-                if (summaryData != null && record != null) {
-                    summaryData.update(record);
-                }
-
-                examinedRecords++;
-
-                if (TimeDuration.fromCurrent(lastLogOutputTime).isLongerThan(30, TimeUnit.SECONDS)) {
-                    final TimeDuration progressDuration = TimeDuration.fromCurrent(startTime);
-                    LOGGER.trace(PwmConstants.REPORTING_SESSION_LABEL,"cache review process in progress, examined "
-                            + examinedRecords + " records in " + progressDuration.asCompactString());
-                    lastLogOutputTime = new Date();
-                }
-            }
-            final TimeDuration totalTime = TimeDuration.fromCurrent(startTime);
-            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,
-                    "completed cache review process of " + examinedRecords + " cached report records in " + totalTime.asCompactString());
-        } finally {
-            if (iterator != null) {
-                iterator.close();
-            }
-        }
-    }
-
-    public boolean updateCachedRecordFromLdap(final UserInfoBean uiBean)
-            throws LocalDBException, PwmUnrecoverableException, ChaiUnavailableException
-    {
-        if (status != STATUS.OPEN) {
-            return false;
-        }
-
-        final UserCacheService.StorageKey storageKey = UserCacheService.StorageKey.fromUserInfoBean(uiBean);
-        return updateCachedRecordFromLdap(uiBean.getUserIdentity(), uiBean, storageKey);
-    }
-
-    private boolean updateCachedRecordFromLdap(final UserIdentity userIdentity)
-            throws ChaiUnavailableException, PwmUnrecoverableException, LocalDBException
-    {
-        if (status != STATUS.OPEN) {
-            return false;
-        }
-
-        final UserCacheService.StorageKey storageKey = UserCacheService.StorageKey.fromUserIdentity(pwmApplication,
-                userIdentity);
-        return updateCachedRecordFromLdap(userIdentity, null, storageKey);
-    }
-
-    private boolean updateCachedRecordFromLdap(
-            final UserIdentity userIdentity,
-            final UserInfoBean userInfoBean,
-            final UserCacheService.StorageKey storageKey
-    )
-            throws ChaiUnavailableException, PwmUnrecoverableException, LocalDBException
-    {
-        final UserCacheRecord userCacheRecord = userCacheService.readStorageKey(storageKey);
-        TimeDuration cacheAge = null;
-        if (userCacheRecord != null && userCacheRecord.getCacheTimestamp() != null) {
-            cacheAge = TimeDuration.fromCurrent(userCacheRecord.getCacheTimestamp());
-        }
-
-        boolean updateCache = false;
-        if (userInfoBean != null) {
-            updateCache = true;
-        } else {
-            if (cacheAge == null) {
-                LOGGER.trace(PwmConstants.REPORTING_SESSION_LABEL,"stored cache for " + userIdentity + " is missing cache storage timestamp, will update");
-                updateCache = true;
-            } else if (cacheAge.isLongerThan(settings.getMinCacheAge())) {
-                LOGGER.trace(PwmConstants.REPORTING_SESSION_LABEL,"stored cache for " + userIdentity + " is " + cacheAge.asCompactString() + " old, will update");
-                updateCache = true;
-            }
-        }
-
-        if (updateCache) {
-            if (userCacheRecord != null) {
-                if (summaryData != null && summaryData.getEpoch() != null && summaryData.getEpoch().equals(userCacheRecord.getSummaryEpoch())) {
-                    summaryData.remove(userCacheRecord);
-                }
-            }
-            final UserInfoBean newUserBean;
-            if (userInfoBean != null) {
-                newUserBean = userInfoBean;
-            } else {
-                newUserBean = new UserInfoBean();
-                final UserStatusReader.Settings readerSettings = new UserStatusReader.Settings();
-                readerSettings.setSkipReportUpdate(true);
-                final ChaiProvider chaiProvider = pwmApplication.getProxyChaiProvider(userIdentity.getLdapProfileID());
-                final UserStatusReader userStatusReader = new UserStatusReader(pwmApplication,PwmConstants.REPORTING_SESSION_LABEL,readerSettings);
-                userStatusReader.populateUserInfoBean(
-                        newUserBean,
-                        PwmConstants.DEFAULT_LOCALE,
-                        userIdentity,
-                        chaiProvider
-                );
-            }
-            final UserCacheRecord newUserCacheRecord = userCacheService.updateUserCache(newUserBean);
-
-            if (summaryData != null && summaryData.getEpoch() != null && newUserCacheRecord != null) {
-                if (!summaryData.getEpoch().equals(newUserCacheRecord.getSummaryEpoch())) {
-                    newUserCacheRecord.setSummaryEpoch(summaryData.getEpoch());
-                    userCacheService.store(newUserCacheRecord);
-                }
-                summaryData.update(newUserCacheRecord);
-            }
-        }
-
-        return updateCache;
-    }
-
-
-    public ReportStatusInfo getReportStatusInfo()
-    {
-        return reportStatus;
-    }
-
-    private List<UserIdentity> getListOfUsers()
-            throws ChaiUnavailableException, ChaiOperationException, PwmUnrecoverableException, PwmOperationalException
-    {
-        return readAllUsersFromLdap(pwmApplication, settings.getSearchFilter(), settings.getMaxSearchSize());
-    }
-
-    private static List<UserIdentity> readAllUsersFromLdap(
-            final PwmApplication pwmApplication,
-            final String searchFilter,
-            final int maxResults
-    )
-            throws ChaiUnavailableException, ChaiOperationException, PwmUnrecoverableException, PwmOperationalException
-    {
-        final UserSearchEngine userSearchEngine = new UserSearchEngine(pwmApplication,null);
-        final UserSearchEngine.SearchConfiguration searchConfiguration = new UserSearchEngine.SearchConfiguration();
-        searchConfiguration.setEnableValueEscaping(false);
-        searchConfiguration.setSearchTimeout(Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.REPORTING_LDAP_SEARCH_TIMEOUT)));
-
-        if (searchFilter == null) {
-            searchConfiguration.setUsername("*");
-        } else {
-            searchConfiguration.setFilter(searchFilter);
-        }
-
-        LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"beginning UserReportService user search using parameters: " + (JsonUtil.serialize(searchConfiguration)));
-
-        final Map<UserIdentity,Map<String,String>> searchResults = userSearchEngine.performMultiUserSearch(searchConfiguration, maxResults, Collections.<String>emptyList());
-        LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"user search found " + searchResults.size() + " users for reporting");
-        final List<UserIdentity> returnList = new ArrayList<>(searchResults.keySet());
-        Collections.shuffle(returnList);
-        return returnList;
-    }
-
-    public RecordIterator<UserCacheRecord> iterator() {
-        return new RecordIterator<>(userCacheService.<UserCacheService.StorageKey>iterator());
-    }
-
-    public class RecordIterator<K> implements ClosableIterator<UserCacheRecord> {
-
-        private UserCacheService.UserStatusCacheBeanIterator<UserCacheService.StorageKey> storageKeyIterator;
-
-        public RecordIterator(UserCacheService.UserStatusCacheBeanIterator<UserCacheService.StorageKey> storageKeyIterator) {
-            this.storageKeyIterator = storageKeyIterator;
-        }
-
-        public boolean hasNext() {
-            return this.storageKeyIterator.hasNext();
-        }
-
-        public UserCacheRecord next()
-        {
-            try {
-                UserCacheRecord returnBean = null;
-                while (returnBean == null && this.storageKeyIterator.hasNext()) {
-                    UserCacheService.StorageKey key = this.storageKeyIterator.next();
-                    returnBean = userCacheService.readStorageKey(key);
-                    if (returnBean != null) {
-                        if (returnBean.getCacheTimestamp() == null) {
-                            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"purging record due to missing cache timestamp: " + JsonUtil.serialize(returnBean));
-                            userCacheService.removeStorageKey(key);
-                        } else if (TimeDuration.fromCurrent(returnBean.getCacheTimestamp()).isLongerThan(settings.getMaxCacheAge())) {
-                            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"purging record due to old age timestamp: " + JsonUtil.serialize(returnBean));
-                            userCacheService.removeStorageKey(key);
-                        } else {
-                            return returnBean;
-                        }
-                    }
-
-                }
-            } catch (LocalDBException e) {
-                throw new IllegalStateException("unexpected iterator traversal error while reading LocalDB: " + e.getMessage());
-            }
-            return null;
-        }
-
-        public void remove()
-        {
-
-        }
-
-        public void close() {
-            storageKeyIterator.close();
-        }
-    }
-
-    public void outputSummaryToCsv(final OutputStream outputStream, final Locale locale)
-            throws IOException
-    {
-        final List<ReportSummaryData.PresentationRow> outputList = summaryData.asPresentableCollection(pwmApplication.getConfig(),locale);
-        final CSVPrinter csvPrinter = Helper.makeCsvPrinter(outputStream);
-
-        for (final ReportSummaryData.PresentationRow presentationRow : outputList) {
-            final List<String> headerRow = new ArrayList<>();
-            headerRow.add(presentationRow.getLabel());
-            headerRow.add(presentationRow.getCount());
-            headerRow.add(presentationRow.getPct());
-            csvPrinter.printRecord(headerRow);
-        }
-
-        csvPrinter.close();
-    }
-
-    public void outputToCsv(final OutputStream outputStream, final boolean includeHeader, final Locale locale)
-            throws IOException, ChaiUnavailableException, ChaiOperationException, PwmUnrecoverableException, PwmOperationalException
-    {
-        final CSVPrinter csvPrinter = Helper.makeCsvPrinter(outputStream);
-        final Configuration config = pwmApplication.getConfig();
-        final Class localeClass = password.pwm.i18n.Admin.class;
-        if (includeHeader) {
-            final List<String> headerRow = new ArrayList<>();
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_UserDN", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_LDAP_Profile", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_Username", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_Email", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_UserGuid", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_LastLogin", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_PwdExpireTime", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_PwdChangeTime", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_ResponseSaveTime", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_HasResponses", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_HasHelpdeskResponses", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_ResponseStorageMethod", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_PwdExpired", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_PwdPreExpired", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_PwdViolatesPolicy", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_PwdWarnPeriod", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_RequiresPasswordUpdate", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_RequiresResponseUpdate", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_RequiresProfileUpdate", config, localeClass));
-            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_RecordCacheTime", config, localeClass));
-            csvPrinter.printRecord(headerRow);
-        }
-
-        ClosableIterator<UserCacheRecord> cacheBeanIterator = null;
-        try {
-            cacheBeanIterator = this.iterator();
-            while (cacheBeanIterator.hasNext()) {
-                final UserCacheRecord userCacheRecord = cacheBeanIterator.next();
-                outputRecordRow(config, locale, userCacheRecord, csvPrinter);
-            }
-        } finally {
-            if (cacheBeanIterator != null) {
-                cacheBeanIterator.close();
-            }
-        }
-
-        csvPrinter.flush();
-    }
-
-    private void outputRecordRow(
-            final Configuration config,
-            final Locale locale,
-            final UserCacheRecord userCacheRecord,
-            final CSVPrinter csvPrinter
-    )
-            throws IOException
-    {
-        final String trueField = Display.getLocalizedMessage(locale, Display.Value_True, config);
-        final String falseField = Display.getLocalizedMessage(locale, Display.Value_False, config);
-        final String naField = Display.getLocalizedMessage(locale, Display.Value_NotApplicable, config);
-        final List<String> csvRow = new ArrayList<>();
-        csvRow.add(userCacheRecord.getUserDN());
-        csvRow.add(userCacheRecord.getLdapProfile());
-        csvRow.add(userCacheRecord.getUsername());
-        csvRow.add(userCacheRecord.getEmail());
-        csvRow.add(userCacheRecord.getUserGUID());
-        csvRow.add(userCacheRecord.getLastLoginTime() == null ? naField : PwmConstants.DEFAULT_DATETIME_FORMAT.format(
-                userCacheRecord.getLastLoginTime()));
-        csvRow.add(userCacheRecord.getPasswordExpirationTime() == null ? naField : PwmConstants.DEFAULT_DATETIME_FORMAT.format(
-                userCacheRecord.getPasswordExpirationTime()));
-        csvRow.add(userCacheRecord.getPasswordChangeTime() == null ? naField : PwmConstants.DEFAULT_DATETIME_FORMAT.format(
-                userCacheRecord.getPasswordChangeTime()));
-        csvRow.add(userCacheRecord.getResponseSetTime() == null ? naField : PwmConstants.DEFAULT_DATETIME_FORMAT.format(
-                userCacheRecord.getResponseSetTime()));
-        csvRow.add(userCacheRecord.isHasResponses() ? trueField : falseField);
-        csvRow.add(userCacheRecord.isHasHelpdeskResponses() ? trueField : falseField);
-        csvRow.add(userCacheRecord.getResponseStorageMethod() == null ? naField : userCacheRecord.getResponseStorageMethod().toString());
-        csvRow.add(userCacheRecord.getPasswordStatus().isExpired() ? trueField : falseField);
-        csvRow.add(userCacheRecord.getPasswordStatus().isPreExpired() ? trueField : falseField);
-        csvRow.add(userCacheRecord.getPasswordStatus().isViolatesPolicy() ? trueField : falseField);
-        csvRow.add(userCacheRecord.getPasswordStatus().isWarnPeriod() ? trueField : falseField);
-        csvRow.add(userCacheRecord.isRequiresPasswordUpdate() ? trueField : falseField);
-        csvRow.add(userCacheRecord.isRequiresResponseUpdate() ? trueField : falseField);
-        csvRow.add(userCacheRecord.isRequiresProfileUpdate() ? trueField : falseField);
-        csvRow.add(userCacheRecord.getCacheTimestamp() == null ? naField : PwmConstants.DEFAULT_DATETIME_FORMAT.format(
-                userCacheRecord.getCacheTimestamp()));
-
-        csvPrinter.printRecord(csvRow);
-    }
-
-    public ReportSummaryData getSummaryData() {
-        return summaryData;
-    }
-
-    public static class AvgTracker {
-        private final int maxSamples;
-        private final Queue<BigInteger> samples = new LinkedList<>();
-
-        public AvgTracker(int maxSamples)
-        {
-            this.maxSamples = maxSamples;
-        }
-
-        public void addSample(final long input) {
-            samples.add(new BigInteger(Long.toString(input)));
-            while (samples.size() > maxSamples) {
-                samples.remove();
-            }
-        }
-
-        public BigDecimal avg() {
-            if (samples.isEmpty()) {
-                throw new IllegalStateException("unable to compute avg without samples");
-            }
-
-            BigInteger total = BigInteger.ZERO;
-            for (final BigInteger sample : samples) {
-                total = total.add(sample);
-            }
-            final BigDecimal maxAsBD = new BigDecimal(Integer.toString(maxSamples));
-            return new BigDecimal(total).divide(maxAsBD, MathContext.DECIMAL32);
-        }
-
-        public long avgAsLong() {
-            return avg().longValue();
-        }
-    }
-
-    private class DredgeTask implements Runnable {
-        @Override
-        public void run()
-        {
-            reportStatus.setCurrentProcess(ReportStatusInfo.ReportEngineProcess.DredgeTask);
-            try {
-                updateCacheFromLdap();
-            } catch (Exception e) {
-                if (e instanceof PwmException) {
-                    if (((PwmException) e).getErrorInformation().getError() == PwmError.ERROR_DIRECTORY_UNAVAILABLE) {
-                        if (executorService != null) {
-                            LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL, "directory unavailable error during background DredgeTask, will retry; error: " + e.getMessage());
-                            executorService.schedule(new DredgeTask(), 10, TimeUnit.MINUTES);
-                        }
-                    } else {
-                        LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL, "error during background DredgeTask: " + e.getMessage());
-                    }
-                }
-            } finally {
-                reportStatus.setCurrentProcess(ReportStatusInfo.ReportEngineProcess.None);
-            }
-        }
-    }
-
-    private class RolloverTask implements Runnable {
-        @Override
-        public void run()
-        {
-            reportStatus.setCurrentProcess(ReportStatusInfo.ReportEngineProcess.RollOver);
-            try {
-                summaryData = ReportSummaryData.newSummaryData(settings.getTrackDays());
-                updateRestingCacheData();
-            } finally {
-                reportStatus.setCurrentProcess(ReportStatusInfo.ReportEngineProcess.None);
-            }
-        }
-    }
-
-    private class InitializationTask implements Runnable {
-        @Override
-        public void run() {
-            try {
-                initTempData();
-            } catch (LocalDBException | PwmUnrecoverableException e) {
-                LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL, "error during initialization: " + e.getMessage());
-                status = STATUS.CLOSED;
-                return;
-            }
-            final long secondsUntilNextDredge = settings.getJobOffsetSeconds() + TimeDuration.fromCurrent(Helper.nextZuluZeroTime()).getTotalSeconds();
-            executorService.scheduleAtFixedRate(new DredgeTask(), secondsUntilNextDredge, TimeDuration.DAY.getTotalSeconds(), TimeUnit.SECONDS);
-            executorService.scheduleAtFixedRate(new RolloverTask(), secondsUntilNextDredge + 1, TimeDuration.DAY.getTotalSeconds(), TimeUnit.SECONDS);
-            executorService.submit(new RolloverTask());
-        }
-    }
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.report;
+
+import com.novell.ldapchai.exception.ChaiOperationException;
+import com.novell.ldapchai.exception.ChaiUnavailableException;
+import com.novell.ldapchai.provider.ChaiProvider;
+import org.apache.commons.csv.CSVPrinter;
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.PwmConstants;
+import password.pwm.bean.UserIdentity;
+import password.pwm.bean.UserInfoBean;
+import password.pwm.config.Configuration;
+import password.pwm.config.PwmSetting;
+import password.pwm.config.option.DataStorageMethod;
+import password.pwm.error.*;
+import password.pwm.health.HealthRecord;
+import password.pwm.i18n.Display;
+import password.pwm.ldap.UserSearchEngine;
+import password.pwm.ldap.UserStatusReader;
+import password.pwm.svc.PwmService;
+import password.pwm.util.*;
+import password.pwm.util.localdb.LocalDB;
+import password.pwm.util.localdb.LocalDBException;
+import password.pwm.util.logging.PwmLogger;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.MathContext;
+import java.util.*;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class ReportService implements PwmService {
+    private static final PwmLogger LOGGER = PwmLogger.forClass(ReportService.class);
+
+    private final AvgTracker avgTracker = new AvgTracker(100);
+
+    private PwmApplication pwmApplication;
+    private STATUS status = STATUS.NEW;
+    private boolean cancelFlag = false;
+    private ReportStatusInfo reportStatus = new ReportStatusInfo("");
+    private ReportSummaryData summaryData = ReportSummaryData.newSummaryData(null);
+    private ScheduledExecutorService executorService;
+
+    private UserCacheService userCacheService;
+    private ReportSettings settings = new ReportSettings();
+
+    public ReportService() {
+    }
+
+    public STATUS status()
+    {
+        return status;
+    }
+
+    public void clear()
+            throws LocalDBException, PwmUnrecoverableException
+    {
+        final Date startTime = new Date();
+        LOGGER.info(PwmConstants.REPORTING_SESSION_LABEL,"clearing cached report data");
+        if (userCacheService != null) {
+            userCacheService.clear();
+        }
+        summaryData = ReportSummaryData.newSummaryData(settings.getTrackDays());
+        reportStatus = new ReportStatusInfo(settings.getSettingsHash());
+        saveTempData();
+        LOGGER.info(PwmConstants.REPORTING_SESSION_LABEL,"finished clearing report " + TimeDuration.fromCurrent(startTime).asCompactString());
+    }
+
+    @Override
+    public void init(PwmApplication pwmApplication)
+            throws PwmException
+    {
+        status = STATUS.OPENING;
+        this.pwmApplication = pwmApplication;
+
+        if (pwmApplication.getApplicationMode() == PwmApplication.MODE.READ_ONLY) {
+            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"application mode is read-only, will remain closed");
+            status = STATUS.CLOSED;
+            return;
+        }
+
+        if (pwmApplication.getLocalDB() == null || LocalDB.Status.OPEN != pwmApplication.getLocalDB().status()) {
+            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"LocalDB is not open, will remain closed");
+            status = STATUS.CLOSED;
+            return;
+        }
+
+        if (!pwmApplication.getConfig().readSettingAsBoolean(PwmSetting.REPORTING_ENABLE)) {
+            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"reporting module is not enabled, will remain closed");
+            status = STATUS.CLOSED;
+            clear();
+            return;
+        }
+
+        try {
+            userCacheService = new UserCacheService();
+            userCacheService.init(pwmApplication);
+        } catch (Exception e) {
+            LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL,"unable to init cache service");
+            status = STATUS.CLOSED;
+            return;
+        }
+
+        settings = ReportSettings.readSettingsFromConfig(pwmApplication.getConfig());
+        summaryData = ReportSummaryData.newSummaryData(settings.getTrackDays());
+
+        executorService = Executors.newSingleThreadScheduledExecutor(
+                Helper.makePwmThreadFactory(
+                        Helper.makeThreadName(pwmApplication,this.getClass()) + "-",
+                        true
+                ));
+
+
+        String startupMsg = "report service started";
+        LOGGER.debug(startupMsg);
+
+        executorService.submit(new InitializationTask());
+
+        status = STATUS.OPEN;
+    }
+
+    @Override
+    public void close()
+    {
+        status = STATUS.CLOSED;
+        saveTempData();
+        pwmApplication.writeAppAttribute(PwmApplication.AppAttribute.REPORT_CLEAN_FLAG, "true");
+        if (userCacheService != null) {
+            userCacheService.close();
+        }
+        if (executorService != null) {
+            executorService.shutdown();
+        }
+        executorService = null;
+    }
+
+    private void saveTempData() {
+        try {
+            final String jsonInfo = JsonUtil.serialize(reportStatus);
+            pwmApplication.writeAppAttribute(PwmApplication.AppAttribute.REPORT_STATUS,jsonInfo);
+        } catch (Exception e) {
+            LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL,"error writing cached report dredge info into memory: " + e.getMessage());
+        }
+    }
+
+    private void initTempData()
+            throws LocalDBException, PwmUnrecoverableException
+    {
+        final String cleanFlag = pwmApplication.readAppAttribute(PwmApplication.AppAttribute.REPORT_CLEAN_FLAG);
+        if (!"true".equals(cleanFlag)) {
+            LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL, "did not shut down cleanly");
+            reportStatus = new ReportStatusInfo(settings.getSettingsHash());
+            reportStatus.setTotal(userCacheService.size());
+        } else {
+            try {
+                final String jsonInfo = pwmApplication.readAppAttribute(PwmApplication.AppAttribute.REPORT_STATUS);
+                if (jsonInfo != null && !jsonInfo.isEmpty()) {
+                    reportStatus = JsonUtil.deserialize(jsonInfo,ReportStatusInfo.class);
+                }
+            } catch (Exception e) {
+                LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL,"error loading cached report status info into memory: " + e.getMessage());
+            }
+        }
+
+        reportStatus = reportStatus == null ? new ReportStatusInfo(settings.getSettingsHash()) : reportStatus; //safety
+
+        final String currentSettingCache = settings.getSettingsHash();
+        if (reportStatus.getSettingsHash() != null && !reportStatus.getSettingsHash().equals(currentSettingCache)) {
+            LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL,"configuration has changed, will clear cached report data");
+            clear();
+        }
+
+        reportStatus.setInProgress(false);
+
+        pwmApplication.writeAppAttribute(PwmApplication.AppAttribute.REPORT_CLEAN_FLAG, "false");
+    }
+
+    @Override
+    public List<HealthRecord> healthCheck()
+    {
+        return null;
+    }
+
+    @Override
+    public ServiceInfo serviceInfo()
+    {
+        return new ServiceInfo(Collections.singletonList(DataStorageMethod.LDAP));
+    }
+
+    public void scheduleImmediateUpdate() {
+        if (!reportStatus.isInProgress()) {
+            executorService.submit(new DredgeTask());
+            LOGGER.trace(PwmConstants.REPORTING_SESSION_LABEL,"submitted new ldap dredge task to executorService");
+        }
+    }
+
+    public void cancelUpdate() {
+        cancelFlag = true;
+    }
+
+    private void updateCacheFromLdap()
+            throws ChaiUnavailableException, ChaiOperationException, PwmOperationalException, PwmUnrecoverableException
+    {
+        LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL, "beginning process to updating user cache records from ldap");
+        if (status != STATUS.OPEN) {
+            return;
+        }
+        cancelFlag = false;
+        reportStatus = new ReportStatusInfo(settings.getSettingsHash());
+        reportStatus.setInProgress(true);
+        reportStatus.setStartDate(new Date());
+        try {
+            final Queue<UserIdentity> allUsers = new LinkedList<>(getListOfUsers());
+            reportStatus.setTotal(allUsers.size());
+            while (status == STATUS.OPEN && !allUsers.isEmpty() && !cancelFlag) {
+                final Date startUpdateTime = new Date();
+                final UserIdentity userIdentity = allUsers.poll();
+                try {
+                    if (updateCachedRecordFromLdap(userIdentity)) {
+                        reportStatus.setUpdated(reportStatus.getUpdated() + 1);
+                    }
+                } catch (Exception e) {
+                    String errorMsg = "error while updating report cache for " + userIdentity.toString() + ", cause: ";
+                    errorMsg += e instanceof PwmException ? ((PwmException) e).getErrorInformation().toDebugStr() : e.getMessage();
+                    final ErrorInformation errorInformation;
+                    errorInformation = new ErrorInformation(PwmError.ERROR_REPORTING_ERROR,errorMsg);
+                    LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL,errorInformation.toDebugStr());
+                    reportStatus.setLastError(errorInformation);
+                    reportStatus.setErrors(reportStatus.getErrors() + 1);
+                }
+                reportStatus.setCount(reportStatus.getCount() + 1);
+                reportStatus.getEventRateMeter().markEvents(1);
+                final TimeDuration totalUpdateTime = TimeDuration.fromCurrent(startUpdateTime);
+                if (settings.isAutoCalcRest()) {
+                    avgTracker.addSample(totalUpdateTime.getTotalMilliseconds());
+                    Helper.pause(avgTracker.avgAsLong());
+                } else {
+                    Helper.pause(settings.getRestTime().getTotalMilliseconds());
+                }
+            }
+            if (cancelFlag) {
+                reportStatus.setLastError(new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,"report cancelled by operator"));
+            }
+        } finally {
+            reportStatus.setFinishDate(new Date());
+            reportStatus.setInProgress(false);
+        }
+        LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"update user cache process completed: " + JsonUtil.serialize(reportStatus));
+    }
+
+    private void updateRestingCacheData() {
+        final long startTime = System.currentTimeMillis();
+        int examinedRecords = 0;
+        ClosableIterator<UserCacheRecord> iterator = null;
+        try {
+            LOGGER.trace(PwmConstants.REPORTING_SESSION_LABEL, "checking size of stored cache records");
+            final int totalRecords = userCacheService.size();
+            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL, "beginning cache review process of " + totalRecords + " records");
+            iterator = iterator();
+            Date lastLogOutputTime = new Date();
+            while (iterator.hasNext() && status == STATUS.OPEN) {
+                final UserCacheRecord record = iterator.next(); // (purge routine is embedded in next();
+
+                if (summaryData != null && record != null) {
+                    summaryData.update(record);
+                }
+
+                examinedRecords++;
+
+                if (TimeDuration.fromCurrent(lastLogOutputTime).isLongerThan(30, TimeUnit.SECONDS)) {
+                    final TimeDuration progressDuration = TimeDuration.fromCurrent(startTime);
+                    LOGGER.trace(PwmConstants.REPORTING_SESSION_LABEL,"cache review process in progress, examined "
+                            + examinedRecords + " records in " + progressDuration.asCompactString());
+                    lastLogOutputTime = new Date();
+                }
+            }
+            final TimeDuration totalTime = TimeDuration.fromCurrent(startTime);
+            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,
+                    "completed cache review process of " + examinedRecords + " cached report records in " + totalTime.asCompactString());
+        } finally {
+            if (iterator != null) {
+                iterator.close();
+            }
+        }
+    }
+
+    public boolean updateCachedRecordFromLdap(final UserInfoBean uiBean)
+            throws LocalDBException, PwmUnrecoverableException, ChaiUnavailableException
+    {
+        if (status != STATUS.OPEN) {
+            return false;
+        }
+
+        final UserCacheService.StorageKey storageKey = UserCacheService.StorageKey.fromUserInfoBean(uiBean);
+        return updateCachedRecordFromLdap(uiBean.getUserIdentity(), uiBean, storageKey);
+    }
+
+    private boolean updateCachedRecordFromLdap(final UserIdentity userIdentity)
+            throws ChaiUnavailableException, PwmUnrecoverableException, LocalDBException
+    {
+        if (status != STATUS.OPEN) {
+            return false;
+        }
+
+        final UserCacheService.StorageKey storageKey = UserCacheService.StorageKey.fromUserIdentity(pwmApplication,
+                userIdentity);
+        return updateCachedRecordFromLdap(userIdentity, null, storageKey);
+    }
+
+    private boolean updateCachedRecordFromLdap(
+            final UserIdentity userIdentity,
+            final UserInfoBean userInfoBean,
+            final UserCacheService.StorageKey storageKey
+    )
+            throws ChaiUnavailableException, PwmUnrecoverableException, LocalDBException
+    {
+        final UserCacheRecord userCacheRecord = userCacheService.readStorageKey(storageKey);
+        TimeDuration cacheAge = null;
+        if (userCacheRecord != null && userCacheRecord.getCacheTimestamp() != null) {
+            cacheAge = TimeDuration.fromCurrent(userCacheRecord.getCacheTimestamp());
+        }
+
+        boolean updateCache = false;
+        if (userInfoBean != null) {
+            updateCache = true;
+        } else {
+            if (cacheAge == null) {
+                LOGGER.trace(PwmConstants.REPORTING_SESSION_LABEL,"stored cache for " + userIdentity + " is missing cache storage timestamp, will update");
+                updateCache = true;
+            } else if (cacheAge.isLongerThan(settings.getMinCacheAge())) {
+                LOGGER.trace(PwmConstants.REPORTING_SESSION_LABEL,"stored cache for " + userIdentity + " is " + cacheAge.asCompactString() + " old, will update");
+                updateCache = true;
+            }
+        }
+
+        if (updateCache) {
+            if (userCacheRecord != null) {
+                if (summaryData != null && summaryData.getEpoch() != null && summaryData.getEpoch().equals(userCacheRecord.getSummaryEpoch())) {
+                    summaryData.remove(userCacheRecord);
+                }
+            }
+            final UserInfoBean newUserBean;
+            if (userInfoBean != null) {
+                newUserBean = userInfoBean;
+            } else {
+                newUserBean = new UserInfoBean();
+                final UserStatusReader.Settings readerSettings = new UserStatusReader.Settings();
+                readerSettings.setSkipReportUpdate(true);
+                final ChaiProvider chaiProvider = pwmApplication.getProxyChaiProvider(userIdentity.getLdapProfileID());
+                final UserStatusReader userStatusReader = new UserStatusReader(pwmApplication,PwmConstants.REPORTING_SESSION_LABEL,readerSettings);
+                userStatusReader.populateUserInfoBean(
+                        newUserBean,
+                        PwmConstants.DEFAULT_LOCALE,
+                        userIdentity,
+                        chaiProvider
+                );
+            }
+            final UserCacheRecord newUserCacheRecord = userCacheService.updateUserCache(newUserBean);
+
+            if (summaryData != null && summaryData.getEpoch() != null && newUserCacheRecord != null) {
+                if (!summaryData.getEpoch().equals(newUserCacheRecord.getSummaryEpoch())) {
+                    newUserCacheRecord.setSummaryEpoch(summaryData.getEpoch());
+                    userCacheService.store(newUserCacheRecord);
+                }
+                summaryData.update(newUserCacheRecord);
+            }
+        }
+
+        return updateCache;
+    }
+
+
+    public ReportStatusInfo getReportStatusInfo()
+    {
+        return reportStatus;
+    }
+
+    private List<UserIdentity> getListOfUsers()
+            throws ChaiUnavailableException, ChaiOperationException, PwmUnrecoverableException, PwmOperationalException
+    {
+        return readAllUsersFromLdap(pwmApplication, settings.getSearchFilter(), settings.getMaxSearchSize());
+    }
+
+    private static List<UserIdentity> readAllUsersFromLdap(
+            final PwmApplication pwmApplication,
+            final String searchFilter,
+            final int maxResults
+    )
+            throws ChaiUnavailableException, ChaiOperationException, PwmUnrecoverableException, PwmOperationalException
+    {
+        final UserSearchEngine userSearchEngine = new UserSearchEngine(pwmApplication,null);
+        final UserSearchEngine.SearchConfiguration searchConfiguration = new UserSearchEngine.SearchConfiguration();
+        searchConfiguration.setEnableValueEscaping(false);
+        searchConfiguration.setSearchTimeout(Long.parseLong(pwmApplication.getConfig().readAppProperty(AppProperty.REPORTING_LDAP_SEARCH_TIMEOUT)));
+
+        if (searchFilter == null) {
+            searchConfiguration.setUsername("*");
+        } else {
+            searchConfiguration.setFilter(searchFilter);
+        }
+
+        LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"beginning UserReportService user search using parameters: " + (JsonUtil.serialize(searchConfiguration)));
+
+        final Map<UserIdentity,Map<String,String>> searchResults = userSearchEngine.performMultiUserSearch(searchConfiguration, maxResults, Collections.<String>emptyList());
+        LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"user search found " + searchResults.size() + " users for reporting");
+        final List<UserIdentity> returnList = new ArrayList<>(searchResults.keySet());
+        Collections.shuffle(returnList);
+        return returnList;
+    }
+
+    public RecordIterator<UserCacheRecord> iterator() {
+        return new RecordIterator<>(userCacheService.<UserCacheService.StorageKey>iterator());
+    }
+
+    public class RecordIterator<K> implements ClosableIterator<UserCacheRecord> {
+
+        private UserCacheService.UserStatusCacheBeanIterator<UserCacheService.StorageKey> storageKeyIterator;
+
+        public RecordIterator(UserCacheService.UserStatusCacheBeanIterator<UserCacheService.StorageKey> storageKeyIterator) {
+            this.storageKeyIterator = storageKeyIterator;
+        }
+
+        public boolean hasNext() {
+            return this.storageKeyIterator.hasNext();
+        }
+
+        public UserCacheRecord next()
+        {
+            try {
+                UserCacheRecord returnBean = null;
+                while (returnBean == null && this.storageKeyIterator.hasNext()) {
+                    UserCacheService.StorageKey key = this.storageKeyIterator.next();
+                    returnBean = userCacheService.readStorageKey(key);
+                    if (returnBean != null) {
+                        if (returnBean.getCacheTimestamp() == null) {
+                            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"purging record due to missing cache timestamp: " + JsonUtil.serialize(returnBean));
+                            userCacheService.removeStorageKey(key);
+                        } else if (TimeDuration.fromCurrent(returnBean.getCacheTimestamp()).isLongerThan(settings.getMaxCacheAge())) {
+                            LOGGER.debug(PwmConstants.REPORTING_SESSION_LABEL,"purging record due to old age timestamp: " + JsonUtil.serialize(returnBean));
+                            userCacheService.removeStorageKey(key);
+                        } else {
+                            return returnBean;
+                        }
+                    }
+
+                }
+            } catch (LocalDBException e) {
+                throw new IllegalStateException("unexpected iterator traversal error while reading LocalDB: " + e.getMessage());
+            }
+            return null;
+        }
+
+        public void remove()
+        {
+
+        }
+
+        public void close() {
+            storageKeyIterator.close();
+        }
+    }
+
+    public void outputSummaryToCsv(final OutputStream outputStream, final Locale locale)
+            throws IOException
+    {
+        final List<ReportSummaryData.PresentationRow> outputList = summaryData.asPresentableCollection(pwmApplication.getConfig(),locale);
+        final CSVPrinter csvPrinter = Helper.makeCsvPrinter(outputStream);
+
+        for (final ReportSummaryData.PresentationRow presentationRow : outputList) {
+            final List<String> headerRow = new ArrayList<>();
+            headerRow.add(presentationRow.getLabel());
+            headerRow.add(presentationRow.getCount());
+            headerRow.add(presentationRow.getPct());
+            csvPrinter.printRecord(headerRow);
+        }
+
+        csvPrinter.close();
+    }
+
+    public void outputToCsv(final OutputStream outputStream, final boolean includeHeader, final Locale locale)
+            throws IOException, ChaiUnavailableException, ChaiOperationException, PwmUnrecoverableException, PwmOperationalException
+    {
+        final CSVPrinter csvPrinter = Helper.makeCsvPrinter(outputStream);
+        final Configuration config = pwmApplication.getConfig();
+        final Class localeClass = password.pwm.i18n.Admin.class;
+        if (includeHeader) {
+            final List<String> headerRow = new ArrayList<>();
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_UserDN", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_LDAP_Profile", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_Username", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_Email", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_UserGuid", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_LastLogin", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_PwdExpireTime", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_PwdChangeTime", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_ResponseSaveTime", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_HasResponses", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_HasHelpdeskResponses", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_ResponseStorageMethod", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_PwdExpired", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_PwdPreExpired", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_PwdViolatesPolicy", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_PwdWarnPeriod", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_RequiresPasswordUpdate", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_RequiresResponseUpdate", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_RequiresProfileUpdate", config, localeClass));
+            headerRow.add(LocaleHelper.getLocalizedMessage(locale, "Field_Report_RecordCacheTime", config, localeClass));
+            csvPrinter.printRecord(headerRow);
+        }
+
+        ClosableIterator<UserCacheRecord> cacheBeanIterator = null;
+        try {
+            cacheBeanIterator = this.iterator();
+            while (cacheBeanIterator.hasNext()) {
+                final UserCacheRecord userCacheRecord = cacheBeanIterator.next();
+                outputRecordRow(config, locale, userCacheRecord, csvPrinter);
+            }
+        } finally {
+            if (cacheBeanIterator != null) {
+                cacheBeanIterator.close();
+            }
+        }
+
+        csvPrinter.flush();
+    }
+
+    private void outputRecordRow(
+            final Configuration config,
+            final Locale locale,
+            final UserCacheRecord userCacheRecord,
+            final CSVPrinter csvPrinter
+    )
+            throws IOException
+    {
+        final String trueField = Display.getLocalizedMessage(locale, Display.Value_True, config);
+        final String falseField = Display.getLocalizedMessage(locale, Display.Value_False, config);
+        final String naField = Display.getLocalizedMessage(locale, Display.Value_NotApplicable, config);
+        final List<String> csvRow = new ArrayList<>();
+        csvRow.add(userCacheRecord.getUserDN());
+        csvRow.add(userCacheRecord.getLdapProfile());
+        csvRow.add(userCacheRecord.getUsername());
+        csvRow.add(userCacheRecord.getEmail());
+        csvRow.add(userCacheRecord.getUserGUID());
+        csvRow.add(userCacheRecord.getLastLoginTime() == null ? naField : PwmConstants.DEFAULT_DATETIME_FORMAT.format(
+                userCacheRecord.getLastLoginTime()));
+        csvRow.add(userCacheRecord.getPasswordExpirationTime() == null ? naField : PwmConstants.DEFAULT_DATETIME_FORMAT.format(
+                userCacheRecord.getPasswordExpirationTime()));
+        csvRow.add(userCacheRecord.getPasswordChangeTime() == null ? naField : PwmConstants.DEFAULT_DATETIME_FORMAT.format(
+                userCacheRecord.getPasswordChangeTime()));
+        csvRow.add(userCacheRecord.getResponseSetTime() == null ? naField : PwmConstants.DEFAULT_DATETIME_FORMAT.format(
+                userCacheRecord.getResponseSetTime()));
+        csvRow.add(userCacheRecord.isHasResponses() ? trueField : falseField);
+        csvRow.add(userCacheRecord.isHasHelpdeskResponses() ? trueField : falseField);
+        csvRow.add(userCacheRecord.getResponseStorageMethod() == null ? naField : userCacheRecord.getResponseStorageMethod().toString());
+        csvRow.add(userCacheRecord.getPasswordStatus().isExpired() ? trueField : falseField);
+        csvRow.add(userCacheRecord.getPasswordStatus().isPreExpired() ? trueField : falseField);
+        csvRow.add(userCacheRecord.getPasswordStatus().isViolatesPolicy() ? trueField : falseField);
+        csvRow.add(userCacheRecord.getPasswordStatus().isWarnPeriod() ? trueField : falseField);
+        csvRow.add(userCacheRecord.isRequiresPasswordUpdate() ? trueField : falseField);
+        csvRow.add(userCacheRecord.isRequiresResponseUpdate() ? trueField : falseField);
+        csvRow.add(userCacheRecord.isRequiresProfileUpdate() ? trueField : falseField);
+        csvRow.add(userCacheRecord.getCacheTimestamp() == null ? naField : PwmConstants.DEFAULT_DATETIME_FORMAT.format(
+                userCacheRecord.getCacheTimestamp()));
+
+        csvPrinter.printRecord(csvRow);
+    }
+
+    public ReportSummaryData getSummaryData() {
+        return summaryData;
+    }
+
+    public static class AvgTracker {
+        private final int maxSamples;
+        private final Queue<BigInteger> samples = new LinkedList<>();
+
+        public AvgTracker(int maxSamples)
+        {
+            this.maxSamples = maxSamples;
+        }
+
+        public void addSample(final long input) {
+            samples.add(new BigInteger(Long.toString(input)));
+            while (samples.size() > maxSamples) {
+                samples.remove();
+            }
+        }
+
+        public BigDecimal avg() {
+            if (samples.isEmpty()) {
+                throw new IllegalStateException("unable to compute avg without samples");
+            }
+
+            BigInteger total = BigInteger.ZERO;
+            for (final BigInteger sample : samples) {
+                total = total.add(sample);
+            }
+            final BigDecimal maxAsBD = new BigDecimal(Integer.toString(maxSamples));
+            return new BigDecimal(total).divide(maxAsBD, MathContext.DECIMAL32);
+        }
+
+        public long avgAsLong() {
+            return avg().longValue();
+        }
+    }
+
+    private class DredgeTask implements Runnable {
+        @Override
+        public void run()
+        {
+            reportStatus.setCurrentProcess(ReportStatusInfo.ReportEngineProcess.DredgeTask);
+            try {
+                updateCacheFromLdap();
+            } catch (Exception e) {
+                if (e instanceof PwmException) {
+                    if (((PwmException) e).getErrorInformation().getError() == PwmError.ERROR_DIRECTORY_UNAVAILABLE) {
+                        if (executorService != null) {
+                            LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL, "directory unavailable error during background DredgeTask, will retry; error: " + e.getMessage());
+                            executorService.schedule(new DredgeTask(), 10, TimeUnit.MINUTES);
+                        }
+                    } else {
+                        LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL, "error during background DredgeTask: " + e.getMessage());
+                    }
+                }
+            } finally {
+                reportStatus.setCurrentProcess(ReportStatusInfo.ReportEngineProcess.None);
+            }
+        }
+    }
+
+    private class RolloverTask implements Runnable {
+        @Override
+        public void run()
+        {
+            reportStatus.setCurrentProcess(ReportStatusInfo.ReportEngineProcess.RollOver);
+            try {
+                summaryData = ReportSummaryData.newSummaryData(settings.getTrackDays());
+                updateRestingCacheData();
+            } finally {
+                reportStatus.setCurrentProcess(ReportStatusInfo.ReportEngineProcess.None);
+            }
+        }
+    }
+
+    private class InitializationTask implements Runnable {
+        @Override
+        public void run() {
+            try {
+                initTempData();
+            } catch (LocalDBException | PwmUnrecoverableException e) {
+                LOGGER.error(PwmConstants.REPORTING_SESSION_LABEL, "error during initialization: " + e.getMessage());
+                status = STATUS.CLOSED;
+                return;
+            }
+            final long secondsUntilNextDredge = settings.getJobOffsetSeconds() + TimeDuration.fromCurrent(Helper.nextZuluZeroTime()).getTotalSeconds();
+            executorService.scheduleAtFixedRate(new DredgeTask(), secondsUntilNextDredge, TimeDuration.DAY.getTotalSeconds(), TimeUnit.SECONDS);
+            executorService.scheduleAtFixedRate(new RolloverTask(), secondsUntilNextDredge + 1, TimeDuration.DAY.getTotalSeconds(), TimeUnit.SECONDS);
+            executorService.submit(new RolloverTask());
+        }
+    }
+}

pwm/servlet/src/password/pwm/util/report/ReportSettings.java → pwm/servlet/src/password/pwm/svc/report/ReportSettings.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.report;
+package password.pwm.svc.report;
 
 import password.pwm.PwmConstants;
 import password.pwm.config.Configuration;

pwm/servlet/src/password/pwm/util/report/ReportStatusInfo.java → pwm/servlet/src/password/pwm/svc/report/ReportStatusInfo.java

@@ -20,11 +20,11 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.report;
+package password.pwm.svc.report;
 
 import password.pwm.error.ErrorInformation;
+import password.pwm.svc.stats.EventRateMeter;
 import password.pwm.util.TimeDuration;
-import password.pwm.util.stats.EventRateMeter;
 
 import java.io.Serializable;
 import java.util.Date;

pwm/servlet/src/password/pwm/util/report/ReportSummaryData.java → pwm/servlet/src/password/pwm/svc/report/ReportSummaryData.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.report;
+package password.pwm.svc.report;
 
 import com.novell.ldapchai.cr.Answer;
 import password.pwm.config.Configuration;

pwm/servlet/src/password/pwm/util/report/UserCacheRecord.java → pwm/servlet/src/password/pwm/svc/report/UserCacheRecord.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.report;
+package password.pwm.svc.report;
 
 import com.novell.ldapchai.cr.Answer;
 import password.pwm.bean.PasswordStatus;

pwm/servlet/src/password/pwm/util/report/UserCacheService.java → pwm/servlet/src/password/pwm/svc/report/UserCacheService.java

@@ -20,12 +20,11 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util.report;
+package password.pwm.svc.report;
 
 import com.google.gson.JsonSyntaxException;
 import com.novell.ldapchai.exception.ChaiUnavailableException;
 import password.pwm.PwmApplication;
-import password.pwm.PwmService;
 import password.pwm.bean.UserIdentity;
 import password.pwm.bean.UserInfoBean;
 import password.pwm.config.option.DataStorageMethod;
@@ -33,6 +32,7 @@ import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
 import password.pwm.ldap.LdapOperationsHelper;
+import password.pwm.svc.PwmService;
 import password.pwm.util.ClosableIterator;
 import password.pwm.util.JsonUtil;
 import password.pwm.util.localdb.LocalDB;

pwm/servlet/src/password/pwm/svc/sessiontrack/SessionTrackService.java

@@ -23,13 +23,13 @@
 package password.pwm.svc.sessiontrack;
 
 import password.pwm.PwmApplication;
-import password.pwm.PwmService;
 import password.pwm.bean.SessionStateBean;
 import password.pwm.bean.SessionStateInfoBean;
 import password.pwm.bean.UserInfoBean;
 import password.pwm.error.PwmException;
 import password.pwm.health.HealthRecord;
 import password.pwm.http.PwmSession;
+import password.pwm.svc.PwmService;
 import password.pwm.util.logging.PwmLogger;
 
 import java.util.*;

pwm/servlet/src/password/pwm/util/AbstractUrlShortener.java → pwm/servlet/src/password/pwm/svc/shorturl/AbstractUrlShortener.java

@@ -1,43 +1,43 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.util;
-
-import password.pwm.PwmApplication;
-import password.pwm.error.PwmUnrecoverableException;
-
-import java.util.Properties;
-
-public interface AbstractUrlShortener {
-	Properties configuration = null;
-
-	/**
-	 * {@link URL}. shorten
-	 * This method should be implemented to read a short replacement
-	 * URL for the input URL.
-	 *
-	 * @param input			the URL to be shortened
-	 *
-	 * @param context		the PwmApplication, used to retrieve configuration
-	 */
-	public String shorten(String input, PwmApplication context) throws PwmUnrecoverableException;
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.shorturl;
+
+import password.pwm.PwmApplication;
+import password.pwm.error.PwmUnrecoverableException;
+
+import java.util.Properties;
+
+public interface AbstractUrlShortener {
+	Properties configuration = null;
+
+	/**
+	 * {@link URL}. shorten
+	 * This method should be implemented to read a short replacement
+	 * URL for the input URL.
+	 *
+	 * @param input			the URL to be shortened
+	 *
+	 * @param context		the PwmApplication, used to retrieve configuration
+	 */
+	String shorten(String input, PwmApplication context) throws PwmUnrecoverableException;
 }

pwm/servlet/src/password/pwm/util/BasicUrlShortener.java → pwm/servlet/src/password/pwm/svc/shorturl/BasicUrlShortener.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util;
+package password.pwm.svc.shorturl;
 
 import password.pwm.PwmApplication;
 import password.pwm.error.PwmUnrecoverableException;

pwm/servlet/src/password/pwm/util/TinyUrlShortener.java → pwm/servlet/src/password/pwm/svc/shorturl/TinyUrlShortener.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util;
+package password.pwm.svc.shorturl;
  
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
@@ -29,6 +29,7 @@ import org.apache.http.util.EntityUtils;
 import password.pwm.PwmApplication;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.http.client.PwmHttpClient;
+import password.pwm.util.StringUtil;
 import password.pwm.util.logging.PwmLogger;
 
 import java.util.Properties;

pwm/servlet/src/password/pwm/util/UrlShortenerService.java → pwm/servlet/src/password/pwm/svc/shorturl/UrlShortenerService.java

@@ -20,16 +20,16 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.util;
+package password.pwm.svc.shorturl;
 
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
-import password.pwm.PwmService;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.option.DataStorageMethod;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
+import password.pwm.svc.PwmService;
 import password.pwm.util.logging.PwmLogger;
 
 import java.util.Arrays;

pwm/servlet/src/password/pwm/util/stats/EventRateMeter.java → pwm/servlet/src/password/pwm/svc/stats/EventRateMeter.java

@@ -1,119 +1,119 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.util.stats;
-
-import password.pwm.util.TimeDuration;
-
-import java.io.Serializable;
-import java.math.BigDecimal;
-
-public class EventRateMeter implements Serializable {
-    private MovingAverage movingAverage;
-    private double remainder = 0;
-
-    public EventRateMeter(final TimeDuration maxDuration) {
-        if (maxDuration == null) {
-            throw new NullPointerException("maxDuration cannot be null");
-        }
-        movingAverage = new MovingAverage(maxDuration.getTotalMilliseconds());
-    }
-
-    public synchronized void markEvents(final int eventCount) {
-        long timeSinceLastUpdate = System.currentTimeMillis() - movingAverage.getLastMillis();
-        if (timeSinceLastUpdate != 0) {
-            double eventRate  = (eventCount + remainder) / timeSinceLastUpdate;
-            movingAverage.update(eventRate * 1000);
-            remainder = 0;
-        } else {
-            remainder += eventCount;
-        }
-    }
-
-    public BigDecimal readEventRate() {
-        return new BigDecimal(this.movingAverage.getAverage());
-    }
-
-    /** MovingAverage.java
-     *
-     * Copyright 2009-2010 Comcast Interactive Media, LLC.
-     *
-     * Licensed under the Apache License, Version 2.0 (the "License");
-     * you may not use this file except in compliance with the License.
-     * You may obtain a copy of the License at
-     *
-     *      http://www.apache.org/licenses/LICENSE-2.0
-     *
-     * Unless required by applicable law or agreed to in writing, software
-     * distributed under the License is distributed on an "AS IS" BASIS,
-     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-     * See the License for the specific language governing permissions and
-     * limitations under the License.
-     *
-     *
-     *  This class implements an exponential moving average, using the
-     *  algorithm described at <a href="http://en.wikipedia.org/wiki/Moving_average">http://en.wikipedia.org/wiki/Moving_average</a>. The average does not
-     *  sample itself; it merely computes the new average when updated with
-     *  a sample by an external mechanism.
-     **/
-    public static class MovingAverage implements Serializable {
-        private long windowMillis;
-        private long lastMillis;
-        private double average;
-
-        /** Construct a {@link MovingAverage}, providing the time window
-         *  we want the average over. For example, providing a value of
-         *  3,600,000 provides a moving average over the last hour.
-         *  @param windowMillis the length of the sliding window in
-         *    milliseconds */
-        public MovingAverage(long windowMillis) {
-            this.windowMillis = windowMillis;
-        }
-
-        /** Updates the average with the latest measurement.
-         *  @param sample the latest measurement in the rolling average */
-        public synchronized void update(double sample) {
-            long now = System.currentTimeMillis();
-
-            if (lastMillis == 0) {  // first sample
-                average = sample;
-                lastMillis = now;
-                return;
-            }
-            long deltaTime = now - lastMillis;
-            double coeff = Math.exp(-1.0 * ((double)deltaTime / windowMillis));
-            average = (1.0 - coeff) * sample + coeff * average;
-
-            lastMillis = now;
-        }
-
-        /** Returns the last computed average value. */
-        public double getAverage() {
-            update(0);
-            return average;
-        }
-
-        public long getLastMillis() {
-            return lastMillis;
-        }
-    }
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.stats;
+
+import password.pwm.util.TimeDuration;
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+
+public class EventRateMeter implements Serializable {
+    private MovingAverage movingAverage;
+    private double remainder = 0;
+
+    public EventRateMeter(final TimeDuration maxDuration) {
+        if (maxDuration == null) {
+            throw new NullPointerException("maxDuration cannot be null");
+        }
+        movingAverage = new MovingAverage(maxDuration.getTotalMilliseconds());
+    }
+
+    public synchronized void markEvents(final int eventCount) {
+        long timeSinceLastUpdate = System.currentTimeMillis() - movingAverage.getLastMillis();
+        if (timeSinceLastUpdate != 0) {
+            double eventRate  = (eventCount + remainder) / timeSinceLastUpdate;
+            movingAverage.update(eventRate * 1000);
+            remainder = 0;
+        } else {
+            remainder += eventCount;
+        }
+    }
+
+    public BigDecimal readEventRate() {
+        return new BigDecimal(this.movingAverage.getAverage());
+    }
+
+    /** MovingAverage.java
+     *
+     * Copyright 2009-2010 Comcast Interactive Media, LLC.
+     *
+     * Licensed under the Apache License, Version 2.0 (the "License");
+     * you may not use this file except in compliance with the License.
+     * You may obtain a copy of the License at
+     *
+     *      http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     *
+     *
+     *  This class implements an exponential moving average, using the
+     *  algorithm described at <a href="http://en.wikipedia.org/wiki/Moving_average">http://en.wikipedia.org/wiki/Moving_average</a>. The average does not
+     *  sample itself; it merely computes the new average when updated with
+     *  a sample by an external mechanism.
+     **/
+    public static class MovingAverage implements Serializable {
+        private long windowMillis;
+        private long lastMillis;
+        private double average;
+
+        /** Construct a {@link MovingAverage}, providing the time window
+         *  we want the average over. For example, providing a value of
+         *  3,600,000 provides a moving average over the last hour.
+         *  @param windowMillis the length of the sliding window in
+         *    milliseconds */
+        public MovingAverage(long windowMillis) {
+            this.windowMillis = windowMillis;
+        }
+
+        /** Updates the average with the latest measurement.
+         *  @param sample the latest measurement in the rolling average */
+        public synchronized void update(double sample) {
+            long now = System.currentTimeMillis();
+
+            if (lastMillis == 0) {  // first sample
+                average = sample;
+                lastMillis = now;
+                return;
+            }
+            long deltaTime = now - lastMillis;
+            double coeff = Math.exp(-1.0 * ((double)deltaTime / windowMillis));
+            average = (1.0 - coeff) * sample + coeff * average;
+
+            lastMillis = now;
+        }
+
+        /** Returns the last computed average value. */
+        public double getAverage() {
+            update(0);
+            return average;
+        }
+
+        public long getLastMillis() {
+            return lastMillis;
+        }
+    }
+}

pwm/servlet/src/password/pwm/util/stats/Statistic.java → pwm/servlet/src/password/pwm/svc/stats/Statistic.java

@@ -1,234 +1,234 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.util.stats;
-
-import password.pwm.PwmApplication;
-import password.pwm.config.PwmSetting;
-import password.pwm.i18n.Admin;
-import password.pwm.util.LocaleHelper;
-import password.pwm.util.TimeDuration;
-import password.pwm.util.logging.PwmLogger;
-
-import java.util.*;
-
-public enum Statistic {
-    AUTHENTICATIONS                     (Type.INCREMENTOR, "Authentications", null),
-    AUTHENTICATION_FAILURES             (Type.INCREMENTOR, "AuthenticationFailures", null),
-    AUTHENTICATION_EXPIRED              (Type.INCREMENTOR, "Authentications_Expired", null),
-    AUTHENTICATION_PRE_EXPIRED          (Type.INCREMENTOR, "Authentications_PreExpired", null),
-    AUTHENTICATION_EXPIRED_WARNING      (Type.INCREMENTOR, "Authentications_ExpiredWarning", null),
-    PWM_STARTUPS                        (Type.INCREMENTOR, "PWM_Startups", null),
-    PWM_UNKNOWN_ERRORS                  (Type.INCREMENTOR, "PWM_UnknownErrors", null),
-    PASSWORD_CHANGES                    (Type.INCREMENTOR, "PasswordChanges", null),
-    FORGOTTEN_USERNAME_FAILURES         (Type.INCREMENTOR, "ForgottenUsernameFailures", null),
-    FORGOTTEN_USERNAME_SUCCESSES        (Type.INCREMENTOR, "ForgottenUsernameSuccesses", null),
-    EMAIL_SEND_SUCCESSES                (Type.INCREMENTOR, "EmailSendSuccesses", null),
-    EMAIL_SEND_FAILURES                 (Type.INCREMENTOR, "EmailSendFailures", null),
-    EMAIL_SEND_DISCARDS                 (Type.INCREMENTOR, "EmailSendDiscards", null),
-    SMS_SEND_SUCCESSES                  (Type.INCREMENTOR, "SmsSendSuccesses", null),
-    SMS_SEND_FAILURES                   (Type.INCREMENTOR, "SmsSendFailures", null),
-    SMS_SEND_DISCARDS                   (Type.INCREMENTOR, "SmsSendDiscards", null),
-    PASSWORD_RULE_CHECKS                (Type.INCREMENTOR, "PasswordRuleChecks", null),
-    HTTP_REQUESTS                       (Type.INCREMENTOR, "HttpRequests", null),
-    HTTP_RESOURCE_REQUESTS              (Type.INCREMENTOR, "HttpResourceRequests", null),
-    HTTP_SESSIONS                       (Type.INCREMENTOR, "HttpSessions", null),
-    ACTIVATED_USERS                     (Type.INCREMENTOR, "ActivatedUsers", null),
-    NEW_USERS                           (Type.INCREMENTOR, "NewUsers", new ConfigSettingDetail(PwmSetting.NEWUSER_ENABLE)),
-    GUESTS                              (Type.INCREMENTOR, "Guests", new ConfigSettingDetail(PwmSetting.GUEST_ENABLE)),
-    UPDATED_GUESTS                      (Type.INCREMENTOR, "UpdatedGuests", new ConfigSettingDetail(PwmSetting.GUEST_ENABLE)),
-    LOCKED_USERS                        (Type.INCREMENTOR, "LockedUsers", null),
-    LOCKED_ADDRESSES                    (Type.INCREMENTOR, "LockedAddresses", null),
-    LOCKED_USERIDS                      (Type.INCREMENTOR, "LockedUserDNs", null),
-    LOCKED_ATTRIBUTES                   (Type.INCREMENTOR, "LockedAttributes", null),
-    LOCKED_TOKENDESTS                   (Type.INCREMENTOR, "LockedTokenDests", null),
-    CAPTCHA_SUCCESSES                   (Type.INCREMENTOR, "CaptchaSuccessess", null),
-    CAPTCHA_FAILURES                    (Type.INCREMENTOR, "CaptchaFailures", null),
-    CAPTCHA_PRESENTATIONS               (Type.INCREMENTOR, "CaptchaPresentations", null),
-    LDAP_UNAVAILABLE_COUNT              (Type.INCREMENTOR, "LdapUnavailableCount", null),
-    DB_UNAVAILABLE_COUNT                (Type.INCREMENTOR, "DatabaseUnavailableCount", null),
-    SETUP_RESPONSES                     (Type.INCREMENTOR, "SetupResponses", null),
-    SETUP_OTP_SECRET                    (Type.INCREMENTOR, "SetupOtpSecret", new ConfigSettingDetail(PwmSetting.OTP_ENABLED)),
-    UPDATE_ATTRIBUTES                   (Type.INCREMENTOR, "UpdateAttributes", new ConfigSettingDetail(PwmSetting.UPDATE_PROFILE_ENABLE)),
-    SHORTCUTS_SELECTED                  (Type.INCREMENTOR, "ShortcutsSelected", new ConfigSettingDetail(PwmSetting.SHORTCUT_ENABLE)),
-    GENERATED_PASSWORDS                 (Type.INCREMENTOR, "GeneratedPasswords", null),
-    RECOVERY_SUCCESSES                  (Type.INCREMENTOR, "RecoverySuccesses", null),
-    RECOVERY_FAILURES                   (Type.INCREMENTOR, "RecoveryFailures", null),
-    TOKENS_SENT                         (Type.INCREMENTOR, "TokensSent",null),
-    TOKENS_PASSSED                      (Type.INCREMENTOR, "TokensPassed",null),
-    RECOVERY_TOKENS_SENT                (Type.INCREMENTOR, "RecoveryTokensSent", null),
-    RECOVERY_TOKENS_PASSED              (Type.INCREMENTOR, "RecoveryTokensPassed", null),
-    RECOVERY_TOKENS_FAILED              (Type.INCREMENTOR, "RecoveryTokensFailed", null),
-    RECOVERY_OTP_PASSED                 (Type.INCREMENTOR, "RecoveryOTPPassed", new ConfigSettingDetail(PwmSetting.OTP_ENABLED)),
-    RECOVERY_OTP_FAILED                 (Type.INCREMENTOR, "RecoveryOTPFailed", new ConfigSettingDetail(PwmSetting.OTP_ENABLED)),
-    PEOPLESEARCH_CACHE_HITS             (Type.INCREMENTOR, "PeopleSearchCacheHits", new ConfigSettingDetail(PwmSetting.PEOPLE_SEARCH_ENABLE)),
-    PEOPLESEARCH_CACHE_MISSES           (Type.INCREMENTOR, "PeopleSearchCacheMisses", new ConfigSettingDetail(PwmSetting.PEOPLE_SEARCH_ENABLE)),
-    PEOPLESEARCH_SEARCHES               (Type.INCREMENTOR, "PeopleSearchSearches", new ConfigSettingDetail(PwmSetting.PEOPLE_SEARCH_ENABLE)),
-    PEOPLESEARCH_DETAILS                (Type.INCREMENTOR, "PeopleSearchDetails", new ConfigSettingDetail(PwmSetting.PEOPLE_SEARCH_ENABLE)),
-    PEOPLESEARCH_ORGCHART               (Type.INCREMENTOR, "PeopleSearchOrgChart", new ConfigSettingDetail(PwmSetting.PEOPLE_SEARCH_ENABLE)),
-    HELPDESK_PASSWORD_SET               (Type.INCREMENTOR, "HelpdeskPasswordSet", null),
-    HELPDESK_USER_LOOKUP                (Type.INCREMENTOR, "HelpdeskUserLookup", null),
-    HELPDESK_TOKENS_SENT                (Type.INCREMENTOR, "HelpdeskTokenSent", null),
-    HELPDESK_UNLOCK                     (Type.INCREMENTOR, "HelpdeskUnlock", null),
-    HELPDESK_VERIFY_OTP                 (Type.INCREMENTOR, "HelpdeskVerifyOTP", null),
-    REST_STATUS                         (Type.INCREMENTOR, "RestStatus", null),
-    REST_CHECKPASSWORD                  (Type.INCREMENTOR, "RestCheckPassword", null),
-    REST_SETPASSWORD                    (Type.INCREMENTOR, "RestSetPassword", null),
-    REST_RANDOMPASSWORD                 (Type.INCREMENTOR, "RestRandomPassword", null),
-    REST_CHALLENGES                     (Type.INCREMENTOR, "RestChallenges", null),
-    REST_HEALTH                         (Type.INCREMENTOR, "RestHealth", null),
-    REST_STATISTICS                     (Type.INCREMENTOR, "RestStatistics", null),
-    REST_VERIFYCHALLENGES               (Type.INCREMENTOR, "RestVerifyChallenges", null),
-    INTRUDER_ATTEMPTS                   (Type.INCREMENTOR, "IntruderAttempts", null),
-
-    AVG_PASSWORD_SYNC_TIME              (Type.AVERAGE, "AvgPasswordSyncTime", null),
-    AVG_AUTHENTICATION_TIME             (Type.AVERAGE, "AvgAuthenticationTime", null),
-    AVG_PASSWORD_STRENGTH               (Type.AVERAGE, "AvgPasswordStrength", null),
-    AVG_LDAP_SEARCH_TIME                (Type.AVERAGE, "AvgLdapSearchTime", null),
-
-    ;
-
-    private final static PwmLogger LOGGER = PwmLogger.forClass(Statistic.class);
-    private final Type type;
-    private final String key;
-    private final StatDetail statDetail;
-
-    Statistic(
-            final Type type,
-            final String key,
-            StatDetail statDetail
-    ) {
-        this.type = type;
-        this.key = key;
-        this.statDetail = statDetail;
-    }
-
-    public String getKey() {
-        return key;
-    }
-
-    public Type getType() {
-        return type;
-    }
-
-    public boolean isActive(final PwmApplication pwmApplication) {
-        if (statDetail == null) {
-            return true;
-        }
-        return statDetail.isActive(pwmApplication);
-    }
-
-    public static SortedSet<Statistic> sortedValues(final Locale locale) {
-        final Comparator<Statistic> comparator = new Comparator<Statistic>() {
-            public int compare(final Statistic o1, final Statistic o2) {
-                return o1.getLabel(locale).compareTo(o2.getLabel(locale));
-            }
-        };
-        final TreeSet<Statistic> set = new TreeSet<>(comparator);
-        set.addAll(Arrays.asList(values()));
-        return set;
-    }
-
-    public enum Type {
-        INCREMENTOR,
-        AVERAGE,
-    }
-
-    public String getLabel(final Locale locale) {
-        try {
-            final String keyName = "Statistic_Label." + this.getKey();
-            return LocaleHelper.getLocalizedMessage(locale, keyName, null, Admin.class);
-        } catch (MissingResourceException e) {
-            return "MISSING STATISTIC LABEL for " + this.getKey();
-        }
-    }
-
-    public String getDescription(final Locale locale) {
-        final String keyName = "Statistic_Description." + this.getKey();
-        try {
-            return LocaleHelper.getLocalizedMessage(locale, keyName, null, Admin.class);
-        } catch (Exception e) {
-            LOGGER.error("unable to load localization for " + keyName + ", error: " + e.getMessage());
-            return "missing localization for " + keyName;
-        }
-    }
-
-    public enum EpsType {
-        PASSWORD_CHANGES(Statistic.PASSWORD_CHANGES),
-        AUTHENTICATION(Statistic.AUTHENTICATIONS),
-        INTRUDER_ATTEMPTS(Statistic.INTRUDER_ATTEMPTS),
-        PWMDB_WRITES(null),
-        PWMDB_READS(null),
-        DB_WRITES(null),
-        DB_READS(null),
-        ;
-
-        private Statistic relatedStatistic;
-
-        private EpsType(Statistic relatedStatistic) {
-            this.relatedStatistic = relatedStatistic;
-        }
-
-        public Statistic getRelatedStatistic() {
-            return relatedStatistic;
-        }
-
-        public String getLabel(final Locale locale) {
-            final String keyName = "Statistic_Label." + EpsType.class.getSimpleName() + "_" + this.name();
-            return LocaleHelper.getLocalizedMessage(locale, keyName, null, Admin.class);
-        }
-    }
-
-    public enum EpsDuration {
-        MINUTE(TimeDuration.MINUTE),
-        HOUR(TimeDuration.HOUR),
-        DAY(TimeDuration.DAY),
-        ;
-
-        private final TimeDuration timeDuration;
-
-        private EpsDuration(TimeDuration timeDuration) {
-            this.timeDuration = timeDuration;
-        }
-
-        public TimeDuration getTimeDuration() {
-            return timeDuration;
-        }
-    }
-
-    interface StatDetail {
-        boolean isActive(PwmApplication pwmApplication);
-    }
-
-    static class ConfigSettingDetail implements StatDetail {
-        final private PwmSetting pwmSetting;
-
-        ConfigSettingDetail(PwmSetting pwmSetting)
-        {
-            this.pwmSetting = pwmSetting;
-        }
-
-        public boolean isActive(final PwmApplication pwmApplication) {
-            return pwmApplication.getConfig().readSettingAsBoolean(pwmSetting);
-        }
-    }
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.stats;
+
+import password.pwm.PwmApplication;
+import password.pwm.config.PwmSetting;
+import password.pwm.i18n.Admin;
+import password.pwm.util.LocaleHelper;
+import password.pwm.util.TimeDuration;
+import password.pwm.util.logging.PwmLogger;
+
+import java.util.*;
+
+public enum Statistic {
+    AUTHENTICATIONS                     (Type.INCREMENTOR, "Authentications", null),
+    AUTHENTICATION_FAILURES             (Type.INCREMENTOR, "AuthenticationFailures", null),
+    AUTHENTICATION_EXPIRED              (Type.INCREMENTOR, "Authentications_Expired", null),
+    AUTHENTICATION_PRE_EXPIRED          (Type.INCREMENTOR, "Authentications_PreExpired", null),
+    AUTHENTICATION_EXPIRED_WARNING      (Type.INCREMENTOR, "Authentications_ExpiredWarning", null),
+    PWM_STARTUPS                        (Type.INCREMENTOR, "PWM_Startups", null),
+    PWM_UNKNOWN_ERRORS                  (Type.INCREMENTOR, "PWM_UnknownErrors", null),
+    PASSWORD_CHANGES                    (Type.INCREMENTOR, "PasswordChanges", null),
+    FORGOTTEN_USERNAME_FAILURES         (Type.INCREMENTOR, "ForgottenUsernameFailures", null),
+    FORGOTTEN_USERNAME_SUCCESSES        (Type.INCREMENTOR, "ForgottenUsernameSuccesses", null),
+    EMAIL_SEND_SUCCESSES                (Type.INCREMENTOR, "EmailSendSuccesses", null),
+    EMAIL_SEND_FAILURES                 (Type.INCREMENTOR, "EmailSendFailures", null),
+    EMAIL_SEND_DISCARDS                 (Type.INCREMENTOR, "EmailSendDiscards", null),
+    SMS_SEND_SUCCESSES                  (Type.INCREMENTOR, "SmsSendSuccesses", null),
+    SMS_SEND_FAILURES                   (Type.INCREMENTOR, "SmsSendFailures", null),
+    SMS_SEND_DISCARDS                   (Type.INCREMENTOR, "SmsSendDiscards", null),
+    PASSWORD_RULE_CHECKS                (Type.INCREMENTOR, "PasswordRuleChecks", null),
+    HTTP_REQUESTS                       (Type.INCREMENTOR, "HttpRequests", null),
+    HTTP_RESOURCE_REQUESTS              (Type.INCREMENTOR, "HttpResourceRequests", null),
+    HTTP_SESSIONS                       (Type.INCREMENTOR, "HttpSessions", null),
+    ACTIVATED_USERS                     (Type.INCREMENTOR, "ActivatedUsers", null),
+    NEW_USERS                           (Type.INCREMENTOR, "NewUsers", new ConfigSettingDetail(PwmSetting.NEWUSER_ENABLE)),
+    GUESTS                              (Type.INCREMENTOR, "Guests", new ConfigSettingDetail(PwmSetting.GUEST_ENABLE)),
+    UPDATED_GUESTS                      (Type.INCREMENTOR, "UpdatedGuests", new ConfigSettingDetail(PwmSetting.GUEST_ENABLE)),
+    LOCKED_USERS                        (Type.INCREMENTOR, "LockedUsers", null),
+    LOCKED_ADDRESSES                    (Type.INCREMENTOR, "LockedAddresses", null),
+    LOCKED_USERIDS                      (Type.INCREMENTOR, "LockedUserDNs", null),
+    LOCKED_ATTRIBUTES                   (Type.INCREMENTOR, "LockedAttributes", null),
+    LOCKED_TOKENDESTS                   (Type.INCREMENTOR, "LockedTokenDests", null),
+    CAPTCHA_SUCCESSES                   (Type.INCREMENTOR, "CaptchaSuccessess", null),
+    CAPTCHA_FAILURES                    (Type.INCREMENTOR, "CaptchaFailures", null),
+    CAPTCHA_PRESENTATIONS               (Type.INCREMENTOR, "CaptchaPresentations", null),
+    LDAP_UNAVAILABLE_COUNT              (Type.INCREMENTOR, "LdapUnavailableCount", null),
+    DB_UNAVAILABLE_COUNT                (Type.INCREMENTOR, "DatabaseUnavailableCount", null),
+    SETUP_RESPONSES                     (Type.INCREMENTOR, "SetupResponses", null),
+    SETUP_OTP_SECRET                    (Type.INCREMENTOR, "SetupOtpSecret", new ConfigSettingDetail(PwmSetting.OTP_ENABLED)),
+    UPDATE_ATTRIBUTES                   (Type.INCREMENTOR, "UpdateAttributes", new ConfigSettingDetail(PwmSetting.UPDATE_PROFILE_ENABLE)),
+    SHORTCUTS_SELECTED                  (Type.INCREMENTOR, "ShortcutsSelected", new ConfigSettingDetail(PwmSetting.SHORTCUT_ENABLE)),
+    GENERATED_PASSWORDS                 (Type.INCREMENTOR, "GeneratedPasswords", null),
+    RECOVERY_SUCCESSES                  (Type.INCREMENTOR, "RecoverySuccesses", null),
+    RECOVERY_FAILURES                   (Type.INCREMENTOR, "RecoveryFailures", null),
+    TOKENS_SENT                         (Type.INCREMENTOR, "TokensSent",null),
+    TOKENS_PASSSED                      (Type.INCREMENTOR, "TokensPassed",null),
+    RECOVERY_TOKENS_SENT                (Type.INCREMENTOR, "RecoveryTokensSent", null),
+    RECOVERY_TOKENS_PASSED              (Type.INCREMENTOR, "RecoveryTokensPassed", null),
+    RECOVERY_TOKENS_FAILED              (Type.INCREMENTOR, "RecoveryTokensFailed", null),
+    RECOVERY_OTP_PASSED                 (Type.INCREMENTOR, "RecoveryOTPPassed", new ConfigSettingDetail(PwmSetting.OTP_ENABLED)),
+    RECOVERY_OTP_FAILED                 (Type.INCREMENTOR, "RecoveryOTPFailed", new ConfigSettingDetail(PwmSetting.OTP_ENABLED)),
+    PEOPLESEARCH_CACHE_HITS             (Type.INCREMENTOR, "PeopleSearchCacheHits", new ConfigSettingDetail(PwmSetting.PEOPLE_SEARCH_ENABLE)),
+    PEOPLESEARCH_CACHE_MISSES           (Type.INCREMENTOR, "PeopleSearchCacheMisses", new ConfigSettingDetail(PwmSetting.PEOPLE_SEARCH_ENABLE)),
+    PEOPLESEARCH_SEARCHES               (Type.INCREMENTOR, "PeopleSearchSearches", new ConfigSettingDetail(PwmSetting.PEOPLE_SEARCH_ENABLE)),
+    PEOPLESEARCH_DETAILS                (Type.INCREMENTOR, "PeopleSearchDetails", new ConfigSettingDetail(PwmSetting.PEOPLE_SEARCH_ENABLE)),
+    PEOPLESEARCH_ORGCHART               (Type.INCREMENTOR, "PeopleSearchOrgChart", new ConfigSettingDetail(PwmSetting.PEOPLE_SEARCH_ENABLE)),
+    HELPDESK_PASSWORD_SET               (Type.INCREMENTOR, "HelpdeskPasswordSet", null),
+    HELPDESK_USER_LOOKUP                (Type.INCREMENTOR, "HelpdeskUserLookup", null),
+    HELPDESK_TOKENS_SENT                (Type.INCREMENTOR, "HelpdeskTokenSent", null),
+    HELPDESK_UNLOCK                     (Type.INCREMENTOR, "HelpdeskUnlock", null),
+    HELPDESK_VERIFY_OTP                 (Type.INCREMENTOR, "HelpdeskVerifyOTP", null),
+    REST_STATUS                         (Type.INCREMENTOR, "RestStatus", null),
+    REST_CHECKPASSWORD                  (Type.INCREMENTOR, "RestCheckPassword", null),
+    REST_SETPASSWORD                    (Type.INCREMENTOR, "RestSetPassword", null),
+    REST_RANDOMPASSWORD                 (Type.INCREMENTOR, "RestRandomPassword", null),
+    REST_CHALLENGES                     (Type.INCREMENTOR, "RestChallenges", null),
+    REST_HEALTH                         (Type.INCREMENTOR, "RestHealth", null),
+    REST_STATISTICS                     (Type.INCREMENTOR, "RestStatistics", null),
+    REST_VERIFYCHALLENGES               (Type.INCREMENTOR, "RestVerifyChallenges", null),
+    INTRUDER_ATTEMPTS                   (Type.INCREMENTOR, "IntruderAttempts", null),
+
+    AVG_PASSWORD_SYNC_TIME              (Type.AVERAGE, "AvgPasswordSyncTime", null),
+    AVG_AUTHENTICATION_TIME             (Type.AVERAGE, "AvgAuthenticationTime", null),
+    AVG_PASSWORD_STRENGTH               (Type.AVERAGE, "AvgPasswordStrength", null),
+    AVG_LDAP_SEARCH_TIME                (Type.AVERAGE, "AvgLdapSearchTime", null),
+
+    ;
+
+    private final static PwmLogger LOGGER = PwmLogger.forClass(Statistic.class);
+    private final Type type;
+    private final String key;
+    private final StatDetail statDetail;
+
+    Statistic(
+            final Type type,
+            final String key,
+            StatDetail statDetail
+    ) {
+        this.type = type;
+        this.key = key;
+        this.statDetail = statDetail;
+    }
+
+    public String getKey() {
+        return key;
+    }
+
+    public Type getType() {
+        return type;
+    }
+
+    public boolean isActive(final PwmApplication pwmApplication) {
+        if (statDetail == null) {
+            return true;
+        }
+        return statDetail.isActive(pwmApplication);
+    }
+
+    public static SortedSet<Statistic> sortedValues(final Locale locale) {
+        final Comparator<Statistic> comparator = new Comparator<Statistic>() {
+            public int compare(final Statistic o1, final Statistic o2) {
+                return o1.getLabel(locale).compareTo(o2.getLabel(locale));
+            }
+        };
+        final TreeSet<Statistic> set = new TreeSet<>(comparator);
+        set.addAll(Arrays.asList(values()));
+        return set;
+    }
+
+    public enum Type {
+        INCREMENTOR,
+        AVERAGE,
+    }
+
+    public String getLabel(final Locale locale) {
+        try {
+            final String keyName = "Statistic_Label." + this.getKey();
+            return LocaleHelper.getLocalizedMessage(locale, keyName, null, Admin.class);
+        } catch (MissingResourceException e) {
+            return "MISSING STATISTIC LABEL for " + this.getKey();
+        }
+    }
+
+    public String getDescription(final Locale locale) {
+        final String keyName = "Statistic_Description." + this.getKey();
+        try {
+            return LocaleHelper.getLocalizedMessage(locale, keyName, null, Admin.class);
+        } catch (Exception e) {
+            LOGGER.error("unable to load localization for " + keyName + ", error: " + e.getMessage());
+            return "missing localization for " + keyName;
+        }
+    }
+
+    public enum EpsType {
+        PASSWORD_CHANGES(Statistic.PASSWORD_CHANGES),
+        AUTHENTICATION(Statistic.AUTHENTICATIONS),
+        INTRUDER_ATTEMPTS(Statistic.INTRUDER_ATTEMPTS),
+        PWMDB_WRITES(null),
+        PWMDB_READS(null),
+        DB_WRITES(null),
+        DB_READS(null),
+        ;
+
+        private Statistic relatedStatistic;
+
+        private EpsType(Statistic relatedStatistic) {
+            this.relatedStatistic = relatedStatistic;
+        }
+
+        public Statistic getRelatedStatistic() {
+            return relatedStatistic;
+        }
+
+        public String getLabel(final Locale locale) {
+            final String keyName = "Statistic_Label." + EpsType.class.getSimpleName() + "_" + this.name();
+            return LocaleHelper.getLocalizedMessage(locale, keyName, null, Admin.class);
+        }
+    }
+
+    public enum EpsDuration {
+        MINUTE(TimeDuration.MINUTE),
+        HOUR(TimeDuration.HOUR),
+        DAY(TimeDuration.DAY),
+        ;
+
+        private final TimeDuration timeDuration;
+
+        private EpsDuration(TimeDuration timeDuration) {
+            this.timeDuration = timeDuration;
+        }
+
+        public TimeDuration getTimeDuration() {
+            return timeDuration;
+        }
+    }
+
+    interface StatDetail {
+        boolean isActive(PwmApplication pwmApplication);
+    }
+
+    static class ConfigSettingDetail implements StatDetail {
+        final private PwmSetting pwmSetting;
+
+        ConfigSettingDetail(PwmSetting pwmSetting)
+        {
+            this.pwmSetting = pwmSetting;
+        }
+
+        public boolean isActive(final PwmApplication pwmApplication) {
+            return pwmApplication.getConfig().readSettingAsBoolean(pwmSetting);
+        }
+    }
+}

pwm/servlet/src/password/pwm/util/stats/StatisticsBundle.java → pwm/servlet/src/password/pwm/svc/stats/StatisticsBundle.java

@@ -1,161 +1,161 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.util.stats;
-
-import password.pwm.util.JsonUtil;
-import password.pwm.util.logging.PwmLogger;
-
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.text.SimpleDateFormat;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.TimeZone;
-
-public class StatisticsBundle {
-
-    private static final PwmLogger LOGGER = PwmLogger.forClass(StatisticsBundle.class);
-
-    final static SimpleDateFormat STORED_DATETIME_FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z");
-
-    static {
-        STORED_DATETIME_FORMATTER.setTimeZone(TimeZone.getTimeZone("Zulu"));
-    }
-
-
-    private final Map<Statistic, String> valueMap = new HashMap<>();
-
-    public StatisticsBundle() {
-    }
-
-    public String output() {
-        return JsonUtil.serializeMap(valueMap);
-    }
-
-    public static StatisticsBundle input(final String inputString) {
-        final Map<Statistic, String> srcMap = new HashMap<>();
-            final Map<String, String> loadedMap = JsonUtil.deserializeStringMap(inputString);
-            for (final String key : loadedMap.keySet()) {
-                try {
-                    srcMap.put(Statistic.valueOf(key),loadedMap.get(key));
-                } catch (IllegalArgumentException e) {
-                    LOGGER.error("error parsing statistic key '" + key + "', reason: " + e.getMessage());
-                }
-            }
-        final StatisticsBundle bundle = new StatisticsBundle();
-
-        for (final Statistic loopStat : Statistic.values()) {
-            final String value = srcMap.get(loopStat);
-            if (value != null && !value.equals("")) {
-                bundle.valueMap.put(loopStat, value);
-            }
-        }
-
-        return bundle;
-    }
-
-    public synchronized void incrementValue(final Statistic statistic) {
-        if (Statistic.Type.INCREMENTOR != statistic.getType()) {
-            LOGGER.error("attempt to increment non-counter/incremental stat " + statistic);
-            return;
-        }
-
-        BigInteger currentValue = BigInteger.ZERO;
-        try {
-            if (valueMap.containsKey(statistic)) {
-                currentValue = new BigInteger(valueMap.get(statistic));
-            } else {
-                currentValue = BigInteger.ZERO;
-            }
-        } catch (NumberFormatException e) {
-            LOGGER.error("error reading counter/incremental stat " + statistic);
-        }
-        final BigInteger newValue = currentValue.add(BigInteger.ONE);
-        valueMap.put(statistic, newValue.toString());
-    }
-
-    public synchronized void updateAverageValue(final Statistic statistic, final long timeDuration) {
-        if (Statistic.Type.AVERAGE != statistic.getType()) {
-            LOGGER.error("attempt to update average value of non-average stat " + statistic);
-            return;
-        }
-
-        final String avgStrValue = valueMap.get(statistic);
-
-        AverageBean avgBean = new AverageBean();
-        if (avgStrValue != null && avgStrValue.length() > 0) {
-            try {
-                avgBean = JsonUtil.deserialize(avgStrValue, AverageBean.class);
-            } catch (Exception e) {
-                LOGGER.trace("unable to parse statistics value for stat " + statistic.toString() + ", value=" + avgStrValue);
-            }
-        }
-
-        avgBean.appendValue(timeDuration);
-        valueMap.put(statistic, JsonUtil.serialize(avgBean));
-    }
-
-    public String getStatistic(final Statistic statistic) {
-        switch (statistic.getType()) {
-            case INCREMENTOR:
-                return valueMap.containsKey(statistic) ? valueMap.get(statistic) : "0";
-
-            case AVERAGE:
-                final String avgStrValue = valueMap.get(statistic);
-
-                AverageBean avgBean = new AverageBean();
-                if (avgStrValue != null && avgStrValue.length() > 0) {
-                    try {
-                        avgBean = JsonUtil.deserialize(avgStrValue, AverageBean.class);
-                    } catch (Exception e) {
-                        LOGGER.trace("unable to parse statistics value for stat " + statistic.toString() + ", value=" + avgStrValue);
-                    }
-                }
-                return avgBean.getAverage().toString();
-
-            default:
-                return "";
-        }
-    }
-
-    private static class AverageBean implements Serializable {
-        BigInteger total = BigInteger.ZERO;
-        BigInteger count = BigInteger.ZERO;
-
-        public AverageBean() {
-        }
-
-        BigInteger getAverage() {
-            if (BigInteger.ZERO.equals(count)) {
-                return BigInteger.ZERO;
-            }
-
-            return total.divide(count);
-        }
-
-        void appendValue(final long value) {
-            count = count.add(BigInteger.ONE);
-            total = total.add(BigInteger.valueOf(value));
-        }
-    }
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.stats;
+
+import password.pwm.util.JsonUtil;
+import password.pwm.util.logging.PwmLogger;
+
+import java.io.Serializable;
+import java.math.BigInteger;
+import java.text.SimpleDateFormat;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TimeZone;
+
+public class StatisticsBundle {
+
+    private static final PwmLogger LOGGER = PwmLogger.forClass(StatisticsBundle.class);
+
+    final static SimpleDateFormat STORED_DATETIME_FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z");
+
+    static {
+        STORED_DATETIME_FORMATTER.setTimeZone(TimeZone.getTimeZone("Zulu"));
+    }
+
+
+    private final Map<Statistic, String> valueMap = new HashMap<>();
+
+    public StatisticsBundle() {
+    }
+
+    public String output() {
+        return JsonUtil.serializeMap(valueMap);
+    }
+
+    public static StatisticsBundle input(final String inputString) {
+        final Map<Statistic, String> srcMap = new HashMap<>();
+            final Map<String, String> loadedMap = JsonUtil.deserializeStringMap(inputString);
+            for (final String key : loadedMap.keySet()) {
+                try {
+                    srcMap.put(Statistic.valueOf(key),loadedMap.get(key));
+                } catch (IllegalArgumentException e) {
+                    LOGGER.error("error parsing statistic key '" + key + "', reason: " + e.getMessage());
+                }
+            }
+        final StatisticsBundle bundle = new StatisticsBundle();
+
+        for (final Statistic loopStat : Statistic.values()) {
+            final String value = srcMap.get(loopStat);
+            if (value != null && !value.equals("")) {
+                bundle.valueMap.put(loopStat, value);
+            }
+        }
+
+        return bundle;
+    }
+
+    public synchronized void incrementValue(final Statistic statistic) {
+        if (Statistic.Type.INCREMENTOR != statistic.getType()) {
+            LOGGER.error("attempt to increment non-counter/incremental stat " + statistic);
+            return;
+        }
+
+        BigInteger currentValue = BigInteger.ZERO;
+        try {
+            if (valueMap.containsKey(statistic)) {
+                currentValue = new BigInteger(valueMap.get(statistic));
+            } else {
+                currentValue = BigInteger.ZERO;
+            }
+        } catch (NumberFormatException e) {
+            LOGGER.error("error reading counter/incremental stat " + statistic);
+        }
+        final BigInteger newValue = currentValue.add(BigInteger.ONE);
+        valueMap.put(statistic, newValue.toString());
+    }
+
+    public synchronized void updateAverageValue(final Statistic statistic, final long timeDuration) {
+        if (Statistic.Type.AVERAGE != statistic.getType()) {
+            LOGGER.error("attempt to update average value of non-average stat " + statistic);
+            return;
+        }
+
+        final String avgStrValue = valueMap.get(statistic);
+
+        AverageBean avgBean = new AverageBean();
+        if (avgStrValue != null && avgStrValue.length() > 0) {
+            try {
+                avgBean = JsonUtil.deserialize(avgStrValue, AverageBean.class);
+            } catch (Exception e) {
+                LOGGER.trace("unable to parse statistics value for stat " + statistic.toString() + ", value=" + avgStrValue);
+            }
+        }
+
+        avgBean.appendValue(timeDuration);
+        valueMap.put(statistic, JsonUtil.serialize(avgBean));
+    }
+
+    public String getStatistic(final Statistic statistic) {
+        switch (statistic.getType()) {
+            case INCREMENTOR:
+                return valueMap.containsKey(statistic) ? valueMap.get(statistic) : "0";
+
+            case AVERAGE:
+                final String avgStrValue = valueMap.get(statistic);
+
+                AverageBean avgBean = new AverageBean();
+                if (avgStrValue != null && avgStrValue.length() > 0) {
+                    try {
+                        avgBean = JsonUtil.deserialize(avgStrValue, AverageBean.class);
+                    } catch (Exception e) {
+                        LOGGER.trace("unable to parse statistics value for stat " + statistic.toString() + ", value=" + avgStrValue);
+                    }
+                }
+                return avgBean.getAverage().toString();
+
+            default:
+                return "";
+        }
+    }
+
+    private static class AverageBean implements Serializable {
+        BigInteger total = BigInteger.ZERO;
+        BigInteger count = BigInteger.ZERO;
+
+        public AverageBean() {
+        }
+
+        BigInteger getAverage() {
+            if (BigInteger.ZERO.equals(count)) {
+                return BigInteger.ZERO;
+            }
+
+            return total.divide(count);
+        }
+
+        void appendValue(final long value) {
+            count = count.add(BigInteger.ONE);
+            total = total.add(BigInteger.valueOf(value));
+        }
+    }
+}

pwm/servlet/src/password/pwm/util/stats/StatisticsManager.java → pwm/servlet/src/password/pwm/svc/stats/StatisticsManager.java

@@ -1,607 +1,607 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.util.stats;
-
-import org.apache.commons.csv.CSVPrinter;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
-import password.pwm.PwmService;
-import password.pwm.bean.StatsPublishBean;
-import password.pwm.config.Configuration;
-import password.pwm.config.PwmSetting;
-import password.pwm.config.option.DataStorageMethod;
-import password.pwm.error.PwmException;
-import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.health.HealthRecord;
-import password.pwm.http.PwmRequest;
-import password.pwm.http.client.PwmHttpClient;
-import password.pwm.util.AlertHandler;
-import password.pwm.util.Helper;
-import password.pwm.util.JsonUtil;
-import password.pwm.util.TimeDuration;
-import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.localdb.LocalDBException;
-import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.secure.PwmRandom;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.math.BigDecimal;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.*;
-
-public class StatisticsManager implements PwmService {
-
-    private static final PwmLogger LOGGER = PwmLogger.forClass(StatisticsManager.class);
-
-    private static final int DB_WRITE_FREQUENCY_MS = 60 * 1000;  // 1 minutes
-
-    private static final String DB_KEY_VERSION = "STATS_VERSION";
-    private static final String DB_KEY_CUMULATIVE = "CUMULATIVE";
-    private static final String DB_KEY_INITIAL_DAILY_KEY = "INITIAL_DAILY_KEY";
-    private static final String DB_KEY_PREFIX_DAILY = "DAILY_";
-    private static final String DB_KEY_TEMP = "TEMP_KEY";
-
-    private static final String DB_VALUE_VERSION = "1";
-
-    public static final String KEY_CURRENT = "CURRENT";
-    public static final String KEY_CUMULATIVE = "CUMULATIVE";
-    public static final String KEY_CLOUD_PUBLISH_TIMESTAMP = "CLOUD_PUB_TIMESTAMP";
-
-    private LocalDB localDB;
-
-    private DailyKey currentDailyKey = new DailyKey(new Date());
-    private DailyKey initialDailyKey = new DailyKey(new Date());
-
-    private Timer daemonTimer;
-
-    private final StatisticsBundle statsCurrent = new StatisticsBundle();
-    private StatisticsBundle statsDaily = new StatisticsBundle();
-    private StatisticsBundle statsCummulative = new StatisticsBundle();
-    private Map<String, EventRateMeter> epsMeterMap = new HashMap<>();
-
-    private PwmApplication pwmApplication;
-
-    private STATUS status = STATUS.NEW;
-
-
-
-    private final Map<String,StatisticsBundle> cachedStoredStats = new LinkedHashMap<String,StatisticsBundle>() {
-        @Override
-        protected boolean removeEldestEntry(final Map.Entry<String, StatisticsBundle> eldest) {
-            return this.size() > 50;
-        }
-    };
-
-    public StatisticsManager() {
-    }
-
-    public synchronized void incrementValue(final Statistic statistic) {
-        statsCurrent.incrementValue(statistic);
-        statsDaily.incrementValue(statistic);
-        statsCummulative.incrementValue(statistic);
-    }
-
-    public synchronized void updateAverageValue(final Statistic statistic, final long value) {
-        statsCurrent.updateAverageValue(statistic,value);
-        statsDaily.updateAverageValue(statistic,value);
-        statsCummulative.updateAverageValue(statistic,value);
-    }
-
-    public Map<String,String> getStatHistory(final Statistic statistic, final int days) {
-        final Map<String,String> returnMap = new LinkedHashMap<>();
-        DailyKey loopKey = currentDailyKey;
-        int counter = days;
-        while (counter > 0) {
-            final StatisticsBundle bundle = getStatBundleForKey(loopKey.toString());
-            if (bundle != null) {
-                final String key = (new SimpleDateFormat("MMM dd")).format(loopKey.calendar().getTime());
-                final String value = bundle.getStatistic(statistic);
-                returnMap.put(key,value);
-            }
-            loopKey = loopKey.previous();
-            counter--;
-        }
-        return returnMap;
-    }
-
-    public StatisticsBundle getStatBundleForKey(final String key) {
-        if (key == null || key.length() < 1 || KEY_CUMULATIVE.equals(key) ) {
-            return statsCummulative;
-        }
-
-        if (KEY_CURRENT.equals(key)) {
-            return statsCurrent;
-        }
-
-        if (currentDailyKey.toString().equals(key)) {
-            return statsDaily;
-        }
-
-        if (cachedStoredStats.containsKey(key)) {
-            return cachedStoredStats.get(key);
-        }
-
-        if (localDB == null) {
-            return null;
-        }
-
-        try {
-            final String storedStat = localDB.get(LocalDB.DB.PWM_STATS, key);
-            final StatisticsBundle returnBundle;
-            if (storedStat != null && storedStat.length() > 0) {
-                returnBundle = StatisticsBundle.input(storedStat);
-            } else {
-                returnBundle = new StatisticsBundle();
-            }
-            cachedStoredStats.put(key, returnBundle);
-            return returnBundle;
-        } catch (LocalDBException e) {
-            LOGGER.error("error retrieving stored stat for " + key + ": " + e.getMessage());
-        }
-
-        return null;
-    }
-
-    public Map<DailyKey,String> getAvailableKeys(final Locale locale) {
-        final DateFormat dateFormatter = SimpleDateFormat.getDateInstance(SimpleDateFormat.DEFAULT, locale);
-        final Map<DailyKey,String> returnMap = new LinkedHashMap<DailyKey,String>();
-
-        // add current time;
-        returnMap.put(currentDailyKey, dateFormatter.format(new Date()));
-
-        // if now historical data then we're done
-        if (currentDailyKey.equals(initialDailyKey)) {
-            return returnMap;
-        }
-
-        DailyKey loopKey = currentDailyKey;
-        int safetyCounter = 0;
-        while (!loopKey.equals(initialDailyKey) && safetyCounter < 5000) {
-            final Calendar c = loopKey.calendar();
-            final String display = dateFormatter.format(c.getTime());
-            returnMap.put(loopKey,display);
-            loopKey = loopKey.previous();
-            safetyCounter++;
-        }
-        return returnMap;
-    }
-
-    public String toString() {
-        final StringBuilder sb = new StringBuilder();
-
-        for (final Statistic m : Statistic.values()) {
-            sb.append(m.toString());
-            sb.append("=");
-            sb.append(statsCurrent.getStatistic(m));
-            sb.append(", ");
-        }
-
-        if (sb.length() > 2) {
-            sb.delete(sb.length() -2 , sb.length());
-        }
-
-        return sb.toString();
-    }
-
-    public void init(PwmApplication pwmApplication) throws PwmException {
-        for (final Statistic.EpsType type : Statistic.EpsType.values()) {
-            for (final Statistic.EpsDuration duration : Statistic.EpsDuration.values()) {
-                epsMeterMap.put(type.toString() + duration.toString(), new EventRateMeter(duration.getTimeDuration()));
-            }
-        }
-
-        status = STATUS.OPENING;
-        this.localDB = pwmApplication.getLocalDB();
-        this.pwmApplication = pwmApplication;
-
-        if (localDB == null) {
-            LOGGER.error("LocalDB is not available, will remain closed");
-            status = STATUS.CLOSED;
-            return;
-        }
-
-        {
-            final String storedCummulativeBundleStr = localDB.get(LocalDB.DB.PWM_STATS, DB_KEY_CUMULATIVE);
-            if (storedCummulativeBundleStr != null && storedCummulativeBundleStr.length() > 0) {
-                statsCummulative = StatisticsBundle.input(storedCummulativeBundleStr);
-            }
-        }
-
-        {
-            for (final Statistic.EpsType loopEpsType : Statistic.EpsType.values()) {
-                for (final Statistic.EpsType loopEpsDuration : Statistic.EpsType.values()) {
-                    final String key = "EPS-" + loopEpsType.toString() + loopEpsDuration.toString();
-                    final String storedValue = localDB.get(LocalDB.DB.PWM_STATS,key);
-                    if (storedValue != null && storedValue.length() > 0) {
-                        try {
-                            final EventRateMeter eventRateMeter = JsonUtil.deserialize(storedValue, EventRateMeter.class);
-                            epsMeterMap.put(loopEpsType.toString() + loopEpsDuration.toString(),eventRateMeter);
-                        } catch (Exception e) {
-                            LOGGER.error("unexpected error reading last EPS rate for " + loopEpsType + " from LocalDB: " + e.getMessage());
-                        }
-                    }
-                }
-            }
-
-        }
-
-        {
-            final String storedInitialString = localDB.get(LocalDB.DB.PWM_STATS, DB_KEY_INITIAL_DAILY_KEY);
-            if (storedInitialString != null && storedInitialString.length() > 0) {
-                initialDailyKey = new DailyKey(storedInitialString);
-            }
-        }
-
-        {
-            currentDailyKey = new DailyKey(new Date());
-            final String storedDailyStr = localDB.get(LocalDB.DB.PWM_STATS, currentDailyKey.toString());
-            if (storedDailyStr != null && storedDailyStr.length() > 0) {
-                statsDaily = StatisticsBundle.input(storedDailyStr);
-            }
-        }
-
-        try {
-            localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_TEMP, PwmConstants.DEFAULT_DATETIME_FORMAT.format(new Date()));
-        } catch (IllegalStateException e) {
-            LOGGER.error("unable to write to localDB, will remain closed, error: " + e.getMessage());
-            status = STATUS.CLOSED;
-            return;
-        }
-
-        localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_VERSION, DB_VALUE_VERSION);
-        localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_INITIAL_DAILY_KEY, initialDailyKey.toString());
-
-        { // setup a timer to roll over at 0 Zula and one to write current stats every 10 seconds
-            final String threadName = Helper.makeThreadName(pwmApplication, this.getClass()) + " timer";
-            daemonTimer = new Timer(threadName, true);
-            daemonTimer.schedule(new FlushTask(), 10 * 1000, DB_WRITE_FREQUENCY_MS);
-            daemonTimer.schedule(new NightlyTask(), Helper.nextZuluZeroTime());
-        }
-
-        if (pwmApplication.getApplicationMode() == PwmApplication.MODE.RUNNING) {
-            if (pwmApplication.getConfig().readSettingAsBoolean(PwmSetting.PUBLISH_STATS_ENABLE)) {
-                long lastPublishTimestamp = pwmApplication.getInstallTime().getTime();
-                {
-                    final String lastPublishDateStr = localDB.get(LocalDB.DB.PWM_STATS,KEY_CLOUD_PUBLISH_TIMESTAMP);
-                    if (lastPublishDateStr != null && lastPublishDateStr.length() > 0) {
-                        try {
-                            lastPublishTimestamp = Long.parseLong(lastPublishDateStr);
-                        } catch (Exception e) {
-                            LOGGER.error("unexpected error reading last publish timestamp from PwmDB: " + e.getMessage());
-                        }
-                    }
-                }
-                final Date nextPublishTime = new Date(lastPublishTimestamp + PwmConstants.STATISTICS_PUBLISH_FREQUENCY_MS + (long) PwmRandom.getInstance().nextInt(3600 * 1000));
-                daemonTimer.schedule(new PublishTask(), nextPublishTime, PwmConstants.STATISTICS_PUBLISH_FREQUENCY_MS);
-            }
-        }
-
-        status = STATUS.OPEN;
-    }
-
-    private void writeDbValues() {
-        if (localDB != null) {
-            try {
-                localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_CUMULATIVE, statsCummulative.output());
-                localDB.put(LocalDB.DB.PWM_STATS, currentDailyKey.toString(), statsDaily.output());
-
-                for (final Statistic.EpsType loopEpsType : Statistic.EpsType.values()) {
-                    for (final Statistic.EpsDuration loopEpsDuration : Statistic.EpsDuration.values()) {
-                        final String key = "EPS-" + loopEpsType.toString();
-                        final String mapKey = loopEpsType.toString() + loopEpsDuration.toString();
-                        final String value = JsonUtil.serialize(this.epsMeterMap.get(mapKey));
-                        localDB.put(LocalDB.DB.PWM_STATS, key, value);
-                    }
-                }
-            } catch (LocalDBException e) {
-                LOGGER.error("error outputting pwm statistics: " + e.getMessage());
-            }
-        }
-
-    }
-
-    private void resetDailyStats() {
-        try {
-            final Map<String, String> emailValues = new LinkedHashMap<>();
-            for (final Statistic statistic : Statistic.values()) {
-                final String key = statistic.getLabel(PwmConstants.DEFAULT_LOCALE);
-                final String value = statsDaily.getStatistic(statistic);
-                emailValues.put(key, value);
-            }
-
-            AlertHandler.alertDailyStats(pwmApplication, emailValues);
-        } catch (Exception e) {
-            LOGGER.error("error while generating daily alert statistics: " + e.getMessage());
-        }
-
-        currentDailyKey = new DailyKey(new Date());
-        statsDaily = new StatisticsBundle();
-        LOGGER.debug("reset daily statistics");
-    }
-
-    public STATUS status() {
-        return status;
-    }
-
-
-    public void close() {
-        try {
-            writeDbValues();
-        } catch (Exception e) {
-            LOGGER.error("unexpected error closing: " + e.getMessage());
-        }
-        if (daemonTimer != null) {
-            daemonTimer.cancel();
-        }
-        status = STATUS.CLOSED;
-    }
-
-    public List<HealthRecord> healthCheck() {
-        return Collections.emptyList();
-    }
-
-
-    private class NightlyTask extends TimerTask {
-        public void run() {
-            writeDbValues();
-            resetDailyStats();
-            daemonTimer.schedule(new NightlyTask(), Helper.nextZuluZeroTime());
-        }
-    }
-
-    private class FlushTask extends TimerTask {
-        public void run() {
-            writeDbValues();
-        }
-    }
-
-    private class PublishTask extends TimerTask {
-        public void run() {
-            try {
-                publishStatisticsToCloud();
-            } catch (Exception e) {
-                LOGGER.error("error publishing statistics to cloud: " + e.getMessage());
-            }
-        }
-    }
-
-    public static class DailyKey {
-        int year;
-        int day;
-
-        public DailyKey(final Date date) {
-            final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("Zulu"));
-            calendar.setTime(date);
-            year = calendar.get(Calendar.YEAR);
-            day = calendar.get(Calendar.DAY_OF_YEAR);
-        }
-
-        public DailyKey(final String value) {
-            final String strippedValue = value.substring(DB_KEY_PREFIX_DAILY.length(),value.length());
-            final String[] splitValue = strippedValue.split("_");
-            year = Integer.valueOf(splitValue[0]);
-            day = Integer.valueOf(splitValue[1]);
-        }
-
-        private DailyKey() {
-        }
-
-        @Override
-        public String toString() {
-            return DB_KEY_PREFIX_DAILY + String.valueOf(year) + "_" + String.valueOf(day);
-        }
-
-        public DailyKey previous() {
-            final Calendar calendar = calendar();
-            calendar.add(Calendar.HOUR,-24);
-            final DailyKey newKey = new DailyKey();
-            newKey.year = calendar.get(Calendar.YEAR);
-            newKey.day = calendar.get(Calendar.DAY_OF_YEAR);
-            return newKey;
-        }
-
-        public Calendar calendar() {
-            final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("Zulu"));
-            calendar.set(Calendar.YEAR,year);
-            calendar.set(Calendar.DAY_OF_YEAR,day);
-            return calendar;
-        }
-
-        @Override
-        public boolean equals(final Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-
-            final DailyKey key = (DailyKey) o;
-
-            if (day != key.day) return false;
-            if (year != key.year) return false;
-
-            return true;
-        }
-
-        @Override
-        public int hashCode() {
-            int result = year;
-            result = 31 * result + day;
-            return result;
-        }
-    }
-
-    public void updateEps(final Statistic.EpsType type, final int itemCount) {
-        for (final Statistic.EpsDuration duration : Statistic.EpsDuration.values()) {
-            epsMeterMap.get(type.toString() + duration.toString()).markEvents(itemCount);
-        }
-    }
-
-    public BigDecimal readEps(final Statistic.EpsType type, final Statistic.EpsDuration duration) {
-        return epsMeterMap.get(type.toString() + duration.toString()).readEventRate();
-    }
-
-    private void publishStatisticsToCloud()
-            throws URISyntaxException, IOException, PwmUnrecoverableException {
-        final StatsPublishBean statsPublishData;
-        {
-            final StatisticsBundle bundle = getStatBundleForKey(KEY_CUMULATIVE);
-            final Map<String,String> statData = new HashMap<>();
-            for (final Statistic loopStat : Statistic.values()) {
-                statData.put(loopStat.getKey(),bundle.getStatistic(loopStat));
-            }
-            final Configuration config = pwmApplication.getConfig();
-            final List<String> configuredSettings = new ArrayList<>();
-            for (final PwmSetting pwmSetting : config.nonDefaultSettings()) {
-                if (!pwmSetting.getCategory().hasProfiles() && !config.isDefaultValue(pwmSetting)) {
-                    configuredSettings.add(pwmSetting.getKey());
-                }
-            }
-            final Map<String,String> otherData = new HashMap<>();
-            otherData.put(StatsPublishBean.KEYS.SITE_URL.toString(),config.readSettingAsString(PwmSetting.PWM_SITE_URL));
-            otherData.put(StatsPublishBean.KEYS.SITE_DESCRIPTION.toString(),config.readSettingAsString(PwmSetting.PUBLISH_STATS_SITE_DESCRIPTION));
-            otherData.put(StatsPublishBean.KEYS.INSTALL_DATE.toString(),PwmConstants.DEFAULT_DATETIME_FORMAT.format(pwmApplication.getInstallTime()));
-
-            try {
-                otherData.put(StatsPublishBean.KEYS.LDAP_VENDOR.toString(),pwmApplication.getProxyChaiProvider(config.getDefaultLdapProfile().getIdentifier()).getDirectoryVendor().toString());
-            } catch (Exception e) {
-                LOGGER.trace("unable to read ldap vendor type for stats publication: " + e.getMessage());
-            }
-
-            statsPublishData = new StatsPublishBean(
-                    pwmApplication.getInstanceID(),
-                    new Date(),
-                    statData,
-                    configuredSettings,
-                    PwmConstants.BUILD_NUMBER,
-                    PwmConstants.BUILD_VERSION,
-                    otherData
-            );
-        }
-        final URI requestURI = new URI(PwmConstants.PWM_URL_CLOUD + "/rest/pwm/statistics");
-        final HttpPost httpPost = new HttpPost(requestURI.toString());
-        final String jsonDataString = JsonUtil.serialize(statsPublishData);
-        httpPost.setEntity(new StringEntity(jsonDataString));
-        httpPost.setHeader("Accept", PwmConstants.AcceptValue.json.getHeaderValue());
-        httpPost.setHeader("Content-Type", PwmConstants.ContentTypeValue.json.getHeaderValue());
-        LOGGER.debug("preparing to send anonymous statistics to " + requestURI.toString() + ", data to send: " + jsonDataString);
-        final HttpResponse httpResponse = PwmHttpClient.getHttpClient(pwmApplication.getConfig()).execute(httpPost);
-        if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
-            throw new IOException("http response error code: " + httpResponse.getStatusLine().getStatusCode());
-        }
-        LOGGER.info("published anonymous statistics to " + requestURI.toString());
-        try {
-            localDB.put(LocalDB.DB.PWM_STATS, KEY_CLOUD_PUBLISH_TIMESTAMP, String.valueOf(System.currentTimeMillis()));
-        } catch (LocalDBException e) {
-            LOGGER.error("unexpected error trying to save last statistics published time to LocalDB: " + e.getMessage());
-        }
-    }
-
-    public int outputStatsToCsv(final OutputStream outputStream, final Locale locale, final boolean includeHeader)
-            throws IOException
-    {
-        LOGGER.trace("beginning output stats to csv process");
-        final Date startTime = new Date();
-
-        final StatisticsManager statsManger = pwmApplication.getStatisticsManager();
-        final CSVPrinter csvPrinter = Helper.makeCsvPrinter(outputStream);
-
-        if (includeHeader) {
-            final List<String> headers = new ArrayList<>();
-            headers.add("KEY");
-            headers.add("YEAR");
-            headers.add("DAY");
-            for (Statistic stat : Statistic.values()) {
-                headers.add(stat.getLabel(locale));
-            }
-            csvPrinter.printRecord(headers);
-        }
-
-        int counter = 0;
-        final Map<StatisticsManager.DailyKey, String> keys = statsManger.getAvailableKeys(PwmConstants.DEFAULT_LOCALE);
-        for (final StatisticsManager.DailyKey loopKey : keys.keySet()) {
-            counter++;
-            final StatisticsBundle bundle = statsManger.getStatBundleForKey(loopKey.toString());
-            final List<String> lineOutput = new ArrayList<>();
-            lineOutput.add(loopKey.toString());
-            lineOutput.add(String.valueOf(loopKey.year));
-            lineOutput.add(String.valueOf(loopKey.day));
-            for (final Statistic stat : Statistic.values()) {
-                lineOutput.add(bundle.getStatistic(stat));
-            }
-            csvPrinter.printRecord(lineOutput);
-        }
-        
-        csvPrinter.flush();
-        LOGGER.trace("completed output stats to csv process; output " + counter + " records in " + TimeDuration.fromCurrent(
-                startTime).asCompactString());
-        return counter;
-    }
-
-    public ServiceInfo serviceInfo()
-    {
-        if (status() == STATUS.OPEN) {
-            return new ServiceInfo(Collections.singletonList(DataStorageMethod.LOCALDB));
-        } else {
-            return new ServiceInfo(Collections.<DataStorageMethod>emptyList());
-        }
-    }
-
-    public static void incrementStat(
-            final PwmRequest pwmRequest,
-            final Statistic statistic
-    )
-    {
-        incrementStat(pwmRequest.getPwmApplication(), statistic);
-    }
-
-    public static void incrementStat(
-            final PwmApplication pwmApplication,
-            final Statistic statistic
-    ) {
-        if (pwmApplication == null) {
-            LOGGER.error("skipping requested statistic increment of " + statistic + " due to null pwmApplication");
-            return;
-        }
-
-        final StatisticsManager statisticsManager = pwmApplication.getStatisticsManager();
-        if (statisticsManager == null) {
-            LOGGER.error("skipping requested statistic increment of " + statistic + " due to null statisticsManager");
-            return;
-        }
-
-        if (statisticsManager.status() != STATUS.OPEN) {
-            LOGGER.trace(
-                    "skipping requested statistic increment of " + statistic + " due to StatisticsManager being closed");
-            return;
-        }
-
-        statisticsManager.incrementValue(statistic);
-    }
-
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.stats;
+
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import password.pwm.PwmApplication;
+import password.pwm.PwmConstants;
+import password.pwm.bean.StatsPublishBean;
+import password.pwm.config.Configuration;
+import password.pwm.config.PwmSetting;
+import password.pwm.config.option.DataStorageMethod;
+import password.pwm.error.PwmException;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.health.HealthRecord;
+import password.pwm.http.PwmRequest;
+import password.pwm.http.client.PwmHttpClient;
+import password.pwm.svc.PwmService;
+import password.pwm.util.AlertHandler;
+import password.pwm.util.Helper;
+import password.pwm.util.JsonUtil;
+import password.pwm.util.TimeDuration;
+import password.pwm.util.localdb.LocalDB;
+import password.pwm.util.localdb.LocalDBException;
+import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.secure.PwmRandom;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.math.BigDecimal;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+public class StatisticsManager implements PwmService {
+
+    private static final PwmLogger LOGGER = PwmLogger.forClass(StatisticsManager.class);
+
+    private static final int DB_WRITE_FREQUENCY_MS = 60 * 1000;  // 1 minutes
+
+    private static final String DB_KEY_VERSION = "STATS_VERSION";
+    private static final String DB_KEY_CUMULATIVE = "CUMULATIVE";
+    private static final String DB_KEY_INITIAL_DAILY_KEY = "INITIAL_DAILY_KEY";
+    private static final String DB_KEY_PREFIX_DAILY = "DAILY_";
+    private static final String DB_KEY_TEMP = "TEMP_KEY";
+
+    private static final String DB_VALUE_VERSION = "1";
+
+    public static final String KEY_CURRENT = "CURRENT";
+    public static final String KEY_CUMULATIVE = "CUMULATIVE";
+    public static final String KEY_CLOUD_PUBLISH_TIMESTAMP = "CLOUD_PUB_TIMESTAMP";
+
+    private LocalDB localDB;
+
+    private DailyKey currentDailyKey = new DailyKey(new Date());
+    private DailyKey initialDailyKey = new DailyKey(new Date());
+
+    private Timer daemonTimer;
+
+    private final StatisticsBundle statsCurrent = new StatisticsBundle();
+    private StatisticsBundle statsDaily = new StatisticsBundle();
+    private StatisticsBundle statsCummulative = new StatisticsBundle();
+    private Map<String, EventRateMeter> epsMeterMap = new HashMap<>();
+
+    private PwmApplication pwmApplication;
+
+    private STATUS status = STATUS.NEW;
+
+
+
+    private final Map<String,StatisticsBundle> cachedStoredStats = new LinkedHashMap<String,StatisticsBundle>() {
+        @Override
+        protected boolean removeEldestEntry(final Map.Entry<String, StatisticsBundle> eldest) {
+            return this.size() > 50;
+        }
+    };
+
+    public StatisticsManager() {
+    }
+
+    public synchronized void incrementValue(final Statistic statistic) {
+        statsCurrent.incrementValue(statistic);
+        statsDaily.incrementValue(statistic);
+        statsCummulative.incrementValue(statistic);
+    }
+
+    public synchronized void updateAverageValue(final Statistic statistic, final long value) {
+        statsCurrent.updateAverageValue(statistic,value);
+        statsDaily.updateAverageValue(statistic,value);
+        statsCummulative.updateAverageValue(statistic, value);
+    }
+
+    public Map<String,String> getStatHistory(final Statistic statistic, final int days) {
+        final Map<String,String> returnMap = new LinkedHashMap<>();
+        DailyKey loopKey = currentDailyKey;
+        int counter = days;
+        while (counter > 0) {
+            final StatisticsBundle bundle = getStatBundleForKey(loopKey.toString());
+            if (bundle != null) {
+                final String key = (new SimpleDateFormat("MMM dd")).format(loopKey.calendar().getTime());
+                final String value = bundle.getStatistic(statistic);
+                returnMap.put(key,value);
+            }
+            loopKey = loopKey.previous();
+            counter--;
+        }
+        return returnMap;
+    }
+
+    public StatisticsBundle getStatBundleForKey(final String key) {
+        if (key == null || key.length() < 1 || KEY_CUMULATIVE.equals(key) ) {
+            return statsCummulative;
+        }
+
+        if (KEY_CURRENT.equals(key)) {
+            return statsCurrent;
+        }
+
+        if (currentDailyKey.toString().equals(key)) {
+            return statsDaily;
+        }
+
+        if (cachedStoredStats.containsKey(key)) {
+            return cachedStoredStats.get(key);
+        }
+
+        if (localDB == null) {
+            return null;
+        }
+
+        try {
+            final String storedStat = localDB.get(LocalDB.DB.PWM_STATS, key);
+            final StatisticsBundle returnBundle;
+            if (storedStat != null && storedStat.length() > 0) {
+                returnBundle = StatisticsBundle.input(storedStat);
+            } else {
+                returnBundle = new StatisticsBundle();
+            }
+            cachedStoredStats.put(key, returnBundle);
+            return returnBundle;
+        } catch (LocalDBException e) {
+            LOGGER.error("error retrieving stored stat for " + key + ": " + e.getMessage());
+        }
+
+        return null;
+    }
+
+    public Map<DailyKey,String> getAvailableKeys(final Locale locale) {
+        final DateFormat dateFormatter = SimpleDateFormat.getDateInstance(SimpleDateFormat.DEFAULT, locale);
+        final Map<DailyKey,String> returnMap = new LinkedHashMap<DailyKey,String>();
+
+        // add current time;
+        returnMap.put(currentDailyKey, dateFormatter.format(new Date()));
+
+        // if now historical data then we're done
+        if (currentDailyKey.equals(initialDailyKey)) {
+            return returnMap;
+        }
+
+        DailyKey loopKey = currentDailyKey;
+        int safetyCounter = 0;
+        while (!loopKey.equals(initialDailyKey) && safetyCounter < 5000) {
+            final Calendar c = loopKey.calendar();
+            final String display = dateFormatter.format(c.getTime());
+            returnMap.put(loopKey,display);
+            loopKey = loopKey.previous();
+            safetyCounter++;
+        }
+        return returnMap;
+    }
+
+    public String toString() {
+        final StringBuilder sb = new StringBuilder();
+
+        for (final Statistic m : Statistic.values()) {
+            sb.append(m.toString());
+            sb.append("=");
+            sb.append(statsCurrent.getStatistic(m));
+            sb.append(", ");
+        }
+
+        if (sb.length() > 2) {
+            sb.delete(sb.length() -2 , sb.length());
+        }
+
+        return sb.toString();
+    }
+
+    public void init(PwmApplication pwmApplication) throws PwmException {
+        for (final Statistic.EpsType type : Statistic.EpsType.values()) {
+            for (final Statistic.EpsDuration duration : Statistic.EpsDuration.values()) {
+                epsMeterMap.put(type.toString() + duration.toString(), new EventRateMeter(duration.getTimeDuration()));
+            }
+        }
+
+        status = STATUS.OPENING;
+        this.localDB = pwmApplication.getLocalDB();
+        this.pwmApplication = pwmApplication;
+
+        if (localDB == null) {
+            LOGGER.error("LocalDB is not available, will remain closed");
+            status = STATUS.CLOSED;
+            return;
+        }
+
+        {
+            final String storedCummulativeBundleStr = localDB.get(LocalDB.DB.PWM_STATS, DB_KEY_CUMULATIVE);
+            if (storedCummulativeBundleStr != null && storedCummulativeBundleStr.length() > 0) {
+                statsCummulative = StatisticsBundle.input(storedCummulativeBundleStr);
+            }
+        }
+
+        {
+            for (final Statistic.EpsType loopEpsType : Statistic.EpsType.values()) {
+                for (final Statistic.EpsType loopEpsDuration : Statistic.EpsType.values()) {
+                    final String key = "EPS-" + loopEpsType.toString() + loopEpsDuration.toString();
+                    final String storedValue = localDB.get(LocalDB.DB.PWM_STATS,key);
+                    if (storedValue != null && storedValue.length() > 0) {
+                        try {
+                            final EventRateMeter eventRateMeter = JsonUtil.deserialize(storedValue, EventRateMeter.class);
+                            epsMeterMap.put(loopEpsType.toString() + loopEpsDuration.toString(),eventRateMeter);
+                        } catch (Exception e) {
+                            LOGGER.error("unexpected error reading last EPS rate for " + loopEpsType + " from LocalDB: " + e.getMessage());
+                        }
+                    }
+                }
+            }
+
+        }
+
+        {
+            final String storedInitialString = localDB.get(LocalDB.DB.PWM_STATS, DB_KEY_INITIAL_DAILY_KEY);
+            if (storedInitialString != null && storedInitialString.length() > 0) {
+                initialDailyKey = new DailyKey(storedInitialString);
+            }
+        }
+
+        {
+            currentDailyKey = new DailyKey(new Date());
+            final String storedDailyStr = localDB.get(LocalDB.DB.PWM_STATS, currentDailyKey.toString());
+            if (storedDailyStr != null && storedDailyStr.length() > 0) {
+                statsDaily = StatisticsBundle.input(storedDailyStr);
+            }
+        }
+
+        try {
+            localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_TEMP, PwmConstants.DEFAULT_DATETIME_FORMAT.format(new Date()));
+        } catch (IllegalStateException e) {
+            LOGGER.error("unable to write to localDB, will remain closed, error: " + e.getMessage());
+            status = STATUS.CLOSED;
+            return;
+        }
+
+        localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_VERSION, DB_VALUE_VERSION);
+        localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_INITIAL_DAILY_KEY, initialDailyKey.toString());
+
+        { // setup a timer to roll over at 0 Zula and one to write current stats every 10 seconds
+            final String threadName = Helper.makeThreadName(pwmApplication, this.getClass()) + " timer";
+            daemonTimer = new Timer(threadName, true);
+            daemonTimer.schedule(new FlushTask(), 10 * 1000, DB_WRITE_FREQUENCY_MS);
+            daemonTimer.schedule(new NightlyTask(), Helper.nextZuluZeroTime());
+        }
+
+        if (pwmApplication.getApplicationMode() == PwmApplication.MODE.RUNNING) {
+            if (pwmApplication.getConfig().readSettingAsBoolean(PwmSetting.PUBLISH_STATS_ENABLE)) {
+                long lastPublishTimestamp = pwmApplication.getInstallTime().getTime();
+                {
+                    final String lastPublishDateStr = localDB.get(LocalDB.DB.PWM_STATS,KEY_CLOUD_PUBLISH_TIMESTAMP);
+                    if (lastPublishDateStr != null && lastPublishDateStr.length() > 0) {
+                        try {
+                            lastPublishTimestamp = Long.parseLong(lastPublishDateStr);
+                        } catch (Exception e) {
+                            LOGGER.error("unexpected error reading last publish timestamp from PwmDB: " + e.getMessage());
+                        }
+                    }
+                }
+                final Date nextPublishTime = new Date(lastPublishTimestamp + PwmConstants.STATISTICS_PUBLISH_FREQUENCY_MS + (long) PwmRandom.getInstance().nextInt(3600 * 1000));
+                daemonTimer.schedule(new PublishTask(), nextPublishTime, PwmConstants.STATISTICS_PUBLISH_FREQUENCY_MS);
+            }
+        }
+
+        status = STATUS.OPEN;
+    }
+
+    private void writeDbValues() {
+        if (localDB != null) {
+            try {
+                localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_CUMULATIVE, statsCummulative.output());
+                localDB.put(LocalDB.DB.PWM_STATS, currentDailyKey.toString(), statsDaily.output());
+
+                for (final Statistic.EpsType loopEpsType : Statistic.EpsType.values()) {
+                    for (final Statistic.EpsDuration loopEpsDuration : Statistic.EpsDuration.values()) {
+                        final String key = "EPS-" + loopEpsType.toString();
+                        final String mapKey = loopEpsType.toString() + loopEpsDuration.toString();
+                        final String value = JsonUtil.serialize(this.epsMeterMap.get(mapKey));
+                        localDB.put(LocalDB.DB.PWM_STATS, key, value);
+                    }
+                }
+            } catch (LocalDBException e) {
+                LOGGER.error("error outputting pwm statistics: " + e.getMessage());
+            }
+        }
+
+    }
+
+    private void resetDailyStats() {
+        try {
+            final Map<String, String> emailValues = new LinkedHashMap<>();
+            for (final Statistic statistic : Statistic.values()) {
+                final String key = statistic.getLabel(PwmConstants.DEFAULT_LOCALE);
+                final String value = statsDaily.getStatistic(statistic);
+                emailValues.put(key, value);
+            }
+
+            AlertHandler.alertDailyStats(pwmApplication, emailValues);
+        } catch (Exception e) {
+            LOGGER.error("error while generating daily alert statistics: " + e.getMessage());
+        }
+
+        currentDailyKey = new DailyKey(new Date());
+        statsDaily = new StatisticsBundle();
+        LOGGER.debug("reset daily statistics");
+    }
+
+    public STATUS status() {
+        return status;
+    }
+
+
+    public void close() {
+        try {
+            writeDbValues();
+        } catch (Exception e) {
+            LOGGER.error("unexpected error closing: " + e.getMessage());
+        }
+        if (daemonTimer != null) {
+            daemonTimer.cancel();
+        }
+        status = STATUS.CLOSED;
+    }
+
+    public List<HealthRecord> healthCheck() {
+        return Collections.emptyList();
+    }
+
+
+    private class NightlyTask extends TimerTask {
+        public void run() {
+            writeDbValues();
+            resetDailyStats();
+            daemonTimer.schedule(new NightlyTask(), Helper.nextZuluZeroTime());
+        }
+    }
+
+    private class FlushTask extends TimerTask {
+        public void run() {
+            writeDbValues();
+        }
+    }
+
+    private class PublishTask extends TimerTask {
+        public void run() {
+            try {
+                publishStatisticsToCloud();
+            } catch (Exception e) {
+                LOGGER.error("error publishing statistics to cloud: " + e.getMessage());
+            }
+        }
+    }
+
+    public static class DailyKey {
+        int year;
+        int day;
+
+        public DailyKey(final Date date) {
+            final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("Zulu"));
+            calendar.setTime(date);
+            year = calendar.get(Calendar.YEAR);
+            day = calendar.get(Calendar.DAY_OF_YEAR);
+        }
+
+        public DailyKey(final String value) {
+            final String strippedValue = value.substring(DB_KEY_PREFIX_DAILY.length(),value.length());
+            final String[] splitValue = strippedValue.split("_");
+            year = Integer.valueOf(splitValue[0]);
+            day = Integer.valueOf(splitValue[1]);
+        }
+
+        private DailyKey() {
+        }
+
+        @Override
+        public String toString() {
+            return DB_KEY_PREFIX_DAILY + String.valueOf(year) + "_" + String.valueOf(day);
+        }
+
+        public DailyKey previous() {
+            final Calendar calendar = calendar();
+            calendar.add(Calendar.HOUR,-24);
+            final DailyKey newKey = new DailyKey();
+            newKey.year = calendar.get(Calendar.YEAR);
+            newKey.day = calendar.get(Calendar.DAY_OF_YEAR);
+            return newKey;
+        }
+
+        public Calendar calendar() {
+            final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("Zulu"));
+            calendar.set(Calendar.YEAR,year);
+            calendar.set(Calendar.DAY_OF_YEAR,day);
+            return calendar;
+        }
+
+        @Override
+        public boolean equals(final Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            final DailyKey key = (DailyKey) o;
+
+            if (day != key.day) return false;
+            if (year != key.year) return false;
+
+            return true;
+        }
+
+        @Override
+        public int hashCode() {
+            int result = year;
+            result = 31 * result + day;
+            return result;
+        }
+    }
+
+    public void updateEps(final Statistic.EpsType type, final int itemCount) {
+        for (final Statistic.EpsDuration duration : Statistic.EpsDuration.values()) {
+            epsMeterMap.get(type.toString() + duration.toString()).markEvents(itemCount);
+        }
+    }
+
+    public BigDecimal readEps(final Statistic.EpsType type, final Statistic.EpsDuration duration) {
+        return epsMeterMap.get(type.toString() + duration.toString()).readEventRate();
+    }
+
+    private void publishStatisticsToCloud()
+            throws URISyntaxException, IOException, PwmUnrecoverableException {
+        final StatsPublishBean statsPublishData;
+        {
+            final StatisticsBundle bundle = getStatBundleForKey(KEY_CUMULATIVE);
+            final Map<String,String> statData = new HashMap<>();
+            for (final Statistic loopStat : Statistic.values()) {
+                statData.put(loopStat.getKey(),bundle.getStatistic(loopStat));
+            }
+            final Configuration config = pwmApplication.getConfig();
+            final List<String> configuredSettings = new ArrayList<>();
+            for (final PwmSetting pwmSetting : config.nonDefaultSettings()) {
+                if (!pwmSetting.getCategory().hasProfiles() && !config.isDefaultValue(pwmSetting)) {
+                    configuredSettings.add(pwmSetting.getKey());
+                }
+            }
+            final Map<String,String> otherData = new HashMap<>();
+            otherData.put(StatsPublishBean.KEYS.SITE_URL.toString(),config.readSettingAsString(PwmSetting.PWM_SITE_URL));
+            otherData.put(StatsPublishBean.KEYS.SITE_DESCRIPTION.toString(),config.readSettingAsString(PwmSetting.PUBLISH_STATS_SITE_DESCRIPTION));
+            otherData.put(StatsPublishBean.KEYS.INSTALL_DATE.toString(),PwmConstants.DEFAULT_DATETIME_FORMAT.format(pwmApplication.getInstallTime()));
+
+            try {
+                otherData.put(StatsPublishBean.KEYS.LDAP_VENDOR.toString(),pwmApplication.getProxyChaiProvider(config.getDefaultLdapProfile().getIdentifier()).getDirectoryVendor().toString());
+            } catch (Exception e) {
+                LOGGER.trace("unable to read ldap vendor type for stats publication: " + e.getMessage());
+            }
+
+            statsPublishData = new StatsPublishBean(
+                    pwmApplication.getInstanceID(),
+                    new Date(),
+                    statData,
+                    configuredSettings,
+                    PwmConstants.BUILD_NUMBER,
+                    PwmConstants.BUILD_VERSION,
+                    otherData
+            );
+        }
+        final URI requestURI = new URI(PwmConstants.PWM_URL_CLOUD + "/rest/pwm/statistics");
+        final HttpPost httpPost = new HttpPost(requestURI.toString());
+        final String jsonDataString = JsonUtil.serialize(statsPublishData);
+        httpPost.setEntity(new StringEntity(jsonDataString));
+        httpPost.setHeader("Accept", PwmConstants.AcceptValue.json.getHeaderValue());
+        httpPost.setHeader("Content-Type", PwmConstants.ContentTypeValue.json.getHeaderValue());
+        LOGGER.debug("preparing to send anonymous statistics to " + requestURI.toString() + ", data to send: " + jsonDataString);
+        final HttpResponse httpResponse = PwmHttpClient.getHttpClient(pwmApplication.getConfig()).execute(httpPost);
+        if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+            throw new IOException("http response error code: " + httpResponse.getStatusLine().getStatusCode());
+        }
+        LOGGER.info("published anonymous statistics to " + requestURI.toString());
+        try {
+            localDB.put(LocalDB.DB.PWM_STATS, KEY_CLOUD_PUBLISH_TIMESTAMP, String.valueOf(System.currentTimeMillis()));
+        } catch (LocalDBException e) {
+            LOGGER.error("unexpected error trying to save last statistics published time to LocalDB: " + e.getMessage());
+        }
+    }
+
+    public int outputStatsToCsv(final OutputStream outputStream, final Locale locale, final boolean includeHeader)
+            throws IOException
+    {
+        LOGGER.trace("beginning output stats to csv process");
+        final Date startTime = new Date();
+
+        final StatisticsManager statsManger = pwmApplication.getStatisticsManager();
+        final CSVPrinter csvPrinter = Helper.makeCsvPrinter(outputStream);
+
+        if (includeHeader) {
+            final List<String> headers = new ArrayList<>();
+            headers.add("KEY");
+            headers.add("YEAR");
+            headers.add("DAY");
+            for (Statistic stat : Statistic.values()) {
+                headers.add(stat.getLabel(locale));
+            }
+            csvPrinter.printRecord(headers);
+        }
+
+        int counter = 0;
+        final Map<StatisticsManager.DailyKey, String> keys = statsManger.getAvailableKeys(PwmConstants.DEFAULT_LOCALE);
+        for (final StatisticsManager.DailyKey loopKey : keys.keySet()) {
+            counter++;
+            final StatisticsBundle bundle = statsManger.getStatBundleForKey(loopKey.toString());
+            final List<String> lineOutput = new ArrayList<>();
+            lineOutput.add(loopKey.toString());
+            lineOutput.add(String.valueOf(loopKey.year));
+            lineOutput.add(String.valueOf(loopKey.day));
+            for (final Statistic stat : Statistic.values()) {
+                lineOutput.add(bundle.getStatistic(stat));
+            }
+            csvPrinter.printRecord(lineOutput);
+        }
+        
+        csvPrinter.flush();
+        LOGGER.trace("completed output stats to csv process; output " + counter + " records in " + TimeDuration.fromCurrent(
+                startTime).asCompactString());
+        return counter;
+    }
+
+    public ServiceInfo serviceInfo()
+    {
+        if (status() == STATUS.OPEN) {
+            return new ServiceInfo(Collections.singletonList(DataStorageMethod.LOCALDB));
+        } else {
+            return new ServiceInfo(Collections.<DataStorageMethod>emptyList());
+        }
+    }
+
+    public static void incrementStat(
+            final PwmRequest pwmRequest,
+            final Statistic statistic
+    )
+    {
+        incrementStat(pwmRequest.getPwmApplication(), statistic);
+    }
+
+    public static void incrementStat(
+            final PwmApplication pwmApplication,
+            final Statistic statistic
+    ) {
+        if (pwmApplication == null) {
+            LOGGER.error("skipping requested statistic increment of " + statistic + " due to null pwmApplication");
+            return;
+        }
+
+        final StatisticsManager statisticsManager = pwmApplication.getStatisticsManager();
+        if (statisticsManager == null) {
+            LOGGER.error("skipping requested statistic increment of " + statistic + " due to null statisticsManager");
+            return;
+        }
+
+        if (statisticsManager.status() != STATUS.OPEN) {
+            LOGGER.trace(
+                    "skipping requested statistic increment of " + statistic + " due to StatisticsManager being closed");
+            return;
+        }
+
+        statisticsManager.incrementValue(statistic);
+    }
+
+}

pwm/servlet/src/password/pwm/token/CryptoTokenMachine.java → pwm/servlet/src/password/pwm/svc/token/CryptoTokenMachine.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.token;
+package password.pwm.svc.token;
 
 import password.pwm.bean.SessionLabel;
 import password.pwm.error.PwmOperationalException;

pwm/servlet/src/password/pwm/token/DBTokenMachine.java → pwm/servlet/src/password/pwm/svc/token/DBTokenMachine.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.token;
+package password.pwm.svc.token;
 
 import password.pwm.bean.SessionLabel;
 import password.pwm.error.PwmOperationalException;

pwm/servlet/src/password/pwm/token/LdapTokenMachine.java → pwm/servlet/src/password/pwm/svc/token/LdapTokenMachine.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.token;
+package password.pwm.svc.token;
 
 import com.novell.ldapchai.ChaiUser;
 import com.novell.ldapchai.exception.ChaiException;

pwm/servlet/src/password/pwm/token/LocalDBTokenMachine.java → pwm/servlet/src/password/pwm/svc/token/LocalDBTokenMachine.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.token;
+package password.pwm.svc.token;
 
 import password.pwm.bean.SessionLabel;
 import password.pwm.error.PwmOperationalException;

pwm/servlet/src/password/pwm/token/TokenMachine.java → pwm/servlet/src/password/pwm/svc/token/TokenMachine.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.token;
+package password.pwm.svc.token;
 
 import password.pwm.bean.SessionLabel;
 import password.pwm.error.PwmOperationalException;

pwm/servlet/src/password/pwm/token/TokenPayload.java → pwm/servlet/src/password/pwm/svc/token/TokenPayload.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm.token;
+package password.pwm.svc.token;
 
 import password.pwm.bean.UserIdentity;
 

pwm/servlet/src/password/pwm/token/TokenService.java → pwm/servlet/src/password/pwm/svc/token/TokenService.java

@@ -1,729 +1,729 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.token;
-
-import com.novell.ldapchai.exception.ChaiUnavailableException;
-import password.pwm.AppProperty;
-import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
-import password.pwm.PwmService;
-import password.pwm.bean.*;
-import password.pwm.config.Configuration;
-import password.pwm.config.PwmSetting;
-import password.pwm.config.option.DataStorageMethod;
-import password.pwm.config.option.MessageSendMethod;
-import password.pwm.config.option.TokenStorageMethod;
-import password.pwm.config.profile.ForgottenPasswordProfile;
-import password.pwm.config.profile.NewUserProfile;
-import password.pwm.error.*;
-import password.pwm.event.AuditEvent;
-import password.pwm.health.HealthMessage;
-import password.pwm.health.HealthRecord;
-import password.pwm.http.PwmSession;
-import password.pwm.ldap.auth.SessionAuthenticator;
-import password.pwm.util.Helper;
-import password.pwm.util.JsonUtil;
-import password.pwm.util.TimeDuration;
-import password.pwm.util.intruder.RecordType;
-import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.operations.PasswordUtility;
-import password.pwm.util.secure.PwmRandom;
-import password.pwm.util.secure.SecureEngine;
-import password.pwm.util.stats.Statistic;
-import password.pwm.util.stats.StatisticsManager;
-
-import java.util.*;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-/**
- * This PWM service is responsible for reading/writing tokens used for forgotten password,
- * new user registration, account activation, and other functions.  Several implementations
- * of the backing storage method are available.
- *
- * @author jrivard@gmail.com
- */
-public class TokenService implements PwmService {
-
-    private static final PwmLogger LOGGER = PwmLogger.forClass(TokenService.class);
-
-    private static final long MAX_CLEANER_INTERVAL_MS = 24 * 60 * 60 * 1000; // one day
-    private static final long MIN_CLEANER_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
-
-    private ScheduledExecutorService executorService;
-
-    private PwmApplication pwmApplication;
-    private Configuration configuration;
-    private TokenStorageMethod storageMethod;
-    private long maxTokenAgeMS;
-    private long maxTokenPurgeAgeMS;
-    private TokenMachine tokenMachine;
-    private long counter;
-
-    private ServiceInfo serviceInfo = new ServiceInfo(Collections.<DataStorageMethod>emptyList());
-    private STATUS status = STATUS.NEW;
-
-    private ErrorInformation errorInformation = null;
-
-    public TokenService()
-            throws PwmOperationalException
-    {
-    }
-
-    public synchronized TokenPayload createTokenPayload(final TokenType name, final Map<String, String> data, final UserIdentity userIdentity, final Set<String> dest) {
-        final long count = counter++;
-        final StringBuilder guid = new StringBuilder();
-        try {
-            guid.append(SecureEngine.md5sum(pwmApplication.getInstanceID() + pwmApplication.getStartupTime().toString()));
-            guid.append("-");
-            guid.append(count);
-        } catch (Exception e) {
-            LOGGER.error("error making payload guid: " + e.getMessage(),e);
-        }
-        return new TokenPayload(name.name(), data, userIdentity, dest, guid.toString());
-    }
-
-    public void init(final PwmApplication pwmApplication)
-            throws PwmException
-    {
-        LOGGER.trace("opening");
-        status = STATUS.OPENING;
-
-        this.pwmApplication = pwmApplication;
-        this.configuration = pwmApplication.getConfig();
-
-        storageMethod = configuration.getTokenStorageMethod();
-        if (storageMethod == null) {
-            final String errorMsg = "no storage method specified";
-            errorInformation = new ErrorInformation(PwmError.ERROR_INVALID_CONFIG,errorMsg);
-            status = STATUS.CLOSED;
-            throw new PwmOperationalException(errorInformation);
-        }
-        try {
-            maxTokenAgeMS = configuration.readSettingAsLong(PwmSetting.TOKEN_LIFETIME) * 1000;
-            maxTokenPurgeAgeMS = maxTokenAgeMS + Long.parseLong(configuration.readAppProperty(AppProperty.TOKEN_REMOVAL_DELAY_MS));
-        } catch (Exception e) {
-            final String errorMsg = "unable to parse max token age value: " + e.getMessage();
-            errorInformation = new ErrorInformation(PwmError.ERROR_INVALID_CONFIG,errorMsg);
-            status = STATUS.CLOSED;
-            throw new PwmOperationalException(errorInformation);
-        }
-
-        try {
-            DataStorageMethod usedStorageMethod = null;
-            switch (storageMethod) {
-                case STORE_LOCALDB:
-                    tokenMachine = new LocalDBTokenMachine(this, pwmApplication.getLocalDB());
-                    usedStorageMethod = DataStorageMethod.LOCALDB;
-                    break;
-
-                case STORE_DB:
-                    tokenMachine = new DBTokenMachine(this, pwmApplication.getDatabaseAccessor());
-                    usedStorageMethod = DataStorageMethod.DB;
-                    break;
-
-                case STORE_CRYPTO:
-                    tokenMachine = new CryptoTokenMachine(this);
-                    usedStorageMethod = DataStorageMethod.CRYPTO;
-                    break;
-
-                case STORE_LDAP:
-                    tokenMachine = new LdapTokenMachine(this, pwmApplication);
-                    usedStorageMethod = DataStorageMethod.LDAP;
-                    break;
-            }
-            serviceInfo = new ServiceInfo(Collections.singletonList(usedStorageMethod));
-        } catch (PwmException e) {
-            final String errorMsg = "unable to start token manager: " + e.getErrorInformation().getDetailedErrorMsg();
-            final ErrorInformation newErrorInformation = new ErrorInformation(e.getError(), errorMsg);
-            errorInformation = newErrorInformation;
-            LOGGER.error(newErrorInformation.toDebugStr());
-            status = STATUS.CLOSED;
-            return;
-        }
-
-        executorService = Executors.newSingleThreadScheduledExecutor(
-                Helper.makePwmThreadFactory(
-                        Helper.makeThreadName(pwmApplication, this.getClass()) + "-",
-                        true
-                ));
-
-        final TimerTask cleanerTask = new CleanerTask();
-
-        final long cleanerFrequency = (maxTokenAgeMS*0.5) > MAX_CLEANER_INTERVAL_MS ? MAX_CLEANER_INTERVAL_MS : (maxTokenAgeMS*0.5) < MIN_CLEANER_INTERVAL_MS ? MIN_CLEANER_INTERVAL_MS : (long)(maxTokenAgeMS*0.5);
-        executorService.scheduleAtFixedRate(cleanerTask, 10 * 1000, cleanerFrequency, TimeUnit.MILLISECONDS);
-        LOGGER.trace("token cleanup will occur every " + TimeDuration.asCompactString(cleanerFrequency));
-
-        final String counterString = pwmApplication.readAppAttribute(PwmApplication.AppAttribute.TOKEN_COUNTER);
-        try {
-            counter = Long.parseLong(counterString);
-        } catch (Exception e) {
-            /* noop */
-        }
-
-        status = STATUS.OPEN;
-        LOGGER.debug("open");
-    }
-
-    public boolean supportsName() {
-        return tokenMachine.supportsName();
-    }
-
-    public String generateNewToken(final TokenPayload tokenPayload, final SessionLabel sessionLabel)
-            throws PwmUnrecoverableException, PwmOperationalException
-    {
-        checkStatus();
-
-        final String tokenKey;
-        try {
-            tokenKey = tokenMachine.generateToken(sessionLabel, tokenPayload);
-            tokenMachine.storeToken(tokenKey, tokenPayload);
-        } catch (PwmException e) {
-            final String errorMsg = "unexpected error trying to store token in datastore: " + e.getMessage();
-            final ErrorInformation errorInformation = new ErrorInformation(e.getError(),errorMsg);
-            throw new PwmOperationalException(errorInformation);
-        }
-
-        pwmApplication.getAuditManager().submit(pwmApplication.getAuditManager().createUserAuditRecord(
-                AuditEvent.TOKEN_ISSUED,
-                tokenPayload.getUserIdentity(),
-                sessionLabel,
-                JsonUtil.serialize(tokenPayload)
-        ));
-
-        return tokenKey;
-    }
-
-
-    public void markTokenAsClaimed(final String tokenKey, final PwmSession pwmSession) throws PwmUnrecoverableException {
-
-        final TokenPayload tokenPayload;
-        try {
-            tokenPayload = retrieveTokenData(tokenKey);
-        } catch (PwmOperationalException e) {
-            return;
-        }
-
-        if (tokenPayload == null || tokenPayload.getUserIdentity() == null) {
-            return;
-        }
-
-        pwmApplication.getAuditManager().submit(pwmApplication.getAuditManager().createUserAuditRecord(
-                AuditEvent.TOKEN_CLAIMED,
-                tokenPayload.getUserIdentity(),
-                pwmSession.getLabel(),
-                JsonUtil.serialize(tokenPayload)
-        ));
-
-        StatisticsManager.incrementStat(pwmApplication, Statistic.TOKENS_PASSSED);
-    }
-
-    public TokenPayload retrieveTokenData(final String tokenKey)
-            throws PwmOperationalException
-    {
-        checkStatus();
-
-        try {
-            final TokenPayload storedToken = tokenMachine.retrieveToken(tokenKey);
-            if (storedToken != null) {
-
-                if (testIfTokenIsExpired(storedToken)) {
-                    throw new PwmOperationalException(new ErrorInformation(PwmError.ERROR_TOKEN_EXPIRED));
-                }
-
-                if (testIfTokenIsPurgable(storedToken)) {
-                    tokenMachine.removeToken(tokenKey);
-                }
-
-                return storedToken;
-            }
-        } catch (PwmException e) {
-            if (e.getError() == PwmError.ERROR_TOKEN_EXPIRED || e.getError() == PwmError.ERROR_TOKEN_INCORRECT || e.getError() == PwmError.ERROR_TOKEN_MISSING_CONTACT) {
-                throw new PwmOperationalException(e.getErrorInformation());
-            }
-            final String errorMsg = "error trying to retrieve token from datastore: " + e.getMessage();
-            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_INCORRECT,errorMsg);
-            throw new PwmOperationalException(errorInformation);
-        }
-        return null;
-    }
-
-    public STATUS status() {
-        return status;
-    }
-
-    public void close() {
-        if (executorService != null) {
-            executorService.shutdown();
-        }
-        status = STATUS.CLOSED;
-    }
-
-    public List<HealthRecord> healthCheck() {
-        final List<HealthRecord> returnRecords = new ArrayList<>();
-
-        if (tokensAreUsedInConfig(configuration)) {
-            if (errorInformation != null) {
-                returnRecords.add(HealthRecord.forMessage(HealthMessage.CryptoTokenWithNewUserVerification,errorInformation.toDebugStr()));
-            }
-        }
-
-        if (storageMethod == TokenStorageMethod.STORE_LDAP) {
-            if (configuration.readSettingAsBoolean(PwmSetting.NEWUSER_ENABLE)) {
-                for (final NewUserProfile newUserProfile : configuration.getNewUserProfiles().values()) {
-                    if (newUserProfile.readSettingAsBoolean(PwmSetting.NEWUSER_EMAIL_VERIFICATION)) {
-                        final String label = PwmSetting.NEWUSER_EMAIL_VERIFICATION.toMenuLocationDebug(newUserProfile.getIdentifier(),PwmConstants.DEFAULT_LOCALE);
-                        final String label2 = PwmSetting.TOKEN_STORAGEMETHOD.toMenuLocationDebug(null,PwmConstants.DEFAULT_LOCALE);
-                        returnRecords.add(HealthRecord.forMessage(HealthMessage.CryptoTokenWithNewUserVerification, label, label2));
-                    }
-                    if (newUserProfile.readSettingAsBoolean(PwmSetting.NEWUSER_SMS_VERIFICATION)) {
-                        final String label = PwmSetting.NEWUSER_SMS_VERIFICATION.toMenuLocationDebug(newUserProfile.getIdentifier(),PwmConstants.DEFAULT_LOCALE);
-                        final String label2 = PwmSetting.TOKEN_STORAGEMETHOD.toMenuLocationDebug(null,PwmConstants.DEFAULT_LOCALE);
-                        returnRecords.add(HealthRecord.forMessage(HealthMessage.CryptoTokenWithNewUserVerification, label, label2));
-                    }
-                }
-            }
-        }
-
-        return returnRecords;
-    }
-
-
-    private boolean testIfTokenIsExpired(final TokenPayload theToken) {
-        if (theToken == null) {
-            return false;
-        }
-        final Date issueDate = theToken.getDate();
-        if (issueDate == null) {
-            LOGGER.error("retrieved token has no issueDate, marking as expired: " + JsonUtil.serialize(theToken));
-            return true;
-        }
-        final TimeDuration duration = new TimeDuration(issueDate,new Date());
-        return duration.isLongerThan(maxTokenAgeMS);
-    }
-
-    private boolean testIfTokenIsPurgable(final TokenPayload theToken) {
-        if (theToken == null) {
-            return false;
-        }
-        final Date issueDate = theToken.getDate();
-        if (issueDate == null) {
-            LOGGER.error("retrieved token has no issueDate, marking as purgable: " + JsonUtil.serialize(theToken));
-            return true;
-        }
-        final TimeDuration duration = new TimeDuration(issueDate,new Date());
-        return duration.isLongerThan(maxTokenPurgeAgeMS);
-    }
-
-
-    void purgeOutdatedTokens() throws
-            PwmUnrecoverableException, PwmOperationalException
-    {
-        final long startTime = System.currentTimeMillis();
-        int cleanedTokens = 0;
-        List<String> tempKeyList = new ArrayList<>();
-        final int purgeBatchSize = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.TOKEN_PURGE_BATCH_SIZE));
-        tempKeyList.addAll(discoverPurgeableTokenKeys(purgeBatchSize));
-        while (status() == STATUS.OPEN && !tempKeyList.isEmpty()) {
-            for (final String loopKey : tempKeyList) {
-                tokenMachine.removeToken(loopKey);
-            }
-            cleanedTokens = cleanedTokens + tempKeyList.size();
-            tempKeyList.clear();
-        }
-        if (cleanedTokens > 0) {
-            LOGGER.trace("cleaner thread removed " + cleanedTokens + " tokens in " + TimeDuration.fromCurrent(startTime).asCompactString());
-        }
-    }
-
-    private List<String> discoverPurgeableTokenKeys(final int maxCount)
-            throws PwmUnrecoverableException, PwmOperationalException
-    {
-        final List<String> returnList = new ArrayList<>();
-        Iterator<String> keyIterator = null;
-
-        try {
-            keyIterator = tokenMachine.keyIterator();
-
-            while (status() == STATUS.OPEN && returnList.size() < maxCount && keyIterator.hasNext()) {
-                final String loopKey = keyIterator.next();
-                final TokenPayload loopInfo = tokenMachine.retrieveToken(loopKey);
-                if (loopInfo != null) {
-                    if (testIfTokenIsPurgable(loopInfo)) {
-                        returnList.add(loopKey);
-                    }
-                }
-            }
-        } catch (Exception e) {
-            LOGGER.error("unexpected error while cleaning expired stored tokens: " + e.getMessage());
-        } finally {
-            if (keyIterator != null && storageMethod == TokenStorageMethod.STORE_LOCALDB) {
-                try {((LocalDB.LocalDBIterator)keyIterator).close(); } catch (Exception e) {LOGGER.error("unexpected error returning LocalDB token DB iterator: " + e.getMessage());}
-            }
-        }
-
-        return returnList;
-    }
-
-
-    private static String makeRandomCode(final Configuration config) {
-        final String RANDOM_CHARS = config.readSettingAsString(PwmSetting.TOKEN_CHARACTERS);
-        final int CODE_LENGTH = (int) config.readSettingAsLong(PwmSetting.TOKEN_LENGTH);
-        final PwmRandom RANDOM = PwmRandom.getInstance();
-
-        return RANDOM.alphaNumericString(RANDOM_CHARS, CODE_LENGTH);
-    }
-
-    private class CleanerTask extends TimerTask {
-        public void run() {
-            try {
-                tokenMachine.cleanup();
-            } catch (Exception e) {
-                LOGGER.warn("unexpected error while cleaning expired stored tokens: " + e.getMessage(),e);
-            }
-        }
-    }
-
-    private void checkStatus() throws PwmOperationalException {
-        if (status != STATUS.OPEN) {
-            throw new PwmOperationalException(new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,"token manager is not open"));
-        }
-    }
-
-    public int size() throws PwmUnrecoverableException {
-        if (status != STATUS.OPEN) {
-            return -1;
-        }
-
-        try {
-            return tokenMachine.size();
-        } catch (Exception e) {
-            LOGGER.error("unexpected error reading size of token storage table: " + e.getMessage());
-        }
-
-        return -1;
-    }
-
-    String makeUniqueTokenForMachine(final SessionLabel sessionLabel, final TokenMachine machine)
-            throws PwmUnrecoverableException, PwmOperationalException
-    {
-        String tokenKey = null;
-        int attempts = 0;
-        final int maxUniqueCreateAttempts = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.TOKEN_MAX_UNIQUE_CREATE_ATTEMPTS));
-        while (tokenKey == null && attempts < maxUniqueCreateAttempts) {
-            tokenKey = makeRandomCode(configuration);
-            LOGGER.trace(sessionLabel, "generated new token random code, checking for uniqueness");
-            if (machine.retrieveToken(tokenKey) != null) {
-                tokenKey = null;
-            }
-            attempts++;
-        }
-
-        if (tokenKey == null) {
-            throw new PwmOperationalException(new ErrorInformation(PwmError.ERROR_UNKNOWN,"unable to generate a unique token key after " + attempts + " attempts"));
-        }
-
-        LOGGER.trace(sessionLabel, "found new, unique, random token value");
-        return tokenKey;
-    }
-
-    static String makeTokenHash(final String tokenKey) throws PwmUnrecoverableException {
-        return SecureEngine.md5sum(tokenKey) + "-hash";
-    }
-
-    private static boolean tokensAreUsedInConfig(final Configuration configuration) {
-        if (configuration.readSettingAsBoolean(PwmSetting.NEWUSER_ENABLE)) {
-            for (final NewUserProfile newUserProfile : configuration.getNewUserProfiles().values()) {
-                if (newUserProfile.readSettingAsBoolean(PwmSetting.NEWUSER_EMAIL_VERIFICATION)) {
-                    return true;
-                }
-            }
-            return true;
-        }
-
-        if (configuration.readSettingAsBoolean(PwmSetting.ACTIVATE_USER_ENABLE) &&
-                MessageSendMethod.NONE != configuration.readSettingAsTokenSendMethod(PwmSetting.ACTIVATE_TOKEN_SEND_METHOD)) {
-            return true;
-        }
-
-        if (configuration.readSettingAsBoolean(PwmSetting.CHALLENGE_ENABLE)) {
-            for (final ForgottenPasswordProfile forgottenPasswordProfile : configuration.getForgottenPasswordProfiles().values()) {
-                final MessageSendMethod messageSendMethod = forgottenPasswordProfile.readSettingAsEnum(PwmSetting.RECOVERY_TOKEN_SEND_METHOD, MessageSendMethod.class);
-                if (messageSendMethod != null && messageSendMethod != MessageSendMethod.NONE) {
-                    return true;
-                }
-            }
-        }
-        return false;
-    }
-
-    String toEncryptedString(final TokenPayload tokenPayload)
-            throws PwmUnrecoverableException, PwmOperationalException
-    {
-        final String jsonPayload = JsonUtil.serialize(tokenPayload);
-        return pwmApplication.getSecureService().encryptToString(jsonPayload);
-    }
-
-    TokenPayload fromEncryptedString(final String inputString)
-            throws PwmOperationalException, PwmUnrecoverableException
-    {
-        final String deWhiteSpacedToken = inputString.replaceAll("\\s","");
-        try {
-            final String decryptedString = pwmApplication.getSecureService().decryptStringValue(deWhiteSpacedToken);
-            return JsonUtil.deserialize(decryptedString, TokenPayload.class);
-        } catch (PwmUnrecoverableException e) {
-            final String errorMsg = "unable to decrypt user supplied token value: " + e.getErrorInformation().toDebugStr();
-            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_INCORRECT,errorMsg);
-            throw new PwmOperationalException(errorInformation);
-        }
-    }
-
-    public ServiceInfo serviceInfo()
-    {
-        return serviceInfo;
-    }
-
-    public TokenPayload processUserEnteredCode(
-            final PwmSession pwmSession,
-            final UserIdentity sessionUserIdentity,
-            final TokenType tokenType,
-            final String userEnteredCode
-    )
-            throws PwmOperationalException, PwmUnrecoverableException
-    {
-        try {
-            final TokenPayload tokenPayload = processUserEnteredCodeImpl(
-                    pwmApplication,
-                    pwmSession,
-                    sessionUserIdentity,
-                    tokenType,
-                    userEnteredCode
-            );
-            if (tokenPayload.getDest() != null) {
-                for (final String dest : tokenPayload.getDest()) {
-                    pwmApplication.getIntruderManager().clear(RecordType.TOKEN_DEST, dest);
-                }
-            }
-            pwmApplication.getTokenService().markTokenAsClaimed(userEnteredCode, pwmSession);
-            return tokenPayload;
-        } catch (Exception e) {
-            final ErrorInformation errorInformation;
-            if (e instanceof PwmException) {
-                errorInformation = ((PwmException) e).getErrorInformation();
-            } else {
-                errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_INCORRECT, e.getMessage());
-            }
-
-            LOGGER.debug(pwmSession, errorInformation.toDebugStr());
-
-            if (sessionUserIdentity != null) {
-                SessionAuthenticator sessionAuthenticator = new SessionAuthenticator(pwmApplication, pwmSession, null);
-                sessionAuthenticator.simulateBadPassword(sessionUserIdentity);
-                pwmApplication.getIntruderManager().convenience().markUserIdentity(sessionUserIdentity, pwmSession);
-            }
-            pwmApplication.getIntruderManager().convenience().markAddressAndSession(pwmSession);
-            pwmApplication.getStatisticsManager().incrementValue(Statistic.RECOVERY_FAILURES);
-            throw new PwmOperationalException(errorInformation);
-        }
-    }
-
-    private static TokenPayload processUserEnteredCodeImpl(
-            final PwmApplication pwmApplication,
-            final PwmSession pwmSession,
-            final UserIdentity sessionUserIdentity,
-            final TokenType tokenType,
-            final String userEnteredCode
-    )
-            throws PwmOperationalException, PwmUnrecoverableException
-    {
-        TokenPayload tokenPayload;
-        try {
-            tokenPayload = pwmApplication.getTokenService().retrieveTokenData(userEnteredCode);
-        } catch (PwmOperationalException e) {
-            final String errorMsg = "unexpected error attempting to read token from storage: " + e.getErrorInformation().toDebugStr();
-            throw new PwmOperationalException(PwmError.ERROR_TOKEN_INCORRECT,errorMsg);
-        }
-
-        if (tokenPayload == null) {
-            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_INCORRECT,"token not found");
-            throw new PwmOperationalException(errorInformation);
-        }
-
-        LOGGER.trace(pwmSession, "retrieved tokenPayload: " + JsonUtil.serialize(tokenPayload));
-
-        if (tokenType != null && pwmApplication.getTokenService().supportsName()) {
-            if (!tokenType.matchesName(tokenPayload.getName()) ) {
-                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_INCORRECT,"incorrect token/name format");
-                throw new PwmOperationalException(errorInformation);
-            }
-        }
-
-        // check current session identity
-        if (tokenPayload.getUserIdentity() != null && sessionUserIdentity != null) {
-            if (!tokenPayload.getUserIdentity().canonicalEquals(sessionUserIdentity, pwmApplication)) {
-                final String errorMsg = "user in session '" + sessionUserIdentity + "' entered code for user '" + tokenPayload.getUserIdentity()+ "', counting as invalid attempt";
-                throw new PwmOperationalException(PwmError.ERROR_TOKEN_INCORRECT,errorMsg);
-            }
-        }
-
-        // check if password-last-modified is same as when tried to read it before.
-        if (tokenPayload.getUserIdentity() != null && tokenPayload.getData() != null && tokenPayload.getData().containsKey(PwmConstants.TOKEN_KEY_PWD_CHG_DATE)) {
-            try {
-                final Date userLastPasswordChange = PasswordUtility.determinePwdLastModified(
-                        pwmApplication,
-                        pwmSession.getLabel(),
-                        tokenPayload.getUserIdentity());
-                final String dateStringInToken = tokenPayload.getData().get(PwmConstants.TOKEN_KEY_PWD_CHG_DATE);
-                if (userLastPasswordChange != null && dateStringInToken != null) {
-                    final String userChangeString = PwmConstants.DEFAULT_DATETIME_FORMAT.format(userLastPasswordChange);
-                    if (!dateStringInToken.equalsIgnoreCase(userChangeString)) {
-                        final String errorString = "user password has changed since token issued, token rejected";
-                        final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_EXPIRED, errorString);
-                        throw new PwmOperationalException(errorInformation);
-                    }
-                }
-            } catch (ChaiUnavailableException | PwmUnrecoverableException e) {
-                final String errorMsg = "unexpected error reading user's last password change time while validating token: " + e.getMessage();
-                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_INCORRECT, errorMsg);
-                throw new PwmOperationalException(errorInformation);
-            }
-        }
-
-        LOGGER.debug(pwmSession, "token validation has been passed");
-        return tokenPayload;
-    }
-
-    public static class TokenSender {
-        public static void sendToken(
-                final PwmApplication pwmApplication,
-                final UserInfoBean userInfoBean,
-                final MacroMachine macroMachine,
-                final EmailItemBean configuredEmailSetting,
-                final MessageSendMethod tokenSendMethod,
-                final String emailAddress,
-                final String smsNumber,
-                final String smsMessage,
-                final String tokenKey
-        )
-                throws PwmUnrecoverableException
-        {
-            final boolean success;
-
-            try {
-                switch (tokenSendMethod) {
-                    case NONE:
-                        // should never read here
-                        LOGGER.error("attempt to send token to destination type 'NONE'");
-                        throw new PwmUnrecoverableException(PwmError.ERROR_UNKNOWN);
-                    case BOTH:
-                        // Send both email and SMS, success if one of both succeeds
-                        final boolean suc1 = sendEmailToken(pwmApplication, userInfoBean, macroMachine, configuredEmailSetting, emailAddress, tokenKey);
-                        final boolean suc2 = sendSmsToken(pwmApplication, userInfoBean, macroMachine, smsNumber, smsMessage, tokenKey);
-                        success = suc1 || suc2;
-                        break;
-                    case EMAILFIRST:
-                        // Send email first, try SMS if email is not available
-                        success = sendEmailToken(pwmApplication, userInfoBean, macroMachine, configuredEmailSetting, emailAddress, tokenKey) ||
-                                sendSmsToken(pwmApplication, userInfoBean, macroMachine, smsNumber, smsMessage, tokenKey);
-                        break;
-                    case SMSFIRST:
-                        // Send SMS first, try email if SMS is not available
-                        success = sendSmsToken(pwmApplication, userInfoBean, macroMachine, smsNumber, smsMessage, tokenKey) ||
-                                sendEmailToken(pwmApplication, userInfoBean, macroMachine, configuredEmailSetting, emailAddress, tokenKey);
-                        break;
-                    case SMSONLY:
-                        // Only try SMS
-                        success = sendSmsToken(pwmApplication, userInfoBean, macroMachine, smsNumber, smsMessage, tokenKey);
-                        break;
-                    case EMAILONLY:
-                    default:
-                        // Only try email
-                        success = sendEmailToken(pwmApplication, userInfoBean, macroMachine, configuredEmailSetting, emailAddress, tokenKey);
-                        break;
-                }
-            } catch (ChaiUnavailableException e) {
-                throw new PwmUnrecoverableException(PwmError.forChaiError(e.getErrorCode()));
-            }
-
-            if (!success) {
-                throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_TOKEN_MISSING_CONTACT));
-            }
-            pwmApplication.getStatisticsManager().incrementValue(Statistic.TOKENS_SENT);
-        }
-
-        public static boolean sendEmailToken(
-                final PwmApplication pwmApplication,
-                final UserInfoBean userInfoBean,
-                final MacroMachine macroMachine,
-                final EmailItemBean configuredEmailSetting,
-                final String toAddress,
-                final String tokenKey
-        )
-                throws PwmUnrecoverableException, ChaiUnavailableException
-        {
-            if (toAddress == null || toAddress.length() < 1) {
-                return false;
-            }
-
-            pwmApplication.getIntruderManager().mark(RecordType.TOKEN_DEST, toAddress, null);
-
-            pwmApplication.getEmailQueue().submitEmail(new EmailItemBean(
-                    toAddress,
-                    configuredEmailSetting.getFrom(),
-                    configuredEmailSetting.getSubject(),
-                    configuredEmailSetting.getBodyPlain().replace("%TOKEN%", tokenKey),
-                    configuredEmailSetting.getBodyHtml().replace("%TOKEN%", tokenKey)
-            ), userInfoBean, macroMachine);
-            LOGGER.debug("token email added to send queue for " + toAddress);
-            return true;
-        }
-
-        public static boolean sendSmsToken(
-                final PwmApplication pwmApplication,
-                final UserInfoBean userInfoBean,
-                final MacroMachine macroMachine,
-                final String smsNumber,
-                final String smsMessage,
-                final String tokenKey
-        )
-                throws PwmUnrecoverableException, ChaiUnavailableException
-        {
-            final Configuration config = pwmApplication.getConfig();
-
-            if (smsNumber == null || smsNumber.length() < 1) {
-                return false;
-            }
-
-            final String modifiedMessage = smsMessage.replaceAll("%TOKEN%", tokenKey);
-
-            pwmApplication.getIntruderManager().mark(RecordType.TOKEN_DEST, smsNumber, null);
-
-            pwmApplication.sendSmsUsingQueue(new SmsItemBean(smsNumber, modifiedMessage), macroMachine);
-            LOGGER.debug("token SMS added to send queue for " + smsNumber);
-            return true;
-        }
-    }
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.token;
+
+import com.novell.ldapchai.exception.ChaiUnavailableException;
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.PwmConstants;
+import password.pwm.bean.*;
+import password.pwm.config.Configuration;
+import password.pwm.config.PwmSetting;
+import password.pwm.config.option.DataStorageMethod;
+import password.pwm.config.option.MessageSendMethod;
+import password.pwm.config.option.TokenStorageMethod;
+import password.pwm.config.profile.ForgottenPasswordProfile;
+import password.pwm.config.profile.NewUserProfile;
+import password.pwm.error.*;
+import password.pwm.health.HealthMessage;
+import password.pwm.health.HealthRecord;
+import password.pwm.http.PwmSession;
+import password.pwm.ldap.auth.SessionAuthenticator;
+import password.pwm.svc.PwmService;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.intruder.RecordType;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.svc.stats.StatisticsManager;
+import password.pwm.util.Helper;
+import password.pwm.util.JsonUtil;
+import password.pwm.util.TimeDuration;
+import password.pwm.util.localdb.LocalDB;
+import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.macro.MacroMachine;
+import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.secure.PwmRandom;
+import password.pwm.util.secure.SecureEngine;
+
+import java.util.*;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This PWM service is responsible for reading/writing tokens used for forgotten password,
+ * new user registration, account activation, and other functions.  Several implementations
+ * of the backing storage method are available.
+ *
+ * @author jrivard@gmail.com
+ */
+public class TokenService implements PwmService {
+
+    private static final PwmLogger LOGGER = PwmLogger.forClass(TokenService.class);
+
+    private static final long MAX_CLEANER_INTERVAL_MS = 24 * 60 * 60 * 1000; // one day
+    private static final long MIN_CLEANER_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
+
+    private ScheduledExecutorService executorService;
+
+    private PwmApplication pwmApplication;
+    private Configuration configuration;
+    private TokenStorageMethod storageMethod;
+    private long maxTokenAgeMS;
+    private long maxTokenPurgeAgeMS;
+    private TokenMachine tokenMachine;
+    private long counter;
+
+    private ServiceInfo serviceInfo = new ServiceInfo(Collections.<DataStorageMethod>emptyList());
+    private STATUS status = STATUS.NEW;
+
+    private ErrorInformation errorInformation = null;
+
+    public TokenService()
+            throws PwmOperationalException
+    {
+    }
+
+    public synchronized TokenPayload createTokenPayload(final TokenType name, final Map<String, String> data, final UserIdentity userIdentity, final Set<String> dest) {
+        final long count = counter++;
+        final StringBuilder guid = new StringBuilder();
+        try {
+            guid.append(SecureEngine.md5sum(pwmApplication.getInstanceID() + pwmApplication.getStartupTime().toString()));
+            guid.append("-");
+            guid.append(count);
+        } catch (Exception e) {
+            LOGGER.error("error making payload guid: " + e.getMessage(),e);
+        }
+        return new TokenPayload(name.name(), data, userIdentity, dest, guid.toString());
+    }
+
+    public void init(final PwmApplication pwmApplication)
+            throws PwmException
+    {
+        LOGGER.trace("opening");
+        status = STATUS.OPENING;
+
+        this.pwmApplication = pwmApplication;
+        this.configuration = pwmApplication.getConfig();
+
+        storageMethod = configuration.getTokenStorageMethod();
+        if (storageMethod == null) {
+            final String errorMsg = "no storage method specified";
+            errorInformation = new ErrorInformation(PwmError.ERROR_INVALID_CONFIG,errorMsg);
+            status = STATUS.CLOSED;
+            throw new PwmOperationalException(errorInformation);
+        }
+        try {
+            maxTokenAgeMS = configuration.readSettingAsLong(PwmSetting.TOKEN_LIFETIME) * 1000;
+            maxTokenPurgeAgeMS = maxTokenAgeMS + Long.parseLong(configuration.readAppProperty(AppProperty.TOKEN_REMOVAL_DELAY_MS));
+        } catch (Exception e) {
+            final String errorMsg = "unable to parse max token age value: " + e.getMessage();
+            errorInformation = new ErrorInformation(PwmError.ERROR_INVALID_CONFIG,errorMsg);
+            status = STATUS.CLOSED;
+            throw new PwmOperationalException(errorInformation);
+        }
+
+        try {
+            DataStorageMethod usedStorageMethod = null;
+            switch (storageMethod) {
+                case STORE_LOCALDB:
+                    tokenMachine = new LocalDBTokenMachine(this, pwmApplication.getLocalDB());
+                    usedStorageMethod = DataStorageMethod.LOCALDB;
+                    break;
+
+                case STORE_DB:
+                    tokenMachine = new DBTokenMachine(this, pwmApplication.getDatabaseAccessor());
+                    usedStorageMethod = DataStorageMethod.DB;
+                    break;
+
+                case STORE_CRYPTO:
+                    tokenMachine = new CryptoTokenMachine(this);
+                    usedStorageMethod = DataStorageMethod.CRYPTO;
+                    break;
+
+                case STORE_LDAP:
+                    tokenMachine = new LdapTokenMachine(this, pwmApplication);
+                    usedStorageMethod = DataStorageMethod.LDAP;
+                    break;
+            }
+            serviceInfo = new ServiceInfo(Collections.singletonList(usedStorageMethod));
+        } catch (PwmException e) {
+            final String errorMsg = "unable to start token manager: " + e.getErrorInformation().getDetailedErrorMsg();
+            final ErrorInformation newErrorInformation = new ErrorInformation(e.getError(), errorMsg);
+            errorInformation = newErrorInformation;
+            LOGGER.error(newErrorInformation.toDebugStr());
+            status = STATUS.CLOSED;
+            return;
+        }
+
+        executorService = Executors.newSingleThreadScheduledExecutor(
+                Helper.makePwmThreadFactory(
+                        Helper.makeThreadName(pwmApplication, this.getClass()) + "-",
+                        true
+                ));
+
+        final TimerTask cleanerTask = new CleanerTask();
+
+        final long cleanerFrequency = (maxTokenAgeMS*0.5) > MAX_CLEANER_INTERVAL_MS ? MAX_CLEANER_INTERVAL_MS : (maxTokenAgeMS*0.5) < MIN_CLEANER_INTERVAL_MS ? MIN_CLEANER_INTERVAL_MS : (long)(maxTokenAgeMS*0.5);
+        executorService.scheduleAtFixedRate(cleanerTask, 10 * 1000, cleanerFrequency, TimeUnit.MILLISECONDS);
+        LOGGER.trace("token cleanup will occur every " + TimeDuration.asCompactString(cleanerFrequency));
+
+        final String counterString = pwmApplication.readAppAttribute(PwmApplication.AppAttribute.TOKEN_COUNTER);
+        try {
+            counter = Long.parseLong(counterString);
+        } catch (Exception e) {
+            /* noop */
+        }
+
+        status = STATUS.OPEN;
+        LOGGER.debug("open");
+    }
+
+    public boolean supportsName() {
+        return tokenMachine.supportsName();
+    }
+
+    public String generateNewToken(final TokenPayload tokenPayload, final SessionLabel sessionLabel)
+            throws PwmUnrecoverableException, PwmOperationalException
+    {
+        checkStatus();
+
+        final String tokenKey;
+        try {
+            tokenKey = tokenMachine.generateToken(sessionLabel, tokenPayload);
+            tokenMachine.storeToken(tokenKey, tokenPayload);
+        } catch (PwmException e) {
+            final String errorMsg = "unexpected error trying to store token in datastore: " + e.getMessage();
+            final ErrorInformation errorInformation = new ErrorInformation(e.getError(),errorMsg);
+            throw new PwmOperationalException(errorInformation);
+        }
+
+        pwmApplication.getAuditManager().submit(pwmApplication.getAuditManager().createUserAuditRecord(
+                AuditEvent.TOKEN_ISSUED,
+                tokenPayload.getUserIdentity(),
+                sessionLabel,
+                JsonUtil.serialize(tokenPayload)
+        ));
+
+        return tokenKey;
+    }
+
+
+    public void markTokenAsClaimed(final String tokenKey, final PwmSession pwmSession) throws PwmUnrecoverableException {
+
+        final TokenPayload tokenPayload;
+        try {
+            tokenPayload = retrieveTokenData(tokenKey);
+        } catch (PwmOperationalException e) {
+            return;
+        }
+
+        if (tokenPayload == null || tokenPayload.getUserIdentity() == null) {
+            return;
+        }
+
+        pwmApplication.getAuditManager().submit(pwmApplication.getAuditManager().createUserAuditRecord(
+                AuditEvent.TOKEN_CLAIMED,
+                tokenPayload.getUserIdentity(),
+                pwmSession.getLabel(),
+                JsonUtil.serialize(tokenPayload)
+        ));
+
+        StatisticsManager.incrementStat(pwmApplication, Statistic.TOKENS_PASSSED);
+    }
+
+    public TokenPayload retrieveTokenData(final String tokenKey)
+            throws PwmOperationalException
+    {
+        checkStatus();
+
+        try {
+            final TokenPayload storedToken = tokenMachine.retrieveToken(tokenKey);
+            if (storedToken != null) {
+
+                if (testIfTokenIsExpired(storedToken)) {
+                    throw new PwmOperationalException(new ErrorInformation(PwmError.ERROR_TOKEN_EXPIRED));
+                }
+
+                if (testIfTokenIsPurgable(storedToken)) {
+                    tokenMachine.removeToken(tokenKey);
+                }
+
+                return storedToken;
+            }
+        } catch (PwmException e) {
+            if (e.getError() == PwmError.ERROR_TOKEN_EXPIRED || e.getError() == PwmError.ERROR_TOKEN_INCORRECT || e.getError() == PwmError.ERROR_TOKEN_MISSING_CONTACT) {
+                throw new PwmOperationalException(e.getErrorInformation());
+            }
+            final String errorMsg = "error trying to retrieve token from datastore: " + e.getMessage();
+            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_INCORRECT,errorMsg);
+            throw new PwmOperationalException(errorInformation);
+        }
+        return null;
+    }
+
+    public STATUS status() {
+        return status;
+    }
+
+    public void close() {
+        if (executorService != null) {
+            executorService.shutdown();
+        }
+        status = STATUS.CLOSED;
+    }
+
+    public List<HealthRecord> healthCheck() {
+        final List<HealthRecord> returnRecords = new ArrayList<>();
+
+        if (tokensAreUsedInConfig(configuration)) {
+            if (errorInformation != null) {
+                returnRecords.add(HealthRecord.forMessage(HealthMessage.CryptoTokenWithNewUserVerification,errorInformation.toDebugStr()));
+            }
+        }
+
+        if (storageMethod == TokenStorageMethod.STORE_LDAP) {
+            if (configuration.readSettingAsBoolean(PwmSetting.NEWUSER_ENABLE)) {
+                for (final NewUserProfile newUserProfile : configuration.getNewUserProfiles().values()) {
+                    if (newUserProfile.readSettingAsBoolean(PwmSetting.NEWUSER_EMAIL_VERIFICATION)) {
+                        final String label = PwmSetting.NEWUSER_EMAIL_VERIFICATION.toMenuLocationDebug(newUserProfile.getIdentifier(),PwmConstants.DEFAULT_LOCALE);
+                        final String label2 = PwmSetting.TOKEN_STORAGEMETHOD.toMenuLocationDebug(null,PwmConstants.DEFAULT_LOCALE);
+                        returnRecords.add(HealthRecord.forMessage(HealthMessage.CryptoTokenWithNewUserVerification, label, label2));
+                    }
+                    if (newUserProfile.readSettingAsBoolean(PwmSetting.NEWUSER_SMS_VERIFICATION)) {
+                        final String label = PwmSetting.NEWUSER_SMS_VERIFICATION.toMenuLocationDebug(newUserProfile.getIdentifier(),PwmConstants.DEFAULT_LOCALE);
+                        final String label2 = PwmSetting.TOKEN_STORAGEMETHOD.toMenuLocationDebug(null,PwmConstants.DEFAULT_LOCALE);
+                        returnRecords.add(HealthRecord.forMessage(HealthMessage.CryptoTokenWithNewUserVerification, label, label2));
+                    }
+                }
+            }
+        }
+
+        return returnRecords;
+    }
+
+
+    private boolean testIfTokenIsExpired(final TokenPayload theToken) {
+        if (theToken == null) {
+            return false;
+        }
+        final Date issueDate = theToken.getDate();
+        if (issueDate == null) {
+            LOGGER.error("retrieved token has no issueDate, marking as expired: " + JsonUtil.serialize(theToken));
+            return true;
+        }
+        final TimeDuration duration = new TimeDuration(issueDate,new Date());
+        return duration.isLongerThan(maxTokenAgeMS);
+    }
+
+    private boolean testIfTokenIsPurgable(final TokenPayload theToken) {
+        if (theToken == null) {
+            return false;
+        }
+        final Date issueDate = theToken.getDate();
+        if (issueDate == null) {
+            LOGGER.error("retrieved token has no issueDate, marking as purgable: " + JsonUtil.serialize(theToken));
+            return true;
+        }
+        final TimeDuration duration = new TimeDuration(issueDate,new Date());
+        return duration.isLongerThan(maxTokenPurgeAgeMS);
+    }
+
+
+    void purgeOutdatedTokens() throws
+            PwmUnrecoverableException, PwmOperationalException
+    {
+        final long startTime = System.currentTimeMillis();
+        int cleanedTokens = 0;
+        List<String> tempKeyList = new ArrayList<>();
+        final int purgeBatchSize = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.TOKEN_PURGE_BATCH_SIZE));
+        tempKeyList.addAll(discoverPurgeableTokenKeys(purgeBatchSize));
+        while (status() == STATUS.OPEN && !tempKeyList.isEmpty()) {
+            for (final String loopKey : tempKeyList) {
+                tokenMachine.removeToken(loopKey);
+            }
+            cleanedTokens = cleanedTokens + tempKeyList.size();
+            tempKeyList.clear();
+        }
+        if (cleanedTokens > 0) {
+            LOGGER.trace("cleaner thread removed " + cleanedTokens + " tokens in " + TimeDuration.fromCurrent(startTime).asCompactString());
+        }
+    }
+
+    private List<String> discoverPurgeableTokenKeys(final int maxCount)
+            throws PwmUnrecoverableException, PwmOperationalException
+    {
+        final List<String> returnList = new ArrayList<>();
+        Iterator<String> keyIterator = null;
+
+        try {
+            keyIterator = tokenMachine.keyIterator();
+
+            while (status() == STATUS.OPEN && returnList.size() < maxCount && keyIterator.hasNext()) {
+                final String loopKey = keyIterator.next();
+                final TokenPayload loopInfo = tokenMachine.retrieveToken(loopKey);
+                if (loopInfo != null) {
+                    if (testIfTokenIsPurgable(loopInfo)) {
+                        returnList.add(loopKey);
+                    }
+                }
+            }
+        } catch (Exception e) {
+            LOGGER.error("unexpected error while cleaning expired stored tokens: " + e.getMessage());
+        } finally {
+            if (keyIterator != null && storageMethod == TokenStorageMethod.STORE_LOCALDB) {
+                try {((LocalDB.LocalDBIterator)keyIterator).close(); } catch (Exception e) {LOGGER.error("unexpected error returning LocalDB token DB iterator: " + e.getMessage());}
+            }
+        }
+
+        return returnList;
+    }
+
+
+    private static String makeRandomCode(final Configuration config) {
+        final String RANDOM_CHARS = config.readSettingAsString(PwmSetting.TOKEN_CHARACTERS);
+        final int CODE_LENGTH = (int) config.readSettingAsLong(PwmSetting.TOKEN_LENGTH);
+        final PwmRandom RANDOM = PwmRandom.getInstance();
+
+        return RANDOM.alphaNumericString(RANDOM_CHARS, CODE_LENGTH);
+    }
+
+    private class CleanerTask extends TimerTask {
+        public void run() {
+            try {
+                tokenMachine.cleanup();
+            } catch (Exception e) {
+                LOGGER.warn("unexpected error while cleaning expired stored tokens: " + e.getMessage(),e);
+            }
+        }
+    }
+
+    private void checkStatus() throws PwmOperationalException {
+        if (status != STATUS.OPEN) {
+            throw new PwmOperationalException(new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,"token manager is not open"));
+        }
+    }
+
+    public int size() throws PwmUnrecoverableException {
+        if (status != STATUS.OPEN) {
+            return -1;
+        }
+
+        try {
+            return tokenMachine.size();
+        } catch (Exception e) {
+            LOGGER.error("unexpected error reading size of token storage table: " + e.getMessage());
+        }
+
+        return -1;
+    }
+
+    String makeUniqueTokenForMachine(final SessionLabel sessionLabel, final TokenMachine machine)
+            throws PwmUnrecoverableException, PwmOperationalException
+    {
+        String tokenKey = null;
+        int attempts = 0;
+        final int maxUniqueCreateAttempts = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.TOKEN_MAX_UNIQUE_CREATE_ATTEMPTS));
+        while (tokenKey == null && attempts < maxUniqueCreateAttempts) {
+            tokenKey = makeRandomCode(configuration);
+            LOGGER.trace(sessionLabel, "generated new token random code, checking for uniqueness");
+            if (machine.retrieveToken(tokenKey) != null) {
+                tokenKey = null;
+            }
+            attempts++;
+        }
+
+        if (tokenKey == null) {
+            throw new PwmOperationalException(new ErrorInformation(PwmError.ERROR_UNKNOWN,"unable to generate a unique token key after " + attempts + " attempts"));
+        }
+
+        LOGGER.trace(sessionLabel, "found new, unique, random token value");
+        return tokenKey;
+    }
+
+    static String makeTokenHash(final String tokenKey) throws PwmUnrecoverableException {
+        return SecureEngine.md5sum(tokenKey) + "-hash";
+    }
+
+    private static boolean tokensAreUsedInConfig(final Configuration configuration) {
+        if (configuration.readSettingAsBoolean(PwmSetting.NEWUSER_ENABLE)) {
+            for (final NewUserProfile newUserProfile : configuration.getNewUserProfiles().values()) {
+                if (newUserProfile.readSettingAsBoolean(PwmSetting.NEWUSER_EMAIL_VERIFICATION)) {
+                    return true;
+                }
+            }
+            return true;
+        }
+
+        if (configuration.readSettingAsBoolean(PwmSetting.ACTIVATE_USER_ENABLE) &&
+                MessageSendMethod.NONE != configuration.readSettingAsTokenSendMethod(PwmSetting.ACTIVATE_TOKEN_SEND_METHOD)) {
+            return true;
+        }
+
+        if (configuration.readSettingAsBoolean(PwmSetting.CHALLENGE_ENABLE)) {
+            for (final ForgottenPasswordProfile forgottenPasswordProfile : configuration.getForgottenPasswordProfiles().values()) {
+                final MessageSendMethod messageSendMethod = forgottenPasswordProfile.readSettingAsEnum(PwmSetting.RECOVERY_TOKEN_SEND_METHOD, MessageSendMethod.class);
+                if (messageSendMethod != null && messageSendMethod != MessageSendMethod.NONE) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    String toEncryptedString(final TokenPayload tokenPayload)
+            throws PwmUnrecoverableException, PwmOperationalException
+    {
+        final String jsonPayload = JsonUtil.serialize(tokenPayload);
+        return pwmApplication.getSecureService().encryptToString(jsonPayload);
+    }
+
+    TokenPayload fromEncryptedString(final String inputString)
+            throws PwmOperationalException, PwmUnrecoverableException
+    {
+        final String deWhiteSpacedToken = inputString.replaceAll("\\s", "");
+        try {
+            final String decryptedString = pwmApplication.getSecureService().decryptStringValue(deWhiteSpacedToken);
+            return JsonUtil.deserialize(decryptedString, TokenPayload.class);
+        } catch (PwmUnrecoverableException e) {
+            final String errorMsg = "unable to decrypt user supplied token value: " + e.getErrorInformation().toDebugStr();
+            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_INCORRECT,errorMsg);
+            throw new PwmOperationalException(errorInformation);
+        }
+    }
+
+    public ServiceInfo serviceInfo()
+    {
+        return serviceInfo;
+    }
+
+    public TokenPayload processUserEnteredCode(
+            final PwmSession pwmSession,
+            final UserIdentity sessionUserIdentity,
+            final TokenType tokenType,
+            final String userEnteredCode
+    )
+            throws PwmOperationalException, PwmUnrecoverableException
+    {
+        try {
+            final TokenPayload tokenPayload = processUserEnteredCodeImpl(
+                    pwmApplication,
+                    pwmSession,
+                    sessionUserIdentity,
+                    tokenType,
+                    userEnteredCode
+            );
+            if (tokenPayload.getDest() != null) {
+                for (final String dest : tokenPayload.getDest()) {
+                    pwmApplication.getIntruderManager().clear(RecordType.TOKEN_DEST, dest);
+                }
+            }
+            pwmApplication.getTokenService().markTokenAsClaimed(userEnteredCode, pwmSession);
+            return tokenPayload;
+        } catch (Exception e) {
+            final ErrorInformation errorInformation;
+            if (e instanceof PwmException) {
+                errorInformation = ((PwmException) e).getErrorInformation();
+            } else {
+                errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_INCORRECT, e.getMessage());
+            }
+
+            LOGGER.debug(pwmSession, errorInformation.toDebugStr());
+
+            if (sessionUserIdentity != null) {
+                SessionAuthenticator sessionAuthenticator = new SessionAuthenticator(pwmApplication, pwmSession, null);
+                sessionAuthenticator.simulateBadPassword(sessionUserIdentity);
+                pwmApplication.getIntruderManager().convenience().markUserIdentity(sessionUserIdentity, pwmSession);
+            }
+            pwmApplication.getIntruderManager().convenience().markAddressAndSession(pwmSession);
+            pwmApplication.getStatisticsManager().incrementValue(Statistic.RECOVERY_FAILURES);
+            throw new PwmOperationalException(errorInformation);
+        }
+    }
+
+    private static TokenPayload processUserEnteredCodeImpl(
+            final PwmApplication pwmApplication,
+            final PwmSession pwmSession,
+            final UserIdentity sessionUserIdentity,
+            final TokenType tokenType,
+            final String userEnteredCode
+    )
+            throws PwmOperationalException, PwmUnrecoverableException
+    {
+        TokenPayload tokenPayload;
+        try {
+            tokenPayload = pwmApplication.getTokenService().retrieveTokenData(userEnteredCode);
+        } catch (PwmOperationalException e) {
+            final String errorMsg = "unexpected error attempting to read token from storage: " + e.getErrorInformation().toDebugStr();
+            throw new PwmOperationalException(PwmError.ERROR_TOKEN_INCORRECT,errorMsg);
+        }
+
+        if (tokenPayload == null) {
+            final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_INCORRECT,"token not found");
+            throw new PwmOperationalException(errorInformation);
+        }
+
+        LOGGER.trace(pwmSession, "retrieved tokenPayload: " + JsonUtil.serialize(tokenPayload));
+
+        if (tokenType != null && pwmApplication.getTokenService().supportsName()) {
+            if (!tokenType.matchesName(tokenPayload.getName()) ) {
+                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_INCORRECT,"incorrect token/name format");
+                throw new PwmOperationalException(errorInformation);
+            }
+        }
+
+        // check current session identity
+        if (tokenPayload.getUserIdentity() != null && sessionUserIdentity != null) {
+            if (!tokenPayload.getUserIdentity().canonicalEquals(sessionUserIdentity, pwmApplication)) {
+                final String errorMsg = "user in session '" + sessionUserIdentity + "' entered code for user '" + tokenPayload.getUserIdentity()+ "', counting as invalid attempt";
+                throw new PwmOperationalException(PwmError.ERROR_TOKEN_INCORRECT,errorMsg);
+            }
+        }
+
+        // check if password-last-modified is same as when tried to read it before.
+        if (tokenPayload.getUserIdentity() != null && tokenPayload.getData() != null && tokenPayload.getData().containsKey(PwmConstants.TOKEN_KEY_PWD_CHG_DATE)) {
+            try {
+                final Date userLastPasswordChange = PasswordUtility.determinePwdLastModified(
+                        pwmApplication,
+                        pwmSession.getLabel(),
+                        tokenPayload.getUserIdentity());
+                final String dateStringInToken = tokenPayload.getData().get(PwmConstants.TOKEN_KEY_PWD_CHG_DATE);
+                if (userLastPasswordChange != null && dateStringInToken != null) {
+                    final String userChangeString = PwmConstants.DEFAULT_DATETIME_FORMAT.format(userLastPasswordChange);
+                    if (!dateStringInToken.equalsIgnoreCase(userChangeString)) {
+                        final String errorString = "user password has changed since token issued, token rejected";
+                        final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_EXPIRED, errorString);
+                        throw new PwmOperationalException(errorInformation);
+                    }
+                }
+            } catch (ChaiUnavailableException | PwmUnrecoverableException e) {
+                final String errorMsg = "unexpected error reading user's last password change time while validating token: " + e.getMessage();
+                final ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_TOKEN_INCORRECT, errorMsg);
+                throw new PwmOperationalException(errorInformation);
+            }
+        }
+
+        LOGGER.debug(pwmSession, "token validation has been passed");
+        return tokenPayload;
+    }
+
+    public static class TokenSender {
+        public static void sendToken(
+                final PwmApplication pwmApplication,
+                final UserInfoBean userInfoBean,
+                final MacroMachine macroMachine,
+                final EmailItemBean configuredEmailSetting,
+                final MessageSendMethod tokenSendMethod,
+                final String emailAddress,
+                final String smsNumber,
+                final String smsMessage,
+                final String tokenKey
+        )
+                throws PwmUnrecoverableException
+        {
+            final boolean success;
+
+            try {
+                switch (tokenSendMethod) {
+                    case NONE:
+                        // should never read here
+                        LOGGER.error("attempt to send token to destination type 'NONE'");
+                        throw new PwmUnrecoverableException(PwmError.ERROR_UNKNOWN);
+                    case BOTH:
+                        // Send both email and SMS, success if one of both succeeds
+                        final boolean suc1 = sendEmailToken(pwmApplication, userInfoBean, macroMachine, configuredEmailSetting, emailAddress, tokenKey);
+                        final boolean suc2 = sendSmsToken(pwmApplication, userInfoBean, macroMachine, smsNumber, smsMessage, tokenKey);
+                        success = suc1 || suc2;
+                        break;
+                    case EMAILFIRST:
+                        // Send email first, try SMS if email is not available
+                        success = sendEmailToken(pwmApplication, userInfoBean, macroMachine, configuredEmailSetting, emailAddress, tokenKey) ||
+                                sendSmsToken(pwmApplication, userInfoBean, macroMachine, smsNumber, smsMessage, tokenKey);
+                        break;
+                    case SMSFIRST:
+                        // Send SMS first, try email if SMS is not available
+                        success = sendSmsToken(pwmApplication, userInfoBean, macroMachine, smsNumber, smsMessage, tokenKey) ||
+                                sendEmailToken(pwmApplication, userInfoBean, macroMachine, configuredEmailSetting, emailAddress, tokenKey);
+                        break;
+                    case SMSONLY:
+                        // Only try SMS
+                        success = sendSmsToken(pwmApplication, userInfoBean, macroMachine, smsNumber, smsMessage, tokenKey);
+                        break;
+                    case EMAILONLY:
+                    default:
+                        // Only try email
+                        success = sendEmailToken(pwmApplication, userInfoBean, macroMachine, configuredEmailSetting, emailAddress, tokenKey);
+                        break;
+                }
+            } catch (ChaiUnavailableException e) {
+                throw new PwmUnrecoverableException(PwmError.forChaiError(e.getErrorCode()));
+            }
+
+            if (!success) {
+                throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_TOKEN_MISSING_CONTACT));
+            }
+            pwmApplication.getStatisticsManager().incrementValue(Statistic.TOKENS_SENT);
+        }
+
+        public static boolean sendEmailToken(
+                final PwmApplication pwmApplication,
+                final UserInfoBean userInfoBean,
+                final MacroMachine macroMachine,
+                final EmailItemBean configuredEmailSetting,
+                final String toAddress,
+                final String tokenKey
+        )
+                throws PwmUnrecoverableException, ChaiUnavailableException
+        {
+            if (toAddress == null || toAddress.length() < 1) {
+                return false;
+            }
+
+            pwmApplication.getIntruderManager().mark(RecordType.TOKEN_DEST, toAddress, null);
+
+            pwmApplication.getEmailQueue().submitEmail(new EmailItemBean(
+                    toAddress,
+                    configuredEmailSetting.getFrom(),
+                    configuredEmailSetting.getSubject(),
+                    configuredEmailSetting.getBodyPlain().replace("%TOKEN%", tokenKey),
+                    configuredEmailSetting.getBodyHtml().replace("%TOKEN%", tokenKey)
+            ), userInfoBean, macroMachine);
+            LOGGER.debug("token email added to send queue for " + toAddress);
+            return true;
+        }
+
+        public static boolean sendSmsToken(
+                final PwmApplication pwmApplication,
+                final UserInfoBean userInfoBean,
+                final MacroMachine macroMachine,
+                final String smsNumber,
+                final String smsMessage,
+                final String tokenKey
+        )
+                throws PwmUnrecoverableException, ChaiUnavailableException
+        {
+            final Configuration config = pwmApplication.getConfig();
+
+            if (smsNumber == null || smsNumber.length() < 1) {
+                return false;
+            }
+
+            final String modifiedMessage = smsMessage.replaceAll("%TOKEN%", tokenKey);
+
+            pwmApplication.getIntruderManager().mark(RecordType.TOKEN_DEST, smsNumber, null);
+
+            pwmApplication.sendSmsUsingQueue(new SmsItemBean(smsNumber, modifiedMessage), macroMachine);
+            LOGGER.debug("token SMS added to send queue for " + smsNumber);
+            return true;
+        }
+    }
+}

pwm/servlet/src/password/pwm/token/TokenType.java → pwm/servlet/src/password/pwm/svc/token/TokenType.java

@@ -1,4 +1,4 @@
-package password.pwm.token;
+package password.pwm.svc.token;
 
 import java.util.Arrays;
 import java.util.Collections;

pwm/servlet/src/password/pwm/wordlist/AbstractWordlist.java → pwm/servlet/src/password/pwm/svc/wordlist/AbstractWordlist.java

@@ -1,400 +1,400 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.wordlist;
-
-import password.pwm.AppProperty;
-import password.pwm.PwmApplication;
-import password.pwm.PwmService;
-import password.pwm.config.option.DataStorageMethod;
-import password.pwm.error.ErrorInformation;
-import password.pwm.error.PwmError;
-import password.pwm.error.PwmException;
-import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.health.HealthRecord;
-import password.pwm.health.HealthStatus;
-import password.pwm.health.HealthTopic;
-import password.pwm.http.ContextManager;
-import password.pwm.util.Helper;
-import password.pwm.util.JsonUtil;
-import password.pwm.util.TimeDuration;
-import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.localdb.LocalDBException;
-import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.secure.PwmHashAlgorithm;
-import password.pwm.util.secure.SecureEngine;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.*;
-
-abstract class AbstractWordlist implements Wordlist, PwmService {
-
-    static final PwmHashAlgorithm CHECKSUM_HASH_ALG = PwmHashAlgorithm.SHA1;
-
-    protected WordlistConfiguration wordlistConfiguration;
-
-    protected volatile STATUS wlStatus = STATUS.NEW;
-    protected LocalDB localDB;
-
-    protected static final PwmLogger LOGGER = PwmLogger.forClass(AbstractWordlist.class);
-    protected String DEBUG_LABEL = "Generic Wordlist";
-
-    protected int storedSize = 0;
-    protected boolean debugTrace;
-
-    private ErrorInformation lastError;
-
-    private PwmApplication pwmApplication;
-    protected Populator populator;
-
-
-
-// --------------------------- CONSTRUCTORS ---------------------------
-
-    protected AbstractWordlist() {
-    }
-
-    public void init(final PwmApplication pwmApplication) throws PwmException {
-        this.pwmApplication = pwmApplication;
-        this.localDB = pwmApplication.getLocalDB();
-        if (pwmApplication.getConfig().isDevDebugMode()) {
-            debugTrace = true;
-        }
-    }
-
-    protected final void startup(final LocalDB localDB, final WordlistConfiguration wordlistConfiguration) {
-        this.wordlistConfiguration = wordlistConfiguration;
-        this.localDB = localDB;
-        wlStatus = STATUS.OPENING;
-
-        if (localDB == null) {
-            final String errorMsg = "LocalDB is not available, " + DEBUG_LABEL + " will remain closed";
-            LOGGER.warn(errorMsg);
-            lastError = new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,errorMsg);
-            close();
-            return;
-        }
-
-        try {
-            checkPopulation();
-        } catch (Exception e) {
-            final String errorMsg = "unexpected error while examining wordlist db: " + e.getMessage();
-            if ((e instanceof PwmUnrecoverableException) || (e instanceof NullPointerException) || (e instanceof LocalDBException)) {
-                LOGGER.warn(errorMsg);
-            } else {
-                LOGGER.warn(errorMsg,e);
-            }
-            lastError = new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,errorMsg);
-            populator = null;
-            close();
-            return;
-        }
-
-        //read stored size
-        storedSize = readMetadata().getSize();
-        wlStatus = STATUS.OPEN;
-    }
-
-    String normalizeWord(final String input) {
-        if (input == null) {
-            return null;
-        }
-
-        String word = input.trim();
-
-        if (!wordlistConfiguration.isCaseSensitive()) {
-            word = word.toLowerCase();
-        }
-
-        return word.length() > 0 ? word : null;
-    }
-
-    protected void checkPopulation()
-            throws Exception
-    {
-        final StoredWordlistDataBean storedWordlistDataBean = readMetadata();
-        boolean needsBuiltinPopulating = false;
-        if (!storedWordlistDataBean.isCompleted()) {
-            needsBuiltinPopulating = true;
-            LOGGER.debug("wordlist stored in database does not have a completed load status, will load built-in wordlist");
-        } else if (storedWordlistDataBean.isBuiltin()) {
-            final String builtInWordlistHash = getBuiltInWordlistHash();
-            if (!builtInWordlistHash.equals(storedWordlistDataBean.getSha1hash())) {
-                LOGGER.debug("wordlist stored in database does not have match checksum with built-in wordlist file, will load built-in wordlist");
-                needsBuiltinPopulating = true;
-            }
-        }
-
-        if (!needsBuiltinPopulating) {
-            return;
-        }
-
-        this.populateBuiltIn();
-    }
-
-
-    public boolean containsWord(final String word) {
-        if (wlStatus != STATUS.OPEN) {
-            return false;
-        }
-
-        final String testWord = normalizeWord(word);
-
-        if (testWord == null || testWord.length() < 1) {
-            return false;
-        }
-
-
-        final Set<String> testWords = chunkWord(testWord, this.wordlistConfiguration.getCheckSize());
-
-        final Date startTime = new Date();
-        try {
-            boolean result = false;
-            for (final String t : testWords) {
-                if (!result) { // stop checking once found
-                    if (localDB.contains(getWordlistDB(), t)) {
-                        result = true;
-                    }
-                }
-            }
-            final TimeDuration timeDuration = TimeDuration.fromCurrent(startTime);
-            if (timeDuration.isLongerThan(100)) {
-                LOGGER.debug("wordlist search time for " + testWords.size() + " wordlist permutations was greater then 100ms: " + timeDuration.asCompactString());
-            }
-            return result;
-        } catch (Exception e) {
-            LOGGER.error("database error checking for word: " + e.getMessage());
-        }
-
-        return false;
-    }
-
-    public int size() {
-        if (populator != null) {
-            return 0;
-        }
-
-        return storedSize;
-    }
-
-    public synchronized void close() {
-        if (populator != null) {
-            try {
-                populator.cancel();
-                populator = null;
-            } catch (PwmUnrecoverableException e) {
-                LOGGER.error("wordlist populator failed to exit");
-            }
-        }
-
-        if (wlStatus != STATUS.CLOSED) {
-            LOGGER.debug("closed");
-        }
-
-        wlStatus = STATUS.CLOSED;
-        localDB = null;
-    }
-
-    public STATUS status() {
-        return wlStatus;
-    }
-
-    public String getDebugStatus() {
-        if (wlStatus == STATUS.OPENING && populator != null) {
-            return populator.makeStatString();
-        } else {
-            return wlStatus.toString();
-        }
-    }
-
-    protected abstract Map<String, String> getWriteTxnForValue(String value);
-
-    protected abstract PwmApplication.AppAttribute getMetaDataAppAttribute();
-
-    protected abstract AppProperty getBuiltInWordlistLocationProperty();
-
-    protected abstract LocalDB.DB getWordlistDB();
-
-    public List<HealthRecord> healthCheck() {
-        if (wlStatus == STATUS.OPENING) {
-
-            final HealthRecord healthRecord = new HealthRecord(HealthStatus.CAUTION, HealthTopic.Application, this.DEBUG_LABEL + " is not yet open: " + this.getDebugStatus());
-            return Collections.singletonList(healthRecord);
-        }
-
-        if (lastError != null) {
-            final HealthRecord healthRecord = new HealthRecord(HealthStatus.WARN, HealthTopic.Application, this.DEBUG_LABEL + " error: " + lastError.toDebugStr());
-            return Collections.singletonList(healthRecord);
-        }
-        return null;
-    }
-
-    public ServiceInfo serviceInfo()
-    {
-        if (status() == STATUS.OPEN) {
-            return new ServiceInfo(Collections.singletonList(DataStorageMethod.LOCALDB));
-        } else {
-            return new ServiceInfo(Collections.<DataStorageMethod>emptyList());
-        }
-    }
-
-    protected Set<String> chunkWord(final String input, final int size) {
-        int checkSize = size == 0 || size > input.length() ? input.length() : size;
-        final TreeSet<String> testWords = new TreeSet<>();
-        while (checkSize <= input.length()) {
-            for (int i = 0; i + checkSize <= input.length(); i++) {
-                final String loopWord = input.substring(i,i + checkSize);
-                testWords.add(loopWord);
-            }
-            checkSize++;
-        }
-
-        return testWords;
-    }
-
-    public StoredWordlistDataBean readMetadata() {
-        final String storedValue = pwmApplication.readAppAttribute(getMetaDataAppAttribute());
-        if (storedValue != null && !storedValue.isEmpty()) {
-            final StoredWordlistDataBean metaDataBean = JsonUtil.deserialize(storedValue, StoredWordlistDataBean.class);
-            if (metaDataBean != null) {
-                return metaDataBean;
-            }
-        }
-        return new StoredWordlistDataBean();
-    }
-
-    void writeMetadata(final StoredWordlistDataBean metadataBean) {
-        final String jsonValue = JsonUtil.serialize(metadataBean);
-        pwmApplication.writeAppAttribute(getMetaDataAppAttribute(),jsonValue);
-    }
-
-    @Override
-    public void populate(final InputStream inputStream)
-            throws IOException, PwmUnrecoverableException
-    {
-        try {
-            populateImpl(inputStream);
-        } finally {
-            if (!readMetadata().isCompleted()) {
-                LOGGER.debug("beginning population using builtin wordlist in background thread");
-                final Thread t = new Thread(new Runnable() {
-                    public void run() {
-                        try {
-                            populateBuiltIn();
-                        } catch (Exception e) {
-                            LOGGER.warn("unexpected error during builtin wordlist population process: " + e.getMessage(),e);
-                        }
-                        populator = null;
-                    }
-                }, Helper.makeThreadName(pwmApplication, WordlistManager.class));
-                t.setDaemon(true);
-                t.start();
-            }
-        }
-    }
-
-    @Override
-    public void populateBuiltIn()
-            throws IOException, PwmUnrecoverableException
-    {
-        populateImpl(getBuiltInWordlist());
-        {
-            final StoredWordlistDataBean storedWordlistDataBean = readMetadata();
-            storedWordlistDataBean.setBuiltin(true);
-            writeMetadata(storedWordlistDataBean);
-        }
-    }
-
-    private void populateImpl(final InputStream inputStream)
-            throws IOException, PwmUnrecoverableException
-    {
-        if (inputStream == null) {
-            throw new NullPointerException("input stream can not be null for populateImpl()");
-        }
-
-        if (wlStatus == STATUS.CLOSED) {
-            return;
-        }
-
-        wlStatus = STATUS.OPENING;
-
-        try {
-            if (populator != null) {
-                populator.cancel();
-
-                final int maxWaitMs = 1000 * 30;
-                final Date startWaitTime = new Date();
-                while (populator.isRunning() && TimeDuration.fromCurrent(startWaitTime).isShorterThan(maxWaitMs)) {
-                    Helper.pause(1000);
-                }
-                if (populator.isRunning() && TimeDuration.fromCurrent(startWaitTime).isShorterThan(maxWaitMs)) {
-                    throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_UNKNOWN, "unable to abort populator"));
-                }
-            }
-
-            { // reset the wordlist metadata
-                final StoredWordlistDataBean storedWordlistDataBean = new StoredWordlistDataBean();
-                writeMetadata(storedWordlistDataBean);
-            }
-
-            populator = new Populator(inputStream, this, pwmApplication);
-            populator.populate();
-        } catch (Exception e) {
-            final ErrorInformation populationError;
-            populationError = e instanceof PwmException
-                    ? ((PwmException) e).getErrorInformation()
-                    : new ErrorInformation(PwmError.ERROR_UNKNOWN, e.getMessage());
-            LOGGER.error("error during wordlist population: " + populationError.toDebugStr());
-            throw new PwmUnrecoverableException(populationError);
-        } finally {
-            populator = null;
-            inputStream.close();
-        }
-
-        wlStatus = STATUS.OPEN;
-    }
-
-    protected InputStream getBuiltInWordlist() throws FileNotFoundException, PwmUnrecoverableException {
-        final ContextManager contextManager = pwmApplication.getPwmEnvironment().getContextManager();
-        if (contextManager != null) {
-            final String wordlistFilename = pwmApplication.getConfig().readAppProperty(getBuiltInWordlistLocationProperty());
-            return contextManager.getResourceAsStream(wordlistFilename);
-        }
-        throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,"unable to locate builtin wordlist file"));
-    }
-
-    protected String getBuiltInWordlistHash() throws IOException, PwmUnrecoverableException {
-
-        InputStream inputStream = null;
-        try {
-            inputStream = getBuiltInWordlist();
-            return SecureEngine.hash(inputStream, CHECKSUM_HASH_ALG);
-        } finally {
-            if (inputStream != null) {
-                inputStream.close();
-            }
-        }
-    }
-
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.config.option.DataStorageMethod;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmException;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.health.HealthRecord;
+import password.pwm.health.HealthStatus;
+import password.pwm.health.HealthTopic;
+import password.pwm.http.ContextManager;
+import password.pwm.svc.PwmService;
+import password.pwm.util.Helper;
+import password.pwm.util.JsonUtil;
+import password.pwm.util.TimeDuration;
+import password.pwm.util.localdb.LocalDB;
+import password.pwm.util.localdb.LocalDBException;
+import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.secure.PwmHashAlgorithm;
+import password.pwm.util.secure.SecureEngine;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.*;
+
+abstract class AbstractWordlist implements Wordlist, PwmService {
+
+    static final PwmHashAlgorithm CHECKSUM_HASH_ALG = PwmHashAlgorithm.SHA1;
+
+    protected WordlistConfiguration wordlistConfiguration;
+
+    protected volatile STATUS wlStatus = STATUS.NEW;
+    protected LocalDB localDB;
+
+    protected static final PwmLogger LOGGER = PwmLogger.forClass(AbstractWordlist.class);
+    protected String DEBUG_LABEL = "Generic Wordlist";
+
+    protected int storedSize = 0;
+    protected boolean debugTrace;
+
+    private ErrorInformation lastError;
+
+    private PwmApplication pwmApplication;
+    protected Populator populator;
+
+
+
+// --------------------------- CONSTRUCTORS ---------------------------
+
+    protected AbstractWordlist() {
+    }
+
+    public void init(final PwmApplication pwmApplication) throws PwmException {
+        this.pwmApplication = pwmApplication;
+        this.localDB = pwmApplication.getLocalDB();
+        if (pwmApplication.getConfig().isDevDebugMode()) {
+            debugTrace = true;
+        }
+    }
+
+    protected final void startup(final LocalDB localDB, final WordlistConfiguration wordlistConfiguration) {
+        this.wordlistConfiguration = wordlistConfiguration;
+        this.localDB = localDB;
+        wlStatus = STATUS.OPENING;
+
+        if (localDB == null) {
+            final String errorMsg = "LocalDB is not available, " + DEBUG_LABEL + " will remain closed";
+            LOGGER.warn(errorMsg);
+            lastError = new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,errorMsg);
+            close();
+            return;
+        }
+
+        try {
+            checkPopulation();
+        } catch (Exception e) {
+            final String errorMsg = "unexpected error while examining wordlist db: " + e.getMessage();
+            if ((e instanceof PwmUnrecoverableException) || (e instanceof NullPointerException) || (e instanceof LocalDBException)) {
+                LOGGER.warn(errorMsg);
+            } else {
+                LOGGER.warn(errorMsg,e);
+            }
+            lastError = new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,errorMsg);
+            populator = null;
+            close();
+            return;
+        }
+
+        //read stored size
+        storedSize = readMetadata().getSize();
+        wlStatus = STATUS.OPEN;
+    }
+
+    String normalizeWord(final String input) {
+        if (input == null) {
+            return null;
+        }
+
+        String word = input.trim();
+
+        if (!wordlistConfiguration.isCaseSensitive()) {
+            word = word.toLowerCase();
+        }
+
+        return word.length() > 0 ? word : null;
+    }
+
+    protected void checkPopulation()
+            throws Exception
+    {
+        final StoredWordlistDataBean storedWordlistDataBean = readMetadata();
+        boolean needsBuiltinPopulating = false;
+        if (!storedWordlistDataBean.isCompleted()) {
+            needsBuiltinPopulating = true;
+            LOGGER.debug("wordlist stored in database does not have a completed load status, will load built-in wordlist");
+        } else if (storedWordlistDataBean.isBuiltin()) {
+            final String builtInWordlistHash = getBuiltInWordlistHash();
+            if (!builtInWordlistHash.equals(storedWordlistDataBean.getSha1hash())) {
+                LOGGER.debug("wordlist stored in database does not have match checksum with built-in wordlist file, will load built-in wordlist");
+                needsBuiltinPopulating = true;
+            }
+        }
+
+        if (!needsBuiltinPopulating) {
+            return;
+        }
+
+        this.populateBuiltIn();
+    }
+
+
+    public boolean containsWord(final String word) {
+        if (wlStatus != STATUS.OPEN) {
+            return false;
+        }
+
+        final String testWord = normalizeWord(word);
+
+        if (testWord == null || testWord.length() < 1) {
+            return false;
+        }
+
+
+        final Set<String> testWords = chunkWord(testWord, this.wordlistConfiguration.getCheckSize());
+
+        final Date startTime = new Date();
+        try {
+            boolean result = false;
+            for (final String t : testWords) {
+                if (!result) { // stop checking once found
+                    if (localDB.contains(getWordlistDB(), t)) {
+                        result = true;
+                    }
+                }
+            }
+            final TimeDuration timeDuration = TimeDuration.fromCurrent(startTime);
+            if (timeDuration.isLongerThan(100)) {
+                LOGGER.debug("wordlist search time for " + testWords.size() + " wordlist permutations was greater then 100ms: " + timeDuration.asCompactString());
+            }
+            return result;
+        } catch (Exception e) {
+            LOGGER.error("database error checking for word: " + e.getMessage());
+        }
+
+        return false;
+    }
+
+    public int size() {
+        if (populator != null) {
+            return 0;
+        }
+
+        return storedSize;
+    }
+
+    public synchronized void close() {
+        if (populator != null) {
+            try {
+                populator.cancel();
+                populator = null;
+            } catch (PwmUnrecoverableException e) {
+                LOGGER.error("wordlist populator failed to exit");
+            }
+        }
+
+        if (wlStatus != STATUS.CLOSED) {
+            LOGGER.debug("closed");
+        }
+
+        wlStatus = STATUS.CLOSED;
+        localDB = null;
+    }
+
+    public STATUS status() {
+        return wlStatus;
+    }
+
+    public String getDebugStatus() {
+        if (wlStatus == STATUS.OPENING && populator != null) {
+            return populator.makeStatString();
+        } else {
+            return wlStatus.toString();
+        }
+    }
+
+    protected abstract Map<String, String> getWriteTxnForValue(String value);
+
+    protected abstract PwmApplication.AppAttribute getMetaDataAppAttribute();
+
+    protected abstract AppProperty getBuiltInWordlistLocationProperty();
+
+    protected abstract LocalDB.DB getWordlistDB();
+
+    public List<HealthRecord> healthCheck() {
+        if (wlStatus == STATUS.OPENING) {
+
+            final HealthRecord healthRecord = new HealthRecord(HealthStatus.CAUTION, HealthTopic.Application, this.DEBUG_LABEL + " is not yet open: " + this.getDebugStatus());
+            return Collections.singletonList(healthRecord);
+        }
+
+        if (lastError != null) {
+            final HealthRecord healthRecord = new HealthRecord(HealthStatus.WARN, HealthTopic.Application, this.DEBUG_LABEL + " error: " + lastError.toDebugStr());
+            return Collections.singletonList(healthRecord);
+        }
+        return null;
+    }
+
+    public ServiceInfo serviceInfo()
+    {
+        if (status() == STATUS.OPEN) {
+            return new ServiceInfo(Collections.singletonList(DataStorageMethod.LOCALDB));
+        } else {
+            return new ServiceInfo(Collections.<DataStorageMethod>emptyList());
+        }
+    }
+
+    protected Set<String> chunkWord(final String input, final int size) {
+        int checkSize = size == 0 || size > input.length() ? input.length() : size;
+        final TreeSet<String> testWords = new TreeSet<>();
+        while (checkSize <= input.length()) {
+            for (int i = 0; i + checkSize <= input.length(); i++) {
+                final String loopWord = input.substring(i,i + checkSize);
+                testWords.add(loopWord);
+            }
+            checkSize++;
+        }
+
+        return testWords;
+    }
+
+    public StoredWordlistDataBean readMetadata() {
+        final String storedValue = pwmApplication.readAppAttribute(getMetaDataAppAttribute());
+        if (storedValue != null && !storedValue.isEmpty()) {
+            final StoredWordlistDataBean metaDataBean = JsonUtil.deserialize(storedValue, StoredWordlistDataBean.class);
+            if (metaDataBean != null) {
+                return metaDataBean;
+            }
+        }
+        return new StoredWordlistDataBean();
+    }
+
+    void writeMetadata(final StoredWordlistDataBean metadataBean) {
+        final String jsonValue = JsonUtil.serialize(metadataBean);
+        pwmApplication.writeAppAttribute(getMetaDataAppAttribute(),jsonValue);
+    }
+
+    @Override
+    public void populate(final InputStream inputStream)
+            throws IOException, PwmUnrecoverableException
+    {
+        try {
+            populateImpl(inputStream);
+        } finally {
+            if (!readMetadata().isCompleted()) {
+                LOGGER.debug("beginning population using builtin wordlist in background thread");
+                final Thread t = new Thread(new Runnable() {
+                    public void run() {
+                        try {
+                            populateBuiltIn();
+                        } catch (Exception e) {
+                            LOGGER.warn("unexpected error during builtin wordlist population process: " + e.getMessage(),e);
+                        }
+                        populator = null;
+                    }
+                }, Helper.makeThreadName(pwmApplication, WordlistManager.class));
+                t.setDaemon(true);
+                t.start();
+            }
+        }
+    }
+
+    @Override
+    public void populateBuiltIn()
+            throws IOException, PwmUnrecoverableException
+    {
+        populateImpl(getBuiltInWordlist());
+        {
+            final StoredWordlistDataBean storedWordlistDataBean = readMetadata();
+            storedWordlistDataBean.setBuiltin(true);
+            writeMetadata(storedWordlistDataBean);
+        }
+    }
+
+    private void populateImpl(final InputStream inputStream)
+            throws IOException, PwmUnrecoverableException
+    {
+        if (inputStream == null) {
+            throw new NullPointerException("input stream can not be null for populateImpl()");
+        }
+
+        if (wlStatus == STATUS.CLOSED) {
+            return;
+        }
+
+        wlStatus = STATUS.OPENING;
+
+        try {
+            if (populator != null) {
+                populator.cancel();
+
+                final int maxWaitMs = 1000 * 30;
+                final Date startWaitTime = new Date();
+                while (populator.isRunning() && TimeDuration.fromCurrent(startWaitTime).isShorterThan(maxWaitMs)) {
+                    Helper.pause(1000);
+                }
+                if (populator.isRunning() && TimeDuration.fromCurrent(startWaitTime).isShorterThan(maxWaitMs)) {
+                    throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_UNKNOWN, "unable to abort populator"));
+                }
+            }
+
+            { // reset the wordlist metadata
+                final StoredWordlistDataBean storedWordlistDataBean = new StoredWordlistDataBean();
+                writeMetadata(storedWordlistDataBean);
+            }
+
+            populator = new Populator(inputStream, this, pwmApplication);
+            populator.populate();
+        } catch (Exception e) {
+            final ErrorInformation populationError;
+            populationError = e instanceof PwmException
+                    ? ((PwmException) e).getErrorInformation()
+                    : new ErrorInformation(PwmError.ERROR_UNKNOWN, e.getMessage());
+            LOGGER.error("error during wordlist population: " + populationError.toDebugStr());
+            throw new PwmUnrecoverableException(populationError);
+        } finally {
+            populator = null;
+            inputStream.close();
+        }
+
+        wlStatus = STATUS.OPEN;
+    }
+
+    protected InputStream getBuiltInWordlist() throws FileNotFoundException, PwmUnrecoverableException {
+        final ContextManager contextManager = pwmApplication.getPwmEnvironment().getContextManager();
+        if (contextManager != null) {
+            final String wordlistFilename = pwmApplication.getConfig().readAppProperty(getBuiltInWordlistLocationProperty());
+            return contextManager.getResourceAsStream(wordlistFilename);
+        }
+        throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_SERVICE_NOT_AVAILABLE,"unable to locate builtin wordlist file"));
+    }
+
+    protected String getBuiltInWordlistHash() throws IOException, PwmUnrecoverableException {
+
+        InputStream inputStream = null;
+        try {
+            inputStream = getBuiltInWordlist();
+            return SecureEngine.hash(inputStream, CHECKSUM_HASH_ALG);
+        } finally {
+            if (inputStream != null) {
+                inputStream.close();
+            }
+        }
+    }
+
+}

pwm/servlet/src/password/pwm/wordlist/Populator.java → pwm/servlet/src/password/pwm/svc/wordlist/Populator.java

@@ -1,286 +1,286 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.wordlist;
-
-import password.pwm.PwmApplication;
-import password.pwm.error.ErrorInformation;
-import password.pwm.error.PwmError;
-import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.util.Helper;
-import password.pwm.util.TimeDuration;
-import password.pwm.util.TransactionSizeCalculator;
-import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.localdb.LocalDBException;
-import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.secure.ChecksumInputStream;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.text.DecimalFormat;
-import java.text.NumberFormat;
-import java.util.Date;
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- * @author Jason D. Rivard
- */
-class Populator {
-// ------------------------------ FIELDS ------------------------------
-
-    private static final PwmLogger LOGGER = PwmLogger.forClass(Populator.class);
-
-
-    private static final int MAX_LINE_LENGTH = 64; // words truncated to this length, prevents massive words if the input
-
-    private static final long DEBUG_OUTPUT_FREQUENCY = 3 * 60 * 1000;  // 3 minutes
-
-    private static final String COMMENT_PREFIX = "!#comment:"; // words tarting with this prefix are ignored.
-    private static final NumberFormat PERCENT_FORMAT = DecimalFormat.getPercentInstance();
-
-    private final ZipReader zipFileReader;
-
-    private volatile boolean running;
-    private volatile boolean abortFlag;
-
-    private final PopulationStats overallStats = new PopulationStats();
-    private PopulationStats perReportStats = new PopulationStats();
-    private TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(600, 10, 50 * 1000);
-    private int loopLines;
-
-    private final Map<String,String> bufferedWords = new TreeMap<>();
-
-    private final LocalDB localDB;
-
-    private final ChecksumInputStream checksumInputStream;
-
-    private final AbstractWordlist rootWordlist;
-
-
-    static {
-        PERCENT_FORMAT.setMinimumFractionDigits(2);
-    }
-
-    public Populator(
-            final InputStream inputStream,
-            final AbstractWordlist rootWordlist,
-            final PwmApplication pwmApplication
-    )
-            throws Exception
-    {
-        this.checksumInputStream = new ChecksumInputStream(AbstractWordlist.CHECKSUM_HASH_ALG, inputStream);
-        this.zipFileReader = new ZipReader(checksumInputStream);
-        this.localDB = pwmApplication.getLocalDB();
-        this.rootWordlist = rootWordlist;
-    }
-
-    private void init() throws LocalDBException, IOException {
-        if (abortFlag) return;
-
-        localDB.truncate(rootWordlist.getWordlistDB());
-
-        if (overallStats.getLines() > 0) {
-            for (int i = 0; i < overallStats.getLines(); i++) {
-                zipFileReader.nextLine();
-            }
-        }
-    }
-
-
-// -------------------------- OTHER METHODS --------------------------
-
-    public String makeStatString()
-    {
-        if (!running) {
-            return "not running";
-        }
-
-        final int lps = perReportStats.getElapsedSeconds() <= 0 ? 0 : perReportStats.getLines() / perReportStats.getElapsedSeconds();
-
-        perReportStats = new PopulationStats();
-        return rootWordlist.DEBUG_LABEL + ", lines/second="
-                + lps + ", line=" + overallStats.getLines() + ")"
-                + " current zipEntry=" + zipFileReader.currentZipName();
-    }
-
-    void populate() throws IOException, LocalDBException, PwmUnrecoverableException {
-        try {
-            rootWordlist.writeMetadata(new StoredWordlistDataBean());
-            running = true;
-            init();
-
-            long lastReportTime = System.currentTimeMillis() - (long) (DEBUG_OUTPUT_FREQUENCY * 0.33);
-
-            String line;
-            while (!abortFlag && (line = zipFileReader.nextLine()) != null) {
-
-                overallStats.incrementLines();
-                perReportStats.incrementLines();
-
-                addLine(line);
-                loopLines++;
-
-                if (TimeDuration.fromCurrent(lastReportTime).isLongerThan(DEBUG_OUTPUT_FREQUENCY)) {
-                    LOGGER.info(makeStatString());
-                    lastReportTime = System.currentTimeMillis();
-                }
-
-                if (bufferedWords.size() > transactionCalculator.getTransactionSize()) {
-                    flushBuffer();
-                }
-            }
-
-            if (abortFlag) {
-                LOGGER.warn("pausing " + rootWordlist.DEBUG_LABEL + " population");
-            } else {
-                populationComplete();
-            }
-        } finally {
-
-            running = false;
-            checksumInputStream.close();
-        }
-    }
-
-    private void addLine(String line)
-            throws IOException
-    {
-        // check for word suitability
-        line = rootWordlist.normalizeWord(line);
-
-        if (line == null || line.length() < 1 || line.startsWith(COMMENT_PREFIX)) {
-            return;
-        }
-
-        if (line.length() > MAX_LINE_LENGTH) {
-            line = line.substring(0,MAX_LINE_LENGTH);
-        }
-
-        final Map<String,String> wordTxn = rootWordlist.getWriteTxnForValue(line);
-        bufferedWords.putAll(wordTxn);
-    }
-
-    private void flushBuffer()
-            throws LocalDBException
-    {
-        final long startTime = System.currentTimeMillis();
-
-        //add the elements
-        localDB.putAll(rootWordlist.getWordlistDB(), bufferedWords);
-
-        if (abortFlag) {
-            return;
-        }
-
-        //mark how long the buffer close took
-        final long commitTime = System.currentTimeMillis() - startTime;
-        transactionCalculator.recordLastTransactionDuration(commitTime);
-
-        if (bufferedWords.size() > 0) {
-            final StringBuilder sb = new StringBuilder();
-            sb.append(rootWordlist.DEBUG_LABEL).append(" ");
-            sb.append("read ").append(loopLines).append(", ");
-            sb.append("saved ");
-            sb.append(bufferedWords.size()).append(" words");
-            sb.append(" (").append(new TimeDuration(commitTime).asCompactString()).append(")");
-
-            LOGGER.trace(sb.toString());
-        }
-
-        //clear the buffers.
-        bufferedWords.clear();
-        loopLines = 0;
-    }
-
-    private void populationComplete()
-            throws LocalDBException, PwmUnrecoverableException, IOException {
-        flushBuffer();
-        LOGGER.info(makeStatString());
-        LOGGER.trace("beginning wordlist size query");
-        final int wordlistSize = localDB.size(rootWordlist.getWordlistDB());
-        if (wordlistSize < 1) {
-            throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_UNKNOWN, rootWordlist.DEBUG_LABEL + " population completed, but no words stored"));
-        }
-
-        final StringBuilder sb = new StringBuilder();
-        sb.append(rootWordlist.DEBUG_LABEL);
-        sb.append(" population complete, added ").append(wordlistSize);
-        sb.append(" total words in ").append(new TimeDuration(overallStats.getElapsedSeconds() * 1000).asCompactString());
-        {
-            StoredWordlistDataBean storedWordlistDataBean = new StoredWordlistDataBean();
-            storedWordlistDataBean.setSha1hash(Helper.binaryArrayToHex(checksumInputStream.closeAndFinalChecksum()));
-            storedWordlistDataBean.setSize(wordlistSize);
-            storedWordlistDataBean.setStoreDate(new Date());
-            if (!abortFlag) {
-                storedWordlistDataBean.setCompleted(true);
-            }
-            rootWordlist.writeMetadata(storedWordlistDataBean);
-        }
-        LOGGER.info(sb.toString());
-    }
-
-    public void cancel() throws PwmUnrecoverableException {
-        LOGGER.debug("cancelling in-progress population");
-        abortFlag = true;
-
-        final int maxWaitMs = 1000 * 30;
-        final Date startWaitTime = new Date();
-        while (isRunning() && TimeDuration.fromCurrent(startWaitTime).isShorterThan(maxWaitMs)) {
-            Helper.pause(1000);
-        }
-        if (isRunning() && TimeDuration.fromCurrent(startWaitTime).isShorterThan(maxWaitMs)) {
-            throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_UNKNOWN, "unable to abort in progress population"));
-        }
-
-    }
-
-    public boolean isRunning() {
-        return running;
-    }
-
-    private static class PopulationStats {
-        // ------------------------------ FIELDS ------------------------------
-
-        private long startTime = System.currentTimeMillis();
-        private int lines;
-
-        // --------------------- GETTER / SETTER METHODS ---------------------
-
-        public int getLines()
-        {
-            return lines;
-        }
-
-        // -------------------------- OTHER METHODS --------------------------
-
-        public void incrementLines()
-        {
-            lines++;
-        }
-
-        public int getElapsedSeconds()
-        {
-            return (int) (System.currentTimeMillis() - startTime) / 1000;
-        }
-    }
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import password.pwm.PwmApplication;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.Helper;
+import password.pwm.util.TimeDuration;
+import password.pwm.util.TransactionSizeCalculator;
+import password.pwm.util.localdb.LocalDB;
+import password.pwm.util.localdb.LocalDBException;
+import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.secure.ChecksumInputStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.text.DecimalFormat;
+import java.text.NumberFormat;
+import java.util.Date;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * @author Jason D. Rivard
+ */
+class Populator {
+// ------------------------------ FIELDS ------------------------------
+
+    private static final PwmLogger LOGGER = PwmLogger.forClass(Populator.class);
+
+
+    private static final int MAX_LINE_LENGTH = 64; // words truncated to this length, prevents massive words if the input
+
+    private static final long DEBUG_OUTPUT_FREQUENCY = 3 * 60 * 1000;  // 3 minutes
+
+    private static final String COMMENT_PREFIX = "!#comment:"; // words tarting with this prefix are ignored.
+    private static final NumberFormat PERCENT_FORMAT = DecimalFormat.getPercentInstance();
+
+    private final ZipReader zipFileReader;
+
+    private volatile boolean running;
+    private volatile boolean abortFlag;
+
+    private final PopulationStats overallStats = new PopulationStats();
+    private PopulationStats perReportStats = new PopulationStats();
+    private TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(600, 10, 50 * 1000);
+    private int loopLines;
+
+    private final Map<String,String> bufferedWords = new TreeMap<>();
+
+    private final LocalDB localDB;
+
+    private final ChecksumInputStream checksumInputStream;
+
+    private final AbstractWordlist rootWordlist;
+
+
+    static {
+        PERCENT_FORMAT.setMinimumFractionDigits(2);
+    }
+
+    public Populator(
+            final InputStream inputStream,
+            final AbstractWordlist rootWordlist,
+            final PwmApplication pwmApplication
+    )
+            throws Exception
+    {
+        this.checksumInputStream = new ChecksumInputStream(AbstractWordlist.CHECKSUM_HASH_ALG, inputStream);
+        this.zipFileReader = new ZipReader(checksumInputStream);
+        this.localDB = pwmApplication.getLocalDB();
+        this.rootWordlist = rootWordlist;
+    }
+
+    private void init() throws LocalDBException, IOException {
+        if (abortFlag) return;
+
+        localDB.truncate(rootWordlist.getWordlistDB());
+
+        if (overallStats.getLines() > 0) {
+            for (int i = 0; i < overallStats.getLines(); i++) {
+                zipFileReader.nextLine();
+            }
+        }
+    }
+
+
+// -------------------------- OTHER METHODS --------------------------
+
+    public String makeStatString()
+    {
+        if (!running) {
+            return "not running";
+        }
+
+        final int lps = perReportStats.getElapsedSeconds() <= 0 ? 0 : perReportStats.getLines() / perReportStats.getElapsedSeconds();
+
+        perReportStats = new PopulationStats();
+        return rootWordlist.DEBUG_LABEL + ", lines/second="
+                + lps + ", line=" + overallStats.getLines() + ")"
+                + " current zipEntry=" + zipFileReader.currentZipName();
+    }
+
+    void populate() throws IOException, LocalDBException, PwmUnrecoverableException {
+        try {
+            rootWordlist.writeMetadata(new StoredWordlistDataBean());
+            running = true;
+            init();
+
+            long lastReportTime = System.currentTimeMillis() - (long) (DEBUG_OUTPUT_FREQUENCY * 0.33);
+
+            String line;
+            while (!abortFlag && (line = zipFileReader.nextLine()) != null) {
+
+                overallStats.incrementLines();
+                perReportStats.incrementLines();
+
+                addLine(line);
+                loopLines++;
+
+                if (TimeDuration.fromCurrent(lastReportTime).isLongerThan(DEBUG_OUTPUT_FREQUENCY)) {
+                    LOGGER.info(makeStatString());
+                    lastReportTime = System.currentTimeMillis();
+                }
+
+                if (bufferedWords.size() > transactionCalculator.getTransactionSize()) {
+                    flushBuffer();
+                }
+            }
+
+            if (abortFlag) {
+                LOGGER.warn("pausing " + rootWordlist.DEBUG_LABEL + " population");
+            } else {
+                populationComplete();
+            }
+        } finally {
+
+            running = false;
+            checksumInputStream.close();
+        }
+    }
+
+    private void addLine(String line)
+            throws IOException
+    {
+        // check for word suitability
+        line = rootWordlist.normalizeWord(line);
+
+        if (line == null || line.length() < 1 || line.startsWith(COMMENT_PREFIX)) {
+            return;
+        }
+
+        if (line.length() > MAX_LINE_LENGTH) {
+            line = line.substring(0,MAX_LINE_LENGTH);
+        }
+
+        final Map<String,String> wordTxn = rootWordlist.getWriteTxnForValue(line);
+        bufferedWords.putAll(wordTxn);
+    }
+
+    private void flushBuffer()
+            throws LocalDBException
+    {
+        final long startTime = System.currentTimeMillis();
+
+        //add the elements
+        localDB.putAll(rootWordlist.getWordlistDB(), bufferedWords);
+
+        if (abortFlag) {
+            return;
+        }
+
+        //mark how long the buffer close took
+        final long commitTime = System.currentTimeMillis() - startTime;
+        transactionCalculator.recordLastTransactionDuration(commitTime);
+
+        if (bufferedWords.size() > 0) {
+            final StringBuilder sb = new StringBuilder();
+            sb.append(rootWordlist.DEBUG_LABEL).append(" ");
+            sb.append("read ").append(loopLines).append(", ");
+            sb.append("saved ");
+            sb.append(bufferedWords.size()).append(" words");
+            sb.append(" (").append(new TimeDuration(commitTime).asCompactString()).append(")");
+
+            LOGGER.trace(sb.toString());
+        }
+
+        //clear the buffers.
+        bufferedWords.clear();
+        loopLines = 0;
+    }
+
+    private void populationComplete()
+            throws LocalDBException, PwmUnrecoverableException, IOException {
+        flushBuffer();
+        LOGGER.info(makeStatString());
+        LOGGER.trace("beginning wordlist size query");
+        final int wordlistSize = localDB.size(rootWordlist.getWordlistDB());
+        if (wordlistSize < 1) {
+            throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_UNKNOWN, rootWordlist.DEBUG_LABEL + " population completed, but no words stored"));
+        }
+
+        final StringBuilder sb = new StringBuilder();
+        sb.append(rootWordlist.DEBUG_LABEL);
+        sb.append(" population complete, added ").append(wordlistSize);
+        sb.append(" total words in ").append(new TimeDuration(overallStats.getElapsedSeconds() * 1000).asCompactString());
+        {
+            StoredWordlistDataBean storedWordlistDataBean = new StoredWordlistDataBean();
+            storedWordlistDataBean.setSha1hash(Helper.binaryArrayToHex(checksumInputStream.closeAndFinalChecksum()));
+            storedWordlistDataBean.setSize(wordlistSize);
+            storedWordlistDataBean.setStoreDate(new Date());
+            if (!abortFlag) {
+                storedWordlistDataBean.setCompleted(true);
+            }
+            rootWordlist.writeMetadata(storedWordlistDataBean);
+        }
+        LOGGER.info(sb.toString());
+    }
+
+    public void cancel() throws PwmUnrecoverableException {
+        LOGGER.debug("cancelling in-progress population");
+        abortFlag = true;
+
+        final int maxWaitMs = 1000 * 30;
+        final Date startWaitTime = new Date();
+        while (isRunning() && TimeDuration.fromCurrent(startWaitTime).isShorterThan(maxWaitMs)) {
+            Helper.pause(1000);
+        }
+        if (isRunning() && TimeDuration.fromCurrent(startWaitTime).isShorterThan(maxWaitMs)) {
+            throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_UNKNOWN, "unable to abort in progress population"));
+        }
+
+    }
+
+    public boolean isRunning() {
+        return running;
+    }
+
+    private static class PopulationStats {
+        // ------------------------------ FIELDS ------------------------------
+
+        private long startTime = System.currentTimeMillis();
+        private int lines;
+
+        // --------------------- GETTER / SETTER METHODS ---------------------
+
+        public int getLines()
+        {
+            return lines;
+        }
+
+        // -------------------------- OTHER METHODS --------------------------
+
+        public void incrementLines()
+        {
+            lines++;
+        }
+
+        public int getElapsedSeconds()
+        {
+            return (int) (System.currentTimeMillis() - startTime) / 1000;
+        }
+    }
+}

pwm/servlet/src/password/pwm/wordlist/SeedlistManager.java → pwm/servlet/src/password/pwm/svc/wordlist/SeedlistManager.java

@@ -1,108 +1,108 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.wordlist;
-
-import password.pwm.AppProperty;
-import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
-import password.pwm.error.PwmException;
-import password.pwm.util.Helper;
-import password.pwm.util.TimeDuration;
-import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.secure.PwmRandom;
-
-import java.util.Collections;
-import java.util.Map;
-
-public class SeedlistManager extends AbstractWordlist implements Wordlist {
-
-    private static final PwmLogger LOGGER = PwmLogger.forClass(SeedlistManager.class);
-
-    private int initialPopulationCounter = 0;
-
-    public SeedlistManager() {
-    }
-
-    public String randomSeed() {
-        if (wlStatus != STATUS.OPEN) {
-            return null;
-        }
-        final long startTime = System.currentTimeMillis();
-        String returnValue = null;
-        try {
-            final int seedCount = size();
-            if (seedCount > 1000) {
-                final int randomKey = PwmRandom.getInstance().nextInt(size());
-                final Object obj = localDB.get(getWordlistDB(), String.valueOf(randomKey));
-                if (obj != null) {
-                    returnValue = obj.toString();
-                }
-            }
-        } catch (Exception e) {
-            LOGGER.warn("error while generating random word: " + e.getMessage());
-        }
-
-        if (debugTrace) {
-            LOGGER.trace("getRandomSeed fetch time: " + TimeDuration.fromCurrent(startTime).asCompactString());
-        }
-        return returnValue;
-    }
-
-    protected Map<String, String> getWriteTxnForValue(final String value) {
-        final Map<String, String> txItem = Collections.singletonMap(String.valueOf(initialPopulationCounter), value);
-        initialPopulationCounter++;
-        return txItem;
-    }
-
-    public void init(final PwmApplication pwmApplication) throws PwmException {
-        super.init(pwmApplication);
-        final WordlistConfiguration wordlistConfiguration = new WordlistConfiguration(true, 0);
-
-        this.DEBUG_LABEL = PwmConstants.PWM_APP_NAME + "-Seedist";
-
-        final Thread t = new Thread(new Runnable() {
-            public void run() {
-                LOGGER.debug(DEBUG_LABEL + " starting up in background thread");
-                startup(pwmApplication.getLocalDB(), wordlistConfiguration);
-            }
-        }, Helper.makeThreadName(pwmApplication,SeedlistManager.class));
-
-        t.start();
-    }
-
-    @Override
-    protected PwmApplication.AppAttribute getMetaDataAppAttribute() {
-        return PwmApplication.AppAttribute.SEEDLIST_METADATA;
-    }
-
-    @Override
-    protected LocalDB.DB getWordlistDB() {
-        return LocalDB.DB.SEEDLIST_WORDS;
-    }
-
-    @Override
-    protected AppProperty getBuiltInWordlistLocationProperty() {
-        return AppProperty.SEEDLIST_BUILTIN_PATH;
-    }
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.PwmConstants;
+import password.pwm.error.PwmException;
+import password.pwm.util.Helper;
+import password.pwm.util.TimeDuration;
+import password.pwm.util.localdb.LocalDB;
+import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.secure.PwmRandom;
+
+import java.util.Collections;
+import java.util.Map;
+
+public class SeedlistManager extends AbstractWordlist implements Wordlist {
+
+    private static final PwmLogger LOGGER = PwmLogger.forClass(SeedlistManager.class);
+
+    private int initialPopulationCounter = 0;
+
+    public SeedlistManager() {
+    }
+
+    public String randomSeed() {
+        if (wlStatus != STATUS.OPEN) {
+            return null;
+        }
+        final long startTime = System.currentTimeMillis();
+        String returnValue = null;
+        try {
+            final int seedCount = size();
+            if (seedCount > 1000) {
+                final int randomKey = PwmRandom.getInstance().nextInt(size());
+                final Object obj = localDB.get(getWordlistDB(), String.valueOf(randomKey));
+                if (obj != null) {
+                    returnValue = obj.toString();
+                }
+            }
+        } catch (Exception e) {
+            LOGGER.warn("error while generating random word: " + e.getMessage());
+        }
+
+        if (debugTrace) {
+            LOGGER.trace("getRandomSeed fetch time: " + TimeDuration.fromCurrent(startTime).asCompactString());
+        }
+        return returnValue;
+    }
+
+    protected Map<String, String> getWriteTxnForValue(final String value) {
+        final Map<String, String> txItem = Collections.singletonMap(String.valueOf(initialPopulationCounter), value);
+        initialPopulationCounter++;
+        return txItem;
+    }
+
+    public void init(final PwmApplication pwmApplication) throws PwmException {
+        super.init(pwmApplication);
+        final WordlistConfiguration wordlistConfiguration = new WordlistConfiguration(true, 0);
+
+        this.DEBUG_LABEL = PwmConstants.PWM_APP_NAME + "-Seedist";
+
+        final Thread t = new Thread(new Runnable() {
+            public void run() {
+                LOGGER.debug(DEBUG_LABEL + " starting up in background thread");
+                startup(pwmApplication.getLocalDB(), wordlistConfiguration);
+            }
+        }, Helper.makeThreadName(pwmApplication,SeedlistManager.class));
+
+        t.start();
+    }
+
+    @Override
+    protected PwmApplication.AppAttribute getMetaDataAppAttribute() {
+        return PwmApplication.AppAttribute.SEEDLIST_METADATA;
+    }
+
+    @Override
+    protected LocalDB.DB getWordlistDB() {
+        return LocalDB.DB.SEEDLIST_WORDS;
+    }
+
+    @Override
+    protected AppProperty getBuiltInWordlistLocationProperty() {
+        return AppProperty.SEEDLIST_BUILTIN_PATH;
+    }
+}

pwm/servlet/src/password/pwm/wordlist/SharedHistoryManager.java → pwm/servlet/src/password/pwm/svc/wordlist/SharedHistoryManager.java

@@ -1,434 +1,434 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.wordlist;
-
-import password.pwm.AppProperty;
-import password.pwm.PwmApplication;
-import password.pwm.PwmService;
-import password.pwm.config.PwmSetting;
-import password.pwm.config.option.DataStorageMethod;
-import password.pwm.error.PwmException;
-import password.pwm.health.HealthRecord;
-import password.pwm.http.PwmSession;
-import password.pwm.util.Helper;
-import password.pwm.util.Sleeper;
-import password.pwm.util.TimeDuration;
-import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.localdb.LocalDBException;
-import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.secure.PwmRandom;
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.*;
-
-
-public class SharedHistoryManager implements PwmService {
-// ------------------------------ FIELDS ------------------------------
-
-    private static final PwmLogger LOGGER = PwmLogger.forClass(SharedHistoryManager.class);
-
-    private static final String KEY_OLDEST_ENTRY = "oldest_entry";
-    private static final String KEY_VERSION = "version";
-    private static final String KEY_SALT = "salt";
-
-    private static final int MIN_CLEANER_FREQUENCY = 1000 * 60 * 60; // 1 hour
-    private static final int MAX_CLEANER_FREQUENCY = 1000 * 60 * 60 * 24; // 1 day
-
-    private static final LocalDB.DB META_DB = LocalDB.DB.SHAREDHISTORY_META;
-    private static final LocalDB.DB WORDS_DB = LocalDB.DB.SHAREDHISTORY_WORDS;
-
-    private volatile PwmService.STATUS status = STATUS.NEW;
-
-    private volatile Timer cleanerTimer = null;
-
-    private LocalDB localDB;
-    private String salt;
-    private long oldestEntry;
-
-    private final Settings settings = new Settings();
-
-
-// --------------------------- CONSTRUCTORS ---------------------------
-
-    public SharedHistoryManager() throws LocalDBException {
-    }
-
-// -------------------------- OTHER METHODS --------------------------
-
-    public void close() {
-        status = STATUS.CLOSED;
-        LOGGER.debug("closed");
-        if (cleanerTimer != null) {
-            cleanerTimer.cancel();
-        }
-        localDB = null;
-    }
-
-    public boolean containsWord(final String word) {
-        if (status != STATUS.OPEN) {
-            return false;
-        }
-
-        final String testWord = normalizeWord(word);
-
-        if (testWord == null) {
-            return false;
-        }
-
-        //final long startTime = System.currentTimeMillis();
-        boolean result = false;
-
-        try {
-            final String hashedWord = hashWord(testWord);
-            final boolean inDB = localDB.contains(WORDS_DB, hashedWord);
-            if (inDB) {
-                final long timeStamp = Long.parseLong(localDB.get(WORDS_DB, hashedWord));
-                final long entryAge = System.currentTimeMillis() - timeStamp;
-                if (entryAge < settings.maxAgeMs) {
-                    result = true;
-                }
-            }
-
-        } catch (Exception e) {
-            LOGGER.warn("error checking global history list: " + e.getMessage());
-        }
-
-        //LOGGER.trace(pwmSession, "successfully checked word, result=" + result + ", duration=" + new TimeDuration(System.currentTimeMillis(), startTime).asCompactString());
-        return result;
-    }
-
-    public PwmService.STATUS status() {
-        return status;
-    }
-
-    public Date getOldestEntryTime() {
-        if (size() > 0) {
-            return new Date(oldestEntry);
-        }
-        return null;
-    }
-
-    public int size() {
-        if (localDB != null) {
-            try {
-                return localDB.size(WORDS_DB);
-            } catch (Exception e) {
-                LOGGER.error("error checking wordlist size: " + e.getMessage());
-                return 0;
-            }
-        } else {
-            return 0;
-        }
-    }
-
-    private boolean checkDbVersion()
-            throws Exception {
-        LOGGER.trace("checking version number stored in LocalDB");
-
-        final Object versionInDB = localDB.get(META_DB, KEY_VERSION);
-        final String currentVersion = "version=" + settings.version;
-        final boolean result = currentVersion.equals(versionInDB);
-
-        if (!result) {
-            LOGGER.info("existing db version does not match current db version db=(" + versionInDB + ")  current=(" + currentVersion + "), clearing db");
-            localDB.truncate(WORDS_DB);
-            localDB.put(META_DB, KEY_VERSION, currentVersion);
-            localDB.remove(META_DB, KEY_OLDEST_ENTRY);
-        } else {
-            LOGGER.trace("existing db version matches current db version db=(" + versionInDB + ")  current=(" + currentVersion + ")");
-        }
-
-        return result;
-    }
-
-    private void init(final PwmApplication pwmApplication, final long maxAgeMs) {
-        status = STATUS.OPENING;
-        final long startTime = System.currentTimeMillis();
-
-        try {
-            checkDbVersion();
-        } catch (Exception e) {
-            LOGGER.error("error checking db version", e);
-            status = STATUS.CLOSED;
-            return;
-        }
-
-
-        try {
-            final String oldestEntryStr = localDB.get(META_DB, KEY_OLDEST_ENTRY);
-            if (oldestEntryStr == null || oldestEntryStr.length() < 1) {
-                oldestEntry = 0;
-                LOGGER.trace("no oldestEntry timestamp stored, will rescan");
-            } else {
-                oldestEntry = Long.parseLong(oldestEntryStr);
-                LOGGER.trace("oldest timestamp loaded from localDB, age is " + TimeDuration.fromCurrent(oldestEntry).asCompactString());
-            }
-        } catch (LocalDBException e) {
-            LOGGER.error("unexpected error loading oldest-entry meta record, will remain closed: " + e.getMessage(), e);
-            status = STATUS.CLOSED;
-            return;
-        }
-
-        try {
-            final int size = localDB.size(WORDS_DB);
-            final StringBuilder sb = new StringBuilder();
-            sb.append("open with ").append(size).append(" words (");
-            sb.append(new TimeDuration(System.currentTimeMillis(), startTime).asCompactString()).append(")");
-            sb.append(", maxAgeMs=").append(new TimeDuration(maxAgeMs).asCompactString());
-            sb.append(", oldestEntry=").append(new TimeDuration(System.currentTimeMillis(), oldestEntry).asCompactString());
-            LOGGER.info(sb.toString());
-        } catch (LocalDBException e) {
-            LOGGER.error("unexpected error examining size of DB, will remain closed: " + e.getMessage(), e);
-            status = STATUS.CLOSED;
-            return;
-        }
-
-        status = STATUS.OPEN;
-        //populateFromWordlist();  //only used for debugging!!!
-
-        if (pwmApplication.getApplicationMode() == PwmApplication.MODE.RUNNING || pwmApplication.getApplicationMode() == PwmApplication.MODE.CONFIGURATION) {
-            long frequencyMs = maxAgeMs > MAX_CLEANER_FREQUENCY ? MAX_CLEANER_FREQUENCY : maxAgeMs;
-            frequencyMs = frequencyMs < MIN_CLEANER_FREQUENCY ? MIN_CLEANER_FREQUENCY : frequencyMs;
-
-            LOGGER.debug("scheduling cleaner task to run once every " + new TimeDuration(frequencyMs).asCompactString());
-            final String threadName = Helper.makeThreadName(pwmApplication, this.getClass()) + " timer";
-            cleanerTimer = new Timer(threadName, true);
-            cleanerTimer.schedule(new CleanerTask(), 1000, frequencyMs);
-        }
-    }
-
-    private String normalizeWord(final String input) {
-        if (input == null) {
-            return null;
-        }
-
-        String word = input.trim();
-
-        if (settings.caseInsensitive) {
-            word = word.toLowerCase();
-        }
-
-        return word.length() > 0 ? word : null;
-    }
-
-    public synchronized void addWord(final PwmSession pwmSession, final String word) {
-        if (status != STATUS.OPEN) {
-            return;
-        }
-
-        final String addWord = normalizeWord(word);
-
-        if (addWord == null) {
-            return;
-        }
-
-        final long startTime = System.currentTimeMillis();
-
-        try {
-            final String hashedWord = hashWord(addWord);
-
-            final boolean preExisting = localDB.contains(WORDS_DB, hashedWord);
-            localDB.put(WORDS_DB, hashedWord, Long.toString(System.currentTimeMillis()));
-
-            {
-                final StringBuilder logOutput = new StringBuilder();
-                logOutput.append(preExisting ? "updated" : "added").append(" word");
-                logOutput.append(" (").append(new TimeDuration(System.currentTimeMillis(), startTime).asCompactString()).append(")");
-                logOutput.append(" (").append(this.size()).append(" total words)");
-                LOGGER.trace(logOutput.toString());
-            }
-        } catch (Exception e) {
-            LOGGER.warn(pwmSession, "error adding word to global history list: " + e.getMessage());
-        }
-    }
-
-    private String hashWord(final String word) throws NoSuchAlgorithmException {
-        final MessageDigest md = MessageDigest.getInstance(settings.hashName);
-        final String wordWithSalt = salt + word;
-        final int hashLoopCount = settings.hashIterations;
-        byte[] hashedAnswer = md.digest((wordWithSalt).getBytes());
-
-        for (int i = 0; i < hashLoopCount; i++) {
-            hashedAnswer = md.digest(hashedAnswer);
-        }
-
-        return Helper.binaryArrayToHex(hashedAnswer);
-    }
-
-    // -------------------------- INNER CLASSES --------------------------
-
-    private class CleanerTask extends TimerTask {
-        final Sleeper sleeper = new Sleeper(10);
-
-        private CleanerTask() {
-        }
-
-        public void run() {
-            try {
-                reduceWordDB();
-            } catch (LocalDBException e) {
-                LOGGER.error("error during old record purge: " + e.getMessage());
-            }
-        }
-
-
-        private void reduceWordDB()
-                throws LocalDBException {
-
-            if (localDB == null || localDB.status() != LocalDB.Status.OPEN) {
-                return;
-            }
-
-            final long oldestEntryAge = System.currentTimeMillis() - oldestEntry;
-            if (oldestEntryAge < settings.maxAgeMs) {
-                LOGGER.debug("skipping wordDB reduce operation, eldestEntry="
-                        + TimeDuration.asCompactString(oldestEntryAge)
-                        + ", maxAge="
-                        + TimeDuration.asCompactString(settings.maxAgeMs));
-                return;
-            }
-
-            final long startTime = System.currentTimeMillis();
-            final int initialSize = size();
-            int removeCount = 0;
-            long localOldestEntry = System.currentTimeMillis();
-
-            LOGGER.debug("beginning wordDB reduce operation, examining " + initialSize + " words for entries older than " + TimeDuration.asCompactString(settings.maxAgeMs));
-
-            LocalDB.LocalDBIterator<String> keyIterator = null;
-            try {
-                keyIterator = localDB.iterator(WORDS_DB);
-                while (status == STATUS.OPEN && keyIterator.hasNext()) {
-                    final String key = keyIterator.next();
-                    final String value = localDB.get(WORDS_DB, key);
-                    final long timeStamp = Long.parseLong(value);
-                    final long entryAge = System.currentTimeMillis() - timeStamp;
-
-                    if (entryAge > settings.maxAgeMs) {
-                        localDB.remove(WORDS_DB, key);
-                        removeCount++;
-
-                        if (removeCount % 1000 == 0) {
-                            LOGGER.trace("wordDB reduce operation in progress, removed=" + removeCount + ", total=" + (initialSize - removeCount));
-                        }
-                    } else {
-                        localOldestEntry = timeStamp < localOldestEntry ? timeStamp : localOldestEntry;
-                    }
-                    sleeper.sleep();
-                }
-            } finally {
-                try {
-                    if (keyIterator != null) {
-                        keyIterator.close();
-                    }
-                } catch (Exception e) {
-                    LOGGER.warn("error returning LocalDB iterator: " + e.getMessage());
-                }
-            }
-
-            //update the oldest entry
-            if (status == STATUS.OPEN) {
-                oldestEntry = localOldestEntry;
-                localDB.put(META_DB, KEY_OLDEST_ENTRY, Long.toString(oldestEntry));
-            }
-
-            LOGGER.debug("completed wordDB reduce operation" + ", removed=" + removeCount
-                    + ", totalRemaining=" + size()
-                    + ", oldestEntry=" + TimeDuration.asCompactString(oldestEntry)
-                    + " in " + TimeDuration.fromCurrent(startTime).asCompactString());
-        }
-    }
-
-    public List<HealthRecord> healthCheck() {
-        return null;
-    }
-
-    public void init(final PwmApplication pwmApplication)
-            throws PwmException
-    {
-        settings.maxAgeMs = 1000 *  pwmApplication.getConfig().readSettingAsLong(PwmSetting.PASSWORD_SHAREDHISTORY_MAX_AGE); // convert to MS;
-        settings.caseInsensitive = Boolean.parseBoolean(pwmApplication.getConfig().readAppProperty(AppProperty.SECURITY_SHAREDHISTORY_CASE_INSENSITIVE));
-        settings.hashName = pwmApplication.getConfig().readAppProperty(AppProperty.SECURITY_SHAREDHISTORY_HASH_NAME);
-        settings.hashIterations = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.SECURITY_SHAREDHISTORY_HASH_ITERATIONS));
-        settings.version = "2" + "_" + settings.hashName + "_" + settings.hashIterations + "_" + settings.caseInsensitive;
-
-        final int SALT_LENGTH = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.SECURITY_SHAREDHISTORY_SALT_LENGTH));
-        this.localDB = pwmApplication.getLocalDB();
-
-        boolean needsClearing = false;
-        if (localDB == null) {
-            LOGGER.info("LocalDB is not available, will remain closed");
-            status = STATUS.CLOSED;
-            return;
-        }
-
-        if (settings.maxAgeMs < 1) {
-            LOGGER.debug("max age=" + settings.maxAgeMs + ", will remain closed");
-            needsClearing = true;
-        }
-
-        {
-            this.salt = localDB.get(META_DB, KEY_SALT);
-            if (salt == null || salt.length() < SALT_LENGTH) {
-                LOGGER.warn("stored global salt value is not present, creating new salt");
-                this.salt = PwmRandom.getInstance().alphaNumericString(SALT_LENGTH);
-                localDB.put(META_DB, KEY_SALT,this.salt);
-                needsClearing = true;
-            }
-        }
-
-        if (needsClearing) {
-            LOGGER.trace("clearing wordlist");
-            try {
-                localDB.truncate(WORDS_DB);
-            } catch (Exception e) {
-                LOGGER.error("error during wordlist truncate", e);
-            }
-        }
-
-        new Thread(new Runnable() {
-            public void run() {
-                LOGGER.debug("starting up in background thread");
-                init(pwmApplication, settings.maxAgeMs);
-            }
-        }, Helper.makeThreadName(pwmApplication, this.getClass()) + " initializer").start();
-    }
-
-    private static class Settings {
-        private String version;
-        private String hashName;
-        private int hashIterations;
-        private long maxAgeMs;
-        private boolean caseInsensitive;
-    }
-
-    public ServiceInfo serviceInfo()
-    {
-        if (status == STATUS.OPEN) {
-            return new ServiceInfo(Collections.singletonList(DataStorageMethod.LOCALDB));
-        } else {
-            return new ServiceInfo(Collections.<DataStorageMethod>emptyList());
-        }
-    }
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.config.PwmSetting;
+import password.pwm.config.option.DataStorageMethod;
+import password.pwm.error.PwmException;
+import password.pwm.health.HealthRecord;
+import password.pwm.http.PwmSession;
+import password.pwm.svc.PwmService;
+import password.pwm.util.Helper;
+import password.pwm.util.Sleeper;
+import password.pwm.util.TimeDuration;
+import password.pwm.util.localdb.LocalDB;
+import password.pwm.util.localdb.LocalDBException;
+import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.secure.PwmRandom;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.*;
+
+
+public class SharedHistoryManager implements PwmService {
+// ------------------------------ FIELDS ------------------------------
+
+    private static final PwmLogger LOGGER = PwmLogger.forClass(SharedHistoryManager.class);
+
+    private static final String KEY_OLDEST_ENTRY = "oldest_entry";
+    private static final String KEY_VERSION = "version";
+    private static final String KEY_SALT = "salt";
+
+    private static final int MIN_CLEANER_FREQUENCY = 1000 * 60 * 60; // 1 hour
+    private static final int MAX_CLEANER_FREQUENCY = 1000 * 60 * 60 * 24; // 1 day
+
+    private static final LocalDB.DB META_DB = LocalDB.DB.SHAREDHISTORY_META;
+    private static final LocalDB.DB WORDS_DB = LocalDB.DB.SHAREDHISTORY_WORDS;
+
+    private volatile PwmService.STATUS status = STATUS.NEW;
+
+    private volatile Timer cleanerTimer = null;
+
+    private LocalDB localDB;
+    private String salt;
+    private long oldestEntry;
+
+    private final Settings settings = new Settings();
+
+
+// --------------------------- CONSTRUCTORS ---------------------------
+
+    public SharedHistoryManager() throws LocalDBException {
+    }
+
+// -------------------------- OTHER METHODS --------------------------
+
+    public void close() {
+        status = STATUS.CLOSED;
+        LOGGER.debug("closed");
+        if (cleanerTimer != null) {
+            cleanerTimer.cancel();
+        }
+        localDB = null;
+    }
+
+    public boolean containsWord(final String word) {
+        if (status != STATUS.OPEN) {
+            return false;
+        }
+
+        final String testWord = normalizeWord(word);
+
+        if (testWord == null) {
+            return false;
+        }
+
+        //final long startTime = System.currentTimeMillis();
+        boolean result = false;
+
+        try {
+            final String hashedWord = hashWord(testWord);
+            final boolean inDB = localDB.contains(WORDS_DB, hashedWord);
+            if (inDB) {
+                final long timeStamp = Long.parseLong(localDB.get(WORDS_DB, hashedWord));
+                final long entryAge = System.currentTimeMillis() - timeStamp;
+                if (entryAge < settings.maxAgeMs) {
+                    result = true;
+                }
+            }
+
+        } catch (Exception e) {
+            LOGGER.warn("error checking global history list: " + e.getMessage());
+        }
+
+        //LOGGER.trace(pwmSession, "successfully checked word, result=" + result + ", duration=" + new TimeDuration(System.currentTimeMillis(), startTime).asCompactString());
+        return result;
+    }
+
+    public PwmService.STATUS status() {
+        return status;
+    }
+
+    public Date getOldestEntryTime() {
+        if (size() > 0) {
+            return new Date(oldestEntry);
+        }
+        return null;
+    }
+
+    public int size() {
+        if (localDB != null) {
+            try {
+                return localDB.size(WORDS_DB);
+            } catch (Exception e) {
+                LOGGER.error("error checking wordlist size: " + e.getMessage());
+                return 0;
+            }
+        } else {
+            return 0;
+        }
+    }
+
+    private boolean checkDbVersion()
+            throws Exception {
+        LOGGER.trace("checking version number stored in LocalDB");
+
+        final Object versionInDB = localDB.get(META_DB, KEY_VERSION);
+        final String currentVersion = "version=" + settings.version;
+        final boolean result = currentVersion.equals(versionInDB);
+
+        if (!result) {
+            LOGGER.info("existing db version does not match current db version db=(" + versionInDB + ")  current=(" + currentVersion + "), clearing db");
+            localDB.truncate(WORDS_DB);
+            localDB.put(META_DB, KEY_VERSION, currentVersion);
+            localDB.remove(META_DB, KEY_OLDEST_ENTRY);
+        } else {
+            LOGGER.trace("existing db version matches current db version db=(" + versionInDB + ")  current=(" + currentVersion + ")");
+        }
+
+        return result;
+    }
+
+    private void init(final PwmApplication pwmApplication, final long maxAgeMs) {
+        status = STATUS.OPENING;
+        final long startTime = System.currentTimeMillis();
+
+        try {
+            checkDbVersion();
+        } catch (Exception e) {
+            LOGGER.error("error checking db version", e);
+            status = STATUS.CLOSED;
+            return;
+        }
+
+
+        try {
+            final String oldestEntryStr = localDB.get(META_DB, KEY_OLDEST_ENTRY);
+            if (oldestEntryStr == null || oldestEntryStr.length() < 1) {
+                oldestEntry = 0;
+                LOGGER.trace("no oldestEntry timestamp stored, will rescan");
+            } else {
+                oldestEntry = Long.parseLong(oldestEntryStr);
+                LOGGER.trace("oldest timestamp loaded from localDB, age is " + TimeDuration.fromCurrent(oldestEntry).asCompactString());
+            }
+        } catch (LocalDBException e) {
+            LOGGER.error("unexpected error loading oldest-entry meta record, will remain closed: " + e.getMessage(), e);
+            status = STATUS.CLOSED;
+            return;
+        }
+
+        try {
+            final int size = localDB.size(WORDS_DB);
+            final StringBuilder sb = new StringBuilder();
+            sb.append("open with ").append(size).append(" words (");
+            sb.append(new TimeDuration(System.currentTimeMillis(), startTime).asCompactString()).append(")");
+            sb.append(", maxAgeMs=").append(new TimeDuration(maxAgeMs).asCompactString());
+            sb.append(", oldestEntry=").append(new TimeDuration(System.currentTimeMillis(), oldestEntry).asCompactString());
+            LOGGER.info(sb.toString());
+        } catch (LocalDBException e) {
+            LOGGER.error("unexpected error examining size of DB, will remain closed: " + e.getMessage(), e);
+            status = STATUS.CLOSED;
+            return;
+        }
+
+        status = STATUS.OPEN;
+        //populateFromWordlist();  //only used for debugging!!!
+
+        if (pwmApplication.getApplicationMode() == PwmApplication.MODE.RUNNING || pwmApplication.getApplicationMode() == PwmApplication.MODE.CONFIGURATION) {
+            long frequencyMs = maxAgeMs > MAX_CLEANER_FREQUENCY ? MAX_CLEANER_FREQUENCY : maxAgeMs;
+            frequencyMs = frequencyMs < MIN_CLEANER_FREQUENCY ? MIN_CLEANER_FREQUENCY : frequencyMs;
+
+            LOGGER.debug("scheduling cleaner task to run once every " + new TimeDuration(frequencyMs).asCompactString());
+            final String threadName = Helper.makeThreadName(pwmApplication, this.getClass()) + " timer";
+            cleanerTimer = new Timer(threadName, true);
+            cleanerTimer.schedule(new CleanerTask(), 1000, frequencyMs);
+        }
+    }
+
+    private String normalizeWord(final String input) {
+        if (input == null) {
+            return null;
+        }
+
+        String word = input.trim();
+
+        if (settings.caseInsensitive) {
+            word = word.toLowerCase();
+        }
+
+        return word.length() > 0 ? word : null;
+    }
+
+    public synchronized void addWord(final PwmSession pwmSession, final String word) {
+        if (status != STATUS.OPEN) {
+            return;
+        }
+
+        final String addWord = normalizeWord(word);
+
+        if (addWord == null) {
+            return;
+        }
+
+        final long startTime = System.currentTimeMillis();
+
+        try {
+            final String hashedWord = hashWord(addWord);
+
+            final boolean preExisting = localDB.contains(WORDS_DB, hashedWord);
+            localDB.put(WORDS_DB, hashedWord, Long.toString(System.currentTimeMillis()));
+
+            {
+                final StringBuilder logOutput = new StringBuilder();
+                logOutput.append(preExisting ? "updated" : "added").append(" word");
+                logOutput.append(" (").append(new TimeDuration(System.currentTimeMillis(), startTime).asCompactString()).append(")");
+                logOutput.append(" (").append(this.size()).append(" total words)");
+                LOGGER.trace(logOutput.toString());
+            }
+        } catch (Exception e) {
+            LOGGER.warn(pwmSession, "error adding word to global history list: " + e.getMessage());
+        }
+    }
+
+    private String hashWord(final String word) throws NoSuchAlgorithmException {
+        final MessageDigest md = MessageDigest.getInstance(settings.hashName);
+        final String wordWithSalt = salt + word;
+        final int hashLoopCount = settings.hashIterations;
+        byte[] hashedAnswer = md.digest((wordWithSalt).getBytes());
+
+        for (int i = 0; i < hashLoopCount; i++) {
+            hashedAnswer = md.digest(hashedAnswer);
+        }
+
+        return Helper.binaryArrayToHex(hashedAnswer);
+    }
+
+    // -------------------------- INNER CLASSES --------------------------
+
+    private class CleanerTask extends TimerTask {
+        final Sleeper sleeper = new Sleeper(10);
+
+        private CleanerTask() {
+        }
+
+        public void run() {
+            try {
+                reduceWordDB();
+            } catch (LocalDBException e) {
+                LOGGER.error("error during old record purge: " + e.getMessage());
+            }
+        }
+
+
+        private void reduceWordDB()
+                throws LocalDBException {
+
+            if (localDB == null || localDB.status() != LocalDB.Status.OPEN) {
+                return;
+            }
+
+            final long oldestEntryAge = System.currentTimeMillis() - oldestEntry;
+            if (oldestEntryAge < settings.maxAgeMs) {
+                LOGGER.debug("skipping wordDB reduce operation, eldestEntry="
+                        + TimeDuration.asCompactString(oldestEntryAge)
+                        + ", maxAge="
+                        + TimeDuration.asCompactString(settings.maxAgeMs));
+                return;
+            }
+
+            final long startTime = System.currentTimeMillis();
+            final int initialSize = size();
+            int removeCount = 0;
+            long localOldestEntry = System.currentTimeMillis();
+
+            LOGGER.debug("beginning wordDB reduce operation, examining " + initialSize + " words for entries older than " + TimeDuration.asCompactString(settings.maxAgeMs));
+
+            LocalDB.LocalDBIterator<String> keyIterator = null;
+            try {
+                keyIterator = localDB.iterator(WORDS_DB);
+                while (status == STATUS.OPEN && keyIterator.hasNext()) {
+                    final String key = keyIterator.next();
+                    final String value = localDB.get(WORDS_DB, key);
+                    final long timeStamp = Long.parseLong(value);
+                    final long entryAge = System.currentTimeMillis() - timeStamp;
+
+                    if (entryAge > settings.maxAgeMs) {
+                        localDB.remove(WORDS_DB, key);
+                        removeCount++;
+
+                        if (removeCount % 1000 == 0) {
+                            LOGGER.trace("wordDB reduce operation in progress, removed=" + removeCount + ", total=" + (initialSize - removeCount));
+                        }
+                    } else {
+                        localOldestEntry = timeStamp < localOldestEntry ? timeStamp : localOldestEntry;
+                    }
+                    sleeper.sleep();
+                }
+            } finally {
+                try {
+                    if (keyIterator != null) {
+                        keyIterator.close();
+                    }
+                } catch (Exception e) {
+                    LOGGER.warn("error returning LocalDB iterator: " + e.getMessage());
+                }
+            }
+
+            //update the oldest entry
+            if (status == STATUS.OPEN) {
+                oldestEntry = localOldestEntry;
+                localDB.put(META_DB, KEY_OLDEST_ENTRY, Long.toString(oldestEntry));
+            }
+
+            LOGGER.debug("completed wordDB reduce operation" + ", removed=" + removeCount
+                    + ", totalRemaining=" + size()
+                    + ", oldestEntry=" + TimeDuration.asCompactString(oldestEntry)
+                    + " in " + TimeDuration.fromCurrent(startTime).asCompactString());
+        }
+    }
+
+    public List<HealthRecord> healthCheck() {
+        return null;
+    }
+
+    public void init(final PwmApplication pwmApplication)
+            throws PwmException
+    {
+        settings.maxAgeMs = 1000 *  pwmApplication.getConfig().readSettingAsLong(PwmSetting.PASSWORD_SHAREDHISTORY_MAX_AGE); // convert to MS;
+        settings.caseInsensitive = Boolean.parseBoolean(pwmApplication.getConfig().readAppProperty(AppProperty.SECURITY_SHAREDHISTORY_CASE_INSENSITIVE));
+        settings.hashName = pwmApplication.getConfig().readAppProperty(AppProperty.SECURITY_SHAREDHISTORY_HASH_NAME);
+        settings.hashIterations = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.SECURITY_SHAREDHISTORY_HASH_ITERATIONS));
+        settings.version = "2" + "_" + settings.hashName + "_" + settings.hashIterations + "_" + settings.caseInsensitive;
+
+        final int SALT_LENGTH = Integer.parseInt(pwmApplication.getConfig().readAppProperty(AppProperty.SECURITY_SHAREDHISTORY_SALT_LENGTH));
+        this.localDB = pwmApplication.getLocalDB();
+
+        boolean needsClearing = false;
+        if (localDB == null) {
+            LOGGER.info("LocalDB is not available, will remain closed");
+            status = STATUS.CLOSED;
+            return;
+        }
+
+        if (settings.maxAgeMs < 1) {
+            LOGGER.debug("max age=" + settings.maxAgeMs + ", will remain closed");
+            needsClearing = true;
+        }
+
+        {
+            this.salt = localDB.get(META_DB, KEY_SALT);
+            if (salt == null || salt.length() < SALT_LENGTH) {
+                LOGGER.warn("stored global salt value is not present, creating new salt");
+                this.salt = PwmRandom.getInstance().alphaNumericString(SALT_LENGTH);
+                localDB.put(META_DB, KEY_SALT,this.salt);
+                needsClearing = true;
+            }
+        }
+
+        if (needsClearing) {
+            LOGGER.trace("clearing wordlist");
+            try {
+                localDB.truncate(WORDS_DB);
+            } catch (Exception e) {
+                LOGGER.error("error during wordlist truncate", e);
+            }
+        }
+
+        new Thread(new Runnable() {
+            public void run() {
+                LOGGER.debug("starting up in background thread");
+                init(pwmApplication, settings.maxAgeMs);
+            }
+        }, Helper.makeThreadName(pwmApplication, this.getClass()) + " initializer").start();
+    }
+
+    private static class Settings {
+        private String version;
+        private String hashName;
+        private int hashIterations;
+        private long maxAgeMs;
+        private boolean caseInsensitive;
+    }
+
+    public ServiceInfo serviceInfo()
+    {
+        if (status == STATUS.OPEN) {
+            return new ServiceInfo(Collections.singletonList(DataStorageMethod.LOCALDB));
+        } else {
+            return new ServiceInfo(Collections.<DataStorageMethod>emptyList());
+        }
+    }
+}

pwm/servlet/src/password/pwm/wordlist/StoredWordlistDataBean.java → pwm/servlet/src/password/pwm/svc/wordlist/StoredWordlistDataBean.java

@@ -1,4 +1,4 @@
-package password.pwm.wordlist;
+package password.pwm.svc.wordlist;
 
 import java.io.Serializable;
 import java.util.Date;

pwm/servlet/src/password/pwm/wordlist/Wordlist.java → pwm/servlet/src/password/pwm/svc/wordlist/Wordlist.java

@@ -1,46 +1,46 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.wordlist;
-
-import password.pwm.PwmService;
-import password.pwm.error.PwmUnrecoverableException;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-
-public interface Wordlist extends PwmService {
-
-    boolean containsWord(final String word);
-
-    int size();
-
-    StoredWordlistDataBean readMetadata();
-
-
-    void populate(InputStream inputStream)
-            throws IOException, PwmUnrecoverableException;
-
-    void populateBuiltIn()
-                    throws IOException, PwmUnrecoverableException;
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.svc.PwmService;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+
+public interface Wordlist extends PwmService {
+
+    boolean containsWord(final String word);
+
+    int size();
+
+    StoredWordlistDataBean readMetadata();
+
+
+    void populate(InputStream inputStream)
+            throws IOException, PwmUnrecoverableException;
+
+    void populateBuiltIn()
+                    throws IOException, PwmUnrecoverableException;
+}

pwm/servlet/src/password/pwm/wordlist/WordlistConfiguration.java → pwm/servlet/src/password/pwm/svc/wordlist/WordlistConfiguration.java

@@ -1,46 +1,46 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.wordlist;
-
-import java.io.Serializable;
-
-public class WordlistConfiguration implements Serializable {
-    final private boolean caseSensitive;
-    final private int checkSize;
-
-    public WordlistConfiguration(
-            final boolean caseSensitive,
-            final int checkSize
-    ) {
-        this.caseSensitive = caseSensitive;
-        this.checkSize = checkSize;
-    }
-
-    public boolean isCaseSensitive() {
-        return caseSensitive;
-    }
-
-    public int getCheckSize() {
-        return checkSize;
-    }
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import java.io.Serializable;
+
+public class WordlistConfiguration implements Serializable {
+    final private boolean caseSensitive;
+    final private int checkSize;
+
+    public WordlistConfiguration(
+            final boolean caseSensitive,
+            final int checkSize
+    ) {
+        this.caseSensitive = caseSensitive;
+        this.checkSize = checkSize;
+    }
+
+    public boolean isCaseSensitive() {
+        return caseSensitive;
+    }
+
+    public int getCheckSize() {
+        return checkSize;
+    }
+}

pwm/servlet/src/password/pwm/wordlist/WordlistManager.java → pwm/servlet/src/password/pwm/svc/wordlist/WordlistManager.java

@@ -1,100 +1,100 @@
-/*
- * Password Management Servlets (PWM)
- * http://code.google.com/p/pwm/
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2015 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  S  ee the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.wordlist;
-
-import password.pwm.AppProperty;
-import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
-import password.pwm.config.PwmSetting;
-import password.pwm.error.PwmException;
-import password.pwm.util.Helper;
-import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.logging.PwmLogger;
-
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-
-/**
- * @author Jason D. Rivard
- */
-public class WordlistManager extends AbstractWordlist implements Wordlist {
-
-    private static final PwmLogger LOGGER = PwmLogger.forClass(WordlistManager.class);
-
-    public WordlistManager() {
-    }
-
-
-    protected Map<String,String> getWriteTxnForValue(final String value) {
-        final Map<String,String> returnSet = new TreeMap<>();
-        final Set<String> chunkedWords = chunkWord(value,this.wordlistConfiguration.getCheckSize());
-        for (final String word : chunkedWords) {
-            returnSet.put(word,"");
-        }
-        return returnSet;
-    }
-
-    public void init(final PwmApplication pwmApplication) throws PwmException {
-        super.init(pwmApplication);
-        final boolean caseSensitive = pwmApplication.getConfig().readSettingAsBoolean(PwmSetting.WORDLIST_CASE_SENSITIVE);
-        final int checkSize = (int)pwmApplication.getConfig().readSettingAsLong(PwmSetting.PASSWORD_WORDLIST_WORDSIZE);
-        final WordlistConfiguration wordlistConfiguration = new WordlistConfiguration(caseSensitive, checkSize);
-
-        this.DEBUG_LABEL = PwmConstants.PWM_APP_NAME + "-Wordlist";
-
-        final Thread t = new Thread(new Runnable() {
-            public void run()
-            {
-                LOGGER.debug(DEBUG_LABEL + " starting up in background thread");
-                try {
-                    startup(pwmApplication.getLocalDB(), wordlistConfiguration);
-                } catch (Exception e) {
-                    try {
-                        LOGGER.warn("error during startup: " + e.getMessage());
-                    } catch (Exception moreE) { /* probably due to shut down */ }
-                }
-            }
-        }, Helper.makeThreadName(pwmApplication, WordlistManager.class));
-
-        t.start();
-    }
-
-    @Override
-    protected PwmApplication.AppAttribute getMetaDataAppAttribute() {
-        return PwmApplication.AppAttribute.WORDLIST_METADATA;
-    }
-
-    @Override
-    protected LocalDB.DB getWordlistDB() {
-        return LocalDB.DB.WORDLIST_WORDS;
-    }
-
-    @Override
-    protected AppProperty getBuiltInWordlistLocationProperty() {
-        return AppProperty.WORDLIST_BUILTIN_PATH;
-    }
-
-
-}
+/*
+ * Password Management Servlets (PWM)
+ * http://code.google.com/p/pwm/
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2015 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  S  ee the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.PwmConstants;
+import password.pwm.config.PwmSetting;
+import password.pwm.error.PwmException;
+import password.pwm.util.Helper;
+import password.pwm.util.localdb.LocalDB;
+import password.pwm.util.logging.PwmLogger;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+
+/**
+ * @author Jason D. Rivard
+ */
+public class WordlistManager extends AbstractWordlist implements Wordlist {
+
+    private static final PwmLogger LOGGER = PwmLogger.forClass(WordlistManager.class);
+
+    public WordlistManager() {
+    }
+
+
+    protected Map<String,String> getWriteTxnForValue(final String value) {
+        final Map<String,String> returnSet = new TreeMap<>();
+        final Set<String> chunkedWords = chunkWord(value,this.wordlistConfiguration.getCheckSize());
+        for (final String word : chunkedWords) {
+            returnSet.put(word,"");
+        }
+        return returnSet;
+    }
+
+    public void init(final PwmApplication pwmApplication) throws PwmException {
+        super.init(pwmApplication);
+        final boolean caseSensitive = pwmApplication.getConfig().readSettingAsBoolean(PwmSetting.WORDLIST_CASE_SENSITIVE);
+        final int checkSize = (int)pwmApplication.getConfig().readSettingAsLong(PwmSetting.PASSWORD_WORDLIST_WORDSIZE);
+        final WordlistConfiguration wordlistConfiguration = new WordlistConfiguration(caseSensitive, checkSize);
+
+        this.DEBUG_LABEL = PwmConstants.PWM_APP_NAME + "-Wordlist";
+
+        final Thread t = new Thread(new Runnable() {
+            public void run()
+            {
+                LOGGER.debug(DEBUG_LABEL + " starting up in background thread");
+                try {
+                    startup(pwmApplication.getLocalDB(), wordlistConfiguration);
+                } catch (Exception e) {
+                    try {
+                        LOGGER.warn("error during startup: " + e.getMessage());
+                    } catch (Exception moreE) { /* probably due to shut down */ }
+                }
+            }
+        }, Helper.makeThreadName(pwmApplication, WordlistManager.class));
+
+        t.start();
+    }
+
+    @Override
+    protected PwmApplication.AppAttribute getMetaDataAppAttribute() {
+        return PwmApplication.AppAttribute.WORDLIST_METADATA;
+    }
+
+    @Override
+    protected LocalDB.DB getWordlistDB() {
+        return LocalDB.DB.WORDLIST_WORDS;
+    }
+
+    @Override
+    protected AppProperty getBuiltInWordlistLocationProperty() {
+        return AppProperty.WORDLIST_BUILTIN_PATH;
+    }
+
+
+}