Merge branch 'l10n-ldap'

server/src/main/java/password/pwm/PwmApplication.java

@@ -1084,7 +1084,14 @@ public class PwmApplication
 
             try
             {
-                executor.execute( runnable );
+                if ( !executor.isShutdown() )
+                {
+                    executor.execute( runnable );
+                }
+                else
+                {
+                    LOGGER.trace( "skipping scheduled job " + runnable + " on shutdown executor + " + executor );
+                }
             }
             catch ( Throwable t )
             {

server/src/main/java/password/pwm/bean/pub/PublicUserInfoBean.java

@@ -22,7 +22,8 @@
 
 package password.pwm.bean.pub;
 
-import lombok.Getter;
+import lombok.Builder;
+import lombok.Value;
 import password.pwm.bean.PasswordStatus;
 import password.pwm.config.Configuration;
 import password.pwm.config.profile.PwmPasswordRule;
@@ -39,7 +40,8 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
-@Getter
+@Value
+@Builder
 public class PublicUserInfoBean implements Serializable
 {
     private String userDN;
@@ -52,6 +54,7 @@ public class PublicUserInfoBean implements Serializable
     private String userSmsNumber;
     private String userSmsNumber2;
     private String userSmsNumber3;
+    private String language;
     private Instant passwordExpirationTime;
     private Instant passwordLastModifiedTime;
     private Instant lastLoginTime;
@@ -75,11 +78,11 @@ public class PublicUserInfoBean implements Serializable
     )
             throws PwmUnrecoverableException
     {
-        final PublicUserInfoBean publicUserInfoBean = new PublicUserInfoBean();
+        final PublicUserInfoBean.PublicUserInfoBeanBuilder publicUserInfoBean = PublicUserInfoBean.builder();
         publicUserInfoBean.userDN = ( userInfoBean.getUserIdentity() == null ) ? "" : userInfoBean.getUserIdentity().getUserDN();
         publicUserInfoBean.ldapProfile = ( userInfoBean.getUserIdentity() == null ) ? "" : userInfoBean.getUserIdentity().getLdapProfileID();
         publicUserInfoBean.userID = userInfoBean.getUsername();
-        publicUserInfoBean.userGUID = publicUserInfoBean.getUserGUID();
+        publicUserInfoBean.userGUID = userInfoBean.getUserGuid();
         publicUserInfoBean.userEmailAddress = userInfoBean.getUserEmailAddress();
         publicUserInfoBean.userEmailAddress2 = userInfoBean.getUserEmailAddress2();
         publicUserInfoBean.userEmailAddress3 = userInfoBean.getUserEmailAddress3();
@@ -97,6 +100,7 @@ public class PublicUserInfoBean implements Serializable
         publicUserInfoBean.requiresUpdateProfile = userInfoBean.isRequiresUpdateProfile();
         publicUserInfoBean.requiresOtpConfig = userInfoBean.isRequiresOtpConfig();
         publicUserInfoBean.requiresInteraction = userInfoBean.isRequiresInteraction();
+        publicUserInfoBean.language = userInfoBean.getLanguage();
 
         publicUserInfoBean.passwordPolicy = new HashMap<>();
         for ( final PwmPasswordRule rule : PwmPasswordRule.values() )
@@ -116,6 +120,6 @@ public class PublicUserInfoBean implements Serializable
             publicUserInfoBean.attributes = Collections.unmodifiableMap( userInfoBean.getCachedAttributeValues() );
         }
 
-        return publicUserInfoBean;
+        return publicUserInfoBean.build();
     }
 }

server/src/main/java/password/pwm/config/PwmSetting.java

@@ -97,6 +97,8 @@ public enum PwmSetting
             "pwm.appProperty.overrides", PwmSettingSyntax.STRING_ARRAY, PwmSettingCategory.GENERAL ),
 
     // clustering
+    CLUSTER_ENABLED(
+            "cluster.enable", PwmSettingSyntax.BOOLEAN, PwmSettingCategory.CLUSTERING ),
     CLUSTER_STORAGE_MODE(
             "cluster.storageMode", PwmSettingSyntax.SELECT, PwmSettingCategory.CLUSTERING ),
     SECURITY_LOGIN_SESSION_MODE(
@@ -278,10 +280,15 @@ public enum PwmSetting
             "peopleSearch.orgChart.assistantAttribute", PwmSettingSyntax.STRING, PwmSettingCategory.LDAP_ATTRIBUTES ),
     LDAP_ATTRIBUTE_ORGCHART_WORKFORCEID(
             "peopleSearch.orgChart.workforceIdAttribute", PwmSettingSyntax.STRING, PwmSettingCategory.LDAP_ATTRIBUTES ),
+    LDAP_ATTRIBUTE_LANGUAGE(
+            "ldap.user.language.attribute", PwmSettingSyntax.STRING, PwmSettingCategory.LDAP_ATTRIBUTES ),
+    LDAP_ATTRIBUTE_PWNOTIFY(
+            "ldap.user.pwNotify.attribute", PwmSettingSyntax.STRING, PwmSettingCategory.LDAP_ATTRIBUTES ),
+    LDAP_AUTO_SET_LANGUAGE_VALUE(
+            "ldap.user.language.autoSet", PwmSettingSyntax.SELECT, PwmSettingCategory.LDAP_ATTRIBUTES ),
     AUTO_ADD_OBJECT_CLASSES(
             "ldap.addObjectClasses", PwmSettingSyntax.STRING_ARRAY, PwmSettingCategory.LDAP_ATTRIBUTES ),
 
-
     // ldap global settings
     LDAP_PROFILE_LIST(
             "ldap.profile.list", PwmSettingSyntax.PROFILE, PwmSettingCategory.INTERNAL ),
@@ -1110,6 +1117,9 @@ public enum PwmSetting
     // pw expiry notice
     PW_EXPY_NOTIFY_ENABLE(
             "pwNotify.enable", PwmSettingSyntax.BOOLEAN, PwmSettingCategory.PW_EXP_NOTIFY ),
+    PW_EXPY_NOTIFY_STORAGE_MODE(
+            "pwNotify.storageMode", PwmSettingSyntax.SELECT, PwmSettingCategory.PW_EXP_NOTIFY ),
+
     PW_EXPY_NOTIFY_PERMISSION(
             "pwNotify.queryString", PwmSettingSyntax.USER_PERMISSION, PwmSettingCategory.PW_EXP_NOTIFY ),
     PW_EXPY_NOTIFY_INTERVAL(

server/src/main/java/password/pwm/config/option/AutoSetLdapUserLanguage.java

@@ -0,0 +1,29 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.config.option;
+
+public enum AutoSetLdapUserLanguage
+{
+    disabled,
+    enabled,
+}

server/src/main/java/password/pwm/error/PwmError.java

@@ -307,6 +307,8 @@ public enum PwmError
             5093, "Error_ClusterServiceError", null ),
     ERROR_WORDLIST_IMPORT_ERROR(
             5094, "Error_WordlistImportError", null ),
+    ERROR_PWNOTIFY_SERVICE_ERROR(
+            5095, "Error_PwNotifyServiceError", null ),
 
     ERROR_REMOTE_ERROR_VALUE(
             6000, "Error_RemoteErrorValue", null, ErrorFlag.Permanent ),

server/src/main/java/password/pwm/health/HealthMessage.java

@@ -88,6 +88,7 @@ public enum HealthMessage
     LocalDBLogger_NOTOPEN( HealthStatus.CAUTION, HealthTopic.LocalDB ),
     LocalDBLogger_HighRecordCount( HealthStatus.CAUTION, HealthTopic.LocalDB ),
     LocalDBLogger_OldRecordPresent( HealthStatus.CAUTION, HealthTopic.LocalDB ),
+    ServiceClosed( HealthStatus.CAUTION, HealthTopic.Application ),
     ServiceClosed_LocalDBUnavail( HealthStatus.CAUTION, HealthTopic.Application ),
     ServiceClosed_AppReadOnly( HealthStatus.CAUTION, HealthTopic.Application ),
     SMS_SendFailure( HealthStatus.WARN, HealthTopic.SMS ),

server/src/main/java/password/pwm/http/servlet/admin/AdminServlet.java

@@ -710,7 +710,7 @@ public class AdminServlet extends ControlledPwmServlet
     }
 
     @ActionHandler( action = "readPwNotifyStatus" )
-    public ProcessStatus restreadPwNotifyStatus( final PwmRequest pwmRequest ) throws IOException, DatabaseException, PwmUnrecoverableException
+    public ProcessStatus restreadPwNotifyStatus( final PwmRequest pwmRequest ) throws IOException, PwmUnrecoverableException
     {
         int key = 0;
         if ( !pwmRequest.getConfig().readSettingAsBoolean( PwmSetting.PW_EXPY_NOTIFY_ENABLE ) )
@@ -722,31 +722,6 @@ public class AdminServlet extends ControlledPwmServlet
             return ProcessStatus.Halt;
         }
 
-
-        {
-            ErrorInformation errorInformation = null;
-            try
-            {
-                if ( !pwmRequest.getPwmApplication().getDatabaseService().getAccessor().isConnected() )
-                {
-                    errorInformation = new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, "database is not connected" );
-                }
-            }
-            catch ( PwmUnrecoverableException e )
-            {
-                errorInformation = e.getErrorInformation();
-            }
-
-            if ( errorInformation != null )
-            {
-                final DisplayElement displayElement = new DisplayElement( String.valueOf( key++ ), DisplayElement.Type.string, "Status",
-                        "Database must be functioning to view Password Notify status.  Current database error: "
-                                + errorInformation.toDebugStr() );
-                pwmRequest.outputJsonResult( RestResultBean.withData( new PwNotifyStatusBean( Collections.singletonList( displayElement ), false ) ) );
-                return ProcessStatus.Halt;
-            }
-        }
-
         final List<DisplayElement> statusData = new ArrayList<>( );
         final Configuration config = pwmRequest.getConfig();
         final Locale locale = pwmRequest.getLocale();
@@ -780,8 +755,11 @@ public class AdminServlet extends ControlledPwmServlet
                         "Last Job Duration", TimeDuration.between( storedJobState.getLastStart(), storedJobState.getLastCompletion() ).asLongString( locale ) ) );
             }
 
-            statusData.add( new DisplayElement( String.valueOf( key++ ), DisplayElement.Type.string,
-                    "Last Job Server Instance",  storedJobState.getServerInstance() ) );
+            if ( !StringUtil.isEmpty( storedJobState.getServerInstance() ) )
+            {
+                statusData.add( new DisplayElement( String.valueOf( key++ ), DisplayElement.Type.string,
+                        "Last Job Server Instance", storedJobState.getServerInstance() ) );
+            }
 
             if ( storedJobState.getLastError() != null )
             {

server/src/main/java/password/pwm/http/servlet/admin/UserDebugDataReader.java

@@ -57,7 +57,9 @@ public class UserDebugDataReader
     )
             throws PwmUnrecoverableException
     {
-        final UserInfo userInfo = UserInfoFactory.newUserInfoUsingProxy( pwmApplication, sessionLabel, userIdentity, locale );
+
+
+        final UserInfo userInfo = UserInfoFactory.newUserInfoUsingProxyForOfflineUser( pwmApplication, sessionLabel, userIdentity );
 
         final Map<Permission, String> permissions = UserDebugDataReader.permissionMap( pwmApplication, sessionLabel, userIdentity );
 

server/src/main/java/password/pwm/ldap/LdapOperationsHelper.java

@@ -26,6 +26,7 @@ import com.novell.ldapchai.ChaiConstant;
 import com.novell.ldapchai.ChaiEntry;
 import com.novell.ldapchai.ChaiUser;
 import com.novell.ldapchai.cr.Answer;
+import com.novell.ldapchai.exception.ChaiException;
 import com.novell.ldapchai.exception.ChaiOperationException;
 import com.novell.ldapchai.exception.ChaiUnavailableException;
 import com.novell.ldapchai.provider.ChaiConfiguration;
@@ -36,12 +37,15 @@ import com.novell.ldapchai.provider.SearchScope;
 import com.novell.ldapchai.util.SearchHelper;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
+import password.pwm.PwmConstants;
 import password.pwm.bean.SessionLabel;
 import password.pwm.bean.UserIdentity;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
+import password.pwm.config.option.AutoSetLdapUserLanguage;
 import password.pwm.config.profile.LdapProfile;
 import password.pwm.config.value.data.FormConfiguration;
+import password.pwm.config.value.data.UserPermission;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmOperationalException;
@@ -53,6 +57,7 @@ import password.pwm.svc.cache.CacheKey;
 import password.pwm.svc.cache.CachePolicy;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsManager;
+import password.pwm.util.LocaleHelper;
 import password.pwm.util.PasswordData;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.StringUtil;
@@ -73,6 +78,7 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
@@ -86,7 +92,7 @@ public class LdapOperationsHelper
             final UserIdentity userIdentity,
             final PwmApplication pwmApplication
     )
-            throws ChaiUnavailableException, PwmUnrecoverableException
+            throws PwmUnrecoverableException
     {
         final LdapProfile ldapProfile = pwmApplication.getConfig().getLdapProfiles().get( userIdentity.getLdapProfileID() );
         final Set<String> newObjClasses = new HashSet<>( ldapProfile.readSettingAsStringArray( PwmSetting.AUTO_ADD_OBJECT_CLASSES ) );
@@ -94,9 +100,16 @@ public class LdapOperationsHelper
         {
             return;
         }
-        final ChaiProvider chaiProvider = pwmApplication.getProxyChaiProvider( userIdentity.getLdapProfileID() );
-        final ChaiUser theUser = chaiProvider.getEntryFactory().newChaiUser( userIdentity.getUserDN() );
-        addUserObjectClass( sessionLabel, theUser, newObjClasses );
+        try
+        {
+            final ChaiProvider chaiProvider = pwmApplication.getProxyChaiProvider( userIdentity.getLdapProfileID() );
+            final ChaiUser theUser = chaiProvider.getEntryFactory().newChaiUser( userIdentity.getUserDN() );
+            addUserObjectClass( sessionLabel, theUser, newObjClasses );
+        }
+        catch ( ChaiUnavailableException e )
+        {
+            throw PwmUnrecoverableException.fromChaiException( e );
+        }
     }
 
     private static void addUserObjectClass(
@@ -825,13 +838,57 @@ public class LdapOperationsHelper
         return Collections.emptyMap();
     }
 
+    public static Iterator<UserIdentity> readUsersFromLdapForPermissions(
+            final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
+            final List<UserPermission> permissionList,
+            final int maxResults
+    )
+            throws PwmUnrecoverableException, PwmOperationalException
+    {
+        final UserSearchEngine userSearchEngine = pwmApplication.getUserSearchEngine();
+        final Queue<UserIdentity> resultSet = new LinkedList<>();
+
+        for ( final UserPermission userPermission : permissionList )
+        {
+            if ( resultSet.size() < maxResults )
+            {
+                final SearchConfiguration searchConfiguration = SearchConfiguration.fromPermission( userPermission );
+                final Map<UserIdentity, Map<String, String>> searchResults = userSearchEngine.performMultiUserSearch(
+                        searchConfiguration,
+                        maxResults - resultSet.size(),
+                        Collections.emptyList(),
+                        sessionLabel
+
+                );
+                resultSet.addAll( searchResults.keySet() );
+            }
+        }
+
+        return new Iterator<UserIdentity>()
+        {
+            @Override
+            public boolean hasNext( )
+            {
+                return resultSet.peek() != null;
+            }
+
+            @Override
+            public UserIdentity next( )
+            {
+                return resultSet.poll();
+            }
+        };
+    }
+
+
     public static Iterator<UserIdentity> readAllUsersFromLdap(
             final PwmApplication pwmApplication,
             final SessionLabel sessionLabel,
             final String searchFilter,
             final int maxResults
     )
-            throws ChaiUnavailableException, ChaiOperationException, PwmUnrecoverableException, PwmOperationalException
+            throws PwmUnrecoverableException, PwmOperationalException
     {
         final UserSearchEngine userSearchEngine = pwmApplication.getUserSearchEngine();
 
@@ -982,4 +1039,71 @@ public class LdapOperationsHelper
         return new PhotoDataBean( mimeType, new ImmutableByteArray( photoData ) );
     }
 
+
+    public static Locale readLdapStoredLanguage(
+            final PwmApplication pwmApplication,
+            final UserIdentity userIdentity
+    )
+            throws PwmUnrecoverableException
+    {
+        final LdapProfile ldapProfile = userIdentity.getLdapProfile( pwmApplication.getConfig() );
+        final String languageAttr = ldapProfile.readSettingAsString( PwmSetting.LDAP_ATTRIBUTE_LANGUAGE );
+        if ( StringUtil.isEmpty( languageAttr ) )
+        {
+            return PwmConstants.DEFAULT_LOCALE;
+        }
+
+        try
+        {
+            final ChaiUser chaiUser = pwmApplication.getProxiedChaiUser( userIdentity );
+            final String storedValue = chaiUser.readStringAttribute( languageAttr );
+            if ( StringUtil.isEmpty( storedValue ) )
+            {
+                return PwmConstants.DEFAULT_LOCALE;
+            }
+
+            return LocaleHelper.parseLocaleString( storedValue );
+        }
+        catch ( ChaiException e )
+        {
+            throw PwmUnrecoverableException.fromChaiException( e );
+        }
+    }
+
+    public static void processAutoUpdateLanguageAttribute(
+            final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
+            final Locale sessionLocale,
+            final UserIdentity userIdentity
+    )
+            throws PwmUnrecoverableException
+    {
+        final LdapProfile ldapProfile = userIdentity.getLdapProfile( pwmApplication.getConfig() );
+        final String languageAttr = ldapProfile.readSettingAsString( PwmSetting.LDAP_ATTRIBUTE_LANGUAGE );
+        if ( StringUtil.isEmpty( languageAttr ) )
+        {
+            return;
+        }
+
+        final AutoSetLdapUserLanguage setting = ldapProfile.readSettingAsEnum( PwmSetting.LDAP_AUTO_SET_LANGUAGE_VALUE, AutoSetLdapUserLanguage.class );
+        if ( setting == null || setting == AutoSetLdapUserLanguage.disabled )
+        {
+            return;
+        }
+
+        if ( setting == AutoSetLdapUserLanguage.enabled )
+        {
+            final String languageCodeValue = LocaleHelper.getBrowserLocaleString( sessionLocale );
+            try
+            {
+                final ChaiUser user = pwmApplication.getProxiedChaiUser( userIdentity );
+                user.writeStringAttribute( languageAttr, languageCodeValue );
+                LOGGER.debug( sessionLabel, "wrote current browser session language value '" + languageCodeValue + "' to user attribute " + languageAttr );
+            }
+            catch ( ChaiException e )
+            {
+                LOGGER.error( sessionLabel, "error writing language value to language attribute '" + languageAttr + "', error: " + e.getMessage() );
+            }
+        }
+    }
 }

server/src/main/java/password/pwm/ldap/UserInfo.java

@@ -44,6 +44,8 @@ public interface UserInfo
 
     Instant getLastLdapLoginTime( ) throws PwmUnrecoverableException;
 
+    String getLanguage( ) throws PwmUnrecoverableException;
+
     ChallengeProfile getChallengeProfile( ) throws PwmUnrecoverableException;
 
     PwmPasswordPolicy getPasswordPolicy( ) throws PwmUnrecoverableException;

server/src/main/java/password/pwm/ldap/UserInfoBean.java

@@ -77,6 +77,8 @@ public class UserInfoBean implements UserInfo
     @Builder.Default
     private final PwmPasswordPolicy passwordPolicy = PwmPasswordPolicy.defaultPolicy();
 
+    private final String language;
+
     private final ChallengeProfile challengeProfile;
     private final ResponseInfoBean responseInfoBean;
     private final OTPUserRecord otpUserRecord;

server/src/main/java/password/pwm/ldap/UserInfoFactory.java

@@ -62,6 +62,18 @@ public class UserInfoFactory
         );
     }
 
+    public static UserInfo newUserInfoUsingProxyForOfflineUser(
+            final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
+            final UserIdentity userIdentity
+    )
+            throws PwmUnrecoverableException
+    {
+        final Locale ldapLocale = LdapOperationsHelper.readLdapStoredLanguage( pwmApplication, userIdentity );
+        final ChaiProvider provider = pwmApplication.getProxyChaiProvider( userIdentity.getLdapProfileID() );
+        return newUserInfo( pwmApplication, sessionLabel, ldapLocale, userIdentity, provider, null );
+    }
+
     public static UserInfo newUserInfoUsingProxy(
             final PwmApplication pwmApplication,
             final SessionLabel sessionLabel,

server/src/main/java/password/pwm/ldap/UserInfoReader.java

@@ -52,6 +52,7 @@ import password.pwm.error.PwmDataValidationException;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.svc.PwmService;
+import password.pwm.util.LocaleHelper;
 import password.pwm.util.PasswordData;
 import password.pwm.util.PwmPasswordRuleValidator;
 import password.pwm.util.form.FormUtility;
@@ -860,4 +861,10 @@ public class UserInfoReader implements UserInfo
                 this.getPasswordStatus()
         );
     }
+
+    @Override
+    public String getLanguage() throws PwmUnrecoverableException
+    {
+        return locale == null ? null : LocaleHelper.getBrowserLocaleString( locale );
+    }
 }

server/src/main/java/password/pwm/ldap/auth/SessionAuthenticator.java

@@ -352,7 +352,7 @@ public class SessionAuthenticator
             final UserIdentity userIdentity,
             final AuthenticationResult authenticationResult
     )
-            throws PwmUnrecoverableException, ChaiUnavailableException
+            throws PwmUnrecoverableException
     {
         final IntruderManager intruderManager = pwmApplication.getIntruderManager();
         final LocalSessionStateBean ssBean = pwmSession.getSessionStateBean();
@@ -429,5 +429,7 @@ public class SessionAuthenticator
         LOGGER.debug( pwmSession, "clearing permission cache" );
         pwmSession.getUserSessionDataCacheBean().clearPermissions();
 
+        // update the users ldap attribute.
+        LdapOperationsHelper.processAutoUpdateLanguageAttribute( pwmApplication, sessionLabel, ssBean.getLocale(), userIdentity );
     }
 }

server/src/main/java/password/pwm/ldap/search/SearchConfiguration.java

@@ -26,8 +26,11 @@ import com.novell.ldapchai.provider.ChaiProvider;
 import lombok.Builder;
 import lombok.Getter;
 import password.pwm.config.value.data.FormConfiguration;
+import password.pwm.config.value.data.UserPermission;
+import password.pwm.util.java.StringUtil;
 
 import java.io.Serializable;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -62,4 +65,24 @@ public class SearchConfiguration implements Serializable
         }
     }
 
+    public static SearchConfiguration fromPermission( final UserPermission permission )
+    {
+        final SearchConfiguration.SearchConfigurationBuilder builder = SearchConfiguration.builder();
+        if ( !StringUtil.isEmpty( permission.getLdapQuery() ) )
+        {
+            builder.filter( permission.getLdapQuery() );
+        }
+
+        if ( !StringUtil.isEmpty( permission.getLdapBase() ) )
+        {
+            builder.contexts( Collections.singletonList( permission.getLdapBase() ) );
+        }
+
+        if ( !StringUtil.isEmpty( permission.getLdapProfileID() ) )
+        {
+            builder.ldapProfile( permission.getLdapProfileID() );
+        }
+
+        return builder.build();
+    }
 }

server/src/main/java/password/pwm/svc/AbstractPwmService.java

@@ -0,0 +1,73 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc;
+
+import password.pwm.error.ErrorInformation;
+import password.pwm.health.HealthMessage;
+import password.pwm.health.HealthRecord;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public abstract class AbstractPwmService
+{
+    private PwmService.STATUS status = PwmService.STATUS.CLOSED;
+    private ErrorInformation startupError;
+
+    public final PwmService.STATUS status()
+    {
+        return status;
+    }
+
+    protected void setStatus( final PwmService.STATUS status )
+    {
+        this.status = status;
+    }
+
+    protected void setStartupError( final ErrorInformation startupError )
+    {
+        this.startupError = startupError;
+    }
+
+    protected ErrorInformation getStartupError()
+    {
+        return startupError;
+    }
+
+    public final List<HealthRecord> healthCheck( )
+    {
+        final List<HealthRecord> returnRecords = new ArrayList<>(  );
+
+        final ErrorInformation startupError = this.startupError;
+        if ( startupError != null )
+        {
+            returnRecords.add( HealthRecord.forMessage( HealthMessage.ServiceClosed, startupError.toDebugStr() ) );
+        }
+
+        returnRecords.addAll( serviceHealthCheck() );
+
+        return returnRecords;
+    }
+
+    protected abstract List<HealthRecord> serviceHealthCheck();
+}

server/src/main/java/password/pwm/svc/cache/CacheService.java

@@ -113,6 +113,8 @@ public class CacheService implements PwmService
     public ServiceInfoBean serviceInfo( )
     {
         final Map<String, String> debugInfo = new TreeMap<>( );
+        debugInfo.put( "itemCount", String.valueOf( memoryCacheStore.itemCount() ) );
+        debugInfo.put( "byteCount", String.valueOf( memoryCacheStore.byteCount() ) );
         debugInfo.putAll( JsonUtil.deserializeStringMap( JsonUtil.serialize( memoryCacheStore.getCacheStoreInfo() ) ) );
         debugInfo.putAll( JsonUtil.deserializeStringMap( JsonUtil.serializeMap( memoryCacheStore.storedClassHistogram( "histogram." ) ) ) );
         return new ServiceInfoBean( Collections.emptyList(), debugInfo );

server/src/main/java/password/pwm/svc/cache/CacheStore.java

@@ -42,4 +42,6 @@ public interface CacheStore
     int itemCount( );
 
     List<CacheDebugItem> getCacheDebugItems( );
+
+    long byteCount();
 }

server/src/main/java/password/pwm/svc/cache/MemoryCacheStore.java

@@ -25,6 +25,7 @@ package password.pwm.svc.cache;
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import lombok.Value;
+import password.pwm.bean.UserIdentity;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.logging.PwmLogger;
@@ -180,4 +181,21 @@ class MemoryCacheStore implements CacheStore
         }
         return output;
     }
+
+    @Override
+    public long byteCount()
+    {
+        long byteCount = 0;
+        for ( Map.Entry<CacheKey, CacheValueWrapper> entry : memoryStore.asMap().entrySet() )
+        {
+            final CacheKey cacheKey = entry.getKey();
+            final UserIdentity userIdentity = cacheKey.getUserIdentity();
+            byteCount += userIdentity == null ? 0 : cacheKey.getUserIdentity().toDelimitedKey().length();
+            final String valueID = cacheKey.getValueID();
+            byteCount += valueID == null ? 0 : cacheKey.getValueID().length();
+            final CacheValueWrapper cacheValueWrapper = entry.getValue();
+            byteCount += cacheValueWrapper.payload.length();
+        }
+        return byteCount;
+    }
 }

server/src/main/java/password/pwm/svc/cluster/ClusterService.java

@@ -66,12 +66,18 @@ public class ClusterService implements PwmService
         status = STATUS.OPENING;
         this.pwmApplication = pwmApplication;
 
+        final boolean serviceEnabled = pwmApplication.getConfig().readSettingAsBoolean( PwmSetting.CLUSTER_ENABLED );
+        if ( !serviceEnabled )
+        {
+            status = STATUS.CLOSED;
+            return;
+        }
 
         try
         {
             final ClusterSettings clusterSettings;
             final ClusterDataServiceProvider clusterDataServiceProvider;
-            dataStore = figureDataStorageMethod( pwmApplication );
+            dataStore = pwmApplication.getConfig().readSettingAsEnum( PwmSetting.CLUSTER_STORAGE_MODE, DataStorageMethod.class );
 
             if ( dataStore != null )
             {
@@ -177,11 +183,9 @@ public class ClusterService implements PwmService
         return Collections.emptyList();
     }
 
-    private DataStorageMethod figureDataStorageMethod( final PwmApplication pwmApplication )
+    private void figureDataStorageMethod( final PwmApplication pwmApplication )
             throws PwmUnrecoverableException
     {
-        final DataStorageMethod method = pwmApplication.getConfig().readSettingAsEnum( PwmSetting.CLUSTER_STORAGE_MODE, DataStorageMethod.class );
-        if ( method == DataStorageMethod.LDAP )
         {
             final UserIdentity userIdentity = pwmApplication.getConfig().getDefaultLdapProfile().getTestUser( pwmApplication );
             if ( userIdentity == null )
@@ -189,21 +193,16 @@ public class ClusterService implements PwmService
                 final String msg = "LDAP storage type selected, but LDAP test user not defined.";
                 LOGGER.debug( msg );
                 startupError = new ErrorInformation( PwmError.ERROR_CLUSTER_SERVICE_ERROR, msg );
-                return null;
             }
         }
 
-        if ( method == DataStorageMethod.DB )
         {
             if ( !pwmApplication.getConfig().hasDbConfigured() )
             {
                 final String msg = "DB storage type selected, but remote DB is not configured.";
                 LOGGER.debug( msg );
                 startupError = new ErrorInformation( PwmError.ERROR_CLUSTER_SERVICE_ERROR, msg );
-                return null;
             }
         }
-
-        return method;
     }
 }

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyDbStorageService.java

@@ -37,16 +37,24 @@ import password.pwm.util.java.StringUtil;
 
 class PwNotifyDbStorageService implements PwNotifyStorageService
 {
+    private static final String DB_STATE_STRING = "PwNotifyJobState";
+
     private static final DatabaseTable TABLE = DatabaseTable.PW_NOTIFY;
     private final PwmApplication pwmApplication;
 
-    PwNotifyDbStorageService( final PwmApplication pwmApplication )
+    PwNotifyDbStorageService( final PwmApplication pwmApplication ) throws PwmUnrecoverableException
     {
         this.pwmApplication = pwmApplication;
+
+        if ( !pwmApplication.getConfig().hasDbConfigured() )
+        {
+            final String msg = "DB storage type selected, but remote DB is not configured.";
+            throw PwmUnrecoverableException.newException( PwmError.ERROR_CLUSTER_SERVICE_ERROR, msg );
+        }
     }
 
     @Override
-    public StoredNotificationState readStoredState(
+    public StoredNotificationState readStoredUserState(
             final UserIdentity userIdentity,
             final SessionLabel sessionLabel
     )
@@ -79,7 +87,7 @@ class PwNotifyDbStorageService implements PwNotifyStorageService
         return JsonUtil.deserialize( rawDbValue, StoredNotificationState.class );
     }
 
-    public void writeStoredState(
+    public void writeStoredUserState(
             final UserIdentity userIdentity,
             final SessionLabel sessionLabel,
             final StoredNotificationState storedNotificationState
@@ -110,4 +118,39 @@ class PwNotifyDbStorageService implements PwNotifyStorageService
             throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, e.getMessage() ) );
         }
     }
+
+    @Override
+    public StoredJobState readStoredJobState()
+            throws PwmUnrecoverableException
+    {
+        try
+        {
+            final String strValue = pwmApplication.getDatabaseService().getAccessor().get( DatabaseTable.PW_NOTIFY, DB_STATE_STRING );
+            if ( StringUtil.isEmpty( strValue ) )
+            {
+                return new StoredJobState( null, null, null, null, false );
+            }
+            return JsonUtil.deserialize( strValue, StoredJobState.class );
+        }
+        catch ( DatabaseException e )
+        {
+            throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, e.getMessage() ) );
+        }
+    }
+
+    @Override
+    public void writeStoredJobState( final StoredJobState storedJobState )
+            throws PwmUnrecoverableException
+    {
+        try
+        {
+            final String strValue = JsonUtil.serialize( storedJobState );
+            pwmApplication.getDatabaseService().getAccessor().put( DatabaseTable.PW_NOTIFY, DB_STATE_STRING, strValue );
+        }
+        catch ( DatabaseException e )
+        {
+            throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, e.getMessage() ) );
+        }
+    }
+
 }

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyEngine.java

@@ -23,10 +23,7 @@
 package password.pwm.svc.pwnotify;
 
 import com.novell.ldapchai.ChaiUser;
-import com.novell.ldapchai.exception.ChaiOperationException;
-import com.novell.ldapchai.exception.ChaiUnavailableException;
 import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
 import password.pwm.bean.EmailItemBean;
 import password.pwm.bean.SessionLabel;
 import password.pwm.bean.UserIdentity;
@@ -36,12 +33,11 @@ import password.pwm.error.PwmError;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.ldap.LdapOperationsHelper;
-import password.pwm.ldap.LdapPermissionTester;
 import password.pwm.ldap.UserInfo;
 import password.pwm.ldap.UserInfoFactory;
-import password.pwm.svc.stats.EventRateMeter;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsManager;
+import password.pwm.util.LocaleHelper;
 import password.pwm.util.java.ConditionalTaskExecutor;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.TimeDuration;
@@ -50,15 +46,18 @@ import password.pwm.util.macro.MacroMachine;
 
 import java.io.IOException;
 import java.io.Writer;
-import java.math.BigDecimal;
 import java.time.Duration;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
 
 public class PwNotifyEngine
 {
@@ -66,31 +65,37 @@ public class PwNotifyEngine
 
     private static final SessionLabel SESSION_LABEL = SessionLabel.PW_EXP_NOTICE_LABEL;
 
+    private static final int MAX_LOG_SIZE = 1024 * 1024 * 1024;
+
     private final PwNotifySettings settings;
     private final PwmApplication pwmApplication;
     private final Writer debugWriter;
     private final StringBuffer internalLog = new StringBuffer(  );
     private final List<UserPermission> permissionList;
+    private final PwNotifyStorageService storageService;
+    private final Supplier<Boolean> cancelFlag;
 
     private final ConditionalTaskExecutor debugOutputTask = new ConditionalTaskExecutor(
             this::periodicDebugOutput,
             new ConditionalTaskExecutor.TimeDurationPredicate( 1, TimeDuration.Unit.MINUTES )
     );
 
-    private EventRateMeter eventRateMeter = new EventRateMeter( TimeDuration.of( 5, TimeDuration.Unit.MINUTES ) );
-
-    private int examinedCount = 0;
-    private int noticeCount = 0;
+    private final AtomicInteger examinedCount = new AtomicInteger( 0 );
+    private final AtomicInteger noticeCount = new AtomicInteger( 0 );
     private Instant startTime;
 
     private volatile boolean running;
 
     PwNotifyEngine(
             final PwmApplication pwmApplication,
+            final PwNotifyStorageService storageService,
+            final Supplier<Boolean> cancelFlag,
             final Writer debugWriter
     )
     {
         this.pwmApplication = pwmApplication;
+        this.cancelFlag = cancelFlag;
+        this.storageService = storageService;
         this.settings = PwNotifySettings.fromConfiguration( pwmApplication.getConfig() );
         this.debugWriter = debugWriter;
         this.permissionList = pwmApplication.getConfig().readSettingAsUserPermission( PwmSetting.PW_EXPY_NOTIFY_PERMISSION );
@@ -125,17 +130,17 @@ public class PwNotifyEngine
     }
 
     void executeJob( )
-            throws ChaiUnavailableException, ChaiOperationException, PwmOperationalException, PwmUnrecoverableException
+            throws PwmOperationalException, PwmUnrecoverableException
     {
         startTime = Instant.now();
-        examinedCount = 0;
-        noticeCount = 0;
+        examinedCount.set( 0 );
+        noticeCount.set( 0 );
         try
         {
             internalLog.delete( 0, internalLog.length() );
             running = true;
 
-            if ( !canRunOnThisServer() )
+            if ( !canRunOnThisServer() || cancelFlag.get() )
             {
                 return;
             }
@@ -150,46 +155,30 @@ public class PwNotifyEngine
             }
 
             log( "starting job, beginning ldap search" );
-            final Iterator<UserIdentity> workQueue = LdapOperationsHelper.readAllUsersFromLdap(
+            final Iterator<UserIdentity> workQueue = LdapOperationsHelper.readUsersFromLdapForPermissions(
                     pwmApplication,
-                    null,
-                    null,
+                    SESSION_LABEL,
+                    permissionList,
                     settings.getMaxLdapSearchSize()
             );
 
             log( "ldap search complete, examining users..." );
+
+            final ThreadPoolExecutor threadPoolExecutor = createExecutor( pwmApplication );
             while ( workQueue.hasNext() )
             {
-                if ( !checkIfRunningOnMaster() )
+                if ( !checkIfRunningOnMaster() || cancelFlag.get() )
                 {
                     final String msg = "job interrupted, server is no longer the cluster master.";
                     log( msg );
                     throw PwmUnrecoverableException.newException( PwmError.ERROR_SERVICE_NOT_AVAILABLE, msg );
                 }
 
-                checkIfRunningOnMaster(  );
-                examinedCount++;
-
-                final List<UserIdentity> batch = new ArrayList<>(  );
-                final int batchSize = settings.getBatchCount();
-
-                while ( batch.size() < batchSize && workQueue.hasNext() )
-                {
-                    batch.add( workQueue.next() );
-                }
+                threadPoolExecutor.submit( new ProcessJob( workQueue.next() ) );
+            }
 
-                final Instant startBatch = Instant.now();
-                examinedCount += batch.size();
-                noticeCount += processBatch( batch );
-                eventRateMeter.markEvents( batchSize );
-                final TimeDuration batchTime = TimeDuration.fromCurrent( startBatch );
-                final TimeDuration pauseTime = TimeDuration.of(
-                        settings.getBatchTimeMultiplier().multiply( new BigDecimal( batchTime.asMillis() ) ).longValue(),
-                        TimeDuration.Unit.MILLISECONDS );
-                pauseTime.pause();
+            JavaHelper.closeAndWaitExecutor( threadPoolExecutor, TimeDuration.DAY );
 
-                debugOutputTask.conditionallyExecuteTask();
-            }
             log( "job complete, " + examinedCount + " users evaluated in " + TimeDuration.fromCurrent( startTime ).asCompactString()
                     + ", sent " + noticeCount + " notices."
             );
@@ -202,63 +191,78 @@ public class PwNotifyEngine
 
     private void periodicDebugOutput()
     {
-        log( "job in progress, " + examinedCount + " users evaluated in " + TimeDuration.fromCurrent( startTime ).asCompactString()
-                + ", sent " + noticeCount + " notices."
-        );
+        final String msg = "job in progress, " + examinedCount + " users evaluated in "
+                + TimeDuration.fromCurrent( startTime ).asCompactString()
+                + ", sent " + noticeCount + " notices.";
+        log( msg );
     }
 
-    private int processBatch( final Collection<UserIdentity> batch )
-            throws PwmUnrecoverableException
+    private class ProcessJob implements Runnable
     {
-        int count = 0;
-        for ( final UserIdentity userIdentity : batch )
+        final UserIdentity userIdentity;
+
+        ProcessJob( final UserIdentity userIdentity )
+        {
+            this.userIdentity = userIdentity;
+        }
+
+        @Override
+        public void run()
         {
-            if ( processUserIdentity( userIdentity ) )
+            try
             {
-                count++;
+                processUserIdentity( userIdentity );
+                debugOutputTask.conditionallyExecuteTask();
+            }
+            catch ( Exception e )
+            {
+                LOGGER.trace( "unexpected error processing user '" + userIdentity.toDisplayString() + "', error: " + e.getMessage() );
             }
         }
-        return count;
     }
 
-    private boolean processUserIdentity(
+    private void processUserIdentity(
             final UserIdentity userIdentity
     )
             throws PwmUnrecoverableException
     {
-        if ( !LdapPermissionTester.testUserPermissions( pwmApplication, SessionLabel.SYSTEM_LABEL, userIdentity, permissionList ) )
+        if ( !canRunOnThisServer() || cancelFlag.get() )
         {
-            return false;
+            return;
         }
 
+        examinedCount.incrementAndGet();
         final ChaiUser theUser = pwmApplication.getProxiedChaiUser( userIdentity );
         final Instant passwordExpirationTime = LdapOperationsHelper.readPasswordExpirationTime( theUser );
 
-        if ( passwordExpirationTime == null || passwordExpirationTime.isBefore( Instant.now() ) )
+        if ( passwordExpirationTime == null )
         {
-            return false;
+            LOGGER.trace( SESSION_LABEL, "skipping user '" + userIdentity.toDisplayString() + "', has no password expiration" );
+            return;
+        }
+
+        if ( passwordExpirationTime.isBefore( Instant.now() ) )
+        {
+            LOGGER.trace( SESSION_LABEL, "skipping user '" + userIdentity.toDisplayString() + "', password expiration is in the past" );
+            return;
         }
 
         final int nextDayInterval = figureNextDayInterval( passwordExpirationTime );
         if ( nextDayInterval < 1 )
         {
-            return false;
+            LOGGER.trace( SESSION_LABEL, "skipping user '" + userIdentity.toDisplayString() + "', password expiration time is not within an interval" );
+            return;
         }
 
         if ( checkIfNoticeAlreadySent( userIdentity, passwordExpirationTime, nextDayInterval ) )
         {
             log( "notice for interval " + nextDayInterval + " already sent for " + userIdentity.toDisplayString() );
-            return false;
+            return;
         }
 
         log( "sending notice to " + userIdentity.toDisplayString() + " for interval " + nextDayInterval );
-        {
-            final PwNotifyDbStorageService dbStorage = new PwNotifyDbStorageService( pwmApplication );
-            dbStorage.writeStoredState( userIdentity, SESSION_LABEL, new StoredNotificationState( passwordExpirationTime, Instant.now(), nextDayInterval ) );
-        }
-
+        storageService.writeStoredUserState( userIdentity, SESSION_LABEL, new StoredNotificationState( passwordExpirationTime, Instant.now(), nextDayInterval ) );
         sendNoticeEmail( userIdentity );
-        return true;
     }
 
     private int figureNextDayInterval(
@@ -292,8 +296,7 @@ public class PwNotifyEngine
     )
             throws PwmUnrecoverableException
     {
-        final PwNotifyDbStorageService dbStorage = new PwNotifyDbStorageService( pwmApplication );
-        final StoredNotificationState storedState = dbStorage.readStoredState( userIdentity, SESSION_LABEL );
+        final StoredNotificationState storedState = storageService.readStoredUserState( userIdentity, SESSION_LABEL );
 
         if ( storedState == null )
         {
@@ -316,18 +319,19 @@ public class PwNotifyEngine
     private void sendNoticeEmail( final UserIdentity userIdentity )
             throws PwmUnrecoverableException
     {
-        final Locale userLocale = PwmConstants.DEFAULT_LOCALE;
-        final EmailItemBean emailItemBean = pwmApplication.getConfig().readSettingAsEmail(
-                PwmSetting.EMAIL_PW_EXPIRATION_NOTICE,
-                userLocale
-        );
-        final MacroMachine macroMachine = MacroMachine.forUser( pwmApplication, userLocale, SESSION_LABEL, userIdentity );
-        final UserInfo userInfoBean = UserInfoFactory.newUserInfoUsingProxy(
+        final UserInfo userInfoBean = UserInfoFactory.newUserInfoUsingProxyForOfflineUser(
                 pwmApplication,
                 SESSION_LABEL,
-                userIdentity, userLocale
+                userIdentity
+        );
+        final Locale ldapLocale = LocaleHelper.parseLocaleString( userInfoBean.getLanguage() );
+        final MacroMachine macroMachine = MacroMachine.forUser( pwmApplication, ldapLocale, SESSION_LABEL, userIdentity );
+        final EmailItemBean emailItemBean = pwmApplication.getConfig().readSettingAsEmail(
+                PwmSetting.EMAIL_PW_EXPIRATION_NOTICE,
+                ldapLocale
         );
 
+        noticeCount.incrementAndGet();
         StatisticsManager.incrementStat( pwmApplication, Statistic.PWNOTIFY_EMAILS_SENT );
         pwmApplication.getEmailQueue().submitEmail( emailItemBean, userInfoBean, macroMachine );
     }
@@ -353,7 +357,7 @@ public class PwNotifyEngine
         }
 
         internalLog.append( msg );
-        while ( internalLog.length() > 1024 * 1024 * 1024 )
+        while ( internalLog.length() > MAX_LOG_SIZE )
         {
             final int nextLf = internalLog.indexOf( "\n" );
             if ( nextLf > 0 )
@@ -368,4 +372,19 @@ public class PwNotifyEngine
 
         LOGGER.trace( SessionLabel.PWNOTIFY_SESSION_LABEL, output );
     }
+
+    private ThreadPoolExecutor createExecutor( final PwmApplication pwmApplication )
+    {
+        final ThreadFactory threadFactory = JavaHelper.makePwmThreadFactory( JavaHelper.makeThreadName( pwmApplication, this.getClass() ), true );
+        final ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(
+                1,
+                10,
+                1,
+                TimeUnit.MINUTES,
+                new LinkedBlockingDeque<>(),
+                threadFactory
+        );
+        threadPoolExecutor.allowCoreThreadTimeOut( true );
+        return threadPoolExecutor;
+    }
 }

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyLdapStorageService.java

@@ -0,0 +1,184 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.pwnotify;
+
+import com.novell.ldapchai.ChaiUser;
+import com.novell.ldapchai.exception.ChaiOperationException;
+import com.novell.ldapchai.exception.ChaiUnavailableException;
+import com.novell.ldapchai.util.ConfigObjectRecord;
+import password.pwm.PwmApplication;
+import password.pwm.PwmConstants;
+import password.pwm.bean.SessionLabel;
+import password.pwm.bean.UserIdentity;
+import password.pwm.config.PwmSetting;
+import password.pwm.config.profile.LdapProfile;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.StringUtil;
+
+class PwNotifyLdapStorageService implements PwNotifyStorageService
+{
+    private final PwmApplication pwmApplication;
+    private final PwNotifySettings settings;
+
+    private enum CoreType
+    {
+        User( "0007" ),
+        ProxyUser( "0005" ),;
+
+        private final String recordID;
+
+        CoreType( final String recordID )
+        {
+            this.recordID = recordID;
+        }
+
+        public String getRecordID()
+        {
+            return recordID;
+        }
+    }
+
+    PwNotifyLdapStorageService( final PwmApplication pwmApplication, final PwNotifySettings settings )
+            throws PwmUnrecoverableException
+    {
+        this.pwmApplication = pwmApplication;
+        this.settings = settings;
+
+        final UserIdentity userIdentity = pwmApplication.getConfig().getDefaultLdapProfile().getTestUser( pwmApplication );
+        if ( userIdentity == null )
+        {
+            final String msg = "LDAP storage type selected, but LDAP test user ("
+                    + PwmSetting.LDAP_TEST_USER_DN.toMenuLocationDebug( pwmApplication.getConfig().getDefaultLdapProfile().getIdentifier(), PwmConstants.DEFAULT_LOCALE )
+                    + ") not defined.";
+            throw new PwmUnrecoverableException( PwmError.ERROR_PWNOTIFY_SERVICE_ERROR, msg );
+        }
+
+        for ( final LdapProfile ldapProfile : pwmApplication.getConfig().getLdapProfiles().values() )
+        {
+            if ( StringUtil.isEmpty( ldapProfile.readSettingAsString( PwmSetting.LDAP_ATTRIBUTE_PWNOTIFY ) ) )
+            {
+                final String msg = "LDAP storage type selected, but setting '"
+                        + PwmSetting.LDAP_ATTRIBUTE_PWNOTIFY.toMenuLocationDebug( ldapProfile.getIdentifier(), PwmConstants.DEFAULT_LOCALE )
+                        + " is not configured ";
+                throw new PwmUnrecoverableException( PwmError.ERROR_PWNOTIFY_SERVICE_ERROR, msg );
+            }
+        }
+
+    }
+
+    @Override
+    public StoredNotificationState readStoredUserState(
+            final UserIdentity userIdentity,
+            final SessionLabel sessionLabel
+    )
+            throws PwmUnrecoverableException
+    {
+        final ConfigObjectRecord configObjectRecord = getUserCOR( userIdentity, CoreType.User );
+        final String payload = configObjectRecord.getPayload();
+        if ( StringUtil.isEmpty( payload ) )
+        {
+            return JsonUtil.deserialize( payload, StoredNotificationState.class );
+        }
+        return null;
+    }
+
+    public void writeStoredUserState(
+            final UserIdentity userIdentity,
+            final SessionLabel sessionLabel,
+            final StoredNotificationState storedNotificationState
+    )
+            throws PwmUnrecoverableException
+    {
+        final ConfigObjectRecord configObjectRecord = getUserCOR( userIdentity, CoreType.User );
+        final String payload = JsonUtil.serialize( storedNotificationState );
+        try
+        {
+            configObjectRecord.updatePayload( payload );
+        }
+        catch ( ChaiOperationException e )
+        {
+            final String msg = "error writing user pwNotifyStatus attribute '" + getLdapUserAttribute( userIdentity ) + ", error: " + e.getMessage();
+            final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_LDAP_DATA_ERROR, msg );
+            throw new PwmUnrecoverableException( errorInformation );
+        }
+        catch ( ChaiUnavailableException e )
+        {
+            throw PwmUnrecoverableException.fromChaiException( e );
+        }
+    }
+
+    @Override
+    public StoredJobState readStoredJobState()
+            throws PwmUnrecoverableException
+    {
+        final UserIdentity proxyUser = pwmApplication.getConfig().getDefaultLdapProfile().getTestUser( pwmApplication );
+        final ConfigObjectRecord configObjectRecord = getUserCOR( proxyUser, CoreType.ProxyUser );
+        final String payload = configObjectRecord.getPayload();
+
+        if ( StringUtil.isEmpty( payload ) )
+        {
+            return new StoredJobState( null, null, null, null, false );
+        }
+        return JsonUtil.deserialize( payload, StoredJobState.class );
+    }
+
+    @Override
+    public void writeStoredJobState( final StoredJobState storedJobState )
+            throws PwmUnrecoverableException
+    {
+        final UserIdentity proxyUser = pwmApplication.getConfig().getDefaultLdapProfile().getTestUser( pwmApplication );
+        final ConfigObjectRecord configObjectRecord = getUserCOR( proxyUser, CoreType.ProxyUser );
+        final String payload = JsonUtil.serialize( storedJobState );
+
+        try
+        {
+            configObjectRecord.updatePayload( payload );
+        }
+        catch ( ChaiOperationException e )
+        {
+            final String msg = "error writing user pwNotifyStatus attribute on proxy user '" + getLdapUserAttribute( proxyUser ) + ", error: " + e.getMessage();
+            final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_LDAP_DATA_ERROR, msg );
+            throw new PwmUnrecoverableException( errorInformation );
+        }
+        catch ( ChaiUnavailableException e )
+        {
+            throw PwmUnrecoverableException.fromChaiException( e );
+        }
+    }
+
+    private ConfigObjectRecord getUserCOR( final UserIdentity userIdentity, final CoreType coreType )
+            throws PwmUnrecoverableException
+    {
+        final String userAttr = getLdapUserAttribute( userIdentity );
+        final ChaiUser chaiUser = pwmApplication.getProxiedChaiUser( userIdentity );
+        return ConfigObjectRecord.createNew( chaiUser, userAttr, coreType.getRecordID(), null, null );
+    }
+
+    private String getLdapUserAttribute( final UserIdentity userIdentity )
+    {
+        return  userIdentity.getLdapProfile( pwmApplication.getConfig() ).readSettingAsString( PwmSetting.LDAP_ATTRIBUTE_PWNOTIFY );
+    }
+}

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyService.java

@@ -25,19 +25,18 @@ package password.pwm.svc.pwnotify;
 import password.pwm.PwmApplication;
 import password.pwm.bean.SessionLabel;
 import password.pwm.config.PwmSetting;
+import password.pwm.config.option.DataStorageMethod;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
+import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsManager;
-import password.pwm.util.db.DatabaseException;
-import password.pwm.util.db.DatabaseTable;
 import password.pwm.util.java.JavaHelper;
-import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
@@ -45,45 +44,37 @@ import password.pwm.util.logging.PwmLogger;
 import java.time.Duration;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutorService;
 
-public class PwNotifyService implements PwmService
+public class PwNotifyService extends AbstractPwmService implements PwmService
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwNotifyService.class );
 
-    private ScheduledExecutorService executorService;
+    private ExecutorService executorService;
     private PwmApplication pwmApplication;
-    private STATUS status = STATUS.NEW;
     private PwNotifyEngine engine;
     private PwNotifySettings settings;
     private Instant nextExecutionTime;
+    private PwNotifyStorageService storageService;
 
-    @Override
-    public STATUS status( )
-    {
-        return status;
-    }
+    private DataStorageMethod storageMethod;
 
-    private static final String DB_STATE_STRING = "PwNotifyJobState";
-
-    private StoredJobState readStoredJobState()
-            throws PwmUnrecoverableException, DatabaseException
+    public StoredJobState getJobState() throws PwmUnrecoverableException
     {
-        final String strValue = pwmApplication.getDatabaseService().getAccessor().get( DatabaseTable.PW_NOTIFY, DB_STATE_STRING );
-        if ( StringUtil.isEmpty( strValue ) )
+        if ( status() != STATUS.OPEN )
         {
-            return new StoredJobState( null, null, null, null, false );
+            if ( getStartupError() != null )
+            {
+                return StoredJobState.builder().lastError( getStartupError() ).build();
+            }
+
+            return StoredJobState.builder().build();
         }
-        return JsonUtil.deserialize( strValue, StoredJobState.class );
-    }
 
-    public StoredJobState getJobState() throws DatabaseException, PwmUnrecoverableException
-    {
-        return readStoredJobState();
+        return storageService.readStoredJobState();
     }
 
     public boolean isRunning()
@@ -93,18 +84,17 @@ public class PwNotifyService implements PwmService
 
     public String debugLog()
     {
-        if ( engine != null )
+        if ( engine != null && !StringUtil.isEmpty( engine.getDebugLog() ) )
         {
             return engine.getDebugLog();
         }
-        return "";
-    }
 
-    private void writeStoredJobState( final StoredJobState storedJobState )
-            throws PwmUnrecoverableException, DatabaseException
-    {
-        final String strValue = JsonUtil.serialize( storedJobState );
-        pwmApplication.getDatabaseService().getAccessor().put( DatabaseTable.PW_NOTIFY, DB_STATE_STRING, strValue );
+        if ( getStartupError(  ) != null )
+        {
+            return getStartupError().toDebugStr();
+        }
+
+        return "";
     }
 
     @Override
@@ -114,23 +104,53 @@ public class PwNotifyService implements PwmService
 
         if ( !pwmApplication.getConfig().readSettingAsBoolean( PwmSetting.PW_EXPY_NOTIFY_ENABLE ) )
         {
-            status = STATUS.CLOSED;
             LOGGER.trace( SessionLabel.PWNOTIFY_SESSION_LABEL, "will remain closed, pw notify feature is not enabled" );
+            setStatus( STATUS.CLOSED );
             return;
         }
 
-        settings = PwNotifySettings.fromConfiguration( pwmApplication.getConfig() );
-        engine = new PwNotifyEngine( pwmApplication, null );
+        try
+        {
+            if ( pwmApplication.getClusterService() == null || pwmApplication.getClusterService().status() != STATUS.OPEN )
+            {
+                throw PwmUnrecoverableException.newException( PwmError.ERROR_PWNOTIFY_SERVICE_ERROR, "will remain closed, cluster service is not running" );
+            }
 
-        executorService = Executors.newSingleThreadScheduledExecutor(
-                JavaHelper.makePwmThreadFactory(
-                        JavaHelper.makeThreadName( pwmApplication, this.getClass() ) + "-",
-                        true
-                ) );
+            settings = PwNotifySettings.fromConfiguration( pwmApplication.getConfig() );
+            storageMethod = pwmApplication.getConfig().readSettingAsEnum( PwmSetting.PW_EXPY_NOTIFY_STORAGE_MODE, DataStorageMethod.class );
 
-        executorService.scheduleWithFixedDelay( new PwNotifyJob(), 1, 1, TimeUnit.MINUTES );
+            switch ( storageMethod )
+            {
+                case LDAP:
+                {
+                    storageService = new PwNotifyLdapStorageService( pwmApplication, settings );
+                }
+                break;
+
+                case DB:
+                {
+                    storageService = new PwNotifyDbStorageService( pwmApplication );
+                }
+                break;
+
+                default:
+                    JavaHelper.unhandledSwitchStatement( storageMethod );
+            }
+
+            executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
+
+            engine = new PwNotifyEngine( pwmApplication, storageService, () -> status() == STATUS.CLOSED, null );
 
-        status = STATUS.OPEN;
+            pwmApplication.scheduleFixedRateJob( new PwNotifyJob(), executorService, TimeDuration.MINUTE, TimeDuration.MINUTE );
+
+            setStatus( STATUS.OPEN );
+        }
+        catch ( PwmUnrecoverableException e )
+        {
+            setStatus( STATUS.CLOSED );
+            LOGGER.trace( SessionLabel.PWNOTIFY_SESSION_LABEL, "will remain closed, pw notify feature is not enabled due to error: " + e.getMessage() );
+            setStartupError( e.getErrorInformation() );
+        }
     }
 
     public Instant getNextExecutionTime( )
@@ -151,9 +171,10 @@ public class PwNotifyService implements PwmService
         }
     }
 
-    private Instant figureNextJobExecutionTime() throws DatabaseException, PwmUnrecoverableException
+    private Instant figureNextJobExecutionTime()
+            throws PwmUnrecoverableException
     {
-        final StoredJobState storedJobState = readStoredJobState();
+        final StoredJobState storedJobState = storageService.readStoredJobState();
         if ( storedJobState != null )
         {
             // never run, or last job not successful.
@@ -183,46 +204,49 @@ public class PwNotifyService implements PwmService
     @Override
     public void close( )
     {
-        status = STATUS.CLOSED;
+        setStatus( STATUS.CLOSED );
         JavaHelper.closeAndWaitExecutor( executorService, TimeDuration.of( 5, TimeDuration.Unit.SECONDS ) );
     }
 
     @Override
-    public List<HealthRecord> healthCheck( )
+    protected List<HealthRecord> serviceHealthCheck( )
     {
-        if ( status != STATUS.OPEN )
+        if ( status() != STATUS.OPEN )
         {
             return Collections.emptyList();
         }
 
+        final List<HealthRecord> returnRecords = new ArrayList<>( );
+
         try
         {
-            final StoredJobState storedJobState = readStoredJobState();
+            final StoredJobState storedJobState = storageService.readStoredJobState();
             if ( storedJobState != null )
             {
                 final ErrorInformation errorInformation = storedJobState.getLastError();
                 if ( errorInformation != null )
                 {
-                    return Collections.singletonList( HealthRecord.forMessage( HealthMessage.PwNotify_Failure, errorInformation.toDebugStr() ) );
+                    returnRecords.add( HealthRecord.forMessage( HealthMessage.PwNotify_Failure, errorInformation.toDebugStr() ) );
                 }
             }
         }
-        catch ( DatabaseException | PwmUnrecoverableException e  )
+        catch ( PwmUnrecoverableException e  )
         {
             LOGGER.error( SessionLabel.PWNOTIFY_SESSION_LABEL, "error while generating health information: " + e.getMessage() );
         }
-        return null;
+
+        return returnRecords;
     }
 
     @Override
     public ServiceInfoBean serviceInfo( )
     {
-        return null;
+        return new ServiceInfoBean( Collections.singleton( storageMethod ), Collections.emptyMap() );
     }
 
     public void executeJob( )
     {
-        if ( status != STATUS.OPEN )
+        if ( status() != STATUS.OPEN )
         {
             LOGGER.trace( SessionLabel.PWNOTIFY_SESSION_LABEL, "ignoring job request start, service is not open" );
             return;
@@ -231,13 +255,18 @@ public class PwNotifyService implements PwmService
         if ( !isRunning() )
         {
             nextExecutionTime = Instant.now();
-            executorService.schedule( new PwNotifyJob(), 1, TimeUnit.SECONDS );
+            pwmApplication.scheduleFutureJob( new PwNotifyJob(), executorService, TimeDuration.ZERO );
         }
     }
 
     public boolean canRunOnThisServer()
     {
-        return engine.canRunOnThisServer();
+        if ( status() == STATUS.OPEN )
+        {
+            return engine.canRunOnThisServer();
+        }
+
+        return false;
     }
 
     class PwNotifyJob implements Runnable
@@ -272,16 +301,17 @@ public class PwNotifyService implements PwmService
 
         private void doJob( )
         {
+            setStartupError( null );
             final Instant start = Instant.now();
             try
             {
-                writeStoredJobState( new StoredJobState( Instant.now(), null, pwmApplication.getInstanceID(), null, false ) );
+                storageService.writeStoredJobState( new StoredJobState( Instant.now(), null, pwmApplication.getInstanceID(), null, false ) );
                 StatisticsManager.incrementStat( pwmApplication, Statistic.PWNOTIFY_JOBS );
                 engine.executeJob();
 
                 final Instant finish = Instant.now();
                 final StoredJobState storedJobState = new StoredJobState( start, finish, pwmApplication.getInstanceID(), null, true );
-                writeStoredJobState( storedJobState );
+                storageService.writeStoredJobState( storedJobState );
             }
             catch ( Exception e )
             {
@@ -301,14 +331,15 @@ public class PwNotifyService implements PwmService
 
                 try
                 {
-                    writeStoredJobState( storedJobState );
+                    storageService.writeStoredJobState( storedJobState );
                 }
                 catch ( Exception e2 )
                 {
                     //no hope
                 }
                 StatisticsManager.incrementStat( pwmApplication, Statistic.PWNOTIFY_JOB_ERRORS );
-                LOGGER.debug( SessionLabel.PWNOTIFY_SESSION_LABEL, "error executing scheduled job: " + e.getMessage() );
+                LOGGER.debug( SessionLabel.PWNOTIFY_SESSION_LABEL, errorInformation );
+                setStartupError( errorInformation );
             }
         }
     }

server/src/main/java/password/pwm/svc/pwnotify/PwNotifySettings.java

@@ -37,7 +37,7 @@ import java.util.List;
 
 @Value
 @Builder
-public class PwNotifySettings implements Serializable
+class PwNotifySettings implements Serializable
 {
     private final List<Integer> notificationIntervals;
     private final TimeDuration maximumSkipWindow;
@@ -66,6 +66,7 @@ public class PwNotifySettings implements Serializable
         builder.batchTimeMultiplier( new BigDecimal( configuration.readAppProperty( AppProperty.PWNOTIFY_BATCH_DELAY_TIME_MULTIPLIER ) ) );
         builder.maximumSkipWindow( TimeDuration.of(
                 Long.parseLong( configuration.readAppProperty( AppProperty.PWNOTIFY_MAX_SKIP_RERUN_WINDOW_SECONDS ) ), TimeDuration.Unit.SECONDS ) );
+
         return builder.build();
     }
 }

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyStorageService.java

@@ -29,11 +29,17 @@ import password.pwm.error.PwmUnrecoverableException;
 interface PwNotifyStorageService
 {
 
-    StoredNotificationState readStoredState(
+    StoredNotificationState readStoredUserState(
             UserIdentity userIdentity,
             SessionLabel sessionLabel
     )
             throws PwmUnrecoverableException;
 
-    void writeStoredState( UserIdentity userIdentity, SessionLabel sessionLabel, StoredNotificationState storedNotificationState ) throws PwmUnrecoverableException;
+    void writeStoredUserState( UserIdentity userIdentity, SessionLabel sessionLabel, StoredNotificationState storedNotificationState ) throws PwmUnrecoverableException;
+
+    StoredJobState readStoredJobState()
+            throws PwmUnrecoverableException;
+
+    void writeStoredJobState( StoredJobState storedJobState )
+                    throws PwmUnrecoverableException;
 }

server/src/main/java/password/pwm/svc/pwnotify/StoredJobState.java

@@ -22,22 +22,20 @@
 
 package password.pwm.svc.pwnotify;
 
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
+import lombok.Builder;
+import lombok.Value;
 import password.pwm.error.ErrorInformation;
 
 import java.io.Serializable;
 import java.time.Instant;
 
-@Getter
-@NoArgsConstructor
-@AllArgsConstructor
+@Value
+@Builder
 public class StoredJobState implements Serializable
 {
-    private Instant lastStart = null;
-    private Instant lastCompletion = null;
-    private String serverInstance = null;
-    private ErrorInformation lastError = null;
+    private Instant lastStart;
+    private Instant lastCompletion;
+    private String serverInstance;
+    private ErrorInformation lastError;
     private boolean jobSuccess;
 }

server/src/main/java/password/pwm/svc/report/ReportService.java

@@ -24,10 +24,8 @@ package password.pwm.svc.report;
 
 import com.novell.ldapchai.exception.ChaiOperationException;
 import com.novell.ldapchai.exception.ChaiUnavailableException;
-import com.novell.ldapchai.provider.ChaiProvider;
 import password.pwm.PwmApplication;
 import password.pwm.PwmApplicationMode;
-import password.pwm.PwmConstants;
 import password.pwm.bean.SessionLabel;
 import password.pwm.bean.UserIdentity;
 import password.pwm.config.PwmSetting;
@@ -627,13 +625,10 @@ public class ReportService implements PwmService
             {
                 summaryData.remove( userCacheRecord );
             }
-            final ChaiProvider chaiProvider = pwmApplication.getProxyChaiProvider( userIdentity.getLdapProfileID() );
-            final UserInfo userInfo = UserInfoFactory.newUserInfo(
+            final UserInfo userInfo = UserInfoFactory.newUserInfoUsingProxyForOfflineUser(
                     pwmApplication,
                     SessionLabel.REPORTING_SESSION_LABEL,
-                    PwmConstants.DEFAULT_LOCALE,
-                    userIdentity,
-                    chaiProvider
+                    userIdentity
             );
             final UserCacheRecord newUserCacheRecord = userCacheService.updateUserCache( userInfo );
 

server/src/main/java/password/pwm/util/LocaleHelper.java

@@ -687,4 +687,9 @@ public class LocaleHelper
             return name1.compareToIgnoreCase( name2 );
         };
     }
+
+    public static String toLongDebug( final Locale locale, final Locale perspectiveLocale )
+    {
+        return locale.getDisplayName( perspectiveLocale ) + " (" + LocaleHelper.getBrowserLocaleString( locale ) + ")";
+    }
 }

server/src/main/java/password/pwm/util/java/ConditionalTaskExecutor.java

@@ -24,7 +24,10 @@ package password.pwm.util.java;
 
 import password.pwm.util.logging.PwmLogger;
 
-import java.util.function.Predicate;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Supplier;
 
 /**
  * <p>Executes a predefined task if a conditional has occurred.  Both the task and the conditional must be supplied by the caller.
@@ -39,43 +42,53 @@ public class ConditionalTaskExecutor
     private static final PwmLogger LOGGER = PwmLogger.forClass( ConditionalTaskExecutor.class );
 
     private Runnable task;
-    private Predicate predicate;
+    private Supplier<Boolean> predicate;
+    private final ReentrantLock lock = new ReentrantLock();
 
     /**
      * Execute the task if the conditional has been met.  Exceptions when running the task will be logged but not returned.
      */
     public void conditionallyExecuteTask( )
     {
-        if ( predicate.test( null ) )
+        lock.lock();
+        try
         {
-            try
+            if ( predicate.get() )
             {
-                task.run();
-            }
-            catch ( Throwable t )
-            {
-                LOGGER.warn( "unexpected error executing conditional task: " + t.getMessage(), t );
-            }
+                try
+                {
+                    task.run();
+                }
+                catch ( Throwable t )
+                {
+                    LOGGER.warn( "unexpected error executing conditional task: " + t.getMessage(), t );
+                }
 
+            }
+        }
+        finally
+        {
+            lock.unlock();
         }
     }
 
-    public ConditionalTaskExecutor( final Runnable task, final Predicate predicate )
+    public ConditionalTaskExecutor( final Runnable task, final Supplier<Boolean> predicate )
     {
         this.task = task;
         this.predicate = predicate;
     }
 
 
-    public static class TimeDurationPredicate implements Predicate
+    public static class TimeDurationPredicate implements Supplier<Boolean>
     {
         private final TimeDuration timeDuration;
-        private long nextExecuteTimestamp;
+        private volatile Instant nextExecuteTimestamp;
 
         public TimeDurationPredicate( final TimeDuration timeDuration )
         {
             this.timeDuration = timeDuration;
-            nextExecuteTimestamp = System.currentTimeMillis() + timeDuration.asMillis();
+            setNextTimeFromNow( timeDuration );
+
         }
 
         public TimeDurationPredicate( final long value, final TimeDuration.Unit unit )
@@ -83,18 +96,18 @@ public class ConditionalTaskExecutor
             this( TimeDuration.of( value, unit ) );
         }
 
-        public TimeDurationPredicate setNextTimeFromNow( final long value, final TimeDuration.Unit unit )
+        public TimeDurationPredicate setNextTimeFromNow( final TimeDuration duration )
         {
-            nextExecuteTimestamp = System.currentTimeMillis() + TimeDuration.of( value, unit ).asMillis();
+            nextExecuteTimestamp = Instant.now().plus( duration.asMillis(), ChronoUnit.MILLIS );
             return this;
         }
 
         @Override
-        public boolean test( final Object o )
+        public Boolean get()
         {
-            if ( nextExecuteTimestamp <= System.currentTimeMillis() )
+            if ( Instant.now().isAfter( nextExecuteTimestamp ) )
             {
-                nextExecuteTimestamp = System.currentTimeMillis() + timeDuration.asMillis();
+                setNextTimeFromNow( timeDuration );
                 return true;
             }
             return false;

server/src/main/java/password/pwm/util/localdb/XodusLocalDB.java

@@ -100,7 +100,7 @@ public class XodusLocalDB implements LocalDBProvider
     private final Map<LocalDB.DB, Store> cachedStoreObjects = new HashMap<>();
 
     private final ConditionalTaskExecutor outputLogExecutor = new ConditionalTaskExecutor(
-            ( ) -> outputStats(), new ConditionalTaskExecutor.TimeDurationPredicate( STATS_OUTPUT_INTERVAL ).setNextTimeFromNow( 1, TimeDuration.Unit.MINUTES )
+            ( ) -> outputStats(), new ConditionalTaskExecutor.TimeDurationPredicate( STATS_OUTPUT_INTERVAL ).setNextTimeFromNow( TimeDuration.MINUTE )
     );
 
     private BindMachine bindMachine = new BindMachine( BindMachine.DEFAULT_ENABLE_COMPRESSION, BindMachine.DEFAULT_MIN_COMPRESSION_LENGTH );
@@ -621,7 +621,7 @@ public class XodusLocalDB implements LocalDBProvider
             final ResourceBundle resourceBundle = ResourceBundle.getBundle( XodusLocalDB.class.getName() );
             final String contents = resourceBundle.getString( "ReadmeContents" );
             final byte[] byteContents = contents.getBytes( PwmConstants.DEFAULT_CHARSET );
-            Files.write( xodusPath.toPath(), byteContents, StandardOpenOption.TRUNCATE_EXISTING );
+            Files.write( xodusPath.toPath(), byteContents, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING );
         }
         catch ( IOException e )
         {

server/src/main/resources/password/pwm/AppProperty.properties

@@ -42,7 +42,7 @@ backup.path=backup
 backup.config.count=20
 backup.localdb.count=10
 cache.enable=true
-cache.memory.maxItems=1000
+cache.memory.maxItems=10000
 cache.pwRuleCheckLifetimeMS=30000
 cache.uniqueFormValueLifetimeMS=30000
 client.ajax.activityMaxEpsRate=100
@@ -259,7 +259,7 @@ peoplesearch.values.verifyUserDN=true
 peoplesearch.values.maxCount=100
 peoplesearch.view.detail.links=
 pwNotify.batch.count=100
-pwNotify.batch.delayTimeMultiplier=1.0
+pwNotify.batch.delayTimeMultiplier=0.1
 pwNotify.maxLdapSearchSize=1000000
 pwNotify.maxSkipRerunWindowSeconds=86400
 queue.email.retryTimeoutMs=10000

server/src/main/resources/password/pwm/config/PwmSetting.xml

@@ -1574,6 +1574,11 @@
             <value>28800</value>
         </default>
     </setting>
+    <setting hidden="false" key="cluster.enable" level="2">
+        <default>
+            <value>true</value>
+        </default>
+    </setting>
     <setting hidden="false" key="cluster.storageMode" level="2">
         <default>
             <value>LDAP</value>
@@ -3321,6 +3326,28 @@
             <value>workforceID</value>
         </default>
     </setting>
+    <setting hidden="false" key="ldap.user.language.attribute" level="1">
+        <ldapPermission actor="proxy" access="write"/>
+        <default>
+            <value/>
+        </default>
+    </setting>
+    <setting hidden="false" key="ldap.user.pwNotify.attribute" level="1">
+        <ldapPermission actor="proxy" access="write"/>
+        <default>
+            <value/>
+        </default>
+    </setting>
+    <setting hidden="false" key="ldap.user.language.autoSet" level="1">
+        <ldapPermission actor="proxy" access="write"/>
+        <options>
+            <option value="disabled">Disabled</option>
+            <option value="enabled">Enabled - Write to LDAP attribute during authentication.</option>
+        </options>
+        <default>
+            <value>disabled</value>
+        </default>
+    </setting>
     <setting hidden="false" key="ldap.edirectory.storeNmasResponses" level="1" required="true">
         <default>
             <value>false</value>
@@ -3790,6 +3817,18 @@
             <value/>
         </default>
     </setting>
+    <setting hidden="false" key="pwNotify.storageMode" level="2">
+        <default>
+            <value>LDAP</value>
+        </default>
+        <default template="DB">
+            <value>DB</value>
+        </default>
+        <options>
+            <option value="LDAP">LDAP Directory</option>
+            <option value="DB">Remote Database</option>
+        </options>
+    </setting>
     <setting hidden="false" key="pwNotify.queryString" level="1">
         <default>
             <value/>

server/src/main/resources/password/pwm/i18n/Error.properties

@@ -164,7 +164,8 @@ Error_FileTypeIncorrect=The file type is not correct.
 Error_FileTooLarge=The file is too large.
 Error_ClusterServiceError=An error occurred with the cluster service: %1%.   Check the log files for more information.
 Error_RemoteErrorValue=Remote Error: %1%
-Error_WordlistImportError=An error occured importing the wordlist: %1%
+Error_WordlistImportError=An error occurred importing the wordlist: %1%
+Error_PwNotifyServiceError=An error occurred while running the password notify service: %1%
 
 Error_ConfigUploadSuccess=File uploaded successfully
 Error_ConfigUploadFailure=File failed to upload.

server/src/main/resources/password/pwm/i18n/Health.properties

@@ -78,6 +78,7 @@ HealthMessage_LocalDB_CLOSED=LocalDB is CLOSED, statistics, online logging, word
 HealthMessage_LocalDBLogger_NOTOPEN=LocalDBLogger is not open, status is %1%
 HealthMessage_LocalDBLogger_HighRecordCount=LocalDBLogger event log record count of %1% records, is more than the configured maximum of %2%.  Excess records are being purged.
 HealthMessage_LocalDBLogger_OldRecordPresent=Oldest LocalDBLogger event log record is %1%, configured maximum is %2%.  Excess records are being purged.
+HealthMessage_ServiceClosed=unable to start %1% service
 HealthMessage_ServiceClosed_LocalDBUnavail=unable to start %1% service, LocalDB is not available
 HealthMessage_ServiceClosed_AppReadOnly=unable to start %1% service, application is in read-only mode
 HealthMessage_Wordlist_AutoImportFailure=Configured word list (%1%) failed to import due to error: %2% at timestamp %3%

server/src/main/resources/password/pwm/i18n/PwmSetting.properties

@@ -267,6 +267,7 @@ Setting_Description_challenge.showConfirmation=Enable this option to show the re
 Setting_Description_challenge.token.sendMethod=Select the methods you want to use for sending the token code or new password to the user.
 Setting_Description_challenge.userAttribute=Specify the attribute to use for response storage when storing responses in an LDAP directory.
 Setting_Description_changePassword.writeAttributes=Add actions to take after a user change password event occurs.  @PwmAppName@ invokes these actions just after writing the password.  You can use macros within the action and are expanded based on the logged in user.
+Setting_Description_cluster.enable=Enable cluster module.
 Setting_Description_cluster.storageMode=Data storage system used for cluster module.  <p>If <b>LDAP</b> is selected, a test user (<a data-gotoSettingLink\="ldap.testuser.username">@PwmSettingReference\:ldap.testuser.username@</a>) must be configured and the response storage attribute (<a data-gotoSettingLink\="ldap.testuser.username">@PwmSettingReference\:ldap.testuser.username@</a>) must be writable by the proxy user.</p><p>If <b>DATABASE</b> is selected then a database must be configured and available for @PwmAppName@ to operate.</p>  
 Setting_Description_command.checkResponses.queryMatch=Controls which users are forced to setup responses.  Users that match this permission will be forced to setup responses. 
 Setting_Description_db.classname=Add the remote database JDBC driver class name.  Consult the database vendor to determine the correct class name for your database.<br/><br/><table><tr><td class\="key">Database Type</td><td class\="key">Example Class Name</td></tr><tr><td>MS-SQL</td><td>com.microsoft.sqlserver.jdbc.SQLServerDriver</td></tr><tr><td>MS-SQL using jTDS</td><td>net.sourceforge.jtds.jdbc.Driver</td></tr><tr><td>Oracle</td><td>oracle.jdbc.OracleDriver</td></tr></table><div class="footnote">The above are examples only, consult your database documentation for the proper setting value.</div>
@@ -475,6 +476,9 @@ Setting_Description_ldap.serverCerts=Import the LDAP Server Certificates.  @PwmA
 Setting_Description_ldap.serverUrls=Add a list of LDAP servers in URL format that @PwmAppName@ uses for a fail-over configuration. @PwmAppName@ uses the servers in order of appearance in this list.  If the first server is unavailable @PwmAppName@ uses the next available server in the list.  @PwmAppName@ periodically checks the first server to see if it has become available.<ul><li>For secure SSL, use the "<i>ldaps\://servername\:636</i>" format</li><li>For plain-text servers, use "<i>ldap\://serverame\:389</i>" format (not recommended)</li></ul><p>When using secure connections, the Java virtual machine must trust the directory server, either because you have manually added the public key certificate from the tree to the Java keystore or you imported the certificate into the setting <i>LDAP Server Certificates</i>.<ul><li>Do not use a non-secure connection for anything but the most basic testing purposes (Many LDAP servers reject password operations on non-secure connections)</li><li>Do not use a load-balancing device for LDAP high availability, instead use the built in LDAP server fail-over functionality</li><li>Do not use a DNS round-robin address</li><li>Avoid using the network address, use the proper fully-qualified domain name address for the server</li></ul>
 Setting_Description_ldap.testuser.username=Specify the fully qualified DN of an LDAP test user that @PwmAppName@ uses to test functionality and for access to the LDAP directory. Configure this user similar to a normal user account with normal access privileges. @PwmAppName@ periodically uses this account to perform a health check, including changing the password of the account. <br/><br/><b>Using a test user account greatly increases the system's ability to detect and alert configuration and health issues.</b><br/><br/>@PwmAppName@ tests the following functionality (if enabled) using the test user account.<ul><li>Authentication</li><li>Password policy reading</li><li>Set password</li><li>Set Challenge/Responses</li><li>Load Challenge/Responses</li></ul>
 Setting_Description_ldap.user.group.attribute=Specify an attribute on the user entry that references group entries.  The value of this attribute in the directory must be a LDAP DN.
+Setting_Description_ldap.user.pwNotify.attribute=Specify an attribute that is used by @PwmAppName@ to store data for the password expiration notification service.
+Setting_Description_ldap.user.language.attribute=Attribute that contains the language of the user in RFC1766 format. (The same format used by web browsers and the HTTP Accept-Language header.)  This value is used only for user interactions when the user does not have an active web session such as an email notification.
+Setting_Description_ldap.user.language.autoSet=When enabled, the user's effective locale for a web session will be written to the LDAP language attribute. 
 Setting_Description_ldap.username.attr=Specify the attribute @PwmAppName@ uses for the user name. If blank, @PwmAppName@ uses the LDAP Naming Attribute.  This option allows fields that display or store the <b>User Name</b> or <b>User ID</b> of a user to show something other then the LDAP Naming Attribute if appropriate.  This value must be unique for this system.
 Setting_Description_ldap.usernameSearchFilter=Specify an LDAP search filter @PwmAppName@ uses for contextless login and other functions to find users in LDAP using user names.  Replace the value <b>%USERNAME%</b> with the actual user name value.
 Setting_Description_ldap.wireTrace.enable=Enable this option to have @PwmAppName@ output all LDAP traffic to the TRACE logging level.<br/><br/><b>WARNING\:</b> enabling this option might allow @PwmAppName@ to write user passwords and other sensitive data to the log files.
@@ -617,6 +621,7 @@ Setting_Description_pwm.securityKey=<p>Specify a Security Key used for cryptogra
 Setting_Description_pwm.seedlist.location=Specify the location of the seed list in the form of a valid URL. When @PwmAppName@ randomly generates passwords, it can generate a "friendly", random password suggestions to users.  It does this by using a "seed" word or words, and then modifying that word randomly until it is sufficiently complex and meets the configured rules computed for the user.<br/><br/>The value must be a valid URL, using the protocol "file" (local file system), "http", or "https".
 Setting_Description_pwm.selfURL=<p>The URL to this application, as seen by users. @PwmAppName@ uses the value in email macros and other user-facing communications.</p><p>The URL must use a valid fully qualified hostname. Do not use a network address.</p><p>In simple environments, the URL will be the base of the URL in the browser you are currently using to view this page, however in more complex environments the URL will typically be an upstream proxy, gateway or network device.</p><p>The URL should include the path to the base application, typically <code>/@Case:lower:[[@PwmAppName@]]@</code>.</p>
 Setting_Description_pwm.wordlist.location=Specify a word list file URL for dictionary checking to prevent users from using commonly used words as passwords.   Using word lists is an important part of password security.  Word lists are used by intruders to guess common passwords.   The default word list included contains commonly used English passwords.  <br/><br/>The first time a startup occurs with a new word list setting, it takes some time to compile the word list into a database.  See the status screen and logs for progress information.  The word list file format is one or more text files containing a single word per line, enclosed in a ZIP file.  The String <i>\!\#comment\:</i> at the beginning of a line indicates a comment. <br/><br/>The value must be a valid URL, using the protocol "file" (local file system), "http", or "https".
+Setting_Description_pwNotify.storageMode=Select storage mode used by cluster module.
 Setting_Description_pwNotify.enable=<p>Enable password expiration notification service.  Operation of this service requires that a remote database be configured.  Status of this service can be viewed on the <code>Administration -> Dashboard -> Password Notification</code> page.  The service will nominally execute once per day on the cluster master server.</p><p>If a job is missed because of an @PwmAppName@, LDAP, or database service interuption it will be run within the next 24 hours as soon as service is restored.  Running a job more than once will not result in duplicate emails sent to the user.</p>
 Setting_Description_pwNotify.queryString=Users that will receive password expiration notifications.
 Setting_Description_pwNotify.intervals=Expiration Notification Day Intervals.  The number of days before a user's password expiration before which an email notice will be set. 
@@ -778,6 +783,7 @@ Setting_Label_challenge.showConfirmation=Show Response Confirmation
 Setting_Label_challenge.token.sendMethod=Token Send Method
 Setting_Label_challenge.userAttribute=Response Storage Attribute
 Setting_Label_changePassword.writeAttributes=Post Password Change Actions
+Setting_Label_cluster.enable=Cluster Enabled
 Setting_Label_cluster.storageMode=Cluster Mode
 Setting_Label_command.checkResponses.queryMatch=Check Responses Match
 Setting_Label_db.classname=Database Class
@@ -986,6 +992,9 @@ Setting_Label_ldap.serverCerts=LDAP Certificates
 Setting_Label_ldap.serverUrls=LDAP URLs
 Setting_Label_ldap.testuser.username=LDAP Test User
 Setting_Label_ldap.user.group.attribute=User Group Attribute
+Setting_Label_ldap.user.pwNotify.attribute=Password Notify Service Attribute
+Setting_Label_ldap.user.language.attribute=User Language Attribute
+Setting_Label_ldap.user.language.autoSet=Auto Set User Language Attribute
 Setting_Label_ldap.username.attr=Attribute to use for User Name
 Setting_Label_ldap.usernameSearchFilter=User Name Search Filter
 Setting_Label_ldap.wireTrace.enable=Enable LDAP Wire Trace
@@ -1128,6 +1137,7 @@ Setting_Label_pwm.securityKey=Security Key
 Setting_Label_pwm.seedlist.location=Seed List File URL
 Setting_Label_pwm.selfURL=Site URL
 Setting_Label_pwm.wordlist.location=Word List File URL
+Setting_Label_pwNotify.storageMode=Storage Mode
 Setting_Label_pwNotify.enable=Enable Password Expiration Notification
 Setting_Label_pwNotify.queryString=Expiration Notification User Match
 Setting_Label_pwNotify.intervals=Expiration Notification Intervals

webapp/src/main/webapp/WEB-INF/jsp/admin-user-debug.jsp

@@ -32,6 +32,7 @@
 <%@ page import="password.pwm.i18n.Display" %>
 <%@ page import="java.util.Map" %>
 <%@ page import="password.pwm.util.java.TimeDuration" %>
+<%@ page import="password.pwm.util.LocaleHelper" %>
 <!DOCTYPE html>
 <%@ page language="java" session="true" isThreadSafe="true" contentType="text/html" %>
 <%@ taglib uri="pwm" prefix="pwm" %>
@@ -241,6 +242,10 @@
                     <%= JspUtility.friendlyWrite(pageContext, userDebugDataBean.isPasswordWithinMinimumLifetime()) %>
                 </td>
             </tr>
+            <tr>
+                <td class="key">Stored Language</td>
+                <td><%=JspUtility.friendlyWrite(pageContext, userInfo.getLanguage() )%></td>
+            </tr>
         </table>
         <br/>
         <table>

webapp/src/main/webapp/public/resources/js/admin.js

@@ -884,7 +884,7 @@ PWM_ADMIN.initPwNotifyPage = function() {
 PWM_ADMIN.loadPwNotifyStatus = function () {
     var processData = function (data) {
         var statusData = data['data']['statusData'];
-        var htmlData = '<tr><td colspan="2" class="title">Password Expiration Notification Status</td></tr>';
+        var htmlData = '<tr><td colspan="2" class="title">Password Expiration Notification Job Status</td></tr>';
         for (var item in statusData) {
             (function(key){
                 var item = statusData[key];