ldap connection and session management improvements

server/src/main/java/password/pwm/AppProperty.java

@@ -200,6 +200,8 @@ public enum AppProperty
     HEALTH_CERTIFICATE_WARN_SECONDS                 ( "health.certificate.warnSeconds" ),
     HEALTH_LDAP_CAUTION_DURATION_MS                 ( "health.ldap.cautionDurationMS" ),
     HEALTH_LDAP_PROXY_WARN_PW_EXPIRE_SECONDS        ( "health.ldap.proxy.pwExpireWarnSeconds" ),
+    HEALTH_LDAP_USER_SEARCH_TERM                    ( "health.ldap.userSearch.searchTerm" ),
+    HEALTH_LDAP_USER_SEARCH_WARN_MS                 ( "health.ldap.userSearch.warnMS" ),
     HEALTH_JAVA_MAX_THREADS                         ( "health.java.maxThreads" ),
     HEALTH_JAVA_MIN_HEAP_BYTES                      ( "health.java.minHeapBytes" ),
     HELPDESK_TOKEN_MAX_AGE                          ( "helpdesk.token.maxAgeSeconds" ),
@@ -215,6 +217,7 @@ public enum AppProperty
     LDAP_PROXY_CONNECTION_PER_PROFILE               ( "ldap.proxy.connectionsPerProfile" ),
     LDAP_PROXY_MAX_CONNECTIONS                      ( "ldap.proxy.maxConnections" ),
     LDAP_PROXY_USE_THREAD_LOCAL                     ( "ldap.proxy.useThreadLocal" ),
+    LDAP_PROXY_IDLE_THREAD_LOCAL_TIMEOUT_MS         ( "ldap.proxy.idleThreadLocal.timeoutMS" ),
     LDAP_EXTENSIONS_NMAS_ENABLE                     ( "ldap.extensions.nmas.enable" ),
     LDAP_CONNECTION_TIMEOUT                         ( "ldap.connection.timeoutMS" ),
     LDAP_PROFILE_RETRY_DELAY                        ( "ldap.profile.retryDelayMS" ),

server/src/main/java/password/pwm/health/HealthMessage.java

@@ -74,6 +74,7 @@ public enum HealthMessage
     LDAP_VendorsNotSame( HealthStatus.CONFIG, HealthTopic.LDAP ),
     LDAP_OK( HealthStatus.GOOD, HealthTopic.LDAP ),
     LDAP_RecentlyUnreachable( HealthStatus.CAUTION, HealthTopic.LDAP ),
+    LDAP_SearchFailure( HealthStatus.WARN, HealthTopic.LDAP ),
     CryptoTokenWithNewUserVerification( HealthStatus.CAUTION, HealthTopic.Configuration ),
     TokenServiceError( HealthStatus.WARN, HealthTopic.TokenService ),
     Java_HighThreads( HealthStatus.CAUTION, HealthTopic.Platform ),

server/src/main/java/password/pwm/health/LDAPHealthChecker.java

@@ -53,12 +53,14 @@ import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.ldap.LdapOperationsHelper;
 import password.pwm.ldap.UserInfo;
 import password.pwm.ldap.UserInfoFactory;
+import password.pwm.ldap.search.SearchConfiguration;
 import password.pwm.util.PasswordData;
 import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.macro.MacroMachine;
 import password.pwm.util.password.PasswordUtility;
 import password.pwm.util.password.RandomPasswordGenerator;
 import password.pwm.ws.server.rest.bean.HealthData;
@@ -150,6 +152,8 @@ public class LDAPHealthChecker implements HealthChecker
                 returnRecords.addAll( checkLdapDNSyntaxValues( pwmApplication ) );
 
                 returnRecords.addAll( checkNewUserPasswordTemplateSetting( pwmApplication, config ) );
+
+     //           returnRecords.addAll( checkUserSearching( pwmApplication ) );
             }
         }
 
@@ -960,6 +964,49 @@ public class LDAPHealthChecker implements HealthChecker
         return Collections.emptyList();
     }
 
+    private static List<HealthRecord> checkUserSearching(
+            final PwmApplication pwmApplication
+    )
+    {
+        final TimeDuration warnDuration = TimeDuration.of(
+                JavaHelper.silentParseLong( pwmApplication.getConfig().readAppProperty( AppProperty.HEALTH_LDAP_USER_SEARCH_WARN_MS ), 10_1000 ),
+                TimeDuration.Unit.MILLISECONDS );
+
+        final Instant startTime = Instant.now();
+
+
+        try
+        {
+            final String healthUsername = MacroMachine.forStatic().expandMacros( pwmApplication.getConfig().readAppProperty( AppProperty.HEALTH_LDAP_USER_SEARCH_TERM ) );
+
+            final SearchConfiguration searchConfiguration = SearchConfiguration.builder()
+                    .enableValueEscaping( false )
+                    .searchTimeout( warnDuration.asMillis() )
+                    .username( healthUsername )
+                    .build();
+
+            pwmApplication.getUserSearchEngine().performMultiUserSearch( searchConfiguration, 1, Collections.singletonList( "cn" ), SessionLabel.HEALTH_SESSION_LABEL );
+        }
+        catch ( final Exception e )
+        {
+            return Collections.singletonList(
+                    HealthRecord.forMessage( HealthMessage.LDAP_SearchFailure,
+                            e.getMessage()
+                    ) );
+        }
+
+        final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
+
+        if ( timeDuration.isLongerThan( warnDuration ) )
+        {
+            return Collections.singletonList(
+                    HealthRecord.forMessage( HealthMessage.LDAP_SearchFailure,
+                            "user search time of " + timeDuration.asLongString() + " exceeded ideal of " + warnDuration.asLongString(  )
+                    ) );
+        }
+
+        return Collections.emptyList();
+    }
 
     private static List<HealthRecord> checkUserPermission(
             final PwmApplication pwmApplication,

server/src/main/java/password/pwm/http/IdleTimeoutCalculator.java

@@ -194,7 +194,7 @@ public class IdleTimeoutCalculator
         {
             if ( config.readSettingAsBoolean( PwmSetting.HELPDESK_ENABLE ) )
             {
-                final HelpdeskProfile helpdeskProfile = pwmSession.getSessionManager().getHelpdeskProfile( pwmApplication );
+                final HelpdeskProfile helpdeskProfile = pwmSession.getSessionManager().getHelpdeskProfile( );
                 if ( helpdeskProfile != null )
                 {
                     final long helpdeskIdleTimeout = helpdeskProfile.readSettingAsLong( PwmSetting.HELPDESK_IDLE_TIMEOUT_SECONDS );
@@ -214,7 +214,7 @@ public class IdleTimeoutCalculator
                         && pwmURL.isPrivateUrl()
                 )
         {
-            final PeopleSearchProfile peopleSearchProfile = pwmSession.getSessionManager().getPeopleSearchProfile( pwmApplication );
+            final PeopleSearchProfile peopleSearchProfile = pwmSession.getSessionManager().getPeopleSearchProfile( );
             if ( peopleSearchProfile != null )
             {
                 final long peopleSearchIdleTimeout = peopleSearchProfile.readSettingAsLong( PwmSetting.PEOPLE_SEARCH_IDLE_TIMEOUT_SECONDS );

server/src/main/java/password/pwm/http/PwmSession.java

@@ -59,15 +59,16 @@ public class PwmSession implements Serializable
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwmSession.class );
 
-    private final LocalSessionStateBean sessionStateBean;
+    private final transient PwmApplication pwmApplication;
+    private final transient LocalSessionStateBean sessionStateBean = new LocalSessionStateBean();
+    private final transient UserSessionDataCacheBean userSessionDataCacheBean = new UserSessionDataCacheBean();
 
     private LoginInfoBean loginInfoBean;
     private transient UserInfo userInfo;
-    private UserSessionDataCacheBean userSessionDataCacheBean;
 
     private static final Object CREATION_LOCK = new Object();
 
-    private transient SessionManager sessionManager;
+    private final SessionManager sessionManager;
 
     public static PwmSession createPwmSession( final PwmApplication pwmApplication )
             throws PwmUnrecoverableException
@@ -87,7 +88,7 @@ public class PwmSession implements Serializable
             throw new IllegalStateException( "PwmApplication must be available during session creation" );
         }
 
-        sessionStateBean = new LocalSessionStateBean();
+        this.pwmApplication = pwmApplication;
         this.sessionStateBean.setSessionID( pwmApplication.getSessionTrackService().generateNewSessionID() );
 
         this.sessionStateBean.setSessionLastAccessedTime( Instant.now() );
@@ -98,6 +99,7 @@ public class PwmSession implements Serializable
         }
 
         pwmApplication.getSessionTrackService().addSessionData( this );
+        this.sessionManager = new SessionManager( pwmApplication, this );
 
         LOGGER.trace( () -> "created new session" );
     }
@@ -105,10 +107,6 @@ public class PwmSession implements Serializable
 
     public SessionManager getSessionManager( )
     {
-        if ( sessionManager == null )
-        {
-            sessionManager = new SessionManager( this );
-        }
         return sessionManager;
     }
 
@@ -188,10 +186,6 @@ public class PwmSession implements Serializable
 
     public UserSessionDataCacheBean getUserSessionDataCacheBean( )
     {
-        if ( userSessionDataCacheBean == null )
-        {
-            userSessionDataCacheBean = new UserSessionDataCacheBean();
-        }
         return userSessionDataCacheBean;
     }
 
@@ -279,7 +273,7 @@ public class PwmSession implements Serializable
 
         userInfo = null;
         loginInfoBean = null;
-        userSessionDataCacheBean = null;
+        userSessionDataCacheBean.clearPermissions();
     }
 
     public TimeDuration getIdleTime( )

server/src/main/java/password/pwm/http/SessionManager.java

@@ -59,13 +59,14 @@ public class SessionManager
 
     private static final PwmLogger LOGGER = PwmLogger.forClass( SessionManager.class );
 
-    private ChaiProvider chaiProvider;
+    private volatile ChaiProvider chaiProvider;
 
+    private final PwmApplication pwmApplication;
     private final PwmSession pwmSession;
 
-
-    public SessionManager( final PwmSession pwmSession )
+    public SessionManager( final PwmApplication pwmApplication, final PwmSession pwmSession )
     {
+        this.pwmApplication = pwmApplication;
         this.pwmSession = pwmSession;
     }
 
@@ -96,7 +97,7 @@ public class SessionManager
         this.chaiProvider = chaiProvider;
     }
 
-    public void updateUserPassword( final PwmApplication pwmApplication, final UserIdentity userIdentity, final PasswordData userPassword )
+    public void updateUserPassword( final UserIdentity userIdentity, final PasswordData userPassword )
             throws PwmUnrecoverableException
     {
         this.closeConnections();
@@ -141,7 +142,7 @@ public class SessionManager
         }
     }
 
-    public ChaiUser getActor( final PwmApplication pwmApplication )
+    public ChaiUser getActor( )
             throws ChaiUnavailableException, PwmUnrecoverableException
     {
 
@@ -160,12 +161,7 @@ public class SessionManager
         return this.getChaiProvider().getEntryFactory().newChaiUser( userDN.getUserDN() );
     }
 
-    public boolean hasActiveLdapConnection( )
-    {
-        return this.chaiProvider != null && this.chaiProvider.isConnected();
-    }
-
-    public ChaiUser getActor( final PwmApplication pwmApplication, final UserIdentity userIdentity )
+    public ChaiUser getActor( final UserIdentity userIdentity )
             throws PwmUnrecoverableException
     {
         try
@@ -246,7 +242,7 @@ public class SessionManager
         return status == Permission.PermissionStatus.GRANTED;
     }
 
-    public MacroMachine getMacroMachine( final PwmApplication pwmApplication )
+    public MacroMachine getMacroMachine( )
             throws PwmUnrecoverableException
     {
         final UserInfo userInfoBean = pwmSession.isAuthenticated()
@@ -269,27 +265,27 @@ public class SessionManager
         return null;
     }
 
-    public HelpdeskProfile getHelpdeskProfile( final PwmApplication pwmApplication ) throws PwmUnrecoverableException
+    public HelpdeskProfile getHelpdeskProfile() throws PwmUnrecoverableException
     {
         return ( HelpdeskProfile ) getProfile( pwmApplication, ProfileDefinition.Helpdesk );
     }
 
-    public SetupOtpProfile getSetupOTPProfile( final PwmApplication pwmApplication ) throws PwmUnrecoverableException
+    public SetupOtpProfile getSetupOTPProfile() throws PwmUnrecoverableException
     {
         return ( SetupOtpProfile ) getProfile( pwmApplication, ProfileDefinition.SetupOTPProfile );
     }
 
-    public UpdateProfileProfile getUpdateAttributeProfile( final PwmApplication pwmApplication ) throws PwmUnrecoverableException
+    public UpdateProfileProfile getUpdateAttributeProfile() throws PwmUnrecoverableException
     {
         return ( UpdateProfileProfile ) getProfile( pwmApplication, ProfileDefinition.UpdateAttributes );
     }
 
-    public PeopleSearchProfile getPeopleSearchProfile( final PwmApplication pwmApplication ) throws PwmUnrecoverableException
+    public PeopleSearchProfile getPeopleSearchProfile() throws PwmUnrecoverableException
     {
         return ( PeopleSearchProfile ) getProfile( pwmApplication, ProfileDefinition.PeopleSearch );
     }
 
-    public DeleteAccountProfile getSelfDeleteProfile( final PwmApplication pwmApplication ) throws PwmUnrecoverableException
+    public DeleteAccountProfile getSelfDeleteProfile() throws PwmUnrecoverableException
     {
         return ( DeleteAccountProfile ) getProfile( pwmApplication, ProfileDefinition.DeleteAccount );
     }

server/src/main/java/password/pwm/http/servlet/ClientApiServlet.java

@@ -355,7 +355,7 @@ public class ClientApiServlet extends ControlledPwmServlet
                     PwmSetting.DISPLAY_PASSWORD_GUIDE_TEXT,
                     pwmSession.getSessionStateBean().getLocale()
             );
-            final MacroMachine macroMachine = pwmSession.getSessionManager().getMacroMachine( pwmApplication );
+            final MacroMachine macroMachine = pwmSession.getSessionManager().getMacroMachine( );
             passwordGuideText = macroMachine.expandMacros( passwordGuideText );
             settingMap.put( "passwordGuideText", passwordGuideText );
         }
@@ -431,7 +431,7 @@ public class ClientApiServlet extends ControlledPwmServlet
         final ResourceBundle bundle = ResourceBundle.getBundle( displayClass.getName() );
         try
         {
-            final MacroMachine macroMachine = pwmSession.getSessionManager().getMacroMachine( pwmApplication );
+            final MacroMachine macroMachine = pwmSession.getSessionManager().getMacroMachine( );
             for ( final String key : new TreeSet<>( Collections.list( bundle.getKeys() ) ) )
             {
                 String displayValue = LocaleHelper.getLocalizedMessage( userLocale, key, config, displayClass );

server/src/main/java/password/pwm/http/servlet/DeleteAccountServlet.java

@@ -96,7 +96,7 @@ public class DeleteAccountServlet extends ControlledPwmServlet
 
     private DeleteAccountProfile getProfile( final PwmRequest pwmRequest ) throws PwmUnrecoverableException
     {
-        return pwmRequest.getPwmSession().getSessionManager().getSelfDeleteProfile( pwmRequest.getPwmApplication() );
+        return pwmRequest.getPwmSession().getSessionManager().getSelfDeleteProfile( );
     }
 
     private DeleteAccountBean getBean( final PwmRequest pwmRequest ) throws PwmUnrecoverableException
@@ -143,7 +143,7 @@ public class DeleteAccountServlet extends ControlledPwmServlet
         {
             if ( !bean.isAgreementPassed() )
             {
-                final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmRequest.getPwmApplication() );
+                final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( );
                 final String expandedText = macroMachine.expandMacros( selfDeleteAgreementText );
                 pwmRequest.setAttribute( PwmRequestAttribute.AgreementText, expandedText );
                 pwmRequest.forwardToJsp( JspUrl.SELF_DELETE_AGREE );
@@ -210,7 +210,7 @@ public class DeleteAccountServlet extends ControlledPwmServlet
 
                 final ActionExecutor actionExecutor = new ActionExecutor.ActionExecutorSettings( pwmApplication, userIdentity )
                         .setExpandPwmMacros( true )
-                        .setMacroMachine( pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmApplication ) )
+                        .setMacroMachine( pwmRequest.getPwmSession().getSessionManager().getMacroMachine( ) )
                         .createActionExecutor();
 
                 try
@@ -234,7 +234,7 @@ public class DeleteAccountServlet extends ControlledPwmServlet
         final String nextUrl = deleteAccountProfile.readSettingAsString( PwmSetting.DELETE_ACCOUNT_NEXT_URL );
         if ( nextUrl != null && !nextUrl.isEmpty() )
         {
-            final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmApplication );
+            final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( );
             final String macroedUrl = macroMachine.expandMacros( nextUrl );
             LOGGER.debug( pwmRequest, () -> "setting forward url to post-delete next url: " + macroedUrl );
             pwmRequest.getPwmSession().getSessionStateBean().setForwardURL( macroedUrl );
@@ -283,7 +283,7 @@ public class DeleteAccountServlet extends ControlledPwmServlet
         pwmRequest.getPwmApplication().getEmailQueue().submitEmail(
                 configuredEmailSetting,
                 pwmRequest.getPwmSession().getUserInfo(),
-                pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmRequest.getPwmApplication() )
+                pwmRequest.getPwmSession().getSessionManager().getMacroMachine( )
         );
     }
 }

server/src/main/java/password/pwm/http/servlet/GuestRegistrationServlet.java

@@ -231,7 +231,7 @@ public class GuestRegistrationServlet extends AbstractPwmServlet
             FormUtility.validateFormValues( config, formValues, ssBean.getLocale() );
 
             //read current values from user.
-            final ChaiUser theGuest = pwmSession.getSessionManager().getActor( pwmApplication, guestRegistrationBean.getUpdateUserIdentity() );
+            final ChaiUser theGuest = pwmSession.getSessionManager().getActor( guestRegistrationBean.getUpdateUserIdentity() );
 
             // check unique fields against ldap
             FormUtility.validateFormValueUniqueness(
@@ -244,7 +244,7 @@ public class GuestRegistrationServlet extends AbstractPwmServlet
             final Instant expirationDate = readExpirationFromRequest( pwmRequest );
 
             // Update user attributes
-            LdapOperationsHelper.writeFormValuesToLdap( theGuest, formValues, pwmSession.getSessionManager().getMacroMachine( pwmApplication ), false );
+            LdapOperationsHelper.writeFormValuesToLdap( theGuest, formValues, pwmSession.getSessionManager().getMacroMachine( ), false );
 
             // Write expirationDate
             if ( expirationDate != null )

server/src/main/java/password/pwm/http/servlet/SetupOtpServlet.java

@@ -117,7 +117,7 @@ public class SetupOtpServlet extends ControlledPwmServlet
 
     public static SetupOtpProfile getSetupOtpProfile( final PwmRequest pwmRequest ) throws PwmUnrecoverableException
     {
-        return pwmRequest.getPwmSession().getSessionManager().getSetupOTPProfile( pwmRequest.getPwmApplication() );
+        return pwmRequest.getPwmSession().getSessionManager().getSetupOTPProfile( );
     }
 
     @Override
@@ -328,7 +328,7 @@ public class SetupOtpServlet extends ControlledPwmServlet
         final UserIdentity theUser = pwmSession.getUserInfo().getUserIdentity();
         try
         {
-            service.clearOTPUserConfiguration( pwmRequest, theUser, pwmSession.getSessionManager().getActor( pwmApplication ) );
+            service.clearOTPUserConfiguration( pwmRequest, theUser, pwmSession.getSessionManager().getActor( ) );
         }
         catch ( final PwmOperationalException e )
         {
@@ -440,7 +440,7 @@ public class SetupOtpServlet extends ControlledPwmServlet
                 final Configuration config = pwmApplication.getConfig();
                 final SetupOtpProfile setupOtpProfile = getSetupOtpProfile( pwmRequest );
                 final String identifierConfigValue = setupOtpProfile.readSettingAsString( PwmSetting.OTP_SECRET_IDENTIFIER );
-                final String identifier = pwmSession.getSessionManager().getMacroMachine( pwmApplication ).expandMacros( identifierConfigValue );
+                final String identifier = pwmSession.getSessionManager().getMacroMachine( ).expandMacros( identifierConfigValue );
                 final OTPUserRecord otpUserRecord = new OTPUserRecord();
                 final List<String> rawRecoveryCodes = pwmApplication.getOtpService().initializeUserRecord(
                         setupOtpProfile,

server/src/main/java/password/pwm/http/servlet/SetupResponsesServlet.java

@@ -210,7 +210,7 @@ public class SetupResponsesServlet extends ControlledPwmServlet
         try
         {
             final String userGUID = pwmSession.getUserInfo().getUserGuid();
-            final ChaiUser theUser = pwmSession.getSessionManager().getActor( pwmApplication );
+            final ChaiUser theUser = pwmSession.getSessionManager().getActor( );
             pwmApplication.getCrService().clearResponses( pwmRequest.getLabel(), pwmRequest.getUserInfoIfLoggedIn(), theUser, userGUID );
             pwmSession.reloadUserInfoBean( pwmRequest );
             pwmRequest.getPwmApplication().getSessionStateService().clearBean( pwmRequest, SetupResponsesBean.class );
@@ -431,7 +431,7 @@ public class SetupResponsesServlet extends ControlledPwmServlet
     {
         final PwmApplication pwmApplication = pwmRequest.getPwmApplication();
         final PwmSession pwmSession = pwmRequest.getPwmSession();
-        final ChaiUser theUser = pwmSession.getSessionManager().getActor( pwmApplication );
+        final ChaiUser theUser = pwmSession.getSessionManager().getActor( );
         final String userGUID = pwmSession.getUserInfo().getUserGuid();
         pwmApplication.getCrService().writeResponses( pwmRequest.getUserInfoIfLoggedIn(), theUser, userGUID, responseInfoBean );
         pwmSession.reloadUserInfoBean( pwmRequest );

server/src/main/java/password/pwm/http/servlet/accountinfo/AccountInformationBean.java

@@ -103,7 +103,7 @@ public class AccountInformationBean implements Serializable
             throws PwmUnrecoverableException
     {
         final PwmPasswordPolicy pwmPasswordPolicy = pwmRequest.getPwmSession().getUserInfo().getPasswordPolicy();
-        final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmRequest.getPwmApplication() );
+        final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine();
         final List<String> rules = PasswordRequirementsTag.getPasswordRequirementsStrings( pwmPasswordPolicy, pwmRequest.getConfig(), pwmRequest.getLocale(), macroMachine );
         return Collections.unmodifiableList( rules );
     }

server/src/main/java/password/pwm/http/servlet/activation/ActivateUserUtils.java

@@ -154,9 +154,8 @@ class ActivateUserUtils
     )
             throws ChaiUnavailableException, PwmDataValidationException, PwmUnrecoverableException
     {
-        final PwmApplication pwmApplication = pwmRequest.getPwmApplication();
         final String searchFilter = figureLdapSearchFilter( pwmRequest );
-        final ChaiProvider chaiProvider = pwmApplication.getProxyChaiProvider( userIdentity.getLdapProfileID() );
+        final ChaiProvider chaiProvider = pwmRequest.getPwmApplication().getProxyChaiProvider( userIdentity.getLdapProfileID() );
         final ChaiUser chaiUser = chaiProvider.getEntryFactory().newChaiUser( userIdentity.getUserDN() );
 
         for ( final Map.Entry<FormConfiguration, String> entry : formValues.entrySet() )
@@ -251,7 +250,7 @@ class ActivateUserUtils
         pwmApplication.getEmailQueue().submitEmail(
                 configuredEmailSetting,
                 pwmSession.getUserInfo(),
-                pwmSession.getSessionManager().getMacroMachine( pwmApplication )
+                pwmSession.getSessionManager().getMacroMachine( )
         );
         return true;
     }
@@ -289,7 +288,7 @@ class ActivateUserUtils
                 toSmsNumber,
                 message,
                 pwmRequest.getLabel(),
-                pwmSession.getSessionManager().getMacroMachine( pwmApplication )
+                pwmSession.getSessionManager().getMacroMachine( )
         );
         return true;
     }

server/src/main/java/password/pwm/http/servlet/changepw/ChangePasswordServlet.java

@@ -189,7 +189,7 @@ public abstract class ChangePasswordServlet extends ControlledPwmServlet
         // check the password meets the requirements
         try
         {
-            final ChaiUser theUser = pwmRequest.getPwmSession().getSessionManager().getActor( pwmRequest.getPwmApplication() );
+            final ChaiUser theUser = pwmRequest.getPwmSession().getSessionManager().getActor( );
             final PwmPasswordRuleValidator pwmPasswordRuleValidator = new PwmPasswordRuleValidator( pwmRequest.getPwmApplication(), userInfo.getPasswordPolicy() );
             final PasswordData oldPassword = pwmRequest.getPwmSession().getLoginInfoBean().getUserCurrentPassword();
             pwmPasswordRuleValidator.testPassword( password1, oldPassword, userInfo, theUser );
@@ -295,7 +295,7 @@ public abstract class ChangePasswordServlet extends ControlledPwmServlet
                     pwmRequest, formItem, ssBean.getLocale() );
 
             ChangePasswordServletUtil.validateParamsAgainstLDAP( formValues, pwmRequest,
-                    pwmRequest.getPwmSession().getSessionManager().getActor( pwmRequest.getPwmApplication() ) );
+                    pwmRequest.getPwmSession().getSessionManager().getActor( ) );
 
             cpb.setFormPassed( true );
         }
@@ -378,7 +378,7 @@ public abstract class ChangePasswordServlet extends ControlledPwmServlet
             pwmRequest.getPwmApplication().getSessionStateService().clearBean( pwmRequest, ChangePasswordBean.class );
             if ( completeMessage != null && !completeMessage.isEmpty() )
             {
-                final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmRequest.getPwmApplication() );
+                final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( );
                 final String expandedText = macroMachine.expandMacros( completeMessage );
                 pwmRequest.setAttribute( PwmRequestAttribute.CompleteText, expandedText );
                 pwmRequest.forwardToJsp( JspUrl.PASSWORD_COMPLETE );
@@ -407,7 +407,7 @@ public abstract class ChangePasswordServlet extends ControlledPwmServlet
         final PasswordUtility.PasswordCheckInfo passwordCheckInfo = PasswordUtility.checkEnteredPassword(
                 pwmRequest.getPwmApplication(),
                 pwmRequest.getLocale(),
-                pwmRequest.getPwmSession().getSessionManager().getActor( pwmRequest.getPwmApplication() ),
+                pwmRequest.getPwmSession().getSessionManager().getActor(),
                 userInfo,
                 pwmRequest.getPwmSession().getLoginInfoBean(),
                 PasswordData.forStringValue( jsonInput.getPassword1() ),
@@ -465,7 +465,7 @@ public abstract class ChangePasswordServlet extends ControlledPwmServlet
         final String agreementMsg = pwmApplication.getConfig().readSettingAsLocalizedString( PwmSetting.PASSWORD_CHANGE_AGREEMENT_MESSAGE, pwmRequest.getLocale() );
         if ( agreementMsg != null && agreementMsg.length() > 0 && !changePasswordBean.isAgreementPassed() )
         {
-            final MacroMachine macroMachine = pwmSession.getSessionManager().getMacroMachine( pwmApplication );
+            final MacroMachine macroMachine = pwmSession.getSessionManager().getMacroMachine();
             final String expandedText = macroMachine.expandMacros( agreementMsg );
             pwmRequest.setAttribute( PwmRequestAttribute.AgreementText, expandedText );
             pwmRequest.forwardToJsp( JspUrl.PASSWORD_AGREEMENT );
@@ -557,7 +557,7 @@ public abstract class ChangePasswordServlet extends ControlledPwmServlet
         final String passwordPolicyChangeMessage = pwmRequest.getPwmSession().getUserInfo().getPasswordPolicy().getRuleHelper().getChangeMessage();
         if ( passwordPolicyChangeMessage.length() > 1 )
         {
-            final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmRequest.getPwmApplication() );
+            final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( );
             macroMachine.expandMacros( passwordPolicyChangeMessage );
             pwmRequest.setAttribute( PwmRequestAttribute.ChangePassword_PasswordPolicyChangeMessage, passwordPolicyChangeMessage );
         }

server/src/main/java/password/pwm/http/servlet/changepw/ChangePasswordServletUtil.java

@@ -161,7 +161,7 @@ public class ChangePasswordServletUtil
                 configuredEmailSetting,
                 pwmRequest.getPwmSession().getUserInfo(),
 
-                pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmApplication ) );
+                pwmRequest.getPwmSession().getSessionManager().getMacroMachine( ) );
     }
 
     static void checkMinimumLifetime(

server/src/main/java/password/pwm/http/servlet/configeditor/ConfigEditorServlet.java

@@ -989,7 +989,7 @@ public class ConfigEditorServlet extends ControlledPwmServlet
             final MacroMachine macroMachine;
             if ( pwmRequest.isAuthenticated() )
             {
-                macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmRequest.getPwmApplication() );
+                macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine();
             }
             else
             {

server/src/main/java/password/pwm/http/servlet/configmanager/DebugItemGenerator.java

@@ -42,6 +42,7 @@ import password.pwm.http.ContextManager;
 import password.pwm.http.servlet.admin.AppDashboardData;
 import password.pwm.http.servlet.admin.UserDebugDataBean;
 import password.pwm.http.servlet.admin.UserDebugDataReader;
+import password.pwm.ldap.LdapConnectionService;
 import password.pwm.ldap.LdapDebugDataGenerator;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.cache.CacheService;
@@ -119,6 +120,7 @@ public class DebugItemGenerator
             ClusterInfoDebugGenerator.class,
             CacheServiceDebugItemGenerator.class,
             RootFileSystemDebugItemGenerator.class,
+            LdapConnectionsDebugItemGenerator.class,
             StatisticsDataDebugItemGenerator.class,
             StatisticsEpsDataDebugItemGenerator.class
     ) );
@@ -805,6 +807,25 @@ public class DebugItemGenerator
         }
     }
 
+    static class LdapConnectionsDebugItemGenerator implements Generator
+    {
+        @Override
+        public String getFilename()
+        {
+            return "ldap-connections.json";
+        }
+
+        @Override
+        public void outputItem( final DebugItemInput debugItemInput, final OutputStream outputStream ) throws Exception
+        {
+            final PwmApplication pwmApplication = debugItemInput.getPwmApplication();
+            final List<LdapConnectionService.ConnectionInfo> connectionInfos = pwmApplication.getLdapConnectionService().getConnectionInfos();
+            final Writer writer = new OutputStreamWriter( outputStream, PwmConstants.DEFAULT_CHARSET );
+            writer.write( JsonUtil.serializeCollection( connectionInfos, JsonUtil.Flag.PrettyPrint ) );
+            writer.flush();
+        }
+    }
+
     static class StatisticsEpsDataDebugItemGenerator implements Generator
     {
         @Override

server/src/main/java/password/pwm/http/servlet/helpdesk/HelpdeskServlet.java

@@ -169,7 +169,7 @@ public class HelpdeskServlet extends ControlledPwmServlet
 
     private HelpdeskProfile getHelpdeskProfile( final PwmRequest pwmRequest ) throws PwmUnrecoverableException
     {
-        return pwmRequest.getPwmSession().getSessionManager().getHelpdeskProfile( pwmRequest.getPwmApplication() );
+        return pwmRequest.getPwmSession().getSessionManager().getHelpdeskProfile( );
     }
 
     @Override
@@ -201,7 +201,7 @@ public class HelpdeskServlet extends ControlledPwmServlet
             return ProcessStatus.Halt;
         }
 
-        final HelpdeskProfile helpdeskProfile = pwmRequest.getPwmSession().getSessionManager().getHelpdeskProfile( pwmApplication );
+        final HelpdeskProfile helpdeskProfile = pwmRequest.getPwmSession().getSessionManager().getHelpdeskProfile( );
         if ( helpdeskProfile == null )
         {
             pwmRequest.respondWithError( PwmError.ERROR_UNAUTHORIZED.toInfo() );
@@ -273,7 +273,7 @@ public class HelpdeskServlet extends ControlledPwmServlet
 
             final ChaiUser chaiUser = useProxy
                     ? pwmRequest.getPwmApplication().getProxiedChaiUser( userIdentity )
-                    : pwmRequest.getPwmSession().getSessionManager().getActor( pwmRequest.getPwmApplication(), userIdentity );
+                    : pwmRequest.getPwmSession().getSessionManager().getActor( userIdentity );
             final MacroMachine macroMachine = MacroMachine.forUser( pwmRequest, userIdentity );
             final ActionExecutor actionExecutor = new ActionExecutor.ActionExecutorSettings( pwmRequest.getPwmApplication(), chaiUser )
                     .setExpandPwmMacros( true )
@@ -973,7 +973,7 @@ public class HelpdeskServlet extends ControlledPwmServlet
         final boolean useProxy = helpdeskProfile.readSettingAsBoolean( PwmSetting.HELPDESK_USE_PROXY );
         return useProxy
                 ? pwmRequest.getPwmApplication().getProxiedChaiUser( userIdentity )
-                : pwmRequest.getPwmSession().getSessionManager().getActor( pwmRequest.getPwmApplication(), userIdentity );
+                : pwmRequest.getPwmSession().getSessionManager().getActor( userIdentity );
     }
 
 
@@ -1144,7 +1144,7 @@ public class HelpdeskServlet extends ControlledPwmServlet
             return ProcessStatus.Halt;
         }
 
-        final HelpdeskProfile helpdeskProfile = pwmRequest.getPwmSession().getSessionManager().getHelpdeskProfile( pwmRequest.getPwmApplication() );
+        final HelpdeskProfile helpdeskProfile = pwmRequest.getPwmSession().getSessionManager().getHelpdeskProfile( );
         HelpdeskServletUtil.checkIfUserIdentityViewable( pwmRequest, helpdeskProfile, userIdentity );
 
         {
@@ -1246,7 +1246,7 @@ public class HelpdeskServlet extends ControlledPwmServlet
     @ActionHandler( action = "setPassword" )
     private ProcessStatus processSetPasswordAction( final PwmRequest pwmRequest ) throws IOException, PwmUnrecoverableException, ChaiUnavailableException
     {
-        final HelpdeskProfile helpdeskProfile = pwmRequest.getPwmSession().getSessionManager().getHelpdeskProfile( pwmRequest.getPwmApplication() );
+        final HelpdeskProfile helpdeskProfile = pwmRequest.getPwmSession().getSessionManager().getHelpdeskProfile( );
 
         final RestSetPasswordServer.JsonInputData jsonInput = JsonUtil.deserialize(
                 pwmRequest.readRequestBodyAsString(),

server/src/main/java/password/pwm/http/servlet/newuser/NewUserServlet.java

@@ -727,7 +727,7 @@ public class NewUserServlet extends ControlledPwmServlet
         final String configuredRedirectUrl = newUserProfile.readSettingAsString( PwmSetting.NEWUSER_REDIRECT_URL );
         if ( !StringUtil.isEmpty( configuredRedirectUrl ) && StringUtil.isEmpty( pwmRequest.getPwmSession().getSessionStateBean().getForwardURL() ) )
         {
-            final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmRequest.getPwmApplication() );
+            final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine();
             final String macroedUrl = macroMachine.expandMacros( configuredRedirectUrl );
             pwmRequest.sendRedirect( macroedUrl );
             return ProcessStatus.Halt;

server/src/main/java/password/pwm/http/servlet/newuser/NewUserUtils.java

@@ -316,7 +316,7 @@ class NewUserUtils
 
                 final ActionExecutor actionExecutor = new ActionExecutor.ActionExecutorSettings( pwmApplication, userIdentity )
                         .setExpandPwmMacros( true )
-                        .setMacroMachine( pwmSession.getSessionManager().getMacroMachine( pwmApplication ) )
+                        .setMacroMachine( pwmSession.getSessionManager().getMacroMachine( ) )
                         .createActionExecutor();
 
                 actionExecutor.executeActions( actions, pwmRequest.getLabel() );
@@ -476,7 +476,7 @@ class NewUserUtils
         pwmRequest.getPwmApplication().getEmailQueue().submitEmail(
                 configuredEmailSetting,
                 pwmSession.getUserInfo(),
-                pwmSession.getSessionManager().getMacroMachine( pwmRequest.getPwmApplication() )
+                pwmSession.getSessionManager().getMacroMachine( )
         );
     }
 

server/src/main/java/password/pwm/http/servlet/peoplesearch/PeopleSearchDataReader.java

@@ -693,7 +693,7 @@ class PeopleSearchDataReader
         final boolean useProxy = useProxy();
         return useProxy
                 ? pwmRequest.getPwmApplication().getProxiedChaiUser( userIdentity )
-                : pwmRequest.getPwmSession().getSessionManager().getActor( pwmRequest.getPwmApplication(), userIdentity );
+                : pwmRequest.getPwmSession().getSessionManager().getActor( userIdentity );
     }
 
     private UserSearchResults doDetailLookup(

server/src/main/java/password/pwm/http/servlet/peoplesearch/PeopleSearchServlet.java

@@ -331,7 +331,7 @@ public abstract class PeopleSearchServlet extends ControlledPwmServlet
 
         if ( pwmRequest.isAuthenticated() )
         {
-            return pwmRequest.getPwmSession().getSessionManager().getPeopleSearchProfile( pwmRequest.getPwmApplication() );
+            return pwmRequest.getPwmSession().getSessionManager().getPeopleSearchProfile();
         }
 
         throw PwmUnrecoverableException.newException( PwmError.ERROR_NO_PROFILE_ASSIGNED, "unable to load peoplesearch profile for authenticated user" );

server/src/main/java/password/pwm/http/servlet/updateprofile/UpdateProfileServlet.java

@@ -140,7 +140,7 @@ public class UpdateProfileServlet extends ControlledPwmServlet
 
     private static UpdateProfileProfile getProfile( final PwmRequest pwmRequest ) throws PwmUnrecoverableException
     {
-        return pwmRequest.getPwmSession().getSessionManager().getUpdateAttributeProfile( pwmRequest.getPwmApplication() );
+        return pwmRequest.getPwmSession().getSessionManager().getUpdateAttributeProfile( );
     }
 
     private static UpdateProfileBean getBean( final PwmRequest pwmRequest ) throws PwmUnrecoverableException
@@ -347,7 +347,7 @@ public class UpdateProfileServlet extends ControlledPwmServlet
             {
                 if ( !updateProfileBean.isAgreementPassed() )
                 {
-                    final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmRequest.getPwmApplication() );
+                    final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( );
                     final String expandedText = macroMachine.expandMacros( updateProfileAgreementText );
                     pwmRequest.setAttribute( PwmRequestAttribute.AgreementText, expandedText );
                     pwmRequest.forwardToJsp( JspUrl.UPDATE_ATTRIBUTES_AGREEMENT );
@@ -406,13 +406,13 @@ public class UpdateProfileServlet extends ControlledPwmServlet
         try
         {
             // write the form values
-            final ChaiUser theUser = pwmSession.getSessionManager().getActor( pwmApplication );
+            final ChaiUser theUser = pwmSession.getSessionManager().getActor( );
             UpdateProfileUtil.doProfileUpdate(
                     pwmRequest.getPwmApplication(),
                     pwmRequest.getLabel(),
                     pwmRequest.getLocale(),
                     pwmSession.getUserInfo(),
-                    pwmSession.getSessionManager().getMacroMachine( pwmApplication ),
+                    pwmSession.getSessionManager().getMacroMachine( ),
                     updateProfileProfile,
                     updateProfileBean.getFormData(),
                     theUser

server/src/main/java/password/pwm/http/tag/DisplayTag.java

@@ -132,7 +132,7 @@ public class DisplayTag extends PwmAbstractTag
 
             if ( pwmRequest != null )
             {
-                final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmRequest.getPwmApplication() );
+                final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( );
                 displayMessage = macroMachine.expandMacros( displayMessage );
             }
 

server/src/main/java/password/pwm/http/tag/ErrorMessageTag.java

@@ -87,7 +87,7 @@ public class ErrorMessageTag extends PwmAbstractTag
 
                 outputMsg = outputMsg.replace( "\n", "<br/>" );
 
-                final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmApplication );
+                final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( );
                 outputMsg = macroMachine.expandMacros( outputMsg );
 
                 pageContext.getOut().write( outputMsg );

server/src/main/java/password/pwm/http/tag/PasswordRequirementsTag.java

@@ -552,7 +552,7 @@ public class PasswordRequirementsTag extends TagSupport
             final Configuration config = pwmApplication.getConfig();
             final Locale locale = pwmSession.getSessionStateBean().getLocale();
 
-            pwmSession.getSessionManager().getMacroMachine( pwmApplication );
+            pwmSession.getSessionManager().getMacroMachine( );
 
             final PwmPasswordPolicy passwordPolicy;
             if ( getForm() != null && getForm().equalsIgnoreCase( "newuser" ) )
@@ -572,7 +572,7 @@ public class PasswordRequirementsTag extends TagSupport
             }
             else
             {
-                final MacroMachine macroMachine = pwmSession.getSessionManager().getMacroMachine( pwmApplication );
+                final MacroMachine macroMachine = pwmSession.getSessionManager().getMacroMachine( );
 
                 final String pre = prepend != null && prepend.length() > 0 ? prepend : "";
                 final String sep = separator != null && separator.length() > 0 ? separator : "<br/>";

server/src/main/java/password/pwm/http/tag/PwmMacroTag.java

@@ -52,7 +52,7 @@ public class PwmMacroTag extends TagSupport
         try
         {
             final PwmRequest pwmRequest = PwmRequest.forRequest( ( HttpServletRequest ) pageContext.getRequest(), ( HttpServletResponse ) pageContext.getResponse() );
-            final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmRequest.getPwmApplication() );
+            final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( );
             final String outputValue = macroMachine.expandMacros( value );
             pageContext.getOut().write( outputValue );
         }

server/src/main/java/password/pwm/http/tag/SuccessMessageTag.java

@@ -54,7 +54,7 @@ public class SuccessMessageTag extends PwmAbstractTag
             {
                 if ( pwmRequest.isAuthenticated() )
                 {
-                    final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmRequest.getPwmApplication() );
+                    final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( );
                     outputMsg = macroMachine.expandMacros( successMsg );
                 }
                 else

server/src/main/java/password/pwm/http/tag/conditional/PwmIfTest.java

@@ -476,7 +476,7 @@ public enum PwmIfTest
             if ( pwmRequest.getConfig().readSettingAsBoolean( PwmSetting.PEOPLE_SEARCH_ENABLE ) )
             {
                 final Optional<PeopleSearchProfile> peopleSearchProfile = pwmRequest.isAuthenticated()
-                        ? Optional.ofNullable( pwmRequest.getPwmSession().getSessionManager().getPeopleSearchProfile( pwmRequest.getPwmApplication() ) )
+                        ? Optional.ofNullable( pwmRequest.getPwmSession().getSessionManager().getPeopleSearchProfile( ) )
                         : pwmRequest.getConfig().getPublicPeopleSearchProfile();
 
                 if ( peopleSearchProfile.isPresent() )
@@ -515,7 +515,7 @@ public enum PwmIfTest
                 return false;
             }
 
-            final SetupOtpProfile setupOtpProfile = pwmRequest.getPwmSession().getSessionManager().getSetupOTPProfile( pwmRequest.getPwmApplication() );
+            final SetupOtpProfile setupOtpProfile = pwmRequest.getPwmSession().getSessionManager().getSetupOTPProfile( );
             return setupOtpProfile != null && setupOtpProfile.readSettingAsBoolean( PwmSetting.OTP_ALLOW_SETUP );
         }
     }

server/src/main/java/password/pwm/http/tag/value/PwmValue.java

@@ -114,8 +114,7 @@ public enum PwmValue
             {
                 try
                 {
-                    final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine(
-                            pwmRequest.getPwmApplication() );
+                    final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine();
                     outputURL = macroMachine.expandMacros( outputURL );
                 }
                 catch ( final PwmUnrecoverableException e )
@@ -158,8 +157,7 @@ public enum PwmValue
             {
                 try
                 {
-                    final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine(
-                            pwmRequest.getPwmApplication() );
+                    final MacroMachine macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine();
                     final String expandedScript = macroMachine.expandMacros( customScript );
                     return expandedScript;
                 }

server/src/main/java/password/pwm/ldap/LdapConnectionService.java

@@ -23,6 +23,11 @@ package password.pwm.ldap;
 import com.google.gson.reflect.TypeToken;
 import com.novell.ldapchai.provider.ChaiProvider;
 import com.novell.ldapchai.provider.ChaiProviderFactory;
+import com.novell.ldapchai.provider.ChaiSetting;
+import com.novell.ldapchai.provider.ProviderStatistics;
+import lombok.Builder;
+import lombok.Data;
+import lombok.Value;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.config.option.DataStorageMethod;
@@ -33,31 +38,73 @@ import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.PwmService;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.AtomicLoopIntIncrementer;
+import password.pwm.util.java.ConditionalTaskExecutor;
+import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
+import java.io.Serializable;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.WeakHashMap;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
 
 public class LdapConnectionService implements PwmService
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( LdapConnectionService.class );
 
-    private final Map<String, Map<Integer, ChaiProvider>> proxyChaiProviders = new ConcurrentHashMap<>();
     private final Map<String, ErrorInformation> lastLdapErrors = new ConcurrentHashMap<>();
+    private final ThreadLocal<ThreadLocalContainer> threadLocalProvider = new ThreadLocal<>();
+    private final Set<ThreadLocalContainer> threadLocalContainers = Collections.synchronizedSet( Collections.newSetFromMap( new WeakHashMap<>() ) );
+    private final ReentrantLock reentrantLock = new ReentrantLock();
+    private final ConditionalTaskExecutor debugLogger = ConditionalTaskExecutor.forPeriodicTask( this::logDebugInfo, TimeDuration.MINUTE );
+    private final ChaiProviderFactory chaiProviderFactory = ChaiProviderFactory.newProviderFactory();
+    private final Map<String, Map<Integer, ChaiProvider>> proxyChaiProviders = new HashMap<>();
 
-    private boolean useThreadLocal;
     private PwmApplication pwmApplication;
-    private STATUS status = STATUS.NEW;
+    private ExecutorService executorService;
     private AtomicLoopIntIncrementer slotIncrementer;
-    private final ThreadLocal<Map<String, ChaiProvider>> threadLocalProvider = new ThreadLocal<>();
-    private ChaiProviderFactory chaiProviderFactory;
+
+    private volatile STATUS status = STATUS.NEW;
+
+    private boolean useThreadLocal;
+    private final AtomicLoopIntIncrementer statCreatedProxies = new AtomicLoopIntIncrementer();
+    private final AtomicLoopIntIncrementer statClearedThreadLocals = new AtomicLoopIntIncrementer();
+
+    private enum DebugKey
+    {
+        /** Providers created since application start. */
+        CreatedProviders,
+
+        /** Currently allocated providers. */
+        Allocated,
+
+        /** Currently allocated providers that have a live connection. */
+        CurrentActive,
+
+        /** Currently allocated thread locals. */
+        ThreadLocals,
+
+        /** Providers discarded since application start. */
+        DiscardedThreadLocals,
+    }
 
     public STATUS status( )
     {
@@ -69,13 +116,19 @@ public class LdapConnectionService implements PwmService
     {
         this.pwmApplication = pwmApplication;
 
-        chaiProviderFactory = ChaiProviderFactory.newProviderFactory();
-
         useThreadLocal = Boolean.parseBoolean( pwmApplication.getConfig().readAppProperty( AppProperty.LDAP_PROXY_USE_THREAD_LOCAL ) );
+        LOGGER.trace( () -> "threadLocal enabled: " + useThreadLocal );
 
         // read the lastLoginTime
         this.lastLdapErrors.putAll( readLastLdapFailure( pwmApplication ) );
 
+        final long idleWeakTimeoutMS = JavaHelper.silentParseLong(
+                pwmApplication.getConfig().readAppProperty( AppProperty.LDAP_PROXY_IDLE_THREAD_LOCAL_TIMEOUT_MS ),
+                60_000 );
+        final TimeDuration idleWeakTimeout = TimeDuration.of( idleWeakTimeoutMS, TimeDuration.Unit.MILLISECONDS );
+        this.executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        this.pwmApplication.getPwmScheduler().scheduleFixedRateJob( new ThreadLocalCleaner(), executorService, idleWeakTimeout, idleWeakTimeout );
+
         final int connectionsPerProfile = maxSlotsPerProfile( pwmApplication );
         LOGGER.trace( () -> "allocating " + connectionsPerProfile + " ldap proxy connections per profile" );
         slotIncrementer = AtomicLoopIntIncrementer.builder().ceiling( connectionsPerProfile ).build();
@@ -91,19 +144,22 @@ public class LdapConnectionService implements PwmService
     public void close( )
     {
         status = STATUS.CLOSED;
+        logDebugInfo();
         LOGGER.trace( () -> "closing ldap proxy connections" );
-        if ( chaiProviderFactory != null )
+
+        try
         {
-            try
-            {
-                chaiProviderFactory.close();
-            }
-            catch ( final Exception e )
-            {
-                LOGGER.error( "error closing ldap proxy connection: " + e.getMessage(), e );
-            }
+            chaiProviderFactory.close();
+        }
+        catch ( final Exception e )
+        {
+            LOGGER.error( "error closing ldap proxy connection: " + e.getMessage(), e );
         }
+
         proxyChaiProviders.clear();
+
+        iterateThreadLocals( container -> container.getProviderMap().clear() );
+        executorService.shutdown();
     }
 
     public List<HealthRecord> healthCheck( )
@@ -133,61 +189,92 @@ public class LdapConnectionService implements PwmService
     public ChaiProvider getProxyChaiProvider( final LdapProfile ldapProfile )
             throws PwmUnrecoverableException
     {
+        if ( status != STATUS.OPEN )
+        {
+            throw new IllegalStateException( "unable to obtain proxy chai provider from closed LdapConnectionService" );
+        }
+
+        debugLogger.conditionallyExecuteTask();
+
         final LdapProfile effectiveProfile = ldapProfile == null
                 ? pwmApplication.getConfig().getDefaultLdapProfile()
                 : ldapProfile;
 
         if ( useThreadLocal )
         {
-            if ( threadLocalProvider.get() != null && threadLocalProvider.get().containsKey( effectiveProfile.getIdentifier() ) )
-            {
-                return threadLocalProvider.get().get( effectiveProfile.getIdentifier() );
-            }
+            return getThreadLocalChaiProvider( effectiveProfile );
         }
 
-        final ChaiProvider chaiProvider = getNewProxyChaiProvider( effectiveProfile );
+        return getSharedLocalChaiProvider( effectiveProfile );
+    }
 
-        if ( useThreadLocal )
+    private ChaiProvider getSharedLocalChaiProvider( final LdapProfile ldapProfile )
+            throws PwmUnrecoverableException
+    {
+        final int slot = slotIncrementer.next();
+        final ChaiProvider proxyChaiProvider = proxyChaiProviders.get( ldapProfile.getIdentifier() ).get( slot );
+
+        if ( proxyChaiProvider == null )
         {
-            if ( threadLocalProvider.get() == null )
-            {
-                threadLocalProvider.set( new ConcurrentHashMap<>() );
-            }
-            threadLocalProvider.get().put( effectiveProfile.getIdentifier(), chaiProvider );
+            final ChaiProvider newProvider = newProxyChaiProvider( ldapProfile );
+            proxyChaiProviders.get( ldapProfile.getIdentifier() ).put( slot, newProvider );
+            return newProvider;
         }
 
-        return chaiProvider;
+        return proxyChaiProvider;
     }
 
-    private ChaiProvider getNewProxyChaiProvider( final LdapProfile ldapProfile )
+    private ChaiProvider getThreadLocalChaiProvider( final LdapProfile ldapProfile )
             throws PwmUnrecoverableException
     {
-        if ( ldapProfile == null )
+        reentrantLock.lock();
+        try
         {
-            throw new NullPointerException( "ldapProfile must not be null" );
-        }
+            if ( threadLocalProvider.get() == null )
+            {
+                final ThreadLocalContainer threadLocalContainer = new ThreadLocalContainer();
+                threadLocalProvider.set( threadLocalContainer );
+                threadLocalContainers.add( threadLocalContainer );
+            }
 
-        final int slot = slotIncrementer.next();
+            final String profileID = ldapProfile.getIdentifier();
+            final ThreadLocalContainer threadLocalContainer = threadLocalProvider.get();
 
-        final ChaiProvider proxyChaiProvider = proxyChaiProviders.get( ldapProfile.getIdentifier() ).get( slot );
+            if ( !threadLocalContainer.getProviderMap().containsKey( profileID ) )
+            {
+                final ChaiProvider chaiProvider = newProxyChaiProvider( ldapProfile );
+                threadLocalContainer.getProviderMap().put( profileID, chaiProvider );
+            }
 
-        if ( proxyChaiProvider != null )
+            threadLocalContainer.setTimestamp( Instant.now() );
+            threadLocalContainer.setThreadName( Thread.currentThread().getName() );
+            return threadLocalContainer.getProviderMap().get( profileID );
+        }
+        finally
         {
-            return proxyChaiProvider;
+            reentrantLock.unlock();
         }
+    }
+
+    private ChaiProvider newProxyChaiProvider( final LdapProfile ldapProfile )
+            throws PwmUnrecoverableException
+    {
+        Objects.requireNonNull( ldapProfile, "ldapProfile must not be null" );
 
         try
         {
-            final ChaiProvider newProvider = LdapOperationsHelper.openProxyChaiProvider(
+            final ChaiProvider chaiProvider = LdapOperationsHelper.openProxyChaiProvider(
                     pwmApplication,
                     null,
                     ldapProfile,
                     pwmApplication.getConfig(),
                     pwmApplication.getStatisticsManager()
             );
-            proxyChaiProviders.get( ldapProfile.getIdentifier() ).put( slot, newProvider );
-
-            return newProvider;
+            LOGGER.trace( () -> "created new system proxy chaiProvider id=" + chaiProvider.toString()
+                    + " for ldap profile '" + ldapProfile.getIdentifier() + "'"
+                    + " thread=" + Thread.currentThread().getName() );
+            statCreatedProxies.next();
+            return chaiProvider;
         }
         catch ( final PwmUnrecoverableException e )
         {
@@ -268,14 +355,11 @@ public class LdapConnectionService implements PwmService
     public int connectionCount( )
     {
         int count = 0;
-        if ( chaiProviderFactory != null )
+        for ( final ChaiProvider chaiProvider : chaiProviderFactory.activeProviders() )
         {
-            for ( final ChaiProvider chaiProvider : chaiProviderFactory.activeProviders() )
+            if ( chaiProvider.isConnected() )
             {
-                if ( chaiProvider.isConnected() )
-                {
-                    count++;
-                }
+                count++;
             }
         }
         return count;
@@ -283,40 +367,123 @@ public class LdapConnectionService implements PwmService
 
     public ChaiProviderFactory getChaiProviderFactory( )
     {
+        if ( status != STATUS.OPEN )
+        {
+            throw new IllegalStateException( "unable to obtain chai provider factory from closed LdapConnectionService" );
+        }
+
         return chaiProviderFactory;
     }
 
-    private enum DebugKey
+    private void logDebugInfo()
     {
-        ALLOCATED_CONNECTIONS,
-        ACTIVE_CONNECTIONS,
-        IDLE_CONNECTIONS,
+        LOGGER.trace( () -> "status: " + StringUtil.mapToString( connectionDebugInfo() ) );
+    }
+
+    public List<ConnectionInfo> getConnectionInfos()
+    {
+        final Map<String, ConnectionInfo> returnData = new TreeMap<>(  );
+        for ( final ChaiProvider chaiProvider : chaiProviderFactory.activeProviders() )
+        {
+            final String bindDN = chaiProvider.getChaiConfiguration().getSetting( ChaiSetting.BIND_DN );
+            final ConnectionInfo connectionInfo = ConnectionInfo.builder()
+                    .bindDN( bindDN )
+                    .active( chaiProvider.isConnected() )
+                    .operationCount( chaiProvider.getProviderStatistics().getIncrementorStatistic( ProviderStatistics.IncrementerStatistic.OPERATION_COUNT ) )
+                    .modifyCount( chaiProvider.getProviderStatistics().getIncrementorStatistic( ProviderStatistics.IncrementerStatistic.MODIFY_COUNT ) )
+                    .readCount( chaiProvider.getProviderStatistics().getIncrementorStatistic( ProviderStatistics.IncrementerStatistic.READ_COUNT ) )
+                    .searchCount( chaiProvider.getProviderStatistics().getIncrementorStatistic( ProviderStatistics.IncrementerStatistic.SEARCH_COUNT ) )
+                    .build();
+
+            returnData.put( bindDN, connectionInfo );
+        }
+        return Collections.unmodifiableList( new ArrayList<>( returnData.values() ) );
+    }
+
+    @Value
+    @Builder
+    public static class ConnectionInfo implements Serializable
+    {
+        private final String bindDN;
+        private final boolean active;
+        private final long operationCount;
+        private final long modifyCount;
+        private final long readCount;
+        private final long searchCount;
     }
 
     private Map<String, String> connectionDebugInfo( )
     {
-        int allocatedConnections = 0;
-        int activeConnections = 0;
-        int idleConnections = 0;
-        if ( chaiProviderFactory != null )
+        final int allocatedConnections = chaiProviderFactory.activeProviders().size();
+        final int activeConnections = connectionCount();
+
+        final AtomicInteger threadLocalConnections = new AtomicInteger( 0 );
+        iterateThreadLocals( container -> threadLocalConnections.set( threadLocalConnections.intValue() + container.getProviderMap().size() ) );
+
+        final Map<DebugKey, String> debugInfo = new TreeMap<>();
+        debugInfo.put( DebugKey.Allocated, String.valueOf( allocatedConnections ) );
+        debugInfo.put( DebugKey.CurrentActive, String.valueOf( activeConnections ) );
+        debugInfo.put( DebugKey.ThreadLocals, String.valueOf( threadLocalConnections.get( ) ) );
+        debugInfo.put( DebugKey.CreatedProviders, String.valueOf( statCreatedProxies.get() ) );
+        debugInfo.put( DebugKey.DiscardedThreadLocals, String.valueOf( statClearedThreadLocals.get() ) );
+        return Collections.unmodifiableMap( JavaHelper.enumMapToStringMap( debugInfo ) );
+    }
+
+    @Data
+    private static class ThreadLocalContainer
+    {
+        private final Map<String, ChaiProvider> providerMap = new ConcurrentHashMap<>();
+        private volatile Instant timestamp = Instant.now();
+        private volatile String threadName;
+    }
+
+    private class ThreadLocalCleaner implements Runnable
+    {
+        @Override
+        public void run()
+        {
+            cleanupIssuedThreadLocals();
+            debugLogger.conditionallyExecuteTask();
+        }
+
+        private void cleanupIssuedThreadLocals()
         {
-            for ( final ChaiProvider chaiProvider : chaiProviderFactory.activeProviders() )
+            final TimeDuration maxIdleTime = TimeDuration.MINUTE;
+
+            iterateThreadLocals( container ->
             {
-                allocatedConnections++;
-                if ( chaiProvider.isConnected() )
+                if ( !container.getProviderMap().isEmpty() )
                 {
-                    activeConnections++;
-                }
-                else
-                {
-                    idleConnections++;
+                    final Instant timestamp = container.getTimestamp();
+                    final TimeDuration age = TimeDuration.fromCurrent( timestamp );
+                    if ( age.isLongerThan( maxIdleTime ) )
+                    {
+                        for ( final ChaiProvider chaiProvider : container.getProviderMap().values() )
+                        {
+                            LOGGER.trace( () -> "discarding idled connection id=" + chaiProvider.toString() + " from orphaned threadLocal, age="
+                                    + age.asCompactString() + ", thread=" + container.getThreadName() );
+                            statClearedThreadLocals.next();
+                        }
+                        container.getProviderMap().clear();
+                    }
                 }
+            } );
+        }
+    }
+
+    private void iterateThreadLocals( final Consumer<ThreadLocalContainer> consumer )
+    {
+        reentrantLock.lock();
+        try
+        {
+            for ( final ThreadLocalContainer container : new HashSet<>( threadLocalContainers ) )
+            {
+                consumer.accept( container );
             }
         }
-        final Map<String, String> debugInfo = new HashMap<>();
-        debugInfo.put( DebugKey.ALLOCATED_CONNECTIONS.name(), String.valueOf( allocatedConnections ) );
-        debugInfo.put( DebugKey.ACTIVE_CONNECTIONS.name(), String.valueOf( activeConnections ) );
-        debugInfo.put( DebugKey.IDLE_CONNECTIONS.name(), String.valueOf( idleConnections ) );
-        return Collections.unmodifiableMap( debugInfo );
+        finally
+        {
+            reentrantLock.unlock();
+        }
     }
 }

server/src/main/java/password/pwm/ldap/LdapOperationsHelper.java

@@ -105,7 +105,7 @@ public class LdapOperationsHelper
         {
             final ChaiProvider chaiProvider = pwmApplication.getProxyChaiProvider( userIdentity.getLdapProfileID() );
             final ChaiUser theUser = chaiProvider.getEntryFactory().newChaiUser( userIdentity.getUserDN() );
-            addUserObjectClass( sessionLabel, theUser, newObjClasses );
+            addUserObjectClass( sessionLabel, userIdentity, theUser, newObjClasses );
         }
         catch ( final ChaiUnavailableException e )
         {
@@ -115,6 +115,7 @@ public class LdapOperationsHelper
 
     private static void addUserObjectClass(
             final SessionLabel sessionLabel,
+            final UserIdentity userIdentity,
             final ChaiUser theUser,
             final Set<String> newObjClasses
     )
@@ -131,19 +132,18 @@ public class LdapOperationsHelper
                 auxClass = newObjClass;
                 theUser.addAttribute( ChaiConstant.ATTR_LDAP_OBJECTCLASS, auxClass );
                 final String finalAuxClass = auxClass;
-                LOGGER.info( sessionLabel, () -> "added objectclass '" + finalAuxClass + "' to user " + theUser.getEntryDN() );
+                LOGGER.info( sessionLabel, () -> "added objectclass '" + finalAuxClass + "' to user " + userIdentity.toDisplayString() );
             }
         }
         catch ( final ChaiOperationException e )
         {
-            final StringBuilder errorMsg = new StringBuilder();
-
-            errorMsg.append( "error adding objectclass '" ).append( auxClass ).append( "' to user " );
-            errorMsg.append( theUser.getEntryDN() );
-            errorMsg.append( ": " );
-            errorMsg.append( e.toString() );
+            final String finalAuxClass = auxClass;
+            LOGGER.error( sessionLabel, () -> "error adding objectclass '" + finalAuxClass + "' to user, error "
+                    + userIdentity.toDisplayString()
+                    + ": "
+                    + e.getMessage()
+            );
 
-            LOGGER.error( sessionLabel, errorMsg.toString() );
         }
     }
 

server/src/main/java/password/pwm/ldap/search/UserSearchEngine.java

@@ -24,9 +24,6 @@ import com.novell.ldapchai.ChaiUser;
 import com.novell.ldapchai.exception.ChaiOperationException;
 import com.novell.ldapchai.exception.ChaiUnavailableException;
 import com.novell.ldapchai.provider.ChaiProvider;
-import com.novell.ldapchai.util.SearchHelper;
-import lombok.AllArgsConstructor;
-import lombok.Getter;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.PwmConstants;
@@ -44,8 +41,8 @@ import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.PwmService;
-import password.pwm.svc.stats.AvgStatistic;
 import password.pwm.util.PwmScheduler;
+import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.ConditionalTaskExecutor;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
@@ -74,7 +71,6 @@ import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class UserSearchEngine implements PwmService
@@ -330,9 +326,12 @@ public class UserSearchEngine implements PwmService
 
         final List<String> errors = new ArrayList<>();
 
+        final int searchID = searchCounter.getAndIncrement();
         final long profileRetryDelayMS = Long.parseLong( pwmApplication.getConfig().readAppProperty( AppProperty.LDAP_PROFILE_RETRY_DELAY ) );
+        final AtomicLoopIntIncrementer jobIncrementer = AtomicLoopIntIncrementer.builder().build();
 
         final List<UserSearchJob> searchJobs = new ArrayList<>();
+
         for ( final LdapProfile ldapProfile : ldapProfiles )
         {
             boolean skipProfile = false;
@@ -352,7 +351,10 @@ public class UserSearchEngine implements PwmService
                             ldapProfile,
                             searchConfiguration,
                             maxResults,
-                            returnAttributes
+                            returnAttributes,
+                            sessionLabel,
+                            searchID,
+                            jobIncrementer
                     ) );
                 }
                 catch ( final PwmUnrecoverableException e )
@@ -378,7 +380,7 @@ public class UserSearchEngine implements PwmService
             }
         }
 
-        final Map<UserIdentity, Map<String, String>> resultsMap = new LinkedHashMap<>( executeSearchJobs( searchJobs, sessionLabel, searchCounter.getAndIncrement() ) );
+        final Map<UserIdentity, Map<String, String>> resultsMap = new LinkedHashMap<>( executeSearchJobs( searchJobs ) );
         final Map<UserIdentity, Map<String, String>> returnMap = trimOrderedMap( resultsMap, maxResults );
         return Collections.unmodifiableMap( returnMap );
     }
@@ -388,7 +390,10 @@ public class UserSearchEngine implements PwmService
             final LdapProfile ldapProfile,
             final SearchConfiguration searchConfiguration,
             final int maxResults,
-            final Collection<String> returnAttributes
+            final Collection<String> returnAttributes,
+            final SessionLabel sessionLabel,
+            final int searchID,
+            final AtomicLoopIntIncrementer jobIncrementer
     )
             throws PwmUnrecoverableException, PwmOperationalException
     {
@@ -399,46 +404,7 @@ public class UserSearchEngine implements PwmService
                 ? searchConfiguration.getFilter()
                 : ldapProfile.readSettingAsString( PwmSetting.LDAP_USERNAME_SEARCH_FILTER );
 
-        final String searchFilter;
-        if ( searchConfiguration.getUsername() != null )
-        {
-            final String inputQuery = searchConfiguration.isEnableValueEscaping()
-                    ? StringUtil.escapeLdapFilter( searchConfiguration.getUsername() )
-                    : searchConfiguration.getUsername();
-
-            if ( searchConfiguration.isEnableSplitWhitespace()
-                    && ( searchConfiguration.getUsername().split( "\\s" ).length > 1 ) )
-            {
-                // split on all whitespace chars
-                final StringBuilder multiSearchFilter = new StringBuilder();
-                multiSearchFilter.append( "(&" );
-                for ( final String queryPart : searchConfiguration.getUsername().split( " " ) )
-                {
-                    multiSearchFilter.append( "(" );
-                    multiSearchFilter.append( inputSearchFilter.replace( PwmConstants.VALUE_REPLACEMENT_USERNAME, queryPart ) );
-                    multiSearchFilter.append( ")" );
-                }
-                multiSearchFilter.append( ")" );
-                searchFilter = multiSearchFilter.toString();
-            }
-            else
-            {
-                searchFilter = inputSearchFilter.replace( PwmConstants.VALUE_REPLACEMENT_USERNAME, inputQuery.trim() );
-            }
-        }
-        else if ( searchConfiguration.getGroupDN() != null )
-        {
-            final String groupAttr = ldapProfile.readSettingAsString( PwmSetting.LDAP_USER_GROUP_ATTRIBUTE );
-            searchFilter = "(" + groupAttr + "=" + searchConfiguration.getGroupDN() + ")";
-        }
-        else if ( searchConfiguration.getFormValues() != null )
-        {
-            searchFilter = figureSearchFilterForParams( searchConfiguration.getFormValues(), inputSearchFilter, searchConfiguration.isEnableValueEscaping() );
-        }
-        else
-        {
-            searchFilter = inputSearchFilter;
-        }
+        final String searchFilter = makeSearchFilter( ldapProfile, searchConfiguration, inputSearchFilter );
 
         final List<String> searchContexts;
         if ( searchConfiguration.getContexts() != null
@@ -462,11 +428,10 @@ public class UserSearchEngine implements PwmService
             searchContexts = ldapProfile.getRootContexts( pwmApplication );
         }
 
-        final long timeLimitMS = searchConfiguration.getSearchTimeout() != 0
+        final long timeLimitMS = searchConfiguration.getSearchTimeout() > 0
                 ? searchConfiguration.getSearchTimeout()
                 : ( ldapProfile.readSettingAsLong( PwmSetting.LDAP_SEARCH_TIMEOUT ) * 1000 );
 
-
         final ChaiProvider chaiProvider = searchConfiguration.getChaiProvider() == null
                 ? pwmApplication.getProxyChaiProvider( ldapProfile.getIdentifier() )
                 : searchConfiguration.getChaiProvider();
@@ -474,7 +439,7 @@ public class UserSearchEngine implements PwmService
         final List<UserSearchJob> returnMap = new ArrayList<>();
         for ( final String loopContext : searchContexts )
         {
-            final UserSearchJob userSearchJob = UserSearchJob.builder()
+            final UserSearchJobParameters userSearchJobParameters = UserSearchJobParameters.builder()
                     .ldapProfile( ldapProfile )
                     .searchFilter( searchFilter )
                     .context( loopContext )
@@ -482,80 +447,63 @@ public class UserSearchEngine implements PwmService
                     .maxResults( maxResults )
                     .chaiProvider( chaiProvider )
                     .timeoutMs( timeLimitMS )
+                    .sessionLabel( sessionLabel )
+                    .searchID( searchID )
+                    .jobId( jobIncrementer.next() )
                     .build();
+            final UserSearchJob userSearchJob = new UserSearchJob( pwmApplication, this, userSearchJobParameters );
             returnMap.add( userSearchJob );
         }
 
         return returnMap;
     }
 
-    private Map<UserIdentity, Map<String, String>> executeSearch(
-            final UserSearchJob userSearchJob,
-            final SessionLabel sessionLabel,
-            final int searchID,
-            final int jobID
-    )
-            throws PwmOperationalException, PwmUnrecoverableException
+    private String makeSearchFilter( final LdapProfile ldapProfile, final SearchConfiguration searchConfiguration, final String inputSearchFilter )
     {
-        debugOutputTask.conditionallyExecuteTask();
-
-        final SearchHelper searchHelper = new SearchHelper();
-        searchHelper.setMaxResults( userSearchJob.getMaxResults() );
-        searchHelper.setFilter( userSearchJob.getSearchFilter() );
-        searchHelper.setAttributes( userSearchJob.getReturnAttributes() );
-        searchHelper.setTimeLimit( ( int ) userSearchJob.getTimeoutMs() );
-
-        final String debugInfo;
+        final String searchFilter;
+        if ( searchConfiguration.getUsername() != null )
         {
-            final Map<String, String> props = new LinkedHashMap<>();
-            props.put( "profile", userSearchJob.getLdapProfile().getIdentifier() );
-            props.put( "base", userSearchJob.getContext() );
-            props.put( "maxCount", String.valueOf( searchHelper.getMaxResults() ) );
-            debugInfo = "[" + StringUtil.mapToString( props ) + "]";
-        }
-        log( PwmLogLevel.TRACE, sessionLabel, searchID, jobID, "performing ldap search for user; " + debugInfo );
+            final String inputQuery = searchConfiguration.isEnableValueEscaping()
+                    ? StringUtil.escapeLdapFilter( searchConfiguration.getUsername() )
+                    : searchConfiguration.getUsername();
 
-        final Instant startTime = Instant.now();
-        final Map<String, Map<String, String>> results;
-        try
-        {
-            results = userSearchJob.getChaiProvider().search( userSearchJob.getContext(), searchHelper );
-        }
-        catch ( final ChaiUnavailableException e )
-        {
-            throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_DIRECTORY_UNAVAILABLE, e.getMessage() ) );
-        }
-        catch ( final ChaiOperationException e )
-        {
-            throw new PwmOperationalException( PwmError.forChaiError( e.getErrorCode() ), "ldap error during searchID="
-                    + searchID + ", error=" + e.getMessage() );
+            if ( searchConfiguration.isEnableSplitWhitespace()
+                    && ( searchConfiguration.getUsername().split( "\\s" ).length > 1 ) )
+            {
+                // split on all whitespace chars
+                final StringBuilder multiSearchFilter = new StringBuilder();
+                multiSearchFilter.append( "(&" );
+                for ( final String queryPart : searchConfiguration.getUsername().split( " " ) )
+                {
+                    multiSearchFilter.append( "(" );
+                    multiSearchFilter.append( inputSearchFilter.replace( PwmConstants.VALUE_REPLACEMENT_USERNAME, queryPart ) );
+                    multiSearchFilter.append( ")" );
+                }
+                multiSearchFilter.append( ")" );
+                searchFilter = multiSearchFilter.toString();
+            }
+            else
+            {
+                searchFilter = inputSearchFilter.replace( PwmConstants.VALUE_REPLACEMENT_USERNAME, inputQuery.trim() );
+            }
         }
-        final TimeDuration searchDuration = TimeDuration.fromCurrent( startTime );
-
-        if ( pwmApplication.getStatisticsManager() != null && pwmApplication.getStatisticsManager().status() == PwmService.STATUS.OPEN )
+        else if ( searchConfiguration.getGroupDN() != null )
         {
-            pwmApplication.getStatisticsManager().updateAverageValue( AvgStatistic.AVG_LDAP_SEARCH_TIME, searchDuration.asMillis() );
+            final String groupAttr = ldapProfile.readSettingAsString( PwmSetting.LDAP_USER_GROUP_ATTRIBUTE );
+            searchFilter = "(" + groupAttr + "=" + searchConfiguration.getGroupDN() + ")";
         }
-
-        if ( results.isEmpty() )
+        else if ( searchConfiguration.getFormValues() != null )
         {
-            log( PwmLogLevel.TRACE, sessionLabel, searchID, jobID, "no matches from search (" + searchDuration.asCompactString() + "); " + debugInfo );
-            return Collections.emptyMap();
+            searchFilter = figureSearchFilterForParams( searchConfiguration.getFormValues(), inputSearchFilter, searchConfiguration.isEnableValueEscaping() );
         }
-
-        log( PwmLogLevel.TRACE, sessionLabel, searchID, jobID, "found " + results.size() + " results in " + searchDuration.asCompactString() + "; " + debugInfo );
-
-        final Map<UserIdentity, Map<String, String>> returnMap = new LinkedHashMap<>();
-        for ( final Map.Entry<String, Map<String, String>> entry : results.entrySet() )
+        else
         {
-            final String userDN = entry.getKey();
-            final Map<String, String> attributeMap = entry.getValue();
-            final UserIdentity userIdentity = new UserIdentity( userDN, userSearchJob.getLdapProfile().getIdentifier() );
-            returnMap.put( userIdentity, attributeMap );
+            searchFilter = inputSearchFilter;
         }
-        return returnMap;
+        return searchFilter;
     }
 
+
     private void validateSpecifiedContext( final LdapProfile profile, final String context )
             throws PwmOperationalException, PwmUnrecoverableException
     {
@@ -599,7 +547,7 @@ public class UserSearchEngine implements PwmService
             final SessionLabel sessionLabel
     )
     {
-        if ( input == null || input.length() < 1 )
+        if ( StringUtil.isEmpty( input ) )
         {
             return false;
         }
@@ -653,39 +601,32 @@ public class UserSearchEngine implements PwmService
     }
 
     private Map<UserIdentity, Map<String, String>> executeSearchJobs(
-            final Collection<UserSearchJob> userSearchJobs,
-            final SessionLabel sessionLabel,
-            final int searchID
+            final Collection<UserSearchJob> userSearchJobs
     )
             throws PwmUnrecoverableException
     {
-        // create jobs
-        final List<JobInfo> jobs = new ArrayList<>();
+        if ( JavaHelper.isEmpty( userSearchJobs ) )
         {
-            int jobID = 0;
-            for ( final UserSearchJob userSearchJob : userSearchJobs )
-            {
-                final int loopJobID = jobID++;
-
-                final FutureTask<Map<UserIdentity, Map<String, String>>> futureTask = new FutureTask<>( ( )
-                        -> executeSearch( userSearchJob, sessionLabel, searchID, loopJobID ) );
+            return Collections.emptyMap();
+        }
 
-                final JobInfo jobInfo = new JobInfo( searchID, loopJobID, userSearchJob, futureTask );
+        debugOutputTask.conditionallyExecuteTask();
 
-                jobs.add( jobInfo );
-            }
-        }
+        final UserSearchJobParameters firstParam = userSearchJobs.iterator().next().getUserSearchJobParameters();
 
         final Instant startTime = Instant.now();
         {
-            final String filterText = jobs.isEmpty() ? "" : ", filter: " + jobs.iterator().next().getUserSearchJob().getSearchFilter();
-            log( PwmLogLevel.DEBUG, sessionLabel, searchID, -1, "beginning user search process with " + jobs.size() + " search jobs" + filterText );
+            final String filterText = ", filter: " + firstParam.getSearchFilter();
+            final SessionLabel sessionLabel = firstParam.getSessionLabel();
+            final int searchID = firstParam.getSearchID();
+            log( PwmLogLevel.DEBUG, sessionLabel, searchID, -1, "beginning user search process with " + userSearchJobs.size() + " search jobs" + filterText );
         }
 
         // execute jobs
-        for ( Iterator<JobInfo> iterator = jobs.iterator(); iterator.hasNext(); )
+        for ( final Iterator<UserSearchJob> iterator = userSearchJobs.iterator(); iterator.hasNext(); )
         {
-            final JobInfo jobInfo = iterator.next();
+
+            final UserSearchJob jobInfo = iterator.next();
 
             boolean submittedToExecutor = false;
 
@@ -714,18 +655,35 @@ public class UserSearchEngine implements PwmService
                 }
                 catch ( final Throwable t )
                 {
-                    log( PwmLogLevel.ERROR, sessionLabel, searchID, jobInfo.getJobID(), "unexpected error running job in local thread: " + t.getMessage() );
+                    log( PwmLogLevel.ERROR, firstParam.getSessionLabel(), firstParam.getSearchID(), firstParam.getJobId(),
+                            "unexpected error running job in local thread: " + t.getMessage() );
                 }
             }
         }
 
-        // aggregate results
+        final Map<UserIdentity, Map<String, String>> results = aggregateJobResults( userSearchJobs );
+
+        log( PwmLogLevel.DEBUG, firstParam.getSessionLabel(), firstParam.getSearchID(), -1, "completed user search process in "
+                + TimeDuration.fromCurrent( startTime ).asCompactString()
+                + ", intermediate result size=" + results.size() );
+
+        return Collections.unmodifiableMap( results );
+    }
+
+    private Map<UserIdentity, Map<String, String>> aggregateJobResults(
+          final Collection<UserSearchJob> userSearchJobs
+    )
+            throws PwmUnrecoverableException
+    {
         final Map<UserIdentity, Map<String, String>> results = new LinkedHashMap<>();
-        for ( final JobInfo jobInfo : jobs )
+
+        for ( final UserSearchJob jobInfo : userSearchJobs )
         {
-            if ( results.size() > jobInfo.getUserSearchJob().getMaxResults() )
+            final UserSearchJobParameters params = jobInfo.getUserSearchJobParameters();
+            final Instant startTime = Instant.now();
+            if ( results.size() > jobInfo.getUserSearchJobParameters().getMaxResults() )
             {
-                final FutureTask futureTask = jobInfo.getFutureTask();
+                final FutureTask<Map<UserIdentity, Map<String, String>>> futureTask = jobInfo.getFutureTask();
                 if ( !futureTask.isDone() )
                 {
                     canceledJobCounter.incrementAndGet();
@@ -734,16 +692,15 @@ public class UserSearchEngine implements PwmService
             }
             else
             {
-                final long maxWaitTime = jobInfo.getUserSearchJob().getTimeoutMs() * 3;
                 try
                 {
-                    results.putAll( jobInfo.getFutureTask().get( maxWaitTime, TimeUnit.MILLISECONDS ) );
+                    results.putAll( jobInfo.getFutureTask().get( ) );
                 }
                 catch ( final InterruptedException e )
                 {
                     final String errorMsg = "unexpected interruption during search job execution: " + e.getMessage();
-                    log( PwmLogLevel.WARN, sessionLabel, searchID, jobInfo.getJobID(), errorMsg );
-                    LOGGER.error( sessionLabel, errorMsg, e );
+                    log( PwmLogLevel.WARN, params.getSessionLabel(), params.getSearchID(), params.getJobId(), errorMsg );
+                    LOGGER.error( params.getSessionLabel(), errorMsg, e );
                     throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_INTERNAL, errorMsg ) );
                 }
                 catch ( final ExecutionException e )
@@ -751,7 +708,7 @@ public class UserSearchEngine implements PwmService
                     final Throwable t = e.getCause();
                     final ErrorInformation errorInformation;
                     final String errorMsg = "unexpected error during ldap search ("
-                            + "profile=" + jobInfo.getUserSearchJob().getLdapProfile().getIdentifier() + ")"
+                            + "profile=" + jobInfo.getUserSearchJobParameters().getLdapProfile().getIdentifier() + ")"
                             + ", error: " + ( t instanceof PwmException ? t.getMessage() : JavaHelper.readHostileExceptionMessage( t ) );
                     if ( t instanceof PwmException )
                     {
@@ -759,36 +716,14 @@ public class UserSearchEngine implements PwmService
                     }
                     else
                     {
-                        errorInformation = new ErrorInformation( PwmError.ERROR_INTERNAL, errorMsg );
+                        errorInformation = new ErrorInformation( PwmError.ERROR_LDAP_DATA_ERROR, errorMsg );
                     }
-                    log( PwmLogLevel.WARN, sessionLabel, searchID, jobInfo.getJobID(), "error during user search: " + errorInformation.toDebugStr() );
+                    log( PwmLogLevel.WARN, params.getSessionLabel(), params.getSearchID(), params.getJobId(), "error during user search: " + errorInformation.toDebugStr() );
                     throw new PwmUnrecoverableException( errorInformation );
                 }
-                catch ( final TimeoutException e )
-                {
-                    final String errorMsg = "background search job timeout after " + jobInfo.getUserSearchJob().getTimeoutMs()
-                            + "ms, to ldapProfile '"
-                            + jobInfo.getUserSearchJob().getLdapProfile() + "'";
-                    log( PwmLogLevel.WARN, sessionLabel, searchID, jobInfo.getJobID(), "error during user search: " + errorMsg );
-                    jobTimeoutCounter.incrementAndGet();
-                }
             }
         }
-
-        log( PwmLogLevel.DEBUG, sessionLabel, searchID, -1, "completed user search process in "
-                + TimeDuration.fromCurrent( startTime ).asCompactString()
-                + ", intermediate result size=" + results.size() );
-        return Collections.unmodifiableMap( results );
-    }
-
-    @Getter
-    @AllArgsConstructor
-    private static class JobInfo
-    {
-        private final int searchID;
-        private final int jobID;
-        private final UserSearchJob userSearchJob;
-        private final FutureTask<Map<UserIdentity, Map<String, String>>> futureTask;
+        return results;
     }
 
     private Map<String, String> debugProperties( )
@@ -822,7 +757,7 @@ public class UserSearchEngine implements PwmService
         LOGGER.debug( () -> "periodic debug status: " + StringUtil.mapToString( debugProperties() ) );
     }
 
-    private void log( final PwmLogLevel level, final SessionLabel sessionLabel, final int searchID, final int jobID, final String message )
+    void log( final PwmLogLevel level, final SessionLabel sessionLabel, final int searchID, final int jobID, final String message )
     {
         final String idMsg = logIdString( searchID, jobID );
         LOGGER.log( level, sessionLabel, idMsg + " " + message );
@@ -865,8 +800,11 @@ public class UserSearchEngine implements PwmService
             final int maxThreads = Integer.parseInt( configuration.readAppProperty( AppProperty.LDAP_SEARCH_PARALLEL_THREAD_MAX ) );
             final int threads = Math.min( maxThreads, ( endPoints ) * factor );
             final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( pwmApplication, UserSearchEngine.class ), true );
+
+            LOGGER.trace( () -> "initialized with " + threads + " max threads" );
+
             return new ThreadPoolExecutor(
-                    threads,
+                    1,
                     threads,
                     1,
                     TimeUnit.MINUTES,

server/src/main/java/password/pwm/ldap/search/UserSearchJob.java

@@ -20,22 +20,122 @@
 
 package password.pwm.ldap.search;
 
-import com.novell.ldapchai.provider.ChaiProvider;
-import lombok.Builder;
-import lombok.Value;
-import password.pwm.config.profile.LdapProfile;
+import com.novell.ldapchai.exception.ChaiOperationException;
+import com.novell.ldapchai.exception.ChaiUnavailableException;
+import com.novell.ldapchai.util.SearchHelper;
+import password.pwm.PwmApplication;
+import password.pwm.bean.UserIdentity;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmOperationalException;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.svc.PwmService;
+import password.pwm.svc.stats.AvgStatistic;
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogLevel;
 
-import java.util.Collection;
+import java.time.Instant;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.FutureTask;
 
-@Value
-@Builder
-public class UserSearchJob
+class UserSearchJob implements Callable<Map<UserIdentity, Map<String, String>>>
 {
-    private final LdapProfile ldapProfile;
-    private final String searchFilter;
-    private final String context;
-    private final Collection<String> returnAttributes;
-    private final int maxResults;
-    private final ChaiProvider chaiProvider;
-    private final long timeoutMs;
+    private final PwmApplication pwmApplication;
+    private final UserSearchJobParameters userSearchJobParameters;
+    private final UserSearchEngine userSearchEngine;
+    private final FutureTask<Map<UserIdentity, Map<String, String>>> futureTask;
+    private final Instant createTime = Instant.now();
+
+    UserSearchJob( final PwmApplication pwmApplication, final UserSearchEngine userSearchEngine, final UserSearchJobParameters userSearchJobParameters )
+    {
+        this.pwmApplication = pwmApplication;
+        this.userSearchJobParameters = userSearchJobParameters;
+        this.userSearchEngine = userSearchEngine;
+        this.futureTask = new FutureTask<>( this );
+    }
+
+    @Override
+    public Map<UserIdentity, Map<String, String>> call()
+            throws PwmOperationalException, PwmUnrecoverableException
+    {
+        final TimeDuration queueLagDuration = TimeDuration.fromCurrent( createTime );
+
+        final SearchHelper searchHelper = new SearchHelper();
+        searchHelper.setMaxResults( userSearchJobParameters.getMaxResults() );
+        searchHelper.setFilter( userSearchJobParameters.getSearchFilter() );
+        searchHelper.setAttributes( userSearchJobParameters.getReturnAttributes() );
+        searchHelper.setTimeLimit( ( int ) userSearchJobParameters.getTimeoutMs() );
+
+        final String debugInfo;
+        {
+            final Map<String, String> props = new LinkedHashMap<>();
+            props.put( "profile", userSearchJobParameters.getLdapProfile().getIdentifier() );
+            props.put( "base", userSearchJobParameters.getContext() );
+            props.put( "maxCount", String.valueOf( searchHelper.getMaxResults() ) );
+            props.put( "queueLag", queueLagDuration.asCompactString() );
+            debugInfo = "[" + StringUtil.mapToString( props ) + "]";
+        }
+
+        userSearchEngine.log( PwmLogLevel.TRACE, userSearchJobParameters.getSessionLabel(), userSearchJobParameters.getSearchID(), userSearchJobParameters.getJobId(),
+                "performing ldap search for user, thread=" + Thread.currentThread().getId()
+                        + ", timeout=" + userSearchJobParameters.getTimeoutMs() + "ms, "
+                        + debugInfo );
+
+        final Instant startTime = Instant.now();
+        final Map<String, Map<String, String>> results;
+        try
+        {
+            results = userSearchJobParameters.getChaiProvider().search( userSearchJobParameters.getContext(), searchHelper );
+        }
+        catch ( final ChaiUnavailableException e )
+        {
+            throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_DIRECTORY_UNAVAILABLE, e.getMessage() ) );
+        }
+        catch ( final ChaiOperationException e )
+        {
+            throw new PwmOperationalException( PwmError.forChaiError( e.getErrorCode() ), "ldap error during searchID="
+                    + userSearchJobParameters.getSearchID() + ", context=" + userSearchJobParameters.getContext() + ", error=" + e.getMessage() );
+        }
+
+        final TimeDuration searchDuration = TimeDuration.fromCurrent( startTime );
+
+        if ( pwmApplication.getStatisticsManager() != null && pwmApplication.getStatisticsManager().status() == PwmService.STATUS.OPEN )
+        {
+            pwmApplication.getStatisticsManager().updateAverageValue( AvgStatistic.AVG_LDAP_SEARCH_TIME, searchDuration.asMillis() );
+        }
+
+        if ( results.isEmpty() )
+        {
+            userSearchEngine.log( PwmLogLevel.TRACE, userSearchJobParameters.getSessionLabel(), userSearchJobParameters.getSearchID(), userSearchJobParameters.getJobId(),
+                    "no matches from search (" + searchDuration.asCompactString() + "); " + debugInfo );
+            return Collections.emptyMap();
+        }
+
+        userSearchEngine.log( PwmLogLevel.TRACE, userSearchJobParameters.getSessionLabel(), userSearchJobParameters.getSearchID(), userSearchJobParameters.getJobId(),
+                "found " + results.size() + " results in " + searchDuration.asCompactString() + "; " + debugInfo );
+
+        final Map<UserIdentity, Map<String, String>> returnMap = new LinkedHashMap<>();
+        for ( final Map.Entry<String, Map<String, String>> entry : results.entrySet() )
+        {
+            final String userDN = entry.getKey();
+            final Map<String, String> attributeMap = entry.getValue();
+            final UserIdentity userIdentity = new UserIdentity( userDN, userSearchJobParameters.getLdapProfile().getIdentifier() );
+            returnMap.put( userIdentity, attributeMap );
+        }
+        return returnMap;
+    }
+
+    public UserSearchJobParameters getUserSearchJobParameters()
+    {
+        return userSearchJobParameters;
+    }
+
+    public FutureTask<Map<UserIdentity, Map<String, String>>> getFutureTask()
+    {
+        return futureTask;
+    }
 }

server/src/main/java/password/pwm/ldap/search/UserSearchJobParameters.java

@@ -0,0 +1,45 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2019 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.ldap.search;
+
+import com.novell.ldapchai.provider.ChaiProvider;
+import lombok.Builder;
+import lombok.Value;
+import password.pwm.bean.SessionLabel;
+import password.pwm.config.profile.LdapProfile;
+
+import java.util.Collection;
+
+@Value
+@Builder
+public class UserSearchJobParameters
+{
+    private final LdapProfile ldapProfile;
+    private final String searchFilter;
+    private final String context;
+    private final Collection<String> returnAttributes;
+    private final int maxResults;
+    private final ChaiProvider chaiProvider;
+    private final long timeoutMs;
+    private final SessionLabel sessionLabel;
+    private final int searchID;
+    private final int jobId;
+}

server/src/main/java/password/pwm/svc/event/AuditRecordFactory.java

@@ -60,13 +60,13 @@ public class AuditRecordFactory
     public AuditRecordFactory( final PwmApplication pwmApplication, final PwmRequest pwmRequest ) throws PwmUnrecoverableException
     {
         this.pwmApplication = pwmApplication;
-        this.macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmApplication );
+        this.macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( );
     }
 
     public AuditRecordFactory( final PwmRequest pwmRequest ) throws PwmUnrecoverableException
     {
         this.pwmApplication = pwmRequest.getPwmApplication();
-        this.macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmApplication );
+        this.macroMachine = pwmRequest.getPwmSession().getSessionManager().getMacroMachine( );
     }
 
     public HelpdeskAuditRecord createHelpdeskAuditRecord(

server/src/main/java/password/pwm/svc/event/AuditService.java

@@ -336,7 +336,7 @@ public class AuditService implements PwmService
     public void submit( final AuditEvent auditEvent, final UserInfo userInfo, final PwmSession pwmSession )
             throws PwmUnrecoverableException
     {
-        final AuditRecordFactory auditRecordFactory = new AuditRecordFactory( pwmApplication, pwmSession.getSessionManager().getMacroMachine( pwmApplication ) );
+        final AuditRecordFactory auditRecordFactory = new AuditRecordFactory( pwmApplication, pwmSession.getSessionManager().getMacroMachine( ) );
         final UserAuditRecord auditRecord = auditRecordFactory.createUserAuditRecord( auditEvent, userInfo, pwmSession );
         submit( auditRecord );
     }

server/src/main/java/password/pwm/svc/httpclient/PwmHttpClient.java

@@ -173,6 +173,7 @@ public class PwmHttpClient implements AutoCloseable
             clientBuilder.setSSLContext( sslContext );
             clientBuilder.setSSLSocketFactory( sslConnectionFactory );
             clientBuilder.setConnectionManager( ccm );
+            clientBuilder.setConnectionManagerShared( true );
         }
         catch ( final Exception e )
         {

server/src/main/java/password/pwm/svc/report/ReportService.java

@@ -528,14 +528,26 @@ public class ReportService implements PwmService
                             {
                                 TimeDuration.of( avgTracker.avgAsLong(), TimeDuration.Unit.MILLISECONDS ).pause();
                             }
+                        }
+                        catch ( final PwmUnrecoverableException e )
+                        {
+
                         }
                         catch ( final Exception e )
                         {
                             String errorMsg = "error while updating report cache for " + userIdentity.toString() + ", cause: ";
-                            errorMsg += e instanceof PwmException ? ( ( PwmException ) e ).getErrorInformation().toDebugStr() : e.getMessage();
-                            final ErrorInformation errorInformation;
-                            errorInformation = new ErrorInformation( PwmError.ERROR_REPORTING_ERROR, errorMsg );
-                            LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, errorInformation.toDebugStr(), e );
+                            errorMsg += e instanceof PwmException
+                                    ? ( ( PwmException ) e ).getErrorInformation().toDebugStr()
+                                    : e.getMessage();
+                            final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_REPORTING_ERROR, errorMsg );
+                            if ( e instanceof PwmException )
+                            {
+                                LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, errorInformation.toDebugStr() );
+                            }
+                            else
+                            {
+                                LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, errorInformation.toDebugStr(), e );
+                            }
                             reportStatus.updateAndGet( reportStatusInfo -> reportStatusInfo.toBuilder()
                                     .lastError( errorInformation )
                                     .errors( reportStatusInfo.getErrors() + 1 )

server/src/main/java/password/pwm/util/PasswordData.java

@@ -73,7 +73,6 @@ public class PasswordData implements Serializable
         catch ( final Exception e )
         {
             LOGGER.fatal( "can't initialize PasswordData handler: " + e.getMessage(), e );
-            e.printStackTrace();
             if ( e instanceof PwmException )
             {
                 newInitializationError = ( ( PwmException ) e ).getErrorInformation();

server/src/main/java/password/pwm/util/java/AtomicLoopIntIncrementer.java

@@ -51,6 +51,11 @@ public class AtomicLoopIntIncrementer
         this.incrementer = new AtomicInteger( initial );
     }
 
+    public int get()
+    {
+        return incrementer.get();
+    }
+
     public int next( )
     {
         return incrementer.getAndUpdate( operand ->

server/src/main/java/password/pwm/util/java/JavaHelper.java

@@ -163,6 +163,12 @@ public class JavaHelper
         return Collections.unmodifiableList( returnList );
     }
 
+    public static <E extends Enum<E>> Map<String, String> enumMapToStringMap( final Map<E, String> inputMap )
+    {
+        return Collections.unmodifiableMap( inputMap.entrySet().stream()
+                .collect( Collectors.toMap( entry -> entry.getKey().name(), Map.Entry::getValue, ( a, b ) -> b, LinkedHashMap::new ) ) );
+    }
+
     public static <E extends Enum<E>> E readEnumFromString( final Class<E> enumClass, final E defaultValue, final String input )
     {
         return readEnumFromString( enumClass, input ).orElse( defaultValue );

server/src/main/java/password/pwm/util/logging/PwmLogger.java

@@ -365,6 +365,11 @@ public class PwmLogger
         doLogEvent( PwmLogLevel.ERROR, sessionLabel, message, null );
     }
 
+    public void error( final SessionLabel sessionLabel, final Supplier<CharSequence> message )
+    {
+        doLogEvent( PwmLogLevel.ERROR, sessionLabel, message, null );
+    }
+
     public void error( final SessionLabel sessionLabel, final ErrorInformation errorInformation )
     {
         doLogEvent( PwmLogLevel.ERROR, sessionLabel, convertErrorInformation( errorInformation ), null );

server/src/main/java/password/pwm/util/operations/otp/LdapOtpOperator.java

@@ -130,7 +130,7 @@ public class LdapOtpOperator extends AbstractOtpOperator
             }
             final ChaiUser theUser = pwmRequest == null
                     ? pwmApplication.getProxiedChaiUser( userIdentity )
-                    : pwmRequest.getPwmSession().getSessionManager().getActor( pwmApplication, userIdentity );
+                    : pwmRequest.getPwmSession().getSessionManager().getActor( userIdentity );
             theUser.writeStringAttribute( ldapStorageAttribute, value );
             LOGGER.info( () -> "saved OTP secret for user to chai-ldap format" );
         }

server/src/main/java/password/pwm/util/password/PasswordUtility.java

@@ -273,7 +273,7 @@ public class PasswordUtility
         try
         {
             final PwmPasswordRuleValidator pwmPasswordRuleValidator = new PwmPasswordRuleValidator( pwmApplication, userInfo.getPasswordPolicy() );
-            pwmPasswordRuleValidator.testPassword( newPassword, null, userInfo, pwmSession.getSessionManager().getActor( pwmApplication ) );
+            pwmPasswordRuleValidator.testPassword( newPassword, null, userInfo, pwmSession.getSessionManager().getActor( ) );
         }
         catch ( final PwmDataValidationException e )
         {
@@ -288,7 +288,7 @@ public class PasswordUtility
         boolean setPasswordWithoutOld = false;
         if ( oldPassword == null )
         {
-            if ( pwmSession.getSessionManager().getActor( pwmApplication ).getChaiProvider().getDirectoryVendor() == DirectoryVendor.ACTIVE_DIRECTORY )
+            if ( pwmSession.getSessionManager().getActor( ).getChaiProvider().getDirectoryVendor() == DirectoryVendor.ACTIVE_DIRECTORY )
             {
                 setPasswordWithoutOld = true;
             }
@@ -316,7 +316,7 @@ public class PasswordUtility
         pwmSession.getLoginInfoBean().setUserCurrentPassword( newPassword );
 
         //close any outstanding ldap connections (since they cache the old password)
-        pwmSession.getSessionManager().updateUserPassword( pwmApplication, userInfo.getUserIdentity(), newPassword );
+        pwmSession.getSessionManager().updateUserPassword( userInfo.getUserIdentity(), newPassword );
 
         // clear the "requires new password flag"
         pwmSession.getLoginInfoBean().getLoginFlags().remove( LoginInfoBean.LoginFlag.forcePwChange );
@@ -328,7 +328,7 @@ public class PasswordUtility
         pwmSession.reloadUserInfoBean( pwmRequest );
 
         // create a proxy user object for pwm to update/read the user.
-        final ChaiUser proxiedUser = pwmSession.getSessionManager().getActor( pwmApplication );
+        final ChaiUser proxiedUser = pwmSession.getSessionManager().getActor();
 
         // update statistics
         {
@@ -498,7 +498,7 @@ public class PasswordUtility
             throw new PwmOperationalException( errorInformation );
         }
 
-        final HelpdeskProfile helpdeskProfile = pwmRequest.getPwmSession().getSessionManager().getHelpdeskProfile( pwmApplication );
+        final HelpdeskProfile helpdeskProfile = pwmRequest.getPwmSession().getSessionManager().getHelpdeskProfile( );
         if ( helpdeskProfile == null )
         {
             final String errorMsg = "attempt to helpdeskSetUserPassword, but user does not have helpdesk permission";
@@ -704,7 +704,7 @@ public class PasswordUtility
 
                 final ActionExecutor actionExecutor = new ActionExecutor.ActionExecutorSettings( pwmApplication, userIdentity )
                         .setExpandPwmMacros( true )
-                        .setMacroMachine( pwmRequest.getPwmSession().getSessionManager().getMacroMachine( pwmApplication ) )
+                        .setMacroMachine( pwmRequest.getPwmSession().getSessionManager().getMacroMachine( ) )
                         .createActionExecutor();
                 actionExecutor.executeActions( configValues, pwmRequest.getLabel() );
             }

server/src/main/resources/password/pwm/AppProperty.properties

@@ -104,6 +104,8 @@ health.certificate.warnSeconds=2592000
 health.disk.minFreeWarning=500000000
 health.ldap.cautionDurationMS=10800000
 health.ldap.proxy.pwExpireWarnSeconds=2592000
+health.ldap.userSearch.warnMS=500
+health.ldap.userSearch.searchTerm=@RandomChar:10@-healthcheck
 health.java.maxThreads=1000
 health.java.minHeapBytes=67108864
 helpdesk.token.maxAgeSeconds=300
@@ -193,6 +195,7 @@ ldap.chaiSettings=
 ldap.proxy.connectionsPerProfile=10
 ldap.proxy.maxConnections=50
 ldap.proxy.useThreadLocal=true
+ldap.proxy.idleThreadLocal.timeoutMS=90000
 ldap.extensions.nmas.enable=true
 ldap.connection.timeoutMS=30000
 ldap.profile.retryDelayMS=30000

server/src/main/resources/password/pwm/config/PwmSetting.xml

@@ -579,7 +579,7 @@
             <value><![CDATA[@LDAP:DN@]]></value>
         </default>
     </setting>
-    <setting hidden="true" key="ldap.search.timeoutSeconds" level="2">
+    <setting hidden="false" key="ldap.search.timeoutSeconds" level="2">
         <default>
             <value>30</value>
         </default>

server/src/main/resources/password/pwm/i18n/Health.properties

@@ -24,6 +24,7 @@ HealthMessage_LDAP_OK=All configured LDAP servers are reachable
 HealthMessage_LDAP_No_Connection=Unable to connect to LDAP server %1%, error: %2%
 HealthMessage_LDAP_ProxyTestSameUser=%1% setting is the same value as the %2% setting
 HealthMessage_LDAP_ProxyUserPwExpired=Proxy user %1% password will expire within %2%.  The proxy user password should never expire. 
+HealthMessage_LDAP_SearchFailure=Error while searching for users: %1%
 HealthMessage_LDAP_TestUserUnavailable=LDAP unavailable error while testing ldap test user %1%, error: %2%
 HealthMessage_LDAP_TestUserUnexpected=Unexpected error while testing ldap test user %1%, error: %2%
 HealthMessage_LDAP_TestUserError=error verifying test user account %1%, error: %2%