improve scheduler/thread management for domain model

server/src/main/java/password/pwm/PwmApplication.java

@@ -67,6 +67,7 @@ import password.pwm.util.java.TimeDuration;
 import password.pwm.util.json.JsonFactory;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.logging.LocalDBLogger;
+import password.pwm.util.logging.PwmLogLevel;
 import password.pwm.util.logging.PwmLogManager;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.secure.PwmRandom;
@@ -94,6 +95,8 @@ public class PwmApplication
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwmApplication.class );
 
+    static final int DOMAIN_STARTUP_THREADS = 10;
+
     private final AtomicInteger activeServletRequests = new AtomicInteger( 0 );
 
     private volatile Map<DomainID, PwmDomain> domains = new HashMap<>();
@@ -152,8 +155,6 @@ public class PwmApplication
 
         initRuntimeNonce();
 
-        domains = PwmDomainUtil.createDomainInstances( this );
-
         // initialize logger
         PwmApplicationUtil.initializeLogging( this );
 
@@ -204,15 +205,15 @@ public class PwmApplication
             }
         }
 
+        // read the instance id
+        instanceID = PwmApplicationUtil.fetchInstanceID( this, localDB );
+        LOGGER.debug( sessionLabel, () -> "using '" + getInstanceID() + "' for instance's ID (instanceID)" );
+
         this.localDBLogger = PwmLogManager.initializeLocalDBLogger( this );
 
         // log the loaded configuration
         LOGGER.debug( sessionLabel, () -> "configuration load completed" );
 
-        // read the pwm servlet instance id
-        instanceID = PwmApplicationUtil.fetchInstanceID( this, localDB );
-        LOGGER.debug( sessionLabel, () -> "using '" + getInstanceID() + "' for instance's ID (instanceID)" );
-
         // read the pwm installation date
         installTime = fetchInstallDate( startupTime );
         LOGGER.debug( sessionLabel, () -> "this application instance first installed on " + StringUtil.toIsoDate( installTime ) );
@@ -223,9 +224,11 @@ public class PwmApplication
 
         pwmScheduler = new PwmScheduler( this );
 
+        domains = PwmDomainUtil.createDomainInstances( this );
+
         pwmServiceManager.initAllServices();
 
-            PwmDomainUtil.initDomains( this, domains().values() );
+        PwmDomainUtil.initDomains( this, domains().values() );
 
         final boolean skipPostInit = pwmEnvironment.isInternalRuntimeInstance()
                 || pwmEnvironment.getFlags().contains( PwmEnvironment.ApplicationFlag.CommandLineInstance );
@@ -237,7 +240,7 @@ public class PwmApplication
             StatisticsClient.incrementStat( this, Statistic.PWM_STARTUPS );
             LOGGER.debug( sessionLabel, () -> "buildTime=" + PwmConstants.BUILD_TIME + ", javaLocale=" + Locale.getDefault() + ", DefaultLocale=" + PwmConstants.DEFAULT_LOCALE );
 
-            pwmScheduler.immediateExecuteRunnableInNewThread( this::postInitTasks, this.getClass().getSimpleName() + " postInit tasks" );
+            pwmScheduler.immediateExecuteRunnableInNewThread( this::postInitTasks, sessionLabel, this.getClass().getSimpleName() + " postInit tasks" );
         }
 
     }
@@ -267,9 +270,6 @@ public class PwmApplication
     {
         final Instant startTime = Instant.now();
 
-        getPwmScheduler().immediateExecuteRunnableInNewThread( UserAgentUtils::initializeCache, "initialize useragent cache" );
-        getPwmScheduler().immediateExecuteRunnableInNewThread( PwmSettingMetaDataReader::initCache, "initialize PwmSetting cache" );
-
         // send system audit event
         AuditServiceClient.submitSystemEvent( this, sessionLabel, AuditEvent.STARTUP );
 
@@ -288,17 +288,25 @@ public class PwmApplication
             PwmApplicationUtil.outputTomcatConf( this );
         }
 
-        if ( Boolean.parseBoolean( getConfig().readAppProperty( AppProperty.LOGGING_OUTPUT_CONFIGURATION ) ) )
+        if ( Boolean.parseBoolean( getConfig().readAppProperty( AppProperty.LOGGING_OUTPUT_CONFIGURATION ) )
+                && LOGGER.isEnabled( PwmLogLevel.TRACE )
+        )
         {
-            getPwmScheduler().immediateExecuteRunnableInNewThread( () ->
-            {
-                PwmApplicationUtil.outputApplicationInfoToLog( this );
-                PwmApplicationUtil.outputConfigurationToLog( this, DomainID.systemId() );
-                PwmApplicationUtil.outputNonDefaultPropertiesToLog( this );
-            }, "output configuration to log" );
+            PwmApplicationUtil.outputApplicationInfoToLog( this );
+            PwmApplicationUtil.outputConfigurationToLog( this, DomainID.systemId() );
+            PwmApplicationUtil.outputNonDefaultPropertiesToLog( this );
         }
 
         MBeanUtility.registerMBean( this );
+
+        getPwmScheduler().immediateExecuteRunnableInNewThread( () ->
+                {
+                    PwmSettingMetaDataReader.initCache();
+                    UserAgentUtils.initializeCache();
+                },
+                sessionLabel,
+                "initialize useragent cache" );
+
         LOGGER.trace( sessionLabel, () -> "completed post init tasks", () -> TimeDuration.fromCurrent( startTime ) );
     }
 
@@ -388,7 +396,7 @@ public class PwmApplication
             final List<Callable<Object>> callables = domains.values().stream()
                     .map( pwmDomain -> Executors.callable( pwmDomain::shutdown ) )
                     .collect( Collectors.toList() );
-            pwmScheduler.executeImmediateThreadPerJobAndAwaitCompletion( callables, "domain shutdown task" );
+            pwmScheduler.executeImmediateThreadPerJobAndAwaitCompletion( DOMAIN_STARTUP_THREADS, callables, sessionLabel, PwmApplication.class );
         }
         catch ( final PwmUnrecoverableException e )
         {

server/src/main/java/password/pwm/PwmApplicationUtil.java

@@ -290,7 +290,7 @@ class PwmApplicationUtil
         final Function<Map.Entry<String, String>, String> valueFormatter = entry ->
         {
             final String spacedValue = entry.getValue().replace( "\n", "\n   " );
-            return " " + entry.getKey() + "\n   " + spacedValue + "\n";
+            return " " + entry.getKey() + "\n   " + spacedValue;
         };
 
         final StoredConfiguration storedConfiguration = pwmApplication.getConfig().getStoredConfiguration();

server/src/main/java/password/pwm/PwmDomain.java

@@ -38,9 +38,11 @@ import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmServiceEnum;
 import password.pwm.svc.PwmServiceManager;
 import password.pwm.svc.cache.CacheService;
+import password.pwm.svc.cr.CrService;
 import password.pwm.svc.event.AuditService;
 import password.pwm.svc.httpclient.HttpClientService;
 import password.pwm.svc.intruder.IntruderDomainService;
+import password.pwm.svc.otp.OtpService;
 import password.pwm.svc.pwnotify.PwNotifyService;
 import password.pwm.svc.secure.DomainSecureService;
 import password.pwm.svc.sessiontrack.SessionTrackService;
@@ -48,17 +50,12 @@ import password.pwm.svc.stats.StatisticsService;
 import password.pwm.svc.token.TokenService;
 import password.pwm.svc.userhistory.UserHistoryService;
 import password.pwm.svc.wordlist.SharedHistoryService;
-import password.pwm.util.DailySummaryJob;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.svc.cr.CrService;
-import password.pwm.svc.otp.OtpService;
 
 import java.time.Instant;
 import java.util.List;
 import java.util.Objects;
-import java.util.concurrent.ExecutorService;
 
 /**
  * A repository for objects common to the servlet context.  A singleton
@@ -93,12 +90,8 @@ public class PwmDomain
     {
         final Instant startTime = Instant.now();
         LOGGER.trace( sessionLabel, () -> "initializing domain " + domainID.stringValue() );
-        pwmServiceManager.initAllServices();
 
-        {
-            final ExecutorService executorService = PwmScheduler.makeSingleThreadExecutorService( getPwmApplication(), DailySummaryJob.class );
-            pwmApplication.getPwmScheduler().scheduleDailyZuluZeroStartJob( new DailySummaryJob( this ), executorService, TimeDuration.ZERO );
-        }
+        pwmServiceManager.initAllServices();
 
         if ( Boolean.parseBoolean( getConfig().readAppProperty( AppProperty.LOGGING_OUTPUT_CONFIGURATION ) ) )
         {

server/src/main/java/password/pwm/PwmDomainUtil.java

@@ -24,7 +24,6 @@ import password.pwm.bean.DomainID;
 import password.pwm.config.AppConfig;
 import password.pwm.config.DomainConfig;
 import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
@@ -84,7 +83,7 @@ class PwmDomainUtil
                 .collect( Collectors.toUnmodifiableList() );
 
         final  List<Optional<PwmUnrecoverableException>> domainStartException = pwmApplication.getPwmScheduler()
-                .executeImmediateThreadPerJobAndAwaitCompletion( callables, "domain initializer" );
+                .executeImmediateThreadPerJobAndAwaitCompletion( PwmApplication.DOMAIN_STARTUP_THREADS, callables, pwmApplication.getSessionLabel(), PwmDomainUtil.class );
 
         final Optional<PwmUnrecoverableException> domainStartupException = domainStartException.stream()
                 .filter( Optional::isPresent )
@@ -169,13 +168,17 @@ class PwmDomainUtil
         // 1 minute later ( to avoid interrupting any in-progress requests, shutdown any obsoleted domains
         if ( !deletedDomains.isEmpty() )
         {
-            pwmApplication.getPwmScheduler().scheduleJob( () ->
+            pwmApplication.getPwmScheduler().immediateExecuteRunnableInNewThread( () ->
                     {
-                        LOGGER.debug( () -> "shutting down obsoleted domain services" );
+                        TimeDuration.MINUTE.pause();
+                        final Instant startTime = Instant.now();
+                        LOGGER.trace( pwmApplication.getSessionLabel(), () -> "shutting down obsoleted domain services" );
                         deletedDomains.forEach( PwmDomain::shutdown );
+                        LOGGER.debug( pwmApplication.getSessionLabel(), () -> "shut down obsoleted domain services completed",
+                                () -> TimeDuration.fromCurrent( startTime ) );
                     },
-                    PwmScheduler.makeSingleThreadExecutorService( pwmApplication, pwmApplication.getClass() ),
-                    TimeDuration.MINUTE );
+                    pwmApplication.getSessionLabel(),
+                    "obsoleted domain cleanup" );
         }
 
     }

server/src/main/java/password/pwm/bean/SessionLabel.java

@@ -24,6 +24,7 @@ import lombok.Builder;
 import lombok.Value;
 import password.pwm.PwmConstants;
 import password.pwm.svc.PwmService;
+import password.pwm.util.java.StringUtil;
 
 import java.io.Serializable;
 
@@ -33,6 +34,7 @@ public class SessionLabel implements Serializable
 {
     private static final String SYSTEM_LABEL_SESSION_ID = "#";
     private static final String RUNTIME_LABEL_SESSION_ID = "#";
+
     public static final SessionLabel SYSTEM_LABEL = SessionLabel.builder().sessionID( SYSTEM_LABEL_SESSION_ID ).username( PwmConstants.PWM_APP_NAME ).build();
     public static final SessionLabel RUNTIME_LABEL = SessionLabel.builder().sessionID( RUNTIME_LABEL_SESSION_ID ).username( "internal" ).build();
     public static final SessionLabel TEST_SESSION_LABEL = SessionLabel.builder().sessionID( SYSTEM_LABEL_SESSION_ID ).username( "test" ).build();
@@ -56,4 +58,41 @@ public class SessionLabel implements Serializable
                 .domain( domainID.stringValue() )
                 .build();
     }
+
+    public String toDebugLabel( )
+    {
+        final StringBuilder sb = new StringBuilder();
+        final String sessionID = getSessionID();
+        final String username = getUsername();
+
+        if ( StringUtil.notEmpty( sessionID ) )
+        {
+            sb.append( sessionID );
+        }
+        if ( StringUtil.notEmpty( domain ) )
+        {
+            if ( sb.length() > 0 )
+            {
+                sb.append( ',' );
+            }
+            sb.append( domain );
+        }
+        if ( StringUtil.notEmpty( username ) )
+        {
+            if ( sb.length() > 0 )
+            {
+                sb.append( ',' );
+            }
+            sb.append( username );
+        }
+
+        if ( sb.length() > 0 )
+        {
+            sb.insert( 0, "{" );
+            sb.append( "} " );
+        }
+
+        return sb.toString();
+    }
+
 }

server/src/main/java/password/pwm/health/HealthService.java

@@ -58,8 +58,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 import java.util.zip.ZipOutputStream;
@@ -76,8 +76,8 @@ public class HealthService extends AbstractPwmService implements PwmService
             new CertificateChecker() );
 
 
-    private ExecutorService executorService;
-    private ExecutorService supportZipWriterService;
+    private ScheduledExecutorService executorService;
+    private ScheduledExecutorService supportZipWriterService;
     private HealthMonitorSettings settings;
 
     private final Map<HealthMonitorFlag, Serializable> healthProperties = new ConcurrentHashMap<>();
@@ -121,8 +121,8 @@ public class HealthService extends AbstractPwmService implements PwmService
             return STATUS.CLOSED;
         }
 
-        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-        supportZipWriterService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        executorService = PwmScheduler.makeBackgroundServiceExecutor( pwmApplication, getSessionLabel(), this.getClass() );
+        supportZipWriterService = PwmScheduler.makeBackgroundServiceExecutor( pwmApplication, getSessionLabel(), this.getClass() );
         scheduleNextZipOutput();
 
         if ( settings.getThreadDumpInterval().as( TimeDuration.Unit.SECONDS ) > 0 )
@@ -164,7 +164,7 @@ public class HealthService extends AbstractPwmService implements PwmService
         {
             final Instant startTime = Instant.now();
             LOGGER.trace( () ->  "begin force immediate check" );
-            final Future future = getPwmApplication().getPwmScheduler().scheduleJob( new ImmediateJob(), executorService, TimeDuration.ZERO );
+            final Future<?> future = getPwmApplication().getPwmScheduler().scheduleJob( new ImmediateJob(), executorService, TimeDuration.ZERO );
             settings.getMaximumForceCheckWait().pause( future::isDone );
             final TimeDuration checkDuration = TimeDuration.fromCurrent( startTime );
             averageStats.update( AverageStatKey.checkProcessTime, checkDuration.asDuration() );

server/src/main/java/password/pwm/http/ContextManager.java

@@ -304,7 +304,7 @@ public class ContextManager implements Serializable
 
         taskMaster = Executors.newSingleThreadScheduledExecutor(
                 PwmScheduler.makePwmThreadFactory(
-                        PwmScheduler.makeThreadName( pwmApplication, this.getClass() ) + "-",
+                        PwmScheduler.makeThreadName( SESSION_LABEL, pwmApplication, this.getClass() ) + "-",
                         true
                 ) );
 

server/src/main/java/password/pwm/http/servlet/peoplesearch/PeopleSearchDataReader.java

@@ -28,8 +28,8 @@ import com.novell.ldapchai.provider.ChaiProvider;
 import lombok.Value;
 import org.apache.commons.csv.CSVPrinter;
 import password.pwm.AppProperty;
-import password.pwm.PwmDomain;
 import password.pwm.PwmConstants;
+import password.pwm.PwmDomain;
 import password.pwm.bean.UserIdentity;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.profile.PeopleSearchProfile;
@@ -49,11 +49,11 @@ import password.pwm.http.servlet.peoplesearch.bean.SearchResultBean;
 import password.pwm.http.servlet.peoplesearch.bean.UserDetailBean;
 import password.pwm.http.servlet.peoplesearch.bean.UserReferenceBean;
 import password.pwm.i18n.Display;
-import password.pwm.ldap.permission.UserPermissionUtility;
 import password.pwm.ldap.PhotoDataBean;
 import password.pwm.ldap.UserInfo;
 import password.pwm.ldap.UserInfoFactory;
 import password.pwm.ldap.permission.UserPermissionType;
+import password.pwm.ldap.permission.UserPermissionUtility;
 import password.pwm.ldap.search.SearchConfiguration;
 import password.pwm.ldap.search.UserSearchEngine;
 import password.pwm.ldap.search.UserSearchResults;
@@ -66,9 +66,9 @@ import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.MiscUtil;
 import password.pwm.util.java.PwmTimeUtil;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroRequest;
 

server/src/main/java/password/pwm/http/servlet/peoplesearch/PeopleSearchService.java

@@ -30,10 +30,7 @@ import password.pwm.util.PwmScheduler;
 
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 
 public class PeopleSearchService extends AbstractPwmService implements PwmService
 {
@@ -45,15 +42,7 @@ public class PeopleSearchService extends AbstractPwmService implements PwmServic
     {
         final int maxThreadCount = 5;
 
-        final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( pwmApplication, PeopleSearchService.class ), true );
-        threadPoolExecutor = new ThreadPoolExecutor(
-                maxThreadCount,
-                maxThreadCount,
-                1,
-                TimeUnit.MINUTES,
-                new ArrayBlockingQueue<>( 5000 ),
-                threadFactory
-        );
+        threadPoolExecutor = PwmScheduler.makeMultiThreadExecutor( maxThreadCount, pwmApplication, getSessionLabel(), PeopleSearchService.class );
 
         return STATUS.OPEN;
     }

server/src/main/java/password/pwm/ldap/LdapConnectionService.java

@@ -41,11 +41,9 @@ import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.ConditionalTaskExecutor;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StatisticCounterBundle;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
@@ -64,7 +62,6 @@ import java.util.Set;
 import java.util.TreeMap;
 import java.util.WeakHashMap;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Consumer;
@@ -83,7 +80,6 @@ public class LdapConnectionService extends AbstractPwmService implements PwmServ
     private final Map<String, Map<Integer, ChaiProvider>> proxyChaiProviders = new HashMap<>();
 
     private PwmDomain pwmDomain;
-    private ExecutorService executorService;
     private ChaiProviderFactory chaiProviderFactory;
     private AtomicLoopIntIncrementer slotIncrementer;
 
@@ -151,13 +147,6 @@ public class LdapConnectionService extends AbstractPwmService implements PwmServ
         // read the lastLoginTime
         this.lastLdapErrors.putAll( pwmApplication.readLastLdapFailure( getDomainID() ) );
 
-        final long idleWeakTimeoutMS = JavaHelper.silentParseLong(
-                pwmDomain.getConfig().readAppProperty( AppProperty.LDAP_PROXY_IDLE_THREAD_LOCAL_TIMEOUT_MS ),
-                60_000 );
-        final TimeDuration idleWeakTimeout = TimeDuration.of( idleWeakTimeoutMS, TimeDuration.Unit.MILLISECONDS );
-        this.executorService = PwmScheduler.makeBackgroundExecutor( pwmDomain.getPwmApplication(), this.getClass() );
-        pwmDomain.getPwmApplication().getPwmScheduler().scheduleFixedRateJob( new ThreadLocalCleaner(), executorService, idleWeakTimeout, idleWeakTimeout );
-
         final int connectionsPerProfile = maxSlotsPerProfile( pwmDomain );
         LOGGER.trace( () -> "allocating " + connectionsPerProfile + " ldap proxy connections per profile" );
         slotIncrementer = AtomicLoopIntIncrementer.builder().ceiling( connectionsPerProfile ).build();
@@ -190,7 +179,6 @@ public class LdapConnectionService extends AbstractPwmService implements PwmServ
         lastLdapErrors.clear();
         iterateThreadLocals( container -> container.getProviderMap().clear() );
         threadLocalContainers.clear();
-        executorService.shutdown();
     }
 
     @Override
@@ -443,7 +431,7 @@ public class LdapConnectionService extends AbstractPwmService implements PwmServ
         debugInfo.put( DebugKey.ThreadLocals, String.valueOf( threadLocalConnections.get( ) ) );
         debugInfo.put( DebugKey.CreatedProviders, String.valueOf( stats.get( StatKey.createdProxies ) ) );
         debugInfo.put( DebugKey.DiscardedThreadLocals, String.valueOf( stats.get( StatKey.clearedThreadLocals ) ) );
-        return Collections.unmodifiableMap( CollectionUtil.enumMapToStringMap( debugInfo ) );
+        return CollectionUtil.enumMapToStringMap( debugInfo );
     }
 
     @Data
@@ -454,38 +442,30 @@ public class LdapConnectionService extends AbstractPwmService implements PwmServ
         private volatile String threadName;
     }
 
-    private class ThreadLocalCleaner implements Runnable
+    void cleanupIssuedThreadLocals()
     {
-        @Override
-        public void run()
-        {
-            cleanupIssuedThreadLocals();
-            debugLogger.conditionallyExecuteTask();
-        }
+        final TimeDuration maxIdleTime = TimeDuration.MINUTE;
 
-        private void cleanupIssuedThreadLocals()
+        iterateThreadLocals( container ->
         {
-            final TimeDuration maxIdleTime = TimeDuration.MINUTE;
-
-            iterateThreadLocals( container ->
+            if ( !container.getProviderMap().isEmpty() )
             {
-                if ( !container.getProviderMap().isEmpty() )
+                final Instant timestamp = container.getTimestamp();
+                final TimeDuration age = TimeDuration.fromCurrent( timestamp );
+                if ( age.isLongerThan( maxIdleTime ) )
                 {
-                    final Instant timestamp = container.getTimestamp();
-                    final TimeDuration age = TimeDuration.fromCurrent( timestamp );
-                    if ( age.isLongerThan( maxIdleTime ) )
+                    for ( final ChaiProvider chaiProvider : container.getProviderMap().values() )
                     {
-                        for ( final ChaiProvider chaiProvider : container.getProviderMap().values() )
-                        {
-                            LOGGER.trace( () -> "discarding idled connection id=" + chaiProvider.toString() + " from orphaned threadLocal, age="
-                                    + age.asCompactString() + ", thread=" + container.getThreadName() );
-                            stats.increment( StatKey.clearedThreadLocals );
-                        }
-                        container.getProviderMap().clear();
+                        LOGGER.trace( () -> "discarding idled connection id=" + chaiProvider.toString() + " from orphaned threadLocal, age="
+                                + age.asCompactString() + ", thread=" + container.getThreadName() );
+                        stats.increment( StatKey.clearedThreadLocals );
                     }
+                    container.getProviderMap().clear();
                 }
-            } );
-        }
+            }
+        } );
+
+        debugLogger.conditionallyExecuteTask();
     }
 
     private void iterateThreadLocals( final Consumer<ThreadLocalContainer> consumer )

server/src/main/java/password/pwm/ldap/LdapSystemService.java

@@ -0,0 +1,80 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.ldap;
+
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.PwmDomain;
+import password.pwm.bean.DomainID;
+import password.pwm.error.PwmException;
+import password.pwm.health.HealthRecord;
+import password.pwm.svc.AbstractPwmService;
+import password.pwm.svc.PwmService;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.TimeDuration;
+
+import java.util.List;
+
+public class LdapSystemService extends AbstractPwmService implements PwmService
+{
+    @Override
+    protected STATUS postAbstractInit( final PwmApplication pwmApplication, final DomainID domainID ) throws PwmException
+    {
+        final long idleWeakTimeoutMS = JavaHelper.silentParseLong(
+                pwmApplication.getConfig().readAppProperty( AppProperty.LDAP_PROXY_IDLE_THREAD_LOCAL_TIMEOUT_MS ),
+                60_000 );
+        final TimeDuration idleWeakTimeout = TimeDuration.of( idleWeakTimeoutMS, TimeDuration.Unit.MILLISECONDS );
+        pwmApplication.getPwmScheduler().scheduleFixedRateJob( new ThreadLocalCleaner(), getExecutorService(), idleWeakTimeout, idleWeakTimeout );
+
+        return STATUS.OPEN;
+    }
+
+    @Override
+    protected void shutdownImpl()
+    {
+
+    }
+
+    @Override
+    protected List<HealthRecord> serviceHealthCheck()
+    {
+        return null;
+    }
+
+    @Override
+    public ServiceInfoBean serviceInfo()
+    {
+        return null;
+    }
+
+    private class ThreadLocalCleaner implements Runnable
+    {
+        @Override
+        public void run()
+        {
+            for ( final PwmDomain pwmDomain : getPwmApplication().domains().values() )
+            {
+                pwmDomain.getLdapConnectionService().cleanupIssuedThreadLocals();
+            }
+        }
+    }
+
+}

server/src/main/java/password/pwm/ldap/search/UserSearchEngine.java

@@ -69,13 +69,10 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 
 
 public class UserSearchEngine extends AbstractPwmService implements PwmService
@@ -787,7 +784,7 @@ public class UserSearchEngine extends AbstractPwmService implements PwmService
         return idMsg;
     }
 
-    private static ThreadPoolExecutor createExecutor( final PwmDomain pwmDomain )
+    private ThreadPoolExecutor createExecutor( final PwmDomain pwmDomain )
     {
         final DomainConfig domainConfig = pwmDomain.getConfig();
 
@@ -813,19 +810,11 @@ public class UserSearchEngine extends AbstractPwmService implements PwmService
             final int factor = Integer.parseInt( domainConfig.readAppProperty( AppProperty.LDAP_SEARCH_PARALLEL_FACTOR ) );
             final int maxThreads = Integer.parseInt( domainConfig.readAppProperty( AppProperty.LDAP_SEARCH_PARALLEL_THREAD_MAX ) );
             final int threads = Math.min( maxThreads, ( endPoints ) * factor );
-            final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( pwmDomain.getPwmApplication(), UserSearchEngine.class ), true );
             final int minThreads = JavaHelper.rangeCheck( 1, 10, endPoints );
 
             LOGGER.trace( () -> "initialized with threads min=" + minThreads + " max=" + threads );
 
-            return new ThreadPoolExecutor(
-                    minThreads,
-                    threads,
-                    1,
-                    TimeUnit.MINUTES,
-                    new ArrayBlockingQueue<>( threads ),
-                    threadFactory
-            );
+            return PwmScheduler.makeMultiThreadExecutor( threads, getPwmApplication(), getSessionLabel(), UserSearchEngine.class );
         }
         return null;
     }

server/src/main/java/password/pwm/svc/AbstractPwmService.java

@@ -27,12 +27,18 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmException;
 import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
+import password.pwm.util.PwmScheduler;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.LazySupplier;
+import password.pwm.util.java.TimeDuration;
 
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.function.Supplier;
 
 public abstract class AbstractPwmService implements PwmService
 {
@@ -42,6 +48,9 @@ public abstract class AbstractPwmService implements PwmService
     private DomainID domainID;
     private SessionLabel sessionLabel;
 
+    private Supplier<ScheduledExecutorService> executorService;
+
+
     public final PwmService.STATUS status()
     {
         return status;
@@ -56,6 +65,8 @@ public abstract class AbstractPwmService implements PwmService
                 ? pwmApplication.getSessionLabel()
                 : pwmApplication.domains().get( domainID ).getSessionLabel();
 
+        executorService = new LazySupplier<>( () -> PwmScheduler.makeBackgroundServiceExecutor( pwmApplication, getSessionLabel(), getClass() ) );
+
         if ( pwmApplication.checkConditions( openConditions() ) )
         {
             setStatus( this.postAbstractInit( pwmApplication, domainID ) );
@@ -79,6 +90,10 @@ public abstract class AbstractPwmService implements PwmService
     public void shutdown()
     {
         this.status = STATUS.CLOSED;
+        if ( executorService != null )
+        {
+            JavaHelper.closeAndWaitExecutor( executorService.get(), TimeDuration.SECONDS_10 );
+        }
         shutdownImpl();
     }
 
@@ -131,4 +146,9 @@ public abstract class AbstractPwmService implements PwmService
     {
         return EnumSet.of( PwmApplication.Condition.RunningMode, PwmApplication.Condition.LocalDBOpen, PwmApplication.Condition.NotInternalInstance );
     }
+
+    protected ScheduledExecutorService getExecutorService()
+    {
+        return executorService.get();
+    }
 }

server/src/main/java/password/pwm/svc/PwmServiceEnum.java

@@ -28,12 +28,12 @@ import password.pwm.svc.intruder.IntruderDomainService;
 import password.pwm.svc.intruder.IntruderSystemService;
 import password.pwm.svc.node.NodeService;
 import password.pwm.svc.pwnotify.PwNotifyService;
+import password.pwm.svc.sms.SmsQueueService;
 import password.pwm.svc.stats.StatisticsService;
 import password.pwm.svc.wordlist.SeedlistService;
 import password.pwm.svc.wordlist.SharedHistoryService;
 import password.pwm.svc.wordlist.WordlistService;
 import password.pwm.util.java.CollectionUtil;
-import password.pwm.svc.sms.SmsQueueService;
 
 import java.util.Arrays;
 import java.util.List;
@@ -56,6 +56,8 @@ public enum PwmServiceEnum
     SmsQueueManager( SmsQueueService.class, PwmSettingScope.SYSTEM ),
     UrlShortenerService( password.pwm.svc.shorturl.UrlShortenerService.class, PwmSettingScope.SYSTEM ),
     CacheService( password.pwm.svc.cache.CacheService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
+    LdapSystemService( password.pwm.ldap.LdapSystemService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
+    TokenSystemService( password.pwm.svc.token.TokenSystemService.class, PwmSettingScope.SYSTEM ),
     HealthMonitor( HealthService.class, PwmSettingScope.SYSTEM ),
     ReportService( password.pwm.svc.report.ReportService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
     SessionTrackService( password.pwm.svc.sessiontrack.SessionTrackService.class, PwmSettingScope.SYSTEM ),

server/src/main/java/password/pwm/svc/db/DatabaseService.java

@@ -35,13 +35,12 @@ import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.EpsStatistic;
 import password.pwm.svc.stats.StatisticsClient;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.PwmTimeUtil;
 import password.pwm.util.java.StringUtil;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.logging.PwmLogger;
 
 import java.sql.Connection;
@@ -58,7 +57,6 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
 
 
 public class DatabaseService extends AbstractPwmService implements PwmService
@@ -80,8 +78,6 @@ public class DatabaseService extends AbstractPwmService implements PwmService
     private AtomicLoopIntIncrementer slotIncrementer;
     private final Map<Integer, DatabaseAccessorImpl> accessors = new ConcurrentHashMap<>();
 
-    private ExecutorService executorService;
-
     private final Map<DatabaseAboutProperty, String> debugInfo = new LinkedHashMap<>();
 
     private volatile boolean initialized = false;
@@ -106,15 +102,11 @@ public class DatabaseService extends AbstractPwmService implements PwmService
     {
         this.dbConfiguration = DBConfiguration.fromConfiguration( getPwmApplication().getConfig() );
 
-
-
-        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-
         final TimeDuration watchdogFrequency = TimeDuration.of(
                 Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.DB_CONNECTIONS_WATCHDOG_FREQUENCY_SECONDS ) ),
                 TimeDuration.Unit.SECONDS );
 
-        pwmApplication.getPwmScheduler().scheduleFixedRateJob( new ConnectionMonitor(), executorService, watchdogFrequency, watchdogFrequency );
+        pwmApplication.getPwmScheduler().scheduleFixedRateJob( new ConnectionMonitor(), getExecutorService(), watchdogFrequency, watchdogFrequency );
 
         return dbInit();
     }
@@ -191,11 +183,6 @@ public class DatabaseService extends AbstractPwmService implements PwmService
     {
         setStatus( STATUS.CLOSED );
 
-        if ( executorService != null )
-        {
-            executorService.shutdown();
-        }
-
         clearCurrentAccessors();
 
         try

server/src/main/java/password/pwm/svc/email/EmailService.java

@@ -130,7 +130,7 @@ public class EmailService extends AbstractPwmService implements PwmService
         final LocalDBStoredQueue localDBStoredQueue = LocalDBStoredQueue.createLocalDBStoredQueue(
                 this.getPwmApplication(), this.getPwmApplication().getLocalDB(), LocalDB.DB.EMAIL_QUEUE );
 
-        workQueueProcessor = new WorkQueueProcessor<>( this.getPwmApplication(), localDBStoredQueue, settings, new EmailItemProcessor(), this.getClass() );
+        workQueueProcessor = new WorkQueueProcessor<>( this.getPwmApplication(), this.getSessionLabel(), localDBStoredQueue, settings, new EmailItemProcessor(), this.getClass() );
 
         connectionPool = new EmailConnectionPool( servers, emailServiceSettings );
 

server/src/main/java/password/pwm/svc/event/AuditService.java

@@ -108,7 +108,7 @@ public class AuditService extends AbstractPwmService implements PwmService
         {
             try
             {
-                syslogManager = new SyslogAuditService( pwmApplication );
+                syslogManager = new SyslogAuditService( pwmApplication, getSessionLabel() );
             }
             catch ( final Exception e )
             {
@@ -118,7 +118,7 @@ public class AuditService extends AbstractPwmService implements PwmService
         }
 
         auditVault = new LocalDbAuditVault();
-        auditVault.init( pwmApplication, pwmApplication.getLocalDB(), settings );
+        auditVault.init( pwmApplication, getSessionLabel(), pwmApplication.getLocalDB(), settings );
 
         return STATUS.OPEN;
     }

server/src/main/java/password/pwm/svc/event/AuditVault.java

@@ -21,6 +21,7 @@
 package password.pwm.svc.event;
 
 import password.pwm.PwmApplication;
+import password.pwm.bean.SessionLabel;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.util.localdb.LocalDB;
@@ -30,7 +31,7 @@ import java.util.Iterator;
 
 public interface AuditVault
 {
-    void init( PwmApplication pwmApplication, LocalDB localDB, AuditSettings settings )
+    void init( PwmApplication pwmApplication, SessionLabel sessionLabel, LocalDB localDB, AuditSettings settings )
             throws PwmException;
 
     void close( );

server/src/main/java/password/pwm/svc/event/LocalDbAuditVault.java

@@ -21,6 +21,7 @@
 package password.pwm.svc.event;
 
 import password.pwm.PwmApplication;
+import password.pwm.bean.SessionLabel;
 import password.pwm.error.PwmException;
 import password.pwm.svc.PwmService;
 import password.pwm.util.PwmScheduler;
@@ -35,7 +36,7 @@ import password.pwm.util.logging.PwmLogger;
 
 import java.time.Instant;
 import java.util.Iterator;
-import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
 
 public class LocalDbAuditVault implements AuditVault
 {
@@ -45,7 +46,7 @@ public class LocalDbAuditVault implements AuditVault
     private AuditSettings settings;
     private Instant oldestRecord;
 
-    private ExecutorService executorService;
+    private ScheduledExecutorService executorService;
     private volatile PwmService.STATUS status = PwmService.STATUS.CLOSED;
 
 
@@ -58,6 +59,7 @@ public class LocalDbAuditVault implements AuditVault
     @Override
     public void init(
             final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
             final LocalDB localDB,
             final AuditSettings settings
     )
@@ -68,7 +70,7 @@ public class LocalDbAuditVault implements AuditVault
 
         readOldestRecord();
 
-        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        executorService = PwmScheduler.makeBackgroundServiceExecutor( pwmApplication, sessionLabel, this.getClass() );
 
         status = PwmService.STATUS.OPEN;
         final TimeDuration jobFrequency = TimeDuration.of( 10, TimeDuration.Unit.MINUTES );

server/src/main/java/password/pwm/svc/event/SyslogAuditService.java

@@ -40,6 +40,7 @@ import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.PwmConstants;
 import password.pwm.bean.DomainID;
+import password.pwm.bean.SessionLabel;
 import password.pwm.config.AppConfig;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.option.SyslogOutputFormat;
@@ -89,9 +90,10 @@ public class SyslogAuditService
     private final AppConfig appConfig;
     private final PwmApplication pwmApplication;
     private final AuditFormatter auditFormatter;
+    private final SessionLabel sessionLabel;
 
 
-    SyslogAuditService( final PwmApplication pwmApplication )
+    SyslogAuditService( final PwmApplication pwmApplication, final SessionLabel sessionLabel )
             throws LocalDBException
     {
         this.pwmApplication = pwmApplication;
@@ -100,6 +102,7 @@ public class SyslogAuditService
         this.syslogInstances = makeSyslogIFs( appConfig );
         this.auditFormatter = makeAuditFormatter( appConfig );
         this.workQueueProcessor = makeWorkQueueProcessor( pwmApplication, appConfig );
+        this.sessionLabel = sessionLabel;
     }
 
     private WorkQueueProcessor<String> makeWorkQueueProcessor(
@@ -117,7 +120,7 @@ public class SyslogAuditService
         final LocalDBStoredQueue localDBStoredQueue = LocalDBStoredQueue.createLocalDBStoredQueue(
                 pwmApplication, pwmApplication.getLocalDB(), LocalDB.DB.SYSLOG_QUEUE );
 
-        return new WorkQueueProcessor<>( pwmApplication, localDBStoredQueue, settings, new SyslogItemProcessor(), this.getClass() );
+        return new WorkQueueProcessor<>( pwmApplication, sessionLabel, localDBStoredQueue, settings, new SyslogItemProcessor(), this.getClass() );
     }
 
     private static AuditFormatter makeAuditFormatter( final AppConfig appConfig )

server/src/main/java/password/pwm/svc/intruder/IntruderSystemService.java

@@ -37,7 +37,6 @@ import password.pwm.health.HealthRecord;
 import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
 import password.pwm.util.DataStore;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.ClosableIterator;
 import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.TimeDuration;
@@ -45,7 +44,6 @@ import password.pwm.util.logging.PwmLogger;
 
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.ExecutorService;
 import java.util.stream.Collectors;
 
 public class IntruderSystemService extends AbstractPwmService implements PwmService
@@ -55,8 +53,6 @@ public class IntruderSystemService extends AbstractPwmService implements PwmServ
     private IntruderRecordStore recordStore;
     private DataStorageMethod dataStorageMethod;
 
-    private ExecutorService executorService;
-
     @Override
     public STATUS postAbstractInit( final PwmApplication pwmApplication, final DomainID domainID ) throws PwmException
     {
@@ -68,8 +64,6 @@ public class IntruderSystemService extends AbstractPwmService implements PwmServ
 
             recordStore = new IntruderDataStore( this, dataStore, this::status );
 
-            executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-
             scheduleCleaner();
         }
         catch ( final Exception e )
@@ -176,7 +170,7 @@ public class IntruderSystemService extends AbstractPwmService implements PwmServ
             }
         };
 
-        getPwmApplication().getPwmScheduler().scheduleFixedRateJob( cleanerJob, executorService, TimeDuration.SECONDS_10, cleanerRunFrequency );
+        getPwmApplication().getPwmScheduler().scheduleFixedRateJob( cleanerJob, getExecutorService(), TimeDuration.SECONDS_10, cleanerRunFrequency );
     }
 
     IntruderRecordStore getRecordStore()

server/src/main/java/password/pwm/svc/node/NodeMachine.java

@@ -26,8 +26,6 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.util.PwmScheduler;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
@@ -37,14 +35,13 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
 
 class NodeMachine
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( NodeMachine.class );
 
     private final PwmApplication pwmApplication;
-    private final ExecutorService executorService;
     private final NodeDataServiceProvider clusterDataServiceProvider;
 
     private ErrorInformation lastError;
@@ -56,6 +53,7 @@ class NodeMachine
 
     NodeMachine(
             final PwmApplication pwmApplication,
+            final ScheduledExecutorService executorService,
             final NodeDataServiceProvider clusterDataServiceProvider,
             final NodeServiceSettings nodeServiceSettings
     )
@@ -64,14 +62,11 @@ class NodeMachine
         this.clusterDataServiceProvider = clusterDataServiceProvider;
         this.settings = nodeServiceSettings;
 
-        this.executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, NodeMachine.class );
-
         pwmApplication.getPwmScheduler().scheduleFixedRateJob( new HeartbeatProcess(), executorService, settings.getHeartbeatInterval(), settings.getHeartbeatInterval() );
     }
 
     public void close( )
     {
-        JavaHelper.closeAndWaitExecutor( executorService, TimeDuration.SECOND );
     }
 
 

server/src/main/java/password/pwm/svc/node/NodeService.java

@@ -94,7 +94,7 @@ public class NodeService extends AbstractPwmService implements PwmService
 
                 }
 
-                nodeMachine = new NodeMachine( pwmApplication, clusterDataServiceProvider, nodeServiceSettings );
+                nodeMachine = new NodeMachine( pwmApplication, getExecutorService(), clusterDataServiceProvider, nodeServiceSettings );
             }
         }
         catch ( final PwmUnrecoverableException e )

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyEngine.java

@@ -56,10 +56,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class PwNotifyEngine
@@ -382,16 +379,10 @@ public class PwNotifyEngine
 
     private ThreadPoolExecutor createExecutor( final PwmDomain pwmDomain )
     {
-        final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( pwmDomain.getPwmApplication(), this.getClass() ), true );
-        final ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(
-                1,
+        return PwmScheduler.makeMultiThreadExecutor(
                 10,
-                1,
-                TimeUnit.MINUTES,
-                new LinkedBlockingDeque<>(),
-                threadFactory
-        );
-        threadPoolExecutor.allowCoreThreadTimeOut( true );
-        return threadPoolExecutor;
+                pwmDomain.getPwmApplication(),
+                pwNotifyService.getSessionLabel(),
+                PwNotifyEngine.class );
     }
 }

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyService.java

@@ -38,7 +38,6 @@ import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsClient;
 import password.pwm.util.PwmScheduler;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.MiscUtil;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
@@ -51,13 +50,11 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.ExecutorService;
 
 public class PwNotifyService extends AbstractPwmService implements PwmService
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwNotifyService.class );
 
-    private ExecutorService executorService;
     private PwmDomain pwmDomain;
     private PwNotifyEngine engine;
     private PwNotifySettings settings;
@@ -141,11 +138,9 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
                     MiscUtil.unhandledSwitchStatement( storageMethod );
             }
 
-            executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-
             engine = new PwNotifyEngine( this, pwmDomain, storageService, null );
 
-            pwmDomain.getPwmApplication().getPwmScheduler().scheduleFixedRateJob( new PwNotifyJob(), executorService, TimeDuration.MINUTE, TimeDuration.MINUTE );
+            pwmDomain.getPwmApplication().getPwmScheduler().scheduleFixedRateJob( new PwNotifyJob(), getExecutorService(), TimeDuration.MINUTE, TimeDuration.MINUTE );
         }
         catch ( final PwmUnrecoverableException e )
         {
@@ -212,7 +207,6 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
     public void shutdownImpl( )
     {
         setStatus( STATUS.CLOSED );
-        JavaHelper.closeAndWaitExecutor( executorService, TimeDuration.of( 5, TimeDuration.Unit.SECONDS ) );
     }
 
     @Override
@@ -262,7 +256,7 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
         if ( !isRunning() )
         {
             nextExecutionTime = Instant.now();
-            pwmDomain.getPwmApplication().getPwmScheduler().scheduleJob( new PwNotifyJob(), executorService, TimeDuration.ZERO );
+            pwmDomain.getPwmApplication().getPwmScheduler().scheduleJob( new PwNotifyJob(), getExecutorService(), TimeDuration.ZERO );
         }
     }
 

server/src/main/java/password/pwm/svc/report/ReportService.java

@@ -47,8 +47,8 @@ import password.pwm.util.java.BlockingThreadPool;
 import password.pwm.util.java.ClosableIterator;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.MiscUtil;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBException;
 import password.pwm.util.localdb.LocalDBStoredQueue;
@@ -63,7 +63,6 @@ import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Queue;
-import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
@@ -78,7 +77,6 @@ public class ReportService extends AbstractPwmService implements PwmService
 
     private final AtomicBoolean cancelFlag = new AtomicBoolean( false );
     private ReportSummaryData summaryData = ReportSummaryData.newSummaryData( null );
-    private ExecutorService executorService;
 
     private ReportRecordLocalDBStorageService userCacheService;
     private ReportSettings settings = ReportSettings.builder().build();
@@ -119,11 +117,9 @@ public class ReportService extends AbstractPwmService implements PwmService
 
         dnQueue = LocalDBStoredQueue.createLocalDBStoredQueue( pwmApplication, getPwmApplication().getLocalDB(), LocalDB.DB.REPORT_QUEUE );
 
-        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-
         if ( !pwmApplication.getPwmEnvironment().isInternalRuntimeInstance() )
         {
-            executorService.submit( new InitializationTask() );
+            getExecutorService().submit( new InitializationTask() );
         }
 
         return STATUS.OPEN;
@@ -135,14 +131,10 @@ public class ReportService extends AbstractPwmService implements PwmService
         setStatus( STATUS.CLOSED );
         cancelFlag.set( true );
 
-        JavaHelper.closeAndWaitExecutor( executorService, TimeDuration.SECONDS_10 );
-
         if ( userCacheService != null )
         {
             userCacheService.shutdown();
         }
-
-        executorService = null;
     }
 
     private void writeReportStatus( )
@@ -186,8 +178,8 @@ public class ReportService extends AbstractPwmService implements PwmService
                         && localReportStatus.getCurrentProcess() != ReportStatusInfo.ReportEngineProcess.SearchLDAP
                 )
                 {
-                    executorService.execute( new ClearTask() );
-                    executorService.execute( new ReadLDAPTask() );
+                    getExecutorService().execute( new ClearTask() );
+                    getExecutorService().execute( new ReadLDAPTask() );
                 }
             }
             break;
@@ -202,7 +194,7 @@ public class ReportService extends AbstractPwmService implements PwmService
             case Clear:
             {
                 cancelFlag.set( true );
-                executorService.execute( new ClearTask() );
+                getExecutorService().execute( new ClearTask() );
             }
             break;
 
@@ -324,7 +316,7 @@ public class ReportService extends AbstractPwmService implements PwmService
             try
             {
                 readUserListFromLdap();
-                executorService.execute( new ProcessWorkQueueTask() );
+                getExecutorService().execute( new ProcessWorkQueueTask() );
             }
             catch ( final Exception e )
             {
@@ -333,11 +325,11 @@ public class ReportService extends AbstractPwmService implements PwmService
                 {
                     if ( ( ( PwmException ) e ).getErrorInformation().getError() == PwmError.ERROR_DIRECTORY_UNAVAILABLE )
                     {
-                        if ( executorService != null )
+                        if ( getExecutorService() != null )
                         {
                             LOGGER.error( getSessionLabel(),
                                     () -> "directory unavailable error during background SearchLDAP, will retry; error: " + e.getMessage() );
-                            getPwmApplication().getPwmScheduler().scheduleJob( new ReadLDAPTask(), executorService, TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
+                            getPwmApplication().getPwmScheduler().scheduleJob( new ReadLDAPTask(), getExecutorService(), TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
                             errorProcessed = true;
                         }
                     }
@@ -447,11 +439,11 @@ public class ReportService extends AbstractPwmService implements PwmService
             {
                 if ( e.getErrorInformation().getError() == PwmError.ERROR_DIRECTORY_UNAVAILABLE )
                 {
-                    if ( executorService != null )
+                    if ( getExecutorService() != null )
                     {
                         LOGGER.error( getSessionLabel(), () -> "directory unavailable error during background ReadData, will retry; error: " + e.getMessage() );
                         getPwmApplication().getPwmScheduler().scheduleJob(
-                                new ProcessWorkQueueTask(), executorService, TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
+                                new ProcessWorkQueueTask(), getExecutorService(), TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
                     }
                 }
                 else
@@ -491,7 +483,7 @@ public class ReportService extends AbstractPwmService implements PwmService
             try
             {
                 LOGGER.trace( getSessionLabel(), () -> "about to begin ldap processing with thread count of " + threadCount );
-                final String threadName = PwmScheduler.makeThreadName( getPwmApplication(), this.getClass() );
+                final String threadName = PwmScheduler.makeThreadName( getSessionLabel(), getPwmApplication(), this.getClass() );
                 final BlockingThreadPool threadService = new BlockingThreadPool( threadCount, threadName );
                 while ( status() == STATUS.OPEN && !dnQueue.isEmpty() && !cancelFlag.get() )
                 {
@@ -643,8 +635,8 @@ public class ReportService extends AbstractPwmService implements PwmService
 
             if ( settings.isDailyJobEnabled() )
             {
-                executorService.execute( new ClearTask() );
-                executorService.execute( new ReadLDAPTask() );
+                getExecutorService().execute( new ClearTask() );
+                getExecutorService().execute( new ReadLDAPTask() );
             }
         }
     }
@@ -674,7 +666,7 @@ public class ReportService extends AbstractPwmService implements PwmService
             if ( reportingEnabled )
             {
                 final TimeDuration jobOffset = TimeDuration.of( settings.getJobOffsetSeconds(), TimeDuration.Unit.SECONDS );
-                getPwmApplication().getPwmScheduler().scheduleDailyZuluZeroStartJob( new DailyJobExecuteTask(), executorService, jobOffset );
+                getPwmApplication().getPwmScheduler().scheduleDailyZuluZeroStartJob( new DailyJobExecuteTask(), getExecutorService(), jobOffset );
             }
         }
 
@@ -729,7 +721,7 @@ public class ReportService extends AbstractPwmService implements PwmService
             if ( !reportStatus.get().isReportComplete() && !dnQueue.isEmpty() )
             {
                 LOGGER.trace( getSessionLabel(), () -> "resuming report data processing" );
-                executorService.execute( new ProcessWorkQueueTask() );
+                getExecutorService().execute( new ProcessWorkQueueTask() );
             }
         }
     }
@@ -792,7 +784,7 @@ public class ReportService extends AbstractPwmService implements PwmService
         final Instant lastFinishDate = reportStatus.get().getFinishDate();
         if ( lastFinishDate != null && TimeDuration.fromCurrent( lastFinishDate ).isLongerThan( settings.getMaxCacheAge() ) )
         {
-            executorService.execute( new ClearTask() );
+            getExecutorService().execute( new ClearTask() );
         }
     }
 }

server/src/main/java/password/pwm/svc/sms/SmsQueueService.java

@@ -164,7 +164,7 @@ public class SmsQueueService extends AbstractPwmService implements PwmService
 
         final LocalDBStoredQueue localDBStoredQueue = LocalDBStoredQueue.createLocalDBStoredQueue( pwmApplication, pwmApplication.getLocalDB(), LocalDB.DB.SMS_QUEUE );
 
-        workQueueProcessor = new WorkQueueProcessor<>( pwmApplication, localDBStoredQueue, settings, new SmsItemProcessor(), this.getClass() );
+        workQueueProcessor = new WorkQueueProcessor<>( pwmApplication, getSessionLabel(), localDBStoredQueue, settings, new SmsItemProcessor(), this.getClass() );
 
         smsSendEngine = new SmsSendEngine( pwmApplication, pwmApplication.getConfig() );
 

server/src/main/java/password/pwm/svc/stats/StatisticsService.java

@@ -29,10 +29,9 @@ import password.pwm.error.PwmException;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
+import password.pwm.util.DailySummaryJob;
 import password.pwm.util.EventRateMeter;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.CollectionUtil;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.MiscUtil;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
@@ -55,7 +54,6 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.TimerTask;
-import java.util.concurrent.ExecutorService;
 import java.util.stream.Collectors;
 
 public class StatisticsService extends AbstractPwmService implements PwmService
@@ -80,8 +78,6 @@ public class StatisticsService extends AbstractPwmService implements PwmService
     private DailyKey currentDailyKey = DailyKey.forToday();
     private DailyKey initialDailyKey = DailyKey.forToday();
 
-    private ExecutorService executorService;
-
     private final StatisticsBundle statsCurrent = new StatisticsBundle();
     private StatisticsBundle statsDaily = new StatisticsBundle();
     private StatisticsBundle statsCummulative = new StatisticsBundle();
@@ -274,9 +270,9 @@ public class StatisticsService extends AbstractPwmService implements PwmService
 
         {
             // setup a timer to roll over at 0 Zulu and one to write current stats regularly
-            executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new FlushTask(), executorService, DB_WRITE_FREQUENCY, DB_WRITE_FREQUENCY );
-            pwmApplication.getPwmScheduler().scheduleDailyZuluZeroStartJob( new NightlyTask(), executorService, TimeDuration.ZERO );
+            pwmApplication.getPwmScheduler().scheduleDailyZuluZeroStartJob( new DailySummaryJob( pwmApplication ), getExecutorService(), TimeDuration.ZERO );
+            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new FlushTask(), getExecutorService(), DB_WRITE_FREQUENCY, DB_WRITE_FREQUENCY );
+            pwmApplication.getPwmScheduler().scheduleDailyZuluZeroStartJob( new NightlyTask(), getExecutorService(), TimeDuration.ZERO );
         }
 
         return STATUS.OPEN;
@@ -328,8 +324,6 @@ public class StatisticsService extends AbstractPwmService implements PwmService
             LOGGER.error( () -> "unexpected error closing: " + e.getMessage() );
         }
 
-        JavaHelper.closeAndWaitExecutor( executorService, TimeDuration.of( 3, TimeDuration.Unit.SECONDS ) );
-
         setStatus( STATUS.CLOSED );
     }
 

server/src/main/java/password/pwm/svc/telemetry/TelemetryService.java

@@ -47,11 +47,10 @@ import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsBundle;
 import password.pwm.svc.stats.StatisticsService;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.CollectionUtil;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroRequest;
@@ -69,14 +68,12 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
-import java.util.concurrent.ExecutorService;
 import java.util.stream.Collectors;
 
 public class TelemetryService extends AbstractPwmService implements PwmService
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( TelemetryService.class );
 
-    private ExecutorService executorService;
     private Settings settings;
 
     private Instant lastPublishTime;
@@ -126,8 +123,6 @@ public class TelemetryService extends AbstractPwmService implements PwmService
                 .orElseGet( pwmApplication::getInstallTime );
         LOGGER.trace( getSessionLabel(), () -> "last publish time was " + StringUtil.toIsoDate( lastPublishTime ) );
 
-        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, TelemetryService.class );
-
         scheduleNextJob();
 
         return STATUS.OPEN;
@@ -197,7 +192,7 @@ public class TelemetryService extends AbstractPwmService implements PwmService
     private void scheduleNextJob( )
     {
         final TimeDuration durationUntilNextPublish = durationUntilNextPublish();
-        getPwmApplication().getPwmScheduler().scheduleJob( new PublishJob(), executorService, durationUntilNextPublish );
+        getPwmApplication().getPwmScheduler().scheduleJob( new PublishJob(), getExecutorService(), durationUntilNextPublish );
         LOGGER.trace( getSessionLabel(), () -> "next publish time: " + durationUntilNextPublish().asCompactString() );
     }
 

server/src/main/java/password/pwm/svc/token/TokenService.java

@@ -65,12 +65,11 @@ import password.pwm.svc.sms.SmsQueueService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsClient;
 import password.pwm.util.DataStore;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.MiscUtil;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.StatisticCounterBundle;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBDataStore;
 import password.pwm.util.logging.PwmLogger;
@@ -83,8 +82,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.TimerTask;
-import java.util.concurrent.ExecutorService;
 
 /**
  * This PWM service is responsible for reading/writing tokens used for forgotten password,
@@ -95,11 +92,8 @@ import java.util.concurrent.ExecutorService;
  */
 public class TokenService extends AbstractPwmService implements PwmService
 {
-
     private static final PwmLogger LOGGER = PwmLogger.forClass( TokenService.class );
 
-    private ExecutorService executorService;
-
     private PwmDomain pwmDomain;
     private DomainConfig domainConfig;
     private TokenStorageMethod storageMethod;
@@ -220,15 +214,6 @@ public class TokenService extends AbstractPwmService implements PwmService
 
         verifyPwModifyTime = Boolean.parseBoolean( domainConfig.readAppProperty( AppProperty.TOKEN_VERIFY_PW_MODIFY_TIME ) );
 
-        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-
-        {
-            final int cleanerFrequencySeconds = Integer.parseInt( domainConfig.readAppProperty( AppProperty.TOKEN_CLEANER_INTERVAL_SECONDS ) );
-            final TimeDuration cleanerFrequency = TimeDuration.of( cleanerFrequencySeconds, TimeDuration.Unit.SECONDS );
-            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new CleanerTask(), executorService, TimeDuration.MINUTE, cleanerFrequency );
-            LOGGER.trace( getSessionLabel(), () -> "token cleanup will occur every " + cleanerFrequency.asCompactString() );
-        }
-
         LOGGER.debug( getSessionLabel(), () -> "open" );
 
         return STATUS.OPEN;
@@ -346,10 +331,6 @@ public class TokenService extends AbstractPwmService implements PwmService
     public void shutdownImpl( )
     {
         setStatus( STATUS.CLOSED );
-        if ( executorService != null )
-        {
-            executorService.shutdown();
-        }
     }
 
     @Override
@@ -419,19 +400,15 @@ public class TokenService extends AbstractPwmService implements PwmService
         return random.alphaNumericString( randomChars, codeLength );
     }
 
-    private class CleanerTask extends TimerTask
+    void cleanup()
     {
-        @Override
-        public void run( )
+        try
         {
-            try
-            {
-                tokenMachine.cleanup();
-            }
-            catch ( final Exception e )
-            {
-                LOGGER.warn( getSessionLabel(), () -> "unexpected error while cleaning expired stored tokens: " + e.getMessage() );
-            }
+            tokenMachine.cleanup();
+        }
+        catch ( final Exception e )
+        {
+            LOGGER.warn( getSessionLabel(), () -> "unexpected error while cleaning expired stored tokens: " + e.getMessage() );
         }
     }
 

server/src/main/java/password/pwm/svc/token/TokenSystemService.java

@@ -0,0 +1,83 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.svc.token;
+
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.PwmDomain;
+import password.pwm.bean.DomainID;
+import password.pwm.error.PwmException;
+import password.pwm.health.HealthRecord;
+import password.pwm.svc.AbstractPwmService;
+import password.pwm.svc.PwmService;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogger;
+
+import java.util.List;
+
+public class TokenSystemService extends AbstractPwmService implements PwmService
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( AbstractPwmService.class );
+
+    @Override
+    protected STATUS postAbstractInit( final PwmApplication pwmApplication, final DomainID domainID ) throws PwmException
+    {
+        {
+            final int cleanerFrequencySeconds = Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.TOKEN_CLEANER_INTERVAL_SECONDS ) );
+            final TimeDuration cleanerFrequency = TimeDuration.of( cleanerFrequencySeconds, TimeDuration.Unit.SECONDS );
+            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new CleanerTask(), getExecutorService(), TimeDuration.MINUTE, cleanerFrequency );
+            LOGGER.trace( getSessionLabel(), () -> "token cleanup will occur every " + cleanerFrequency.asCompactString() );
+        }
+
+        return STATUS.OPEN;
+    }
+
+    @Override
+    protected void shutdownImpl()
+    {
+
+    }
+
+    @Override
+    protected List<HealthRecord> serviceHealthCheck()
+    {
+        return null;
+    }
+
+    @Override
+    public ServiceInfoBean serviceInfo()
+    {
+        return null;
+    }
+
+    private class CleanerTask implements Runnable
+    {
+        @Override
+        public void run( )
+        {
+            for ( final PwmDomain pwmDomain : getPwmApplication().domains().values() )
+            {
+                pwmDomain.getTokenService().cleanup();
+            }
+        }
+    }
+
+}

server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java

@@ -36,11 +36,11 @@ import password.pwm.svc.PwmService;
 import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.ConditionalTaskExecutor;
 import password.pwm.util.java.JavaHelper;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.Percent;
 import password.pwm.util.java.PwmCallable;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.logging.PwmLogger;
 
 import java.io.InputStream;
@@ -51,7 +51,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.BooleanSupplier;
@@ -62,7 +62,7 @@ abstract class AbstractWordlist extends AbstractPwmService implements Wordlist,
 
     private WordlistConfiguration wordlistConfiguration;
     private WordlistBucket wordlistBucket;
-    private ExecutorService executorService;
+    private ScheduledExecutorService executorService;
     private volatile Set<WordType> wordTypesCache = null;
 
     private volatile ErrorInformation lastError;
@@ -121,7 +121,7 @@ abstract class AbstractWordlist extends AbstractPwmService implements Wordlist,
         }
 
         inhibitBackgroundImportFlag.set( false );
-        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        executorService = PwmScheduler.makeBackgroundServiceExecutor( pwmApplication, getSessionLabel(), this.getClass() );
 
         if ( !pwmApplication.getPwmEnvironment().isInternalRuntimeInstance() )
         {

server/src/main/java/password/pwm/svc/wordlist/SharedHistoryService.java

@@ -35,7 +35,6 @@ import password.pwm.error.PwmException;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.localdb.LocalDB;
@@ -51,7 +50,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.TimerTask;
-import java.util.concurrent.ExecutorService;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -67,16 +65,14 @@ public class SharedHistoryService extends AbstractPwmService implements PwmServi
     private static final String DATA_FORMAT_VERSION = "2";
 
     // 1 hour
-    private static final int MIN_CLEANER_FREQUENCY = 1000 * 60 * 60;
+    private static final TimeDuration MIN_CLEANER_FREQUENCY = TimeDuration.HOUR;
 
     // 1 day
-    private static final int MAX_CLEANER_FREQUENCY = 1000 * 60 * 60 * 24;
+    private static final TimeDuration MAX_CLEANER_FREQUENCY = TimeDuration.DAY;
 
     private static final LocalDB.DB META_DB = LocalDB.DB.SHAREDHISTORY_META;
     private static final LocalDB.DB WORDS_DB = LocalDB.DB.SHAREDHISTORY_WORDS;
 
-    private ExecutorService executorService;
-
     private LocalDB localDB;
     private String salt;
     private Instant oldestEntry;
@@ -92,10 +88,6 @@ public class SharedHistoryService extends AbstractPwmService implements PwmServi
     public void shutdownImpl( )
     {
         setStatus( STATUS.CLOSED );
-        if ( executorService != null )
-        {
-            executorService.shutdown();
-        }
         localDB = null;
     }
 
@@ -187,7 +179,7 @@ public class SharedHistoryService extends AbstractPwmService implements PwmServi
         }
     }
 
-    private void init( final PwmApplication pwmApplication, final TimeDuration maxAge )
+    private void initImpl( final PwmApplication pwmApplication, final TimeDuration maxAge )
     {
         final Instant startTime = Instant.now();
 
@@ -244,12 +236,11 @@ public class SharedHistoryService extends AbstractPwmService implements PwmServi
 
         if ( pwmApplication.getApplicationMode() == PwmApplicationMode.RUNNING || pwmApplication.getApplicationMode() == PwmApplicationMode.CONFIGURATION )
         {
-            final long frequencyMs = JavaHelper.rangeCheck( MIN_CLEANER_FREQUENCY, MAX_CLEANER_FREQUENCY, maxAge.asMillis() );
+            final long frequencyMs = JavaHelper.rangeCheck( MIN_CLEANER_FREQUENCY.asMillis(), MAX_CLEANER_FREQUENCY.asMillis(), maxAge.asMillis() );
             final TimeDuration frequency = TimeDuration.of( frequencyMs, TimeDuration.Unit.MILLISECONDS );
 
             LOGGER.debug( () -> "scheduling cleaner task to run once every " + frequency.asCompactString() );
-            executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new CleanerTask(), executorService, null, frequency );
+            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new CleanerTask(), getExecutorService(), TimeDuration.ZERO, frequency );
         }
     }
 
@@ -475,8 +466,10 @@ public class SharedHistoryService extends AbstractPwmService implements PwmServi
         pwmApplication.getPwmScheduler().immediateExecuteRunnableInNewThread( () ->
         {
             LOGGER.debug( getSessionLabel(), () -> "starting up in background thread" );
-            init( pwmApplication, settings.getMaxAge() );
-        }, "shared history initializer" );
+            initImpl( pwmApplication, settings.getMaxAge() );
+        },
+                getSessionLabel(),
+                "shared history initializer" );
 
         return STATUS.OPEN;
     }

server/src/main/java/password/pwm/util/DailySummaryJob.java

@@ -24,6 +24,7 @@ import lombok.Builder;
 import lombok.Value;
 import org.apache.commons.text.WordUtils;
 import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
 import password.pwm.PwmDomain;
 import password.pwm.PwmConstants;
 import password.pwm.bean.EmailItemBean;
@@ -53,13 +54,11 @@ public class DailySummaryJob implements Runnable
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( DailySummaryJob.class );
 
-    private final PwmDomain pwmDomain;
-    private final DailySummaryJobSettings settings;
+    private final PwmApplication pwmApplication;
 
-    public DailySummaryJob( final PwmDomain pwmDomain )
+    public DailySummaryJob( final PwmApplication pwmDomain )
     {
-        this.pwmDomain = pwmDomain;
-        this.settings = DailySummaryJobSettings.fromConfig( pwmDomain.getConfig() );
+        this.pwmApplication = pwmDomain;
     }
 
     @Value
@@ -87,22 +86,27 @@ public class DailySummaryJob implements Runnable
     @Override
     public void run()
     {
-        try
+        for ( final PwmDomain pwmDomain : pwmApplication.domains().values() )
         {
-            alertDailyStats();
-        }
-        catch ( final Exception e )
-        {
-            LOGGER.error( () -> "error while generating daily alert statistics: " + e.getMessage() );
-        }
+            final DailySummaryJobSettings dailySummaryJobSettings = DailySummaryJobSettings.fromConfig( pwmDomain.getConfig() );
+            try
+            {
+                alertDailyStats( pwmDomain, dailySummaryJobSettings );
+            }
+            catch ( final Exception e )
+            {
+                LOGGER.error( () -> "error while generating daily alert statistics: " + e.getMessage() );
+            }
+    }
     }
 
-    private void alertDailyStats(
-
+    private static void alertDailyStats(
+        final PwmDomain pwmDomain,
+        final DailySummaryJobSettings settings
     )
             throws PwmUnrecoverableException
     {
-        if ( !checkIfEnabled( pwmDomain ) )
+        if ( !checkIfEnabled( pwmDomain, settings ) )
         {
             LOGGER.trace( () -> "skipping daily summary alert job, setting "
                     + PwmSetting.EVENTS_ALERT_DAILY_SUMMARY.toMenuLocationDebug( null, PwmConstants.DEFAULT_LOCALE )
@@ -126,15 +130,16 @@ public class DailySummaryJob implements Runnable
             final String subject = Display.getLocalizedMessage( locale, Display.Title_Application, pwmDomain.getConfig() ) + " - Daily Summary";
             final StringBuilder textBody = new StringBuilder();
             final StringBuilder htmlBody = new StringBuilder();
-            makeEmailBody( pwmDomain, dailyStatistics, locale, textBody, htmlBody );
+            makeEmailBody( pwmDomain, settings, dailyStatistics, locale, textBody, htmlBody );
             final EmailItemBean emailItem = new EmailItemBean( toAddress, fromAddress, subject, textBody.toString(), htmlBody.toString() );
             LOGGER.debug( () -> "sending daily summary email to " + toAddress );
             pwmDomain.getPwmApplication().getEmailQueue().submitEmail( emailItem, null, MacroRequest.forNonUserSpecific( pwmDomain.getPwmApplication(), null ) );
         }
     }
 
-    private void makeEmailBody(
+    private static void makeEmailBody(
             final PwmDomain pwmDomain,
+            final DailySummaryJobSettings settings,
             final Map<String, String> dailyStatistics,
             final Locale locale,
             final StringBuilder textBody,
@@ -266,7 +271,7 @@ public class DailySummaryJob implements Runnable
 
     }
 
-    private boolean checkIfEnabled( final PwmDomain pwmDomain )
+    private static boolean checkIfEnabled( final PwmDomain pwmDomain, final DailySummaryJobSettings settings )
     {
         if ( pwmDomain == null )
         {

server/src/main/java/password/pwm/util/PwmScheduler.java

@@ -20,11 +20,10 @@
 
 package password.pwm.util;
 
-import org.jetbrains.annotations.NotNull;
 import password.pwm.PwmApplication;
 import password.pwm.PwmConstants;
+import password.pwm.bean.SessionLabel;
 import password.pwm.error.PwmError;
-import password.pwm.error.PwmInternalException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.StringUtil;
@@ -46,85 +45,58 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 public class PwmScheduler
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwmScheduler.class );
     private static final AtomicLoopIntIncrementer THREAD_ID_COUNTER = new AtomicLoopIntIncrementer();
 
-    private final ScheduledExecutorService applicationExecutorService;
     private final PwmApplication pwmApplication;
 
     public PwmScheduler( final PwmApplication pwmApplication )
     {
         this.pwmApplication = Objects.requireNonNull( pwmApplication );
-        applicationExecutorService = makeSingleThreadExecutorService( pwmApplication.getInstanceID(), this.getClass() );
     }
 
     public void shutdown()
     {
-        applicationExecutorService.shutdown();
     }
 
     public void immediateExecuteRunnableInNewThread(
             final Runnable runnable,
-            final String threadName
-    )
-    {
-        immediateExecuteCallableInNewThread( Executors.callable( runnable ), threadName );
-    }
-
-    public <V> Future<V> immediateExecuteCallableInNewThread(
-            final Callable<V> callable,
+            final SessionLabel sessionLabel,
             final String threadName
     )
     {
         checkIfSchedulerClosed();
 
-        Objects.requireNonNull( callable );
-
-        final String name = "runtime thread #" + THREAD_ID_COUNTER.next() + " " + threadName;
-
-        final ScheduledExecutorService executor = makeSingleThreadExecutorService( pwmApplication.getInstanceID(), callable.getClass() );
+        Objects.requireNonNull( runnable );
 
-        final Callable<V> runnableWrapper = () ->
-        {
-            final Instant itemStartTime = Instant.now();
-            LOGGER.trace( () -> "started " + name );
-            try
-            {
-                final V result = callable.call();
-                LOGGER.trace( () -> "completed " + name, () -> TimeDuration.fromCurrent( itemStartTime ) );
-                executor.shutdown();
-                return result;
-            }
-            catch ( final Exception e )
-            {
-                LOGGER.error( () -> "error running scheduled immediate task: " + name + ", error: " + e.getMessage(), e );
-                throw e;
-            }
-        };
+        final ExecutorService executor = makeMultiThreadExecutor( 1, pwmApplication, sessionLabel, runnable.getClass() );
 
-        return executor.submit( runnableWrapper );
+        executor.submit( runnable );
     }
 
     public void scheduleDailyZuluZeroStartJob(
             final Runnable runnable,
-            final ExecutorService executorService,
-            final TimeDuration offset
+            final ScheduledExecutorService executorService,
+            final TimeDuration zuluOffset
     )
     {
         final TimeDuration delayTillNextZulu = TimeDuration.fromCurrent( nextZuluZeroTime() );
-        final TimeDuration delayTillNextOffset = delayTillNextZulu.add( offset );
+        final TimeDuration delayTillNextOffset = zuluOffset == null ? TimeDuration.ZERO : delayTillNextZulu.add( zuluOffset );
         scheduleFixedRateJob( runnable, executorService, delayTillNextOffset, TimeDuration.DAY );
     }
 
-    public Future<?> scheduleJob(
+    public ScheduledFuture<?> scheduleJob(
             final Runnable runnable,
-            final ExecutorService executor,
+            final ScheduledExecutorService executor,
             final TimeDuration delay
     )
     {
@@ -134,29 +106,25 @@ public class PwmScheduler
         Objects.requireNonNull( executor );
         Objects.requireNonNull( delay );
 
-        if ( applicationExecutorService.isShutdown() )
-        {
-            throw new IllegalStateException( "can not schedule job with shutdown scheduler" );
-        }
-
-        final FutureRunner<Object> wrappedRunner = new FutureRunner<>( runnable, executor );
-        applicationExecutorService.schedule( wrappedRunner, delay.asMillis(), TimeUnit.MILLISECONDS );
-        return wrappedRunner.getFuture();
+        return executor.schedule( runnable, delay.asMillis(), TimeUnit.MILLISECONDS );
     }
 
     public <T> List<T> executeImmediateThreadPerJobAndAwaitCompletion(
-            final List<Callable<T>> runnableList,
-            final String threadNames
+            final int maxThreadCount,
+            final List<Callable<T>> callables,
+            final SessionLabel sessionLabel,
+            final Class<?> theClass
     )
             throws PwmUnrecoverableException
     {
         checkIfSchedulerClosed();
 
-        final List<Future<T>> futures = new ArrayList<>( runnableList.size() );
-        for ( final Callable<T> callable : runnableList )
-        {
-            futures.add( this.immediateExecuteCallableInNewThread( callable, threadNames ) );
-        }
+        final ExecutorService executor = makeMultiThreadExecutor( maxThreadCount, pwmApplication, sessionLabel, theClass );
+
+        final List<Future<T>> futures = callables.stream()
+                .map( executor::submit )
+                .collect( Collectors.toUnmodifiableList() );
+
 
         final List<T> results = new ArrayList<>();
         for ( final Future<T> f : futures )
@@ -164,6 +132,7 @@ public class PwmScheduler
             results.add( awaitFutureCompletion( f ) );
         }
 
+        executor.shutdown();
         return Collections.unmodifiableList( results );
     }
 
@@ -195,48 +164,20 @@ public class PwmScheduler
 
     public void scheduleFixedRateJob(
             final Runnable runnable,
-            final ExecutorService executor,
+            final ScheduledExecutorService executor,
             final TimeDuration initialDelay,
             final TimeDuration frequency
     )
     {
         checkIfSchedulerClosed();
 
-
-        if ( initialDelay != null )
-        {
-            applicationExecutorService.schedule( new FutureRunner( runnable, executor ), initialDelay.asMillis(), TimeUnit.MILLISECONDS );
-        }
-
-        final Runnable jobWithNextScheduler = () ->
-        {
-            new FutureRunner( runnable, executor ).run();
-            scheduleFixedRateJob( runnable, executor, null, frequency );
-        };
-
-        applicationExecutorService.schedule(  jobWithNextScheduler, frequency.asMillis(), TimeUnit.MILLISECONDS );
+        executor.scheduleAtFixedRate( runnable, initialDelay.asMillis(), frequency.asMillis(), TimeUnit.MILLISECONDS );
     }
 
-    public static ExecutorService makeBackgroundExecutor(
+    public static String makeThreadName(
+            final SessionLabel sessionLabel,
             final PwmApplication pwmApplication,
-            final Class<?> clazz
-    )
-    {
-        final ThreadPoolExecutor executor = new ThreadPoolExecutor(
-                1,
-                1,
-                10, TimeUnit.SECONDS,
-                new LinkedBlockingQueue<>(),
-                makePwmThreadFactory(
-                        makeThreadName( pwmApplication, clazz ) + "-",
-                        true
-                ) );
-        executor.allowCoreThreadTimeOut( true );
-        return executor;
-    }
-
-
-    public static String makeThreadName( final PwmApplication pwmApplication, final Class<?> theClass )
+            final Class<?> theClass )
     {
         String instanceName = "-";
         if ( pwmApplication != null )
@@ -244,18 +185,37 @@ public class PwmScheduler
             instanceName = pwmApplication.getInstanceID();
         }
 
-        return makeThreadName( instanceName, theClass );
+        return makeThreadName( sessionLabel, instanceName, theClass );
     }
 
-    public static String makeThreadName( final String instanceID, final Class<?> theClass )
+    public static String makeThreadName(
+            final SessionLabel sessionLabel,
+            final String instanceID,
+            final Class<?> theClass )
     {
-        String instanceName = "-";
+        final StringBuilder output = new StringBuilder();
+
+        output.append( PwmConstants.PWM_APP_NAME );
+
         if ( StringUtil.notEmpty( instanceID ) )
         {
-            instanceName = instanceID;
+            output.append( "-" );
+            output.append( instanceID );
+        }
+
+        if ( theClass != null )
+        {
+            output.append( "-" );
+            output.append( theClass.getSimpleName() );
         }
 
-        return PwmConstants.PWM_APP_NAME + "-" + instanceName + "-" + theClass.getSimpleName();
+        if ( sessionLabel != null && !StringUtil.isEmpty( sessionLabel.getDomain() ) )
+        {
+            output.append( "-" );
+            output.append( sessionLabel.getDomain() );
+        }
+
+        return output.toString();
     }
 
     public static ThreadFactory makePwmThreadFactory( final String namePrefix, final boolean daemon )
@@ -279,108 +239,55 @@ public class PwmScheduler
         };
     }
 
-    public static ScheduledExecutorService makeSingleThreadExecutorService(
+    public static ThreadPoolExecutor makeMultiThreadExecutor(
+            final int maxThreadCount,
             final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
             final Class<?> theClass
     )
     {
-        return makeSingleThreadExecutorService( pwmApplication.getInstanceID(), theClass );
+        return makeMultiThreadExecutor( maxThreadCount, pwmApplication.getInstanceID(), sessionLabel, theClass );
     }
 
-    public static ScheduledExecutorService makeSingleThreadExecutorService(
+    public static ThreadPoolExecutor makeMultiThreadExecutor(
+            final int maxThreadCount,
             final String instanceID,
+            final SessionLabel sessionLabel,
             final Class<?> theClass
     )
     {
-        return Executors.newSingleThreadScheduledExecutor(
+        final ThreadPoolExecutor executor = new ThreadPoolExecutor(
+                1,
+                maxThreadCount,
+                1, TimeUnit.SECONDS,
+                new LinkedBlockingQueue<>(),
                 makePwmThreadFactory(
-                        makeThreadName( instanceID, theClass ) + "-",
+                        makeThreadName( sessionLabel, instanceID, theClass ) + "-",
                         true
                 ) );
+        executor.allowCoreThreadTimeOut( true );
+        return executor;
     }
 
-    private static class FutureRunner<T> implements Runnable
+    public static ScheduledExecutorService makeBackgroundServiceExecutor(
+            final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
+            final Class<?> clazz
+    )
     {
-        private final Runnable runnable;
-        private final ExecutorService executor;
-        private volatile Future<Object> innerFuture;
-        private volatile boolean hasFailed;
-
-        enum Flag
-        {
-            ShutdownExecutorAfterExecution,
-        }
-
-        FutureRunner( final Runnable runnable, final ExecutorService executor )
-        {
-            this.runnable = runnable;
-            this.executor = executor;
-        }
-
-        Future<T> getFuture()
-        {
-            return new Future<T>()
-            {
-                @Override
-                public boolean cancel( final boolean mayInterruptIfRunning )
-                {
-                    return false;
-                }
-
-                @Override
-                public boolean isCancelled()
-                {
-                    return hasFailed;
-                }
-
-                @Override
-                public boolean isDone()
-                {
-                    return hasFailed || ( innerFuture != null && innerFuture.isDone() );
-                }
-
-                @Override
-                public T get()
-                {
-                    return null;
-                }
-
-                @Override
-                public T get( final long timeout, @NotNull final TimeUnit unit )
-                {
-                    return null;
-                }
-            };
-        }
-
-        @Override
-        public void run()
-        {
-            try
-            {
-                if ( !executor.isShutdown() )
-                {
-                    innerFuture = executor.submit( Executors.callable( runnable ) );
-                }
-                else
-                {
-                    hasFailed = true;
-                }
-            }
-            catch ( final Throwable t )
-            {
-                LOGGER.error( () -> "unexpected error running scheduled job: " + t.getMessage(), t );
-                hasFailed = true;
-            }
-        }
+        final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(
+                1,
+                makePwmThreadFactory(
+                        makeThreadName( sessionLabel, pwmApplication, clazz ) + "-",
+                        true
+                ) );
+        executor.setKeepAliveTime( 1, TimeUnit.MINUTES );
+        executor.allowCoreThreadTimeOut( true );
+        return executor;
     }
 
     private void checkIfSchedulerClosed()
     {
-        if ( applicationExecutorService.isShutdown() )
-        {
-                throw new PwmInternalException( "scheduler is closed" );
-        }
     }
 
     public static Instant nextZuluZeroTime( )

server/src/main/java/password/pwm/util/localdb/WorkQueueProcessor.java

@@ -24,6 +24,7 @@ import com.google.gson.annotations.SerializedName;
 import lombok.Builder;
 import lombok.Value;
 import password.pwm.PwmApplication;
+import password.pwm.bean.SessionLabel;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmOperationalException;
@@ -102,10 +103,11 @@ public final class WorkQueueProcessor<W extends Serializable>
 
     public WorkQueueProcessor(
             final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
             final Deque<String> queue,
             final Settings settings,
             final ItemProcessor<W> itemProcessor,
-            final Class sourceClass
+            final Class<?> sourceClass
     )
     {
         this.settings = settings;
@@ -121,12 +123,12 @@ public final class WorkQueueProcessor<W extends Serializable>
 
         this.workerThread = new WorkerThread();
         workerThread.setDaemon( true );
-        workerThread.setName( PwmScheduler.makeThreadName( pwmApplication, sourceClass ) + "-worker-" );
+        workerThread.setName( PwmScheduler.makeThreadName( sessionLabel, pwmApplication, sourceClass ) + "-worker-" );
         workerThread.start();
 
         if ( settings.getPreThreads() > 0 )
         {
-            final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( pwmApplication, sourceClass ), true );
+            final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( sessionLabel, pwmApplication, sourceClass ), true );
             executorService = new ThreadPoolExecutor(
                     1,
                     settings.getPreThreads(),

server/src/main/java/password/pwm/util/logging/LocalDBLogger.java

@@ -150,13 +150,13 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
 
         cleanerService = Executors.newSingleThreadScheduledExecutor(
                 PwmScheduler.makePwmThreadFactory(
-                        PwmScheduler.makeThreadName( pwmApplication, this.getClass() ) + "-cleaner-",
+                        PwmScheduler.makeThreadName( getSessionLabel(), pwmApplication, this.getClass() ) + "-cleaner-",
                         true
                 ) );
 
         writerService = Executors.newSingleThreadScheduledExecutor(
                 PwmScheduler.makePwmThreadFactory(
-                        PwmScheduler.makeThreadName( pwmApplication, this.getClass() ) + "-writer-",
+                        PwmScheduler.makeThreadName( getSessionLabel(), pwmApplication, this.getClass() ) + "-writer-",
                         true
                 ) );
 

server/src/main/java/password/pwm/util/logging/PwmLogEvent.java

@@ -171,38 +171,14 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
 
     private String getDebugLabel( )
     {
-        final StringBuilder sb = new StringBuilder();
-        final String sessionID = getSessionID();
-        final String username = getUsername();
-
-        if ( StringUtil.notEmpty( sessionID ) )
-        {
-            sb.append( sessionID );
-        }
-        if ( StringUtil.notEmpty( domain ) )
-        {
-            if ( sb.length() > 0 )
-            {
-                sb.append( ',' );
-            }
-            sb.append( domain );
-        }
-        if ( StringUtil.notEmpty( username ) )
-        {
-            if ( sb.length() > 0 )
-            {
-                sb.append( ',' );
-            }
-            sb.append( username );
-        }
-
-        if ( sb.length() > 0 )
-        {
-            sb.insert( 0, "{" );
-            sb.append( "} " );
-        }
-
-        return sb.toString();
+        return SessionLabel.builder()
+                .sessionID( getSessionID() )
+                .requestID( getRequestID() )
+                .username( getUsername() )
+                .sourceAddress( getSourceAddress() )
+                .domain( getDomain() )
+                .build()
+                .toDebugLabel();
     }
 
     public String toLogString( )