PwmScheduler refactoring

server/src/main/java/password/pwm/PwmApplication.java

@@ -62,6 +62,7 @@ import password.pwm.svc.wordlist.WordlistService;
 import password.pwm.util.MBeanUtility;
 import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.FileSystemUtility;
+import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.json.JsonFactory;
@@ -268,6 +269,8 @@ public class PwmApplication
 
     private void postInitTasks()
     {
+        System.out.println( JavaHelper.stackTraceToString( new Throwable() ) );
+
         final Instant startTime = Instant.now();
 
         // send system audit event
@@ -282,15 +285,11 @@ public class PwmApplication
             LOGGER.warn( sessionLabel, () -> "error while clearing config manager-intruder-username from intruder table: " + e.getMessage() );
         }
 
-        if ( !pwmEnvironment.isInternalRuntimeInstance() )
-        {
-            PwmApplicationUtil.outputKeystore( this );
-            PwmApplicationUtil.outputTomcatConf( this );
-        }
+        PwmApplicationUtil.outputKeystore( this );
+        PwmApplicationUtil.outputTomcatConf( this );
 
-        if ( Boolean.parseBoolean( getConfig().readAppProperty( AppProperty.LOGGING_OUTPUT_CONFIGURATION ) )
-                && LOGGER.isEnabled( PwmLogLevel.TRACE )
-        )
+        if ( LOGGER.isEnabled( PwmLogLevel.TRACE )
+                && Boolean.parseBoolean( getConfig().readAppProperty( AppProperty.LOGGING_OUTPUT_CONFIGURATION ) ) )
         {
             PwmApplicationUtil.outputApplicationInfoToLog( this );
             PwmApplicationUtil.outputConfigurationToLog( this, DomainID.systemId() );
@@ -299,13 +298,9 @@ public class PwmApplication
 
         MBeanUtility.registerMBean( this );
 
-        getPwmScheduler().immediateExecuteRunnableInNewThread( () ->
-                {
-                    PwmSettingMetaDataReader.initCache();
-                    UserAgentUtils.initializeCache();
-                },
-                sessionLabel,
-                "initialize useragent cache" );
+        UserAgentUtils.initializeCache();
+
+        PwmSettingMetaDataReader.initCache();
 
         LOGGER.trace( sessionLabel, () -> "completed post init tasks", () -> TimeDuration.fromCurrent( startTime ) );
     }
@@ -389,20 +384,24 @@ public class PwmApplication
 
         MBeanUtility.unregisterMBean( this );
 
-        pwmServiceManager.shutdownAllServices();
-
         try
         {
             final List<Callable<Object>> callables = domains.values().stream()
                     .map( pwmDomain -> Executors.callable( pwmDomain::shutdown ) )
                     .collect( Collectors.toList() );
+
+            final Instant startDomainShutdown = Instant.now();
+            LOGGER.trace( sessionLabel, () -> "beginning shutdown of " + callables.size() + " running domains" );
             pwmScheduler.executeImmediateThreadPerJobAndAwaitCompletion( DOMAIN_STARTUP_THREADS, callables, sessionLabel, PwmApplication.class );
+            LOGGER.trace( sessionLabel, () -> "shutdown of " + callables.size() + " running domains completed", () -> TimeDuration.fromCurrent( startDomainShutdown ) );
         }
         catch ( final PwmUnrecoverableException e )
         {
             LOGGER.error( sessionLabel, () -> "error shutting down domain services: " + e.getMessage(), e );
         }
 
+        pwmServiceManager.shutdownAllServices();
+
         if ( localDBLogger != null )
         {
             try

server/src/main/java/password/pwm/PwmDomain.java

@@ -51,6 +51,7 @@ import password.pwm.svc.token.TokenService;
 import password.pwm.svc.userhistory.UserHistoryService;
 import password.pwm.svc.wordlist.SharedHistoryService;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogLevel;
 import password.pwm.util.logging.PwmLogger;
 
 import java.time.Instant;
@@ -93,7 +94,9 @@ public class PwmDomain
 
         pwmServiceManager.initAllServices();
 
-        if ( Boolean.parseBoolean( getConfig().readAppProperty( AppProperty.LOGGING_OUTPUT_CONFIGURATION ) ) )
+        if ( LOGGER.isEnabled( PwmLogLevel.TRACE )
+                && !pwmApplication.getPwmEnvironment().isInternalRuntimeInstance()
+                && Boolean.parseBoolean( getConfig().readAppProperty( AppProperty.LOGGING_OUTPUT_CONFIGURATION ) ) )
         {
             PwmApplicationUtil.outputConfigurationToLog( pwmApplication, domainID );
         }
@@ -244,8 +247,11 @@ public class PwmDomain
 
     public void shutdown()
     {
+        final Instant startTime = Instant.now();
         LOGGER.trace( sessionLabel, () -> "beginning shutdown domain " + domainID.stringValue() );
+        TimeDuration.SECONDS_10.pause();
         pwmServiceManager.shutdownAllServices();
+        LOGGER.trace( sessionLabel, () -> "shutdown domain " + domainID.stringValue() + " completed", () -> TimeDuration.fromCurrent( startTime ) );
     }
 
     public DomainID getDomainID()

server/src/main/java/password/pwm/http/HttpEventManager.java

@@ -35,6 +35,7 @@ import javax.servlet.ServletContextEvent;
 import javax.servlet.ServletContextListener;
 import javax.servlet.ServletRequestEvent;
 import javax.servlet.ServletRequestListener;
+import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpSession;
 import javax.servlet.http.HttpSessionActivationListener;
 import javax.servlet.http.HttpSessionEvent;
@@ -191,6 +192,16 @@ public class HttpEventManager implements
     @Override
     public void requestDestroyed( final ServletRequestEvent sre )
     {
+        try
+        {
+            final PwmRequest pwmRequest = PwmRequest.forRequest( ( HttpServletRequest ) sre.getServletRequest(), null );
+            pwmRequest.cleanThreadLocals();
+        }
+        catch ( final Exception e )
+        {
+            LOGGER.debug( () -> "error cleaning request thread locals: " + e.getMessage() );
+        }
+
         ServletRequestListener.super.requestDestroyed( sre );
     }
 

server/src/main/java/password/pwm/http/PwmRequest.java

@@ -76,6 +76,7 @@ import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -102,6 +103,8 @@ public class PwmRequest extends PwmHttpRequestWrapper
     private final DomainID domainID;
     private final Lock cspCreationLock = new ReentrantLock();
 
+    private final Set<ThreadLocal> threadLocals = new HashSet<>();
+
     private static final Lock CREATE_LOCK = new ReentrantLock();
 
     public static PwmRequest forRequest(
@@ -712,4 +715,14 @@ public class PwmRequest extends PwmHttpRequestWrapper
         return ( AccountInformationProfile ) getProfile( getPwmDomain(), ProfileDefinition.AccountInformation );
     }
 
+    public void registerThreadLocal( final ThreadLocal<?> threadLocal )
+    {
+        threadLocals.add( threadLocal );
+    }
+
+    public void cleanThreadLocals()
+    {
+        threadLocals.forEach( ThreadLocal::remove );
+        threadLocals.clear();
+    }
 }

server/src/main/java/password/pwm/ldap/LdapSystemService.java

@@ -31,6 +31,7 @@ import password.pwm.svc.PwmService;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.TimeDuration;
 
+import java.util.Collections;
 import java.util.List;
 
 public class LdapSystemService extends AbstractPwmService implements PwmService
@@ -56,7 +57,7 @@ public class LdapSystemService extends AbstractPwmService implements PwmService
     @Override
     protected List<HealthRecord> serviceHealthCheck()
     {
-        return null;
+        return Collections.emptyList();
     }
 
     @Override

server/src/main/java/password/pwm/svc/AbstractPwmService.java

@@ -134,7 +134,11 @@ public abstract class AbstractPwmService implements PwmService
 
         if ( status() == STATUS.OPEN )
         {
-            returnRecords.addAll( serviceHealthCheck() );
+            final List<HealthRecord> records = serviceHealthCheck();
+            if ( records != null )
+            {
+                returnRecords.addAll( records );
+            }
         }
 
         return returnRecords;

server/src/main/java/password/pwm/svc/intruder/IntruderDataStore.java

@@ -39,17 +39,21 @@ import java.time.Instant;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Optional;
+import java.util.concurrent.atomic.LongAdder;
 import java.util.function.Supplier;
 
 class IntruderDataStore implements IntruderRecordStore
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( IntruderDataStore.class );
+    private static final long CLEANER_MIN_WRITE_COUNT = 1_000;
 
     private final DataStore dataStore;
     private final Supplier<PwmService.STATUS> serviceStatus;
     private final StatisticCounterBundle<DebugKeys> stats = new StatisticCounterBundle<>( DebugKeys.class );
     private final PwmService intruderService;
 
+    private final LongAdder writeCounter = new LongAdder();
+
     private Instant eldestRecord;
 
     IntruderDataStore( final PwmService intruderService, final DataStore dataStore, final Supplier<PwmService.STATUS> serviceStatus )
@@ -57,6 +61,7 @@ class IntruderDataStore implements IntruderRecordStore
         this.intruderService = intruderService;
         this.dataStore = dataStore;
         this.serviceStatus = serviceStatus;
+        writeCounter.add( CLEANER_MIN_WRITE_COUNT );
     }
 
     @Override
@@ -120,6 +125,7 @@ class IntruderDataStore implements IntruderRecordStore
         try
         {
             dataStore.put( key, jsonRecord );
+            writeCounter.increment();
         }
         catch ( final PwmDataStoreException e )
         {
@@ -220,6 +226,11 @@ class IntruderDataStore implements IntruderRecordStore
     @Override
     public void cleanup( final TimeDuration maxRecordAge )
     {
+        if ( writeCounter.longValue() < CLEANER_MIN_WRITE_COUNT )
+        {
+            return;
+        }
+
         if ( eldestRecord != null && TimeDuration.fromCurrent( eldestRecord ).isShorterThan( maxRecordAge ) )
         {
             LOGGER.trace( intruderService.getSessionLabel(), () -> "skipping table cleanup: eldest record is younger than max age" );
@@ -266,5 +277,7 @@ class IntruderDataStore implements IntruderRecordStore
                     + TimeDuration.compactFromCurrent( startTime ) + ", recordsExamined="
                     + finalExamined + ", recordsRemoved=" + finalRemoved );
         }
+
+        writeCounter.reset();
     }
 }

server/src/main/java/password/pwm/svc/token/TokenSystemService.java

@@ -31,6 +31,7 @@ import password.pwm.svc.PwmService;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
+import java.util.Collections;
 import java.util.List;
 
 public class TokenSystemService extends AbstractPwmService implements PwmService
@@ -59,7 +60,7 @@ public class TokenSystemService extends AbstractPwmService implements PwmService
     @Override
     protected List<HealthRecord> serviceHealthCheck()
     {
-        return null;
+        return Collections.emptyList();
     }
 
     @Override

server/src/main/java/password/pwm/util/PwmScheduler.java

@@ -78,7 +78,7 @@ public class PwmScheduler
 
         Objects.requireNonNull( runnable );
 
-        final ExecutorService executor = makeMultiThreadExecutor( 1, pwmApplication, sessionLabel, runnable.getClass() );
+        final ExecutorService executor = makeMultiThreadExecutor( 1, pwmApplication, sessionLabel, runnable.getClass(), threadName );
 
         executor.submit( runnable );
     }
@@ -119,7 +119,7 @@ public class PwmScheduler
     {
         checkIfSchedulerClosed();
 
-        final ExecutorService executor = makeMultiThreadExecutor( maxThreadCount, pwmApplication, sessionLabel, theClass );
+        final ExecutorService executor = makeMultiThreadExecutor( maxThreadCount, pwmApplication, sessionLabel, theClass, null );
 
         final List<Future<T>> futures = callables.stream()
                 .map( executor::submit )
@@ -179,19 +179,31 @@ public class PwmScheduler
             final PwmApplication pwmApplication,
             final Class<?> theClass )
     {
-        String instanceName = "-";
-        if ( pwmApplication != null )
-        {
-            instanceName = pwmApplication.getInstanceID();
-        }
+        return makeThreadName( sessionLabel, pwmApplication, theClass, null );
+    }
+
+    public static String makeThreadName(
+            final SessionLabel sessionLabel,
+            final PwmApplication pwmApplication,
+            final Class<?> theClass,
+            final String threadNameSuffix
+    )
+    {
+        final String instanceName = pwmApplication == null
+                ? "-"
+                : pwmApplication.getInstanceID();
 
-        return makeThreadName( sessionLabel, instanceName, theClass );
+
+        return makeThreadName( sessionLabel, instanceName, theClass, threadNameSuffix );
     }
 
     public static String makeThreadName(
             final SessionLabel sessionLabel,
             final String instanceID,
-            final Class<?> theClass )
+            final Class<?> theClass,
+            final String threadNameSuffix
+
+    )
     {
         final StringBuilder output = new StringBuilder();
 
@@ -215,6 +227,12 @@ public class PwmScheduler
             output.append( sessionLabel.getDomain() );
         }
 
+        if ( !StringUtil.isEmpty( threadNameSuffix ) )
+        {
+            output.append( "-" );
+            output.append( threadNameSuffix );
+        }
+
         return output.toString();
     }
 
@@ -246,14 +264,26 @@ public class PwmScheduler
             final Class<?> theClass
     )
     {
-        return makeMultiThreadExecutor( maxThreadCount, pwmApplication.getInstanceID(), sessionLabel, theClass );
+        return makeMultiThreadExecutor( maxThreadCount, pwmApplication, sessionLabel, theClass, null );
+    }
+
+    public static ThreadPoolExecutor makeMultiThreadExecutor(
+            final int maxThreadCount,
+            final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
+            final Class<?> theClass,
+            final String threadNameSuffix
+    )
+    {
+        return makeMultiThreadExecutor( maxThreadCount, pwmApplication.getInstanceID(), sessionLabel, theClass, threadNameSuffix );
     }
 
     public static ThreadPoolExecutor makeMultiThreadExecutor(
             final int maxThreadCount,
             final String instanceID,
             final SessionLabel sessionLabel,
-            final Class<?> theClass
+            final Class<?> theClass,
+            final String threadNameSuffix
     )
     {
         final ThreadPoolExecutor executor = new ThreadPoolExecutor(
@@ -262,7 +292,7 @@ public class PwmScheduler
                 1, TimeUnit.SECONDS,
                 new LinkedBlockingQueue<>(),
                 makePwmThreadFactory(
-                        makeThreadName( sessionLabel, instanceID, theClass ) + "-",
+                        makeThreadName( sessionLabel, instanceID, theClass, threadNameSuffix ) + "-",
                         true
                 ) );
         executor.allowCoreThreadTimeOut( true );
@@ -274,14 +304,24 @@ public class PwmScheduler
             final SessionLabel sessionLabel,
             final Class<?> clazz
     )
+    {
+        return makeBackgroundServiceExecutor( pwmApplication, sessionLabel, clazz, null );
+    }
+
+    public static ScheduledExecutorService makeBackgroundServiceExecutor(
+            final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
+            final Class<?> clazz,
+            final String threadNameSuffix
+    )
     {
         final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(
                 1,
                 makePwmThreadFactory(
-                        makeThreadName( sessionLabel, pwmApplication, clazz ) + "-",
+                        makeThreadName( sessionLabel, pwmApplication, clazz, threadNameSuffix ) + "-",
                         true
                 ) );
-        executor.setKeepAliveTime( 1, TimeUnit.MINUTES );
+        executor.setKeepAliveTime( 5, TimeUnit.SECONDS );
         executor.allowCoreThreadTimeOut( true );
         return executor;
     }

server/src/main/java/password/pwm/util/logging/LocalDBLogger.java

@@ -54,7 +54,6 @@ import java.util.Optional;
 import java.util.Queue;
 import java.util.TreeMap;
 import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -148,17 +147,11 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
 
         setStatus( STATUS.OPEN );
 
-        cleanerService = Executors.newSingleThreadScheduledExecutor(
-                PwmScheduler.makePwmThreadFactory(
-                        PwmScheduler.makeThreadName( getSessionLabel(), pwmApplication, this.getClass() ) + "-cleaner-",
-                        true
-                ) );
+        cleanerService = PwmScheduler.makeBackgroundServiceExecutor(
+                pwmApplication, getSessionLabel(), LocalDBLogger.class, "cleaner" );
 
-        writerService = Executors.newSingleThreadScheduledExecutor(
-                PwmScheduler.makePwmThreadFactory(
-                        PwmScheduler.makeThreadName( getSessionLabel(), pwmApplication, this.getClass() ) + "-writer-",
-                        true
-                ) );
+        writerService = PwmScheduler.makeBackgroundServiceExecutor(
+                pwmApplication, getSessionLabel(), LocalDBLogger.class, "writer" );
 
         cleanerService.scheduleAtFixedRate( new CleanupTask(), 0, this.settings.cleanerFrequency().asMillis(), TimeUnit.MILLISECONDS );
 
@@ -197,6 +190,7 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
 
     private void scheduleNextFlush()
     {
+
         if ( tempMemoryEventQueue.size() > settings.getMaxBufferSize() / 2 )
         {
             writerService.schedule( new FlushTask(), 0, TimeUnit.SECONDS );

server/src/main/java/password/pwm/util/logging/PwmLogger.java

@@ -83,7 +83,7 @@ public class PwmLogger
         PwmLogger.fileAppender = rollingFileAppender;
     }
 
-    public static PwmLogger forClass( final Class className )
+    public static PwmLogger forClass( final Class<?> className )
     {
         return new PwmLogger( className.getName(), false );
     }
@@ -94,7 +94,7 @@ public class PwmLogger
     }
 
     public static PwmLogger forClass(
-            final Class className,
+            final Class<?> className,
             final boolean localDBDisabled
     )
     {