logging improvements

lib-util/src/main/java/password/pwm/util/java/MutableReference.java

@@ -0,0 +1,42 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.util.java;
+
+
+/**
+ * Simple mutable reference useful for some scenarios with lambdas that require only final parameters.
+ *
+ * @param <T> the reference type.
+ */
+public class MutableReference<T>
+{
+    private T reference;
+
+    public T get()
+    {
+        return reference;
+    }
+
+    public void set( final T reference )
+    {
+        this.reference = reference;
+    }
+}

server/pom.xml

@@ -171,13 +171,7 @@
         <dependency>
             <groupId>com.github.ldapchai</groupId>
             <artifactId>ldapchai</artifactId>
-            <version>0.8.2</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>log4j</groupId>
-                    <artifactId>log4j</artifactId>
-                </exclusion>
-            </exclusions>
+            <version>0.8.3-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>org.jrivard.xmlchai</groupId>

server/src/main/java/password/pwm/AppProperty.java

@@ -20,7 +20,7 @@
 
 package password.pwm;
 
-import password.pwm.util.java.CollectionUtil;
+import password.pwm.util.java.JavaHelper;
 
 import java.util.Objects;
 import java.util.Optional;
@@ -433,6 +433,11 @@ public enum AppProperty
         return defaultValue;
     }
 
+    public boolean isDefaultValue( final String value )
+    {
+        return Objects.equals( defaultValue, value );
+    }
+
     public String getDescription( )
     {
         return readAppPropertiesBundle( this.getKey() + DESCRIPTION_SUFFIX );
@@ -445,8 +450,6 @@ public enum AppProperty
 
     public static Optional<AppProperty> forKey( final String key )
     {
-        return CollectionUtil.enumStream( AppProperty.class )
-                .filter( loopProperty -> Objects.equals( loopProperty.getKey(), key ) )
-                .findFirst();
+        return JavaHelper.readEnumFromPredicate( AppProperty.class, appProperty -> Objects.equals( appProperty.getKey(), key ) );
     }
 }

server/src/main/java/password/pwm/PwmAboutProperty.java

@@ -37,10 +37,8 @@ import java.lang.management.ManagementFactory;
 import java.nio.charset.Charset;
 import java.security.NoSuchAlgorithmException;
 import java.time.Instant;
-import java.util.Collections;
 import java.util.Map;
 import java.util.Optional;
-import java.util.TreeMap;
 import java.util.function.Function;
 
 public enum PwmAboutProperty
@@ -176,18 +174,6 @@ public enum PwmAboutProperty
         return label == null ? this.name() : label;
     }
 
-    public static Map<String, String> toStringMap( final Map<PwmAboutProperty, String> infoBeanMap )
-    {
-        final Map<String, String> outputProps = new TreeMap<>( );
-        for ( final Map.Entry<PwmAboutProperty, String> entry : infoBeanMap.entrySet() )
-        {
-            final PwmAboutProperty aboutProperty = entry.getKey();
-            final String value = entry.getValue();
-            outputProps.put( aboutProperty.toString().replace( '_', '.' ), value );
-        }
-        return Collections.unmodifiableMap( outputProps );
-    }
-
     private static String readSslVersions()
     {
         try

server/src/main/java/password/pwm/bean/SessionLabel.java

@@ -51,6 +51,7 @@ public class SessionLabel implements Serializable
 
     private static final String SYSTEM_LABEL_SESSION_ID = "#";
     private static final String RUNTIME_LABEL_SESSION_ID = "!";
+    private static final String HEALTH_LABEL_SESSION_ID = "H";
 
     public static final SessionLabel SYSTEM_LABEL = SessionLabel.forNonUserType( ActorType.system, DomainID.systemId() );
     public static final SessionLabel HEALTH_LABEL = SessionLabel.forNonUserType( ActorType.health, DomainID.systemId() );
@@ -70,22 +71,34 @@ public class SessionLabel implements Serializable
 
     public enum ActorType
     {
-        user,
-        system,
-        runtime,
-        health,
-        test,
-        cli,
-        onejar,
-        context,
-        rest,
+        user( null ),
+        system( "#" ),
+        runtime( "!" ),
+        health( "-HEALTH" ),
+        test( "-TEST" ),
+        cli( "-CLI" ),
+        onejar( "-ONEJAR" ),
+        context( "-CONTEXT" ),
+        rest( null ),;
+
+        private final String defaultSessionId;
+
+        ActorType( final String defaultSessionId )
+        {
+            this.defaultSessionId = defaultSessionId;
+        }
+
+        public String defaultSessionId()
+        {
+            return defaultSessionId;
+        }
     }
 
     private static SessionLabel forNonUserType( final ActorType actorType, final DomainID domainID )
     {
         Objects.requireNonNull( actorType );
 
-        final String sessionID = domainID == null || domainID.isSystem() ? SYSTEM_LABEL_SESSION_ID : RUNTIME_LABEL_SESSION_ID;
+        final String sessionID = actorType.defaultSessionId();
         final String domainSting = domainID == null ? DomainID.systemId().stringValue() : domainID.stringValue();
 
         return SessionLabel.builder()
@@ -159,6 +172,10 @@ public class SessionLabel implements Serializable
                 }
             }
         }
+        else
+        {
+            builder.sessionID( "-" );
+        }
 
         return builder.build();
     }

server/src/main/java/password/pwm/config/AppConfig.java

@@ -50,12 +50,10 @@ import password.pwm.util.secure.PwmRandom;
 import password.pwm.util.secure.PwmSecurityKey;
 
 import java.security.cert.X509Certificate;
-import java.util.Collections;
 import java.util.EnumMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.SortedMap;
@@ -329,8 +327,9 @@ public class AppConfig implements SettingReader
 
     private static Map<AppProperty, String> makeAppPropertyOverrides( final SettingReader settingReader )
     {
-        final Map<String, String> stringMap =  StringUtil.convertStringListToNameValuePair(
-                settingReader.readSettingAsStringArray( PwmSetting.APP_PROPERTY_OVERRIDES ), "=" );
+        final List<String> settingValues = settingReader.readSettingAsStringArray( PwmSetting.APP_PROPERTY_OVERRIDES );
+
+        final Map<String, String> stringMap =  StringUtil.convertStringListToNameValuePair( settingValues, "=" );
 
         final Map<AppProperty, String> appPropertyMap = new EnumMap<>( AppProperty.class );
         for ( final Map.Entry<String, String> stringEntry : stringMap.entrySet() )
@@ -338,16 +337,14 @@ public class AppConfig implements SettingReader
             AppProperty.forKey( stringEntry.getKey() )
                     .ifPresent( appProperty ->
                     {
-                        final String defaultValue = appProperty.getDefaultValue();
-                        final String value = stringEntry.getValue();
-                        if ( !Objects.equals( defaultValue, value ) )
+                        if ( !appProperty.isDefaultValue( stringEntry.getValue() ) )
                         {
-                            appPropertyMap.put( appProperty, value );
+                            appPropertyMap.put( appProperty, stringEntry.getValue() );
                         }
                     } );
         }
 
-        return Collections.unmodifiableMap( appPropertyMap );
+        return CollectionUtil.unmodifiableEnumMap( appPropertyMap, AppProperty.class );
     }
 
     public boolean isSmsConfigured()
@@ -355,12 +352,13 @@ public class AppConfig implements SettingReader
         final String gatewayUrl = readSettingAsString( PwmSetting.SMS_GATEWAY_URL );
         final String gatewayUser = readSettingAsString( PwmSetting.SMS_GATEWAY_USER );
         final PasswordData gatewayPass = readSettingAsPassword( PwmSetting.SMS_GATEWAY_PASSWORD );
-        if ( gatewayUrl == null || gatewayUrl.length() < 1 )
+
+        if ( StringUtil.isEmpty( gatewayUrl ) )
         {
             return false;
         }
 
-        if ( gatewayUser != null && gatewayUser.length() > 0 && ( gatewayPass == null ) )
+        if ( !StringUtil.isEmpty( gatewayUser ) && gatewayPass == null )
         {
             return false;
         }

server/src/main/java/password/pwm/health/DebugOutputService.java

@@ -0,0 +1,216 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.health;
+
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.bean.DomainID;
+import password.pwm.bean.SessionLabel;
+import password.pwm.error.PwmException;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.svc.AbstractPwmService;
+import password.pwm.svc.PwmService;
+import password.pwm.util.debug.DebugItemGenerator;
+import password.pwm.util.java.AtomicLoopIntIncrementer;
+import password.pwm.util.java.FileSystemUtility;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogLevel;
+import password.pwm.util.logging.PwmLogManager;
+import password.pwm.util.logging.PwmLogger;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.nio.file.Files;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.ZipOutputStream;
+
+public class DebugOutputService extends AbstractPwmService implements PwmService
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( DebugOutputService.class );
+
+
+    @Override
+    public STATUS postAbstractInit( final PwmApplication pwmApplication, final DomainID domainID )
+            throws PwmException
+    {
+
+        scheduleNextZipOutput();
+
+        final TimeDuration threadDumpInterval = TimeDuration.of(
+            Long.parseLong( pwmApplication.getConfig().readAppProperty( AppProperty.LOGGING_EXTRA_PERIODIC_THREAD_DUMP_INTERVAL ) ), TimeDuration.Unit.SECONDS );
+
+        if ( threadDumpInterval.as( TimeDuration.Unit.SECONDS ) > 0 )
+        {
+            scheduleFixedRateJob(
+                    new ThreadDumpLogger( getSessionLabel() ),
+                    TimeDuration.SECOND,
+                    threadDumpInterval );
+        }
+
+        return STATUS.OPEN;
+    }
+
+
+    @Override
+    public void shutdownImpl( )
+    {
+    }
+
+    @Override
+    public List<HealthRecord> serviceHealthCheck( )
+    {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public ServiceInfoBean serviceInfo( )
+    {
+        return ServiceInfoBean.builder()
+                .build();
+    }
+
+
+    private void scheduleNextZipOutput()
+    {
+        final int intervalSeconds = JavaHelper.silentParseInt( getPwmApplication().getConfig().readAppProperty( AppProperty.HEALTH_SUPPORT_BUNDLE_WRITE_INTERVAL_SECONDS ), 0 );
+        if ( intervalSeconds > 0 )
+        {
+            final TimeDuration intervalDuration = TimeDuration.of( intervalSeconds, TimeDuration.Unit.SECONDS );
+            scheduleJob( new SupportZipFileWriter( getPwmApplication() ), intervalDuration );
+        }
+    }
+
+    private class SupportZipFileWriter implements Runnable
+    {
+        private final PwmApplication pwmApplication;
+
+        SupportZipFileWriter( final PwmApplication pwmApplication )
+        {
+            this.pwmApplication = pwmApplication;
+        }
+
+        @Override
+        public void run()
+        {
+            PwmLogManager.executeWithThreadSessionData( getSessionLabel(), () ->
+                    {
+                        try
+                        {
+                            writeSupportZipToAppPath();
+                        }
+                        catch ( final Exception e )
+                        {
+                            LOGGER.debug( getSessionLabel(), () -> "error writing support zip to file system: " + e.getMessage() );
+                        }
+
+                        scheduleNextZipOutput();
+                    } );
+
+        }
+
+        private void writeSupportZipToAppPath()
+                throws IOException, PwmUnrecoverableException
+        {
+            final File appPath = getPwmApplication().getPwmEnvironment().getApplicationPath();
+            if ( !appPath.exists() )
+            {
+                return;
+            }
+
+            final int rotationCount = JavaHelper.silentParseInt( pwmApplication.getConfig().readAppProperty( AppProperty.HEALTH_SUPPORT_BUNDLE_FILE_WRITE_COUNT ), 10 );
+            final DebugItemGenerator debugItemGenerator = new DebugItemGenerator( pwmApplication, getSessionLabel() );
+
+            final File supportPath = new File( appPath.getPath() + File.separator + "support" );
+
+            Files.createDirectories( supportPath.toPath() );
+
+            final File supportFile = new File ( supportPath.getPath() + File.separator + debugItemGenerator.getFilename() );
+
+            FileSystemUtility.rotateBackups( supportFile, rotationCount );
+
+            final File newSupportFile = new File ( supportFile.getPath() + ".new" );
+            Files.deleteIfExists( newSupportFile.toPath() );
+
+            try ( ZipOutputStream zipOutputStream = new ZipOutputStream( new FileOutputStream( newSupportFile ) ) )
+            {
+                LOGGER.trace( getSessionLabel(), () -> "beginning periodic support bundle filesystem output" );
+                debugItemGenerator.outputZipDebugFile( zipOutputStream );
+            }
+
+            Files.move( newSupportFile.toPath(), supportFile.toPath() );
+        }
+    }
+
+    private static class ThreadDumpLogger implements Runnable
+    {
+        private static final PwmLogger LOGGER = PwmLogger.forClass( ThreadDumpLogger.class );
+        private static final AtomicLoopIntIncrementer COUNTER = new AtomicLoopIntIncrementer();
+
+        private final SessionLabel sessionLabel;
+
+        ThreadDumpLogger( final SessionLabel sessionLabel )
+        {
+            this.sessionLabel = sessionLabel;
+        }
+
+        @Override
+        public void run()
+        {
+            if ( !LOGGER.isInterestingLevel( PwmLogLevel.TRACE ) )
+            {
+                return;
+            }
+
+            final int count = COUNTER.next();
+            output( "---BEGIN OUTPUT THREAD DUMP #" + count + "---" );
+
+            {
+                final Map<String, String> debugValues = new LinkedHashMap<>();
+                debugValues.put( "Memory Free", Long.toString( Runtime.getRuntime().freeMemory() ) );
+                debugValues.put( "Memory Allocated", Long.toString( Runtime.getRuntime().totalMemory() ) );
+                debugValues.put( "Memory Max", Long.toString( Runtime.getRuntime().maxMemory() ) );
+                debugValues.put( "Thread Count", Integer.toString( Thread.activeCount() ) );
+                output( "Thread Data #: " + StringUtil.mapToString( debugValues ) );
+            }
+
+            final ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true );
+            for ( final ThreadInfo threadInfo : threads )
+            {
+                output( JavaHelper.threadInfoToString( threadInfo ) );
+            }
+
+            output( "---END OUTPUT THREAD DUMP #" + count + "---" );
+        }
+
+        private void output( final CharSequence output )
+        {
+            LOGGER.trace( sessionLabel, () -> output );
+        }
+    }
+}

server/src/main/java/password/pwm/health/HealthMonitorSettings.java

@@ -37,7 +37,6 @@ class HealthMonitorSettings implements Serializable
     private TimeDuration minimumCheckInterval;
     private TimeDuration maximumRecordAge;
     private TimeDuration maximumForceCheckWait;
-    private TimeDuration threadDumpInterval;
 
     static HealthMonitorSettings fromConfiguration( final AppConfig config )
     {
@@ -47,8 +46,6 @@ class HealthMonitorSettings implements Serializable
                 .minimumCheckInterval( TimeDuration.of( Long.parseLong( config.readAppProperty( AppProperty.HEALTHCHECK_MIN_CHECK_INTERVAL ) ), TimeDuration.Unit.SECONDS ) )
                 .maximumRecordAge( TimeDuration.of( Long.parseLong( config.readAppProperty( AppProperty.HEALTHCHECK_MAX_RECORD_AGE ) ), TimeDuration.Unit.SECONDS ) )
                 .maximumForceCheckWait( TimeDuration.of( Long.parseLong( config.readAppProperty( AppProperty.HEALTHCHECK_MAX_FORCE_WAIT ) ), TimeDuration.Unit.SECONDS ) )
-                .threadDumpInterval( TimeDuration.of(
-                        Long.parseLong( config.readAppProperty( AppProperty.LOGGING_EXTRA_PERIODIC_THREAD_DUMP_INTERVAL ) ), TimeDuration.Unit.SECONDS ) )
                 .build();
     }
 }

server/src/main/java/password/pwm/health/HealthService.java

@@ -26,43 +26,27 @@ import password.pwm.PwmApplication;
 import password.pwm.bean.DomainID;
 import password.pwm.bean.SessionLabel;
 import password.pwm.error.PwmException;
-import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
-import password.pwm.util.PwmScheduler;
-import password.pwm.util.debug.DebugItemGenerator;
-import password.pwm.util.java.AtomicLoopIntIncrementer;
-import password.pwm.util.java.FileSystemUtility;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StatisticAverageBundle;
 import password.pwm.util.java.StatisticCounterBundle;
-import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
-import password.pwm.util.logging.PwmLogLevel;
+import password.pwm.util.logging.PwmLogManager;
 import password.pwm.util.logging.PwmLogger;
 
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
 import java.io.Serializable;
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadInfo;
-import java.nio.file.Files;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
-import java.util.zip.ZipOutputStream;
 
 public class HealthService extends AbstractPwmService implements PwmService
 {
@@ -76,8 +60,6 @@ public class HealthService extends AbstractPwmService implements PwmService
             new CertificateChecker() );
 
 
-    private ScheduledExecutorService executorService;
-    private ScheduledExecutorService supportZipWriterService;
     private HealthMonitorSettings settings;
 
     private final Map<HealthMonitorFlag, Serializable> healthProperties = new ConcurrentHashMap<>();
@@ -121,19 +103,6 @@ public class HealthService extends AbstractPwmService implements PwmService
             return STATUS.CLOSED;
         }
 
-        executorService = PwmScheduler.makeBackgroundServiceExecutor( pwmApplication, getSessionLabel(), this.getClass() );
-        supportZipWriterService = PwmScheduler.makeBackgroundServiceExecutor( pwmApplication, getSessionLabel(), this.getClass() );
-        scheduleNextZipOutput();
-
-        if ( settings.getThreadDumpInterval().as( TimeDuration.Unit.SECONDS ) > 0 )
-        {
-            pwmApplication.getPwmScheduler().scheduleFixedRateJob(
-                    new ThreadDumpLogger( getSessionLabel() ),
-                    executorService,
-                    TimeDuration.SECOND,
-                    settings.getThreadDumpInterval() );
-        }
-
         return STATUS.OPEN;
     }
 
@@ -168,7 +137,7 @@ public class HealthService extends AbstractPwmService implements PwmService
         {
             final Instant startTime = Instant.now();
             LOGGER.trace( getSessionLabel(), () -> "begin force immediate check" );
-            final Future<?> future = getPwmApplication().getPwmScheduler().scheduleJob( new ImmediateJob(), executorService, TimeDuration.ZERO );
+            final Future<?> future = scheduleJob( new ImmediateJob(), TimeDuration.ZERO );
             settings.getMaximumForceCheckWait().pause( future::isDone );
             final TimeDuration checkDuration = TimeDuration.fromCurrent( startTime );
             averageStats.update( AverageStatKey.checkProcessTime, checkDuration.asDuration() );
@@ -176,7 +145,7 @@ public class HealthService extends AbstractPwmService implements PwmService
             LOGGER.trace( getSessionLabel(), () -> "exit force immediate check, done=" + future.isDone(), checkDuration );
         }
 
-        getPwmApplication().getPwmScheduler().scheduleJob( new UpdateJob(), executorService, settings.getNominalCheckInterval() );
+        scheduleJob( new UpdateJob(), settings.getNominalCheckInterval() );
 
         {
             final HealthData localHealthData = this.healthData;
@@ -192,14 +161,6 @@ public class HealthService extends AbstractPwmService implements PwmService
     @Override
     public void shutdownImpl( )
     {
-        if ( executorService != null )
-        {
-            executorService.shutdown();
-        }
-        if ( supportZipWriterService != null )
-        {
-            supportZipWriterService.shutdown();
-        }
         healthData = emptyHealthData();
         setStatus( STATUS.CLOSED );
     }
@@ -319,14 +280,17 @@ public class HealthService extends AbstractPwmService implements PwmService
         @Override
         public void run( )
         {
-            try
-            {
-                doHealthChecks();
-            }
-            catch ( final Throwable e )
-            {
-                LOGGER.error( getSessionLabel(), () -> "error during health check execution: " + e.getMessage(), e );
-            }
+            PwmLogManager.executeWithThreadSessionData( getSessionLabel(), () ->
+                    {
+                        try
+                        {
+                            doHealthChecks();
+                        }
+                        catch ( final Throwable e )
+                        {
+                            LOGGER.error( getSessionLabel(), () -> "error during health check execution: " + e.getMessage(), e );
+                        }
+                    } );
         }
     }
 
@@ -346,118 +310,4 @@ public class HealthService extends AbstractPwmService implements PwmService
             return TimeDuration.fromCurrent( this.getTimeStamp() ).isLongerThan( settings.getMaximumRecordAge() );
         }
     }
-
-    private void scheduleNextZipOutput()
-    {
-        final int intervalSeconds = JavaHelper.silentParseInt( getPwmApplication().getConfig().readAppProperty( AppProperty.HEALTH_SUPPORT_BUNDLE_WRITE_INTERVAL_SECONDS ), 0 );
-        if ( intervalSeconds > 0 )
-        {
-            final TimeDuration intervalDuration = TimeDuration.of( intervalSeconds, TimeDuration.Unit.SECONDS );
-            getPwmApplication().getPwmScheduler().scheduleJob( new SupportZipFileWriter( getPwmApplication() ), supportZipWriterService, intervalDuration );
-        }
-    }
-
-    private class SupportZipFileWriter implements Runnable
-    {
-        private final PwmApplication pwmApplication;
-
-        SupportZipFileWriter( final PwmApplication pwmApplication )
-        {
-            this.pwmApplication = pwmApplication;
-        }
-
-        @Override
-        public void run()
-        {
-            try
-            {
-                writeSupportZipToAppPath();
-            }
-            catch ( final Exception e )
-            {
-                LOGGER.debug( getSessionLabel(), () -> "error writing support zip to file system: " + e.getMessage() );
-            }
-
-            scheduleNextZipOutput();
-        }
-
-        private void writeSupportZipToAppPath()
-                throws IOException, PwmUnrecoverableException
-        {
-            final File appPath = getPwmApplication().getPwmEnvironment().getApplicationPath();
-            if ( !appPath.exists() )
-            {
-                return;
-            }
-
-            final int rotationCount = JavaHelper.silentParseInt( pwmApplication.getConfig().readAppProperty( AppProperty.HEALTH_SUPPORT_BUNDLE_FILE_WRITE_COUNT ), 10 );
-            final DebugItemGenerator debugItemGenerator = new DebugItemGenerator( pwmApplication, getSessionLabel() );
-
-            final File supportPath = new File( appPath.getPath() + File.separator + "support" );
-
-            Files.createDirectories( supportPath.toPath() );
-
-            final File supportFile = new File ( supportPath.getPath() + File.separator + debugItemGenerator.getFilename() );
-
-            FileSystemUtility.rotateBackups( supportFile, rotationCount );
-
-            final File newSupportFile = new File ( supportFile.getPath() + ".new" );
-            Files.deleteIfExists( newSupportFile.toPath() );
-
-            try ( ZipOutputStream zipOutputStream = new ZipOutputStream( new FileOutputStream( newSupportFile ) ) )
-            {
-                LOGGER.trace( getSessionLabel(), () -> "beginning periodic support bundle filesystem output" );
-                debugItemGenerator.outputZipDebugFile( zipOutputStream );
-            }
-
-            Files.move( newSupportFile.toPath(), supportFile.toPath() );
-        }
-    }
-
-    private static class ThreadDumpLogger implements Runnable
-    {
-        private static final PwmLogger LOGGER = PwmLogger.forClass( ThreadDumpLogger.class );
-        private static final AtomicLoopIntIncrementer COUNTER = new AtomicLoopIntIncrementer();
-
-        private final SessionLabel sessionLabel;
-
-        ThreadDumpLogger( final SessionLabel sessionLabel )
-        {
-            this.sessionLabel = sessionLabel;
-        }
-
-        @Override
-        public void run()
-        {
-            if ( !LOGGER.isInterestingLevel( PwmLogLevel.TRACE ) )
-            {
-                return;
-            }
-
-            final int count = COUNTER.next();
-            output( "---BEGIN OUTPUT THREAD DUMP #" + count + "---" );
-
-            {
-                final Map<String, String> debugValues = new LinkedHashMap<>();
-                debugValues.put( "Memory Free", Long.toString( Runtime.getRuntime().freeMemory() ) );
-                debugValues.put( "Memory Allocated", Long.toString( Runtime.getRuntime().totalMemory() ) );
-                debugValues.put( "Memory Max", Long.toString( Runtime.getRuntime().maxMemory() ) );
-                debugValues.put( "Thread Count", Integer.toString( Thread.activeCount() ) );
-                output( "Thread Data #: " + StringUtil.mapToString( debugValues ) );
-            }
-
-            final ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true );
-            for ( final ThreadInfo threadInfo : threads )
-            {
-                output( JavaHelper.threadInfoToString( threadInfo ) );
-            }
-
-            output( "---END OUTPUT THREAD DUMP #" + count + "---" );
-        }
-
-        private void output( final CharSequence output )
-        {
-            LOGGER.trace( sessionLabel, () -> output );
-        }
-    }
 }

server/src/main/java/password/pwm/http/filter/RequestInitializationFilter.java

@@ -48,8 +48,10 @@ import password.pwm.svc.intruder.IntruderServiceClient;
 import password.pwm.svc.stats.EpsStatistic;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsService;
+import password.pwm.util.java.MutableReference;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogManager;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroRequest;
 import password.pwm.util.secure.PwmRandom;
@@ -197,10 +199,48 @@ public class RequestInitializationFilter implements Filter
             return;
         }
 
+        final PwmRequest pwmRequest;
         try
         {
-            final PwmRequest pwmRequest = PwmRequest.forRequest( req, resp );
+            pwmRequest = PwmRequest.forRequest( req, resp );
+            doTheThing( pwmRequest );
+        }
+        catch ( final Throwable t )
+        {
+            processInitThrowable( t, req, resp );
+            return;
+        }
+
+        final MutableReference<Throwable> throwableReference = new MutableReference<>();
+
+        PwmLogManager.executeWithThreadSessionData( pwmRequest.getLabel(), () ->
+        {
+            try
+            {
+                pwmRequest.getPwmDomain().getActiveServletRequests().incrementAndGet();
+                filterChain.doFilter( req, resp );
+            }
+            catch ( final Throwable e )
+            {
+                throwableReference.set( e );
+            }
+            finally
+            {
+                pwmRequest.getPwmDomain().getActiveServletRequests().decrementAndGet();
+            }
+        } );
 
+        if ( throwableReference.get() != null )
+        {
+            processInitThrowable( throwableReference.get(), req, resp );
+        }
+    }
+
+    private void doTheThing( final PwmRequest pwmRequest )
+            throws ServletException, PwmUnrecoverableException, IOException
+    {
+        try
+        {
             checkIfSessionRecycleNeeded( pwmRequest );
 
             handleRequestInitialization( pwmRequest );
@@ -211,56 +251,41 @@ public class RequestInitializationFilter implements Filter
 
             updateStats( pwmRequest.getPwmApplication() );
 
-            try
-            {
-                handleRequestSecurityChecks( pwmRequest );
-            }
-            catch ( final PwmUnrecoverableException e )
+            handleRequestSecurityChecks( pwmRequest );
+        }
+        catch ( final PwmUnrecoverableException e )
+        {
+            LOGGER.error( pwmRequest, e.getErrorInformation() );
+            pwmRequest.respondWithError( e.getErrorInformation() );
+            if ( PwmError.ERROR_INTRUDER_SESSION != e.getError() )
             {
-                LOGGER.error( pwmRequest, e.getErrorInformation() );
-                pwmRequest.respondWithError( e.getErrorInformation() );
-                if ( PwmError.ERROR_INTRUDER_SESSION != e.getError() )
-                {
-                    pwmRequest.invalidateSession();
-                }
-                return;
+                pwmRequest.invalidateSession();
             }
+        }
+    }
 
-
-            try
-            {
-                pwmRequest.getPwmDomain().getActiveServletRequests().incrementAndGet();
-                filterChain.doFilter( req, resp );
-            }
-            finally
-            {
-                pwmRequest.getPwmDomain().getActiveServletRequests().decrementAndGet();
-            }
+    private void processInitThrowable( final Throwable e, final HttpServletRequest req, final HttpServletResponse resp )
+            throws ServletException, IOException
+    {
+        final String logMsg = "can't init request: " + e.getMessage();
+        if ( e instanceof PwmException && ( ( PwmException ) e ).getError() != PwmError.ERROR_INTERNAL )
+        {
+            LOGGER.error( () -> logMsg );
         }
-        catch ( final Throwable e )
+        else
         {
-            final String logMsg = "can't init request: " + e.getMessage();
-            if ( e instanceof PwmException && ( ( PwmException ) e ).getError() != PwmError.ERROR_INTERNAL )
-            {
-                LOGGER.error( () -> logMsg );
-            }
-            else
-            {
-                LOGGER.error( () -> logMsg, e );
-            }
-            try
-            {
-                if ( !( PwmURL.create( req ).isResourceURL() ) )
-                {
-                    respondWithUnavailableError( req, resp );
-                    return;
-                }
-            }
-            catch ( final PwmUnrecoverableException pwmUnrecoverableException )
+            LOGGER.error( () -> logMsg, e );
+        }
+        try
+        {
+            if ( !( PwmURL.create( req ).isResourceURL() ) )
             {
-                LOGGER.debug( () -> "error initializing http request for " + PwmConstants.PWM_APP_NAME + ": " + e.getMessage() );
+                respondWithUnavailableError( req, resp );
             }
-            return;
+        }
+        catch ( final PwmUnrecoverableException pwmUnrecoverableException )
+        {
+            LOGGER.debug( () -> "error initializing http request for " + PwmConstants.PWM_APP_NAME + ": " + e.getMessage() );
         }
     }
 

server/src/main/java/password/pwm/http/servlet/ClientApiServlet.java

@@ -92,9 +92,14 @@ import java.util.stream.Collectors;
 )
 public class ClientApiServlet extends ControlledPwmServlet
 {
-
     private static final PwmLogger LOGGER = PwmLogger.forClass( ClientApiServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     @Data
     public static class AppData implements Serializable
     {

server/src/main/java/password/pwm/http/servlet/ControlledPwmServlet.java

@@ -30,6 +30,7 @@ import password.pwm.http.ProcessStatus;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmResponse;
 import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
 import javax.servlet.ServletException;
@@ -38,6 +39,7 @@ import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.time.Instant;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
@@ -54,7 +56,7 @@ public abstract class ControlledPwmServlet extends AbstractPwmServlet implements
     {
         for ( final PwmServletDefinition pwmServletDefinition : PwmServletDefinition.values() )
         {
-            final Class pwmServletClass = pwmServletDefinition.getPwmServletClass();
+            final Class<? extends PwmServlet> pwmServletClass = pwmServletDefinition.getPwmServletClass();
             if ( pwmServletClass.isInstance( this ) )
             {
                 return pwmServletDefinition;
@@ -89,7 +91,11 @@ public abstract class ControlledPwmServlet extends AbstractPwmServlet implements
             final Method interestedMethod = actionMethodCache.get( action.get() );
             if ( interestedMethod != null )
             {
-                return ( ProcessStatus ) interestedMethod.invoke( this, pwmRequest );
+                final Instant startTime = Instant.now();
+                getLogger().trace( pwmRequest, () -> "entering process action for '" + interestedMethod.getName() + '\'' );
+                final ProcessStatus result =  ( ProcessStatus ) interestedMethod.invoke( this, pwmRequest );
+                getLogger().trace( pwmRequest, () -> "completed process action for '" +  interestedMethod.getName() + '\'', TimeDuration.fromCurrent( startTime ) );
+                return result;
             }
         }
         catch ( final InvocationTargetException e )
@@ -121,6 +127,8 @@ public abstract class ControlledPwmServlet extends AbstractPwmServlet implements
         throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_INTERNAL, msg ) );
     }
 
+    protected abstract PwmLogger getLogger();
+
     @Override
     protected void processAction( final PwmRequest pwmRequest )
             throws ServletException, IOException, ChaiUnavailableException, PwmUnrecoverableException
@@ -183,7 +191,7 @@ public abstract class ControlledPwmServlet extends AbstractPwmServlet implements
     public @interface ActionHandler
     {
         String action( );
-            }
+    }
 
     private Map<? extends ProcessAction, Method> createMethodCache()
     {

server/src/main/java/password/pwm/http/servlet/DeleteAccountServlet.java

@@ -67,9 +67,14 @@ import java.util.Locale;
 )
 public class DeleteAccountServlet extends ControlledPwmServlet
 {
-
     private static final PwmLogger LOGGER = PwmLogger.forClass( DeleteAccountServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     public enum DeleteAccountAction implements AbstractPwmServlet.ProcessAction
     {
         agree( HttpMethod.POST ),

server/src/main/java/password/pwm/http/servlet/FullPageHealthServlet.java

@@ -51,6 +51,12 @@ public class FullPageHealthServlet extends ControlledPwmServlet
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( FullPageHealthServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     public enum FullPageHealthAction implements AbstractPwmServlet.ProcessAction
     {
         value( HttpMethod.GET ),;

server/src/main/java/password/pwm/http/servlet/GuestRegistrationServlet.java

@@ -98,6 +98,12 @@ public class GuestRegistrationServlet extends ControlledPwmServlet
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( GuestRegistrationServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     public static final String HTTP_PARAM_EXPIRATION_DATE = "_expirationDateFormInput";
 
     public enum Page

server/src/main/java/password/pwm/http/servlet/LoginServlet.java

@@ -75,6 +75,13 @@ public class LoginServlet extends ControlledPwmServlet
 {
     private static final PwmLogger LOGGER = PwmLogger.getLogger( LoginServlet.class.getName() );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
+
     public enum LoginServletAction implements ProcessAction
     {
         login( HttpMethod.POST ),

server/src/main/java/password/pwm/http/servlet/LogoutServlet.java

@@ -57,6 +57,12 @@ public class LogoutServlet extends ControlledPwmServlet
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( LogoutServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     private static final String PARAM_URL = "url";
     private static final String PARAM_IDLE = "idle";
     private static final String PARAM_PASSWORD_MODIFIED = "passwordModified";

server/src/main/java/password/pwm/http/servlet/SetupOtpServlet.java

@@ -83,6 +83,12 @@ public class SetupOtpServlet extends ControlledPwmServlet
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( SetupOtpServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     public enum SetupOtpAction implements AbstractPwmServlet.ProcessAction
     {
         clearOtp( HttpMethod.POST ),

server/src/main/java/password/pwm/http/servlet/ShortcutServlet.java

@@ -67,9 +67,14 @@ import java.util.stream.Collectors;
 )
 public class ShortcutServlet extends ControlledPwmServlet
 {
-
     private static final PwmLogger LOGGER = PwmLogger.forClass( ShortcutServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     public enum ShortcutAction implements AbstractPwmServlet.ProcessAction
     {
         selectShortcut,;

server/src/main/java/password/pwm/http/servlet/accountinfo/AccountInformationServlet.java

@@ -56,6 +56,12 @@ public class AccountInformationServlet extends ControlledPwmServlet
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( AccountInformationServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     public enum AccountInformationAction implements AbstractPwmServlet.ProcessAction
     {
         read( HttpMethod.GET ),;

server/src/main/java/password/pwm/http/servlet/activation/ActivateUserServlet.java

@@ -96,6 +96,12 @@ public class ActivateUserServlet extends ControlledPwmServlet
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( ActivateUserServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     public enum ActivateUserAction implements ProcessAction
     {
         search( HttpMethod.POST ),

server/src/main/java/password/pwm/http/servlet/admin/AdminServlet.java

@@ -112,6 +112,12 @@ public class AdminServlet extends ControlledPwmServlet
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( AdminServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     public enum AdminAction implements AbstractPwmServlet.ProcessAction
     {
         viewLogWindow( HttpMethod.GET ),

server/src/main/java/password/pwm/http/servlet/changepw/ChangePasswordServlet.java

@@ -91,6 +91,12 @@ public abstract class ChangePasswordServlet extends ControlledPwmServlet
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( ChangePasswordServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     public enum ChangePasswordAction implements ProcessAction
     {
         checkProgress( HttpMethod.POST ),

server/src/main/java/password/pwm/http/servlet/command/CommandServlet.java

@@ -47,9 +47,14 @@ import java.util.Collection;
 
 public abstract class CommandServlet extends ControlledPwmServlet
 {
-
     private static final PwmLogger LOGGER = PwmLogger.forClass( CommandServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     @Override
     public Class<? extends ProcessAction> getProcessActionsClass( )
     {

server/src/main/java/password/pwm/http/servlet/configeditor/ConfigEditorServlet.java

@@ -128,6 +128,12 @@ public class ConfigEditorServlet extends ControlledPwmServlet
     public static final String REQ_PARAM_PROFILE = "profile";
     public static final String REQ_PARAM_TEMPLATE = "template";
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     public enum ConfigEditorAction implements AbstractPwmServlet.ProcessAction
     {
         readSetting( HttpMethod.POST ),
@@ -552,8 +558,6 @@ public class ConfigEditorServlet extends ControlledPwmServlet
     )
             throws IOException, PwmUnrecoverableException
     {
-        LOGGER.debug( pwmRequest, () -> "beginning restLdapHealthCheck" );
-
         final Instant startTime = Instant.now();
         final ConfigManagerBean configManagerBean = getBean( pwmRequest );
         final ProfileID profileID = ProfileID.create( pwmRequest.readParameterAsString( REQ_PARAM_PROFILE ) );
@@ -573,7 +577,6 @@ public class ConfigEditorServlet extends ControlledPwmServlet
         final RestResultBean<PublicHealthData> restResultBean = RestResultBean.withData( healthData, PublicHealthData.class );
 
         pwmRequest.outputJsonResult( restResultBean );
-        LOGGER.debug( pwmRequest, () -> "completed restLdapHealthCheck in " + TimeDuration.fromCurrent( startTime ).asCompactString() );
         return ProcessStatus.Halt;
     }
 
@@ -585,14 +588,12 @@ public class ConfigEditorServlet extends ControlledPwmServlet
     {
         final Instant startTime = Instant.now();
         final ConfigManagerBean configManagerBean = getBean( pwmRequest );
-        LOGGER.debug( pwmRequest, () -> "beginning restDatabaseHealthCheck" );
         final AppConfig config = AppConfig.forStoredConfig( configManagerBean.getStoredConfiguration() );
         final DomainID domainID = DomainStateReader.forRequest( pwmRequest ).getDomainID( PwmSetting.LDAP_SERVER_URLS );
         final List<HealthRecord> healthRecords = DatabaseStatusChecker.checkNewDatabaseStatus( pwmRequest.getPwmApplication(), config );
         final PublicHealthData healthData = HealthRecord.asHealthDataBean( config.getDomainConfigs().get( domainID ), pwmRequest.getLocale(), healthRecords );
         final RestResultBean restResultBean = RestResultBean.withData( healthData, PublicHealthData.class );
         pwmRequest.outputJsonResult( restResultBean );
-        LOGGER.debug( pwmRequest, () -> "completed restDatabaseHealthCheck in " + TimeDuration.fromCurrent( startTime ).asCompactString() );
         return ProcessStatus.Halt;
     }
 
@@ -604,7 +605,6 @@ public class ConfigEditorServlet extends ControlledPwmServlet
     {
         final Instant startTime = Instant.now();
         final ConfigManagerBean configManagerBean = getBean( pwmRequest );
-        LOGGER.debug( pwmRequest, () -> "beginning restSmsHealthCheck" );
 
         final DomainID domainID = DomainStateReader.forRequest( pwmRequest ).getDomainID( PwmSetting.LDAP_SERVER_URLS );
         final DomainConfig config = AppConfig.forStoredConfig( configManagerBean.getStoredConfiguration() ).getDomainConfigs().get( domainID );
@@ -644,7 +644,6 @@ public class ConfigEditorServlet extends ControlledPwmServlet
 
         final RestResultBean<String> restResultBean = RestResultBean.withData( output.toString(), String.class );
         pwmRequest.outputJsonResult( restResultBean );
-        LOGGER.debug( pwmRequest, () -> "completed restSmsHealthCheck in " + TimeDuration.fromCurrent( startTime ).asCompactString() );
         return ProcessStatus.Halt;
     }
 
@@ -658,8 +657,6 @@ public class ConfigEditorServlet extends ControlledPwmServlet
         final ConfigManagerBean configManagerBean = getBean( pwmRequest );
         final ProfileID profileID = ProfileID.create( pwmRequest.readParameterAsString( REQ_PARAM_PROFILE ) );
 
-        LOGGER.debug( pwmRequest, () -> "beginning restEmailHealthCheck" );
-
         final Map<String, String> params = pwmRequest.readBodyAsJsonStringMap();
         final EmailItemBean testEmailItem = new EmailItemBean(
                 params.get( "to" ),
@@ -705,7 +702,6 @@ public class ConfigEditorServlet extends ControlledPwmServlet
 
         final RestResultBean<String> restResultBean = RestResultBean.withData( output.toString(), String.class );
         pwmRequest.outputJsonResult( restResultBean );
-        LOGGER.debug( pwmRequest, () -> "completed restEmailHealthCheck in " + TimeDuration.fromCurrent( startTime ).asCompactString() );
         return ProcessStatus.Halt;
     }
 
@@ -771,7 +767,6 @@ public class ConfigEditorServlet extends ControlledPwmServlet
                 storedConfiguration,
                 navTreeSettings );
 
-        LOGGER.trace( pwmRequest, () -> "completed navigation tree data request in " + TimeDuration.fromCurrent( startTime ).asCompactString() );
         pwmRequest.outputJsonResult( RestResultBean.withData( navigationData, List.class ) );
         return ProcessStatus.Halt;
     }

server/src/main/java/password/pwm/http/servlet/configguide/ConfigGuideServlet.java

@@ -103,9 +103,14 @@ import java.util.Optional;
 )
 public class ConfigGuideServlet extends ControlledPwmServlet
 {
-
     private static final PwmLogger LOGGER = PwmLogger.getLogger( ConfigGuideServlet.class.getName() );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     private static final ProfileID LDAP_PROFILE_KEY = ProfileID.PROFILE_ID_DEFAULT;
     public static final String PARAM_STEP = "step";
     public static final String PARAM_KEY = "key";

server/src/main/java/password/pwm/http/servlet/forgottenpw/ForgottenPasswordServlet.java

@@ -128,6 +128,13 @@ public class ForgottenPasswordServlet extends ControlledPwmServlet
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( ForgottenPasswordServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
+
     public enum ForgottenPasswordAction implements AbstractPwmServlet.ProcessAction
     {
         search( HttpMethod.POST ),

server/src/main/java/password/pwm/http/servlet/helpdesk/HelpdeskServlet.java

@@ -128,9 +128,14 @@ import java.util.concurrent.Callable;
 )
 public class HelpdeskServlet extends ControlledPwmServlet
 {
-
     private static final PwmLogger LOGGER = PwmLogger.forClass( HelpdeskServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     public enum HelpdeskAction implements AbstractPwmServlet.ProcessAction
     {
         unlockIntruder( HttpMethod.POST ),

server/src/main/java/password/pwm/http/servlet/newuser/NewUserServlet.java

@@ -103,6 +103,12 @@ public class NewUserServlet extends ControlledPwmServlet
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( NewUserServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     static final String FIELD_PASSWORD1 = "password1";
     static final String FIELD_PASSWORD2 = "password2";
     static final String TOKEN_PAYLOAD_ATTR = "p";

server/src/main/java/password/pwm/http/servlet/peoplesearch/PeopleSearchServlet.java

@@ -63,6 +63,13 @@ public abstract class PeopleSearchServlet extends ControlledPwmServlet
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PeopleSearchServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
+
     private static final String PARAM_USERKEY = "userKey";
     private static final String PARAM_DEPTH = "depth";
 

server/src/main/java/password/pwm/http/servlet/setupresponses/SetupResponsesServlet.java

@@ -85,6 +85,12 @@ public class SetupResponsesServlet extends ControlledPwmServlet
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( SetupResponsesServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     public static final String PARAM_RESPONSE_MODE = "responseMode";
 
     public enum SetupResponsesAction implements ProcessAction

server/src/main/java/password/pwm/http/servlet/updateprofile/UpdateProfileServlet.java

@@ -94,9 +94,14 @@ import java.util.Optional;
 )
 public class UpdateProfileServlet extends ControlledPwmServlet
 {
-
     private static final PwmLogger LOGGER = PwmLogger.forClass( UpdateProfileServlet.class );
 
+    @Override
+    protected PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
     @Data
     public static class ValidateResponse implements Serializable
     {

server/src/main/java/password/pwm/ldap/search/UserSearchJob.java

@@ -23,10 +23,12 @@ package password.pwm.ldap.search;
 import com.novell.ldapchai.exception.ChaiOperationException;
 import com.novell.ldapchai.exception.ChaiUnavailableException;
 import com.novell.ldapchai.util.SearchHelper;
+import org.jetbrains.annotations.NotNull;
 import password.pwm.PwmDomain;
 import password.pwm.bean.UserIdentity;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
+import password.pwm.error.PwmInternalException;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.svc.PwmService;
@@ -34,6 +36,7 @@ import password.pwm.svc.stats.AvgStatistic;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogLevel;
+import password.pwm.util.logging.PwmLogManager;
 
 import java.time.Instant;
 import java.util.Collections;
@@ -41,6 +44,7 @@ import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.concurrent.FutureTask;
+import java.util.function.Supplier;
 
 class UserSearchJob implements Callable<Map<UserIdentity, Map<String, String>>>
 {
@@ -49,6 +53,7 @@ class UserSearchJob implements Callable<Map<UserIdentity, Map<String, String>>>
     private final UserSearchService userSearchService;
     private final FutureTask<Map<UserIdentity, Map<String, String>>> futureTask;
     private final Instant createTime = Instant.now();
+    private final SearchHelper searchHelper;
 
     UserSearchJob( final PwmDomain pwmDomain, final UserSearchService userSearchService, final UserSearchJobParameters userSearchJobParameters )
     {
@@ -56,42 +61,36 @@ class UserSearchJob implements Callable<Map<UserIdentity, Map<String, String>>>
         this.userSearchJobParameters = userSearchJobParameters;
         this.userSearchService = userSearchService;
         this.futureTask = new FutureTask<>( this );
+        this.searchHelper = makeSearchHelper( userSearchJobParameters );
     }
 
-    @Override
-    public Map<UserIdentity, Map<String, String>> call()
-            throws PwmOperationalException, PwmUnrecoverableException
+    private static SearchHelper makeSearchHelper( final UserSearchJobParameters userSearchJobParameters )
     {
-        final TimeDuration queueLagDuration = TimeDuration.fromCurrent( createTime );
-
         final SearchHelper searchHelper = new SearchHelper();
         searchHelper.setMaxResults( userSearchJobParameters.getMaxResults() );
         searchHelper.setFilter( userSearchJobParameters.getSearchFilter() );
         searchHelper.setAttributes( userSearchJobParameters.getReturnAttributes() );
         searchHelper.setTimeLimit( ( int ) userSearchJobParameters.getTimeoutMs() );
         searchHelper.setSearchScope( userSearchJobParameters.getSearchScope().getChaiSearchScope() );
+        return searchHelper;
+    }
 
-
-        final String debugInfo;
-        {
-            final Map<String, String> props = new LinkedHashMap<>();
-            props.put( "profile", userSearchJobParameters.getLdapProfile().getId().stringValue() );
-            props.put( "base", userSearchJobParameters.getContext() );
-            props.put( "maxCount", String.valueOf( searchHelper.getMaxResults() ) );
-            props.put( "queueLag", queueLagDuration.asCompactString() );
-            debugInfo = "[" + StringUtil.mapToString( props ) + "]";
-        }
-
-        userSearchService.log( PwmLogLevel.TRACE, userSearchJobParameters.getSessionLabel(), userSearchJobParameters.getSearchID(), userSearchJobParameters.getJobId(),
-                "performing ldap search for user, thread=" + Thread.currentThread().getId()
-                        + ", timeout=" + userSearchJobParameters.getTimeoutMs() + "ms, "
-                        + debugInfo );
+    @Override
+    public Map<UserIdentity, Map<String, String>> call()
+            throws PwmOperationalException, PwmUnrecoverableException
+    {
+        log( () -> "performing ldap search for user, thread=" + Thread.currentThread().getId()
+                + ", timeout=" + userSearchJobParameters.getTimeoutMs() + "ms" );
 
         final Instant startTime = Instant.now();
         final Map<String, Map<String, String>> results = new LinkedHashMap<>();
         try
         {
-            results.putAll( userSearchJobParameters.getChaiProvider().search( userSearchJobParameters.getContext(), searchHelper ) );
+            PwmLogManager.executeWithThreadSessionData( userSearchJobParameters.getSessionLabel(), () ->
+            {
+                results.putAll( userSearchJobParameters.getChaiProvider().search( userSearchJobParameters.getContext(), searchHelper ) );
+                return null;
+            } );
         }
         catch ( final ChaiUnavailableException e )
         {
@@ -103,10 +102,16 @@ class UserSearchJob implements Callable<Map<UserIdentity, Map<String, String>>>
             {
                 final PwmError pwmError = PwmError.forChaiError( e.getErrorCode() ).orElse( PwmError.ERROR_INTERNAL );
                 final String msg = "ldap error during searchID="
-                        + userSearchJobParameters.getSearchID() + ", context=" + userSearchJobParameters.getContext() + ", error=" + e.getMessage();
+                        + userSearchJobParameters.getSearchID() + ", context=" + userSearchJobParameters.getContext()
+                        + ", error=" + e.getMessage();
                 throw new PwmOperationalException( pwmError, msg );
             }
         }
+        catch ( final Exception e )
+        {
+            throw new PwmInternalException( e );
+        }
+
 
         final TimeDuration searchDuration = TimeDuration.fromCurrent( startTime );
 
@@ -117,13 +122,11 @@ class UserSearchJob implements Callable<Map<UserIdentity, Map<String, String>>>
 
         if ( results.isEmpty() )
         {
-            userSearchService.log( PwmLogLevel.TRACE, userSearchJobParameters.getSessionLabel(), userSearchJobParameters.getSearchID(), userSearchJobParameters.getJobId(),
-                    "no matches from search (" + searchDuration.asCompactString() + "); " + debugInfo );
+            log( () -> "no matches from search (" + searchDuration.asCompactString() );
             return Collections.emptyMap();
         }
 
-        userSearchService.log( PwmLogLevel.TRACE, userSearchJobParameters.getSessionLabel(), userSearchJobParameters.getSearchID(), userSearchJobParameters.getJobId(),
-                "found " + results.size() + " results in " + searchDuration.asCompactString() + "; " + debugInfo );
+        log( () -> "found " + results.size() + " results in " + searchDuration.asCompactString() );
 
         final Map<UserIdentity, Map<String, String>> returnMap = new LinkedHashMap<>( results.size() );
         for ( final Map.Entry<String, Map<String, String>> entry : results.entrySet() )
@@ -140,6 +143,29 @@ class UserSearchJob implements Callable<Map<UserIdentity, Map<String, String>>>
         return returnMap;
     }
 
+    private void log( final Supplier<String> message )
+    {
+        final TimeDuration queueLagDuration = TimeDuration.fromCurrent( createTime );
+
+        userSearchService.log(
+                PwmLogLevel.TRACE,
+                userSearchJobParameters.getSessionLabel(),
+                userSearchJobParameters.getSearchID(),
+                userSearchJobParameters.getJobId(),
+                () -> message.get() + "; " + makeDebugString( queueLagDuration ) );
+    }
+
+    @NotNull
+    private String makeDebugString( final TimeDuration queueLagDuration )
+    {
+        final Map<String, String> props = new LinkedHashMap<>();
+        props.put( "profile", userSearchJobParameters.getLdapProfile().getId().stringValue() );
+        props.put( "base", userSearchJobParameters.getContext() );
+        props.put( "maxCount", String.valueOf( searchHelper.getMaxResults() ) );
+        props.put( "queueLag", queueLagDuration.asCompactString() );
+        return "[" + StringUtil.mapToString( props ) + "]";
+    }
+
     public UserSearchJobParameters getUserSearchJobParameters()
     {
         return userSearchJobParameters;

server/src/main/java/password/pwm/ldap/search/UserSearchService.java

@@ -71,6 +71,7 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ThreadPoolExecutor;
+import java.util.function.Supplier;
 
 
 public class UserSearchService extends AbstractPwmService implements PwmService
@@ -589,7 +590,7 @@ public class UserSearchService extends AbstractPwmService implements PwmService
             final String filterText = ", filter: " + firstParam.getSearchFilter();
             final SessionLabel sessionLabel = firstParam.getSessionLabel();
             final int searchID = firstParam.getSearchID();
-            log( PwmLogLevel.DEBUG, sessionLabel, searchID, -1, "beginning user search process with " + userSearchJobs.size() + " search jobs" + filterText );
+            log( PwmLogLevel.DEBUG, sessionLabel, searchID, -1, () -> "beginning user search process with " + userSearchJobs.size() + " search jobs" + filterText );
         }
 
         // execute jobs
@@ -626,14 +627,15 @@ public class UserSearchService extends AbstractPwmService implements PwmService
                 catch ( final Throwable t )
                 {
                     log( PwmLogLevel.ERROR, firstParam.getSessionLabel(), firstParam.getSearchID(), firstParam.getJobId(),
-                            "unexpected error running job in local thread: " + t.getMessage() );
+                            () -> "unexpected error running job in local thread: " + t.getMessage() );
                 }
             }
         }
 
         final Map<UserIdentity, Map<String, String>> results = aggregateJobResults( userSearchJobs );
 
-        log( PwmLogLevel.DEBUG, firstParam.getSessionLabel(), firstParam.getSearchID(), -1, "completed user search process in "
+        log( PwmLogLevel.DEBUG, firstParam.getSessionLabel(), firstParam.getSearchID(), -1,
+                () -> "completed user search process in "
                 + TimeDuration.fromCurrent( startTime ).asCompactString()
                 + ", intermediate result size=" + results.size() );
 
@@ -667,10 +669,10 @@ public class UserSearchService extends AbstractPwmService implements PwmService
                 }
                 catch ( final InterruptedException e )
                 {
-                    final String errorMsg = "unexpected interruption during search job execution: " + e.getMessage();
+                    final Supplier<String> errorMsg = () -> "unexpected interruption during search job execution: " + e.getMessage();
                     log( PwmLogLevel.WARN, params.getSessionLabel(), params.getSearchID(), params.getJobId(), errorMsg );
-                    LOGGER.error( params.getSessionLabel(), () -> errorMsg, e );
-                    throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_INTERNAL, errorMsg ) );
+                    LOGGER.error( params.getSessionLabel(), errorMsg, e );
+                    throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_INTERNAL, errorMsg.get() ) );
                 }
                 catch ( final ExecutionException e )
                 {
@@ -688,7 +690,8 @@ public class UserSearchService extends AbstractPwmService implements PwmService
                     {
                         errorInformation = new ErrorInformation( PwmError.ERROR_LDAP_DATA_ERROR, errorMsg );
                     }
-                    log( PwmLogLevel.WARN, params.getSessionLabel(), params.getSearchID(), params.getJobId(), "error during user search: " + errorInformation.toDebugStr() );
+                    log( PwmLogLevel.WARN, params.getSessionLabel(), params.getSearchID(), params.getJobId(),
+                            () -> "error during user search: " + errorInformation.toDebugStr() );
                     throw new PwmUnrecoverableException( errorInformation );
                 }
             }
@@ -721,10 +724,10 @@ public class UserSearchService extends AbstractPwmService implements PwmService
         LOGGER.trace( getSessionLabel(), () -> "periodic debug status: " + StringUtil.mapToString( debugProperties() ) );
     }
 
-    void log( final PwmLogLevel level, final SessionLabel sessionLabel, final int searchID, final int jobID, final String message )
+    void log( final PwmLogLevel level, final SessionLabel sessionLabel, final int searchID, final int jobID, final Supplier<String> message )
     {
         final String idMsg = "domain=" + pwmDomain.getDomainID() + " " + logIdString( searchID, jobID );
-        LOGGER.log( level, sessionLabel, () -> idMsg + " " + message );
+        LOGGER.log( level, sessionLabel, () -> idMsg + " " + message.get() );
     }
 
     private static String logIdString( final int searchID, final int jobID )

server/src/main/java/password/pwm/svc/AbstractPwmService.java

@@ -33,6 +33,7 @@ import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.LazySupplier;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogManager;
 
 import java.util.ArrayList;
 import java.util.EnumSet;
@@ -40,6 +41,7 @@ import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
 
 public abstract class AbstractPwmService implements PwmService
 {
@@ -171,8 +173,46 @@ public abstract class AbstractPwmService implements PwmService
         return EnumSet.of( PwmApplication.Condition.RunningMode, PwmApplication.Condition.LocalDBOpen, PwmApplication.Condition.NotInternalInstance );
     }
 
-    protected ScheduledExecutorService getExecutorService()
+    protected void scheduleFixedRateJob( final Runnable runnable, final TimeDuration initialDelay, final TimeDuration repeatInterval )
     {
-        return executorService.get();
+        pwmApplication.getPwmScheduler().scheduleFixedRateJob( new WrappedRunnable( runnable ),
+                executorService.get(),
+                initialDelay,
+                repeatInterval );
+    }
+
+    protected ScheduledFuture<?> scheduleJob( final Runnable runnable )
+    {
+        return scheduleJob( runnable, TimeDuration.ZERO );
+    }
+
+    protected ScheduledFuture<?> scheduleJob( final Runnable runnable, final TimeDuration initialDelay )
+    {
+        return pwmApplication.getPwmScheduler().scheduleJob( new WrappedRunnable( runnable ),
+                executorService.get(),
+                initialDelay );
+    }
+
+    protected void scheduleDailyZuluZeroStartJob( final Runnable runnable, final TimeDuration zuluOffset )
+    {
+        pwmApplication.getPwmScheduler().scheduleDailyZuluZeroStartJob( new WrappedRunnable( runnable ),
+                executorService.get(),
+                zuluOffset );
+    }
+
+    private class WrappedRunnable implements Runnable
+    {
+        private final Runnable runnable;
+
+        WrappedRunnable( final Runnable runnable )
+        {
+            this.runnable = runnable;
+        }
+
+        @Override
+        public void run()
+        {
+            PwmLogManager.executeWithThreadSessionData( getSessionLabel(), runnable );
+        }
     }
 }

server/src/main/java/password/pwm/svc/PwmServiceEnum.java

@@ -59,6 +59,7 @@ public enum PwmServiceEnum
     LdapSystemService( password.pwm.ldap.LdapSystemService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
     TokenSystemService( password.pwm.svc.token.TokenSystemService.class, PwmSettingScope.SYSTEM ),
     HealthMonitor( HealthService.class, PwmSettingScope.SYSTEM ),
+    DebugOutputService( password.pwm.health.DebugOutputService.class, PwmSettingScope.SYSTEM ),
     ReportService( password.pwm.svc.report.ReportService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
     SessionTrackService( password.pwm.svc.sessiontrack.SessionTrackService.class, PwmSettingScope.SYSTEM ),
     SessionStateSvc( password.pwm.http.state.SessionStateService.class, PwmSettingScope.SYSTEM ),

server/src/main/java/password/pwm/svc/db/DatabaseService.java

@@ -106,7 +106,7 @@ public class DatabaseService extends AbstractPwmService implements PwmService
                 Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.DB_CONNECTIONS_WATCHDOG_FREQUENCY_SECONDS ) ),
                 TimeDuration.Unit.SECONDS );
 
-        pwmApplication.getPwmScheduler().scheduleFixedRateJob( new ConnectionMonitor(), getExecutorService(), watchdogFrequency, watchdogFrequency );
+        scheduleFixedRateJob( new ConnectionMonitor(), watchdogFrequency, watchdogFrequency );
 
         return dbInit();
     }

server/src/main/java/password/pwm/svc/event/AuditEvent.java

@@ -24,10 +24,12 @@ import password.pwm.config.SettingReader;
 import password.pwm.i18n.Admin;
 import password.pwm.i18n.Message;
 import password.pwm.i18n.PwmDisplayBundle;
+import password.pwm.util.java.JavaHelper;
 import password.pwm.util.json.JsonFactory;
 
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.ResourceBundle;
 
@@ -107,20 +109,11 @@ public enum AuditEvent
 
     public static Optional<AuditEvent> forKey( final String key )
     {
-        for ( final AuditEvent loopEvent : AuditEvent.values() )
+        return JavaHelper.readEnumFromPredicate( AuditEvent.class, auditEvent ->
         {
-            final Message message = loopEvent.getMessage();
-            if ( message != null )
-            {
-                final String resourceKey = message.getKey();
-                if ( resourceKey.equals( key ) )
-                {
-                    return Optional.of( loopEvent );
-                }
-            }
-        }
-
-        return Optional.empty();
+            final Message message = auditEvent.getMessage();
+            return message != null && Objects.equals( message.getKey(), key );
+        } );
     }
 
     public String getLocalizedString( final SettingReader config, final Locale locale )

server/src/main/java/password/pwm/svc/intruder/IntruderSystemService.java

@@ -169,7 +169,7 @@ public class IntruderSystemService extends AbstractPwmService implements PwmServ
             }
         };
 
-        getPwmApplication().getPwmScheduler().scheduleFixedRateJob( cleanerJob, getExecutorService(), TimeDuration.SECONDS_10, cleanerRunFrequency );
+        scheduleFixedRateJob( cleanerJob, TimeDuration.SECONDS_10, cleanerRunFrequency );
     }
 
     IntruderRecordStore getRecordStore()

server/src/main/java/password/pwm/svc/node/NodeMachine.java

@@ -35,7 +35,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ScheduledExecutorService;
 
 class NodeMachine
 {
@@ -53,7 +52,6 @@ class NodeMachine
 
     NodeMachine(
             final PwmApplication pwmApplication,
-            final ScheduledExecutorService executorService,
             final NodeDataServiceProvider clusterDataServiceProvider,
             final NodeServiceSettings nodeServiceSettings
     )
@@ -61,8 +59,6 @@ class NodeMachine
         this.pwmApplication = pwmApplication;
         this.clusterDataServiceProvider = clusterDataServiceProvider;
         this.settings = nodeServiceSettings;
-
-        pwmApplication.getPwmScheduler().scheduleFixedRateJob( new HeartbeatProcess(), executorService, settings.getHeartbeatInterval(), settings.getHeartbeatInterval() );
     }
 
     public void close( )
@@ -153,6 +149,11 @@ class NodeMachine
         return lastError;
     }
 
+    protected HeartbeatProcess getHeartbeatProcess()
+    {
+        return new HeartbeatProcess();
+    }
+
     private class HeartbeatProcess implements Runnable
     {
         @Override

server/src/main/java/password/pwm/svc/node/NodeService.java

@@ -94,7 +94,8 @@ public class NodeService extends AbstractPwmService implements PwmService
 
                 }
 
-                nodeMachine = new NodeMachine( pwmApplication, getExecutorService(), clusterDataServiceProvider, nodeServiceSettings );
+                nodeMachine = new NodeMachine( pwmApplication, clusterDataServiceProvider, nodeServiceSettings );
+                scheduleFixedRateJob( nodeMachine.getHeartbeatProcess(), nodeServiceSettings.getHeartbeatInterval(), nodeServiceSettings.getHeartbeatInterval() );
             }
         }
         catch ( final PwmUnrecoverableException e )

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyService.java

@@ -140,7 +140,7 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
 
             engine = new PwNotifyEngine( this, pwmDomain, storageService, null );
 
-            pwmDomain.getPwmApplication().getPwmScheduler().scheduleFixedRateJob( new PwNotifyJob(), getExecutorService(), TimeDuration.MINUTE, TimeDuration.MINUTE );
+            scheduleFixedRateJob( new PwNotifyJob(), TimeDuration.MINUTE, TimeDuration.MINUTE );
         }
         catch ( final PwmUnrecoverableException e )
         {
@@ -256,7 +256,7 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
         if ( !isRunning() )
         {
             nextExecutionTime = Instant.now();
-            pwmDomain.getPwmApplication().getPwmScheduler().scheduleJob( new PwNotifyJob(), getExecutorService(), TimeDuration.ZERO );
+           scheduleJob( new PwNotifyJob() );
         }
     }
 

server/src/main/java/password/pwm/svc/report/ReportService.java

@@ -119,7 +119,7 @@ public class ReportService extends AbstractPwmService implements PwmService
 
         if ( !pwmApplication.getPwmEnvironment().isInternalRuntimeInstance() )
         {
-            getExecutorService().submit( new InitializationTask() );
+            scheduleJob( new InitializationTask() );
         }
 
         return STATUS.OPEN;
@@ -178,8 +178,8 @@ public class ReportService extends AbstractPwmService implements PwmService
                         && localReportStatus.getCurrentProcess() != ReportStatusInfo.ReportEngineProcess.SearchLDAP
                 )
                 {
-                    getExecutorService().execute( new ClearTask() );
-                    getExecutorService().execute( new ReadLDAPTask() );
+                    scheduleJob( new ClearTask() );
+                    scheduleJob( new ReadLDAPTask() );
                 }
             }
             break;
@@ -194,7 +194,7 @@ public class ReportService extends AbstractPwmService implements PwmService
             case Clear:
             {
                 cancelFlag.set( true );
-                getExecutorService().execute( new ClearTask() );
+                scheduleJob( new ClearTask() );
             }
             break;
 
@@ -316,7 +316,7 @@ public class ReportService extends AbstractPwmService implements PwmService
             try
             {
                 readUserListFromLdap();
-                getExecutorService().execute( new ProcessWorkQueueTask() );
+                scheduleJob( new ProcessWorkQueueTask() );
             }
             catch ( final Exception e )
             {
@@ -325,13 +325,10 @@ public class ReportService extends AbstractPwmService implements PwmService
                 {
                     if ( ( ( PwmException ) e ).getErrorInformation().getError() == PwmError.ERROR_DIRECTORY_UNAVAILABLE )
                     {
-                        if ( getExecutorService() != null )
-                        {
                             LOGGER.error( getSessionLabel(),
                                     () -> "directory unavailable error during background SearchLDAP, will retry; error: " + e.getMessage() );
-                            getPwmApplication().getPwmScheduler().scheduleJob( new ReadLDAPTask(), getExecutorService(), TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
+                            scheduleJob( new ReadLDAPTask(), TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
                             errorProcessed = true;
-                        }
                     }
                 }
 
@@ -439,12 +436,8 @@ public class ReportService extends AbstractPwmService implements PwmService
             {
                 if ( e.getErrorInformation().getError() == PwmError.ERROR_DIRECTORY_UNAVAILABLE )
                 {
-                    if ( getExecutorService() != null )
-                    {
                         LOGGER.error( getSessionLabel(), () -> "directory unavailable error during background ReadData, will retry; error: " + e.getMessage() );
-                        getPwmApplication().getPwmScheduler().scheduleJob(
-                                new ProcessWorkQueueTask(), getExecutorService(), TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
-                    }
+                        scheduleJob( new ProcessWorkQueueTask(), TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
                 }
                 else
                 {
@@ -635,8 +628,8 @@ public class ReportService extends AbstractPwmService implements PwmService
 
             if ( settings.isDailyJobEnabled() )
             {
-                getExecutorService().execute( new ClearTask() );
-                getExecutorService().execute( new ReadLDAPTask() );
+                scheduleJob( new ClearTask() );
+                scheduleJob( new ReadLDAPTask() );
             }
         }
     }
@@ -666,7 +659,7 @@ public class ReportService extends AbstractPwmService implements PwmService
             if ( reportingEnabled )
             {
                 final TimeDuration jobOffset = TimeDuration.of( settings.getJobOffsetSeconds(), TimeDuration.Unit.SECONDS );
-                getPwmApplication().getPwmScheduler().scheduleDailyZuluZeroStartJob( new DailyJobExecuteTask(), getExecutorService(), jobOffset );
+                scheduleJob( new DailyJobExecuteTask(), jobOffset );
             }
         }
 
@@ -721,7 +714,7 @@ public class ReportService extends AbstractPwmService implements PwmService
             if ( !reportStatus.get().isReportComplete() && !dnQueue.isEmpty() )
             {
                 LOGGER.trace( getSessionLabel(), () -> "resuming report data processing" );
-                getExecutorService().execute( new ProcessWorkQueueTask() );
+                scheduleJob( new ProcessWorkQueueTask() );
             }
         }
     }
@@ -784,7 +777,7 @@ public class ReportService extends AbstractPwmService implements PwmService
         final Instant lastFinishDate = reportStatus.get().getFinishDate();
         if ( lastFinishDate != null && TimeDuration.fromCurrent( lastFinishDate ).isLongerThan( settings.getMaxCacheAge() ) )
         {
-            getExecutorService().execute( new ClearTask() );
+            scheduleJob( new ClearTask() );
         }
     }
 }

server/src/main/java/password/pwm/svc/stats/StatisticsService.java

@@ -273,9 +273,9 @@ public class StatisticsService extends AbstractPwmService implements PwmService
 
         {
             // setup a timer to roll over at 0 Zulu and one to write current stats regularly
-            pwmApplication.getPwmScheduler().scheduleDailyZuluZeroStartJob( new DailySummaryJob( pwmApplication ), getExecutorService(), TimeDuration.ZERO );
-            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new FlushTask(), getExecutorService(), DB_WRITE_FREQUENCY, DB_WRITE_FREQUENCY );
-            pwmApplication.getPwmScheduler().scheduleDailyZuluZeroStartJob( new NightlyTask(), getExecutorService(), TimeDuration.ZERO );
+            scheduleDailyZuluZeroStartJob( new DailySummaryJob( pwmApplication ), TimeDuration.ZERO );
+            scheduleFixedRateJob( new FlushTask(), DB_WRITE_FREQUENCY, DB_WRITE_FREQUENCY );
+            scheduleDailyZuluZeroStartJob( new NightlyTask(), TimeDuration.ZERO );
         }
 
         return STATUS.OPEN;

server/src/main/java/password/pwm/svc/telemetry/TelemetryService.java

@@ -193,7 +193,7 @@ public class TelemetryService extends AbstractPwmService implements PwmService
     private void scheduleNextJob( )
     {
         final TimeDuration durationUntilNextPublish = durationUntilNextPublish();
-        getPwmApplication().getPwmScheduler().scheduleJob( new PublishJob(), getExecutorService(), durationUntilNextPublish );
+        scheduleJob( new PublishJob(), durationUntilNextPublish );
         LOGGER.trace( getSessionLabel(), () -> "next publish time: " + durationUntilNextPublish().asCompactString() );
     }
 

server/src/main/java/password/pwm/svc/token/TokenSystemService.java

@@ -44,7 +44,7 @@ public class TokenSystemService extends AbstractPwmService implements PwmService
         {
             final int cleanerFrequencySeconds = Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.TOKEN_CLEANER_INTERVAL_SECONDS ) );
             final TimeDuration cleanerFrequency = TimeDuration.of( cleanerFrequencySeconds, TimeDuration.Unit.SECONDS );
-            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new CleanerTask(), getExecutorService(), TimeDuration.MINUTE, cleanerFrequency );
+            scheduleFixedRateJob( new CleanerTask(), TimeDuration.MINUTE, cleanerFrequency );
             LOGGER.trace( getSessionLabel(), () -> "token cleanup will occur every " + cleanerFrequency.asCompactString() );
         }
 

server/src/main/java/password/pwm/svc/version/VersionCheckService.java

@@ -62,7 +62,6 @@ import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.TimeUnit;
 
 public class VersionCheckService extends AbstractPwmService
 {
@@ -156,7 +155,7 @@ public class VersionCheckService extends AbstractPwmService
 
         final TimeDuration delayUntilNextExecution = TimeDuration.fromCurrent( this.nextScheduledCheck );
 
-        getExecutorService().schedule( new PeriodicCheck(), delayUntilNextExecution.asMillis(), TimeUnit.MILLISECONDS );
+        scheduleJob( new PeriodicCheck(), delayUntilNextExecution );
 
         LOGGER.trace( getSessionLabel(), () -> "scheduled next check execution at " + StringUtil.toIsoDate( nextScheduledCheck )
                 + " in " + delayUntilNextExecution.asCompactString() );

server/src/main/java/password/pwm/svc/wordlist/SharedHistoryService.java

@@ -240,7 +240,7 @@ public class SharedHistoryService extends AbstractPwmService implements PwmServi
             final TimeDuration frequency = TimeDuration.of( frequencyMs, TimeDuration.Unit.MILLISECONDS );
 
             LOGGER.debug( () -> "scheduling cleaner task to run once every " + frequency.asCompactString() );
-            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new CleanerTask(), getExecutorService(), TimeDuration.ZERO, frequency );
+            scheduleFixedRateJob( new CleanerTask(), TimeDuration.ZERO, frequency );
         }
     }
 

server/src/main/java/password/pwm/util/logging/LocalDBLogbackAppender.java

@@ -22,9 +22,6 @@ package password.pwm.util.logging;
 
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.UnsynchronizedAppenderBase;
-import org.slf4j.Marker;
-
-import java.util.List;
 
 public class LocalDBLogbackAppender extends UnsynchronizedAppenderBase<ILoggingEvent>
 {
@@ -38,8 +35,7 @@ public class LocalDBLogbackAppender extends UnsynchronizedAppenderBase<ILoggingE
     @Override
     protected void append( final ILoggingEvent event )
     {
-        final List<Marker> markerList = event.getMarkerList();
-        if ( markerList != null && markerList.contains( PwmLogbackMarker.singleton() ) )
+        if ( PwmLogUtil.eventIsPwmGenerated( event ) )
         {
             // event was generated by PWM, which has already sent the event to the localDB
             return;

server/src/main/java/password/pwm/util/logging/LocalDBLogger.java

@@ -23,6 +23,7 @@ package password.pwm.util.logging;
 import password.pwm.AppAttribute;
 import password.pwm.PwmApplication;
 import password.pwm.bean.DomainID;
+import password.pwm.bean.SessionLabel;
 import password.pwm.config.option.DataStorageMethod;
 import password.pwm.error.PwmException;
 import password.pwm.health.HealthMessage;
@@ -70,6 +71,8 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( LocalDBLogger.class );
 
+    private static final SessionLabel SESSION_LABEL = SessionLabel.SYSTEM_LABEL;
+
     private final LocalDBLoggerSettings settings;
     private final LocalDBStoredQueue localDBListQueue;
     private final Queue<PwmLogMessage> tempMemoryEventQueue;
@@ -130,7 +133,7 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
 
         if ( this.settings.getMaxEvents() == 0 )
         {
-            LOGGER.info( () -> "maxEvents set to zero, clearing LocalDBLogger history and LocalDBLogger will remain closed" );
+            LOGGER.info( SESSION_LABEL, () -> "maxEvents set to zero, clearing LocalDBLogger history and LocalDBLogger will remain closed" );
             localDBListQueue.clear();
             throw new IllegalArgumentException( "maxEvents=0, will remain closed" );
         }
@@ -144,7 +147,7 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
                     {
                         if ( !STORAGE_FORMAT_VERSION.equals( currentFormat ) )
                         {
-                            LOGGER.warn( () -> "localdb logger is using outdated format, clearing existing records (existing='"
+                            LOGGER.warn( SESSION_LABEL, () -> "localdb logger is using outdated format, clearing existing records (existing='"
                                     + currentFormat + "', current='" + STORAGE_FORMAT_VERSION + "')" );
 
                             localDBListQueue.clear();
@@ -190,7 +193,7 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
         }
         catch ( final Exception e )
         {
-            LOGGER.error( () -> "unexpected error attempting to determine tail event timestamp: " + e.getMessage() );
+            LOGGER.error( SESSION_LABEL, () -> "unexpected error attempting to determine tail event timestamp: " + e.getMessage() );
         }
 
         return Optional.empty();
@@ -239,7 +242,7 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
     {
         if ( lastLogOutput.get() + stats.get( CounterStat.EventsWritten ) > LOG_OUTPUT_INCREMENTS )
         {
-            LOGGER.trace( () -> "periodic debug output: " + StringUtil.mapToString( debugStats() ) );
+            LOGGER.trace( SESSION_LABEL, () -> "periodic debug output: " + StringUtil.mapToString( debugStats() ) );
             lastLogOutput.set( stats.get( CounterStat.EventsWritten ) );
         }
     }
@@ -252,7 +255,7 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
         final int flushedEvents;
         if ( status() != STATUS.CLOSED )
         {
-            LOGGER.trace( () -> "LocalDBLogger closing" );
+            LOGGER.trace( SESSION_LABEL, () -> "LocalDBLogger closing" );
             flushedEvents = tempMemoryEventQueue.size();
             if ( cleanerService != null )
             {
@@ -270,11 +273,11 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
 
         if ( flushedEvents > 0 )
         {
-            LOGGER.trace( () -> "LocalDBLogger close completed (flushed during close: " + flushedEvents + ")", TimeDuration.fromCurrent( startTime ) );
+            LOGGER.trace( SESSION_LABEL, () -> "LocalDBLogger close completed (flushed during close: " + flushedEvents + ")", TimeDuration.fromCurrent( startTime ) );
         }
         else
         {
-            LOGGER.trace( () -> "LocalDBLogger close completed", TimeDuration.fromCurrent( startTime ) );
+            LOGGER.trace( SESSION_LABEL, () -> "LocalDBLogger close completed", TimeDuration.fromCurrent( startTime ) );
         }
     }
 
@@ -349,7 +352,7 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
             if ( !hasShownReadError )
             {
                 hasShownReadError = true;
-                LOGGER.error( () -> "error reading localDBLogger event: " + e.getMessage() );
+                LOGGER.error( SESSION_LABEL, () -> "error reading localDBLogger event: " + e.getMessage() );
             }
         }
         return null;
@@ -385,7 +388,7 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
         }
         catch ( final PatternSyntaxException e )
         {
-            LOGGER.trace( () -> "invalid regex syntax for " + searchParameters.getUsername() + ", reverting to plaintext search" );
+            LOGGER.trace( SESSION_LABEL, () -> "invalid regex syntax for " + searchParameters.getUsername() + ", reverting to plaintext search" );
         }
 
         if ( pattern != null )
@@ -473,7 +476,7 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
         {
             if ( TimeDuration.fromCurrent( startTime ).isLongerThan( settings.getMaxBufferWaitTime() ) )
             {
-                LOGGER.warn( () -> "discarded event after waiting max buffer wait time of " + settings.getMaxBufferWaitTime().asCompactString() );
+                LOGGER.warn( SESSION_LABEL, () -> "discarded event after waiting max buffer wait time of " + settings.getMaxBufferWaitTime().asCompactString() );
                 return;
             }
             TimeDuration.of( 100, TimeDuration.Unit.MILLISECONDS ).pause();
@@ -519,7 +522,7 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
         }
         catch ( final Exception e )
         {
-            LOGGER.error( () -> "error writing to localDBLogger: " + e.getMessage(), e );
+            LOGGER.error( SESSION_LABEL, () -> "error writing to localDBLogger: " + e.getMessage(), e );
         }
 
         debugOutputter.conditionallyExecuteTask();

server/src/main/java/password/pwm/util/logging/PwmLogEvent.java

@@ -44,6 +44,7 @@ import java.util.List;
 @Value
 @AllArgsConstructor( access = AccessLevel.PRIVATE )
 @Builder( access = AccessLevel.PRIVATE, toBuilder = true )
+@SuppressWarnings( "checkstyle:ParameterNumber" )
 public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
 {
     private static final int MAX_MESSAGE_LENGTH = 10_000;
@@ -60,6 +61,7 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
     private final String domain;
     private final String sourceAddress;
     private final Duration duration;
+    private final String threadName;
 
     private static final Comparator<PwmLogEvent> COMPARATOR = Comparator.comparing(
             PwmLogEvent::getTimestamp,
@@ -87,7 +89,8 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
             final SessionLabel sessionLabel,
             final LoggedThrowable loggedThrowable,
             final PwmLogLevel level,
-            final Duration duration
+            final Duration duration,
+            final String threadName
     )
     {
         if ( timestamp == null )
@@ -105,6 +108,7 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
         this.message = StringUtil.truncate( message, MAX_MESSAGE_LENGTH, " [truncated message]" );
         this.loggedThrowable = loggedThrowable;
         this.level = level;
+        this.threadName = threadName == null ? "" : StringUtil.truncate( threadName, MAX_FIELD_LENGTH );
 
         this.sessionID = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getSessionID(), MAX_FIELD_LENGTH );
         this.requestID = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getRequestID(), MAX_FIELD_LENGTH );
@@ -112,6 +116,7 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
         this.domain = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getDomain(), MAX_FIELD_LENGTH );
         this.sourceAddress = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getSourceAddress(), MAX_FIELD_LENGTH );
         this.duration = duration == null ? Duration.ZERO : duration;
+
     }
 
     public static PwmLogEvent createPwmLogEvent(
@@ -121,10 +126,11 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
             final SessionLabel sessionLabel,
             final Throwable throwable,
             final PwmLogLevel level,
-            final Duration duration
+            final Duration duration,
+            final String threadName
     )
     {
-        return new PwmLogEvent( timestamp, topic, message, sessionLabel, LoggedThrowable.fromThrowable( throwable ), level, duration );
+        return new PwmLogEvent( timestamp, topic, message, sessionLabel, LoggedThrowable.fromThrowable( throwable ), level, duration, threadName );
     }
 
     String getEnhancedMessage( )

server/src/main/java/password/pwm/util/logging/PwmLogManager.java

@@ -43,6 +43,7 @@ import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.PwmApplicationMode;
 import password.pwm.PwmConstants;
+import password.pwm.bean.SessionLabel;
 import password.pwm.config.AppConfig;
 import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.localdb.LocalDB;
@@ -53,6 +54,7 @@ import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.Callable;
 
 public class PwmLogManager
 {
@@ -62,6 +64,8 @@ public class PwmLogManager
     private static final String LOGGER_NAME_FILE = "pwmFileLogger";
     private static final String LOGGER_NAME_CONSOLE = "pwmConsoleLogger";
 
+    private static final ThreadLocal<SessionLabel> THREAD_SESSION_DATA = new ThreadLocal<>();
+
     private static PwmApplication pwmApplication;
     private static LocalDBLogger localDBLogger;
     private static PwmLogSettings pwmLogSettings = PwmLogSettings.defaultSettings();
@@ -130,13 +134,10 @@ public class PwmLogManager
         // all initialize lines start here (above line disables everything !!)
 
         PwmLogManager.pwmLogSettings = pwmLogSettings;
-
         PwmLogManager.pwmApplication = pwmApplication;
-
-        lowestLogLevelConfigured = pwmLogSettings.calculateLowestLevel();
+        PwmLogManager.lowestLogLevelConfigured = pwmLogSettings.calculateLowestLevel();
 
         initConsoleLogger( config, pwmLogSettings.getStdoutLevel() );
-
         initFileLogger( config, pwmLogSettings.getFileLevel(), pwmApplicationPath );
     }
 
@@ -385,4 +386,52 @@ public class PwmLogManager
         }
         return null;
     }
+
+    static SessionLabel getThreadSessionData()
+    {
+        return THREAD_SESSION_DATA.get();
+    }
+
+    public static void executeWithThreadSessionData( final SessionLabel sessionLabel, final Runnable runnable )
+    {
+        final SessionLabel previous = THREAD_SESSION_DATA.get();
+        THREAD_SESSION_DATA.set( sessionLabel );
+        try
+        {
+            runnable.run();
+        }
+        finally
+        {
+            if ( previous == null )
+            {
+                THREAD_SESSION_DATA.remove();
+            }
+            else
+            {
+                THREAD_SESSION_DATA.set( previous );
+            }
+        }
+    }
+
+    public static <V> V executeWithThreadSessionData( final SessionLabel sessionLabel, final Callable<V> callable )
+            throws Exception
+    {
+        final SessionLabel previous = THREAD_SESSION_DATA.get();
+        THREAD_SESSION_DATA.set( sessionLabel );
+        try
+        {
+            return callable.call();
+        }
+        finally
+        {
+            if ( previous == null )
+            {
+                THREAD_SESSION_DATA.remove();
+            }
+            else
+            {
+                THREAD_SESSION_DATA.set( previous );
+            }
+        }
+    }
 }

server/src/main/java/password/pwm/util/logging/PwmLogMessage.java

@@ -37,6 +37,7 @@ import java.util.function.Supplier;
  */
 @Value
 @AllArgsConstructor( access = AccessLevel.PRIVATE )
+@SuppressWarnings( "checkstyle:ParameterNumber" )
 public class PwmLogMessage
 {
     private Instant timestamp;
@@ -46,6 +47,7 @@ public class PwmLogMessage
     private Supplier<? extends CharSequence> message;
     private TimeDuration duration;
     private Throwable throwable;
+    private String threadName;
 
     private final Supplier<String> enhancedMessage = LazySupplier.create( () -> PwmLogMessage.createEnhancedMessage( this ) );
 
@@ -56,10 +58,11 @@ public class PwmLogMessage
             final SessionLabel sessionLabel,
             final Supplier<? extends CharSequence> message,
             final TimeDuration duration,
-            final Throwable throwable
+            final Throwable throwable,
+            final String threadName
     )
     {
-        return new PwmLogMessage( timestamp, topic, level, sessionLabel, message, duration, throwable );
+        return new PwmLogMessage( timestamp, topic, level, sessionLabel, message, duration, throwable, threadName );
     }
 
     String messageToString()
@@ -76,7 +79,8 @@ public class PwmLogMessage
                 sessionLabel,
                 throwable,
                 level,
-                duration == null ? null : duration.asDuration() );
+                duration == null ? null : duration.asDuration(),
+                threadName );
     }
 
     String getEnhancedMessage()
@@ -93,24 +97,32 @@ public class PwmLogMessage
                 pwmLogMessage.getDuration() );
     }
 
-    static PwmLogMessage fromLogbackEvent( final ILoggingEvent event  )
+    static Throwable throwableFromILoggingEvent( final ILoggingEvent event )
     {
-        final Supplier<? extends CharSequence> message = ( Supplier<CharSequence> ) event::getMessage;
-        final Throwable throwableInformation = event.getThrowableProxy() instanceof ThrowableProxy
+        return event.getThrowableProxy() instanceof ThrowableProxy
                 ? ( ( ThrowableProxy ) event.getThrowableProxy() ).getThrowable()
                 : null;
+    }
+
+    static PwmLogMessage fromLogbackEvent( final ILoggingEvent event  )
+    {
+        final Supplier<? extends CharSequence> message = ( Supplier<CharSequence> ) event::getMessage;
+        final Throwable throwableInformation = throwableFromILoggingEvent( event );
         final PwmLogLevel level = PwmLogLevel.fromLogbackLevel( event.getLevel() );
         final Instant timeStamp = Instant.ofEpochMilli( event.getTimeStamp() );
         final String sourceLogger = event.getLoggerName();
+        final String threadName = event.getThreadName();
+
+        final SessionLabel sessionLabel = PwmLogManager.getThreadSessionData();
 
         return PwmLogMessage.create(
                 timeStamp,
                 sourceLogger,
                 level,
-                null,
+                sessionLabel,
                 message,
-                TimeDuration.ZERO,
-                throwableInformation
-        );
+                null,
+                throwableInformation,
+                threadName );
     }
 }

server/src/main/java/password/pwm/util/logging/PwmLogUtil.java

@@ -22,12 +22,14 @@ package password.pwm.util.logging;
 
 import ch.qos.logback.classic.LoggerContext;
 import ch.qos.logback.classic.joran.JoranConfigurator;
+import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.joran.spi.JoranException;
 import ch.qos.logback.core.util.StatusPrinter;
 import lombok.Builder;
 import lombok.Value;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.Marker;
 import password.pwm.PwmApplication;
 import password.pwm.PwmConstants;
 import password.pwm.bean.LoginInfoBean;
@@ -43,6 +45,7 @@ import password.pwm.util.json.JsonFactory;
 
 import java.io.File;
 import java.io.Serializable;
+import java.util.List;
 import java.util.function.Function;
 import java.util.function.Supplier;
 
@@ -212,7 +215,7 @@ class PwmLogUtil
             output.append( JavaHelper.throwableToString( throwable ) );
         }
 
-        if ( duration != null )
+        if ( duration != null && duration.isLongerThan( TimeDuration.ZERO ) )
         {
             output.append( " (" ).append( duration.asCompactString() ).append( ")" );
         }
@@ -250,4 +253,10 @@ class PwmLogUtil
         }
     }
 
+    static boolean eventIsPwmGenerated( final ILoggingEvent event )
+    {
+        final List<Marker> markerList = event.getMarkerList();
+        return markerList != null && markerList.contains( PwmLogbackMarker.singleton() );
+    }
+
 }

server/src/main/java/password/pwm/util/logging/PwmLogbackPattern.java

@@ -40,14 +40,22 @@ public class PwmLogbackPattern extends LayoutBase<ILoggingEvent>
             return "[null logback event]";
         }
 
+        final String logMsg = PwmLogUtil.eventIsPwmGenerated( event )
+                ? event.getMessage()
+                : PwmLogUtil.createEnhancedMessage(
+                        PwmLogManager.getThreadSessionData(),
+                        event.getMessage(),
+                        PwmLogMessage.throwableFromILoggingEvent( event ),
+                        null );
+
         return StringUtil.toIsoDate( Instant.ofEpochMilli( event.getTimeStamp() ) )
                 + ", "
                 + printLevel( event.getLevel() )
                 + ", "
                 + printLoggerName( event.getLoggerName() )
                 + ", "
-                + event.getFormattedMessage()
-                + "\n";
+                + logMsg
+                + '\n';
     }
 
     private static String printLevel( final Level pwmLogLevel )

server/src/main/java/password/pwm/util/logging/PwmLogger.java

@@ -358,6 +358,8 @@ public class PwmLogger
             final TimeDuration timeDuration
     )
     {
+        final String threadName = Thread.currentThread().getName();
+
         final PwmLogMessage pwmLogMessage = PwmLogMessage.create(
                 Instant.now(),
                 logbackLogger.getName(),
@@ -365,7 +367,8 @@ public class PwmLogger
                 sessionLabel,
                 message,
                 timeDuration,
-                e );
+                e,
+                threadName );
 
         doLogEvent( pwmLogMessage );
     }

server/src/main/java/password/pwm/util/logging/PwmLoggerAppendable.java

@@ -76,6 +76,8 @@ class PwmLoggerAppendable implements Appendable
     {
         buffer.append( charSequence );
 
+        final String threadName = Thread.currentThread().getName();
+
         final PwmLogMessage pwmLogMessage = PwmLogMessage.create(
                 Instant.now(),
                 pwmLogger.getName(),
@@ -83,7 +85,8 @@ class PwmLoggerAppendable implements Appendable
                 sessionLabel,
                 () -> charSequence,
                 null,
-                null );
+                null,
+                threadName );
 
         int length = buffer.indexOf( "\n" );
         while ( length > 0 )

server/src/main/java/password/pwm/util/password/PasswordUtility.java

@@ -970,7 +970,7 @@ public class PasswordUtility
 
     public static PwmPasswordPolicy determineConfiguredPolicyProfileForUser(
             final PwmDomain pwmDomain,
-            final SessionLabel pwmSession,
+            final SessionLabel sessionLabel,
             final UserIdentity userIdentity
     )
             throws PwmUnrecoverableException
@@ -985,10 +985,10 @@ public class PasswordUtility
         {
             final PwmPasswordPolicy loopPolicy = pwmDomain.getConfig().getPasswordPolicy( profile );
             final List<UserPermission> userPermissions = loopPolicy.getUserPermissions();
-            LOGGER.debug( pwmSession, () -> "testing password policy profile '" + profile + "'" );
+            LOGGER.debug( sessionLabel, () -> "testing password policy profile '" + profile + "'" );
             try
             {
-                final boolean match = UserPermissionUtility.testUserPermission( pwmDomain, pwmSession, userIdentity, userPermissions );
+                final boolean match = UserPermissionUtility.testUserPermission( pwmDomain, sessionLabel, userIdentity, userPermissions );
                 if ( match )
                 {
                     return loopPolicy;
@@ -996,7 +996,7 @@ public class PasswordUtility
             }
             catch ( final PwmUnrecoverableException e )
             {
-                LOGGER.error( pwmSession, () -> "unexpected error while testing password policy profile '" + profile + "', error: " + e.getMessage() );
+                LOGGER.error( sessionLabel, () -> "unexpected error while testing password policy profile '" + profile + "', error: " + e.getMessage() );
             }
         }
 

server/src/main/java/password/pwm/ws/server/RestServlet.java

@@ -48,10 +48,12 @@ import password.pwm.svc.stats.StatisticsClient;
 import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.AtomicLoopLongIncrementer;
 import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.MutableReference;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.json.JsonFactory;
 import password.pwm.util.logging.PwmLogLevel;
+import password.pwm.util.logging.PwmLogManager;
 import password.pwm.util.logging.PwmLogger;
 
 import javax.servlet.ServletException;
@@ -129,60 +131,67 @@ public abstract class RestServlet extends HttpServlet
             return;
         }
 
-        final RestResultBean restResultBean = executeRequest( req, resp, locale, pwmApplication, pwmDomain, sessionLabel );
+        final MutableReference<RestResultBean> mutableReference = new MutableReference<>();
 
+        PwmLogManager.executeWithThreadSessionData( sessionLabel, () ->
+        {
+            mutableReference.set( executeRequest( req, resp, locale, pwmApplication, pwmDomain, sessionLabel ) );
+        } );
+
+
+        final RestResultBean restResultBean = mutableReference.get();
         outputRestResultBean( restResultBean, req, resp );
         final boolean success = restResultBean != null && !restResultBean.isError();
         LOGGER.trace( sessionLabel, () -> "completed rest invocation, success=" + success, TimeDuration.fromCurrent( startTime ) );
     }
 
-   private RestResultBean executeRequest(
-           final HttpServletRequest req,
-           final HttpServletResponse resp,
-           final Locale locale,
-           final PwmApplication pwmApplication,
-           final PwmDomain pwmDomain,
-           final SessionLabel sessionLabel
-   )
-   {
-       try
-       {
-           final RestAuthentication restAuthentication = new RestAuthenticationProcessor( pwmDomain, sessionLabel, req ).readRestAuthentication();
-           LOGGER.debug( sessionLabel, () -> "rest request authentication status: " + JsonFactory.get().serialize( restAuthentication ) );
-
-           final RestRequest restRequest = RestRequest.forRequest( pwmDomain, restAuthentication, sessionLabel, req );
-
-           RequestInitializationFilter.addStaticResponseHeaders( pwmApplication, req, resp );
-
-           preCheck( restRequest );
-
-           preCheckRequest( restRequest );
-
-           return invokeWebService( restRequest );
-       }
-       catch ( final PwmUnrecoverableException e )
-       {
-           return RestResultBean.fromError(
-                   e.getErrorInformation(),
-                   pwmDomain,
-                   locale,
-                   pwmDomain.getConfig(),
-                   pwmDomain.determineIfDetailErrorMsgShown()
-           );
-       }
-       catch ( final Throwable e )
-       {
-           final String errorMsg = "internal error during rest service invocation: " + e.getMessage();
-           final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_INTERNAL, errorMsg );
-           LOGGER.error( sessionLabel, errorInformation, e );
-           return RestResultBean.fromError(
-                   errorInformation,
-                   pwmDomain,
-                   locale,
-                   pwmDomain.getConfig(),
-                   pwmDomain.determineIfDetailErrorMsgShown() );
-       }
-   }
+    private RestResultBean executeRequest(
+            final HttpServletRequest req,
+            final HttpServletResponse resp,
+            final Locale locale,
+            final PwmApplication pwmApplication,
+            final PwmDomain pwmDomain,
+            final SessionLabel sessionLabel
+    )
+    {
+        try
+        {
+            final RestAuthentication restAuthentication = new RestAuthenticationProcessor( pwmDomain, sessionLabel, req ).readRestAuthentication();
+            LOGGER.debug( sessionLabel, () -> "rest request authentication status: " + JsonFactory.get().serialize( restAuthentication ) );
+
+            final RestRequest restRequest = RestRequest.forRequest( pwmDomain, restAuthentication, sessionLabel, req );
+
+            RequestInitializationFilter.addStaticResponseHeaders( pwmApplication, req, resp );
+
+            preCheck( restRequest );
+
+            preCheckRequest( restRequest );
+
+            return invokeWebService( restRequest );
+        }
+        catch ( final PwmUnrecoverableException e )
+        {
+            return RestResultBean.fromError(
+                    e.getErrorInformation(),
+                    pwmDomain,
+                    locale,
+                    pwmDomain.getConfig(),
+                    pwmDomain.determineIfDetailErrorMsgShown()
+            );
+        }
+        catch ( final Throwable e )
+        {
+            final String errorMsg = "internal error during rest service invocation: " + e.getMessage();
+            final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_INTERNAL, errorMsg );
+            LOGGER.error( sessionLabel, errorInformation, e );
+            return RestResultBean.fromError(
+                    errorInformation,
+                    pwmDomain,
+                    locale,
+                    pwmDomain.getConfig(),
+                    pwmDomain.determineIfDetailErrorMsgShown() );
+        }
+    }
 
     private static void logHttpRequest(
             final PwmApplication pwmApplication,

server/src/test/java/password/pwm/util/json/JsonProviderTest.java

@@ -351,7 +351,8 @@ public class JsonProviderTest
                         SessionLabel.TEST_SESSION_LABEL,
                         throwable,
                         PwmLogLevel.TRACE,
-                        Duration.ZERO
+                        Duration.ZERO,
+                        "threadName"
                 );
             }
         }

server/src/test/java/password/pwm/util/localdb/LocalDBLoggerExtendedTest.java

@@ -189,7 +189,8 @@ public class LocalDBLoggerExtendedTest
                     SessionLabel.TEST_SESSION_LABEL,
                     description,
                     TimeDuration.ZERO,
-                    null );
+                    null,
+                    "threadName" );
             events.add( event );
         }