Merge branch 'master' into enh-domainadmin

# Conflicts:
#	server/src/main/java/password/pwm/svc/pwnotify/PwNotifyEngine.java
#	server/src/main/java/password/pwm/svc/report/ReportRecordLocalDBStorageService.java
#	server/src/main/java/password/pwm/svc/report/ReportService.java
#	server/src/main/java/password/pwm/util/DailySummaryJob.java
#	server/src/main/java/password/pwm/util/PwmScheduler.java

lib-util/src/main/java/password/pwm/util/java/JavaHelper.java

@@ -24,8 +24,10 @@ import org.apache.commons.io.IOUtils;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.io.StringWriter;
@@ -205,12 +207,25 @@ public class JavaHelper
     }
 
 
-    public static String copyToString( final InputStream input, final Charset charset )
+    public static Optional<String> copyToString( final InputStream input, final Charset charset, final int maximumLength )
             throws IOException
     {
-        final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
-        JavaHelper.copy( input, byteArrayOutputStream );
-        return byteArrayOutputStream.toString( charset );
+        if ( input == null )
+        {
+            return Optional.empty();
+        }
+        final StringWriter stringWriter = new StringWriter();
+        final InputStreamReader reader = new InputStreamReader( input, charset );
+        IOUtils.copyLarge( reader, stringWriter, 0, maximumLength );
+        final String value = stringWriter.toString();
+        return ( value.length() > 0 )
+                ? Optional.of( value )
+                : Optional.empty();
+    }
+
+    public static void closeQuietly( final Closeable closable )
+    {
+        IOUtils.closeQuietly( closable );
     }
 
     public static ImmutableByteArray copyToBytes( final InputStream inputStream, final int maxLength )
@@ -510,7 +525,7 @@ public class JavaHelper
             throws IOException
     {
         try ( ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
-             GZIPOutputStream gzipOutputStream = new GZIPOutputStream( byteArrayOutputStream ) )
+              GZIPOutputStream gzipOutputStream = new GZIPOutputStream( byteArrayOutputStream ) )
         {
             gzipOutputStream.write( bytes );
             gzipOutputStream.close();
@@ -574,7 +589,7 @@ public class JavaHelper
         }
         catch ( final InterruptedException e )
         {
-           /* ignore */
+            /* ignore */
         }
     }
 

rest-test-service/pom.xml

@@ -86,6 +86,12 @@
             <artifactId>commons-io</artifactId>
             <version>1.3.2</version>
         </dependency>
+        <dependency>
+            <groupId>org.pwm-project</groupId>
+            <artifactId>pwm-lib-util</artifactId>
+            <version>2.1.0-SNAPSHOT</version>
+            <scope>compile</scope>
+        </dependency>
         <!-- / library dependencies -->
     </dependencies>
 

rest-test-service/src/main/java/password/pwm/resttest/RestTestExternalTokenDestinationServlet.java

@@ -22,7 +22,7 @@ package password.pwm.resttest;
 
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
-import org.apache.commons.io.IOUtils;
+import password.pwm.util.java.JavaHelper;
 
 import javax.servlet.annotation.WebServlet;
 import javax.servlet.http.HttpServlet;
@@ -31,6 +31,7 @@ import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
 
 @WebServlet(
         name = "RestTestExternalTokenDestinationServlet",
@@ -45,7 +46,7 @@ public class RestTestExternalTokenDestinationServlet extends HttpServlet
     {
         System.out.println( "--External Token Destination--" );
         final InputStream inputStream = req.getInputStream();
-        final String body = IOUtils.toString( inputStream );
+        final String body = JavaHelper.copyToString( inputStream, StandardCharsets.UTF_8, Integer.MAX_VALUE ).orElseThrow();
         final JsonObject jsonObject = JsonParser.parseString( body ).getAsJsonObject();
         final String email = jsonObject.getAsJsonObject( "tokenDestination" ).get( "email" ).getAsString();
         final String sms = jsonObject.getAsJsonObject( "tokenDestination" ).get( "sms" ).getAsString();

rest-test-service/src/main/java/password/pwm/resttest/RestTestUtilities.java

@@ -21,6 +21,7 @@
 package password.pwm.resttest;
 
 import org.apache.commons.io.IOUtils;
+import password.pwm.util.java.JavaHelper;
 
 import javax.servlet.http.HttpServletRequest;
 import java.io.IOException;
@@ -51,7 +52,7 @@ public class RestTestUtilities
         }
         finally
         {
-            IOUtils.closeQuietly( readerStream );
+            JavaHelper.closeQuietly( readerStream );
         }
 
         return stringWriter.toString();

server/src/main/java/password/pwm/PwmAboutProperty.java

@@ -48,13 +48,13 @@ import java.util.stream.Collectors;
 public enum PwmAboutProperty
 {
 
-    app_version( null, pwmApplication -> PwmConstants.SERVLET_VERSION ),
-    app_chaiApiVersion( null, pwmApplication -> PwmConstants.CHAI_API_VERSION ),
-    app_currentTime( null, pwmApplication -> format( Instant.now() ) ),
-    app_startTime( null, pwmApplication -> format( pwmApplication.getStartupTime() ) ),
-    app_installTime( null, pwmApplication -> format( pwmApplication.getInstallTime() ) ),
-    app_siteUrl( null, pwmApplication -> pwmApplication.getConfig().readSettingAsString( PwmSetting.PWM_SITE_URL ) ),
-    app_instanceID( null, PwmApplication::getInstanceID ),
+    app_version( "App Version", pwmApplication -> PwmConstants.SERVLET_VERSION ),
+    app_chaiApiVersion( "App Chai Version", pwmApplication -> PwmConstants.CHAI_API_VERSION ),
+    app_currentTime( "App Current Time", pwmApplication -> format( Instant.now() ) ),
+    app_startTime( "App Startup Time", pwmApplication -> format( pwmApplication.getStartupTime() ) ),
+    app_installTime( "App Install Time", pwmApplication -> format( pwmApplication.getInstallTime() ) ),
+    app_siteUrl( "App Site URL", pwmApplication -> pwmApplication.getConfig().readSettingAsString( PwmSetting.PWM_SITE_URL ) ),
+    app_instanceID( "App InstanceID", PwmApplication::getInstanceID ),
     app_trialMode( null, pwmApplication -> Boolean.toString( PwmConstants.TRIAL_MODE ) ),
     app_mode_appliance( null, pwmApplication -> Boolean.toString( pwmApplication.getPwmEnvironment().getFlags().contains( PwmEnvironment.ApplicationFlag.Appliance ) ) ),
     app_mode_docker( null, pwmApplication -> Boolean.toString( pwmApplication.getPwmEnvironment().getFlags().contains( PwmEnvironment.ApplicationFlag.Docker ) ) ),
@@ -79,8 +79,9 @@ public enum PwmAboutProperty
     app_secureHashAlgorithm( null, pwmApplication -> pwmApplication.getSecureService().getDefaultHashAlgorithm().toString() ),
     app_ldapProfileCount( null, pwmApplication -> Integer.toString( LdapConnectionService.totalLdapProfileCount( pwmApplication ) ) ),
     app_ldapConnectionCount( null, pwmApplication -> Long.toString( LdapConnectionService.totalLdapConnectionCount( pwmApplication ) ) ),
-    app_activeSessionCount( "Active Session Count", pwmApplication -> Integer.toString( pwmApplication.getSessionTrackService().sessionCount() ) ),
-    app_activeRequestCount( "Active Request Count", pwmApplication -> Integer.toString( pwmApplication.getActiveServletRequests().get() ) ),
+    app_activeSessionCount( "App Active Session Count", pwmApplication -> Integer.toString( pwmApplication.getSessionTrackService().sessionCount() ) ),
+    app_activeRequestCount( "App Active Request Count", pwmApplication -> Integer.toString( pwmApplication.getActiveServletRequests().get() ) ),
+    app_definedDomainCount( "App Defined Domain Count", pwmApplication -> Integer.toString( pwmApplication.domains().size() ) ),
 
     build_Time( "Build Time", pwmApplication -> PwmConstants.BUILD_TIME ),
     build_Number( "Build Number", pwmApplication -> PwmConstants.BUILD_NUMBER ),

server/src/main/java/password/pwm/PwmApplication.java

@@ -23,14 +23,10 @@ package password.pwm;
 import lombok.Value;
 import password.pwm.bean.DomainID;
 import password.pwm.bean.SessionLabel;
-import password.pwm.bean.SmsItemBean;
 import password.pwm.config.AppConfig;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.PwmSettingMetaDataReader;
 import password.pwm.config.PwmSettingScope;
-import password.pwm.config.stored.StoredConfigKey;
-import password.pwm.config.stored.StoredConfiguration;
-import password.pwm.config.stored.StoredConfigurationUtil;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
@@ -64,34 +60,20 @@ import password.pwm.svc.wordlist.SeedlistService;
 import password.pwm.svc.wordlist.SharedHistoryService;
 import password.pwm.svc.wordlist.WordlistService;
 import password.pwm.util.MBeanUtility;
-import password.pwm.util.PasswordData;
 import password.pwm.util.PwmScheduler;
-import password.pwm.util.cli.commands.ExportHttpsTomcatConfigCommand;
-import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.json.JsonFactory;
 import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.localdb.LocalDBFactory;
 import password.pwm.util.logging.LocalDBLogger;
 import password.pwm.util.logging.PwmLogLevel;
 import password.pwm.util.logging.PwmLogManager;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.macro.MacroRequest;
-import password.pwm.util.secure.HttpsServerCertificateManager;
 import password.pwm.util.secure.PwmRandom;
-import password.pwm.util.secure.X509Utils;
 
-import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.io.Serializable;
-import java.net.URI;
-import java.nio.file.Files;
-import java.security.KeyStore;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -103,42 +85,47 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.TreeMap;
 import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
-import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 public class PwmApplication
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwmApplication.class );
-    private static final String DEFAULT_INSTANCE_ID = "-1";
+
+    static final int DOMAIN_STARTUP_THREADS = 10;
 
     private final AtomicInteger activeServletRequests = new AtomicInteger( 0 );
 
-    private Map<DomainID, PwmDomain> domains;
+    private volatile Map<DomainID, PwmDomain> domains = new HashMap<>();
     private String runtimeNonce = PwmRandom.getInstance().randomUUID().toString();
 
-    private final PwmServiceManager pwmServiceManager = new PwmServiceManager(
-            SessionLabel.SYSTEM_LABEL,
-            this, DomainID.systemId(), PwmServiceEnum.forScope( PwmSettingScope.SYSTEM ) );
+    private final SessionLabel sessionLabel;
+    private final PwmServiceManager pwmServiceManager;
 
     private final Instant startupTime = Instant.now();
+
     private Instant installTime = Instant.now();
     private ErrorInformation lastLocalDBFailure;
     private PwmEnvironment pwmEnvironment;
     private FileLocker fileLocker;
     private PwmScheduler pwmScheduler;
-    private String instanceID = DEFAULT_INSTANCE_ID;
+    private String instanceID = PwmApplicationUtil.DEFAULT_INSTANCE_ID;
     private LocalDB localDB;
     private LocalDBLogger localDBLogger;
 
-
     public PwmApplication( final PwmEnvironment pwmEnvironment )
             throws PwmUnrecoverableException
     {
         this.pwmEnvironment = Objects.requireNonNull( pwmEnvironment );
+        this.sessionLabel = pwmEnvironment.isInternalRuntimeInstance()
+                ? SessionLabel.RUNTIME_LABEL
+                : SessionLabel.SYSTEM_LABEL;
+
+        this.pwmServiceManager = new PwmServiceManager(
+                sessionLabel, this, DomainID.systemId(), PwmServiceEnum.forScope( PwmSettingScope.SYSTEM ) );
 
         if ( !pwmEnvironment.isInternalRuntimeInstance() )
         {
@@ -151,46 +138,25 @@ public class PwmApplication
         }
         catch ( final PwmUnrecoverableException e )
         {
-            LOGGER.fatal( e::getMessage );
+            LOGGER.fatal( sessionLabel, e::getMessage );
             throw e;
         }
     }
 
-    public static Optional<String> deriveLocalServerHostname( final AppConfig appConfig )
+    private void initRuntimeNonce()
     {
-        if ( appConfig != null )
-        {
-            final String siteUrl = appConfig.readSettingAsString( PwmSetting.PWM_SITE_URL );
-            if ( StringUtil.notEmpty( siteUrl ) )
-            {
-                try
-                {
-                    final URI parsedUri = URI.create( siteUrl );
-                    {
-                        final String uriHost = parsedUri.getHost();
-                        return Optional.ofNullable( uriHost );
-                    }
-                }
-                catch ( final IllegalArgumentException e )
-                {
-                    LOGGER.trace( () -> " error parsing siteURL hostname: " + e.getMessage() );
-                }
-            }
-        }
-        return Optional.empty();
+        runtimeNonce = PwmRandom.getInstance().randomUUID().toString();
     }
 
-    private void initialize( )
+    private void initialize()
             throws PwmUnrecoverableException
     {
         final Instant startTime = Instant.now();
 
-        runtimeNonce = PwmRandom.getInstance().randomUUID().toString();
-
-        this.domains = Initializer.initializeDomains( this );
+        initRuntimeNonce();
 
-        // initialize log4j
-        Initializer.initializeLogging( this );
+        // initialize logger
+        PwmApplicationUtil.initializeLogging( this );
 
         // get file lock
         if ( !pwmEnvironment.isInternalRuntimeInstance() )
@@ -205,7 +171,7 @@ public class PwmApplication
             final File tempFileDirectory = getTempDirectory();
             try
             {
-                LOGGER.debug( () -> "deleting directory (and sub-directory) contents in " + tempFileDirectory );
+                LOGGER.debug( sessionLabel, () -> "deleting directory (and sub-directory) contents in " + tempFileDirectory );
                 FileSystemUtility.deleteDirectoryContentsRecursively( tempFileDirectory.toPath() );
             }
             catch ( final Exception e )
@@ -218,7 +184,7 @@ public class PwmApplication
 
         if ( getApplicationMode() != PwmApplicationMode.READ_ONLY )
         {
-            LOGGER.info( () -> "initializing, application mode=" + getApplicationMode()
+            LOGGER.info( sessionLabel, () -> "initializing, application mode=" + getApplicationMode()
                     + ", applicationPath=" + ( pwmEnvironment.getApplicationPath() == null ? "null" : pwmEnvironment.getApplicationPath().getAbsolutePath() )
                     + ", configFile=" + ( pwmEnvironment.getConfigurationFile() == null ? "null" : pwmEnvironment.getConfigurationFile().getAbsolutePath() )
             );
@@ -228,39 +194,41 @@ public class PwmApplication
         {
             if ( getApplicationMode() == PwmApplicationMode.ERROR )
             {
-                LOGGER.warn( () -> "skipping LocalDB open due to application mode " + getApplicationMode() );
+                LOGGER.warn( sessionLabel, () -> "skipping LocalDB open due to application mode " + getApplicationMode() );
             }
             else
             {
                 if ( localDB == null )
                 {
-                    this.localDB = Initializer.initializeLocalDB( this, pwmEnvironment );
+                    this.localDB = PwmApplicationUtil.initializeLocalDB( this, pwmEnvironment );
                 }
             }
         }
 
+        // read the instance id
+        instanceID = PwmApplicationUtil.fetchInstanceID( this, localDB );
+        LOGGER.debug( sessionLabel, () -> "using '" + getInstanceID() + "' for instance's ID (instanceID)" );
+
         this.localDBLogger = PwmLogManager.initializeLocalDBLogger( this );
 
         // log the loaded configuration
-        LOGGER.debug( () -> "configuration load completed" );
-
-        // read the pwm servlet instance id
-        instanceID = fetchInstanceID( localDB, this );
-        LOGGER.debug( () -> "using '" + getInstanceID() + "' for instance's ID (instanceID)" );
+        LOGGER.debug( sessionLabel, () -> "configuration load completed" );
 
         // read the pwm installation date
         installTime = fetchInstallDate( startupTime );
-        LOGGER.debug( () -> "this application instance first installed on " + StringUtil.toIsoDate( installTime ) );
+        LOGGER.debug( sessionLabel, () -> "this application instance first installed on " + StringUtil.toIsoDate( installTime ) );
 
-        LOGGER.debug( () -> "application environment flags: " + JsonFactory.get().serializeCollection( pwmEnvironment.getFlags() ) );
-        LOGGER.debug( () -> "application environment parameters: "
+        LOGGER.debug( sessionLabel, () -> "application environment flags: " + JsonFactory.get().serializeCollection( pwmEnvironment.getFlags() ) );
+        LOGGER.debug( sessionLabel, () -> "application environment parameters: "
                 + JsonFactory.get().serializeMap( pwmEnvironment.getParameters(), PwmEnvironment.ApplicationParameter.class, String.class ) );
 
         pwmScheduler = new PwmScheduler( this );
 
+        domains = PwmDomainUtil.createDomainInstances( this );
+
         pwmServiceManager.initAllServices();
 
-        initAllDomains();
+        PwmDomainUtil.initDomains( this, domains().values() );
 
         final boolean skipPostInit = pwmEnvironment.isInternalRuntimeInstance()
                 || pwmEnvironment.getFlags().contains( PwmEnvironment.ApplicationFlag.CommandLineInstance );
@@ -268,68 +236,42 @@ public class PwmApplication
         if ( !skipPostInit )
         {
             final TimeDuration totalTime = TimeDuration.fromCurrent( startTime );
-            LOGGER.info( () -> PwmConstants.PWM_APP_NAME + " " + PwmConstants.SERVLET_VERSION + " open for bidness! (" + totalTime.asCompactString() + ")" );
+            LOGGER.info( sessionLabel, () -> PwmConstants.PWM_APP_NAME + " " + PwmConstants.SERVLET_VERSION + " open for bidness! (" + totalTime.asCompactString() + ")" );
             StatisticsClient.incrementStat( this, Statistic.PWM_STARTUPS );
-            LOGGER.debug( () -> "buildTime=" + PwmConstants.BUILD_TIME + ", javaLocale=" + Locale.getDefault() + ", DefaultLocale=" + PwmConstants.DEFAULT_LOCALE );
+            LOGGER.debug( sessionLabel, () -> "buildTime=" + PwmConstants.BUILD_TIME + ", javaLocale=" + Locale.getDefault() + ", DefaultLocale=" + PwmConstants.DEFAULT_LOCALE );
 
-            pwmScheduler.immediateExecuteRunnableInNewThread( this::postInitTasks, this.getClass().getSimpleName() + " postInit tasks" );
+            pwmScheduler.immediateExecuteRunnableInNewThread( this::postInitTasks, sessionLabel, this.getClass().getSimpleName() + " postInit tasks" );
         }
 
     }
 
-    private void initAllDomains()
-            throws PwmUnrecoverableException
+    public void reInit( final PwmEnvironment pwmEnvironment )
+            throws PwmException
     {
-        final Instant domainInitStartTime = Instant.now();
-        LOGGER.trace( () -> "beginning domain initializations" );
+        final Instant startTime = Instant.now();
+        LOGGER.debug( sessionLabel, () -> "beginning application restart" );
+        final AppConfig oldConfig = this.pwmEnvironment.getConfig();
+        this.pwmEnvironment = pwmEnvironment;
+        final AppConfig newConfig = this.pwmEnvironment.getConfig();
 
-        final List<Callable<?>> callables = domains.values().stream().<Callable<?>>map( pwmDomain -> () ->
+        if ( !Objects.equals( oldConfig.getValueHash(), newConfig.getValueHash() ) )
         {
-            pwmDomain.initialize();
-            return null;
-        } ).collect( Collectors.toList() );
-        pwmScheduler.executeImmediateThreadPerJobAndAwaitCompletion( callables, "domain initializer" );
+            processPwmAppRestart( );
+        }
 
-        LOGGER.trace( () -> "completed domain initialization for all domains", () -> TimeDuration.fromCurrent( domainInitStartTime ) );
-    }
+        domains = PwmDomainUtil.reInitDomains( this, newConfig, oldConfig );
 
+        runtimeNonce = PwmRandom.getInstance().randomUUID().toString();
 
-    public void reInit( final PwmEnvironment pwmEnvironment )
-            throws PwmException
-    {
-        final Instant startTime = Instant.now();
-        LOGGER.debug( () -> "beginning application restart" );
-        shutdown( true );
-        this.pwmEnvironment = pwmEnvironment;
-        initialize();
-        LOGGER.debug( () -> "completed application restart", () -> TimeDuration.fromCurrent( startTime ) );
+        LOGGER.debug( sessionLabel, () -> "completed application restart", () -> TimeDuration.fromCurrent( startTime ) );
     }
 
-    private void postInitTasks( )
+    private void postInitTasks()
     {
         final Instant startTime = Instant.now();
 
-        getPwmScheduler().immediateExecuteRunnableInNewThread( UserAgentUtils::initializeCache, "initialize useragent cache" );
-        getPwmScheduler().immediateExecuteRunnableInNewThread( PwmSettingMetaDataReader::initCache, "initialize PwmSetting cache" );
-
-        if ( Boolean.parseBoolean( getConfig().readAppProperty( AppProperty.LOGGING_OUTPUT_CONFIGURATION ) ) )
-        {
-            outputConfigurationToLog( this );
-            outputNonDefaultPropertiesToLog( this );
-        }
-
         // send system audit event
-        AuditServiceClient.submitSystemEvent( this, SessionLabel.SYSTEM_LABEL, AuditEvent.STARTUP );
-
-        try
-        {
-            final Map<PwmAboutProperty, String> infoMap = PwmAboutProperty.makeInfoBean( this );
-            LOGGER.trace( () ->  "application info: " + JsonFactory.get().serializeMap( infoMap, PwmAboutProperty.class, String.class ) );
-        }
-        catch ( final Exception e )
-        {
-            LOGGER.error( () -> "error generating about application bean: " + e.getMessage(), e );
-        }
+        AuditServiceClient.submitSystemEvent( this, sessionLabel, AuditEvent.STARTUP );
 
         try
         {
@@ -337,41 +279,35 @@ public class PwmApplication
         }
         catch ( final Exception e )
         {
-            LOGGER.warn( () -> "error while clearing configmanager-intruder-username from intruder table: " + e.getMessage() );
+            LOGGER.warn( sessionLabel, () -> "error while clearing config manager-intruder-username from intruder table: " + e.getMessage() );
         }
 
         if ( !pwmEnvironment.isInternalRuntimeInstance() )
         {
-            try
-            {
-                outputKeystore( this );
-            }
-            catch ( final Exception e )
-            {
-                LOGGER.debug( () -> "error while generating keystore output: " + e.getMessage() );
-            }
-
-            try
-            {
-                outputTomcatConf( this );
-            }
-            catch ( final Exception e )
-            {
-                LOGGER.debug( () -> "error while generating tomcat conf output: " + e.getMessage() );
-            }
+            PwmApplicationUtil.outputKeystore( this );
+            PwmApplicationUtil.outputTomcatConf( this );
         }
 
-        if ( Boolean.parseBoolean( getConfig().readAppProperty( AppProperty.LOGGING_OUTPUT_CONFIGURATION ) ) )
+        if ( Boolean.parseBoolean( getConfig().readAppProperty( AppProperty.LOGGING_OUTPUT_CONFIGURATION ) )
+                && LOGGER.isEnabled( PwmLogLevel.TRACE )
+        )
         {
-            getPwmScheduler().immediateExecuteRunnableInNewThread( () ->
-            {
-                outputConfigurationToLog( this );
-                outputNonDefaultPropertiesToLog( this );
-            }, "output configuration to log" );
+            PwmApplicationUtil.outputApplicationInfoToLog( this );
+            PwmApplicationUtil.outputConfigurationToLog( this, DomainID.systemId() );
+            PwmApplicationUtil.outputNonDefaultPropertiesToLog( this );
         }
 
         MBeanUtility.registerMBean( this );
-        LOGGER.trace( () -> "completed post init tasks", () -> TimeDuration.fromCurrent( startTime ) );
+
+        getPwmScheduler().immediateExecuteRunnableInNewThread( () ->
+                {
+                    PwmSettingMetaDataReader.initCache();
+                    UserAgentUtils.initializeCache();
+                },
+                sessionLabel,
+                "initialize useragent cache" );
+
+        LOGGER.trace( sessionLabel, () -> "completed post init tasks", () -> TimeDuration.fromCurrent( startTime ) );
     }
 
     public static PwmApplication createPwmApplication( final PwmEnvironment pwmEnvironment )
@@ -380,6 +316,11 @@ public class PwmApplication
         return new PwmApplication( pwmEnvironment );
     }
 
+    public SessionLabel getSessionLabel()
+    {
+        return sessionLabel;
+    }
+
     public Map<DomainID, PwmDomain> domains()
     {
         return domains;
@@ -416,75 +357,76 @@ public class PwmApplication
         return domains().get( getConfig().getAdminDomainID() );
     }
 
-    public void shutdown( )
+    private void processPwmAppRestart()
+            throws PwmUnrecoverableException
     {
-        shutdown( false );
+        LOGGER.debug( sessionLabel, () -> "system config settings modified, system services restart required" );
+        AuditServiceClient.submitSystemEvent( this, sessionLabel, AuditEvent.RESTART );
+
+        initRuntimeNonce();
+
+        pwmServiceManager.shutdownAllServices();
+
+
+        // initialize logger
+        PwmApplicationUtil.initializeLogging( this );
+
+        pwmServiceManager.initAllServices();
+
+        if ( !pwmEnvironment.isInternalRuntimeInstance() )
+        {
+            PwmApplicationUtil.outputKeystore( this );
+            PwmApplicationUtil.outputTomcatConf( this );
+        }
     }
 
-    public void shutdown( final boolean keepServicesRunning )
+    public void shutdown( )
     {
         final Instant startTime = Instant.now();
 
-        if ( keepServicesRunning )
-        {
-            LOGGER.warn( () -> "preparing for restart" );
-            AuditServiceClient.submitSystemEvent( this, SessionLabel.SYSTEM_LABEL, AuditEvent.RESTART );
-        }
-        else
-        {
-            LOGGER.warn( () -> "shutting down" );
-            AuditServiceClient.submitSystemEvent( this, SessionLabel.SYSTEM_LABEL, AuditEvent.SHUTDOWN );
-        }
+        LOGGER.warn( sessionLabel, () -> "shutting down" );
+        AuditServiceClient.submitSystemEvent( this, sessionLabel, AuditEvent.SHUTDOWN );
 
         MBeanUtility.unregisterMBean( this );
 
-        if ( !keepServicesRunning )
-        {
-            try
-            {
-                final List<Callable<?>> callables = domains.values().stream().<Callable<?>>map( pwmDomain -> () ->
-                {
-                    pwmDomain.shutdown();
-                    return null;
-                } ).collect( Collectors.toList() );
-                pwmScheduler.executeImmediateThreadPerJobAndAwaitCompletion( callables, "domain shutdown task" );
-            }
-            catch ( final PwmUnrecoverableException e )
-            {
-                LOGGER.error( () -> "error shutting down domain services: " + e.getMessage(), e );
-            }
+        pwmServiceManager.shutdownAllServices();
 
-            pwmServiceManager.shutdownAllServices();
+        try
+        {
+            final List<Callable<Object>> callables = domains.values().stream()
+                    .map( pwmDomain -> Executors.callable( pwmDomain::shutdown ) )
+                    .collect( Collectors.toList() );
+            pwmScheduler.executeImmediateThreadPerJobAndAwaitCompletion( DOMAIN_STARTUP_THREADS, callables, sessionLabel, PwmApplication.class );
+        }
+        catch ( final PwmUnrecoverableException e )
+        {
+            LOGGER.error( sessionLabel, () -> "error shutting down domain services: " + e.getMessage(), e );
         }
 
         if ( localDBLogger != null )
         {
             try
             {
-                localDBLogger.close();
+                localDBLogger.shutdownImpl();
             }
             catch ( final Exception e )
             {
-                LOGGER.error( () -> "error closing localDBLogger: " + e.getMessage(), e );
+                LOGGER.error( sessionLabel, () -> "error closing localDBLogger: " + e.getMessage(), e );
             }
             localDBLogger = null;
         }
 
-        if ( keepServicesRunning )
-        {
-            LOGGER.trace( () -> "skipping close of LocalDB (restart request)" );
-        }
-        else if ( localDB != null )
+        if ( localDB != null )
         {
             try
             {
                 final Instant startCloseDbTime = Instant.now();
-                LOGGER.debug( () -> "beginning close of LocalDB" );
+                LOGGER.debug( sessionLabel, () -> "beginning close of LocalDB" );
                 localDB.close();
                 final TimeDuration closeLocalDbDuration = TimeDuration.fromCurrent( startCloseDbTime );
                 if ( closeLocalDbDuration.isLongerThan( TimeDuration.SECONDS_10 ) )
                 {
-                    LOGGER.info( () -> "completed close of LocalDB", () -> closeLocalDbDuration );
+                    LOGGER.info( sessionLabel, () -> "completed close of LocalDB", () -> closeLocalDbDuration );
                 }
             }
             catch ( final Exception e )
@@ -501,133 +443,10 @@ public class PwmApplication
 
         pwmScheduler.shutdown();
 
-        LOGGER.info( () -> PwmConstants.PWM_APP_NAME + " " + PwmConstants.SERVLET_VERSION
+        LOGGER.info( sessionLabel, () -> PwmConstants.PWM_APP_NAME + " " + PwmConstants.SERVLET_VERSION
                 + " closed for bidness, cya!", () -> TimeDuration.fromCurrent( startTime ) );
     }
 
-    private static void outputKeystore( final PwmApplication pwmApplication ) throws Exception
-    {
-        final Map<PwmEnvironment.ApplicationParameter, String> applicationParams = pwmApplication.getPwmEnvironment().getParameters();
-        final String keystoreFileString = applicationParams.get( PwmEnvironment.ApplicationParameter.AutoExportHttpsKeyStoreFile );
-        if ( StringUtil.isEmpty( keystoreFileString ) )
-        {
-            return;
-        }
-
-        final File keyStoreFile = new File( keystoreFileString );
-        final String password = applicationParams.get( PwmEnvironment.ApplicationParameter.AutoExportHttpsKeyStorePassword );
-        final String alias = applicationParams.get( PwmEnvironment.ApplicationParameter.AutoExportHttpsKeyStoreAlias );
-        final KeyStore keyStore = HttpsServerCertificateManager.keyStoreForApplication( pwmApplication, new PasswordData( password ), alias );
-        X509Utils.outputKeystore( keyStore, keyStoreFile, password );
-        PwmApplication.LOGGER.info( () -> "exported application https key to keystore file " + keyStoreFile.getAbsolutePath() );
-    }
-
-    private static void outputTomcatConf( final PwmApplication pwmDomain ) throws IOException
-    {
-        final Map<PwmEnvironment.ApplicationParameter, String> applicationParams = pwmDomain.getPwmEnvironment().getParameters();
-        final String tomcatOutputFileStr = applicationParams.get( PwmEnvironment.ApplicationParameter.AutoWriteTomcatConfOutputFile );
-        if ( tomcatOutputFileStr != null && !tomcatOutputFileStr.isEmpty() )
-        {
-            LOGGER.trace( () -> "attempting to output tomcat configuration file as configured by environment parameters to " + tomcatOutputFileStr );
-            final File tomcatOutputFile = new File( tomcatOutputFileStr );
-            final File tomcatSourceFile;
-            {
-                final String tomcatSourceFileStr = applicationParams.get( PwmEnvironment.ApplicationParameter.AutoWriteTomcatConfSourceFile );
-                if ( tomcatSourceFileStr != null && !tomcatSourceFileStr.isEmpty() )
-                {
-                    tomcatSourceFile = new File( tomcatSourceFileStr );
-                    if ( !tomcatSourceFile.exists() )
-                    {
-                        LOGGER.error( () -> "can not output tomcat configuration file, source file does not exist: " + tomcatSourceFile.getAbsolutePath() );
-                        return;
-                    }
-                }
-                else
-                {
-                    LOGGER.error( () -> "can not output tomcat configuration file, source file parameter '"
-                            + PwmEnvironment.ApplicationParameter.AutoWriteTomcatConfSourceFile + "' is not specified." );
-                    return;
-                }
-            }
-
-            try ( ByteArrayOutputStream outputContents = new ByteArrayOutputStream() )
-            {
-                try ( InputStream fileInputStream = Files.newInputStream( tomcatOutputFile.toPath() ) )
-                {
-                    ExportHttpsTomcatConfigCommand.TomcatConfigWriter.writeOutputFile(
-                            pwmDomain.getConfig(),
-                            fileInputStream,
-                            outputContents
-                    );
-                }
-
-                if ( tomcatOutputFile.exists() )
-                {
-                    LOGGER.trace( () -> "deleting existing tomcat configuration file " + tomcatOutputFile.getAbsolutePath() );
-                    if ( tomcatOutputFile.delete() )
-                    {
-                        LOGGER.trace( () -> "deleted existing tomcat configuration file: " + tomcatOutputFile.getAbsolutePath() );
-                    }
-                }
-
-                try ( OutputStream fileOutputStream = Files.newOutputStream( tomcatOutputFile.toPath() ) )
-                {
-                    fileOutputStream.write( outputContents.toByteArray() );
-                }
-            }
-
-            LOGGER.info( () -> "successfully wrote tomcat configuration to file " + tomcatOutputFile.getAbsolutePath() );
-        }
-    }
-
-    private static void outputConfigurationToLog( final PwmApplication pwmApplication )
-    {
-        final Instant startTime = Instant.now();
-
-        final Function<Map.Entry<String, String>, String> valueFormatter = entry ->
-        {
-            final String spacedValue = entry.getValue().replace( "\n", "\n   " );
-            return " " + entry.getKey() + "\n   " + spacedValue + "\n";
-        };
-
-        final StoredConfiguration storedConfiguration = pwmApplication.getConfig().getStoredConfiguration();
-        final List<StoredConfigKey> keys = CollectionUtil.iteratorToStream( storedConfiguration.keys() ).collect( Collectors.toList() );
-        final Map<String, String> debugStrings = StoredConfigurationUtil.makeDebugMap(
-                storedConfiguration,
-                keys,
-                PwmConstants.DEFAULT_LOCALE );
-
-        LOGGER.trace( () -> "--begin current configuration output--" );
-        final long itemCount = debugStrings.entrySet().stream()
-                .map( valueFormatter )
-                .map( s -> ( Supplier<CharSequence> ) () -> s )
-                .peek( LOGGER::trace )
-                .count();
-
-        LOGGER.trace( () -> "--end current configuration output of " + itemCount + " items --",
-                () -> TimeDuration.fromCurrent( startTime ) );
-    }
-
-    private static void outputNonDefaultPropertiesToLog( final PwmApplication pwmApplication )
-    {
-        final Instant startTime = Instant.now();
-
-        final Map<AppProperty, String> nonDefaultProperties = pwmApplication.getConfig().readAllNonDefaultAppProperties();
-        if ( !CollectionUtil.isEmpty( nonDefaultProperties ) )
-        {
-            LOGGER.trace( () -> "--begin non-default app properties output--" );
-            nonDefaultProperties.entrySet().stream()
-                    .map( entry -> "AppProperty: " + entry.getKey().getKey() + " -> " + entry.getValue() )
-                    .map( s -> ( Supplier<CharSequence> ) () -> s )
-                    .forEach( LOGGER::trace );
-            LOGGER.trace( () -> "--end non-default app properties output--", () -> TimeDuration.fromCurrent( startTime ) );
-        }
-        else
-        {
-            LOGGER.trace( () -> "no non-default app properties in configuration" );
-        }
-    }
-
     public String getInstanceID( )
     {
         return instanceID;
@@ -815,47 +634,6 @@ public class PwmApplication
         return Instant.now();
     }
 
-    private String fetchInstanceID( final LocalDB localDB, final PwmApplication pwmApplication )
-    {
-        {
-            final String newInstanceID = pwmApplication.getPwmEnvironment().getParameters().get( PwmEnvironment.ApplicationParameter.InstanceID );
-
-            if ( !StringUtil.isTrimEmpty( newInstanceID ) )
-            {
-                return newInstanceID;
-            }
-        }
-
-        if ( pwmApplication.getLocalDB() == null || pwmApplication.getApplicationMode() != PwmApplicationMode.RUNNING )
-        {
-            return DEFAULT_INSTANCE_ID;
-        }
-
-        {
-            final Optional<String> optionalStoredInstanceID = readAppAttribute( AppAttribute.INSTANCE_ID, String.class );
-            if ( optionalStoredInstanceID.isPresent() )
-            {
-                final String instanceID = optionalStoredInstanceID.get();
-                if ( !StringUtil.isTrimEmpty( instanceID ) )
-                {
-                    LOGGER.trace( () -> "retrieved instanceID " + instanceID + "" + " from localDB" );
-                    return instanceID;
-                }
-            }
-        }
-
-        final PwmRandom pwmRandom = PwmRandom.getInstance();
-        final String newInstanceID = Long.toHexString( pwmRandom.nextLong() ).toUpperCase();
-        LOGGER.debug( () -> "generated new random instanceID " + newInstanceID );
-
-        if ( localDB != null )
-        {
-            writeAppAttribute( AppAttribute.INSTANCE_ID, newInstanceID );
-        }
-
-        return newInstanceID;
-    }
-
     public SharedHistoryService getSharedHistoryManager( )
     {
         return ( SharedHistoryService ) pwmServiceManager.getService( PwmServiceEnum.SharedHistoryManager );
@@ -953,6 +731,11 @@ public class PwmApplication
         return lastLocalDBFailure;
     }
 
+    void setLastLocalDBFailure( final ErrorInformation lastLocalDBFailure )
+    {
+        this.lastLocalDBFailure = lastLocalDBFailure;
+    }
+
     public SessionTrackService getSessionTrackService( )
     {
         return ( SessionTrackService ) pwmServiceManager.getService( PwmServiceEnum.SessionTrackService );
@@ -1011,134 +794,6 @@ public class PwmApplication
         return this.getConfig().isMultiDomain();
     }
 
-    public void sendSmsUsingQueue(
-            final String to,
-            final String message,
-            final SessionLabel sessionLabel,
-            final MacroRequest macroRequest
-    )
-    {
-        final SmsQueueService smsQueue = getSmsQueue();
-        if ( smsQueue == null )
-        {
-            LOGGER.error( sessionLabel, () -> "SMS queue is unavailable, unable to send SMS to: " + to );
-            return;
-        }
-
-        final SmsItemBean smsItemBean = new SmsItemBean(
-                macroRequest.expandMacros( to ),
-                macroRequest.expandMacros( message ),
-                sessionLabel
-        );
-
-        try
-        {
-            smsQueue.addSmsToQueue( smsItemBean );
-        }
-        catch ( final PwmUnrecoverableException e )
-        {
-            LOGGER.warn( () -> "unable to add sms to queue: " + e.getMessage() );
-        }
-    }
-
-    private static class Initializer
-    {
-        public static LocalDB initializeLocalDB( final PwmApplication pwmApplication, final PwmEnvironment pwmEnvironment )
-                throws PwmUnrecoverableException
-        {
-            final File databaseDirectory;
-
-            try
-            {
-                final String localDBLocationSetting = pwmApplication.getConfig().readAppProperty( AppProperty.LOCALDB_LOCATION );
-                databaseDirectory = FileSystemUtility.figureFilepath( localDBLocationSetting, pwmApplication.pwmEnvironment.getApplicationPath() );
-            }
-            catch ( final Exception e )
-            {
-                pwmApplication.lastLocalDBFailure = new ErrorInformation( PwmError.ERROR_LOCALDB_UNAVAILABLE, "error locating configured LocalDB directory: " + e.getMessage() );
-                LOGGER.warn( () -> pwmApplication.lastLocalDBFailure.toDebugStr() );
-                throw new PwmUnrecoverableException( pwmApplication.lastLocalDBFailure );
-            }
-
-            LOGGER.debug( () -> "using localDB path " + databaseDirectory );
-
-            // initialize the localDB
-            try
-            {
-                final boolean readOnly = pwmApplication.getApplicationMode() == PwmApplicationMode.READ_ONLY;
-                return LocalDBFactory.getInstance( databaseDirectory, readOnly, pwmEnvironment, pwmApplication.getConfig() );
-            }
-            catch ( final Exception e )
-            {
-                pwmApplication.lastLocalDBFailure = new ErrorInformation( PwmError.ERROR_LOCALDB_UNAVAILABLE, "unable to initialize LocalDB: " + e.getMessage() );
-                LOGGER.warn( () -> pwmApplication.lastLocalDBFailure.toDebugStr() );
-                throw new PwmUnrecoverableException( pwmApplication.lastLocalDBFailure );
-            }
-        }
-
-        private static Map<DomainID, PwmDomain> initializeDomains( final PwmApplication pwmApplication )
-        {
-            final Map<DomainID, PwmDomain> domainMap = new TreeMap<>();
-            for ( final String domainIdString : pwmApplication.getPwmEnvironment().getConfig().getDomainIDs() )
-            {
-                final DomainID domainID = DomainID.create( domainIdString );
-                final PwmDomain newDomain = new PwmDomain( pwmApplication, domainID );
-                domainMap.put( domainID, newDomain );
-            }
-
-            return Collections.unmodifiableMap( domainMap );
-        }
-
-        public static void initializeLogging( final PwmApplication pwmApplication )
-        {
-            final PwmEnvironment pwmEnvironment = pwmApplication.getPwmEnvironment();
-
-            if ( !pwmEnvironment.isInternalRuntimeInstance() && !pwmEnvironment.getFlags().contains( PwmEnvironment.ApplicationFlag.CommandLineInstance ) )
-            {
-                final String log4jFileName = pwmEnvironment.getConfig().readSettingAsString( PwmSetting.EVENTS_JAVA_LOG4JCONFIG_FILE );
-                final File log4jFile = FileSystemUtility.figureFilepath( log4jFileName, pwmEnvironment.getApplicationPath() );
-                final String consoleLevel;
-                final String fileLevel;
-
-                switch ( pwmApplication.getApplicationMode() )
-                {
-                    case ERROR:
-                    case NEW:
-                        consoleLevel = PwmLogLevel.TRACE.toString();
-                        fileLevel = PwmLogLevel.TRACE.toString();
-                        break;
-
-                    default:
-                        consoleLevel = pwmEnvironment.getConfig().readSettingAsString( PwmSetting.EVENTS_JAVA_STDOUT_LEVEL );
-                        fileLevel = pwmEnvironment.getConfig().readSettingAsString( PwmSetting.EVENTS_FILE_LEVEL );
-                        break;
-                }
-
-                PwmLogManager.initializeLogger(
-                        pwmApplication,
-                        pwmApplication.getConfig(),
-                        log4jFile,
-                        consoleLevel,
-                        pwmEnvironment.getApplicationPath(),
-                        fileLevel );
-
-                switch ( pwmApplication.getApplicationMode() )
-                {
-                    case RUNNING:
-                        break;
-
-                    case ERROR:
-                        LOGGER.fatal( () -> "starting up in ERROR mode! Check log or health check information for cause" );
-                        break;
-
-                    default:
-                        LOGGER.trace( () -> "setting log level to TRACE because application mode is " + pwmApplication.getApplicationMode() );
-                        break;
-                }
-            }
-        }
-    }
-
     public File getTempDirectory( ) throws PwmUnrecoverableException
     {
         if ( pwmEnvironment.getApplicationPath() == null )
@@ -1224,4 +879,5 @@ public class PwmApplication
 
         return conditions.stream().allMatch( ( c ) -> c.matches( this ) );
     }
+
 }

server/src/main/java/password/pwm/PwmApplicationUtil.java

@@ -0,0 +1,355 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm;
+
+import password.pwm.bean.DomainID;
+import password.pwm.config.PwmSetting;
+import password.pwm.config.stored.StoredConfigKey;
+import password.pwm.config.stored.StoredConfiguration;
+import password.pwm.config.stored.StoredConfigurationUtil;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.PasswordData;
+import password.pwm.util.cli.commands.ExportHttpsTomcatConfigCommand;
+import password.pwm.util.java.CollectionUtil;
+import password.pwm.util.java.FileSystemUtility;
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.localdb.LocalDB;
+import password.pwm.util.localdb.LocalDBFactory;
+import password.pwm.util.logging.PwmLogLevel;
+import password.pwm.util.logging.PwmLogManager;
+import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.secure.HttpsServerCertificateManager;
+import password.pwm.util.secure.PwmRandom;
+import password.pwm.util.secure.X509Utils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.security.KeyStore;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+class PwmApplicationUtil
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( PwmApplicationUtil.class );
+
+    static final String DEFAULT_INSTANCE_ID = "-1";
+
+    static LocalDB initializeLocalDB( final PwmApplication pwmApplication, final PwmEnvironment pwmEnvironment )
+            throws PwmUnrecoverableException
+    {
+        final File databaseDirectory;
+
+        try
+        {
+            final String localDBLocationSetting = pwmApplication.getConfig().readAppProperty( AppProperty.LOCALDB_LOCATION );
+            databaseDirectory = FileSystemUtility.figureFilepath( localDBLocationSetting, pwmApplication.getPwmEnvironment().getApplicationPath() );
+        }
+        catch ( final Exception e )
+        {
+            pwmApplication.setLastLocalDBFailure( new ErrorInformation( PwmError.ERROR_LOCALDB_UNAVAILABLE, "error locating configured LocalDB directory: " + e.getMessage() ) );
+            LOGGER.warn( pwmApplication.getSessionLabel(), () -> pwmApplication.getLastLocalDBFailure().toDebugStr() );
+            throw new PwmUnrecoverableException( pwmApplication.getLastLocalDBFailure() );
+        }
+
+        LOGGER.debug( pwmApplication.getSessionLabel(), () -> "using localDB path " + databaseDirectory );
+
+        // initialize the localDB
+        try
+        {
+            final boolean readOnly = pwmApplication.getApplicationMode() == PwmApplicationMode.READ_ONLY;
+            return LocalDBFactory.getInstance( databaseDirectory, readOnly, pwmEnvironment, pwmApplication.getConfig() );
+        }
+        catch ( final Exception e )
+        {
+            pwmApplication.setLastLocalDBFailure( new ErrorInformation( PwmError.ERROR_LOCALDB_UNAVAILABLE, "unable to initialize LocalDB: " + e.getMessage() ) );
+            LOGGER.warn( pwmApplication.getSessionLabel(), () -> pwmApplication.getLastLocalDBFailure().toDebugStr() );
+            throw new PwmUnrecoverableException( pwmApplication.getLastLocalDBFailure() );
+        }
+    }
+
+    static void initializeLogging( final PwmApplication pwmApplication )
+    {
+        final PwmEnvironment pwmEnvironment = pwmApplication.getPwmEnvironment();
+
+        if ( !pwmEnvironment.isInternalRuntimeInstance() && !pwmEnvironment.getFlags().contains( PwmEnvironment.ApplicationFlag.CommandLineInstance ) )
+        {
+            final String log4jFileName = pwmEnvironment.getConfig().readSettingAsString( PwmSetting.EVENTS_JAVA_LOG4JCONFIG_FILE );
+            final File log4jFile = FileSystemUtility.figureFilepath( log4jFileName, pwmEnvironment.getApplicationPath() );
+            final String consoleLevel;
+            final String fileLevel;
+
+            switch ( pwmApplication.getApplicationMode() )
+            {
+                case ERROR:
+                case NEW:
+                    consoleLevel = PwmLogLevel.TRACE.toString();
+                    fileLevel = PwmLogLevel.TRACE.toString();
+                    break;
+
+                default:
+                    consoleLevel = pwmEnvironment.getConfig().readSettingAsString( PwmSetting.EVENTS_JAVA_STDOUT_LEVEL );
+                    fileLevel = pwmEnvironment.getConfig().readSettingAsString( PwmSetting.EVENTS_FILE_LEVEL );
+                    break;
+            }
+
+            PwmLogManager.initializeLogger(
+                    pwmApplication,
+                    pwmApplication.getConfig(),
+                    log4jFile,
+                    consoleLevel,
+                    pwmEnvironment.getApplicationPath(),
+                    fileLevel );
+
+            switch ( pwmApplication.getApplicationMode() )
+            {
+                case RUNNING:
+                    break;
+
+                case ERROR:
+                    LOGGER.fatal( pwmApplication.getSessionLabel(), () -> "starting up in ERROR mode! Check log or health check information for cause" );
+                    break;
+
+                default:
+                    LOGGER.trace( pwmApplication.getSessionLabel(), () -> "setting log level to TRACE because application mode is " + pwmApplication.getApplicationMode() );
+                    break;
+            }
+        }
+    }
+
+    static String fetchInstanceID(
+            final PwmApplication pwmApplication,
+            final LocalDB localDB
+    )
+    {
+        {
+            final String newInstanceID = pwmApplication.getPwmEnvironment().getParameters().get( PwmEnvironment.ApplicationParameter.InstanceID );
+
+            if ( !StringUtil.isTrimEmpty( newInstanceID ) )
+            {
+                return newInstanceID;
+            }
+        }
+
+        if ( pwmApplication.getLocalDB() == null || pwmApplication.getApplicationMode() != PwmApplicationMode.RUNNING )
+        {
+            return DEFAULT_INSTANCE_ID;
+        }
+
+        {
+            final Optional<String> optionalStoredInstanceID = pwmApplication.readAppAttribute( AppAttribute.INSTANCE_ID, String.class );
+            if ( optionalStoredInstanceID.isPresent() )
+            {
+                final String instanceID = optionalStoredInstanceID.get();
+                if ( !StringUtil.isTrimEmpty( instanceID ) )
+                {
+                    LOGGER.trace( pwmApplication.getSessionLabel(), () -> "retrieved instanceID " + instanceID + "" + " from localDB" );
+                    return instanceID;
+                }
+            }
+        }
+
+        final PwmRandom pwmRandom = PwmRandom.getInstance();
+        final String newInstanceID = Long.toHexString( pwmRandom.nextLong() ).toUpperCase();
+        LOGGER.debug( pwmApplication.getSessionLabel(), () -> "generated new random instanceID " + newInstanceID );
+
+        if ( localDB != null )
+        {
+            pwmApplication.writeAppAttribute( AppAttribute.INSTANCE_ID, newInstanceID );
+        }
+
+        return newInstanceID;
+    }
+
+    static void outputKeystore( final PwmApplication pwmApplication )
+    {
+        try
+        {
+
+            final Map<PwmEnvironment.ApplicationParameter, String> applicationParams = pwmApplication.getPwmEnvironment().getParameters();
+            final String keystoreFileString = applicationParams.get( PwmEnvironment.ApplicationParameter.AutoExportHttpsKeyStoreFile );
+            if ( StringUtil.isEmpty( keystoreFileString ) )
+            {
+                return;
+            }
+
+            final File keyStoreFile = new File( keystoreFileString );
+            final String password = applicationParams.get( PwmEnvironment.ApplicationParameter.AutoExportHttpsKeyStorePassword );
+            final String alias = applicationParams.get( PwmEnvironment.ApplicationParameter.AutoExportHttpsKeyStoreAlias );
+            final KeyStore keyStore = HttpsServerCertificateManager.keyStoreForApplication( pwmApplication, new PasswordData( password ), alias );
+            X509Utils.outputKeystore( keyStore, keyStoreFile, password );
+            LOGGER.info( pwmApplication.getSessionLabel(), () -> "exported application https key to keystore file " + keyStoreFile.getAbsolutePath() );
+        }
+        catch ( final Exception e )
+        {
+            LOGGER.debug( pwmApplication.getSessionLabel(), () -> "error while generating keystore output: " + e.getMessage() );
+        }
+    }
+
+    static void outputTomcatConf( final PwmApplication pwmApplication )
+    {
+        try
+        {
+            final Map<PwmEnvironment.ApplicationParameter, String> applicationParams = pwmApplication.getPwmEnvironment().getParameters();
+            final String tomcatOutputFileStr = applicationParams.get( PwmEnvironment.ApplicationParameter.AutoWriteTomcatConfOutputFile );
+            if ( tomcatOutputFileStr != null && !tomcatOutputFileStr.isEmpty() )
+            {
+                LOGGER.trace( pwmApplication.getSessionLabel(),
+                        () -> "attempting to output tomcat configuration file as configured by environment parameters to " + tomcatOutputFileStr );
+                final File tomcatOutputFile = new File( tomcatOutputFileStr );
+                final File tomcatSourceFile;
+                {
+                    final String tomcatSourceFileStr = applicationParams.get( PwmEnvironment.ApplicationParameter.AutoWriteTomcatConfSourceFile );
+                    if ( tomcatSourceFileStr != null && !tomcatSourceFileStr.isEmpty() )
+                    {
+                        tomcatSourceFile = new File( tomcatSourceFileStr );
+                        if ( !tomcatSourceFile.exists() )
+                        {
+                            LOGGER.error( pwmApplication.getSessionLabel(),
+                                    () -> "can not output tomcat configuration file, source file does not exist: " + tomcatSourceFile.getAbsolutePath() );
+                            return;
+                        }
+                    }
+                    else
+                    {
+                        LOGGER.error( pwmApplication.getSessionLabel(),
+                                () -> "can not output tomcat configuration file, source file parameter '"
+                                        + PwmEnvironment.ApplicationParameter.AutoWriteTomcatConfSourceFile + "' is not specified." );
+                        return;
+                    }
+                }
+
+                try ( ByteArrayOutputStream outputContents = new ByteArrayOutputStream() )
+                {
+                    try ( InputStream fileInputStream = Files.newInputStream( tomcatOutputFile.toPath() ) )
+                    {
+                        ExportHttpsTomcatConfigCommand.TomcatConfigWriter.writeOutputFile(
+                                pwmApplication.getConfig(),
+                                fileInputStream,
+                                outputContents
+                        );
+                    }
+
+                    if ( tomcatOutputFile.exists() )
+                    {
+                        LOGGER.trace( pwmApplication.getSessionLabel(), () -> "deleting existing tomcat configuration file " + tomcatOutputFile.getAbsolutePath() );
+                        if ( tomcatOutputFile.delete() )
+                        {
+                            LOGGER.trace( pwmApplication.getSessionLabel(), () -> "deleted existing tomcat configuration file: " + tomcatOutputFile.getAbsolutePath() );
+                        }
+                    }
+
+                    try ( OutputStream fileOutputStream = Files.newOutputStream( tomcatOutputFile.toPath() ) )
+                    {
+                        fileOutputStream.write( outputContents.toByteArray() );
+                    }
+                }
+
+                LOGGER.info( pwmApplication.getSessionLabel(), () -> "successfully wrote tomcat configuration to file " + tomcatOutputFile.getAbsolutePath() );
+            }
+        }
+        catch ( final Exception e )
+        {
+            LOGGER.debug( pwmApplication.getSessionLabel(), () -> "error while generating tomcat conf output: " + e.getMessage() );
+        }
+
+    }
+
+    static void outputConfigurationToLog( final PwmApplication pwmApplication, final DomainID domainID )
+    {
+        final Instant startTime = Instant.now();
+
+        final Function<Map.Entry<String, String>, String> valueFormatter = entry ->
+        {
+            final String spacedValue = entry.getValue().replace( "\n", "\n   " );
+            return " " + entry.getKey() + "\n   " + spacedValue;
+        };
+
+        final StoredConfiguration storedConfiguration = pwmApplication.getConfig().getStoredConfiguration();
+        final List<StoredConfigKey> keys = CollectionUtil.iteratorToStream( storedConfiguration.keys() )
+                .filter( key -> key.getDomainID().equals( domainID ) )
+                .collect( Collectors.toList() );
+        final Map<String, String> debugStrings = StoredConfigurationUtil.makeDebugMap(
+                storedConfiguration,
+                keys,
+                PwmConstants.DEFAULT_LOCALE );
+
+        LOGGER.trace( pwmApplication.getSessionLabel(), () -> "--begin current configuration output for domainID '" + domainID + "'--" );
+        debugStrings.entrySet().stream()
+                .map( valueFormatter )
+                .map( s -> ( Supplier<CharSequence> ) () -> s )
+                .forEach( s -> LOGGER.trace( pwmApplication.getSessionLabel(), s ) );
+
+        final long itemCount = debugStrings.size();
+        LOGGER.trace( pwmApplication.getSessionLabel(), () -> "--end current configuration output of " + itemCount + " items --",
+                () -> TimeDuration.fromCurrent( startTime ) );
+    }
+
+    static void outputNonDefaultPropertiesToLog( final PwmApplication pwmApplication )
+    {
+        final Instant startTime = Instant.now();
+
+        final Map<AppProperty, String> nonDefaultProperties = pwmApplication.getConfig().readAllNonDefaultAppProperties();
+        if ( !CollectionUtil.isEmpty( nonDefaultProperties ) )
+        {
+            LOGGER.trace( pwmApplication.getSessionLabel(), () -> "--begin non-default app properties output--" );
+            nonDefaultProperties.entrySet().stream()
+                    .map( entry -> "AppProperty: " + entry.getKey().getKey() + " -> " + entry.getValue() )
+                    .map( s -> ( Supplier<CharSequence> ) () -> s )
+                    .forEach( s -> LOGGER.trace( pwmApplication.getSessionLabel(), s ) );
+            LOGGER.trace( pwmApplication.getSessionLabel(), () -> "--end non-default app properties output--", () -> TimeDuration.fromCurrent( startTime ) );
+        }
+        else
+        {
+            LOGGER.trace( pwmApplication.getSessionLabel(), () -> "no non-default app properties in configuration" );
+        }
+    }
+
+    static void outputApplicationInfoToLog( final PwmApplication pwmApplication )
+    {
+        final Instant startTime = Instant.now();
+
+        final Map<PwmAboutProperty, String> aboutProperties = PwmAboutProperty.makeInfoBean( pwmApplication );
+        if ( !CollectionUtil.isEmpty( aboutProperties ) )
+        {
+            LOGGER.trace( pwmApplication.getSessionLabel(), () -> "--begin application info--" );
+            aboutProperties.entrySet().stream()
+                    .map( entry -> "AppProperty: " + entry.getKey().getLabel() + " -> " + entry.getValue() )
+                    .map( s -> ( Supplier<CharSequence> ) () -> s )
+                    .forEach( s -> LOGGER.trace( pwmApplication.getSessionLabel(), s ) );
+            LOGGER.trace( pwmApplication.getSessionLabel(), () -> "--end application info--", () -> TimeDuration.fromCurrent( startTime ) );
+        }
+        else
+        {
+            LOGGER.trace( pwmApplication.getSessionLabel(), () -> "no non-default app properties in configuration" );
+        }
+    }
+}

server/src/main/java/password/pwm/PwmDomain.java

@@ -38,9 +38,11 @@ import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmServiceEnum;
 import password.pwm.svc.PwmServiceManager;
 import password.pwm.svc.cache.CacheService;
+import password.pwm.svc.cr.CrService;
 import password.pwm.svc.event.AuditService;
 import password.pwm.svc.httpclient.HttpClientService;
 import password.pwm.svc.intruder.IntruderDomainService;
+import password.pwm.svc.otp.OtpService;
 import password.pwm.svc.pwnotify.PwNotifyService;
 import password.pwm.svc.secure.DomainSecureService;
 import password.pwm.svc.sessiontrack.SessionTrackService;
@@ -48,17 +50,12 @@ import password.pwm.svc.stats.StatisticsService;
 import password.pwm.svc.token.TokenService;
 import password.pwm.svc.userhistory.UserHistoryService;
 import password.pwm.svc.wordlist.SharedHistoryService;
-import password.pwm.util.DailySummaryJob;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.svc.cr.CrService;
-import password.pwm.svc.otp.OtpService;
 
 import java.time.Instant;
 import java.util.List;
 import java.util.Objects;
-import java.util.concurrent.ExecutorService;
 
 /**
  * A repository for objects common to the servlet context.  A singleton
@@ -72,15 +69,18 @@ public class PwmDomain
 
     private final PwmApplication pwmApplication;
     private final DomainID domainID;
-
     private final PwmServiceManager pwmServiceManager;
+    private final SessionLabel sessionLabel;
 
     public PwmDomain( final PwmApplication pwmApplication, final DomainID domainID )
     {
         this.pwmApplication = Objects.requireNonNull( pwmApplication );
         this.domainID = Objects.requireNonNull( domainID );
 
-        final SessionLabel sessionLabel = SessionLabel.builder().domain( domainID.stringValue() ).build();
+        this.sessionLabel = pwmApplication.getPwmEnvironment().isInternalRuntimeInstance()
+                ? SessionLabel.RUNTIME_LABEL.toBuilder().domain( domainID.stringValue() ).build()
+                : SessionLabel.SYSTEM_LABEL.toBuilder().domain( domainID.stringValue() ).build();
+
         this.pwmServiceManager = new PwmServiceManager( sessionLabel, pwmApplication, domainID, PwmServiceEnum.forScope( PwmSettingScope.DOMAIN ) );
     }
 
@@ -89,15 +89,16 @@ public class PwmDomain
 
     {
         final Instant startTime = Instant.now();
-        LOGGER.trace( () -> "initializing domain " + domainID.stringValue() );
+        LOGGER.trace( sessionLabel, () -> "initializing domain " + domainID.stringValue() );
+
         pwmServiceManager.initAllServices();
 
+        if ( Boolean.parseBoolean( getConfig().readAppProperty( AppProperty.LOGGING_OUTPUT_CONFIGURATION ) ) )
         {
-            final ExecutorService executorService = PwmScheduler.makeSingleThreadExecutorService( getPwmApplication(), DailySummaryJob.class );
-            pwmApplication.getPwmScheduler().scheduleDailyZuluZeroStartJob( new DailySummaryJob( this ), executorService, TimeDuration.ZERO );
+            PwmApplicationUtil.outputConfigurationToLog( pwmApplication, domainID );
         }
 
-        LOGGER.trace( () -> "completed initializing domain " + domainID.stringValue(), () -> TimeDuration.fromCurrent( startTime ) );
+        LOGGER.trace( sessionLabel, () -> "completed initializing domain " + domainID.stringValue(), () -> TimeDuration.fromCurrent( startTime ) );
     }
 
     public DomainConfig getConfig( )
@@ -236,9 +237,14 @@ public class PwmDomain
         return ( UserHistoryService ) pwmServiceManager.getService( PwmServiceEnum.UserHistoryService );
     }
 
+    public SessionLabel getSessionLabel()
+    {
+        return sessionLabel;
+    }
+
     public void shutdown()
     {
-        LOGGER.trace( () -> "beginning shutdown domain " + domainID.stringValue() );
+        LOGGER.trace( sessionLabel, () -> "beginning shutdown domain " + domainID.stringValue() );
         pwmServiceManager.shutdownAllServices();
     }
 

server/src/main/java/password/pwm/PwmDomainUtil.java

@@ -0,0 +1,233 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm;
+
+import password.pwm.bean.DomainID;
+import password.pwm.config.AppConfig;
+import password.pwm.config.DomainConfig;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogger;
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+
+class PwmDomainUtil
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( PwmDomainUtil.class );
+
+    static Map<DomainID, PwmDomain> createDomainInstances( final PwmApplication pwmApplication )
+            throws PwmUnrecoverableException
+    {
+        final Map<DomainID, PwmDomain> domainMap = new TreeMap<>();
+
+        if ( pwmApplication.getPwmEnvironment().isInternalRuntimeInstance() )
+        {
+            final DomainID domainID = pwmApplication.getPwmEnvironment().getConfig().getAdminDomainID();
+            domainMap.put( domainID, new PwmDomain( pwmApplication, domainID ) );
+        }
+        else
+        {
+            for ( final String domainIdString : pwmApplication.getPwmEnvironment().getConfig().getDomainIDs() )
+            {
+                final DomainID domainID = DomainID.create( domainIdString );
+                final PwmDomain newDomain = new PwmDomain( pwmApplication, domainID );
+                domainMap.put( domainID, newDomain );
+            }
+        }
+
+        return Collections.unmodifiableMap( domainMap );
+    }
+
+    static void initDomains(
+            final PwmApplication pwmApplication,
+            final Collection<PwmDomain> domains
+
+    )
+            throws PwmUnrecoverableException
+    {
+        final Instant domainInitStartTime = Instant.now();
+        LOGGER.trace( () -> "beginning domain initializations" );
+
+        final List<Callable<Optional<PwmUnrecoverableException>>> callables = domains.stream()
+                .map( DomainInitializingCallable::new )
+                .collect( Collectors.toUnmodifiableList() );
+
+        final  List<Optional<PwmUnrecoverableException>> domainStartException = pwmApplication.getPwmScheduler()
+                .executeImmediateThreadPerJobAndAwaitCompletion( PwmApplication.DOMAIN_STARTUP_THREADS, callables, pwmApplication.getSessionLabel(), PwmDomainUtil.class );
+
+        final Optional<PwmUnrecoverableException> domainStartupException = domainStartException.stream()
+                .filter( Optional::isPresent )
+                .map( Optional::get )
+                .findAny();
+
+        if ( domainStartupException.isPresent() )
+        {
+            throw domainStartupException.get();
+        }
+
+        LOGGER.trace( () -> "completed domain initialization for domains", () -> TimeDuration.fromCurrent( domainInitStartTime ) );
+    }
+
+    private static class DomainInitializingCallable implements Callable<Optional<PwmUnrecoverableException>>
+    {
+        private final PwmDomain pwmDomain;
+
+        DomainInitializingCallable( final PwmDomain pwmDomain )
+        {
+            this.pwmDomain = pwmDomain;
+        }
+
+        @Override
+        public Optional<PwmUnrecoverableException> call()
+                throws Exception
+        {
+            try
+            {
+                pwmDomain.initialize();
+                return Optional.empty();
+            }
+            catch ( final PwmUnrecoverableException e )
+            {
+                return Optional.of( e );
+            }
+        }
+    }
+
+    static Map<DomainID, PwmDomain> reInitDomains(
+            final PwmApplication pwmApplication,
+            final AppConfig newConfig,
+            final AppConfig oldConfig
+    )
+            throws PwmUnrecoverableException
+    {
+        final Map<DomainModifyCategory, Set<DomainID>> categorizedDomains = categorizeDomainModifications( newConfig, oldConfig );
+
+        final Set<PwmDomain> deletedDomains = pwmApplication.domains().entrySet().stream()
+                .filter( e -> categorizedDomains.get( DomainModifyCategory.obsolete ).contains( e.getKey() ) )
+                .map( Map.Entry::getValue ).collect( Collectors.toSet() );
+
+        final Set<PwmDomain> newDomains = pwmApplication.domains().entrySet().stream()
+                .filter( e -> categorizedDomains.get( DomainModifyCategory.created ).contains( e.getKey() ) )
+                .map( Map.Entry::getValue ).collect( Collectors.toSet() );
+
+        final Map<DomainID, PwmDomain> returnDomainMap = new TreeMap<>( pwmApplication.domains().entrySet().stream()
+                .filter( e -> categorizedDomains.get( DomainModifyCategory.unchanged ).contains( e.getKey() ) )
+                .collect( Collectors.toMap( Map.Entry::getKey, Map.Entry::getValue ) ) );
+
+        for ( final DomainID modifiedDomainID : categorizedDomains.get( DomainModifyCategory.modified ) )
+        {
+            deletedDomains.add( pwmApplication.domains().get( modifiedDomainID ) );
+            final PwmDomain newDomain = new PwmDomain( pwmApplication, modifiedDomainID );
+            newDomains.add( newDomain );
+            returnDomainMap.put( modifiedDomainID, newDomain );
+        }
+
+
+        initDomains( pwmApplication, newDomains );
+
+        processDeletedDomains( pwmApplication, deletedDomains );
+
+        return Collections.unmodifiableMap( returnDomainMap );
+    }
+
+    private static void processDeletedDomains(
+            final PwmApplication pwmApplication,
+            final Set<PwmDomain> deletedDomains
+    )
+    {
+        // 1 minute later ( to avoid interrupting any in-progress requests, shutdown any obsoleted domains
+        if ( !deletedDomains.isEmpty() )
+        {
+            pwmApplication.getPwmScheduler().immediateExecuteRunnableInNewThread( () ->
+                    {
+                        TimeDuration.MINUTE.pause();
+                        final Instant startTime = Instant.now();
+                        LOGGER.trace( pwmApplication.getSessionLabel(), () -> "shutting down obsoleted domain services" );
+                        deletedDomains.forEach( PwmDomain::shutdown );
+                        LOGGER.debug( pwmApplication.getSessionLabel(), () -> "shut down obsoleted domain services completed",
+                                () -> TimeDuration.fromCurrent( startTime ) );
+                    },
+                    pwmApplication.getSessionLabel(),
+                    "obsoleted domain cleanup" );
+        }
+
+    }
+
+    enum DomainModifyCategory
+    {
+        obsolete,
+        unchanged,
+        modified,
+        created,
+    }
+
+    private static Map<DomainModifyCategory, Set<DomainID>> categorizeDomainModifications(
+            final AppConfig newConfig,
+            final AppConfig oldConfig
+    )
+    {
+        final Map<DomainModifyCategory, Set<DomainID>> types = new EnumMap<>( DomainModifyCategory.class );
+
+        {
+            final Set<DomainID> obsoleteDomains = new HashSet<>( oldConfig.getDomainConfigs().keySet() );
+            obsoleteDomains.removeAll( newConfig.getDomainConfigs().keySet() );
+            types.put( DomainModifyCategory.obsolete, Collections.unmodifiableSet( obsoleteDomains ) );
+        }
+
+        {
+            final Set<DomainID> createdDomains = new HashSet<>( newConfig.getDomainConfigs().keySet() );
+            createdDomains.removeAll( oldConfig.getDomainConfigs().keySet() );
+            types.put( DomainModifyCategory.created, Collections.unmodifiableSet( createdDomains ) );
+        }
+
+        final Set<DomainID> unchangedDomains = new HashSet<>();
+        final Set<DomainID> modifiedDomains = new HashSet<>();
+        for ( final DomainID domainID : newConfig.getDomainConfigs().keySet() )
+        {
+            final DomainConfig newDomainConfig = newConfig.getDomainConfigs().get( domainID );
+            final String oldValueHash = oldConfig.getDomainConfigs().get( newDomainConfig.getDomainID() ).getValueHash();
+
+            if ( Objects.equals( oldValueHash, newDomainConfig.getValueHash() ) )
+            {
+                unchangedDomains.add( domainID );
+            }
+            else
+            {
+                modifiedDomains.add( domainID );
+            }
+        }
+        types.put( DomainModifyCategory.unchanged, Collections.unmodifiableSet( unchangedDomains ) );
+        types.put( DomainModifyCategory.modified, Collections.unmodifiableSet( modifiedDomains ) );
+        return Collections.unmodifiableMap( types );
+    }
+}

server/src/main/java/password/pwm/bean/SessionLabel.java

@@ -24,6 +24,7 @@ import lombok.Builder;
 import lombok.Value;
 import password.pwm.PwmConstants;
 import password.pwm.svc.PwmService;
+import password.pwm.util.java.StringUtil;
 
 import java.io.Serializable;
 
@@ -31,11 +32,14 @@ import java.io.Serializable;
 @Builder( toBuilder = true )
 public class SessionLabel implements Serializable
 {
-    public static final String SESSION_LABEL_SESSION_ID = "#";
-    public static final SessionLabel SYSTEM_LABEL = SessionLabel.builder().sessionID( SESSION_LABEL_SESSION_ID ).username( PwmConstants.PWM_APP_NAME ).build();
-    public static final SessionLabel TEST_SESSION_LABEL = SessionLabel.builder().sessionID( SESSION_LABEL_SESSION_ID ).username( "test" ).build();
-    public static final SessionLabel CLI_SESSION_LABEL = SessionLabel.builder().sessionID( SESSION_LABEL_SESSION_ID ).username( "cli" ).build();
-    public static final SessionLabel CONTEXT_SESSION_LABEL = SessionLabel.builder().sessionID( SESSION_LABEL_SESSION_ID ).username( "context" ).build();
+    private static final String SYSTEM_LABEL_SESSION_ID = "#";
+    private static final String RUNTIME_LABEL_SESSION_ID = "#";
+
+    public static final SessionLabel SYSTEM_LABEL = SessionLabel.builder().sessionID( SYSTEM_LABEL_SESSION_ID ).username( PwmConstants.PWM_APP_NAME ).build();
+    public static final SessionLabel RUNTIME_LABEL = SessionLabel.builder().sessionID( RUNTIME_LABEL_SESSION_ID ).username( "internal" ).build();
+    public static final SessionLabel TEST_SESSION_LABEL = SessionLabel.builder().sessionID( SYSTEM_LABEL_SESSION_ID ).username( "test" ).build();
+    public static final SessionLabel CLI_SESSION_LABEL = SessionLabel.builder().sessionID( SYSTEM_LABEL_SESSION_ID ).username( "cli" ).build();
+    public static final SessionLabel CONTEXT_SESSION_LABEL = SessionLabel.builder().sessionID( SYSTEM_LABEL_SESSION_ID ).username( "context" ).build();
 
     private final String sessionID;
     private final String requestID;
@@ -49,9 +53,46 @@ public class SessionLabel implements Serializable
     public static SessionLabel forPwmService( final PwmService pwmService, final DomainID domainID )
     {
         return SessionLabel.builder()
-                .sessionID( SESSION_LABEL_SESSION_ID )
+                .sessionID( SYSTEM_LABEL_SESSION_ID )
                 .username( pwmService.getClass().getSimpleName() )
                 .domain( domainID.stringValue() )
                 .build();
     }
+
+    public String toDebugLabel( )
+    {
+        final StringBuilder sb = new StringBuilder();
+        final String sessionID = getSessionID();
+        final String username = getUsername();
+
+        if ( StringUtil.notEmpty( sessionID ) )
+        {
+            sb.append( sessionID );
+        }
+        if ( StringUtil.notEmpty( domain ) )
+        {
+            if ( sb.length() > 0 )
+            {
+                sb.append( ',' );
+            }
+            sb.append( domain );
+        }
+        if ( StringUtil.notEmpty( username ) )
+        {
+            if ( sb.length() > 0 )
+            {
+                sb.append( ',' );
+            }
+            sb.append( username );
+        }
+
+        if ( sb.length() > 0 )
+        {
+            sb.insert( 0, "{" );
+            sb.append( "} " );
+        }
+
+        return sb.toString();
+    }
+
 }

server/src/main/java/password/pwm/config/AppConfig.java

@@ -433,4 +433,10 @@ public class AppConfig implements SettingReader
         }
         return Collections.unmodifiableMap( localeFlagMap );
     }
+
+    @Override
+    public String getValueHash()
+    {
+        return settingReader.getValueHash();
+    }
 }

server/src/main/java/password/pwm/config/DomainConfig.java

@@ -21,7 +21,6 @@
 package password.pwm.config;
 
 import password.pwm.AppProperty;
-import password.pwm.PwmConstants;
 import password.pwm.bean.DomainID;
 import password.pwm.bean.EmailItemBean;
 import password.pwm.bean.PrivateKeyCertificate;
@@ -44,7 +43,6 @@ import password.pwm.config.profile.UpdateProfileProfile;
 import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationUtil;
 import password.pwm.config.value.FileValue;
-import password.pwm.config.value.StoredValue;
 import password.pwm.config.value.data.ActionConfiguration;
 import password.pwm.config.value.data.FormConfiguration;
 import password.pwm.config.value.data.NamedSecretData;
@@ -57,10 +55,8 @@ import password.pwm.util.PasswordData;
 import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.secure.PwmHashAlgorithm;
 import password.pwm.util.secure.PwmSecurityKey;
 
-import java.security.MessageDigest;
 import java.security.cert.X509Certificate;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -115,7 +111,7 @@ public class DomainConfig implements SettingReader
                 ) ) );
 
         this.ldapProfiles = makeLdapProfileMap( this );
-        this.domainSecurityKey = makeDomainSecurityKey( appConfig, domainID );
+        this.domainSecurityKey = makeDomainSecurityKey( appConfig, settingReader.getValueHash() );
     }
 
     public AppConfig getAppConfig()
@@ -415,13 +411,12 @@ public class DomainConfig implements SettingReader
 
     private static PwmSecurityKey makeDomainSecurityKey(
             final AppConfig appConfig,
-            final DomainID domainID
+            final String valueHash
     )
     {
         try
         {
-            final String hashedData = valueHash( appConfig.getStoredConfiguration(), domainID );
-            final PwmSecurityKey domainKey = new PwmSecurityKey( hashedData );
+            final PwmSecurityKey domainKey = new PwmSecurityKey( valueHash );
             return appConfig.getSecurityKey().add( domainKey );
         }
         catch ( final PwmUnrecoverableException e )
@@ -430,19 +425,9 @@ public class DomainConfig implements SettingReader
         }
     }
 
-
-    private static String valueHash( final StoredConfiguration storedConfiguration, final DomainID domainID )
+    @Override
+    public String getValueHash()
     {
-        final MessageDigest messageDigest = PwmHashAlgorithm.SHA512.newMessageDigest();
-        messageDigest.update( domainID.stringValue().getBytes( PwmConstants.DEFAULT_CHARSET ) );
-
-        CollectionUtil.iteratorToStream( storedConfiguration.keys() )
-                .filter( key -> Objects.equals( key.getDomainID(), domainID ) )
-                .map( storedConfiguration::readStoredValue )
-                .flatMap( Optional::stream )
-                .map( StoredValue::valueHash )
-                .forEach( s -> messageDigest.update( s.getBytes( PwmConstants.DEFAULT_CHARSET ) ) );
-
-        return JavaHelper.binaryArrayToHex( messageDigest.digest() );
+        return settingReader.getValueHash();
     }
 }

server/src/main/java/password/pwm/config/SettingReader.java

@@ -62,4 +62,6 @@ public interface SettingReader
     PasswordData readSettingAsPassword( PwmSetting setting );
 
     Optional<Map<Locale, String>> readLocalizedBundle( PwmLocaleBundle className, String keyName );
+
+    String getValueHash();
 }

server/src/main/java/password/pwm/config/StoredSettingReader.java

@@ -20,6 +20,7 @@
 
 package password.pwm.config;
 
+import password.pwm.PwmConstants;
 import password.pwm.bean.DomainID;
 import password.pwm.bean.EmailItemBean;
 import password.pwm.bean.PrivateKeyCertificate;
@@ -49,8 +50,10 @@ import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.secure.PwmHashAlgorithm;
 
 import java.lang.reflect.InvocationTargetException;
+import java.security.MessageDigest;
 import java.security.cert.X509Certificate;
 import java.util.Arrays;
 import java.util.Collections;
@@ -74,12 +77,14 @@ public class StoredSettingReader implements SettingReader
     private final DomainID domainID;
 
     private final Map<ProfileDefinition, Map> profileCache;
+    private final String valueHash;
 
     public StoredSettingReader( final StoredConfiguration storedConfiguration, final String profileID, final DomainID domainID )
     {
         this.storedConfiguration = Objects.requireNonNull( storedConfiguration );
         this.profileID = profileID;
         this.domainID = Objects.requireNonNull( domainID );
+        this.valueHash = valueHash( storedConfiguration, domainID );
         this.profileCache = profileID == null
                 ? ProfileReader.makeCacheMap( storedConfiguration, domainID )
                 : Collections.emptyMap();
@@ -351,4 +356,25 @@ public class StoredSettingReader implements SettingReader
                 Map.Entry::getValue
         ) ) );
     }
+
+    @Override
+    public String getValueHash()
+    {
+        return valueHash;
+    }
+
+    private static String valueHash( final StoredConfiguration storedConfiguration, final DomainID domainID )
+    {
+        final MessageDigest messageDigest = PwmHashAlgorithm.SHA512.newMessageDigest();
+        messageDigest.update( domainID.stringValue().getBytes( PwmConstants.DEFAULT_CHARSET ) );
+
+        CollectionUtil.iteratorToStream( storedConfiguration.keys() )
+                .filter( key -> Objects.equals( key.getDomainID(), domainID ) )
+                .map( storedConfiguration::readStoredValue )
+                .flatMap( Optional::stream )
+                .map( StoredValue::valueHash )
+                .forEach( s -> messageDigest.update( s.getBytes( PwmConstants.DEFAULT_CHARSET ) ) );
+
+        return JavaHelper.binaryArrayToHex( messageDigest.digest() );
+    }
 }

server/src/main/java/password/pwm/config/stored/ConfigurationReader.java → server/src/main/java/password/pwm/config/stored/ConfigurationFileManager.java

@@ -57,13 +57,13 @@ import java.util.Optional;
 import java.util.stream.Collectors;
 
 /**
- * Read the PWM configuration.
+ * Read and write the PWM configuration XML file from the filesystem.
  *
  * @author Jason D. Rivard
  */
-public class ConfigurationReader
+public class ConfigurationFileManager
 {
-    private static final PwmLogger LOGGER = PwmLogger.getLogger( ConfigurationReader.class.getName() );
+    private static final PwmLogger LOGGER = PwmLogger.getLogger( ConfigurationFileManager.class.getName() );
 
     private final File configFile;
     private final String configFileChecksum;
@@ -75,7 +75,7 @@ public class ConfigurationReader
 
     private volatile boolean saveInProgress;
 
-    public ConfigurationReader( final File configFile ) throws PwmUnrecoverableException
+    public ConfigurationFileManager( final File configFile ) throws PwmUnrecoverableException
     {
         this.configFile = configFile;
 

server/src/main/java/password/pwm/config/stored/StoredConfigZipJsonSerializer.java

@@ -145,19 +145,22 @@ public class StoredConfigZipJsonSerializer implements StoredConfigSerializer
         {
             if ( SETTINGS_FILENAME.equals( zipEntry.getName() ) )
             {
-                final String stringData = JavaHelper.copyToString( zipInputStream, PwmConstants.DEFAULT_CHARSET );
+                final String stringData = JavaHelper.copyToString( zipInputStream, PwmConstants.DEFAULT_CHARSET, Integer.MAX_VALUE )
+                        .orElse( "" );
                 final List<SerializedValue> readComponents = JsonFactory.get().deserializeList( stringData, SerializedValue.class );
                 serializedValues.addAll( readComponents );
             }
             else if ( META_VALUES_FILENAME.equals( zipEntry.getName() ) )
             {
-                final String stringData = JavaHelper.copyToString( zipInputStream, PwmConstants.DEFAULT_CHARSET );
+                final String stringData = JavaHelper.copyToString( zipInputStream, PwmConstants.DEFAULT_CHARSET, Integer.MAX_VALUE )
+                        .orElse( "" );
                 final List<SerializedMetaValue> readMetaValues = JsonFactory.get().deserializeList( stringData, SerializedMetaValue.class );
                 serializedMetaValues.addAll( readMetaValues );
             }
             else if ( META_FILENAME.equals( zipEntry.getName() ) )
             {
-                final String stringData = JavaHelper.copyToString( zipInputStream, PwmConstants.DEFAULT_CHARSET );
+                final String stringData = JavaHelper.copyToString( zipInputStream, PwmConstants.DEFAULT_CHARSET, Integer.MAX_VALUE )
+                        .orElse( "" );
                 metaData = JsonFactory.get().deserialize( stringData, MetaData.class );
             }
             else if ( zipEntry.getName().endsWith( XREF_SUFFIX ) )

server/src/main/java/password/pwm/health/ApplianceStatusChecker.java

@@ -1,192 +0,0 @@
-/*
- * Password Management Servlets (PWM)
- * http://www.pwm-project.org
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2021 The PWM Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package password.pwm.health;
-
-import org.apache.commons.io.FileUtils;
-import password.pwm.PwmApplication;
-import password.pwm.PwmEnvironment;
-import password.pwm.bean.DomainID;
-import password.pwm.error.ErrorInformation;
-import password.pwm.error.PwmError;
-import password.pwm.error.PwmOperationalException;
-import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.http.HttpMethod;
-import password.pwm.svc.httpclient.PwmHttpClient;
-import password.pwm.svc.httpclient.PwmHttpClientConfiguration;
-import password.pwm.svc.httpclient.PwmHttpClientRequest;
-import password.pwm.svc.httpclient.PwmHttpClientResponse;
-import password.pwm.util.json.JsonFactory;
-import password.pwm.util.java.StringUtil;
-import password.pwm.util.logging.PwmLogger;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Supplier;
-
-public class ApplianceStatusChecker implements HealthSupplier
-{
-    private static final PwmLogger LOGGER = PwmLogger.forClass( ApplianceStatusChecker.class );
-
-    private static class UpdateStatus implements Serializable
-    {
-        boolean pendingInstallation;
-        boolean autoUpdatesEnabled;
-        boolean updateServiceConfigured;
-    }
-
-    @Override
-    public List<Supplier<List<HealthRecord>>> jobs( final HealthSupplier.HealthSupplierRequest request )
-    {
-        final Supplier<List<HealthRecord>> supplier = () -> doHealthCheck( request );
-        return Collections.singletonList( supplier );
-    }
-
-    private List<HealthRecord> doHealthCheck( final HealthSupplier.HealthSupplierRequest request )
-    {
-        final PwmApplication pwmApplication = request.getPwmApplication();
-        final boolean isApplianceAvailable = pwmApplication.getPwmEnvironment().getFlags().contains( PwmEnvironment.ApplicationFlag.Appliance );
-
-        if ( !isApplianceAvailable )
-        {
-            return Collections.emptyList();
-        }
-
-        try
-        {
-            return List.copyOf( readApplianceHealthStatus( request ) );
-        }
-        catch ( final Exception e )
-        {
-            LOGGER.error( request.getSessionLabel(), () -> "error communicating with client " + e.getMessage() );
-        }
-
-        return Collections.emptyList();
-    }
-
-    private List<HealthRecord> readApplianceHealthStatus( final HealthSupplier.HealthSupplierRequest request  )
-            throws PwmUnrecoverableException, PwmOperationalException
-    {
-        final PwmApplication pwmApplication = request.getPwmApplication();
-        final List<HealthRecord> healthRecords = new ArrayList<>();
-
-        final String url = figureUrl( request );
-        final Map<String, String> requestHeaders = Collections.singletonMap( "sspr-authorization-token", getApplianceAccessToken( pwmApplication ) );
-
-        final PwmHttpClientConfiguration pwmHttpClientConfiguration = PwmHttpClientConfiguration.builder()
-                .trustManagerType( PwmHttpClientConfiguration.TrustManagerType.promiscuous )
-                .build();
-
-        final PwmHttpClient pwmHttpClient = pwmApplication.getHttpClientService().getPwmHttpClient( pwmHttpClientConfiguration );
-        final PwmHttpClientRequest pwmHttpClientRequest = PwmHttpClientRequest.builder()
-                .method( HttpMethod.GET )
-                .url( url )
-                .headers( requestHeaders )
-                .build();
-
-        final PwmHttpClientResponse response = pwmHttpClient.makeRequest( pwmHttpClientRequest, request.getSessionLabel() );
-
-        LOGGER.trace( request.getSessionLabel(), () -> "https response from appliance server request: " + response.getBody() );
-
-        final String jsonString = response.getBody();
-
-        LOGGER.debug( () -> "response from /sspr/appliance-update-status: " + jsonString );
-
-        final UpdateStatus updateStatus = JsonFactory.get().deserialize( jsonString, UpdateStatus.class );
-
-        if ( updateStatus.pendingInstallation )
-        {
-            healthRecords.add( HealthRecord.forMessage( DomainID.systemId(), HealthMessage.Appliance_PendingUpdates ) );
-        }
-
-        if ( !updateStatus.autoUpdatesEnabled )
-        {
-            healthRecords.add( HealthRecord.forMessage( DomainID.systemId(), HealthMessage.Appliance_UpdatesNotEnabled ) );
-        }
-
-        if ( !updateStatus.updateServiceConfigured )
-        {
-            healthRecords.add( HealthRecord.forMessage( DomainID.systemId(), HealthMessage.Appliance_UpdateServiceNotConfigured ) );
-        }
-
-        return Collections.unmodifiableList( healthRecords );
-
-    }
-
-    private String getApplianceAccessToken( final PwmApplication pwmApplication ) throws PwmOperationalException
-    {
-        final String tokenFile = pwmApplication.getPwmEnvironment().getParameters().get( PwmEnvironment.ApplicationParameter.ApplianceTokenFile );
-        if ( StringUtil.isEmpty( tokenFile ) )
-        {
-            final String msg = "unable to determine appliance token, token file environment param "
-                    + PwmEnvironment.ApplicationParameter.ApplianceTokenFile + " is not set";
-            throw new PwmOperationalException( new ErrorInformation( PwmError.ERROR_INTERNAL, msg ) );
-        }
-        final String fileInput = readFileContents( tokenFile );
-        if ( fileInput != null )
-        {
-            return fileInput.trim();
-        }
-        return "";
-    }
-
-    private String figureUrl( final HealthSupplier.HealthSupplierRequest request ) throws PwmOperationalException
-    {
-        final PwmApplication pwmApplication = request.getPwmApplication();
-        final String hostnameFile = pwmApplication.getPwmEnvironment().getParameters().get( PwmEnvironment.ApplicationParameter.ApplianceHostnameFile );
-        if ( StringUtil.isEmpty( hostnameFile ) )
-        {
-            final String msg = "unable to determine appliance hostname, hostname file environment param "
-                    + PwmEnvironment.ApplicationParameter.ApplianceHostnameFile + " is not set";
-            throw new PwmOperationalException( new ErrorInformation( PwmError.ERROR_INTERNAL, msg ) );
-        }
-
-        final String hostname = readFileContents( hostnameFile );
-        final String port = pwmApplication.getPwmEnvironment().getParameters().get( PwmEnvironment.ApplicationParameter.AppliancePort );
-
-        final String url = "https://" + hostname + ":" + port + "/sspr/appliance-update-status";
-        LOGGER.trace( request.getSessionLabel(), () -> "calculated appliance host url as: " + url );
-        return url;
-    }
-
-    private String readFileContents( final String filename ) throws PwmOperationalException
-    {
-        try
-        {
-            final String fileInput = FileUtils.readFileToString( new File( filename ) );
-            if ( fileInput != null )
-            {
-                final String trimmedStr = fileInput.trim();
-                return trimmedStr.replace( "\n", "" );
-            }
-            return "";
-        }
-        catch ( final IOException e )
-        {
-            final String msg = "unable to read contents of file '" + filename + "', error: " + e.getMessage();
-            throw new PwmOperationalException( new ErrorInformation( PwmError.ERROR_INTERNAL, msg ), e );
-        }
-    }
-}

server/src/main/java/password/pwm/health/HealthService.java

@@ -58,8 +58,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 import java.util.zip.ZipOutputStream;
@@ -73,12 +73,11 @@ public class HealthService extends AbstractPwmService implements PwmService
             new JavaChecker(),
             new ConfigurationChecker(),
             new LocalDBHealthChecker(),
-            new ApplianceStatusChecker(),
             new CertificateChecker() );
 
 
-    private ExecutorService executorService;
-    private ExecutorService supportZipWriterService;
+    private ScheduledExecutorService executorService;
+    private ScheduledExecutorService supportZipWriterService;
     private HealthMonitorSettings settings;
 
     private final Map<HealthMonitorFlag, Serializable> healthProperties = new ConcurrentHashMap<>();
@@ -122,8 +121,8 @@ public class HealthService extends AbstractPwmService implements PwmService
             return STATUS.CLOSED;
         }
 
-        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-        supportZipWriterService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        executorService = PwmScheduler.makeBackgroundServiceExecutor( pwmApplication, getSessionLabel(), this.getClass() );
+        supportZipWriterService = PwmScheduler.makeBackgroundServiceExecutor( pwmApplication, getSessionLabel(), this.getClass() );
         scheduleNextZipOutput();
 
         if ( settings.getThreadDumpInterval().as( TimeDuration.Unit.SECONDS ) > 0 )
@@ -165,7 +164,7 @@ public class HealthService extends AbstractPwmService implements PwmService
         {
             final Instant startTime = Instant.now();
             LOGGER.trace( () ->  "begin force immediate check" );
-            final Future future = getPwmApplication().getPwmScheduler().scheduleJob( new ImmediateJob(), executorService, TimeDuration.ZERO );
+            final Future<?> future = getPwmApplication().getPwmScheduler().scheduleJob( new ImmediateJob(), executorService, TimeDuration.ZERO );
             settings.getMaximumForceCheckWait().pause( future::isDone );
             final TimeDuration checkDuration = TimeDuration.fromCurrent( startTime );
             averageStats.update( AverageStatKey.checkProcessTime, checkDuration.asDuration() );
@@ -187,7 +186,7 @@ public class HealthService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         if ( executorService != null )
         {

server/src/main/java/password/pwm/http/ContextManager.java

@@ -31,8 +31,8 @@ import password.pwm.config.AppConfig;
 import password.pwm.config.DomainConfig;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.profile.LdapProfile;
+import password.pwm.config.stored.ConfigurationFileManager;
 import password.pwm.config.stored.ConfigurationProperty;
-import password.pwm.config.stored.ConfigurationReader;
 import password.pwm.config.stored.StoredConfigKey;
 import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationModifier;
@@ -93,7 +93,7 @@ public class ContextManager implements Serializable
     private transient ScheduledExecutorService taskMaster;
 
     private transient volatile PwmApplication pwmApplication;
-    private transient ConfigurationReader configReader;
+    private transient ConfigurationFileManager configReader;
     private ErrorInformation startupErrorInformation;
 
     private final AtomicInteger restartCount = new AtomicInteger( 0 );
@@ -246,7 +246,7 @@ public class ContextManager implements Serializable
         {
             configurationFile = locateConfigurationFile( applicationPath, PwmConstants.DEFAULT_CONFIG_FILE_FILENAME );
 
-            configReader = new ConfigurationReader( configurationFile );
+            configReader = new ConfigurationFileManager( configurationFile );
             appConfig = configReader.getConfiguration();
 
             mode = startupErrorInformation == null ? configReader.getConfigMode() : PwmApplicationMode.ERROR;
@@ -304,7 +304,7 @@ public class ContextManager implements Serializable
 
         taskMaster = Executors.newSingleThreadScheduledExecutor(
                 PwmScheduler.makePwmThreadFactory(
-                        PwmScheduler.makeThreadName( pwmApplication, this.getClass() ) + "-",
+                        PwmScheduler.makeThreadName( SESSION_LABEL, pwmApplication, this.getClass() ) + "-",
                         true
                 ) );
 
@@ -338,7 +338,7 @@ public class ContextManager implements Serializable
     }
 
     private void checkConfigForAutoImportLdapCerts(
-            final ConfigurationReader configReader
+            final ConfigurationFileManager configReader
     )
     {
         if ( configReader == null || configReader.getStoredConfiguration() == null )
@@ -418,7 +418,7 @@ public class ContextManager implements Serializable
         taskMaster.schedule( new RestartFlagWatcher(), 0, TimeUnit.MILLISECONDS );
     }
 
-    public ConfigurationReader getConfigReader( )
+    public ConfigurationFileManager getConfigReader( )
     {
         return configReader;
     }
@@ -533,7 +533,7 @@ public class ContextManager implements Serializable
 
                 try
                 {
-                        reInitialize();
+                    reInitialize();
                 }
                 catch ( final Exception e )
                 {

server/src/main/java/password/pwm/http/PwmRequest.java

@@ -24,7 +24,6 @@ import lombok.Value;
 import org.apache.commons.fileupload.FileItemIterator;
 import org.apache.commons.fileupload.FileItemStream;
 import org.apache.commons.fileupload.servlet.ServletFileUpload;
-import org.apache.commons.io.IOUtils;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.PwmApplicationMode;
@@ -52,12 +51,13 @@ import password.pwm.config.value.data.FormConfiguration;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.util.java.ImmutableByteArray;
 import password.pwm.http.servlet.AbstractPwmServlet;
 import password.pwm.http.servlet.PwmRequestID;
 import password.pwm.http.servlet.PwmServletDefinition;
 import password.pwm.ldap.UserInfo;
 import password.pwm.util.Validator;
+import password.pwm.util.java.ImmutableByteArray;
+import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.LazySupplier;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
@@ -69,7 +69,6 @@ import password.pwm.ws.server.RestResultBean;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Serializable;
@@ -282,20 +281,18 @@ public class PwmRequest extends PwmHttpRequestWrapper
                 {
                     final FileItemStream item = iter.next();
                     final InputStream inputStream = item.openStream();
-                    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-                    final long length = IOUtils.copyLarge( inputStream, baos, 0, maxFileSize + 1 );
-                    if ( length > maxFileSize )
+                    final ImmutableByteArray fileContents = JavaHelper.copyToBytes( inputStream, maxFileSize + 1 );
+                    if ( fileContents.size() > maxFileSize )
                     {
                         final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_INTERNAL, "upload file size limit exceeded" );
                         LOGGER.error( this, errorInformation );
                         respondWithError( errorInformation );
                         return Collections.emptyMap();
                     }
-                    final byte[] outputFile = baos.toByteArray();
                     final FileUploadItem fileUploadItem = new FileUploadItem(
                             item.getName(),
                             item.getContentType(),
-                            ImmutableByteArray.of( outputFile )
+                            fileContents
                     );
                     returnObj.put( item.getFieldName(), fileUploadItem );
                 }

server/src/main/java/password/pwm/http/filter/ConfigAccessFilter.java

@@ -23,7 +23,7 @@ package password.pwm.http.filter;
 import password.pwm.AppProperty;
 import password.pwm.Permission;
 import password.pwm.PwmApplicationMode;
-import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.config.stored.ConfigurationFileManager;
 import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationUtil;
 import password.pwm.error.ErrorInformation;
@@ -99,7 +99,7 @@ public class ConfigAccessFilter extends AbstractPwmFilter
     )
             throws IOException, PwmUnrecoverableException, ServletException
     {
-        final ConfigurationReader runningConfigReader = ContextManager.getContextManager( pwmRequest.getHttpServletRequest().getSession() ).getConfigReader();
+        final ConfigurationFileManager runningConfigReader = ContextManager.getContextManager( pwmRequest.getHttpServletRequest().getSession() ).getConfigReader();
         final StoredConfiguration storedConfig = runningConfigReader.getStoredConfiguration();
 
         checkPreconditions( pwmRequest, storedConfig );

server/src/main/java/password/pwm/http/servlet/ForgottenUsernameServlet.java

@@ -44,6 +44,7 @@ import password.pwm.ldap.UserInfoFactory;
 import password.pwm.ldap.search.SearchConfiguration;
 import password.pwm.ldap.search.UserSearchEngine;
 import password.pwm.svc.intruder.IntruderServiceClient;
+import password.pwm.svc.sms.SmsQueueService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsClient;
 import password.pwm.util.CaptchaUtility;
@@ -326,7 +327,7 @@ public class ForgottenUsernameServlet extends AbstractPwmServlet
 
         final MacroRequest macroRequest = MacroRequest.forUser( pwmDomain.getPwmApplication(), sessionLabel, userInfo, null );
 
-        pwmDomain.getPwmApplication().sendSmsUsingQueue( toNumber, smsMessage, sessionLabel, macroRequest );
+        SmsQueueService.sendSmsUsingQueue( pwmDomain.getPwmApplication(), toNumber, smsMessage, sessionLabel, macroRequest );
         return null;
     }
 

server/src/main/java/password/pwm/http/servlet/activation/ActivateUserUtils.java

@@ -54,6 +54,7 @@ import password.pwm.ldap.auth.PwmAuthenticationSource;
 import password.pwm.ldap.auth.SessionAuthenticator;
 import password.pwm.svc.event.AuditEvent;
 import password.pwm.svc.event.AuditServiceClient;
+import password.pwm.svc.sms.SmsQueueService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsClient;
 import password.pwm.util.form.FormUtility;
@@ -285,7 +286,8 @@ class ActivateUserUtils
             return false;
         }
 
-        pwmRequest.getPwmApplication().sendSmsUsingQueue(
+        SmsQueueService.sendSmsUsingQueue(
+                pwmRequest.getPwmApplication(),
                 toSmsNumber,
                 message,
                 pwmRequest.getLabel(),

server/src/main/java/password/pwm/http/servlet/configguide/ConfigGuideUtils.java

@@ -33,7 +33,7 @@ import password.pwm.config.AppConfig;
 import password.pwm.config.PwmSetting;
 import password.pwm.http.servlet.configeditor.function.UserMatchViewerFunction;
 import password.pwm.config.stored.ConfigurationProperty;
-import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.config.stored.ConfigurationFileManager;
 import password.pwm.config.stored.StoredConfigKey;
 import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationFactory;
@@ -106,7 +106,7 @@ public class ConfigGuideUtils
     )
             throws PwmOperationalException, PwmUnrecoverableException
     {
-        final ConfigurationReader configReader = contextManager.getConfigReader();
+        final ConfigurationFileManager configReader = contextManager.getConfigReader();
         final PwmApplication pwmApplication = contextManager.getPwmApplication();
 
         try

server/src/main/java/password/pwm/http/servlet/configmanager/ConfigManagerLoginServlet.java

@@ -30,7 +30,7 @@ import password.pwm.PwmConstants;
 import password.pwm.PwmDomain;
 import password.pwm.bean.UserIdentity;
 import password.pwm.config.stored.ConfigurationProperty;
-import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.config.stored.ConfigurationFileManager;
 import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationUtil;
 import password.pwm.error.ErrorInformation;
@@ -138,7 +138,7 @@ public class ConfigManagerLoginServlet extends AbstractPwmServlet
             throws PwmUnrecoverableException, IOException, ServletException
     {
         final PwmDomain pwmDomain = pwmRequest.getPwmDomain();
-        final ConfigurationReader runningConfigReader = ContextManager.getContextManager( pwmRequest.getHttpServletRequest().getSession() ).getConfigReader();
+        final ConfigurationFileManager runningConfigReader = ContextManager.getContextManager( pwmRequest.getHttpServletRequest().getSession() ).getConfigReader();
         final StoredConfiguration storedConfig = runningConfigReader.getStoredConfiguration();
 
         final String password = pwmRequest.readParameterAsString( "password" );
@@ -321,7 +321,7 @@ public class ConfigManagerLoginServlet extends AbstractPwmServlet
             return;
         }
 
-        final ConfigurationReader runningConfigReader = ContextManager.getContextManager( pwmRequest.getHttpServletRequest().getSession() ).getConfigReader();
+        final ConfigurationFileManager runningConfigReader = ContextManager.getContextManager( pwmRequest.getHttpServletRequest().getSession() ).getConfigReader();
         final StoredConfiguration storedConfig = runningConfigReader.getStoredConfiguration();
 
         try

server/src/main/java/password/pwm/http/servlet/configmanager/ConfigManagerServlet.java

@@ -22,14 +22,13 @@ package password.pwm.http.servlet.configmanager;
 
 import com.novell.ldapchai.exception.ChaiUnavailableException;
 import org.apache.commons.csv.CSVPrinter;
-import org.apache.commons.io.IOUtils;
 import password.pwm.AppProperty;
 import password.pwm.Permission;
 import password.pwm.PwmConstants;
 import password.pwm.PwmDomain;
 import password.pwm.config.AppConfig;
 import password.pwm.config.stored.ConfigurationProperty;
-import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.config.stored.ConfigurationFileManager;
 import password.pwm.config.stored.StoredConfigKey;
 import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationFactory;
@@ -194,12 +193,12 @@ public class ConfigManagerServlet extends AbstractPwmServlet
     void initRequestAttributes( final PwmRequest pwmRequest )
             throws PwmUnrecoverableException
     {
-        final ConfigurationReader configurationReader = pwmRequest.getContextManager().getConfigReader();
+        final ConfigurationFileManager configurationFileManager = pwmRequest.getContextManager().getConfigReader();
         pwmRequest.setAttribute( PwmRequestAttribute.PageTitle, LocaleHelper.getLocalizedMessage( Config.Title_ConfigManager, pwmRequest ) );
         pwmRequest.setAttribute( PwmRequestAttribute.ApplicationPath, pwmRequest.getPwmApplication().getPwmEnvironment().getApplicationPath().getAbsolutePath() );
-        pwmRequest.setAttribute( PwmRequestAttribute.ConfigFilename, configurationReader.getConfigFile().getAbsolutePath() );
+        pwmRequest.setAttribute( PwmRequestAttribute.ConfigFilename, configurationFileManager.getConfigFile().getAbsolutePath() );
         {
-            final Instant lastModifyTime = configurationReader.getStoredConfiguration().modifyTime();
+            final Instant lastModifyTime = configurationFileManager.getStoredConfiguration().modifyTime();
             final String output = lastModifyTime == null
                     ? LocaleHelper.getLocalizedMessage( Display.Value_NotApplicable, pwmRequest )
                     : StringUtil.toIsoDate( lastModifyTime );
@@ -209,7 +208,7 @@ public class ConfigManagerServlet extends AbstractPwmServlet
         pwmRequest.setAttribute(
                 PwmRequestAttribute.ConfigHasPassword,
                 LocaleHelper.booleanString(
-                        StoredConfigurationUtil.hasPassword( configurationReader.getStoredConfiguration() ),
+                        StoredConfigurationUtil.hasPassword( configurationFileManager.getStoredConfiguration() ),
                         pwmRequest.getLocale(),
                         pwmRequest.getDomainConfig()
                 )
@@ -419,11 +418,9 @@ public class ConfigManagerServlet extends AbstractPwmServlet
                 pwmRequest.getDomainConfig().readAppProperty( AppProperty.DOWNLOAD_FILENAME_LDAP_PERMISSION_CSV )
         );
 
-        final CSVPrinter csvPrinter = MiscUtil.makeCsvPrinter( pwmRequest.getPwmResponse().getOutputStream() );
-        try
+        try ( CSVPrinter csvPrinter = MiscUtil.makeCsvPrinter( pwmRequest.getPwmResponse().getOutputStream() ) )
         {
 
-            final StoredConfiguration storedConfiguration = readCurrentConfiguration( pwmRequest );
             final LDAPPermissionCalculator ldapPermissionCalculator = new LDAPPermissionCalculator( pwmRequest.getDomainConfig() );
 
             for ( final LDAPPermissionCalculator.PermissionRecord permissionRecord : ldapPermissionCalculator.getPermissionRecords() )
@@ -446,10 +443,6 @@ public class ConfigManagerServlet extends AbstractPwmServlet
             LOGGER.error( pwmRequest, errorInformation );
             pwmRequest.respondWithError( errorInformation );
         }
-        finally
-        {
-            IOUtils.closeQuietly( csvPrinter );
-        }
     }
 }
 

server/src/main/java/password/pwm/http/servlet/peoplesearch/PeopleSearchDataReader.java

@@ -28,8 +28,8 @@ import com.novell.ldapchai.provider.ChaiProvider;
 import lombok.Value;
 import org.apache.commons.csv.CSVPrinter;
 import password.pwm.AppProperty;
-import password.pwm.PwmDomain;
 import password.pwm.PwmConstants;
+import password.pwm.PwmDomain;
 import password.pwm.bean.UserIdentity;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.profile.PeopleSearchProfile;
@@ -49,11 +49,11 @@ import password.pwm.http.servlet.peoplesearch.bean.SearchResultBean;
 import password.pwm.http.servlet.peoplesearch.bean.UserDetailBean;
 import password.pwm.http.servlet.peoplesearch.bean.UserReferenceBean;
 import password.pwm.i18n.Display;
-import password.pwm.ldap.permission.UserPermissionUtility;
 import password.pwm.ldap.PhotoDataBean;
 import password.pwm.ldap.UserInfo;
 import password.pwm.ldap.UserInfoFactory;
 import password.pwm.ldap.permission.UserPermissionType;
+import password.pwm.ldap.permission.UserPermissionUtility;
 import password.pwm.ldap.search.SearchConfiguration;
 import password.pwm.ldap.search.UserSearchEngine;
 import password.pwm.ldap.search.UserSearchResults;
@@ -66,9 +66,9 @@ import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.MiscUtil;
 import password.pwm.util.java.PwmTimeUtil;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroRequest;
 

server/src/main/java/password/pwm/http/servlet/peoplesearch/PeopleSearchService.java

@@ -30,10 +30,7 @@ import password.pwm.util.PwmScheduler;
 
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 
 public class PeopleSearchService extends AbstractPwmService implements PwmService
 {
@@ -45,21 +42,13 @@ public class PeopleSearchService extends AbstractPwmService implements PwmServic
     {
         final int maxThreadCount = 5;
 
-        final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( pwmApplication, PeopleSearchService.class ), true );
-        threadPoolExecutor = new ThreadPoolExecutor(
-                maxThreadCount,
-                maxThreadCount,
-                1,
-                TimeUnit.MINUTES,
-                new ArrayBlockingQueue<>( 5000 ),
-                threadFactory
-        );
+        threadPoolExecutor = PwmScheduler.makeMultiThreadExecutor( maxThreadCount, pwmApplication, getSessionLabel(), PeopleSearchService.class );
 
         return STATUS.OPEN;
     }
 
     @Override
-    public void close()
+    public void shutdownImpl()
     {
         threadPoolExecutor.shutdown();
     }

server/src/main/java/password/pwm/http/servlet/resource/ResourceServletConfiguration.java

@@ -22,20 +22,19 @@ package password.pwm.http.servlet.resource;
 
 import lombok.Value;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
 import password.pwm.AppProperty;
 import password.pwm.PwmDomain;
 import password.pwm.bean.SessionLabel;
 import password.pwm.config.DomainConfig;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.value.FileValue;
-import password.pwm.util.java.ImmutableByteArray;
 import password.pwm.util.java.CollectionUtil;
-import password.pwm.util.json.JsonFactory;
+import password.pwm.util.java.ImmutableByteArray;
+import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.logging.PwmLogger;
 
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
@@ -55,6 +54,9 @@ class ResourceServletConfiguration
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( ResourceServletConfiguration.class );
 
+    private static final int MAX_RESOURCE_SIZE = 10_000_000;
+    private static final int MAX_RESOURCE_COUNT = 1000;
+
     // settings with default values, values are set by app properties.
     private int maxCacheItems;
     private long cacheExpireSeconds;
@@ -180,7 +182,10 @@ class ResourceServletConfiguration
         return new ResourceServletConfiguration();
     }
 
-    private static Map<String, FileResource> makeMemoryFileMapFromZipInput( final SessionLabel sessionLabel, final ImmutableByteArray content )
+    private static Map<String, FileResource> makeMemoryFileMapFromZipInput(
+            final SessionLabel sessionLabel,
+            final ImmutableByteArray content
+    )
             throws IOException
     {
         final ZipInputStream stream = new ZipInputStream( content.newByteArrayInputStream() );
@@ -193,11 +198,22 @@ class ResourceServletConfiguration
             {
                 final String name = entry.getName();
                 final Instant lastModified = Instant.ofEpochMilli( entry.getTime() );
-                final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
-                IOUtils.copy( stream, byteArrayOutputStream );
-                final ImmutableByteArray contents = ImmutableByteArray.of( byteArrayOutputStream.toByteArray() );
-                memoryMap.put( name, new MemoryFileResource( name, contents, lastModified ) );
+                final ImmutableByteArray contents = JavaHelper.copyToBytes( stream, MAX_RESOURCE_SIZE + 1 );
+                if ( contents.size() > MAX_RESOURCE_COUNT )
+                {
+                    final String entryName = entry.getName();
+                    LOGGER.error( () -> "ignoring resource bundle zip file entry '" + entryName
+                            + "' due to size being greater than max of " + MAX_RESOURCE_SIZE + " bytes " );
+                }
+                else if ( memoryMap.size() > MAX_RESOURCE_COUNT )
+                {
+                    final String entryName = entry.getName();
+                    LOGGER.error( () -> "ignoring resource bundle zip file entry '" + entryName
+                            + "' due to total resource count being greater than max of " + MAX_RESOURCE_COUNT );
+                }
+                else
                 {
+                    memoryMap.put( name, new MemoryFileResource( name, contents, lastModified ) );
                     final String finalEntry = entry.getName();
                     LOGGER.trace( sessionLabel, () -> "discovered file in configured resource bundle: " + finalEntry );
                 }

server/src/main/java/password/pwm/http/servlet/resource/ResourceServletService.java

@@ -176,7 +176,7 @@ public class ResourceServletService extends AbstractPwmService implements PwmSer
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         setStatus( STATUS.CLOSED );
     }

server/src/main/java/password/pwm/http/state/SessionStateService.java

@@ -116,7 +116,7 @@ public class SessionStateService extends AbstractPwmService implements PwmServic
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
     }
 

server/src/main/java/password/pwm/ldap/LdapConnectionService.java

@@ -41,11 +41,9 @@ import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.ConditionalTaskExecutor;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StatisticCounterBundle;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
@@ -64,7 +62,6 @@ import java.util.Set;
 import java.util.TreeMap;
 import java.util.WeakHashMap;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Consumer;
@@ -83,7 +80,6 @@ public class LdapConnectionService extends AbstractPwmService implements PwmServ
     private final Map<String, Map<Integer, ChaiProvider>> proxyChaiProviders = new HashMap<>();
 
     private PwmDomain pwmDomain;
-    private ExecutorService executorService;
     private ChaiProviderFactory chaiProviderFactory;
     private AtomicLoopIntIncrementer slotIncrementer;
 
@@ -151,13 +147,6 @@ public class LdapConnectionService extends AbstractPwmService implements PwmServ
         // read the lastLoginTime
         this.lastLdapErrors.putAll( pwmApplication.readLastLdapFailure( getDomainID() ) );
 
-        final long idleWeakTimeoutMS = JavaHelper.silentParseLong(
-                pwmDomain.getConfig().readAppProperty( AppProperty.LDAP_PROXY_IDLE_THREAD_LOCAL_TIMEOUT_MS ),
-                60_000 );
-        final TimeDuration idleWeakTimeout = TimeDuration.of( idleWeakTimeoutMS, TimeDuration.Unit.MILLISECONDS );
-        this.executorService = PwmScheduler.makeBackgroundExecutor( pwmDomain.getPwmApplication(), this.getClass() );
-        pwmDomain.getPwmApplication().getPwmScheduler().scheduleFixedRateJob( new ThreadLocalCleaner(), executorService, idleWeakTimeout, idleWeakTimeout );
-
         final int connectionsPerProfile = maxSlotsPerProfile( pwmDomain );
         LOGGER.trace( () -> "allocating " + connectionsPerProfile + " ldap proxy connections per profile" );
         slotIncrementer = AtomicLoopIntIncrementer.builder().ceiling( connectionsPerProfile ).build();
@@ -171,7 +160,7 @@ public class LdapConnectionService extends AbstractPwmService implements PwmServ
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         setStatus( STATUS.CLOSED );
         logDebugInfo();
@@ -190,7 +179,6 @@ public class LdapConnectionService extends AbstractPwmService implements PwmServ
         lastLdapErrors.clear();
         iterateThreadLocals( container -> container.getProviderMap().clear() );
         threadLocalContainers.clear();
-        executorService.shutdown();
     }
 
     @Override
@@ -443,7 +431,7 @@ public class LdapConnectionService extends AbstractPwmService implements PwmServ
         debugInfo.put( DebugKey.ThreadLocals, String.valueOf( threadLocalConnections.get( ) ) );
         debugInfo.put( DebugKey.CreatedProviders, String.valueOf( stats.get( StatKey.createdProxies ) ) );
         debugInfo.put( DebugKey.DiscardedThreadLocals, String.valueOf( stats.get( StatKey.clearedThreadLocals ) ) );
-        return Collections.unmodifiableMap( CollectionUtil.enumMapToStringMap( debugInfo ) );
+        return CollectionUtil.enumMapToStringMap( debugInfo );
     }
 
     @Data
@@ -454,38 +442,30 @@ public class LdapConnectionService extends AbstractPwmService implements PwmServ
         private volatile String threadName;
     }
 
-    private class ThreadLocalCleaner implements Runnable
+    void cleanupIssuedThreadLocals()
     {
-        @Override
-        public void run()
-        {
-            cleanupIssuedThreadLocals();
-            debugLogger.conditionallyExecuteTask();
-        }
+        final TimeDuration maxIdleTime = TimeDuration.MINUTE;
 
-        private void cleanupIssuedThreadLocals()
+        iterateThreadLocals( container ->
         {
-            final TimeDuration maxIdleTime = TimeDuration.MINUTE;
-
-            iterateThreadLocals( container ->
+            if ( !container.getProviderMap().isEmpty() )
             {
-                if ( !container.getProviderMap().isEmpty() )
+                final Instant timestamp = container.getTimestamp();
+                final TimeDuration age = TimeDuration.fromCurrent( timestamp );
+                if ( age.isLongerThan( maxIdleTime ) )
                 {
-                    final Instant timestamp = container.getTimestamp();
-                    final TimeDuration age = TimeDuration.fromCurrent( timestamp );
-                    if ( age.isLongerThan( maxIdleTime ) )
+                    for ( final ChaiProvider chaiProvider : container.getProviderMap().values() )
                     {
-                        for ( final ChaiProvider chaiProvider : container.getProviderMap().values() )
-                        {
-                            LOGGER.trace( () -> "discarding idled connection id=" + chaiProvider.toString() + " from orphaned threadLocal, age="
-                                    + age.asCompactString() + ", thread=" + container.getThreadName() );
-                            stats.increment( StatKey.clearedThreadLocals );
-                        }
-                        container.getProviderMap().clear();
+                        LOGGER.trace( () -> "discarding idled connection id=" + chaiProvider.toString() + " from orphaned threadLocal, age="
+                                + age.asCompactString() + ", thread=" + container.getThreadName() );
+                        stats.increment( StatKey.clearedThreadLocals );
                     }
+                    container.getProviderMap().clear();
                 }
-            } );
-        }
+            }
+        } );
+
+        debugLogger.conditionallyExecuteTask();
     }
 
     private void iterateThreadLocals( final Consumer<ThreadLocalContainer> consumer )

server/src/main/java/password/pwm/ldap/LdapSystemService.java

@@ -0,0 +1,80 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.ldap;
+
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.PwmDomain;
+import password.pwm.bean.DomainID;
+import password.pwm.error.PwmException;
+import password.pwm.health.HealthRecord;
+import password.pwm.svc.AbstractPwmService;
+import password.pwm.svc.PwmService;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.TimeDuration;
+
+import java.util.List;
+
+public class LdapSystemService extends AbstractPwmService implements PwmService
+{
+    @Override
+    protected STATUS postAbstractInit( final PwmApplication pwmApplication, final DomainID domainID ) throws PwmException
+    {
+        final long idleWeakTimeoutMS = JavaHelper.silentParseLong(
+                pwmApplication.getConfig().readAppProperty( AppProperty.LDAP_PROXY_IDLE_THREAD_LOCAL_TIMEOUT_MS ),
+                60_000 );
+        final TimeDuration idleWeakTimeout = TimeDuration.of( idleWeakTimeoutMS, TimeDuration.Unit.MILLISECONDS );
+        pwmApplication.getPwmScheduler().scheduleFixedRateJob( new ThreadLocalCleaner(), getExecutorService(), idleWeakTimeout, idleWeakTimeout );
+
+        return STATUS.OPEN;
+    }
+
+    @Override
+    protected void shutdownImpl()
+    {
+
+    }
+
+    @Override
+    protected List<HealthRecord> serviceHealthCheck()
+    {
+        return null;
+    }
+
+    @Override
+    public ServiceInfoBean serviceInfo()
+    {
+        return null;
+    }
+
+    private class ThreadLocalCleaner implements Runnable
+    {
+        @Override
+        public void run()
+        {
+            for ( final PwmDomain pwmDomain : getPwmApplication().domains().values() )
+            {
+                pwmDomain.getLdapConnectionService().cleanupIssuedThreadLocals();
+            }
+        }
+    }
+
+}

server/src/main/java/password/pwm/ldap/search/UserSearchEngine.java

@@ -69,13 +69,10 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 
 
 public class UserSearchEngine extends AbstractPwmService implements PwmService
@@ -126,7 +123,7 @@ public class UserSearchEngine extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         if ( executor != null )
         {
@@ -787,7 +784,7 @@ public class UserSearchEngine extends AbstractPwmService implements PwmService
         return idMsg;
     }
 
-    private static ThreadPoolExecutor createExecutor( final PwmDomain pwmDomain )
+    private ThreadPoolExecutor createExecutor( final PwmDomain pwmDomain )
     {
         final DomainConfig domainConfig = pwmDomain.getConfig();
 
@@ -813,19 +810,11 @@ public class UserSearchEngine extends AbstractPwmService implements PwmService
             final int factor = Integer.parseInt( domainConfig.readAppProperty( AppProperty.LDAP_SEARCH_PARALLEL_FACTOR ) );
             final int maxThreads = Integer.parseInt( domainConfig.readAppProperty( AppProperty.LDAP_SEARCH_PARALLEL_THREAD_MAX ) );
             final int threads = Math.min( maxThreads, ( endPoints ) * factor );
-            final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( pwmDomain.getPwmApplication(), UserSearchEngine.class ), true );
             final int minThreads = JavaHelper.rangeCheck( 1, 10, endPoints );
 
             LOGGER.trace( () -> "initialized with threads min=" + minThreads + " max=" + threads );
 
-            return new ThreadPoolExecutor(
-                    minThreads,
-                    threads,
-                    1,
-                    TimeUnit.MINUTES,
-                    new ArrayBlockingQueue<>( threads ),
-                    threadFactory
-            );
+            return PwmScheduler.makeMultiThreadExecutor( threads, getPwmApplication(), getSessionLabel(), UserSearchEngine.class );
         }
         return null;
     }

server/src/main/java/password/pwm/svc/AbstractPwmService.java

@@ -27,25 +27,33 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmException;
 import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
+import password.pwm.util.PwmScheduler;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.LazySupplier;
+import password.pwm.util.java.TimeDuration;
 
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.function.Supplier;
 
 public abstract class AbstractPwmService implements PwmService
 {
     private PwmApplication pwmApplication;
-    private final AtomicReference<PwmService.STATUS> status = new AtomicReference<>( PwmService.STATUS.CLOSED );
+    private volatile PwmService.STATUS status = PwmService.STATUS.CLOSED;
     private ErrorInformation startupError;
     private DomainID domainID;
     private SessionLabel sessionLabel;
 
+    private Supplier<ScheduledExecutorService> executorService;
+
+
     public final PwmService.STATUS status()
     {
-        return status.get();
+        return status;
     }
 
     public final void init( final PwmApplication pwmApplication, final DomainID domainID )
@@ -53,7 +61,11 @@ public abstract class AbstractPwmService implements PwmService
     {
         this.pwmApplication = Objects.requireNonNull( pwmApplication );
         this.domainID = Objects.requireNonNull( domainID );
-        this.sessionLabel = SessionLabel.forPwmService( this, domainID );
+        this.sessionLabel = domainID.isSystem()
+                ? pwmApplication.getSessionLabel()
+                : pwmApplication.domains().get( domainID ).getSessionLabel();
+
+        executorService = new LazySupplier<>( () -> PwmScheduler.makeBackgroundServiceExecutor( pwmApplication, getSessionLabel(), getClass() ) );
 
         if ( pwmApplication.checkConditions( openConditions() ) )
         {
@@ -71,9 +83,22 @@ public abstract class AbstractPwmService implements PwmService
 
     protected void setStatus( final PwmService.STATUS status )
     {
-        this.status.set( status );
+        this.status = Objects.requireNonNull( status );
+    }
+
+    @Override
+    public void shutdown()
+    {
+        this.status = STATUS.CLOSED;
+        if ( executorService != null )
+        {
+            JavaHelper.closeAndWaitExecutor( executorService.get(), TimeDuration.SECONDS_10 );
+        }
+        shutdownImpl();
     }
 
+    protected abstract void shutdownImpl();
+
     public DomainID getDomainID()
     {
         return domainID;
@@ -121,4 +146,9 @@ public abstract class AbstractPwmService implements PwmService
     {
         return EnumSet.of( PwmApplication.Condition.RunningMode, PwmApplication.Condition.LocalDBOpen, PwmApplication.Condition.NotInternalInstance );
     }
+
+    protected ScheduledExecutorService getExecutorService()
+    {
+        return executorService.get();
+    }
 }

server/src/main/java/password/pwm/svc/PwmService.java

@@ -53,7 +53,7 @@ public interface PwmService
 
     void init( PwmApplication pwmApplication, DomainID domainID ) throws PwmException;
 
-    void close( );
+    void shutdown( );
 
     List<HealthRecord> healthCheck( );
 

server/src/main/java/password/pwm/svc/PwmServiceEnum.java

@@ -28,12 +28,12 @@ import password.pwm.svc.intruder.IntruderDomainService;
 import password.pwm.svc.intruder.IntruderSystemService;
 import password.pwm.svc.node.NodeService;
 import password.pwm.svc.pwnotify.PwNotifyService;
+import password.pwm.svc.sms.SmsQueueService;
 import password.pwm.svc.stats.StatisticsService;
 import password.pwm.svc.wordlist.SeedlistService;
 import password.pwm.svc.wordlist.SharedHistoryService;
 import password.pwm.svc.wordlist.WordlistService;
 import password.pwm.util.java.CollectionUtil;
-import password.pwm.svc.sms.SmsQueueService;
 
 import java.util.Arrays;
 import java.util.List;
@@ -56,6 +56,8 @@ public enum PwmServiceEnum
     SmsQueueManager( SmsQueueService.class, PwmSettingScope.SYSTEM ),
     UrlShortenerService( password.pwm.svc.shorturl.UrlShortenerService.class, PwmSettingScope.SYSTEM ),
     CacheService( password.pwm.svc.cache.CacheService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
+    LdapSystemService( password.pwm.ldap.LdapSystemService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
+    TokenSystemService( password.pwm.svc.token.TokenSystemService.class, PwmSettingScope.SYSTEM ),
     HealthMonitor( HealthService.class, PwmSettingScope.SYSTEM ),
     ReportService( password.pwm.svc.report.ReportService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
     SessionTrackService( password.pwm.svc.sessiontrack.SessionTrackService.class, PwmSettingScope.SYSTEM ),

server/src/main/java/password/pwm/svc/PwmServiceManager.java

@@ -205,7 +205,7 @@ public class PwmServiceManager
         try
         {
             final Instant startTime = Instant.now();
-            serviceInstance.close();
+            serviceInstance.shutdown();
             final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
             LOGGER.trace( () -> "successfully closed service " + pwmServiceEnum.serviceName( domainID ) + " (" + timeDuration.asCompactString() + ")" );
         }

server/src/main/java/password/pwm/svc/cache/CacheService.java

@@ -81,7 +81,7 @@ public class CacheService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         setStatus( STATUS.CLOSED );
     }

server/src/main/java/password/pwm/svc/cr/CrService.java

@@ -105,7 +105,7 @@ public class CrService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         for ( final CrOperator operator : operatorMap.values() )
         {

server/src/main/java/password/pwm/svc/db/DatabaseService.java

@@ -35,13 +35,12 @@ import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.EpsStatistic;
 import password.pwm.svc.stats.StatisticsClient;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.PwmTimeUtil;
 import password.pwm.util.java.StringUtil;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.logging.PwmLogger;
 
 import java.sql.Connection;
@@ -58,7 +57,6 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
 
 
 public class DatabaseService extends AbstractPwmService implements PwmService
@@ -80,8 +78,6 @@ public class DatabaseService extends AbstractPwmService implements PwmService
     private AtomicLoopIntIncrementer slotIncrementer;
     private final Map<Integer, DatabaseAccessorImpl> accessors = new ConcurrentHashMap<>();
 
-    private ExecutorService executorService;
-
     private final Map<DatabaseAboutProperty, String> debugInfo = new LinkedHashMap<>();
 
     private volatile boolean initialized = false;
@@ -106,15 +102,11 @@ public class DatabaseService extends AbstractPwmService implements PwmService
     {
         this.dbConfiguration = DBConfiguration.fromConfiguration( getPwmApplication().getConfig() );
 
-
-
-        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-
         final TimeDuration watchdogFrequency = TimeDuration.of(
                 Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.DB_CONNECTIONS_WATCHDOG_FREQUENCY_SECONDS ) ),
                 TimeDuration.Unit.SECONDS );
 
-        pwmApplication.getPwmScheduler().scheduleFixedRateJob( new ConnectionMonitor(), executorService, watchdogFrequency, watchdogFrequency );
+        pwmApplication.getPwmScheduler().scheduleFixedRateJob( new ConnectionMonitor(), getExecutorService(), watchdogFrequency, watchdogFrequency );
 
         return dbInit();
     }
@@ -187,15 +179,10 @@ public class DatabaseService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         setStatus( STATUS.CLOSED );
 
-        if ( executorService != null )
-        {
-            executorService.shutdown();
-        }
-
         clearCurrentAccessors();
 
         try

server/src/main/java/password/pwm/svc/email/EmailService.java

@@ -130,7 +130,7 @@ public class EmailService extends AbstractPwmService implements PwmService
         final LocalDBStoredQueue localDBStoredQueue = LocalDBStoredQueue.createLocalDBStoredQueue(
                 this.getPwmApplication(), this.getPwmApplication().getLocalDB(), LocalDB.DB.EMAIL_QUEUE );
 
-        workQueueProcessor = new WorkQueueProcessor<>( this.getPwmApplication(), localDBStoredQueue, settings, new EmailItemProcessor(), this.getClass() );
+        workQueueProcessor = new WorkQueueProcessor<>( this.getPwmApplication(), this.getSessionLabel(), localDBStoredQueue, settings, new EmailItemProcessor(), this.getClass() );
 
         connectionPool = new EmailConnectionPool( servers, emailServiceSettings );
 
@@ -140,7 +140,7 @@ public class EmailService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         setStatus( STATUS.CLOSED );
         if ( workQueueProcessor != null )

server/src/main/java/password/pwm/svc/event/AuditService.java

@@ -108,7 +108,7 @@ public class AuditService extends AbstractPwmService implements PwmService
         {
             try
             {
-                syslogManager = new SyslogAuditService( pwmApplication );
+                syslogManager = new SyslogAuditService( pwmApplication, getSessionLabel() );
             }
             catch ( final Exception e )
             {
@@ -118,13 +118,13 @@ public class AuditService extends AbstractPwmService implements PwmService
         }
 
         auditVault = new LocalDbAuditVault();
-        auditVault.init( pwmApplication, pwmApplication.getLocalDB(), settings );
+        auditVault.init( pwmApplication, getSessionLabel(), pwmApplication.getLocalDB(), settings );
 
         return STATUS.OPEN;
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         if ( syslogManager != null )
         {

server/src/main/java/password/pwm/svc/event/AuditVault.java

@@ -21,6 +21,7 @@
 package password.pwm.svc.event;
 
 import password.pwm.PwmApplication;
+import password.pwm.bean.SessionLabel;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.util.localdb.LocalDB;
@@ -30,7 +31,7 @@ import java.util.Iterator;
 
 public interface AuditVault
 {
-    void init( PwmApplication pwmApplication, LocalDB localDB, AuditSettings settings )
+    void init( PwmApplication pwmApplication, SessionLabel sessionLabel, LocalDB localDB, AuditSettings settings )
             throws PwmException;
 
     void close( );

server/src/main/java/password/pwm/svc/event/CEFAuditFormatter.java

@@ -27,6 +27,7 @@ import password.pwm.PwmApplication;
 import password.pwm.PwmConstants;
 import password.pwm.bean.SessionLabel;
 import password.pwm.config.AppConfig;
+import password.pwm.config.PwmSetting;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.JavaHelper;
@@ -35,6 +36,7 @@ import password.pwm.util.java.StringUtil;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroRequest;
 
+import java.net.URI;
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -59,6 +61,30 @@ public class CEFAuditFormatter implements AuditFormatter
         CEF_VALUE_ESCAPES = Collections.unmodifiableMap( map );
     }
 
+    private static Optional<String> deriveLocalServerHostname( final AppConfig appConfig )
+    {
+        if ( appConfig != null )
+        {
+            final String siteUrl = appConfig.readSettingAsString( PwmSetting.PWM_SITE_URL );
+            if ( StringUtil.notEmpty( siteUrl ) )
+            {
+                try
+                {
+                    final URI parsedUri = URI.create( siteUrl );
+                    {
+                        final String uriHost = parsedUri.getHost();
+                        return Optional.ofNullable( uriHost );
+                    }
+                }
+                catch ( final IllegalArgumentException e )
+                {
+                    LOGGER.trace( () -> " error parsing siteURL hostname: " + e.getMessage() );
+                }
+            }
+        }
+        return Optional.empty();
+    }
+
     enum CEFAuditField
     {
         cat( AuditField.type ),
@@ -100,7 +126,7 @@ public class CEFAuditFormatter implements AuditFormatter
         final String auditRecordAsJson = JsonFactory.get().serialize( auditRecord );
         final Map<String, Object> auditRecordMap = JsonFactory.get().deserializeMap( auditRecordAsJson, String.class, Object.class );
 
-        final Optional<String> srcHost = PwmApplication.deriveLocalServerHostname( pwmApplication.getConfig() );
+        final Optional<String> srcHost = deriveLocalServerHostname( pwmApplication.getConfig() );
 
         final StringBuilder cefOutput = new StringBuilder(  );
 

server/src/main/java/password/pwm/svc/event/LocalDbAuditVault.java

@@ -21,6 +21,7 @@
 package password.pwm.svc.event;
 
 import password.pwm.PwmApplication;
+import password.pwm.bean.SessionLabel;
 import password.pwm.error.PwmException;
 import password.pwm.svc.PwmService;
 import password.pwm.util.PwmScheduler;
@@ -35,7 +36,7 @@ import password.pwm.util.logging.PwmLogger;
 
 import java.time.Instant;
 import java.util.Iterator;
-import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
 
 public class LocalDbAuditVault implements AuditVault
 {
@@ -45,7 +46,7 @@ public class LocalDbAuditVault implements AuditVault
     private AuditSettings settings;
     private Instant oldestRecord;
 
-    private ExecutorService executorService;
+    private ScheduledExecutorService executorService;
     private volatile PwmService.STATUS status = PwmService.STATUS.CLOSED;
 
 
@@ -58,6 +59,7 @@ public class LocalDbAuditVault implements AuditVault
     @Override
     public void init(
             final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
             final LocalDB localDB,
             final AuditSettings settings
     )
@@ -68,7 +70,7 @@ public class LocalDbAuditVault implements AuditVault
 
         readOldestRecord();
 
-        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        executorService = PwmScheduler.makeBackgroundServiceExecutor( pwmApplication, sessionLabel, this.getClass() );
 
         status = PwmService.STATUS.OPEN;
         final TimeDuration jobFrequency = TimeDuration.of( 10, TimeDuration.Unit.MINUTES );

server/src/main/java/password/pwm/svc/event/SyslogAuditService.java

@@ -40,6 +40,7 @@ import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.PwmConstants;
 import password.pwm.bean.DomainID;
+import password.pwm.bean.SessionLabel;
 import password.pwm.config.AppConfig;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.option.SyslogOutputFormat;
@@ -89,9 +90,10 @@ public class SyslogAuditService
     private final AppConfig appConfig;
     private final PwmApplication pwmApplication;
     private final AuditFormatter auditFormatter;
+    private final SessionLabel sessionLabel;
 
 
-    SyslogAuditService( final PwmApplication pwmApplication )
+    SyslogAuditService( final PwmApplication pwmApplication, final SessionLabel sessionLabel )
             throws LocalDBException
     {
         this.pwmApplication = pwmApplication;
@@ -100,6 +102,7 @@ public class SyslogAuditService
         this.syslogInstances = makeSyslogIFs( appConfig );
         this.auditFormatter = makeAuditFormatter( appConfig );
         this.workQueueProcessor = makeWorkQueueProcessor( pwmApplication, appConfig );
+        this.sessionLabel = sessionLabel;
     }
 
     private WorkQueueProcessor<String> makeWorkQueueProcessor(
@@ -117,7 +120,7 @@ public class SyslogAuditService
         final LocalDBStoredQueue localDBStoredQueue = LocalDBStoredQueue.createLocalDBStoredQueue(
                 pwmApplication, pwmApplication.getLocalDB(), LocalDB.DB.SYSLOG_QUEUE );
 
-        return new WorkQueueProcessor<>( pwmApplication, localDBStoredQueue, settings, new SyslogItemProcessor(), this.getClass() );
+        return new WorkQueueProcessor<>( pwmApplication, sessionLabel, localDBStoredQueue, settings, new SyslogItemProcessor(), this.getClass() );
     }
 
     private static AuditFormatter makeAuditFormatter( final AppConfig appConfig )

server/src/main/java/password/pwm/svc/httpclient/HttpClientService.java

@@ -95,7 +95,7 @@ public class HttpClientService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close()
+    public void shutdownImpl()
     {
         for ( final PwmHttpClient pwmHttpClient : new HashSet<>( issuedClients.keySet() ) )
         {

server/src/main/java/password/pwm/svc/intruder/IntruderDomainService.java

@@ -121,7 +121,7 @@ public class IntruderDomainService extends AbstractPwmService implements PwmServ
             final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, "unexpected error starting intruder manager: " + e.getMessage() );
             LOGGER.error( errorInformation::toDebugStr );
             setStartupError( errorInformation );
-            close();
+            shutdown();
             return STATUS.CLOSED;
         }
 
@@ -193,7 +193,7 @@ public class IntruderDomainService extends AbstractPwmService implements PwmServ
 
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         setStatus( STATUS.CLOSED );
     }

server/src/main/java/password/pwm/svc/intruder/IntruderSystemService.java

@@ -37,7 +37,6 @@ import password.pwm.health.HealthRecord;
 import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
 import password.pwm.util.DataStore;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.ClosableIterator;
 import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.TimeDuration;
@@ -45,7 +44,6 @@ import password.pwm.util.logging.PwmLogger;
 
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.ExecutorService;
 import java.util.stream.Collectors;
 
 public class IntruderSystemService extends AbstractPwmService implements PwmService
@@ -55,8 +53,6 @@ public class IntruderSystemService extends AbstractPwmService implements PwmServ
     private IntruderRecordStore recordStore;
     private DataStorageMethod dataStorageMethod;
 
-    private ExecutorService executorService;
-
     @Override
     public STATUS postAbstractInit( final PwmApplication pwmApplication, final DomainID domainID ) throws PwmException
     {
@@ -68,8 +64,6 @@ public class IntruderSystemService extends AbstractPwmService implements PwmServ
 
             recordStore = new IntruderDataStore( this, dataStore, this::status );
 
-            executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-
             scheduleCleaner();
         }
         catch ( final Exception e )
@@ -77,7 +71,7 @@ public class IntruderSystemService extends AbstractPwmService implements PwmServ
             final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, "unexpected error starting intruder manager: " + e.getMessage() );
             LOGGER.error( errorInformation::toDebugStr );
             setStartupError( errorInformation );
-            close();
+            shutdown();
             return STATUS.CLOSED;
         }
 
@@ -85,7 +79,7 @@ public class IntruderSystemService extends AbstractPwmService implements PwmServ
     }
 
     @Override
-    public void close()
+    public void shutdownImpl()
     {
         setStatus( STATUS.CLOSED );
     }
@@ -176,7 +170,7 @@ public class IntruderSystemService extends AbstractPwmService implements PwmServ
             }
         };
 
-        getPwmApplication().getPwmScheduler().scheduleFixedRateJob( cleanerJob, executorService, TimeDuration.SECONDS_10, cleanerRunFrequency );
+        getPwmApplication().getPwmScheduler().scheduleFixedRateJob( cleanerJob, getExecutorService(), TimeDuration.SECONDS_10, cleanerRunFrequency );
     }
 
     IntruderRecordStore getRecordStore()

server/src/main/java/password/pwm/svc/node/NodeMachine.java

@@ -26,8 +26,6 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.util.PwmScheduler;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
@@ -37,14 +35,13 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
 
 class NodeMachine
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( NodeMachine.class );
 
     private final PwmApplication pwmApplication;
-    private final ExecutorService executorService;
     private final NodeDataServiceProvider clusterDataServiceProvider;
 
     private ErrorInformation lastError;
@@ -56,6 +53,7 @@ class NodeMachine
 
     NodeMachine(
             final PwmApplication pwmApplication,
+            final ScheduledExecutorService executorService,
             final NodeDataServiceProvider clusterDataServiceProvider,
             final NodeServiceSettings nodeServiceSettings
     )
@@ -64,14 +62,11 @@ class NodeMachine
         this.clusterDataServiceProvider = clusterDataServiceProvider;
         this.settings = nodeServiceSettings;
 
-        this.executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, NodeMachine.class );
-
         pwmApplication.getPwmScheduler().scheduleFixedRateJob( new HeartbeatProcess(), executorService, settings.getHeartbeatInterval(), settings.getHeartbeatInterval() );
     }
 
     public void close( )
     {
-        JavaHelper.closeAndWaitExecutor( executorService, TimeDuration.SECOND );
     }
 
 

server/src/main/java/password/pwm/svc/node/NodeService.java

@@ -94,7 +94,7 @@ public class NodeService extends AbstractPwmService implements PwmService
 
                 }
 
-                nodeMachine = new NodeMachine( pwmApplication, clusterDataServiceProvider, nodeServiceSettings );
+                nodeMachine = new NodeMachine( pwmApplication, getExecutorService(), clusterDataServiceProvider, nodeServiceSettings );
             }
         }
         catch ( final PwmUnrecoverableException e )
@@ -114,7 +114,7 @@ public class NodeService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         if ( nodeMachine != null )
         {

server/src/main/java/password/pwm/svc/otp/OtpService.java

@@ -293,7 +293,7 @@ public class OtpService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         for ( final OtpOperator operator : operatorMap.values() )
         {

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyEngine.java

@@ -52,15 +52,11 @@ import java.io.Writer;
 import java.time.Duration;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
-import java.util.LinkedList;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
-import java.util.Queue;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class PwNotifyEngine
@@ -158,15 +154,16 @@ public class PwNotifyEngine
             }
 
             log( "starting job, beginning ldap search" );
-            final Queue<UserIdentity> workQueue = new LinkedList<>( UserPermissionUtility.discoverMatchingUsers(
+            final Iterator<UserIdentity> workQueue = UserPermissionUtility.discoverMatchingUsers(
                     pwmDomain,
                     permissionList, pwNotifyService.getSessionLabel(), settings.getMaxLdapSearchSize(),
-                    settings.getSearchTimeout() ) );
+                    settings.getSearchTimeout()
+            );
 
             log( "ldap search complete, examining users..." );
 
             final ThreadPoolExecutor threadPoolExecutor = createExecutor( pwmDomain );
-            while ( workQueue.peek() != null )
+            while ( workQueue.hasNext() )
             {
                 if ( !checkIfRunningOnMaster() || pwNotifyService.status() == PwmService.STATUS.CLOSED )
                 {
@@ -175,7 +172,7 @@ public class PwNotifyEngine
                     throw PwmUnrecoverableException.newException( PwmError.ERROR_SERVICE_NOT_AVAILABLE, msg );
                 }
 
-                threadPoolExecutor.submit( new ProcessJob( workQueue.poll() ) );
+                threadPoolExecutor.submit( new ProcessJob( workQueue.next() ) );
             }
 
             JavaHelper.closeAndWaitExecutor( threadPoolExecutor, TimeDuration.DAY );
@@ -382,16 +379,10 @@ public class PwNotifyEngine
 
     private ThreadPoolExecutor createExecutor( final PwmDomain pwmDomain )
     {
-        final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( pwmDomain.getPwmApplication(), this.getClass() ), true );
-        final ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(
-                1,
+        return PwmScheduler.makeMultiThreadExecutor(
                 10,
-                1,
-                TimeUnit.MINUTES,
-                new LinkedBlockingDeque<>(),
-                threadFactory
-        );
-        threadPoolExecutor.allowCoreThreadTimeOut( true );
-        return threadPoolExecutor;
+                pwmDomain.getPwmApplication(),
+                pwNotifyService.getSessionLabel(),
+                PwNotifyEngine.class );
     }
 }

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyService.java

@@ -38,7 +38,6 @@ import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsClient;
 import password.pwm.util.PwmScheduler;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.MiscUtil;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
@@ -51,13 +50,11 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.ExecutorService;
 
 public class PwNotifyService extends AbstractPwmService implements PwmService
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwNotifyService.class );
 
-    private ExecutorService executorService;
     private PwmDomain pwmDomain;
     private PwNotifyEngine engine;
     private PwNotifySettings settings;
@@ -141,11 +138,9 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
                     MiscUtil.unhandledSwitchStatement( storageMethod );
             }
 
-            executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-
             engine = new PwNotifyEngine( this, pwmDomain, storageService, null );
 
-            pwmDomain.getPwmApplication().getPwmScheduler().scheduleFixedRateJob( new PwNotifyJob(), executorService, TimeDuration.MINUTE, TimeDuration.MINUTE );
+            pwmDomain.getPwmApplication().getPwmScheduler().scheduleFixedRateJob( new PwNotifyJob(), getExecutorService(), TimeDuration.MINUTE, TimeDuration.MINUTE );
         }
         catch ( final PwmUnrecoverableException e )
         {
@@ -209,10 +204,9 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         setStatus( STATUS.CLOSED );
-        JavaHelper.closeAndWaitExecutor( executorService, TimeDuration.of( 5, TimeDuration.Unit.SECONDS ) );
     }
 
     @Override
@@ -262,7 +256,7 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
         if ( !isRunning() )
         {
             nextExecutionTime = Instant.now();
-            pwmDomain.getPwmApplication().getPwmScheduler().scheduleJob( new PwNotifyJob(), executorService, TimeDuration.ZERO );
+            pwmDomain.getPwmApplication().getPwmScheduler().scheduleJob( new PwNotifyJob(), getExecutorService(), TimeDuration.ZERO );
         }
     }
 

server/src/main/java/password/pwm/svc/secure/AbstractSecureService.java

@@ -118,7 +118,7 @@ public abstract class AbstractSecureService extends AbstractPwmService implement
             throws PwmException;
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
     }
 

server/src/main/java/password/pwm/svc/sessiontrack/SessionTrackService.java

@@ -82,7 +82,7 @@ public class SessionTrackService extends AbstractPwmService implements PwmServic
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         pwmSessions.clear();
     }

server/src/main/java/password/pwm/svc/shorturl/UrlShortenerService.java

@@ -91,7 +91,7 @@ public class UrlShortenerService extends AbstractPwmService implements PwmServic
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         setStatus( PwmService.STATUS.CLOSED );
     }

server/src/main/java/password/pwm/svc/sms/SmsQueueService.java

@@ -57,6 +57,7 @@ import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBStoredQueue;
 import password.pwm.util.localdb.WorkQueueProcessor;
 import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.macro.MacroRequest;
 import password.pwm.util.secure.PwmRandom;
 
 import java.time.Instant;
@@ -75,6 +76,37 @@ public class SmsQueueService extends AbstractPwmService implements PwmService
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( SmsQueueService.class );
 
+    public static void sendSmsUsingQueue(
+            final PwmApplication pwmApplication,
+            final String to,
+            final String message,
+            final SessionLabel sessionLabel,
+            final MacroRequest macroRequest
+    )
+    {
+        final SmsQueueService smsQueue = pwmApplication.getSmsQueue();
+        if ( smsQueue == null )
+        {
+            LOGGER.error( sessionLabel, () -> "SMS queue is unavailable, unable to send SMS to: " + to );
+            return;
+        }
+
+        final SmsItemBean smsItemBean = new SmsItemBean(
+                macroRequest.expandMacros( to ),
+                macroRequest.expandMacros( message ),
+                sessionLabel
+        );
+
+        try
+        {
+            smsQueue.addSmsToQueue( smsItemBean );
+        }
+        catch ( final PwmUnrecoverableException e )
+        {
+            LOGGER.warn( () -> "unable to add sms to queue: " + e.getMessage() );
+        }
+    }
+
     public enum SmsNumberFormat
     {
         RAW,
@@ -132,7 +164,7 @@ public class SmsQueueService extends AbstractPwmService implements PwmService
 
         final LocalDBStoredQueue localDBStoredQueue = LocalDBStoredQueue.createLocalDBStoredQueue( pwmApplication, pwmApplication.getLocalDB(), LocalDB.DB.SMS_QUEUE );
 
-        workQueueProcessor = new WorkQueueProcessor<>( pwmApplication, localDBStoredQueue, settings, new SmsItemProcessor(), this.getClass() );
+        workQueueProcessor = new WorkQueueProcessor<>( pwmApplication, getSessionLabel(), localDBStoredQueue, settings, new SmsItemProcessor(), this.getClass() );
 
         smsSendEngine = new SmsSendEngine( pwmApplication, pwmApplication.getConfig() );
 
@@ -259,7 +291,7 @@ public class SmsQueueService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         if ( workQueueProcessor != null )
         {

server/src/main/java/password/pwm/svc/stats/StatisticsService.java

@@ -29,10 +29,9 @@ import password.pwm.error.PwmException;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
+import password.pwm.util.DailySummaryJob;
 import password.pwm.util.EventRateMeter;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.CollectionUtil;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.MiscUtil;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
@@ -55,7 +54,6 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.TimerTask;
-import java.util.concurrent.ExecutorService;
 import java.util.stream.Collectors;
 
 public class StatisticsService extends AbstractPwmService implements PwmService
@@ -80,8 +78,6 @@ public class StatisticsService extends AbstractPwmService implements PwmService
     private DailyKey currentDailyKey = DailyKey.forToday();
     private DailyKey initialDailyKey = DailyKey.forToday();
 
-    private ExecutorService executorService;
-
     private final StatisticsBundle statsCurrent = new StatisticsBundle();
     private StatisticsBundle statsDaily = new StatisticsBundle();
     private StatisticsBundle statsCummulative = new StatisticsBundle();
@@ -274,9 +270,9 @@ public class StatisticsService extends AbstractPwmService implements PwmService
 
         {
             // setup a timer to roll over at 0 Zulu and one to write current stats regularly
-            executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new FlushTask(), executorService, DB_WRITE_FREQUENCY, DB_WRITE_FREQUENCY );
-            pwmApplication.getPwmScheduler().scheduleDailyZuluZeroStartJob( new NightlyTask(), executorService, TimeDuration.ZERO );
+            pwmApplication.getPwmScheduler().scheduleDailyZuluZeroStartJob( new DailySummaryJob( pwmApplication ), getExecutorService(), TimeDuration.ZERO );
+            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new FlushTask(), getExecutorService(), DB_WRITE_FREQUENCY, DB_WRITE_FREQUENCY );
+            pwmApplication.getPwmScheduler().scheduleDailyZuluZeroStartJob( new NightlyTask(), getExecutorService(), TimeDuration.ZERO );
         }
 
         return STATUS.OPEN;
@@ -317,7 +313,7 @@ public class StatisticsService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         try
         {
@@ -328,8 +324,6 @@ public class StatisticsService extends AbstractPwmService implements PwmService
             LOGGER.error( () -> "unexpected error closing: " + e.getMessage() );
         }
 
-        JavaHelper.closeAndWaitExecutor( executorService, TimeDuration.of( 3, TimeDuration.Unit.SECONDS ) );
-
         setStatus( STATUS.CLOSED );
     }
 

server/src/main/java/password/pwm/svc/telemetry/TelemetryService.java

@@ -47,11 +47,10 @@ import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsBundle;
 import password.pwm.svc.stats.StatisticsService;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.CollectionUtil;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroRequest;
@@ -69,14 +68,12 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
-import java.util.concurrent.ExecutorService;
 import java.util.stream.Collectors;
 
 public class TelemetryService extends AbstractPwmService implements PwmService
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( TelemetryService.class );
 
-    private ExecutorService executorService;
     private Settings settings;
 
     private Instant lastPublishTime;
@@ -126,8 +123,6 @@ public class TelemetryService extends AbstractPwmService implements PwmService
                 .orElseGet( pwmApplication::getInstallTime );
         LOGGER.trace( getSessionLabel(), () -> "last publish time was " + StringUtil.toIsoDate( lastPublishTime ) );
 
-        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, TelemetryService.class );
-
         scheduleNextJob();
 
         return STATUS.OPEN;
@@ -197,7 +192,7 @@ public class TelemetryService extends AbstractPwmService implements PwmService
     private void scheduleNextJob( )
     {
         final TimeDuration durationUntilNextPublish = durationUntilNextPublish();
-        getPwmApplication().getPwmScheduler().scheduleJob( new PublishJob(), executorService, durationUntilNextPublish );
+        getPwmApplication().getPwmScheduler().scheduleJob( new PublishJob(), getExecutorService(), durationUntilNextPublish );
         LOGGER.trace( getSessionLabel(), () -> "next publish time: " + durationUntilNextPublish().asCompactString() );
     }
 
@@ -222,7 +217,7 @@ public class TelemetryService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
 
     }

server/src/main/java/password/pwm/svc/token/TokenService.java

@@ -61,15 +61,15 @@ import password.pwm.svc.event.AuditRecordFactory;
 import password.pwm.svc.event.AuditServiceClient;
 import password.pwm.svc.intruder.IntruderRecordType;
 import password.pwm.svc.intruder.IntruderServiceClient;
+import password.pwm.svc.sms.SmsQueueService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsClient;
 import password.pwm.util.DataStore;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.MiscUtil;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.StatisticCounterBundle;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBDataStore;
 import password.pwm.util.logging.PwmLogger;
@@ -82,8 +82,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.TimerTask;
-import java.util.concurrent.ExecutorService;
 
 /**
  * This PWM service is responsible for reading/writing tokens used for forgotten password,
@@ -94,11 +92,8 @@ import java.util.concurrent.ExecutorService;
  */
 public class TokenService extends AbstractPwmService implements PwmService
 {
-
     private static final PwmLogger LOGGER = PwmLogger.forClass( TokenService.class );
 
-    private ExecutorService executorService;
-
     private PwmDomain pwmDomain;
     private DomainConfig domainConfig;
     private TokenStorageMethod storageMethod;
@@ -219,15 +214,6 @@ public class TokenService extends AbstractPwmService implements PwmService
 
         verifyPwModifyTime = Boolean.parseBoolean( domainConfig.readAppProperty( AppProperty.TOKEN_VERIFY_PW_MODIFY_TIME ) );
 
-        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-
-        {
-            final int cleanerFrequencySeconds = Integer.parseInt( domainConfig.readAppProperty( AppProperty.TOKEN_CLEANER_INTERVAL_SECONDS ) );
-            final TimeDuration cleanerFrequency = TimeDuration.of( cleanerFrequencySeconds, TimeDuration.Unit.SECONDS );
-            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new CleanerTask(), executorService, TimeDuration.MINUTE, cleanerFrequency );
-            LOGGER.trace( getSessionLabel(), () -> "token cleanup will occur every " + cleanerFrequency.asCompactString() );
-        }
-
         LOGGER.debug( getSessionLabel(), () -> "open" );
 
         return STATUS.OPEN;
@@ -342,13 +328,9 @@ public class TokenService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         setStatus( STATUS.CLOSED );
-        if ( executorService != null )
-        {
-            executorService.shutdown();
-        }
     }
 
     @Override
@@ -418,19 +400,15 @@ public class TokenService extends AbstractPwmService implements PwmService
         return random.alphaNumericString( randomChars, codeLength );
     }
 
-    private class CleanerTask extends TimerTask
+    void cleanup()
     {
-        @Override
-        public void run( )
+        try
         {
-            try
-            {
-                tokenMachine.cleanup();
-            }
-            catch ( final Exception e )
-            {
-                LOGGER.warn( getSessionLabel(), () -> "unexpected error while cleaning expired stored tokens: " + e.getMessage() );
-            }
+            tokenMachine.cleanup();
+        }
+        catch ( final Exception e )
+        {
+            LOGGER.warn( getSessionLabel(), () -> "unexpected error while cleaning expired stored tokens: " + e.getMessage() );
         }
     }
 
@@ -806,7 +784,7 @@ public class TokenService extends AbstractPwmService implements PwmService
             final PwmDomain pwmDomain = tokenSendInfo.getPwmDomain();
             pwmDomain.getIntruderService().mark( IntruderRecordType.TOKEN_DEST, smsNumber, tokenSendInfo.getSessionLabel() );
 
-            pwmDomain.getPwmApplication().sendSmsUsingQueue( smsNumber, modifiedMessage, tokenSendInfo.getSessionLabel(), tokenSendInfo.getMacroRequest() );
+            SmsQueueService.sendSmsUsingQueue( pwmDomain.getPwmApplication(), smsNumber, modifiedMessage, tokenSendInfo.getSessionLabel(), tokenSendInfo.getMacroRequest() );
             LOGGER.debug( tokenSendInfo.getSessionLabel(), () -> "token SMS added to send queue for " + smsNumber );
             return true;
         }

server/src/main/java/password/pwm/svc/token/TokenSystemService.java

@@ -0,0 +1,83 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.svc.token;
+
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.PwmDomain;
+import password.pwm.bean.DomainID;
+import password.pwm.error.PwmException;
+import password.pwm.health.HealthRecord;
+import password.pwm.svc.AbstractPwmService;
+import password.pwm.svc.PwmService;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogger;
+
+import java.util.List;
+
+public class TokenSystemService extends AbstractPwmService implements PwmService
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( AbstractPwmService.class );
+
+    @Override
+    protected STATUS postAbstractInit( final PwmApplication pwmApplication, final DomainID domainID ) throws PwmException
+    {
+        {
+            final int cleanerFrequencySeconds = Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.TOKEN_CLEANER_INTERVAL_SECONDS ) );
+            final TimeDuration cleanerFrequency = TimeDuration.of( cleanerFrequencySeconds, TimeDuration.Unit.SECONDS );
+            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new CleanerTask(), getExecutorService(), TimeDuration.MINUTE, cleanerFrequency );
+            LOGGER.trace( getSessionLabel(), () -> "token cleanup will occur every " + cleanerFrequency.asCompactString() );
+        }
+
+        return STATUS.OPEN;
+    }
+
+    @Override
+    protected void shutdownImpl()
+    {
+
+    }
+
+    @Override
+    protected List<HealthRecord> serviceHealthCheck()
+    {
+        return null;
+    }
+
+    @Override
+    public ServiceInfoBean serviceInfo()
+    {
+        return null;
+    }
+
+    private class CleanerTask implements Runnable
+    {
+        @Override
+        public void run( )
+        {
+            for ( final PwmDomain pwmDomain : getPwmApplication().domains().values() )
+            {
+                pwmDomain.getTokenService().cleanup();
+            }
+        }
+    }
+
+}

server/src/main/java/password/pwm/svc/userhistory/UserHistoryService.java

@@ -137,7 +137,7 @@ public class UserHistoryService extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         setStatus( STATUS.CLOSED );
     }

server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java

@@ -36,11 +36,11 @@ import password.pwm.svc.PwmService;
 import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.ConditionalTaskExecutor;
 import password.pwm.util.java.JavaHelper;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.Percent;
 import password.pwm.util.java.PwmCallable;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.logging.PwmLogger;
 
 import java.io.InputStream;
@@ -51,7 +51,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.BooleanSupplier;
@@ -62,7 +62,7 @@ abstract class AbstractWordlist extends AbstractPwmService implements Wordlist,
 
     private WordlistConfiguration wordlistConfiguration;
     private WordlistBucket wordlistBucket;
-    private ExecutorService executorService;
+    private ScheduledExecutorService executorService;
     private volatile Set<WordType> wordTypesCache = null;
 
     private volatile ErrorInformation lastError;
@@ -121,7 +121,7 @@ abstract class AbstractWordlist extends AbstractPwmService implements Wordlist,
         }
 
         inhibitBackgroundImportFlag.set( false );
-        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        executorService = PwmScheduler.makeBackgroundServiceExecutor( pwmApplication, getSessionLabel(), this.getClass() );
 
         if ( !pwmApplication.getPwmEnvironment().isInternalRuntimeInstance() )
         {
@@ -292,7 +292,7 @@ abstract class AbstractWordlist extends AbstractPwmService implements Wordlist,
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         final TimeDuration closeWaitTime = TimeDuration.of( 1, TimeDuration.Unit.MINUTES );
 

server/src/main/java/password/pwm/svc/wordlist/SharedHistoryService.java

@@ -35,7 +35,6 @@ import password.pwm.error.PwmException;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
-import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.localdb.LocalDB;
@@ -51,7 +50,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.TimerTask;
-import java.util.concurrent.ExecutorService;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -67,16 +65,14 @@ public class SharedHistoryService extends AbstractPwmService implements PwmServi
     private static final String DATA_FORMAT_VERSION = "2";
 
     // 1 hour
-    private static final int MIN_CLEANER_FREQUENCY = 1000 * 60 * 60;
+    private static final TimeDuration MIN_CLEANER_FREQUENCY = TimeDuration.HOUR;
 
     // 1 day
-    private static final int MAX_CLEANER_FREQUENCY = 1000 * 60 * 60 * 24;
+    private static final TimeDuration MAX_CLEANER_FREQUENCY = TimeDuration.DAY;
 
     private static final LocalDB.DB META_DB = LocalDB.DB.SHAREDHISTORY_META;
     private static final LocalDB.DB WORDS_DB = LocalDB.DB.SHAREDHISTORY_WORDS;
 
-    private ExecutorService executorService;
-
     private LocalDB localDB;
     private String salt;
     private Instant oldestEntry;
@@ -89,13 +85,9 @@ public class SharedHistoryService extends AbstractPwmService implements PwmServi
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         setStatus( STATUS.CLOSED );
-        if ( executorService != null )
-        {
-            executorService.shutdown();
-        }
         localDB = null;
     }
 
@@ -187,7 +179,7 @@ public class SharedHistoryService extends AbstractPwmService implements PwmServi
         }
     }
 
-    private void init( final PwmApplication pwmApplication, final TimeDuration maxAge )
+    private void initImpl( final PwmApplication pwmApplication, final TimeDuration maxAge )
     {
         final Instant startTime = Instant.now();
 
@@ -244,12 +236,11 @@ public class SharedHistoryService extends AbstractPwmService implements PwmServi
 
         if ( pwmApplication.getApplicationMode() == PwmApplicationMode.RUNNING || pwmApplication.getApplicationMode() == PwmApplicationMode.CONFIGURATION )
         {
-            final long frequencyMs = JavaHelper.rangeCheck( MIN_CLEANER_FREQUENCY, MAX_CLEANER_FREQUENCY, maxAge.asMillis() );
+            final long frequencyMs = JavaHelper.rangeCheck( MIN_CLEANER_FREQUENCY.asMillis(), MAX_CLEANER_FREQUENCY.asMillis(), maxAge.asMillis() );
             final TimeDuration frequency = TimeDuration.of( frequencyMs, TimeDuration.Unit.MILLISECONDS );
 
             LOGGER.debug( () -> "scheduling cleaner task to run once every " + frequency.asCompactString() );
-            executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
-            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new CleanerTask(), executorService, null, frequency );
+            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new CleanerTask(), getExecutorService(), TimeDuration.ZERO, frequency );
         }
     }
 
@@ -475,8 +466,10 @@ public class SharedHistoryService extends AbstractPwmService implements PwmServi
         pwmApplication.getPwmScheduler().immediateExecuteRunnableInNewThread( () ->
         {
             LOGGER.debug( getSessionLabel(), () -> "starting up in background thread" );
-            init( pwmApplication, settings.getMaxAge() );
-        }, "shared history initializer" );
+            initImpl( pwmApplication, settings.getMaxAge() );
+        },
+                getSessionLabel(),
+                "shared history initializer" );
 
         return STATUS.OPEN;
     }

server/src/main/java/password/pwm/svc/wordlist/WordlistImporter.java

@@ -20,18 +20,18 @@
 
 package password.pwm.svc.wordlist;
 
-import org.apache.commons.io.IOUtils;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.TransactionSizeCalculator;
 import password.pwm.util.java.ConditionalTaskExecutor;
+import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.MiscUtil;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.Percent;
 import password.pwm.util.java.StatisticAverageBundle;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.logging.PwmLogger;
 
 import java.time.Instant;
@@ -277,7 +277,7 @@ class WordlistImporter implements Runnable
         }
         finally
         {
-            IOUtils.closeQuietly( zipFileReader );
+            JavaHelper.closeQuietly( zipFileReader );
         }
     }
 

server/src/main/java/password/pwm/svc/wordlist/WordlistSource.java

@@ -20,7 +20,6 @@
 
 package password.pwm.svc.wordlist;
 
-import org.apache.commons.io.IOUtils;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.bean.SessionLabel;
@@ -36,9 +35,9 @@ import password.pwm.svc.httpclient.PwmHttpClientRequest;
 import password.pwm.svc.httpclient.PwmHttpClientResponse;
 import password.pwm.util.java.ConditionalTaskExecutor;
 import password.pwm.util.java.JavaHelper;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.logging.PwmLogger;
 
 import java.io.IOException;
@@ -247,7 +246,7 @@ class WordlistSource
         finally
         {
             closeStreams( pwmLogger, processId, sessionLabel, inputStream );
-            IOUtils.closeQuietly( zipInputStream );
+            JavaHelper.closeQuietly( zipInputStream );
         }
 
         bytes = zipInputStream.getByteCount();
@@ -315,7 +314,7 @@ class WordlistSource
         pwmLogger.trace( sessionLabel, () -> processIdLabel( processId ) + "beginning close of remote wordlist read process" );
         for ( final InputStream inputStream : inputStreams )
         {
-            IOUtils.closeQuietly( inputStream );
+            JavaHelper.closeQuietly( inputStream );
         }
         pwmLogger.trace( sessionLabel, () -> processIdLabel( processId ) + "completed close of remote wordlist read process",
                 () -> TimeDuration.fromCurrent( startClose ) );

server/src/main/java/password/pwm/util/DailySummaryJob.java

@@ -24,8 +24,9 @@ import lombok.Builder;
 import lombok.Value;
 import org.apache.commons.text.WordUtils;
 import password.pwm.AppProperty;
-import password.pwm.PwmConstants;
+import password.pwm.PwmApplication;
 import password.pwm.PwmDomain;
+import password.pwm.PwmConstants;
 import password.pwm.bean.EmailItemBean;
 import password.pwm.config.DomainConfig;
 import password.pwm.config.PwmSetting;
@@ -33,6 +34,7 @@ import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
 import password.pwm.i18n.Display;
 import password.pwm.svc.PwmService;
+import password.pwm.svc.report.ReportSummaryData;
 import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.PwmTimeUtil;
 import password.pwm.util.java.StringUtil;
@@ -52,13 +54,11 @@ public class DailySummaryJob implements Runnable
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( DailySummaryJob.class );
 
-    private final PwmDomain pwmDomain;
-    private final DailySummaryJobSettings settings;
+    private final PwmApplication pwmApplication;
 
-    public DailySummaryJob( final PwmDomain pwmDomain )
+    public DailySummaryJob( final PwmApplication pwmDomain )
     {
-        this.pwmDomain = pwmDomain;
-        this.settings = DailySummaryJobSettings.fromConfig( pwmDomain.getConfig() );
+        this.pwmApplication = pwmDomain;
     }
 
     @Value
@@ -86,22 +86,27 @@ public class DailySummaryJob implements Runnable
     @Override
     public void run()
     {
-        try
-        {
-            alertDailyStats();
-        }
-        catch ( final Exception e )
+        for ( final PwmDomain pwmDomain : pwmApplication.domains().values() )
         {
-            LOGGER.error( () -> "error while generating daily alert statistics: " + e.getMessage() );
-        }
+            final DailySummaryJobSettings dailySummaryJobSettings = DailySummaryJobSettings.fromConfig( pwmDomain.getConfig() );
+            try
+            {
+                alertDailyStats( pwmDomain, dailySummaryJobSettings );
+            }
+            catch ( final Exception e )
+            {
+                LOGGER.error( () -> "error while generating daily alert statistics: " + e.getMessage() );
+            }
+    }
     }
 
-    private void alertDailyStats(
-
+    private static void alertDailyStats(
+        final PwmDomain pwmDomain,
+        final DailySummaryJobSettings settings
     )
             throws PwmUnrecoverableException
     {
-        if ( !checkIfEnabled( pwmDomain ) )
+        if ( !checkIfEnabled( pwmDomain, settings ) )
         {
             LOGGER.trace( () -> "skipping daily summary alert job, setting "
                     + PwmSetting.EVENTS_ALERT_DAILY_SUMMARY.toMenuLocationDebug( null, PwmConstants.DEFAULT_LOCALE )
@@ -125,15 +130,16 @@ public class DailySummaryJob implements Runnable
             final String subject = Display.getLocalizedMessage( locale, Display.Title_Application, pwmDomain.getConfig() ) + " - Daily Summary";
             final StringBuilder textBody = new StringBuilder();
             final StringBuilder htmlBody = new StringBuilder();
-            makeEmailBody( pwmDomain, dailyStatistics, locale, textBody, htmlBody );
+            makeEmailBody( pwmDomain, settings, dailyStatistics, locale, textBody, htmlBody );
             final EmailItemBean emailItem = new EmailItemBean( toAddress, fromAddress, subject, textBody.toString(), htmlBody.toString() );
             LOGGER.debug( () -> "sending daily summary email to " + toAddress );
             pwmDomain.getPwmApplication().getEmailQueue().submitEmail( emailItem, null, MacroRequest.forNonUserSpecific( pwmDomain.getPwmApplication(), null ) );
         }
     }
 
-    private void makeEmailBody(
+    private static void makeEmailBody(
             final PwmDomain pwmDomain,
+            final DailySummaryJobSettings settings,
             final Map<String, String> dailyStatistics,
             final Locale locale,
             final StringBuilder textBody,
@@ -214,11 +220,7 @@ public class DailySummaryJob implements Runnable
 
         if ( settings.isReportingEnableDailyJob() )
         {
-
-
-            /*
             final List<ReportSummaryData.PresentationRow> summaryData = pwmDomain.getPwmApplication().getReportService()
-
                     .getSummaryData().asPresentableCollection( pwmDomain.getPwmApplication().getConfig(), locale );
 
             textBody.append( "-- Directory Report Summary --\n" );
@@ -233,7 +235,6 @@ public class DailySummaryJob implements Runnable
             }
 
             htmlBody.append( "<h2>Directory Report Summary</h2>" );
-
             htmlBody.append( "<table border='1'>" );
             for ( final ReportSummaryData.PresentationRow record : summaryData )
             {
@@ -244,7 +245,6 @@ public class DailySummaryJob implements Runnable
                 htmlBody.append( "</tr>" );
             }
             htmlBody.append( "</table>" );
-            */
         }
 
         textBody.append( '\n' );
@@ -271,7 +271,7 @@ public class DailySummaryJob implements Runnable
 
     }
 
-    private boolean checkIfEnabled( final PwmDomain pwmDomain )
+    private static boolean checkIfEnabled( final PwmDomain pwmDomain, final DailySummaryJobSettings settings )
     {
         if ( pwmDomain == null )
         {

server/src/main/java/password/pwm/util/OnejarHelper.java

@@ -26,7 +26,7 @@ import password.pwm.PwmConstants;
 import password.pwm.PwmEnvironment;
 import password.pwm.config.AppConfig;
 import password.pwm.config.PwmSetting;
-import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.config.stored.ConfigurationFileManager;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.cli.commands.ExportHttpsTomcatConfigCommand;
 import password.pwm.util.java.StringUtil;
@@ -110,7 +110,7 @@ public class OnejarHelper
             throws Exception
     {
         final File configFile = new File( applicationPath + File.separator + PwmConstants.DEFAULT_CONFIG_FILE_FILENAME );
-        final ConfigurationReader configReader = new ConfigurationReader( configFile );
+        final ConfigurationFileManager configReader = new ConfigurationFileManager( configFile );
         final AppConfig config = configReader.getConfiguration();
         final PwmEnvironment pwmEnvironment = PwmEnvironment.builder()
                 .config( config )

server/src/main/java/password/pwm/util/PwmScheduler.java

@@ -20,9 +20,9 @@
 
 package password.pwm.util;
 
-import org.jetbrains.annotations.NotNull;
 import password.pwm.PwmApplication;
 import password.pwm.PwmConstants;
+import password.pwm.bean.SessionLabel;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.java.AtomicLoopIntIncrementer;
@@ -33,6 +33,7 @@ import password.pwm.util.logging.PwmLogger;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Calendar;
+import java.util.Collections;
 import java.util.GregorianCalendar;
 import java.util.List;
 import java.util.Objects;
@@ -44,144 +45,103 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 public class PwmScheduler
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwmScheduler.class );
     private static final AtomicLoopIntIncrementer THREAD_ID_COUNTER = new AtomicLoopIntIncrementer();
 
-    private final ScheduledExecutorService applicationExecutorService;
     private final PwmApplication pwmApplication;
 
     public PwmScheduler( final PwmApplication pwmApplication )
     {
         this.pwmApplication = Objects.requireNonNull( pwmApplication );
-        applicationExecutorService = makeSingleThreadExecutorService( pwmApplication.getInstanceID(), this.getClass() );
     }
 
     public void shutdown()
     {
-        applicationExecutorService.shutdown();
     }
 
-    public Future<?> immediateExecuteRunnableInNewThread(
+    public void immediateExecuteRunnableInNewThread(
             final Runnable runnable,
+            final SessionLabel sessionLabel,
             final String threadName
     )
     {
-        return immediateExecuteCallableInNewThread( Executors.callable( runnable ), threadName );
-    }
-
-    public <V> Future<V> immediateExecuteCallableInNewThread(
-            final Callable<V> callable,
-            final String threadName
-    )
-    {
-        if ( checkIfSchedulerClosed() )
-        {
-            return null;
-        }
-
-        Objects.requireNonNull( callable );
+        checkIfSchedulerClosed();
 
-        final String name = "runtime thread #" + THREAD_ID_COUNTER.next() + " " + threadName;
+        Objects.requireNonNull( runnable );
 
-        final ScheduledExecutorService executor = makeSingleThreadExecutorService( pwmApplication.getInstanceID(), callable.getClass() );
+        final ExecutorService executor = makeMultiThreadExecutor( 1, pwmApplication, sessionLabel, runnable.getClass() );
 
-        final Callable<V> runnableWrapper = () ->
-        {
-            final Instant itemStartTime = Instant.now();
-            LOGGER.trace( () -> "started " + name );
-            try
-            {
-                final V result = callable.call();
-                LOGGER.trace( () -> "completed " + name, () -> TimeDuration.fromCurrent( itemStartTime ) );
-                executor.shutdown();
-                return result;
-            }
-            catch ( final Exception e )
-            {
-                LOGGER.error( () -> "error running scheduled immediate task: " + name + ", error: " + e.getMessage(), e );
-                throw e;
-            }
-        };
-
-        return executor.submit( runnableWrapper );
+        executor.submit( runnable );
     }
 
     public void scheduleDailyZuluZeroStartJob(
             final Runnable runnable,
-            final ExecutorService executorService,
-            final TimeDuration offset
+            final ScheduledExecutorService executorService,
+            final TimeDuration zuluOffset
     )
     {
         final TimeDuration delayTillNextZulu = TimeDuration.fromCurrent( nextZuluZeroTime() );
-        final TimeDuration delayTillNextOffset = delayTillNextZulu.add( offset );
+        final TimeDuration delayTillNextOffset = zuluOffset == null ? TimeDuration.ZERO : delayTillNextZulu.add( zuluOffset );
         scheduleFixedRateJob( runnable, executorService, delayTillNextOffset, TimeDuration.DAY );
     }
 
-    public Future<?> scheduleJob(
+    public ScheduledFuture<?> scheduleJob(
             final Runnable runnable,
-            final ExecutorService executor,
+            final ScheduledExecutorService executor,
             final TimeDuration delay
     )
     {
-        if ( checkIfSchedulerClosed() )
-        {
-            return null;
-        }
+        checkIfSchedulerClosed();
 
         Objects.requireNonNull( runnable );
         Objects.requireNonNull( executor );
         Objects.requireNonNull( delay );
 
-        if ( applicationExecutorService.isShutdown() )
-        {
-            throw new IllegalStateException( "can not schedule job with shutdown scheduler" );
-        }
-
-        final FutureRunner wrappedRunner = new FutureRunner( runnable, executor );
-        applicationExecutorService.schedule( wrappedRunner, delay.asMillis(), TimeUnit.MILLISECONDS );
-        return wrappedRunner.getFuture();
+        return executor.schedule( runnable, delay.asMillis(), TimeUnit.MILLISECONDS );
     }
 
-    public void executeImmediateThreadPerJobAndAwaitCompletion(
-            final List<Callable<?>> runnableList,
-            final String threadNames
+    public <T> List<T> executeImmediateThreadPerJobAndAwaitCompletion(
+            final int maxThreadCount,
+            final List<Callable<T>> callables,
+            final SessionLabel sessionLabel,
+            final Class<?> theClass
     )
             throws PwmUnrecoverableException
     {
-        if ( checkIfSchedulerClosed() )
-        {
-            return;
-        }
+        checkIfSchedulerClosed();
 
+        final ExecutorService executor = makeMultiThreadExecutor( maxThreadCount, pwmApplication, sessionLabel, theClass );
 
-        final List<Future<?>> futures = new ArrayList<>( runnableList.size() );
-        for ( final Callable<?> callable : runnableList )
-        {
-            futures.add( this.immediateExecuteCallableInNewThread( () ->
-            {
-                callable.call();
-                return null;
-            }, threadNames ) );
-        }
+        final List<Future<T>> futures = callables.stream()
+                .map( executor::submit )
+                .collect( Collectors.toUnmodifiableList() );
 
-        for ( final Future<?> f : futures )
+
+        final List<T> results = new ArrayList<>();
+        for ( final Future<T> f : futures )
         {
-            awaitFutureCompletion( f );
+            results.add( awaitFutureCompletion( f ) );
         }
+
+        executor.shutdown();
+        return Collections.unmodifiableList( results );
     }
 
-    private static void awaitFutureCompletion( final Future<?> future )
+    private static <T> T awaitFutureCompletion( final Future<T> future )
             throws PwmUnrecoverableException
     {
         try
         {
-            future.get();
+            return future.get();
         }
         catch ( final InterruptedException e )
         {
@@ -204,56 +164,20 @@ public class PwmScheduler
 
     public void scheduleFixedRateJob(
             final Runnable runnable,
-            final ExecutorService executor,
+            final ScheduledExecutorService executor,
             final TimeDuration initialDelay,
             final TimeDuration frequency
     )
     {
-        if ( checkIfSchedulerClosed() )
-        {
-            return;
-        }
-
-
-        if ( initialDelay != null )
-        {
-            applicationExecutorService.schedule( new FutureRunner( runnable, executor ), initialDelay.asMillis(), TimeUnit.MILLISECONDS );
-        }
-
-        final Runnable jobWithNextScheduler = () ->
-        {
-            new FutureRunner( runnable, executor ).run();
-            scheduleFixedRateJob( runnable, executor, null, frequency );
-        };
+        checkIfSchedulerClosed();
 
-        applicationExecutorService.schedule(  jobWithNextScheduler, frequency.asMillis(), TimeUnit.MILLISECONDS );
+        executor.scheduleAtFixedRate( runnable, initialDelay.asMillis(), frequency.asMillis(), TimeUnit.MILLISECONDS );
     }
 
-    public static ExecutorService makeBackgroundExecutor(
+    public static String makeThreadName(
+            final SessionLabel sessionLabel,
             final PwmApplication pwmApplication,
-            final Class clazz
-    )
-    {
-        if ( pwmApplication.getPwmScheduler().checkIfSchedulerClosed() )
-        {
-            return null;
-        }
-
-        final ThreadPoolExecutor executor = new ThreadPoolExecutor(
-                1,
-                1,
-                10, TimeUnit.SECONDS,
-                new LinkedBlockingQueue<>(),
-                makePwmThreadFactory(
-                        makeThreadName( pwmApplication, clazz ) + "-",
-                        true
-                ) );
-        executor.allowCoreThreadTimeOut( true );
-        return executor;
-    }
-
-
-    public static String makeThreadName( final PwmApplication pwmApplication, final Class theClass )
+            final Class<?> theClass )
     {
         String instanceName = "-";
         if ( pwmApplication != null )
@@ -261,18 +185,37 @@ public class PwmScheduler
             instanceName = pwmApplication.getInstanceID();
         }
 
-        return makeThreadName( instanceName, theClass );
+        return makeThreadName( sessionLabel, instanceName, theClass );
     }
 
-    public static String makeThreadName( final String instanceID, final Class theClass )
+    public static String makeThreadName(
+            final SessionLabel sessionLabel,
+            final String instanceID,
+            final Class<?> theClass )
     {
-        String instanceName = "-";
+        final StringBuilder output = new StringBuilder();
+
+        output.append( PwmConstants.PWM_APP_NAME );
+
         if ( StringUtil.notEmpty( instanceID ) )
         {
-            instanceName = instanceID;
+            output.append( "-" );
+            output.append( instanceID );
         }
 
-        return PwmConstants.PWM_APP_NAME + "-" + instanceName + "-" + theClass.getSimpleName();
+        if ( theClass != null )
+        {
+            output.append( "-" );
+            output.append( theClass.getSimpleName() );
+        }
+
+        if ( sessionLabel != null && !StringUtil.isEmpty( sessionLabel.getDomain() ) )
+        {
+            output.append( "-" );
+            output.append( sessionLabel.getDomain() );
+        }
+
+        return output.toString();
     }
 
     public static ThreadFactory makePwmThreadFactory( final String namePrefix, final boolean daemon )
@@ -296,129 +239,55 @@ public class PwmScheduler
         };
     }
 
-    public static ScheduledExecutorService makeSingleThreadExecutorService(
+    public static ThreadPoolExecutor makeMultiThreadExecutor(
+            final int maxThreadCount,
             final PwmApplication pwmApplication,
-            final Class theClass
+            final SessionLabel sessionLabel,
+            final Class<?> theClass
     )
     {
-        return makeSingleThreadExecutorService( pwmApplication.getInstanceID(), theClass );
+        return makeMultiThreadExecutor( maxThreadCount, pwmApplication.getInstanceID(), sessionLabel, theClass );
     }
 
-    public static ExecutorService makeMultiThreadExecutorService(
-            final PwmApplication pwmApplication,
-            final Class theClass,
-            final int maxThreads
+    public static ThreadPoolExecutor makeMultiThreadExecutor(
+            final int maxThreadCount,
+            final String instanceID,
+            final SessionLabel sessionLabel,
+            final Class<?> theClass
     )
     {
-        final String threadName = makeThreadName( pwmApplication.getInstanceID(), theClass ) + "-";
-        final ThreadFactory threadFactory = makePwmThreadFactory( threadName, true );
-        final ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(
-                maxThreads,
-                maxThreads,
-                10,
-                TimeUnit.SECONDS,
+        final ThreadPoolExecutor executor = new ThreadPoolExecutor(
+                1,
+                maxThreadCount,
+                1, TimeUnit.SECONDS,
                 new LinkedBlockingQueue<>(),
-                threadFactory );
-        threadPoolExecutor.allowCoreThreadTimeOut( true );
-        return threadPoolExecutor;
+                makePwmThreadFactory(
+                        makeThreadName( sessionLabel, instanceID, theClass ) + "-",
+                        true
+                ) );
+        executor.allowCoreThreadTimeOut( true );
+        return executor;
     }
 
-    public static ScheduledExecutorService makeSingleThreadExecutorService(
-            final String instanceID,
-            final Class theClass
+    public static ScheduledExecutorService makeBackgroundServiceExecutor(
+            final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
+            final Class<?> clazz
     )
     {
-        return Executors.newSingleThreadScheduledExecutor(
+        final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(
+                1,
                 makePwmThreadFactory(
-                        makeThreadName( instanceID, theClass ) + "-",
+                        makeThreadName( sessionLabel, pwmApplication, clazz ) + "-",
                         true
                 ) );
+        executor.setKeepAliveTime( 1, TimeUnit.MINUTES );
+        executor.allowCoreThreadTimeOut( true );
+        return executor;
     }
 
-    private static class FutureRunner implements Runnable
-    {
-        private final Runnable runnable;
-        private final ExecutorService executor;
-        private volatile Future innerFuture;
-        private volatile boolean hasFailed;
-
-        enum Flag
-        {
-            ShutdownExecutorAfterExecution,
-        }
-
-        FutureRunner( final Runnable runnable, final ExecutorService executor )
-        {
-            this.runnable = runnable;
-            this.executor = executor;
-        }
-
-        Future getFuture()
-        {
-            return new Future()
-            {
-                @Override
-                public boolean cancel( final boolean mayInterruptIfRunning )
-                {
-                    return false;
-                }
-
-                @Override
-                public boolean isCancelled()
-                {
-                    return hasFailed;
-                }
-
-                @Override
-                public boolean isDone()
-                {
-                    return hasFailed || ( innerFuture != null && innerFuture.isDone() );
-                }
-
-                @Override
-                public Object get()
-                {
-                    return null;
-                }
-
-                @Override
-                public Object get( final long timeout, @NotNull final TimeUnit unit )
-                {
-                    return null;
-                }
-            };
-        }
-
-        @Override
-        public void run()
-        {
-            try
-            {
-                if ( !executor.isShutdown() )
-                {
-                    innerFuture = executor.submit( runnable );
-                }
-                else
-                {
-                    hasFailed = true;
-                }
-            }
-            catch ( final Throwable t )
-            {
-                LOGGER.error( () -> "unexpected error running scheduled job: " + t.getMessage(), t );
-                hasFailed = true;
-            }
-        }
-    }
-
-    private boolean checkIfSchedulerClosed()
+    private void checkIfSchedulerClosed()
     {
-        return false;
-        /*
-        return pwmApplication.getApplicationMode() == PwmApplicationMode.READ_ONLY
-                || pwmApplication.getPwmEnvironment().isInternalRuntimeInstance()
-                || applicationExecutorService.isShutdown();
-        */
     }
 
     public static Instant nextZuluZeroTime( )

server/src/main/java/password/pwm/util/ServletUtility.java

@@ -20,17 +20,14 @@
 
 package password.pwm.util;
 
-import org.apache.commons.io.IOUtils;
 import password.pwm.PwmConstants;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.java.JavaHelper;
 
 import javax.servlet.http.HttpServletRequest;
 import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.io.StringWriter;
 
 public final class ServletUtility
 {
@@ -41,32 +38,14 @@ public final class ServletUtility
     public static String readRequestBodyAsString( final HttpServletRequest req, final int maxChars )
             throws IOException, PwmUnrecoverableException
     {
-        final StringWriter stringWriter = new StringWriter();
-        final Reader readerStream = new InputStreamReader(
-                req.getInputStream(),
-                PwmConstants.DEFAULT_CHARSET
-        );
+        final String value = JavaHelper.copyToString( req.getInputStream(), PwmConstants.DEFAULT_CHARSET, maxChars + 1 )
+                .orElse( "" );
 
-        try
+        if ( value.length() > maxChars )
         {
-            IOUtils.copy( readerStream, stringWriter );
-        }
-        catch ( final Exception e )
-        {
-            final String errorMsg = "error reading request body stream: " + e.getMessage();
-            throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_INTERNAL, errorMsg ) );
-        }
-        finally
-        {
-            IOUtils.closeQuietly( readerStream );
-        }
-
-        final String stringValue = stringWriter.toString();
-        if ( stringValue.length() > maxChars )
-        {
-            final String msg = "input request body is to big, size=" + stringValue.length() + ", max=" + maxChars;
+            final String msg = "input request body is to big, size=" + value.length() + ", max=" + maxChars;
             throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_INTERNAL, msg ) );
         }
-        return stringValue;
+        return value;
     }
 }

server/src/main/java/password/pwm/util/cli/CliEnvironment.java

@@ -24,7 +24,7 @@ import lombok.Builder;
 import lombok.Value;
 import password.pwm.PwmApplication;
 import password.pwm.config.AppConfig;
-import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.config.stored.ConfigurationFileManager;
 import password.pwm.util.localdb.LocalDB;
 
 import java.io.File;
@@ -35,7 +35,7 @@ import java.util.Map;
 @Builder( toBuilder = true )
 public class CliEnvironment
 {
-    final ConfigurationReader configurationReader;
+    final ConfigurationFileManager configurationFileManager;
     final File configurationFile;
     final AppConfig config;
     final File applicationPath;

server/src/main/java/password/pwm/util/cli/MainClass.java

@@ -30,7 +30,7 @@ import password.pwm.PwmApplicationMode;
 import password.pwm.PwmConstants;
 import password.pwm.PwmEnvironment;
 import password.pwm.config.AppConfig;
-import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.config.stored.ConfigurationFileManager;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
@@ -190,7 +190,7 @@ public class MainClass
 
         final File configurationFile = locateConfigurationFile( applicationPath );
 
-        final ConfigurationReader configReader = loadConfiguration( configurationFile );
+        final ConfigurationFileManager configReader = loadConfiguration( configurationFile );
         final AppConfig config = configReader.getConfiguration();
 
         final PwmApplication pwmApplication;
@@ -217,7 +217,7 @@ public class MainClass
 
         final Writer outputStream = new OutputStreamWriter( System.out, PwmConstants.DEFAULT_CHARSET );
         return CliEnvironment.builder()
-                .configurationReader( configReader )
+                .configurationFileManager( configReader )
                 .configurationFile( configurationFile )
                 .config( config )
                 .applicationPath( applicationPath )
@@ -441,9 +441,9 @@ public class MainClass
         return LocalDBFactory.getInstance( databaseDirectory, readonly, null, config );
     }
 
-    private static ConfigurationReader loadConfiguration( final File configurationFile ) throws Exception
+    private static ConfigurationFileManager loadConfiguration( final File configurationFile ) throws Exception
     {
-        final ConfigurationReader reader = new ConfigurationReader( configurationFile );
+        final ConfigurationFileManager reader = new ConfigurationFileManager( configurationFile );
 
         if ( reader.getConfigMode() == PwmApplicationMode.ERROR )
         {

server/src/main/java/password/pwm/util/cli/commands/ConfigLockCommand.java

@@ -22,7 +22,7 @@ package password.pwm.util.cli.commands;
 
 import password.pwm.bean.SessionLabel;
 import password.pwm.config.stored.ConfigurationProperty;
-import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.config.stored.ConfigurationFileManager;
 import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationModifier;
 import password.pwm.util.cli.CliParameters;
@@ -35,8 +35,8 @@ public class ConfigLockCommand extends AbstractCliCommand
     public void doCommand( )
             throws Exception
     {
-        final ConfigurationReader configurationReader = cliEnvironment.getConfigurationReader();
-        final StoredConfiguration storedConfiguration = configurationReader.getStoredConfiguration();
+        final ConfigurationFileManager configurationFileManager = cliEnvironment.getConfigurationFileManager();
+        final StoredConfiguration storedConfiguration = configurationFileManager.getStoredConfiguration();
         final Optional<String> configIsEditable = storedConfiguration.readConfigProperty( ConfigurationProperty.CONFIG_IS_EDITABLE );
         if ( configIsEditable.isPresent() && !Boolean.parseBoolean( configIsEditable.get() ) )
         {
@@ -46,7 +46,7 @@ public class ConfigLockCommand extends AbstractCliCommand
 
         final StoredConfigurationModifier modifier = StoredConfigurationModifier.newModifier( storedConfiguration );
         modifier.writeConfigProperty( ConfigurationProperty.CONFIG_IS_EDITABLE, Boolean.toString( false ) );
-        configurationReader.saveConfiguration( modifier.newStoredConfiguration(), cliEnvironment.getPwmApplication(), SessionLabel.CLI_SESSION_LABEL );
+        configurationFileManager.saveConfiguration( modifier.newStoredConfiguration(), cliEnvironment.getPwmApplication(), SessionLabel.CLI_SESSION_LABEL );
         out( "success: configuration has been locked" );
     }
 

server/src/main/java/password/pwm/util/cli/commands/ConfigResetHttpsCommand.java

@@ -24,7 +24,7 @@ import password.pwm.bean.DomainID;
 import password.pwm.bean.SessionLabel;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.PwmSettingCategory;
-import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.config.stored.ConfigurationFileManager;
 import password.pwm.config.stored.StoredConfigKey;
 import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationModifier;
@@ -51,8 +51,8 @@ public class ConfigResetHttpsCommand
             return;
         }
 
-        final ConfigurationReader configurationReader = new ConfigurationReader( cliEnvironment.getConfigurationFile() );
-        final StoredConfiguration storedConfiguration = configurationReader.getStoredConfiguration();
+        final ConfigurationFileManager configurationFileManager = new ConfigurationFileManager( cliEnvironment.getConfigurationFile() );
+        final StoredConfiguration storedConfiguration = configurationFileManager.getStoredConfiguration();
 
         final StoredConfigurationModifier modifier = StoredConfigurationModifier.newModifier( storedConfiguration );
         for ( final PwmSetting setting : PwmSettingCategory.HTTPS_SERVER.getSettings() )
@@ -60,7 +60,7 @@ public class ConfigResetHttpsCommand
             final StoredConfigKey key = StoredConfigKey.forSetting( setting, null, DomainID.systemId() );
             modifier.resetSetting( key, null );
         }
-        configurationReader.saveConfiguration( modifier.newStoredConfiguration(), cliEnvironment.getPwmApplication(), SessionLabel.CLI_SESSION_LABEL );
+        configurationFileManager.saveConfiguration( modifier.newStoredConfiguration(), cliEnvironment.getPwmApplication(), SessionLabel.CLI_SESSION_LABEL );
         out( "success" );
     }
 

server/src/main/java/password/pwm/util/cli/commands/ConfigSetPasswordCommand.java

@@ -21,7 +21,7 @@
 package password.pwm.util.cli.commands;
 
 import password.pwm.bean.SessionLabel;
-import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.config.stored.ConfigurationFileManager;
 import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationModifier;
 import password.pwm.config.stored.StoredConfigurationUtil;
@@ -36,12 +36,12 @@ public class ConfigSetPasswordCommand extends AbstractCliCommand
     public void doCommand( )
             throws Exception
     {
-        final ConfigurationReader configurationReader = cliEnvironment.getConfigurationReader();
-        final StoredConfiguration storedConfiguration = configurationReader.getStoredConfiguration();
+        final ConfigurationFileManager configurationFileManager = cliEnvironment.getConfigurationFileManager();
+        final StoredConfiguration storedConfiguration = configurationFileManager.getStoredConfiguration();
         final StoredConfigurationModifier modifier = StoredConfigurationModifier.newModifier( storedConfiguration );
         final String password = getOptionalPassword();
         StoredConfigurationUtil.setPassword( modifier, password );
-        configurationReader.saveConfiguration( modifier.newStoredConfiguration(), cliEnvironment.getPwmApplication(), SessionLabel.CLI_SESSION_LABEL );
+        configurationFileManager.saveConfiguration( modifier.newStoredConfiguration(), cliEnvironment.getPwmApplication(), SessionLabel.CLI_SESSION_LABEL );
         out( "success: new password has been set" );
     }
 

server/src/main/java/password/pwm/util/cli/commands/ConfigUnlockCommand.java

@@ -22,7 +22,7 @@ package password.pwm.util.cli.commands;
 
 import password.pwm.bean.SessionLabel;
 import password.pwm.config.stored.ConfigurationProperty;
-import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.config.stored.ConfigurationFileManager;
 import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationModifier;
 import password.pwm.util.cli.CliParameters;
@@ -35,8 +35,8 @@ public class ConfigUnlockCommand extends AbstractCliCommand
     public void doCommand( )
             throws Exception
     {
-        final ConfigurationReader configurationReader = cliEnvironment.getConfigurationReader();
-        final StoredConfiguration storedConfiguration = configurationReader.getStoredConfiguration();
+        final ConfigurationFileManager configurationFileManager = cliEnvironment.getConfigurationFileManager();
+        final StoredConfiguration storedConfiguration = configurationFileManager.getStoredConfiguration();
 
         final Optional<String> configIsEditable = storedConfiguration.readConfigProperty( ConfigurationProperty.CONFIG_IS_EDITABLE );
         if ( configIsEditable.isPresent() && Boolean.parseBoolean( configIsEditable.get() ) )
@@ -47,7 +47,7 @@ public class ConfigUnlockCommand extends AbstractCliCommand
 
         final StoredConfigurationModifier modifier = StoredConfigurationModifier.newModifier( storedConfiguration );
         modifier.writeConfigProperty( ConfigurationProperty.CONFIG_IS_EDITABLE, Boolean.toString( true ) );
-        configurationReader.saveConfiguration( modifier.newStoredConfiguration(), cliEnvironment.getPwmApplication(), SessionLabel.CLI_SESSION_LABEL );
+        configurationFileManager.saveConfiguration( modifier.newStoredConfiguration(), cliEnvironment.getPwmApplication(), SessionLabel.CLI_SESSION_LABEL );
         out( "success: configuration has been unlocked" );
     }
 

server/src/main/java/password/pwm/util/cli/commands/ExportHttpsTomcatConfigCommand.java

@@ -20,12 +20,12 @@
 
 package password.pwm.util.cli.commands;
 
-import org.apache.commons.io.IOUtils;
 import password.pwm.PwmConstants;
 import password.pwm.config.AppConfig;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.option.TLSVersion;
 import password.pwm.util.cli.CliParameters;
+import password.pwm.util.java.JavaHelper;
 
 import java.io.File;
 import java.io.IOException;
@@ -118,7 +118,7 @@ public class ExportHttpsTomcatConfigCommand extends AbstractCliCommand
         )
                 throws IOException
         {
-            String fileContents = IOUtils.toString( sourceFile, PwmConstants.DEFAULT_CHARSET.toString() );
+            String fileContents = JavaHelper.copyToString( sourceFile, PwmConstants.DEFAULT_CHARSET, Integer.MAX_VALUE ).orElse( "" );
             fileContents = fileContents.replace( TOKEN_TLS_PROTOCOLS, getTlsProtocolsValue( appConfig ) );
             final String tlsCiphers = appConfig.readSettingAsString( PwmSetting.HTTPS_CIPHERS );
             fileContents = fileContents.replace( TOKEN_TLS_CIPHERS, tlsCiphers );

server/src/main/java/password/pwm/util/cli/commands/ImportHttpsKeyStoreCommand.java

@@ -21,7 +21,7 @@
 package password.pwm.util.cli.commands;
 
 import password.pwm.bean.SessionLabel;
-import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.config.stored.ConfigurationFileManager;
 import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationModifier;
 import password.pwm.util.PasswordData;
@@ -64,8 +64,8 @@ public class ImportHttpsKeyStoreCommand extends AbstractCliCommand
         final String keyStorePassword = getOptionalPassword();
         final String inputAliasName = ( String ) cliEnvironment.getOptions().get( ALIAS_OPTIONNAME );
 
-        final ConfigurationReader configurationReader = new ConfigurationReader( cliEnvironment.getConfigurationFile() );
-        final StoredConfiguration storedConfiguration = configurationReader.getStoredConfiguration();
+        final ConfigurationFileManager configurationFileManager = new ConfigurationFileManager( cliEnvironment.getConfigurationFile() );
+        final StoredConfiguration storedConfiguration = configurationFileManager.getStoredConfiguration();
         final StoredConfigurationModifier modifier = StoredConfigurationModifier.newModifier( storedConfiguration );
 
         try ( FileInputStream fileInputStream = new FileInputStream( inputFile ) )
@@ -84,7 +84,7 @@ public class ImportHttpsKeyStoreCommand extends AbstractCliCommand
             return;
         }
 
-        configurationReader.saveConfiguration( modifier.newStoredConfiguration(), cliEnvironment.getPwmApplication(), SessionLabel.CLI_SESSION_LABEL );
+        configurationFileManager.saveConfiguration( modifier.newStoredConfiguration(), cliEnvironment.getPwmApplication(), SessionLabel.CLI_SESSION_LABEL );
         out( "success: keystore has been imported" );
     }
 

server/src/main/java/password/pwm/util/localdb/LocalDBService.java

@@ -72,7 +72,7 @@ public class LocalDBService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdown( )
     {
         //no-op
     }

server/src/main/java/password/pwm/util/localdb/WorkQueueProcessor.java

@@ -24,6 +24,7 @@ import com.google.gson.annotations.SerializedName;
 import lombok.Builder;
 import lombok.Value;
 import password.pwm.PwmApplication;
+import password.pwm.bean.SessionLabel;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmOperationalException;
@@ -102,10 +103,11 @@ public final class WorkQueueProcessor<W extends Serializable>
 
     public WorkQueueProcessor(
             final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
             final Deque<String> queue,
             final Settings settings,
             final ItemProcessor<W> itemProcessor,
-            final Class sourceClass
+            final Class<?> sourceClass
     )
     {
         this.settings = settings;
@@ -121,12 +123,12 @@ public final class WorkQueueProcessor<W extends Serializable>
 
         this.workerThread = new WorkerThread();
         workerThread.setDaemon( true );
-        workerThread.setName( PwmScheduler.makeThreadName( pwmApplication, sourceClass ) + "-worker-" );
+        workerThread.setName( PwmScheduler.makeThreadName( sessionLabel, pwmApplication, sourceClass ) + "-worker-" );
         workerThread.start();
 
         if ( settings.getPreThreads() > 0 )
         {
-            final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( pwmApplication, sourceClass ), true );
+            final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( sessionLabel, pwmApplication, sourceClass ), true );
             executorService = new ThreadPoolExecutor(
                     1,
                     settings.getPreThreads(),

server/src/main/java/password/pwm/util/logging/LocalDBLogger.java

@@ -150,13 +150,13 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
 
         cleanerService = Executors.newSingleThreadScheduledExecutor(
                 PwmScheduler.makePwmThreadFactory(
-                        PwmScheduler.makeThreadName( pwmApplication, this.getClass() ) + "-cleaner-",
+                        PwmScheduler.makeThreadName( getSessionLabel(), pwmApplication, this.getClass() ) + "-cleaner-",
                         true
                 ) );
 
         writerService = Executors.newSingleThreadScheduledExecutor(
                 PwmScheduler.makePwmThreadFactory(
-                        PwmScheduler.makeThreadName( pwmApplication, this.getClass() ) + "-writer-",
+                        PwmScheduler.makeThreadName( getSessionLabel(), pwmApplication, this.getClass() ) + "-writer-",
                         true
                 ) );
 
@@ -239,7 +239,7 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void shutdownImpl( )
     {
         final Instant startTime = Instant.now();
         int flushedEvents = 0;

server/src/main/java/password/pwm/util/logging/PwmLogEvent.java

@@ -171,38 +171,14 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
 
     private String getDebugLabel( )
     {
-        final StringBuilder sb = new StringBuilder();
-        final String sessionID = getSessionID();
-        final String username = getUsername();
-
-        if ( StringUtil.notEmpty( sessionID ) )
-        {
-            sb.append( sessionID );
-        }
-        if ( StringUtil.notEmpty( domain ) )
-        {
-            if ( sb.length() > 0 )
-            {
-                sb.append( ',' );
-            }
-            sb.append( domain );
-        }
-        if ( StringUtil.notEmpty( username ) )
-        {
-            if ( sb.length() > 0 )
-            {
-                sb.append( ',' );
-            }
-            sb.append( username );
-        }
-
-        if ( sb.length() > 0 )
-        {
-            sb.insert( 0, "{" );
-            sb.append( "} " );
-        }
-
-        return sb.toString();
+        return SessionLabel.builder()
+                .sessionID( getSessionID() )
+                .requestID( getRequestID() )
+                .username( getUsername() )
+                .sourceAddress( getSourceAddress() )
+                .domain( getDomain() )
+                .build()
+                .toDebugLabel();
     }
 
     public String toLogString( )

server/src/main/java/password/pwm/util/password/PasswordUtility.java

@@ -79,6 +79,7 @@ import password.pwm.svc.event.AuditEvent;
 import password.pwm.svc.event.AuditRecordFactory;
 import password.pwm.svc.event.AuditServiceClient;
 import password.pwm.svc.event.HelpdeskAuditRecord;
+import password.pwm.svc.sms.SmsQueueService;
 import password.pwm.svc.stats.AvgStatistic;
 import password.pwm.svc.stats.EpsStatistic;
 import password.pwm.svc.stats.Statistic;
@@ -178,7 +179,7 @@ public class PasswordUtility
 
         message = message.replace( "%TOKEN%", newPassword.getStringValue() );
 
-        pwmDomain.getPwmApplication().sendSmsUsingQueue( toNumber, message, null, macroRequest );
+        SmsQueueService.sendSmsUsingQueue( pwmDomain.getPwmApplication(), toNumber, message, null, macroRequest );
         LOGGER.debug( () -> String.format( "password SMS added to send queue for %s", toNumber ) );
         return null;
     }

server/src/main/resources/password/pwm/i18n/PwmSetting.properties

@@ -352,7 +352,7 @@ Setting_Description_display.updateAttributes.agreement=<p>Specify a message to d
 Setting_Description_domain.list=List of domains supported by this application instance.  Domain order is unimportant.  The value of the domain(s) may be used in public URLs and parameters.<p>Domains provide a way for multiple systems/sites/tenants/customers to use a single instance of this @PwmAppName@ application.  Typically only a single instance is required.  If multiple domains are listed, the configuration editor will allow per-domain configuration of many settings.  Other settings are system-level and apply to the entire application instance.</p><p>Saving the configuration after increasing or decreasing the number of domains beyond a single domain may cause application URLs to change, and this configuration editor will change to allow editing of multiple domain configurations</p>
 Setting_Description_domain.hosts=Domain Hostnames
 Setting_Description_domain.system.adminDomain=Administrative Domain
-Setting_Description_domain.system.domainPaths=If enabled, domain IDs will be added to the path, and all access will required the domainID in the path.
+Setting_Description_domain.system.domainPaths=If enabled, domain IDs will be added to the path, and URL paths will required the inclusion of the domainID in the path.  Example: "/pwm/private/login" will become "/pwm/default/private/login".  Regardless of this setting, the domain is always accessible if the host header (the browser url) is matched by the setting in  
 Setting_Description_email.activation=Define this template to send an email to users after a successful activation.
 Setting_Description_email.activation.token=Define this template to send an email during the activation verification process. You can use %TOKEN% to insert the token value into the email.
 Setting_Description_email.adminAlert.toAddress=Define this template to send an email when System Audit events occur to the defined email addresses.

server/src/test/java/password/pwm/svc/sms/EmailQueueManagerTest.java

@@ -23,16 +23,17 @@ package password.pwm.svc.sms;
 import jakarta.mail.Message;
 import jakarta.mail.MessagingException;
 import jakarta.mail.internet.InternetAddress;
-import org.apache.commons.io.IOUtils;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.Mockito;
 import password.pwm.AppProperty;
+import password.pwm.PwmConstants;
 import password.pwm.bean.EmailItemBean;
 import password.pwm.config.AppConfig;
 import password.pwm.svc.email.EmailServer;
 import password.pwm.svc.email.EmailServerUtil;
 import password.pwm.svc.email.EmailService;
+import password.pwm.util.java.JavaHelper;
 
 import java.io.IOException;
 import java.util.List;
@@ -62,20 +63,26 @@ public class EmailQueueManagerTest
         final List<Message> messages = EmailServerUtil.convertEmailItemToMessages( emailItemBean, config, emailServer );
         Assert.assertEquals( 2, messages.size() );
 
-        Message message = messages.get( 0 );
-        Assert.assertEquals( new InternetAddress( "fred@flintstones.tv" ), message.getRecipients( Message.RecipientType.TO )[0] );
-        Assert.assertEquals( new InternetAddress( "bedrock-admin@flintstones.tv" ), message.getFrom()[0] );
-        Assert.assertEquals( "Test Subject", message.getSubject() );
-        String content = IOUtils.toString( message.getInputStream() );
-        Assert.assertTrue( content.contains( "bodyPlain" ) );
-        Assert.assertTrue( content.contains( "bodyHtml" ) );
+        {
+            final Message message = messages.get( 0 );
+            Assert.assertEquals( new InternetAddress( "fred@flintstones.tv" ), message.getRecipients( Message.RecipientType.TO )[0] );
+            Assert.assertEquals( new InternetAddress( "bedrock-admin@flintstones.tv" ), message.getFrom()[0] );
+            Assert.assertEquals( "Test Subject", message.getSubject() );
+            final String content = JavaHelper.copyToString( message.getInputStream(), PwmConstants.DEFAULT_CHARSET, Integer.MAX_VALUE )
+                    .orElse( "" );
+            Assert.assertTrue( content.contains( "bodyPlain" ) );
+            Assert.assertTrue( content.contains( "bodyHtml" ) );
+        }
 
-        message = messages.get( 1 );
-        Assert.assertEquals( new InternetAddress( "barney@flintstones.tv" ), message.getRecipients( Message.RecipientType.TO )[0] );
-        Assert.assertEquals( new InternetAddress( "bedrock-admin@flintstones.tv" ), message.getFrom()[0] );
-        Assert.assertEquals( "Test Subject", message.getSubject() );
-        content = IOUtils.toString( message.getInputStream() );
-        Assert.assertTrue( content.contains( "bodyPlain" ) );
-        Assert.assertTrue( content.contains( "bodyHtml" ) );
+        {
+            final Message message = messages.get( 1 );
+            Assert.assertEquals( new InternetAddress( "barney@flintstones.tv" ), message.getRecipients( Message.RecipientType.TO )[0] );
+            Assert.assertEquals( new InternetAddress( "bedrock-admin@flintstones.tv" ), message.getFrom()[0] );
+            Assert.assertEquals( "Test Subject", message.getSubject() );
+            final String content = JavaHelper.copyToString( message.getInputStream(), PwmConstants.DEFAULT_CHARSET, Integer.MAX_VALUE )
+                    .orElse( "" );
+            Assert.assertTrue( content.contains( "bodyPlain" ) );
+            Assert.assertTrue( content.contains( "bodyHtml" ) );
+        }
     }
 }

server/src/test/java/password/pwm/util/java/XmlFactoryBenchmarkExtendedTest.java

@@ -20,7 +20,6 @@
 
 package password.pwm.util.java;
 
-import org.apache.commons.io.output.NullOutputStream;
 import org.jrivard.xmlchai.AccessMode;
 import org.jrivard.xmlchai.XmlChai;
 import org.jrivard.xmlchai.XmlDocument;
@@ -34,6 +33,7 @@ import org.openjdk.jmh.runner.options.OptionsBuilder;
 import org.openjdk.jmh.runner.options.TimeValue;
 
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.concurrent.TimeUnit;
 
 public class XmlFactoryBenchmarkExtendedTest
@@ -72,6 +72,6 @@ public class XmlFactoryBenchmarkExtendedTest
         final XmlFactory xmlFactory = XmlChai.getFactory();
         final InputStream xmlFactoryTestXmlFile = XmlFactoryTest.class.getResourceAsStream( "XmlFactoryTest.xml" );
         final XmlDocument xmlDocument = xmlFactory.parse( xmlFactoryTestXmlFile, AccessMode.IMMUTABLE );
-        xmlFactory.output( xmlDocument, new NullOutputStream() );
+        xmlFactory.output( xmlDocument, OutputStream.nullOutputStream() );
     }
 }