scheduler refactoring

data-service/src/main/java/password/pwm/receiver/PwmReceiverApp.java

@@ -20,6 +20,7 @@
 
 package password.pwm.receiver;
 
+import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StatisticCounterBundle;
 import password.pwm.util.java.StatisticRateBundle;
@@ -29,6 +30,7 @@ import java.io.IOException;
 import java.time.Instant;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 
 public class PwmReceiverApp
@@ -39,12 +41,14 @@ public class PwmReceiverApp
     private Storage storage;
     private Settings settings;
 
-    private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
+    private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( THREAD_FACTORY );
     private final Status status = new Status();
-    private final StatisticCounterBundle statisticCounterBundle = new StatisticCounterBundle( CounterStatsKey.class );
-    private final StatisticRateBundle statisticRateBundle = new StatisticRateBundle( EpsStatKey.class );
+    private final StatisticCounterBundle<CounterStatsKey> statisticCounterBundle = new StatisticCounterBundle<>( CounterStatsKey.class );
+    private final StatisticRateBundle<EpsStatKey> statisticRateBundle = new StatisticRateBundle<>( EpsStatKey.class );
     private final Instant startupTime = Instant.now();
 
+    private static final ThreadFactory THREAD_FACTORY = makeThreadFactory();
+
     public enum EpsStatKey
     {
         VersionCheckRequests,
@@ -119,7 +123,7 @@ public class PwmReceiverApp
     void close( )
     {
         storage.close();
-        scheduledExecutorService.shutdown();
+        scheduledExecutorService.shutdownNow();
     }
 
     public Status getStatus( )
@@ -127,12 +131,12 @@ public class PwmReceiverApp
         return status;
     }
 
-    public StatisticCounterBundle getStatisticCounterBundle()
+    public StatisticCounterBundle<CounterStatsKey> getStatisticCounterBundle()
     {
         return statisticCounterBundle;
     }
 
-    public StatisticRateBundle getStatisticEpsBundle()
+    public StatisticRateBundle<EpsStatKey> getStatisticEpsBundle()
     {
         return statisticRateBundle;
     }
@@ -141,4 +145,21 @@ public class PwmReceiverApp
     {
         return startupTime;
     }
+
+    private static ThreadFactory makeThreadFactory()
+    {
+        return new ThreadFactory()
+        {
+            private final AtomicLoopIntIncrementer counter = new AtomicLoopIntIncrementer();
+
+            @Override
+            public Thread newThread( final Runnable runnable )
+            {
+                final Thread t = new Thread( runnable );
+                t.setDaemon( true );
+                t.setName( PwmReceiverApp.class.getName() + "-" + counter.next() );
+                return t;
+            }
+        };
+    }
 }

lib-util/src/main/java/password/pwm/util/java/JavaHelper.java

@@ -49,8 +49,6 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Properties;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.LongAccumulator;
 import java.util.function.Predicate;
 import java.util.zip.GZIPInputStream;
@@ -497,30 +495,6 @@ public final class JavaHelper
         return newByteArray;
     }
 
-    /**
-     * Close executor and wait up to the specified TimeDuration for all executor jobs to terminate.  There is no guarantee that either all jobs will
-     * terminate or the entire duration will be waited for, though the duration should not be exceeded.
-     * @param executor Executor close
-     * @param timeDuration TimeDuration to wait for
-     */
-    public static void closeAndWaitExecutor( final ExecutorService executor, final TimeDuration timeDuration )
-    {
-        if ( executor == null )
-        {
-            return;
-        }
-
-        executor.shutdown();
-        try
-        {
-            executor.awaitTermination( timeDuration.asMillis(), TimeUnit.MILLISECONDS );
-        }
-        catch ( final InterruptedException e )
-        {
-            /* ignore */
-        }
-    }
-
     public static String stackTraceToString( final Throwable e )
     {
         final Writer stackTraceOutput = new StringWriter();

lib-util/src/main/java/password/pwm/util/java/LazySupplier.java

@@ -42,14 +42,14 @@ public interface LazySupplier<T> extends Supplier<T>
     }
 
     /**
-     * Synchronized wrapper for any other {@code LazySupplier} implementation that
+     * Synchronized wrapper for any other {@code Supplier} implementation that
      * guarantee thread safety.  In particular, the backing realSupplier will only ever be called
      * a single time unless {@code #clear} is invoked.
      * @param realSupplier another {@code LazySupplier} instance
      * @param <T> return type.
      * @return a {@code LazyWrapper} thread safe synchronization.
      */
-    static <T> LazySupplier<T> synchronizedSupplier( final LazySupplier<T> realSupplier )
+    static <T> LazySupplier<T> createSynchronized( final Supplier<T> realSupplier )
     {
         return new LazySupplierImpl.LockingSupplier<>( realSupplier );
     }
@@ -59,7 +59,7 @@ public interface LazySupplier<T> extends Supplier<T>
         return new LazySupplierImpl.StandardLazySupplier<T>( realSupplier );
     }
 
-    static <T> LazySupplier<T> soft( final Supplier<T> realSupplier )
+    static <T> LazySupplier<T> createSoft( final Supplier<T> realSupplier )
     {
         return new LazySupplierImpl.SoftLazySupplier<T>( realSupplier );
     }

lib-util/src/main/java/password/pwm/util/java/LazySupplierImpl.java

@@ -24,6 +24,8 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 
 import java.lang.ref.SoftReference;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 
 class LazySupplierImpl
@@ -107,8 +109,8 @@ class LazySupplierImpl
     static class LockingSupplier<T> implements LazySupplier<T>
     {
         private final Supplier<T> realSupplier;
-        private volatile T value;
-        private volatile boolean supplied = false;
+        private final AtomicReference<T> value = new AtomicReference<>();
+        private final AtomicBoolean  supplied = new AtomicBoolean();
 
         private final FunctionalReentrantLock lock = new FunctionalReentrantLock();
 
@@ -122,20 +124,20 @@ class LazySupplierImpl
         {
             return lock.exec( () ->
             {
-                if ( !supplied )
+                if ( !supplied.get() )
                 {
-                    value = realSupplier.get();
-                    supplied = true;
+                    value.set( realSupplier.get() );
+                    supplied.set( true );
                 }
 
-                return value;
+                return value.get();
             } );
         }
 
         @Override
         public boolean isSupplied()
         {
-            return lock.exec( () -> supplied );
+            return lock.exec( supplied::get );
         }
 
         @Override
@@ -143,8 +145,8 @@ class LazySupplierImpl
         {
             lock.exec( () ->
             {
-                supplied = false;
-                value = null;
+                supplied.set( false );
+                value.set( null );
             } );
         }
     }

lib-util/src/main/java/password/pwm/util/java/StringUtil.java

@@ -795,4 +795,37 @@ public final class StringUtil
         final List<String> values = new ArrayList<>( Arrays.asList( inputString.split( separator ) ) );
         return Collections.unmodifiableList( values );
     }
+
+    /**
+     * Strip leading and trailing characters of a string.
+     * @param input input string
+     * @param charToStrip character to trip from input.
+     * @return A non-null string.
+     */
+    public static String stripEdgeChars( final String input, final char charToStrip )
+    {
+        if ( StringUtil.isEmpty( input ) )
+        {
+            return "";
+        }
+
+        if ( input.charAt( 0 ) != charToStrip && input.charAt( input.length() - 1 ) != charToStrip )
+        {
+            return input;
+        }
+
+        final StringBuilder stringBuilder = new StringBuilder( input );
+
+        while ( stringBuilder.length() > 0 && stringBuilder.charAt( 0 ) == charToStrip )
+        {
+            stringBuilder.deleteCharAt( 0 );
+        }
+
+        while ( stringBuilder.length() > 0 && stringBuilder.charAt( stringBuilder.length() - 1 ) == charToStrip )
+        {
+            stringBuilder.deleteCharAt( stringBuilder.length() - 1 );
+        }
+
+        return stringBuilder.toString();
+    }
 }

lib-util/src/test/java/password/pwm/util/java/StringUtilTest.java

@@ -330,4 +330,18 @@ public class StringUtilTest
         Assertions.assertEquals( "TEST", StringUtil.padLeft( "TEST", 3, ' ' ) );
         Assertions.assertEquals( "TEST", StringUtil.padLeft( "TEST", -3, ' ' ) );
     }
+
+    @Test
+    public void stripEdgeCharsTest()
+    {
+        Assertions.assertEquals( "testvalue", StringUtil.stripEdgeChars( "testvalue", '-' ) );
+        Assertions.assertEquals( "testvalue", StringUtil.stripEdgeChars( "testvalue-", '-' ) );
+        Assertions.assertEquals( "testvalue", StringUtil.stripEdgeChars( "testvalue--", '-' ) );
+        Assertions.assertEquals( "testvalue", StringUtil.stripEdgeChars( "-testvalue", '-' ) );
+        Assertions.assertEquals( "testvalue", StringUtil.stripEdgeChars( "--testvalue", '-' ) );
+        Assertions.assertEquals( "testvalue", StringUtil.stripEdgeChars( "-testvalue-", '-' ) );
+        Assertions.assertEquals( "testvalue", StringUtil.stripEdgeChars( "--testvalue--", '-' ) );
+        Assertions.assertEquals( "testvalue", StringUtil.stripEdgeChars( "---testvalue---", '-' ) );
+        Assertions.assertEquals( "testvalue", StringUtil.stripEdgeChars( "----testvalue----", '-' ) );
+    }
 }

server/src/main/java/password/pwm/PwmApplication.java

@@ -47,7 +47,6 @@ import password.pwm.svc.httpclient.HttpClientService;
 import password.pwm.svc.intruder.IntruderRecordType;
 import password.pwm.svc.intruder.IntruderSystemService;
 import password.pwm.svc.node.NodeService;
-import password.pwm.svc.report.ReportService;
 import password.pwm.svc.secure.SystemSecureService;
 import password.pwm.svc.sessiontrack.SessionTrackService;
 import password.pwm.svc.sessiontrack.UserAgentUtils;
@@ -684,11 +683,6 @@ public class PwmApplication
         return ( WordlistService ) pwmServiceManager.getService( PwmServiceEnum.WordlistService );
     }
 
-    public ReportService getReportService( )
-    {
-        return ( ReportService ) pwmServiceManager.getService( PwmServiceEnum.ReportService );
-    }
-
     public EmailService getEmailQueue( )
     {
         return ( EmailService ) pwmServiceManager.getService( PwmServiceEnum.EmailService );

server/src/main/java/password/pwm/PwmDomain.java

@@ -45,6 +45,7 @@ import password.pwm.svc.httpclient.HttpClientService;
 import password.pwm.svc.intruder.IntruderDomainService;
 import password.pwm.svc.otp.OtpService;
 import password.pwm.svc.pwnotify.PwNotifyService;
+import password.pwm.svc.report.ReportService;
 import password.pwm.svc.secure.DomainSecureService;
 import password.pwm.svc.sessiontrack.SessionTrackService;
 import password.pwm.svc.stats.StatisticsService;
@@ -238,6 +239,11 @@ public class PwmDomain
         return ( ResourceServletService ) pwmServiceManager.getService( PwmServiceEnum.ResourceServletService );
     }
 
+    public ReportService getReportService( )
+    {
+        return ( ReportService ) pwmServiceManager.getService( PwmServiceEnum.ReportService );
+    }
+
     public UserHistoryService getUserHistoryService()
     {
         return ( UserHistoryService ) pwmServiceManager.getService( PwmServiceEnum.UserHistoryService );

server/src/main/java/password/pwm/PwmDomainUtil.java

@@ -95,7 +95,8 @@ class PwmDomainUtil
             throw domainStartupException.get();
         }
 
-        LOGGER.trace( pwmApplication.getSessionLabel(), () -> "completed domain initialization for domains", TimeDuration.fromCurrent( domainInitStartTime ) );
+        LOGGER.trace( pwmApplication.getSessionLabel(), () -> "completed domain initialization for domains",
+                TimeDuration.fromCurrent( domainInitStartTime ) );
     }
 
     private static class DomainInitializingCallable implements Callable<Optional<PwmUnrecoverableException>>
@@ -130,6 +131,7 @@ class PwmDomainUtil
             throws PwmUnrecoverableException
     {
         final Map<DomainModifyCategory, Set<DomainID>> categorizedDomains = categorizeDomainModifications( newConfig, oldConfig );
+
         categorizedDomains.forEach( (  modifyCategory, domainIDSet ) -> domainIDSet.forEach( domainID ->
                 LOGGER.trace( pwmApplication.getSessionLabel(), () -> "domain '" + domainID
                         + "' configuration modification detected as: " + modifyCategory ) ) );
@@ -206,40 +208,46 @@ class PwmDomainUtil
             final AppConfig oldConfig
     )
     {
-
         {
             final Instant newInstant = newConfig.getStoredConfiguration().modifyTime();
             final Instant oldInstant = oldConfig.getStoredConfiguration().modifyTime();
             if ( newInstant != null && oldInstant != null && newInstant.isBefore( oldInstant ) )
             {
-                throw new IllegalStateException( "refusing request to categorize changes due to oldConfig being newer than new config" );
+                throw new IllegalStateException( "refusing request to categorize changes due to oldConfig "
+                        + "being newer than new config" );
             }
         }
 
-        final Set<StoredConfigKey> modifiedValues = StoredConfigurationUtil.changedValues( newConfig.getStoredConfiguration(), oldConfig.getStoredConfiguration() );
+        final Set<StoredConfigKey> modifiedValues = StoredConfigurationUtil.changedValues(
+                newConfig.getStoredConfiguration(),
+                oldConfig.getStoredConfiguration() );
 
-        return CATEGORIZERS.entrySet().stream()
+        return CLASSIFIERS.entrySet().stream()
                 .collect( Collectors.toUnmodifiableMap(
                         Map.Entry::getKey,
                         entry -> entry.getValue().categorize( newConfig, oldConfig, modifiedValues )
                 ) );
     }
 
-    interface DomainModificationCategorizer
+    interface DomainModificationClassifier
     {
         Set<DomainID> categorize( AppConfig newConfig, AppConfig oldConfig, Set<StoredConfigKey> modifiedValues );
     }
 
-    private static final Map<DomainModifyCategory, DomainModificationCategorizer> CATEGORIZERS = Map.of(
-            DomainModifyCategory.removed, new RemovalCategorizer(),
-            DomainModifyCategory.created, new CreationCategorizer(),
-            DomainModifyCategory.unchanged, new UnchangedCategorizer(),
-            DomainModifyCategory.modified, new ModifiedCategorizer() );
+    private static final Map<DomainModifyCategory, DomainModificationClassifier> CLASSIFIERS = Map.of(
+            DomainModifyCategory.removed, new RemovalClassifier(),
+            DomainModifyCategory.created, new CreationClassifier(),
+            DomainModifyCategory.unchanged, new UnchangedClassifier(),
+            DomainModifyCategory.modified, new ModifiedClassifier() );
 
-    private static class RemovalCategorizer implements DomainModificationCategorizer
+    private static class RemovalClassifier implements DomainModificationClassifier
     {
         @Override
-        public Set<DomainID> categorize( final AppConfig newConfig, final AppConfig oldConfig, final Set<StoredConfigKey> modifiedValues )
+        public Set<DomainID> categorize(
+                final AppConfig newConfig,
+                final AppConfig oldConfig,
+                final Set<StoredConfigKey> modifiedValues
+        )
         {
             final Set<DomainID> removedDomains = new HashSet<>( oldConfig.getDomainConfigs().keySet() );
             removedDomains.removeAll( newConfig.getDomainConfigs().keySet() );
@@ -247,10 +255,14 @@ class PwmDomainUtil
         }
     }
 
-    private static class CreationCategorizer implements DomainModificationCategorizer
+    private static class CreationClassifier implements DomainModificationClassifier
     {
         @Override
-        public Set<DomainID> categorize( final AppConfig newConfig, final AppConfig oldConfig, final Set<StoredConfigKey> modifiedValues )
+        public Set<DomainID> categorize(
+                final AppConfig newConfig,
+                final AppConfig oldConfig,
+                final Set<StoredConfigKey> modifiedValues
+        )
         {
             final Set<DomainID> createdDomains = new HashSet<>( newConfig.getDomainConfigs().keySet() );
             createdDomains.removeAll( oldConfig.getDomainConfigs().keySet() );
@@ -258,27 +270,39 @@ class PwmDomainUtil
         }
     }
 
-    private static class UnchangedCategorizer implements DomainModificationCategorizer
+    private static class UnchangedClassifier implements DomainModificationClassifier
     {
         @Override
-        public Set<DomainID> categorize( final AppConfig newConfig, final AppConfig oldConfig, final Set<StoredConfigKey> modifiedValues )
+        public Set<DomainID> categorize(
+                final AppConfig newConfig,
+                final AppConfig oldConfig,
+                final Set<StoredConfigKey> modifiedValues
+        )
         {
-            final Set<DomainID> persistentDomains = new HashSet<>( CollectionUtil.setUnion( newConfig.getDomainConfigs().keySet(), oldConfig.getDomainConfigs().keySet() ) );
+            final Set<DomainID> persistentDomains = new HashSet<>(
+                    CollectionUtil.setUnion(
+                            newConfig.getDomainConfigs().keySet(),
+                            oldConfig.getDomainConfigs().keySet() ) );
             persistentDomains.removeAll( StoredConfigKey.uniqueDomains( modifiedValues ) );
             return Set.copyOf( persistentDomains );
         }
     }
 
-    private static class ModifiedCategorizer implements DomainModificationCategorizer
+    private static class ModifiedClassifier implements DomainModificationClassifier
     {
         @Override
-        public Set<DomainID> categorize( final AppConfig newConfig, final AppConfig oldConfig, final Set<StoredConfigKey> modifiedValues )
+        public Set<DomainID> categorize(
+                final AppConfig newConfig,
+                final AppConfig oldConfig,
+                final Set<StoredConfigKey> modifiedValues
+        )
         {
-            final Set<DomainID> persistentDomains = new HashSet<>( CollectionUtil.setUnion( newConfig.getDomainConfigs().keySet(), oldConfig.getDomainConfigs().keySet() ) );
-            persistentDomains.retainAll( StoredConfigKey.uniqueDomains( modifiedValues ) );
-            return Set.copyOf( persistentDomains );
+            final Set<DomainID> modifiedDomains = new HashSet<>(
+                    CollectionUtil.setUnion(
+                            newConfig.getDomainConfigs().keySet(),
+                            oldConfig.getDomainConfigs().keySet() ) );
+            modifiedDomains.retainAll( StoredConfigKey.uniqueDomains( modifiedValues ) );
+            return Set.copyOf( modifiedDomains );
         }
     }
-
-
 }

server/src/main/java/password/pwm/config/PwmSettingXml.java

@@ -24,6 +24,7 @@ import org.jrivard.xmlchai.AccessMode;
 import org.jrivard.xmlchai.XmlChai;
 import org.jrivard.xmlchai.XmlDocument;
 import org.jrivard.xmlchai.XmlElement;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.LazySupplier;
 import password.pwm.util.java.TimeDuration;
@@ -72,8 +73,8 @@ public class PwmSettingXml
 
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwmSettingXml.class );
 
-    private static final LazySupplier<XmlDocument> XML_DOC_CACHE = LazySupplier.synchronizedSupplier(
-            LazySupplier.create( PwmSettingXml::readXml ) );
+    private static final LazySupplier<XmlDocument> XML_DOC_CACHE = LazySupplier.createSynchronized(
+            PwmSettingXml::readXml );
 
     private static final AtomicInteger LOAD_COUNTER = new AtomicInteger( 0 );
 
@@ -86,8 +87,14 @@ public class PwmSettingXml
             final TimeDuration parseDuration = TimeDuration.fromCurrent( startTime );
             LOGGER.trace( () -> "parsed PwmSettingXml in " + parseDuration.asCompactString() + ", loads=" + LOAD_COUNTER.getAndIncrement() );
 
-            final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
-            scheduledExecutorService.schedule( XML_DOC_CACHE::clear, 30, TimeUnit.SECONDS );
+            final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(
+                    PwmScheduler.makePwmThreadFactory( "pwmSetting-xml-cache-timeout-thread" ) );
+
+            scheduledExecutorService.schedule( () ->
+            {
+                XML_DOC_CACHE.clear();
+                scheduledExecutorService.shutdownNow();
+            }, 30, TimeUnit.SECONDS );
 
             return newDoc;
         }

server/src/main/java/password/pwm/http/ContextManager.java

@@ -287,8 +287,7 @@ public class ContextManager implements Serializable
 
         taskMaster = Executors.newSingleThreadScheduledExecutor(
                 PwmScheduler.makePwmThreadFactory(
-                        PwmScheduler.makeThreadName( SESSION_LABEL, pwmApplication, this.getClass() ) + "-",
-                        true
+                        PwmScheduler.makeThreadName( SESSION_LABEL, pwmApplication, this.getClass() ) + "-"
                 ) );
 
         boolean reloadOnChange = true;
@@ -388,7 +387,7 @@ public class ContextManager implements Serializable
                 LOGGER.error( () -> "unexpected error attempting to close application: " + e.getMessage() );
             }
         }
-        taskMaster.shutdown();
+        taskMaster.shutdownNow();
 
 
         this.pwmApplication = null;

server/src/main/java/password/pwm/http/servlet/admin/domain/AdminReportServlet.java

@@ -154,7 +154,7 @@ public class AdminReportServlet extends ControlledPwmServlet
 
         try ( OutputStream outputStream = pwmRequest.getPwmResponse().getOutputStream() )
         {
-            final ReportService reportService = pwmRequest.getPwmApplication().getReportService();
+            final ReportService reportService = pwmRequest.getPwmDomain().getReportService();
             try ( ReportProcess reportProcess = reportService.createReportProcess( reportProcessRequest ) )
             {
                 pwmRequest.getPwmSession().setReportProcess( reportProcess );

server/src/main/java/password/pwm/http/servlet/configeditor/ConfigEditorServletUtils.java

@@ -391,7 +391,7 @@ public class ConfigEditorServletUtils
 
         try
         {
-            return PwmScheduler.timeoutExecutor( pwmRequest.getPwmApplication(), pwmRequest.getLabel(), configEditorSettings.getMaxWaitSettingsFunction(), callable );
+            return PwmScheduler.executeWithTimeout( pwmRequest.getPwmApplication(), pwmRequest.getLabel(), configEditorSettings.getMaxWaitSettingsFunction(), callable );
         }
         catch ( final PwmUnrecoverableException e )
         {

server/src/main/java/password/pwm/http/servlet/peoplesearch/PeopleSearchService.java

@@ -52,7 +52,7 @@ public class PeopleSearchService extends AbstractPwmService implements PwmServic
     {
         if ( threadPoolExecutor != null )
         {
-            threadPoolExecutor.shutdown();
+            threadPoolExecutor.shutdownNow();
             threadPoolExecutor = null;
         }
     }

server/src/main/java/password/pwm/ldap/search/UserSearchService.java

@@ -126,7 +126,7 @@ public class UserSearchService extends AbstractPwmService implements PwmService
     {
         if ( executor != null )
         {
-            executor.shutdown();
+            executor.shutdownNow();
         }
         executor = null;
     }

server/src/main/java/password/pwm/svc/AbstractPwmService.java

@@ -30,10 +30,10 @@ import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
 import password.pwm.util.PwmScheduler;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.LazySupplier;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogManager;
+import password.pwm.util.logging.PwmLogger;
 
 import java.util.ArrayList;
 import java.util.EnumSet;
@@ -103,8 +103,7 @@ public abstract class AbstractPwmService implements PwmService
         this.status = STATUS.CLOSED;
         if ( executorService != null && executorService.isSupplied() )
         {
-            executorService.get().shutdownNow();
-            JavaHelper.closeAndWaitExecutor( executorService.get(), TimeDuration.SECONDS_10 );
+            PwmScheduler.closeAndWaitExecutor( executorService.get(), TimeDuration.SECONDS_10,  PwmLogger.forClass( this.getClass() ), getSessionLabel() );
         }
         shutdownImpl();
     }

server/src/main/java/password/pwm/svc/PwmServiceEnum.java

@@ -45,6 +45,7 @@ public enum PwmServiceEnum
 {
     LocalDBService( password.pwm.util.localdb.LocalDBService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
     SystemSecureService( password.pwm.svc.secure.SystemSecureService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
+    EmailService( EmailService.class, PwmSettingScope.SYSTEM ),
     HttpClientService( password.pwm.svc.httpclient.HttpClientService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
     DatabaseService( password.pwm.svc.db.DatabaseService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
     SharedHistoryManager( SharedHistoryService.class, PwmSettingScope.SYSTEM ),
@@ -52,7 +53,6 @@ public enum PwmServiceEnum
     StatisticsService( StatisticsService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
     WordlistService( WordlistService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
     IntruderSystemService( IntruderSystemService.class, PwmSettingScope.SYSTEM ),
-    EmailService( EmailService.class, PwmSettingScope.SYSTEM ),
     SmsQueueManager( SmsQueueService.class, PwmSettingScope.SYSTEM ),
     UrlShortenerService( password.pwm.svc.shorturl.UrlShortenerService.class, PwmSettingScope.SYSTEM ),
     CacheService( password.pwm.svc.cache.CacheService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),

server/src/main/java/password/pwm/svc/cr/NMASCrOperator.java

@@ -239,7 +239,7 @@ public class NMASCrOperator implements CrOperator
                 if ( localTimer != null )
                 {
                     LOGGER.debug( pwmDomain.getSessionLabel(), () -> "discontinuing NMASCrOperator watchdog timer, no active threads" );
-                    localTimer.shutdown();
+                    localTimer.shutdownNow();
                     executorService = null;
                 }
             }

server/src/main/java/password/pwm/svc/db/DBConfiguration.java

@@ -30,7 +30,6 @@ import password.pwm.config.value.FileValue;
 import password.pwm.data.ImmutableByteArray;
 import password.pwm.util.PasswordData;
 import password.pwm.util.java.CollectionUtil;
-import password.pwm.util.java.StringUtil;
 
 import java.io.Serializable;
 import java.util.Arrays;
@@ -59,15 +58,6 @@ public class DBConfiguration implements Serializable
         return jdbcDriver;
     }
 
-    public boolean isEnabled( )
-    {
-        return
-                StringUtil.notEmpty( driverClassname )
-                        && StringUtil.notEmpty( connectionString )
-                        && StringUtil.notEmpty( username )
-                        && !( password == null );
-    }
-
     static DBConfiguration fromConfiguration( final AppConfig config )
     {
         final Map<FileValue.FileInformation, FileValue.FileContent> fileValue = config.readSettingAsFile(

server/src/main/java/password/pwm/svc/db/DatabaseService.java

@@ -106,24 +106,39 @@ public class DatabaseService extends AbstractPwmService implements PwmService
                 Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.DB_CONNECTIONS_WATCHDOG_FREQUENCY_SECONDS ) ),
                 TimeDuration.Unit.SECONDS );
 
+        if ( !dbShouldOpen() )
+        {
+            initialized = true;
+            return STATUS.CLOSED;
+        }
+
         scheduleFixedRateJob( new ConnectionMonitor(), watchdogFrequency, watchdogFrequency );
 
         return dbInit();
     }
 
-    private STATUS dbInit( )
+    private boolean dbShouldOpen()
     {
-        if ( initialized )
+        if ( getPwmApplication().getPwmEnvironment().isInternalRuntimeInstance() )
         {
-            return STATUS.OPEN;
+            return false;
         }
 
-        if ( !dbConfiguration.isEnabled() )
+        if ( !getPwmApplication().getConfig().hasDbConfigured() )
         {
             setStatus( STATUS.CLOSED );
-            LOGGER.debug( getSessionLabel(), () -> "skipping database connection open, no connection parameters configured" );
-            initialized = true;
-            return STATUS.CLOSED;
+            LOGGER.debug( getSessionLabel(), () -> "skipping database connection open, connection parameters are not configured" );
+            return false;
+        }
+
+        return true;
+    }
+
+    private STATUS dbInit( )
+    {
+        if ( initialized )
+        {
+            return STATUS.OPEN;
         }
 
         final Instant startTime = Instant.now();
@@ -282,7 +297,7 @@ public class DatabaseService extends AbstractPwmService implements PwmService
     {
 
         final String errorMsg;
-        if ( dbConfiguration != null && !dbConfiguration.isEnabled() )
+        if ( dbConfiguration != null && !getPwmApplication().getConfig().hasDbConfigured() )
         {
             errorMsg = "database is not configured";
         }

server/src/main/java/password/pwm/svc/event/LocalDbAuditVault.java

@@ -80,7 +80,7 @@ public class LocalDbAuditVault implements AuditVault
     @Override
     public void close( )
     {
-        executorService.shutdown();
+        executorService.shutdownNow();
         status = PwmService.STATUS.CLOSED;
     }
 

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyEngine.java

@@ -30,18 +30,17 @@ import password.pwm.error.PwmError;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.ldap.LdapOperationsHelper;
-import password.pwm.user.UserInfo;
 import password.pwm.ldap.UserInfoFactory;
 import password.pwm.ldap.permission.UserPermissionUtility;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.node.NodeService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsClient;
+import password.pwm.user.UserInfo;
 import password.pwm.util.PwmScheduler;
 import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.ConditionalTaskExecutor;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
@@ -62,7 +61,7 @@ public class PwNotifyEngine
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwNotifyEngine.class );
 
-    private static final int MAX_LOG_SIZE = 1024 * 1024 * 1024;
+    private static final int MAX_LOG_SIZE = 10_1024_1024;
 
     private final PwNotifyService pwNotifyService;
     private final PwNotifySettings settings;
@@ -174,7 +173,7 @@ public class PwNotifyEngine
                 threadPoolExecutor.submit( new ProcessJob( userIdentity ) );
             }
 
-            JavaHelper.closeAndWaitExecutor( threadPoolExecutor, TimeDuration.DAY );
+            PwmScheduler.closeAndWaitExecutor( threadPoolExecutor, TimeDuration.DAY, LOGGER, pwNotifyService.getSessionLabel() );
 
             log( "job complete, " + examinedCount + " users evaluated in " + TimeDuration.fromCurrent( startTime ).asCompactString()
                     + ", sent " + noticeCount + " notices."

server/src/main/java/password/pwm/svc/report/ReportProcess.java

@@ -288,7 +288,7 @@ public class ReportProcess implements AutoCloseable
 
                     log( PwmLogLevel.TRACE, () -> msg, null );
 
-                    if ( recordErrorMessages.size() < 1000 )
+                    if ( recordErrorMessages.size() < reportSettings.getMaxErrorRecords() )
                     {
                         recordErrorMessages.add( msg );
                     }
@@ -305,7 +305,7 @@ public class ReportProcess implements AutoCloseable
         }
         finally
         {
-            completion.getExecutorService().shutdown();
+            completion.getExecutorService().shutdownNow();
         }
 
         final Instant finishTime = Instant.now();

server/src/main/java/password/pwm/svc/report/ReportSettings.java

@@ -76,6 +76,8 @@ public class ReportSettings implements Serializable
     @Builder.Default
     private JobIntensity reportJobIntensity = JobIntensity.LOW;
 
+    @Builder.Default
+    private int maxErrorRecords = 1_000;
 
     @Builder.Default
     private TimeDuration reportJobTimeout = TimeDuration.MINUTE;

server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java

@@ -33,10 +33,9 @@ import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
+import password.pwm.util.Percent;
 import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.ConditionalTaskExecutor;
-import password.pwm.util.java.JavaHelper;
-import password.pwm.util.Percent;
 import password.pwm.util.java.PwmCallable;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
@@ -301,9 +300,8 @@ abstract class AbstractWordlist extends AbstractPwmService implements Wordlist,
 
         if ( executorService != null )
         {
-            executorService.shutdown();
+            PwmScheduler.closeAndWaitExecutor( executorService, closeWaitTime, getLogger(), getSessionLabel() );
 
-            JavaHelper.closeAndWaitExecutor( executorService, closeWaitTime );
             if ( backgroundImportRunning.isLocked() )
             {
                 getLogger().warn( getSessionLabel(), () -> "background thread still running after waiting " + closeWaitTime.asCompactString() );

server/src/main/java/password/pwm/util/PwmScheduler.java

@@ -42,7 +42,6 @@ import java.util.TimeZone;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ScheduledExecutorService;
@@ -57,7 +56,6 @@ import java.util.stream.Collectors;
 public class PwmScheduler
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwmScheduler.class );
-    private static final AtomicLoopIntIncrementer THREAD_ID_COUNTER = new AtomicLoopIntIncrementer();
 
     private final PwmApplication pwmApplication;
 
@@ -134,7 +132,7 @@ public class PwmScheduler
             results.add( awaitFutureCompletion( f ) );
         }
 
-        executor.shutdown();
+        executor.shutdownNow();
         return Collections.unmodifiableList( results );
     }
 
@@ -204,7 +202,6 @@ public class PwmScheduler
             final String instanceID,
             final Class<?> theClass,
             final String threadNameSuffix
-
     )
     {
         final StringBuilder output = new StringBuilder();
@@ -238,22 +235,19 @@ public class PwmScheduler
         return output.toString();
     }
 
-    public static ThreadFactory makePwmThreadFactory( final String namePrefix, final boolean daemon )
+    public static ThreadFactory makePwmThreadFactory( final String namePrefix )
     {
         return new ThreadFactory()
         {
-            private final ThreadFactory realThreadFactory = Executors.defaultThreadFactory();
+            private final AtomicLoopIntIncrementer counter = new AtomicLoopIntIncrementer();
 
             @Override
             public Thread newThread( final Runnable runnable )
             {
-                final Thread t = realThreadFactory.newThread( runnable );
-                t.setDaemon( daemon );
-                if ( namePrefix != null )
-                {
-                    final String newName = namePrefix + t.getName();
-                    t.setName( newName );
-                }
+                final String strippedNamePrefix = StringUtil.stripEdgeChars( namePrefix, '-' );
+                final Thread t = new Thread( runnable );
+                t.setDaemon( true );
+                t.setName( strippedNamePrefix + "-" + counter.next() );
                 return t;
             }
         };
@@ -293,10 +287,7 @@ public class PwmScheduler
                 maxThreadCount,
                 1, TimeUnit.SECONDS,
                 new LinkedBlockingQueue<>(),
-                makePwmThreadFactory(
-                        makeThreadName( sessionLabel, instanceID, theClass, threadNameSuffix ) + "-",
-                        true
-                ) );
+                makePwmThreadFactory( makeThreadName( sessionLabel, instanceID, theClass, threadNameSuffix ) + "-" ) );
         executor.allowCoreThreadTimeOut( true );
         return executor;
     }
@@ -320,9 +311,7 @@ public class PwmScheduler
         final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(
                 1,
                 makePwmThreadFactory(
-                        makeThreadName( sessionLabel, pwmApplication, clazz, threadNameSuffix ) + "-",
-                        true
-                ) );
+                        makeThreadName( sessionLabel, pwmApplication, clazz, threadNameSuffix ) + "-" ) );
         executor.setKeepAliveTime( 5, TimeUnit.SECONDS );
         executor.allowCoreThreadTimeOut( true );
         return executor;
@@ -356,7 +345,7 @@ public class PwmScheduler
      * @throws PwmUnrecoverableException if the task times out.  Uses {@link PwmError#ERROR_TIMEOUT}.
      * @throws Throwable any throwable generated by the {@code callable}
      */
-    public static <T> T timeoutExecutor(
+    public static <T> T executeWithTimeout(
             final PwmApplication pwmApplication,
             final SessionLabel label,
             final TimeDuration maxWaitDuration,
@@ -391,4 +380,46 @@ public class PwmScheduler
             executor.shutdownNow();
         }
     }
+
+    /**
+     * Close executor and wait up to the specified TimeDuration for all executor jobs to terminate.  There is no guarantee that either all jobs will
+     * terminate or the entire duration will be waited for, though the duration should not be exceeded.
+     * @param executor Executor to close
+     * @param timeDuration TimeDuration to wait for
+     * @param pwmLogger log errors or failed closures to.
+     */
+    public static void closeAndWaitExecutor(
+            final ExecutorService executor,
+            final TimeDuration timeDuration,
+            final PwmLogger pwmLogger,
+            final SessionLabel sessionLabel
+    )
+    {
+        if ( executor == null )
+        {
+            return;
+        }
+
+        executor.shutdownNow();
+
+        try
+        {
+            executor.awaitTermination( timeDuration.asMillis(), TimeUnit.MILLISECONDS );
+        }
+        catch ( final InterruptedException e )
+        {
+            pwmLogger.error( sessionLabel, () -> "attempted to shutdown thread '" + executor + "' however shutdown was interrupted: " + e.getMessage() );
+        }
+
+        if ( pwmLogger != null && !executor.isShutdown() )
+        {
+            pwmLogger.error( sessionLabel, () -> "attempted to shutdown thread '" + executor + "' however thread is not shutdown" );
+        }
+
+        if ( pwmLogger != null && !executor.isTerminated() )
+        {
+            pwmLogger.error( sessionLabel, () -> "attempted to shutdown thread '" + executor + "' however thread is not terminated" );
+        }
+    }
+
 }

server/src/main/java/password/pwm/util/cli/commands/UserReportCommand.java

@@ -23,6 +23,7 @@ package password.pwm.util.cli.commands;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import password.pwm.PwmApplication;
 import password.pwm.PwmConstants;
+import password.pwm.bean.DomainID;
 import password.pwm.bean.SessionLabel;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
@@ -57,7 +58,7 @@ public class UserReportCommand extends AbstractCliCommand
 
             final PwmApplication pwmApplication = cliEnvironment.getPwmApplication();
 
-            final ReportService reportService = pwmApplication.getReportService();
+            final ReportService reportService = pwmApplication.domains().get( DomainID.DOMAIN_ID_DEFAULT ).getReportService();
             if ( reportService.status() != PwmService.STATUS.OPEN )
             {
                 out( "report service is not open or enabled" );

server/src/main/java/password/pwm/util/java/BlockingThreadPool.java

@@ -1,54 +0,0 @@
-/*
- * Password Management Servlets (PWM)
- * http://www.pwm-project.org
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2021 The PWM Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package password.pwm.util.java;
-
-import password.pwm.util.PwmScheduler;
-
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-public class BlockingThreadPool extends ThreadPoolExecutor
-{
-
-    private final Semaphore semaphore;
-
-    public BlockingThreadPool( final int bound, final String name )
-    {
-        super( bound, bound, 0, TimeUnit.MILLISECONDS, new LinkedBlockingDeque<>(), PwmScheduler.makePwmThreadFactory( name, true ) );
-        semaphore = new Semaphore( bound );
-    }
-
-    public Future<?> blockingSubmit( final Runnable task )
-    {
-        semaphore.acquireUninterruptibly();
-        return super.submit( task );
-    }
-
-    @Override
-    protected void afterExecute( final Runnable r, final Throwable t )
-    {
-        super.afterExecute( r, t );
-        semaphore.release();
-    }
-}

server/src/main/java/password/pwm/util/localdb/WorkQueueProcessor.java

@@ -127,7 +127,7 @@ public final class WorkQueueProcessor<W extends Serializable>
 
         if ( settings.getPreThreads() > 0 )
         {
-            final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( sessionLabel, pwmApplication, sourceClass ), true );
+            final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( sessionLabel, pwmApplication, sourceClass ) );
             executorService = new ThreadPoolExecutor(
                     1,
                     settings.getPreThreads(),
@@ -143,14 +143,14 @@ public final class WorkQueueProcessor<W extends Serializable>
 
     public void close( )
     {
-        if ( workerThread == null )
+        if ( executorService != null )
         {
-            return;
+            executorService.shutdownNow();
         }
 
-        if ( executorService != null )
+        if ( workerThread == null )
         {
-            executorService.shutdown();
+            return;
         }
 
         final WorkerThread localWorkerThread = workerThread;

server/src/main/java/password/pwm/util/logging/LocalDBLogger.java

@@ -259,10 +259,10 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
             flushedEvents = tempMemoryEventQueue.size();
             if ( cleanerService != null )
             {
-                cleanerService.shutdown();
+                cleanerService.shutdownNow();
             }
             writerService.execute( new FlushTask() );
-            JavaHelper.closeAndWaitExecutor( writerService, TimeDuration.SECONDS_10 );
+            PwmScheduler.closeAndWaitExecutor( writerService, TimeDuration.SECONDS_10, LOGGER, SESSION_LABEL );
         }
         else
         {

server/src/test/java/password/pwm/util/localdb/LocalDBLoggerExtendedTest.java

@@ -147,7 +147,7 @@ public class LocalDBLoggerExtendedTest
             threadPoolExecutor.execute( new PopulatorThread() );
         }
 
-        threadPoolExecutor.shutdown();
+        threadPoolExecutor.shutdownNow();
         threadPoolExecutor.awaitTermination( 1, TimeUnit.DAYS );
         timer.cancel();
         out( "bulk operations completed" );