ソースを参照

introduce pwmScheduler

jrivard@gmail.com 6 年 前
コミット
bd3ed4c7a5
22 ファイル変更375 行追加259 行削除
  1. 10 129
      server/src/main/java/password/pwm/PwmApplication.java
  2. 4 3
      server/src/main/java/password/pwm/health/HealthMonitor.java
  3. 3 2
      server/src/main/java/password/pwm/http/ContextManager.java
  4. 2 2
      server/src/main/java/password/pwm/http/servlet/peoplesearch/PeopleSearchService.java
  5. 2 1
      server/src/main/java/password/pwm/ldap/search/UserSearchEngine.java
  6. 3 3
      server/src/main/java/password/pwm/svc/event/LocalDbAuditVault.java
  7. 2 1
      server/src/main/java/password/pwm/svc/intruder/IntruderManager.java
  8. 3 2
      server/src/main/java/password/pwm/svc/node/NodeMachine.java
  9. 2 1
      server/src/main/java/password/pwm/svc/pwnotify/PwNotifyEngine.java
  10. 5 4
      server/src/main/java/password/pwm/svc/pwnotify/PwNotifyService.java
  11. 6 8
      server/src/main/java/password/pwm/svc/report/ReportService.java
  12. 4 4
      server/src/main/java/password/pwm/svc/stats/StatisticsManager.java
  13. 3 2
      server/src/main/java/password/pwm/svc/telemetry/TelemetryService.java
  14. 3 2
      server/src/main/java/password/pwm/svc/token/TokenService.java
  15. 3 2
      server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java
  16. 4 3
      server/src/main/java/password/pwm/svc/wordlist/SharedHistoryManager.java
  17. 302 0
      server/src/main/java/password/pwm/util/PwmScheduler.java
  18. 3 2
      server/src/main/java/password/pwm/util/db/DatabaseService.java
  19. 3 1
      server/src/main/java/password/pwm/util/java/BlockingThreadPool.java
  20. 0 81
      server/src/main/java/password/pwm/util/java/JavaHelper.java
  21. 3 2
      server/src/main/java/password/pwm/util/localdb/WorkQueueProcessor.java
  22. 5 4
      server/src/main/java/password/pwm/util/logging/LocalDBLogger.java

+ 10 - 129
server/src/main/java/password/pwm/PwmApplication.java

@@ -25,7 +25,6 @@ package password.pwm;
 import com.novell.ldapchai.ChaiUser;
 import com.novell.ldapchai.ChaiUser;
 import com.novell.ldapchai.exception.ChaiUnavailableException;
 import com.novell.ldapchai.exception.ChaiUnavailableException;
 import com.novell.ldapchai.provider.ChaiProvider;
 import com.novell.ldapchai.provider.ChaiProvider;
-import org.jetbrains.annotations.NotNull;
 import password.pwm.bean.SessionLabel;
 import password.pwm.bean.SessionLabel;
 import password.pwm.bean.SmsItemBean;
 import password.pwm.bean.SmsItemBean;
 import password.pwm.bean.UserIdentity;
 import password.pwm.bean.UserIdentity;
@@ -66,6 +65,7 @@ import password.pwm.svc.wordlist.WordlistService;
 import password.pwm.util.DailySummaryJob;
 import password.pwm.util.DailySummaryJob;
 import password.pwm.util.MBeanUtility;
 import password.pwm.util.MBeanUtility;
 import password.pwm.util.PasswordData;
 import password.pwm.util.PasswordData;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.cli.commands.ExportHttpsTomcatConfigCommand;
 import password.pwm.util.cli.commands.ExportHttpsTomcatConfigCommand;
 import password.pwm.util.db.DatabaseAccessor;
 import password.pwm.util.db.DatabaseAccessor;
 import password.pwm.util.db.DatabaseService;
 import password.pwm.util.db.DatabaseService;
@@ -101,11 +101,8 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.List;
 import java.util.Locale;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Map;
-import java.util.Objects;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicInteger;
 
 
 /**
 /**
@@ -169,7 +166,7 @@ public class PwmApplication
 
 
     private final PwmServiceManager pwmServiceManager = new PwmServiceManager( this );
     private final PwmServiceManager pwmServiceManager = new PwmServiceManager( this );
 
 
-    private ScheduledExecutorService applicationExecutorService;
+    private PwmScheduler pwmScheduler;
 
 
     public PwmApplication( final PwmEnvironment pwmEnvironment )
     public PwmApplication( final PwmEnvironment pwmEnvironment )
             throws PwmUnrecoverableException
             throws PwmUnrecoverableException
@@ -287,7 +284,7 @@ public class PwmApplication
         LOGGER.debug( () -> "application environment flags: " + JsonUtil.serializeCollection( pwmEnvironment.getFlags() ) );
         LOGGER.debug( () -> "application environment flags: " + JsonUtil.serializeCollection( pwmEnvironment.getFlags() ) );
         LOGGER.debug( () -> "application environment parameters: " + JsonUtil.serializeMap( pwmEnvironment.getParameters() ) );
         LOGGER.debug( () -> "application environment parameters: " + JsonUtil.serializeMap( pwmEnvironment.getParameters() ) );
 
 
-        applicationExecutorService = JavaHelper.makeSingleThreadExecutorService( this, this.getClass() );
+        pwmScheduler = new PwmScheduler( getInstanceID() );
 
 
         pwmServiceManager.initAllServices();
         pwmServiceManager.initAllServices();
 
 
@@ -301,8 +298,7 @@ public class PwmApplication
             StatisticsManager.incrementStat( this, Statistic.PWM_STARTUPS );
             StatisticsManager.incrementStat( this, Statistic.PWM_STARTUPS );
             LOGGER.debug( () -> "buildTime=" + PwmConstants.BUILD_TIME + ", javaLocale=" + Locale.getDefault() + ", DefaultLocale=" + PwmConstants.DEFAULT_LOCALE );
             LOGGER.debug( () -> "buildTime=" + PwmConstants.BUILD_TIME + ", javaLocale=" + Locale.getDefault() + ", DefaultLocale=" + PwmConstants.DEFAULT_LOCALE );
 
 
-            final ExecutorService executorService = JavaHelper.makeSingleThreadExecutorService( this, PwmApplication.class );
-            scheduleFutureJob( this::postInitTasks, executorService, TimeDuration.ZERO );
+            pwmScheduler.immediateExecuteInNewThread( this::postInitTasks );
         }
         }
 
 
 
 
@@ -424,8 +420,8 @@ public class PwmApplication
         }
         }
 
 
         {
         {
-            final ExecutorService executorService = JavaHelper.makeSingleThreadExecutorService( this, PwmApplication.class );
-            this.scheduleFixedRateJob( new DailySummaryJob( this ), executorService, TimeDuration.fromCurrent( JavaHelper.nextZuluZeroTime() ), TimeDuration.DAY );
+            final ExecutorService executorService = PwmScheduler.makeSingleThreadExecutorService( this, PwmApplication.class );
+            pwmScheduler.scheduleDailyZuluZeroStartJob( new DailySummaryJob( this ), executorService, TimeDuration.ZERO );
         }
         }
 
 
         LOGGER.trace( () -> "completed post init tasks in " + TimeDuration.fromCurrent( startTime ).asCompactString() );
         LOGGER.trace( () -> "completed post init tasks in " + TimeDuration.fromCurrent( startTime ).asCompactString() );
@@ -796,7 +792,7 @@ public class PwmApplication
 
 
     public void shutdown( )
     public void shutdown( )
     {
     {
-        applicationExecutorService.shutdown();
+        pwmScheduler.shutdown();
 
 
         LOGGER.warn( "shutting down" );
         LOGGER.warn( "shutting down" );
         {
         {
@@ -1035,124 +1031,9 @@ public class PwmApplication
         return inprogressRequests;
         return inprogressRequests;
     }
     }
 
 
-
-    public Future scheduleFutureJob(
-            final Runnable runnable,
-            final ExecutorService executor,
-            final TimeDuration delay
-    )
-    {
-        Objects.requireNonNull( runnable );
-        Objects.requireNonNull( executor );
-        Objects.requireNonNull( delay );
-
-        if ( applicationExecutorService.isShutdown() )
-        {
-            return null;
-        }
-
-        final WrappedRunner wrappedRunner = new WrappedRunner( runnable, executor );
-        applicationExecutorService.schedule( wrappedRunner, delay.asMillis(), TimeUnit.MILLISECONDS );
-        return wrappedRunner.getFuture();
-    }
-
-    public void scheduleFixedRateJob(
-            final Runnable runnable,
-            final ExecutorService executor,
-            final TimeDuration initialDelay,
-            final TimeDuration frequency
-    )
-    {
-        if ( applicationExecutorService.isShutdown() )
-        {
-            return;
-        }
-
-        if ( initialDelay != null )
-        {
-            applicationExecutorService.schedule( new WrappedRunner( runnable, executor ), initialDelay.asMillis(), TimeUnit.MILLISECONDS );
-        }
-
-        final Runnable jobWithNextScheduler = () ->
-        {
-            new WrappedRunner( runnable, executor ).run();
-            scheduleFixedRateJob( runnable, executor, null, frequency );
-        };
-
-        applicationExecutorService.schedule(  jobWithNextScheduler, frequency.asMillis(), TimeUnit.MILLISECONDS );
-    }
-
-    private static class WrappedRunner implements Runnable
+    public PwmScheduler getPwmScheduler()
     {
     {
-        private final Runnable runnable;
-        private final ExecutorService executor;
-        private volatile Future innerFuture;
-        private volatile boolean hasFailed;
-
-        WrappedRunner( final Runnable runnable, final ExecutorService executor )
-        {
-            this.runnable = runnable;
-            this.executor = executor;
-        }
-
-        Future getFuture()
-        {
-            return new Future()
-            {
-                @Override
-                public boolean cancel( final boolean mayInterruptIfRunning )
-                {
-                    return false;
-                }
-
-                @Override
-                public boolean isCancelled()
-                {
-                    return hasFailed;
-                }
-
-                @Override
-                public boolean isDone()
-                {
-                    return hasFailed || innerFuture != null && innerFuture.isDone();
-                }
-
-                @Override
-                public Object get()
-                {
-                    return null;
-                }
-
-                @Override
-                public Object get( final long timeout, @NotNull final TimeUnit unit )
-                {
-                    return null;
-                }
-            };
-        }
-
-
-        @Override
-        public void run()
-        {
-            try
-            {
-                if ( !executor.isShutdown() )
-                {
-                    innerFuture = executor.submit( runnable );
-                }
-                else
-                {
-                    hasFailed = true;
-                    LOGGER.trace( () -> "skipping scheduled job " + runnable + " on shutdown executor + " + executor );
-                }
-            }
-            catch ( Throwable t )
-            {
-                LOGGER.error( "unexpected error running scheduled job: " + t.getMessage(), t );
-                hasFailed = true;
-            }
-        }
+        return pwmScheduler;
     }
     }
 }
 }
 
 

+ 4 - 3
server/src/main/java/password/pwm/health/HealthMonitor.java

@@ -27,6 +27,7 @@ import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.PwmApplication;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmException;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmService;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.logging.PwmLogger;
@@ -135,7 +136,7 @@ public class HealthMonitor implements PwmService
             return;
             return;
         }
         }
 
 
-        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
 
 
         status = STATUS.OPEN;
         status = STATUS.OPEN;
     }
     }
@@ -151,12 +152,12 @@ public class HealthMonitor implements PwmService
         {
         {
             final Instant startTime = Instant.now();
             final Instant startTime = Instant.now();
             LOGGER.trace( () ->  "begin force immediate check" );
             LOGGER.trace( () ->  "begin force immediate check" );
-            final Future future = pwmApplication.scheduleFutureJob( new ImmediateJob(), executorService, TimeDuration.ZERO );
+            final Future future = pwmApplication.getPwmScheduler().scheduleFutureJob( new ImmediateJob(), executorService, TimeDuration.ZERO );
             JavaHelper.pause( settings.getMaximumForceCheckWait().asMillis(), 500, o -> future.isDone() );
             JavaHelper.pause( settings.getMaximumForceCheckWait().asMillis(), 500, o -> future.isDone() );
             LOGGER.trace( () ->  "exit force immediate check, done=" + future.isDone() + ", " + TimeDuration.compactFromCurrent( startTime ) );
             LOGGER.trace( () ->  "exit force immediate check, done=" + future.isDone() + ", " + TimeDuration.compactFromCurrent( startTime ) );
         }
         }
 
 
-        pwmApplication.scheduleFutureJob( new UpdateJob(), executorService, settings.getNominalCheckInterval() );
+        pwmApplication.getPwmScheduler().scheduleFutureJob( new UpdateJob(), executorService, settings.getNominalCheckInterval() );
 
 
         {
         {
             final HealthData localHealthData = this.healthData;
             final HealthData localHealthData = this.healthData;

+ 3 - 2
server/src/main/java/password/pwm/http/ContextManager.java

@@ -36,6 +36,7 @@ import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.PropertyConfigurationImporter;
 import password.pwm.util.PropertyConfigurationImporter;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.logging.PwmLogger;
@@ -274,8 +275,8 @@ public class ContextManager implements Serializable
         }
         }
 
 
         taskMaster = Executors.newSingleThreadScheduledExecutor(
         taskMaster = Executors.newSingleThreadScheduledExecutor(
-                JavaHelper.makePwmThreadFactory(
-                        JavaHelper.makeThreadName( pwmApplication, this.getClass() ) + "-",
+                PwmScheduler.makePwmThreadFactory(
+                        PwmScheduler.makeThreadName( pwmApplication, this.getClass() ) + "-",
                         true
                         true
                 ) );
                 ) );
 
 

+ 2 - 2
server/src/main/java/password/pwm/http/servlet/peoplesearch/PeopleSearchService.java

@@ -26,7 +26,7 @@ import password.pwm.PwmApplication;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmException;
 import password.pwm.health.HealthRecord;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmService;
-import password.pwm.util.java.JavaHelper;
+import password.pwm.util.PwmScheduler;
 
 
 import java.util.List;
 import java.util.List;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ArrayBlockingQueue;
@@ -52,7 +52,7 @@ public class PeopleSearchService implements PwmService
 
 
         final int maxThreadCount = 5;
         final int maxThreadCount = 5;
 
 
-        final ThreadFactory threadFactory = JavaHelper.makePwmThreadFactory( JavaHelper.makeThreadName( pwmApplication, PeopleSearchService.class ), true );
+        final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( pwmApplication, PeopleSearchService.class ), true );
         threadPoolExecutor = new ThreadPoolExecutor(
         threadPoolExecutor = new ThreadPoolExecutor(
                 maxThreadCount,
                 maxThreadCount,
                 maxThreadCount,
                 maxThreadCount,

+ 2 - 1
server/src/main/java/password/pwm/ldap/search/UserSearchEngine.java

@@ -47,6 +47,7 @@ import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.Statistic;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.ConditionalTaskExecutor;
 import password.pwm.util.java.ConditionalTaskExecutor;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.JsonUtil;
@@ -844,7 +845,7 @@ public class UserSearchEngine implements PwmService
             final int factor = Integer.parseInt( configuration.readAppProperty( AppProperty.LDAP_SEARCH_PARALLEL_FACTOR ) );
             final int factor = Integer.parseInt( configuration.readAppProperty( AppProperty.LDAP_SEARCH_PARALLEL_FACTOR ) );
             final int maxThreads = Integer.parseInt( configuration.readAppProperty( AppProperty.LDAP_SEARCH_PARALLEL_THREAD_MAX ) );
             final int maxThreads = Integer.parseInt( configuration.readAppProperty( AppProperty.LDAP_SEARCH_PARALLEL_THREAD_MAX ) );
             final int threads = Math.min( maxThreads, ( endPoints ) * factor );
             final int threads = Math.min( maxThreads, ( endPoints ) * factor );
-            final ThreadFactory threadFactory = JavaHelper.makePwmThreadFactory( JavaHelper.makeThreadName( pwmApplication, UserSearchEngine.class ), true );
+            final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( pwmApplication, UserSearchEngine.class ), true );
             return new ThreadPoolExecutor(
             return new ThreadPoolExecutor(
                     threads,
                     threads,
                     threads,
                     threads,

+ 3 - 3
server/src/main/java/password/pwm/svc/event/LocalDbAuditVault.java

@@ -26,8 +26,8 @@ import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.PwmApplication;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmException;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmService;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.TransactionSizeCalculator;
 import password.pwm.util.TransactionSizeCalculator;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.Percent;
 import password.pwm.util.java.Percent;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.java.TimeDuration;
@@ -74,11 +74,11 @@ public class LocalDbAuditVault implements AuditVault
 
 
         readOldestRecord();
         readOldestRecord();
 
 
-        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
 
 
         status = PwmService.STATUS.OPEN;
         status = PwmService.STATUS.OPEN;
         final TimeDuration jobFrequency = TimeDuration.of( 10, TimeDuration.Unit.MINUTES );
         final TimeDuration jobFrequency = TimeDuration.of( 10, TimeDuration.Unit.MINUTES );
-        pwmApplication.scheduleFixedRateJob( new TrimmerThread(), executorService, TimeDuration.SECONDS_10, jobFrequency );
+        pwmApplication.getPwmScheduler().scheduleFixedRateJob( new TrimmerThread(), executorService, TimeDuration.SECONDS_10, jobFrequency );
     }
     }
 
 
     public void close( )
     public void close( )

+ 2 - 1
server/src/main/java/password/pwm/svc/intruder/IntruderManager.java

@@ -53,6 +53,7 @@ import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.DataStore;
 import password.pwm.util.DataStore;
 import password.pwm.util.DataStoreFactory;
 import password.pwm.util.DataStoreFactory;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.db.DatabaseDataStore;
 import password.pwm.util.db.DatabaseDataStore;
 import password.pwm.util.db.DatabaseTable;
 import password.pwm.util.db.DatabaseTable;
@@ -171,7 +172,7 @@ public class IntruderManager implements PwmService
         final RecordStore recordStore;
         final RecordStore recordStore;
         {
         {
             recordStore = new DataStoreRecordStore( dataStore, this );
             recordStore = new DataStoreRecordStore( dataStore, this );
-            final String threadName = JavaHelper.makeThreadName( pwmApplication, this.getClass() ) + " timer";
+            final String threadName = PwmScheduler.makeThreadName( pwmApplication, this.getClass() ) + " timer";
             timer = new Timer( threadName, true );
             timer = new Timer( threadName, true );
             final long maxRecordAge = Long.parseLong( pwmApplication.getConfig().readAppProperty( AppProperty.INTRUDER_RETENTION_TIME_MS ) );
             final long maxRecordAge = Long.parseLong( pwmApplication.getConfig().readAppProperty( AppProperty.INTRUDER_RETENTION_TIME_MS ) );
             final long cleanerRunFrequency = Long.parseLong( pwmApplication.getConfig().readAppProperty( AppProperty.INTRUDER_CLEANUP_FREQUENCY_MS ) );
             final long cleanerRunFrequency = Long.parseLong( pwmApplication.getConfig().readAppProperty( AppProperty.INTRUDER_CLEANUP_FREQUENCY_MS ) );

+ 3 - 2
server/src/main/java/password/pwm/svc/node/NodeMachine.java

@@ -27,6 +27,7 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.logging.PwmLogger;
@@ -65,9 +66,9 @@ class NodeMachine
         this.clusterDataServiceProvider = clusterDataServiceProvider;
         this.clusterDataServiceProvider = clusterDataServiceProvider;
         this.settings = nodeServiceSettings;
         this.settings = nodeServiceSettings;
 
 
-        this.executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, NodeMachine.class );
+        this.executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, NodeMachine.class );
 
 
-        pwmApplication.scheduleFixedRateJob( new HeartbeatProcess(), executorService, settings.getHeartbeatInterval(), settings.getHeartbeatInterval() );
+        pwmApplication.getPwmScheduler().scheduleFixedRateJob( new HeartbeatProcess(), executorService, settings.getHeartbeatInterval(), settings.getHeartbeatInterval() );
     }
     }
 
 
     public void close( )
     public void close( )

+ 2 - 1
server/src/main/java/password/pwm/svc/pwnotify/PwNotifyEngine.java

@@ -37,6 +37,7 @@ import password.pwm.ldap.UserInfo;
 import password.pwm.ldap.UserInfoFactory;
 import password.pwm.ldap.UserInfoFactory;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.svc.stats.StatisticsManager;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.ConditionalTaskExecutor;
 import password.pwm.util.java.ConditionalTaskExecutor;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
@@ -377,7 +378,7 @@ public class PwNotifyEngine
 
 
     private ThreadPoolExecutor createExecutor( final PwmApplication pwmApplication )
     private ThreadPoolExecutor createExecutor( final PwmApplication pwmApplication )
     {
     {
-        final ThreadFactory threadFactory = JavaHelper.makePwmThreadFactory( JavaHelper.makeThreadName( pwmApplication, this.getClass() ), true );
+        final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( pwmApplication, this.getClass() ), true );
         final ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(
         final ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(
                 1,
                 1,
                 10,
                 10,

+ 5 - 4
server/src/main/java/password/pwm/svc/pwnotify/PwNotifyService.java

@@ -37,6 +37,7 @@ import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.svc.stats.StatisticsManager;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.java.TimeDuration;
@@ -139,11 +140,11 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
                     JavaHelper.unhandledSwitchStatement( storageMethod );
                     JavaHelper.unhandledSwitchStatement( storageMethod );
             }
             }
 
 
-            executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
+            executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
 
 
             engine = new PwNotifyEngine( pwmApplication, storageService, () -> status() == STATUS.CLOSED, null );
             engine = new PwNotifyEngine( pwmApplication, storageService, () -> status() == STATUS.CLOSED, null );
 
 
-            pwmApplication.scheduleFixedRateJob( new PwNotifyJob(), executorService, TimeDuration.MINUTE, TimeDuration.MINUTE );
+            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new PwNotifyJob(), executorService, TimeDuration.MINUTE, TimeDuration.MINUTE );
 
 
             setStatus( STATUS.OPEN );
             setStatus( STATUS.OPEN );
         }
         }
@@ -193,7 +194,7 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
             }
             }
         }
         }
 
 
-        final Instant nextZuluZeroTime = JavaHelper.nextZuluZeroTime();
+        final Instant nextZuluZeroTime = PwmScheduler.nextZuluZeroTime();
         final Instant adjustedNextZuluZeroTime = nextZuluZeroTime.plus( settings.getZuluOffset().as( TimeDuration.Unit.SECONDS ), ChronoUnit.SECONDS );
         final Instant adjustedNextZuluZeroTime = nextZuluZeroTime.plus( settings.getZuluOffset().as( TimeDuration.Unit.SECONDS ), ChronoUnit.SECONDS );
         final Instant previousAdjustedZuluZeroTime = adjustedNextZuluZeroTime.minus( 1, ChronoUnit.DAYS );
         final Instant previousAdjustedZuluZeroTime = adjustedNextZuluZeroTime.minus( 1, ChronoUnit.DAYS );
 
 
@@ -258,7 +259,7 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
         if ( !isRunning() )
         if ( !isRunning() )
         {
         {
             nextExecutionTime = Instant.now();
             nextExecutionTime = Instant.now();
-            pwmApplication.scheduleFutureJob( new PwNotifyJob(), executorService, TimeDuration.ZERO );
+            pwmApplication.getPwmScheduler().scheduleFutureJob( new PwNotifyJob(), executorService, TimeDuration.ZERO );
         }
         }
     }
     }
 
 

+ 6 - 8
server/src/main/java/password/pwm/svc/report/ReportService.java

@@ -41,6 +41,7 @@ import password.pwm.ldap.UserInfo;
 import password.pwm.ldap.UserInfoFactory;
 import password.pwm.ldap.UserInfoFactory;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.EventRateMeter;
 import password.pwm.svc.stats.EventRateMeter;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.TransactionSizeCalculator;
 import password.pwm.util.TransactionSizeCalculator;
 import password.pwm.util.java.BlockingThreadPool;
 import password.pwm.util.java.BlockingThreadPool;
 import password.pwm.util.java.ClosableIterator;
 import password.pwm.util.java.ClosableIterator;
@@ -142,7 +143,7 @@ public class ReportService implements PwmService
 
 
         dnQueue = LocalDBStoredQueue.createLocalDBStoredQueue( pwmApplication, pwmApplication.getLocalDB(), LocalDB.DB.REPORT_QUEUE );
         dnQueue = LocalDBStoredQueue.createLocalDBStoredQueue( pwmApplication, pwmApplication.getLocalDB(), LocalDB.DB.REPORT_QUEUE );
 
 
-        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
 
 
         LOGGER.debug( () -> "report service started" );
         LOGGER.debug( () -> "report service started" );
 
 
@@ -397,7 +398,7 @@ public class ReportService implements PwmService
                         if ( executorService != null )
                         if ( executorService != null )
                         {
                         {
                             LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, "directory unavailable error during background SearchLDAP, will retry; error: " + e.getMessage() );
                             LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, "directory unavailable error during background SearchLDAP, will retry; error: " + e.getMessage() );
-                            pwmApplication.scheduleFutureJob( new ReadLDAPTask(), executorService, TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
+                            pwmApplication.getPwmScheduler().scheduleFutureJob( new ReadLDAPTask(), executorService, TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
                             errorProcessed = true;
                             errorProcessed = true;
                         }
                         }
                     }
                     }
@@ -476,7 +477,7 @@ public class ReportService implements PwmService
                         if ( executorService != null )
                         if ( executorService != null )
                         {
                         {
                             LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, "directory unavailable error during background ReadData, will retry; error: " + e.getMessage() );
                             LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, "directory unavailable error during background ReadData, will retry; error: " + e.getMessage() );
-                            pwmApplication.scheduleFutureJob( new ProcessWorkQueueTask(), executorService, TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
+                            pwmApplication.getPwmScheduler().scheduleFutureJob( new ProcessWorkQueueTask(), executorService, TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
                         }
                         }
                     }
                     }
                     else
                     else
@@ -664,11 +665,8 @@ public class ReportService implements PwmService
             final boolean reportingEnabled = pwmApplication.getConfig().readSettingAsBoolean( PwmSetting.REPORTING_ENABLE_DAILY_JOB );
             final boolean reportingEnabled = pwmApplication.getConfig().readSettingAsBoolean( PwmSetting.REPORTING_ENABLE_DAILY_JOB );
             if ( reportingEnabled )
             if ( reportingEnabled )
             {
             {
-                final Instant nextZuluZeroTime = JavaHelper.nextZuluZeroTime();
-                final long secondsUntilNextDredge = settings.getJobOffsetSeconds() + TimeDuration.fromCurrent( nextZuluZeroTime ).as( TimeDuration.Unit.SECONDS );
-                final TimeDuration initialDelay = TimeDuration.of( secondsUntilNextDredge, TimeDuration.Unit.SECONDS );
-                pwmApplication.scheduleFixedRateJob( new DailyJobExecuteTask(), executorService, initialDelay, TimeDuration.DAY );
-                LOGGER.debug( () -> "scheduled daily execution, next task will be at " + nextZuluZeroTime.toString() );
+                final TimeDuration jobOffset = TimeDuration.of( settings.getJobOffsetSeconds(), TimeDuration.Unit.SECONDS );
+                pwmApplication.getPwmScheduler().scheduleDailyZuluZeroStartJob( new DailyJobExecuteTask(), executorService, jobOffset );
             }
             }
         }
         }
 
 

+ 4 - 4
server/src/main/java/password/pwm/svc/stats/StatisticsManager.java

@@ -30,6 +30,7 @@ import password.pwm.error.PwmException;
 import password.pwm.health.HealthRecord;
 import password.pwm.health.HealthRecord;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmRequest;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmService;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.java.TimeDuration;
@@ -329,10 +330,9 @@ public class StatisticsManager implements PwmService
 
 
         {
         {
             // setup a timer to roll over at 0 Zulu and one to write current stats regularly
             // setup a timer to roll over at 0 Zulu and one to write current stats regularly
-            executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
-            pwmApplication.scheduleFixedRateJob( new FlushTask(), executorService, DB_WRITE_FREQUENCY, DB_WRITE_FREQUENCY );
-            final TimeDuration delayTillNextZulu = TimeDuration.fromCurrent( JavaHelper.nextZuluZeroTime() );
-            pwmApplication.scheduleFixedRateJob( new NightlyTask(), executorService, delayTillNextZulu, TimeDuration.DAY );
+            executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
+            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new FlushTask(), executorService, DB_WRITE_FREQUENCY, DB_WRITE_FREQUENCY );
+            pwmApplication.getPwmScheduler().scheduleDailyZuluZeroStartJob( new NightlyTask(), executorService, TimeDuration.ZERO );
         }
         }
 
 
         status = STATUS.OPEN;
         status = STATUS.OPEN;

+ 3 - 2
server/src/main/java/password/pwm/svc/telemetry/TelemetryService.java

@@ -45,6 +45,7 @@ import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsBundle;
 import password.pwm.svc.stats.StatisticsBundle;
 import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.svc.stats.StatisticsManager;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.StringUtil;
@@ -143,7 +144,7 @@ public class TelemetryService implements PwmService
             LOGGER.trace( SessionLabel.TELEMETRY_SESSION_LABEL, () -> "last publish time was " + JavaHelper.toIsoDate( lastPublishTime ) );
             LOGGER.trace( SessionLabel.TELEMETRY_SESSION_LABEL, () -> "last publish time was " + JavaHelper.toIsoDate( lastPublishTime ) );
         }
         }
 
 
-        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, TelemetryService.class );
+        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, TelemetryService.class );
 
 
         scheduleNextJob();
         scheduleNextJob();
 
 
@@ -214,7 +215,7 @@ public class TelemetryService implements PwmService
     private void scheduleNextJob( )
     private void scheduleNextJob( )
     {
     {
         final TimeDuration durationUntilNextPublish = durationUntilNextPublish();
         final TimeDuration durationUntilNextPublish = durationUntilNextPublish();
-        pwmApplication.scheduleFutureJob( new PublishJob(), executorService, durationUntilNextPublish );
+        pwmApplication.getPwmScheduler().scheduleFutureJob( new PublishJob(), executorService, durationUntilNextPublish );
         LOGGER.trace( SessionLabel.TELEMETRY_SESSION_LABEL, () -> "next publish time: " + durationUntilNextPublish().asCompactString() );
         LOGGER.trace( SessionLabel.TELEMETRY_SESSION_LABEL, () -> "next publish time: " + durationUntilNextPublish().asCompactString() );
     }
     }
 
 

+ 3 - 2
server/src/main/java/password/pwm/svc/token/TokenService.java

@@ -57,6 +57,7 @@ import password.pwm.svc.intruder.RecordType;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.DataStore;
 import password.pwm.util.DataStore;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.db.DatabaseDataStore;
 import password.pwm.util.db.DatabaseDataStore;
 import password.pwm.util.db.DatabaseTable;
 import password.pwm.util.db.DatabaseTable;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
@@ -194,12 +195,12 @@ public class TokenService implements PwmService
 
 
         verifyPwModifyTime = Boolean.parseBoolean( configuration.readAppProperty( AppProperty.TOKEN_VERIFY_PW_MODIFY_TIME ) );
         verifyPwModifyTime = Boolean.parseBoolean( configuration.readAppProperty( AppProperty.TOKEN_VERIFY_PW_MODIFY_TIME ) );
 
 
-        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
 
 
         {
         {
             final int cleanerFrequencySeconds = Integer.parseInt( configuration.readAppProperty( AppProperty.TOKEN_CLEANER_INTERVAL_SECONDS ) );
             final int cleanerFrequencySeconds = Integer.parseInt( configuration.readAppProperty( AppProperty.TOKEN_CLEANER_INTERVAL_SECONDS ) );
             final TimeDuration cleanerFrequency = TimeDuration.of( cleanerFrequencySeconds, TimeDuration.Unit.SECONDS );
             final TimeDuration cleanerFrequency = TimeDuration.of( cleanerFrequencySeconds, TimeDuration.Unit.SECONDS );
-            pwmApplication.scheduleFixedRateJob( new CleanerTask(), executorService, TimeDuration.MINUTE, cleanerFrequency );
+            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new CleanerTask(), executorService, TimeDuration.MINUTE, cleanerFrequency );
             LOGGER.trace( () -> "token cleanup will occur every " + cleanerFrequency.asCompactString() );
             LOGGER.trace( () -> "token cleanup will occur every " + cleanerFrequency.asCompactString() );
         }
         }
 
 

+ 3 - 2
server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java

@@ -35,6 +35,7 @@ import password.pwm.health.HealthRecord;
 import password.pwm.health.HealthStatus;
 import password.pwm.health.HealthStatus;
 import password.pwm.health.HealthTopic;
 import password.pwm.health.HealthTopic;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmService;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.localdb.LocalDBException;
 import password.pwm.util.localdb.LocalDBException;
@@ -93,7 +94,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
 
 
         this.wordklistBucket = new WordlistBucket( pwmApplication, wordlistConfiguration, type );
         this.wordklistBucket = new WordlistBucket( pwmApplication, wordlistConfiguration, type );
 
 
-        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
 
 
         if ( pwmApplication.getLocalDB() != null )
         if ( pwmApplication.getLocalDB() != null )
         {
         {
@@ -107,7 +108,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
             lastError = new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, errorMsg );
             lastError = new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, errorMsg );
         }
         }
 
 
-        pwmApplication.scheduleFixedRateJob( new InspectorJob(), executorService, TimeDuration.SECOND, wordlistConfiguration.getInspectorFrequency() );
+        pwmApplication.getPwmScheduler().scheduleFixedRateJob( new InspectorJob(), executorService, TimeDuration.SECOND, wordlistConfiguration.getInspectorFrequency() );
     }
     }
 
 
     boolean containsWord( final String word ) throws PwmUnrecoverableException
     boolean containsWord( final String word ) throws PwmUnrecoverableException

+ 4 - 3
server/src/main/java/password/pwm/svc/wordlist/SharedHistoryManager.java

@@ -32,6 +32,7 @@ import password.pwm.config.option.DataStorageMethod;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmException;
 import password.pwm.health.HealthRecord;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmService;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.Sleeper;
 import password.pwm.util.java.Sleeper;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.java.TimeDuration;
@@ -252,8 +253,8 @@ public class SharedHistoryManager implements PwmService
             final TimeDuration frequency = TimeDuration.of( frequencyMs, TimeDuration.Unit.MILLISECONDS );
             final TimeDuration frequency = TimeDuration.of( frequencyMs, TimeDuration.Unit.MILLISECONDS );
 
 
             LOGGER.debug( () -> "scheduling cleaner task to run once every " + frequency.asCompactString() );
             LOGGER.debug( () -> "scheduling cleaner task to run once every " + frequency.asCompactString() );
-            executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
-            pwmApplication.scheduleFixedRateJob( new CleanerTask(), executorService, null, frequency );
+            executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
+            pwmApplication.getPwmScheduler().scheduleFixedRateJob( new CleanerTask(), executorService, null, frequency );
         }
         }
     }
     }
 
 
@@ -497,7 +498,7 @@ public class SharedHistoryManager implements PwmService
                 LOGGER.debug( () -> "starting up in background thread" );
                 LOGGER.debug( () -> "starting up in background thread" );
                 init( pwmApplication, settings.maxAgeMs );
                 init( pwmApplication, settings.maxAgeMs );
             }
             }
-        }, JavaHelper.makeThreadName( pwmApplication, this.getClass() ) + " initializer" ).start();
+        }, PwmScheduler.makeThreadName( pwmApplication, this.getClass() ) + " initializer" ).start();
     }
     }
 
 
     private static class Settings
     private static class Settings

+ 302 - 0
server/src/main/java/password/pwm/util/PwmScheduler.java

@@ -0,0 +1,302 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.util;
+
+import org.jetbrains.annotations.NotNull;
+import password.pwm.PwmApplication;
+import password.pwm.PwmConstants;
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogger;
+
+import java.time.Instant;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+import java.util.Objects;
+import java.util.TimeZone;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+public class PwmScheduler
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( PwmScheduler.class );
+    private final ScheduledExecutorService applicationExecutorService;
+    private final String instanceID;
+
+    public PwmScheduler( final String instanceID )
+    {
+        this.instanceID = instanceID;
+        applicationExecutorService = makeSingleThreadExecutorService( instanceID, this.getClass() );
+    }
+
+    public void shutdown()
+    {
+        applicationExecutorService.shutdown();
+    }
+
+    public Future immediateExecuteInNewThread(
+            final Runnable runnable
+    )
+    {
+        Objects.requireNonNull( runnable );
+
+        final ExecutorService executor = makeSingleThreadExecutorService( instanceID, runnable.getClass() );
+
+        if ( applicationExecutorService.isShutdown() )
+        {
+            return null;
+        }
+
+        final WrappedRunner wrappedRunner = new WrappedRunner( runnable, executor );
+        applicationExecutorService.schedule( wrappedRunner, 0, TimeUnit.MILLISECONDS );
+        executor.shutdown();
+        return wrappedRunner.getFuture();
+    }
+
+    public void scheduleDailyZuluZeroStartJob(
+            final Runnable runnable,
+            final ExecutorService executorService,
+            final TimeDuration offset
+    )
+    {
+        final TimeDuration delayTillNextZulu = TimeDuration.fromCurrent( nextZuluZeroTime() );
+        final TimeDuration delayTillNextOFfiset = delayTillNextZulu.add( offset );
+        scheduleFixedRateJob( runnable, executorService, delayTillNextOFfiset, TimeDuration.DAY );
+    }
+
+    public Future scheduleFutureJob(
+            final Runnable runnable,
+            final ExecutorService executor,
+            final TimeDuration delay
+    )
+    {
+        Objects.requireNonNull( runnable );
+        Objects.requireNonNull( executor );
+        Objects.requireNonNull( delay );
+
+        if ( applicationExecutorService.isShutdown() )
+        {
+            return null;
+        }
+
+        final WrappedRunner wrappedRunner = new WrappedRunner( runnable, executor );
+        applicationExecutorService.schedule( wrappedRunner, delay.asMillis(), TimeUnit.MILLISECONDS );
+        return wrappedRunner.getFuture();
+    }
+
+    public void scheduleFixedRateJob(
+            final Runnable runnable,
+            final ExecutorService executor,
+            final TimeDuration initialDelay,
+            final TimeDuration frequency
+    )
+    {
+        if ( applicationExecutorService.isShutdown() )
+        {
+            return;
+        }
+
+        if ( initialDelay != null )
+        {
+            applicationExecutorService.schedule( new WrappedRunner( runnable, executor ), initialDelay.asMillis(), TimeUnit.MILLISECONDS );
+        }
+
+        final Runnable jobWithNextScheduler = () ->
+        {
+            new WrappedRunner( runnable, executor ).run();
+            scheduleFixedRateJob( runnable, executor, null, frequency );
+        };
+
+        applicationExecutorService.schedule(  jobWithNextScheduler, frequency.asMillis(), TimeUnit.MILLISECONDS );
+    }
+
+    public static ExecutorService makeBackgroundExecutor(
+            final PwmApplication pwmApplication,
+            final Class clazz
+    )
+    {
+        final ThreadPoolExecutor executor = new ThreadPoolExecutor(
+                1,
+                1,
+                10, TimeUnit.SECONDS,
+                new LinkedBlockingQueue<>(),
+                makePwmThreadFactory(
+                        makeThreadName( pwmApplication, clazz ) + "-",
+                        true
+                ) );
+        executor.allowCoreThreadTimeOut( true );
+        return executor;
+    }
+
+
+    public static String makeThreadName( final PwmApplication pwmApplication, final Class theClass )
+    {
+        String instanceName = "-";
+        if ( pwmApplication != null )
+        {
+            instanceName = pwmApplication.getInstanceID();
+        }
+
+        return makeThreadName( instanceName, theClass );
+    }
+
+    public static String makeThreadName( final String instanceID, final Class theClass )
+    {
+        String instanceName = "-";
+        if ( !StringUtil.isEmpty( instanceID ) )
+        {
+            instanceName = instanceID;
+        }
+
+        return PwmConstants.PWM_APP_NAME + "-" + instanceName + "-" + theClass.getSimpleName();
+    }
+
+    public static ThreadFactory makePwmThreadFactory( final String namePrefix, final boolean daemon )
+    {
+        return new ThreadFactory()
+        {
+            private final ThreadFactory realThreadFactory = Executors.defaultThreadFactory();
+
+            @Override
+            public Thread newThread( final Runnable runnable )
+            {
+                final Thread t = realThreadFactory.newThread( runnable );
+                t.setDaemon( daemon );
+                if ( namePrefix != null )
+                {
+                    final String newName = namePrefix + t.getName();
+                    t.setName( newName );
+                }
+                return t;
+            }
+        };
+    }
+
+    public static ScheduledExecutorService makeSingleThreadExecutorService(
+            final PwmApplication pwmApplication,
+            final Class theClass
+    )
+    {
+        return makeSingleThreadExecutorService( pwmApplication.getInstanceID(), theClass );
+    }
+
+    public static ScheduledExecutorService makeSingleThreadExecutorService(
+            final String instanceID,
+            final Class theClass
+    )
+    {
+        return Executors.newSingleThreadScheduledExecutor(
+                makePwmThreadFactory(
+                        makeThreadName( instanceID, theClass ) + "-",
+                        true
+                ) );
+    }
+
+    private static class WrappedRunner implements Runnable
+    {
+        private final Runnable runnable;
+        private final ExecutorService executor;
+        private volatile Future innerFuture;
+        private volatile boolean hasFailed;
+
+        WrappedRunner( final Runnable runnable, final ExecutorService executor )
+        {
+            this.runnable = runnable;
+            this.executor = executor;
+        }
+
+        Future getFuture()
+        {
+            return new Future()
+            {
+                @Override
+                public boolean cancel( final boolean mayInterruptIfRunning )
+                {
+                    return false;
+                }
+
+                @Override
+                public boolean isCancelled()
+                {
+                    return hasFailed;
+                }
+
+                @Override
+                public boolean isDone()
+                {
+                    return hasFailed || innerFuture != null && innerFuture.isDone();
+                }
+
+                @Override
+                public Object get()
+                {
+                    return null;
+                }
+
+                @Override
+                public Object get( final long timeout, @NotNull final TimeUnit unit )
+                {
+                    return null;
+                }
+            };
+        }
+
+        @Override
+        public void run()
+        {
+            try
+            {
+                if ( !executor.isShutdown() )
+                {
+                    innerFuture = executor.submit( runnable );
+                }
+                else
+                {
+                    hasFailed = true;
+                    LOGGER.trace( () -> "skipping scheduled job " + runnable + " on shutdown executor + " + executor );
+                }
+            }
+            catch ( Throwable t )
+            {
+                LOGGER.error( "unexpected error running scheduled job: " + t.getMessage(), t );
+                hasFailed = true;
+            }
+        }
+    }
+
+    public static Instant nextZuluZeroTime( )
+    {
+        final Calendar nextZuluMidnight = GregorianCalendar.getInstance( TimeZone.getTimeZone( "Zulu" ) );
+        nextZuluMidnight.set( Calendar.HOUR_OF_DAY, 0 );
+        nextZuluMidnight.set( Calendar.MINUTE, 0 );
+        nextZuluMidnight.set( Calendar.SECOND, 0 );
+        nextZuluMidnight.add( Calendar.HOUR, 24 );
+        return nextZuluMidnight.getTime().toInstant();
+    }
+}

+ 3 - 2
server/src/main/java/password/pwm/util/db/DatabaseService.java

@@ -39,6 +39,7 @@ import password.pwm.health.HealthTopic;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.EpsStatistic;
 import password.pwm.svc.stats.EpsStatistic;
 import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.svc.stats.StatisticsManager;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.JsonUtil;
@@ -110,12 +111,12 @@ public class DatabaseService implements PwmService
         this.pwmApplication = pwmApplication;
         this.pwmApplication = pwmApplication;
         init();
         init();
 
 
-        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
 
 
         final TimeDuration watchdogFrequency = TimeDuration.of(
         final TimeDuration watchdogFrequency = TimeDuration.of(
                 Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.DB_CONNECTIONS_WATCHDOG_FREQUENCY_SECONDS ) ),
                 Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.DB_CONNECTIONS_WATCHDOG_FREQUENCY_SECONDS ) ),
                 TimeDuration.Unit.SECONDS );
                 TimeDuration.Unit.SECONDS );
-        pwmApplication.scheduleFixedRateJob( new ConnectionMonitor(), executorService, watchdogFrequency, watchdogFrequency );
+        pwmApplication.getPwmScheduler().scheduleFixedRateJob( new ConnectionMonitor(), executorService, watchdogFrequency, watchdogFrequency );
     }
     }
 
 
     private synchronized void init( )
     private synchronized void init( )

+ 3 - 1
server/src/main/java/password/pwm/util/java/BlockingThreadPool.java

@@ -22,6 +22,8 @@
 
 
 package password.pwm.util.java;
 package password.pwm.util.java;
 
 
+import password.pwm.util.PwmScheduler;
+
 import java.util.concurrent.Future;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.Semaphore;
@@ -35,7 +37,7 @@ public class BlockingThreadPool extends ThreadPoolExecutor
 
 
     public BlockingThreadPool( final int bound, final String name )
     public BlockingThreadPool( final int bound, final String name )
     {
     {
-        super( bound, bound, 0, TimeUnit.MILLISECONDS, new LinkedBlockingDeque<>(), JavaHelper.makePwmThreadFactory( name, true ) );
+        super( bound, bound, 0, TimeUnit.MILLISECONDS, new LinkedBlockingDeque<>(), PwmScheduler.makePwmThreadFactory( name, true ) );
         semaphore = new Semaphore( bound );
         semaphore = new Semaphore( bound );
     }
     }
 
 

+ 0 - 81
server/src/main/java/password/pwm/util/java/JavaHelper.java

@@ -26,7 +26,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import jetbrains.exodus.core.dataStructures.hash.LinkedHashMap;
 import jetbrains.exodus.core.dataStructures.hash.LinkedHashMap;
 import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.IOUtils;
-import password.pwm.PwmApplication;
 import password.pwm.PwmConstants;
 import password.pwm.PwmConstants;
 import password.pwm.config.Configuration;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.PwmSetting;
@@ -54,13 +53,11 @@ import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Arrays;
-import java.util.Calendar;
 import java.util.Collection;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.Date;
 import java.util.Enumeration;
 import java.util.Enumeration;
-import java.util.GregorianCalendar;
 import java.util.LinkedHashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.List;
 import java.util.Map;
 import java.util.Map;
@@ -68,13 +65,7 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.Optional;
 import java.util.Properties;
 import java.util.Properties;
 import java.util.Set;
 import java.util.Set;
-import java.util.TimeZone;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Predicate;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Collectors;
@@ -208,16 +199,6 @@ public class JavaHelper
         return new String( chars );
         return new String( chars );
     }
     }
 
 
-    public static Instant nextZuluZeroTime( )
-    {
-        final Calendar nextZuluMidnight = GregorianCalendar.getInstance( TimeZone.getTimeZone( "Zulu" ) );
-        nextZuluMidnight.set( Calendar.HOUR_OF_DAY, 0 );
-        nextZuluMidnight.set( Calendar.MINUTE, 0 );
-        nextZuluMidnight.set( Calendar.SECOND, 0 );
-        nextZuluMidnight.add( Calendar.HOUR, 24 );
-        return nextZuluMidnight.getTime().toInstant();
-    }
-
     public static <E extends Enum<E>> List<E> readEnumListFromStringCollection( final Class<E> enumClass, final Collection<String> inputs )
     public static <E extends Enum<E>> List<E> readEnumListFromStringCollection( final Class<E> enumClass, final Collection<String> inputs )
     {
     {
         final List<E> returnList = new ArrayList<E>();
         final List<E> returnList = new ArrayList<E>();
@@ -420,38 +401,6 @@ public class JavaHelper
         return false;
         return false;
     }
     }
 
 
-    public static String makeThreadName( final PwmApplication pwmApplication, final Class theClass )
-    {
-        String instanceName = "-";
-        if ( pwmApplication != null && pwmApplication.getInstanceID() != null )
-        {
-            instanceName = pwmApplication.getInstanceID();
-        }
-
-        return PwmConstants.PWM_APP_NAME + "-" + instanceName + "-" + theClass.getSimpleName();
-    }
-
-    public static ThreadFactory makePwmThreadFactory( final String namePrefix, final boolean daemon )
-    {
-        return new ThreadFactory()
-        {
-            private final ThreadFactory realThreadFactory = Executors.defaultThreadFactory();
-
-            @Override
-            public Thread newThread( final Runnable r )
-            {
-                final Thread t = realThreadFactory.newThread( r );
-                t.setDaemon( daemon );
-                if ( namePrefix != null )
-                {
-                    final String newName = namePrefix + t.getName();
-                    t.setName( newName );
-                }
-                return t;
-            }
-        };
-    }
-
     public static Collection<Method> getAllMethodsForClass( final Class clazz )
     public static Collection<Method> getAllMethodsForClass( final Class clazz )
     {
     {
         final LinkedHashSet<Method> methods = new LinkedHashSet<>();
         final LinkedHashSet<Method> methods = new LinkedHashSet<>();
@@ -474,36 +423,6 @@ public class JavaHelper
         return new CSVPrinter( new OutputStreamWriter( outputStream, PwmConstants.DEFAULT_CHARSET ), PwmConstants.DEFAULT_CSV_FORMAT );
         return new CSVPrinter( new OutputStreamWriter( outputStream, PwmConstants.DEFAULT_CHARSET ), PwmConstants.DEFAULT_CSV_FORMAT );
     }
     }
 
 
-    public static ScheduledExecutorService makeSingleThreadExecutorService(
-            final PwmApplication pwmApplication,
-            final Class clazz
-    )
-    {
-        return Executors.newSingleThreadScheduledExecutor(
-                makePwmThreadFactory(
-                        JavaHelper.makeThreadName( pwmApplication, clazz ) + "-",
-                        true
-                ) );
-    }
-
-    public static ExecutorService makeBackgroundExecutor(
-            final PwmApplication pwmApplication,
-            final Class clazz
-    )
-    {
-        final ThreadPoolExecutor executor = new ThreadPoolExecutor(
-                1,
-                1,
-                10, TimeUnit.SECONDS,
-                new LinkedBlockingQueue<>(),
-                JavaHelper.makePwmThreadFactory(
-                        JavaHelper.makeThreadName( pwmApplication, clazz ) + "-",
-                        true
-                ) );
-        executor.allowCoreThreadTimeOut( true );
-        return executor;
-    }
-
     /**
     /**
      * Copy of {@link ThreadInfo#toString()} but with the MAX_FRAMES changed from 8 to 256.
      * Copy of {@link ThreadInfo#toString()} but with the MAX_FRAMES changed from 8 to 256.
      * @param threadInfo thread information
      * @param threadInfo thread information

+ 3 - 2
server/src/main/java/password/pwm/util/localdb/WorkQueueProcessor.java

@@ -30,6 +30,7 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.svc.stats.EventRateMeter;
 import password.pwm.svc.stats.EventRateMeter;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.JsonUtil;
@@ -113,12 +114,12 @@ public final class WorkQueueProcessor<W extends Serializable>
 
 
         this.workerThread = new WorkerThread();
         this.workerThread = new WorkerThread();
         workerThread.setDaemon( true );
         workerThread.setDaemon( true );
-        workerThread.setName( JavaHelper.makeThreadName( pwmApplication, sourceClass ) + "-worker-" );
+        workerThread.setName( PwmScheduler.makeThreadName( pwmApplication, sourceClass ) + "-worker-" );
         workerThread.start();
         workerThread.start();
 
 
         if ( settings.getPreThreads() > 0 )
         if ( settings.getPreThreads() > 0 )
         {
         {
-            final ThreadFactory threadFactory = JavaHelper.makePwmThreadFactory( JavaHelper.makeThreadName( pwmApplication, sourceClass ), true );
+            final ThreadFactory threadFactory = PwmScheduler.makePwmThreadFactory( PwmScheduler.makeThreadName( pwmApplication, sourceClass ), true );
             executorService = new ThreadPoolExecutor(
             executorService = new ThreadPoolExecutor(
                     1,
                     1,
                     settings.getPreThreads(),
                     settings.getPreThreads(),

+ 5 - 4
server/src/main/java/password/pwm/util/logging/LocalDBLogger.java

@@ -28,6 +28,7 @@ import password.pwm.error.PwmException;
 import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.PwmService;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.PwmNumberFormat;
 import password.pwm.util.java.PwmNumberFormat;
@@ -124,14 +125,14 @@ public class LocalDBLogger implements PwmService
         status = STATUS.OPEN;
         status = STATUS.OPEN;
 
 
         cleanerService = Executors.newSingleThreadScheduledExecutor(
         cleanerService = Executors.newSingleThreadScheduledExecutor(
-                JavaHelper.makePwmThreadFactory(
-                        JavaHelper.makeThreadName( pwmApplication, this.getClass() ) + "-cleaner-",
+                PwmScheduler.makePwmThreadFactory(
+                        PwmScheduler.makeThreadName( pwmApplication, this.getClass() ) + "-cleaner-",
                         true
                         true
                 ) );
                 ) );
 
 
         writerService = Executors.newSingleThreadScheduledExecutor(
         writerService = Executors.newSingleThreadScheduledExecutor(
-                JavaHelper.makePwmThreadFactory(
-                        JavaHelper.makeThreadName( pwmApplication, this.getClass() ) + "-writer-",
+                PwmScheduler.makePwmThreadFactory(
+                        PwmScheduler.makeThreadName( pwmApplication, this.getClass() ) + "-writer-",
                         true
                         true
                 ) );
                 ) );