allow reports to process across restarts

server/src/main/java/password/pwm/ldap/LdapOperationsHelper.java

@@ -59,7 +59,7 @@ import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.PasswordData;
 import password.pwm.util.i18n.LocaleHelper;
-import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
@@ -864,12 +864,18 @@ public class LdapOperationsHelper
     {
         final UserSearchEngine userSearchEngine = pwmApplication.getUserSearchEngine();
         final Queue<UserIdentity> resultSet = new LinkedList<>();
+        final long searchTimeoutMs = JavaHelper.silentParseLong(
+                pwmApplication.getConfig().readAppProperty( AppProperty.REPORTING_LDAP_SEARCH_TIMEOUT ),
+                30_000 );
 
         for ( final UserPermission userPermission : permissionList )
         {
             if ( resultSet.size() < maxResults )
             {
-                final SearchConfiguration searchConfiguration = SearchConfiguration.fromPermission( userPermission );
+                final SearchConfiguration searchConfiguration = SearchConfiguration.fromPermission( userPermission )
+                        .toBuilder()
+                        .searchTimeout( searchTimeoutMs )
+                        .build();
                 final Map<UserIdentity, Map<String, String>> searchResults = userSearchEngine.performMultiUserSearch(
                         searchConfiguration,
                         maxResults - resultSet.size(),
@@ -897,65 +903,6 @@ public class LdapOperationsHelper
         };
     }
 
-
-    public static Iterator<UserIdentity> readAllUsersFromLdap(
-            final PwmApplication pwmApplication,
-            final SessionLabel sessionLabel,
-            final String searchFilter,
-            final int maxResults
-    )
-            throws PwmUnrecoverableException, PwmOperationalException
-    {
-        final UserSearchEngine userSearchEngine = pwmApplication.getUserSearchEngine();
-
-        final SearchConfiguration searchConfiguration;
-        {
-            final SearchConfiguration.SearchConfigurationBuilder builder = SearchConfiguration.builder();
-
-            builder.enableValueEscaping( false );
-            builder.searchTimeout( Long.parseLong( pwmApplication.getConfig().readAppProperty( AppProperty.REPORTING_LDAP_SEARCH_TIMEOUT ) ) );
-
-            if ( searchFilter == null )
-            {
-                builder.username( "*" );
-            }
-            else
-            {
-                builder.filter( searchFilter );
-            }
-
-            searchConfiguration = builder.build();
-        }
-
-        LOGGER.debug( sessionLabel, () -> "beginning user search using parameters: " + ( JsonUtil.serialize( searchConfiguration ) ) );
-
-        final Map<UserIdentity, Map<String, String>> searchResults = userSearchEngine.performMultiUserSearch(
-                searchConfiguration,
-                maxResults,
-                Collections.emptyList(),
-                sessionLabel
-
-        );
-        LOGGER.debug( sessionLabel, () -> "user search found " + searchResults.size() + " users" );
-
-        final Queue<UserIdentity> tempQueue = new LinkedList<>( searchResults.keySet() );
-
-        return new Iterator<UserIdentity>()
-        {
-            @Override
-            public boolean hasNext( )
-            {
-                return tempQueue.peek() != null;
-            }
-
-            @Override
-            public UserIdentity next( )
-            {
-                return tempQueue.poll();
-            }
-        };
-    }
-
     public static Instant readPasswordExpirationTime( final ChaiUser theUser )
     {
         try

server/src/main/java/password/pwm/ldap/search/SearchConfiguration.java

@@ -22,7 +22,7 @@ package password.pwm.ldap.search;
 
 import com.novell.ldapchai.provider.ChaiProvider;
 import lombok.Builder;
-import lombok.Getter;
+import lombok.Value;
 import password.pwm.PwmConstants;
 import password.pwm.config.value.data.FormConfiguration;
 import password.pwm.config.value.data.UserPermission;
@@ -35,11 +35,10 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
-@Builder
-@Getter
+@Value
+@Builder( toBuilder = true )
 public class SearchConfiguration implements Serializable
 {
-
     private String filter;
     private String ldapProfile;
     private String username;

server/src/main/java/password/pwm/svc/report/ReportCsvUtility.java

@@ -187,7 +187,7 @@ public class ReportCsvUtility
         csvPrinter.printRecord( csvRow );
     }
 
-    public ReportService.RecordIterator<UserCacheRecord> iterator( )
+    public ClosableIterator<UserCacheRecord> iterator( )
     {
         return reportService.iterator();
     }

server/src/main/java/password/pwm/svc/report/ReportService.java

@@ -20,8 +20,6 @@
 
 package password.pwm.svc.report;
 
-import com.novell.ldapchai.exception.ChaiOperationException;
-import com.novell.ldapchai.exception.ChaiUnavailableException;
 import password.pwm.PwmApplication;
 import password.pwm.PwmApplicationMode;
 import password.pwm.bean.SessionLabel;
@@ -41,6 +39,7 @@ import password.pwm.svc.PwmService;
 import password.pwm.util.EventRateMeter;
 import password.pwm.util.PwmScheduler;
 import password.pwm.util.TransactionSizeCalculator;
+import password.pwm.util.java.AverageTracker;
 import password.pwm.util.java.BlockingThreadPool;
 import password.pwm.util.java.ClosableIterator;
 import password.pwm.util.java.JavaHelper;
@@ -52,29 +51,28 @@ import password.pwm.util.localdb.LocalDBStoredQueue;
 import password.pwm.util.logging.PwmLogger;
 
 import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.math.MathContext;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
+import java.util.Objects;
 import java.util.Queue;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
+
 public class ReportService implements PwmService
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( ReportService.class );
 
-    private final AvgTracker avgTracker = new AvgTracker( 100 );
+    private final AverageTracker avgTracker = new AverageTracker( 100 );
 
     private PwmApplication pwmApplication;
     private STATUS status = STATUS.NEW;
-    private boolean cancelFlag = false;
-    private ReportStatusInfo reportStatus = ReportStatusInfo.builder().build();
+    private volatile boolean cancelFlag = false;
     private ReportSummaryData summaryData = ReportSummaryData.newSummaryData( null );
     private ExecutorService executorService;
 
@@ -83,7 +81,8 @@ public class ReportService implements PwmService
 
     private Queue<String> dnQueue;
 
-    private final EventRateMeter eventRateMeter = new EventRateMeter( TimeDuration.MINUTE );
+    private final AtomicReference<ReportStatusInfo> reportStatus = new AtomicReference<>( ReportStatusInfo.builder().build() );
+    private final EventRateMeter processRateMeter = new EventRateMeter( TimeDuration.MINUTE );
 
 
     public ReportService( )
@@ -143,8 +142,6 @@ public class ReportService implements PwmService
 
         executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
 
-        LOGGER.debug( () -> "report service started" );
-
         executorService.submit( new InitializationTask() );
 
         status = STATUS.OPEN;
@@ -153,6 +150,7 @@ public class ReportService implements PwmService
     @Override
     public void close( )
     {
+        status = STATUS.CLOSED;
         cancelFlag = true;
 
         JavaHelper.closeAndWaitExecutor( executorService, TimeDuration.SECONDS_10 );
@@ -162,16 +160,15 @@ public class ReportService implements PwmService
             userCacheService.close();
         }
 
-        status = STATUS.CLOSED;
         executorService = null;
-        saveTempData();
+        writeReportStatus();
     }
 
-    private void saveTempData( )
+    private void writeReportStatus( )
     {
         try
         {
-            pwmApplication.writeAppAttribute( PwmApplication.AppAttribute.REPORT_STATUS, reportStatus );
+            pwmApplication.writeAppAttribute( PwmApplication.AppAttribute.REPORT_STATUS, reportStatus.get() );
         }
         catch ( Exception e )
         {
@@ -198,13 +195,13 @@ public class ReportService implements PwmService
         {
             case Start:
             {
-                if ( reportStatus.getCurrentProcess() != ReportStatusInfo.ReportEngineProcess.ReadData
-                        && reportStatus.getCurrentProcess() != ReportStatusInfo.ReportEngineProcess.SearchLDAP
+                final ReportStatusInfo localReportStatus = reportStatus.get();
+                if ( localReportStatus.getCurrentProcess() != ReportStatusInfo.ReportEngineProcess.ReadData
+                        && localReportStatus.getCurrentProcess() != ReportStatusInfo.ReportEngineProcess.SearchLDAP
                 )
                 {
                     executorService.execute( new ClearTask() );
                     executorService.execute( new ReadLDAPTask() );
-                    LOGGER.trace( SessionLabel.REPORTING_SESSION_LABEL, () -> "submitted new ldap dredge task to executorService" );
                 }
             }
             break;
@@ -228,14 +225,9 @@ public class ReportService implements PwmService
         }
     }
 
-    PwmApplication getPwmApplication( )
-    {
-        return pwmApplication;
-    }
-
     public BigDecimal getEventRate( )
     {
-        return eventRateMeter.readEventRate();
+        return processRateMeter.readEventRate();
     }
 
     public long getTotalRecords( )
@@ -245,38 +237,39 @@ public class ReportService implements PwmService
 
     private void clearWorkQueue( )
     {
-        reportStatus.setCount( 0 );
-        reportStatus.setJobDuration( TimeDuration.ZERO );
+        reportStatus.updateAndGet( reportStatusInfo -> reportStatusInfo.toBuilder()
+                .count( 0 )
+                .jobDuration( TimeDuration.ZERO )
+                .build() );
+
         dnQueue.clear();
     }
 
     private void resetJobStatus( )
     {
         cancelFlag = false;
-        eventRateMeter.reset();
-
-        reportStatus.setLastError( null );
-        reportStatus.setErrors( 0 );
+        processRateMeter.reset();
 
-        reportStatus.setFinishDate( null );
+        reportStatus.updateAndGet( reportStatusInfo -> reportStatusInfo.toBuilder()
+                .lastError( null )
+                .errors( 0 )
+                .finishDate( null )
+                .reportComplete( false )
+                .build() );
 
-        pwmApplication.writeAppAttribute( PwmApplication.AppAttribute.REPORT_STATUS, null );
+        writeReportStatus();
     }
 
 
     public ReportStatusInfo getReportStatusInfo( )
     {
-        return reportStatus;
+        return reportStatus.get();
     }
 
 
-    public interface RecordIterator<K> extends ClosableIterator<UserCacheRecord>
-    {
-    }
-
-    public RecordIterator<UserCacheRecord> iterator( )
+    public ClosableIterator<UserCacheRecord> iterator( )
     {
-        return new RecordIterator<UserCacheRecord>()
+        return new ClosableIterator<UserCacheRecord>()
         {
             private UserCacheService.UserStatusCacheBeanIterator<UserCacheService.StorageKey> storageKeyIterator = userCacheService.iterator();
 
@@ -334,53 +327,14 @@ public class ReportService implements PwmService
         return dnQueue.size();
     }
 
-    public static class AvgTracker
-    {
-        private final int maxSamples;
-        private final Queue<BigInteger> samples = new LinkedList<>();
-
-        public AvgTracker( final int maxSamples )
-        {
-            this.maxSamples = maxSamples;
-        }
-
-        public void addSample( final long input )
-        {
-            samples.add( new BigInteger( Long.toString( input ) ) );
-            while ( samples.size() > maxSamples )
-            {
-                samples.remove();
-            }
-        }
-
-        public BigDecimal avg( )
-        {
-            if ( samples.isEmpty() )
-            {
-                throw new IllegalStateException( "unable to compute avg without samples" );
-            }
-
-            BigInteger total = BigInteger.ZERO;
-            for ( final BigInteger sample : samples )
-            {
-                total = total.add( sample );
-            }
-            final BigDecimal maxAsBD = new BigDecimal( Integer.toString( maxSamples ) );
-            return new BigDecimal( total ).divide( maxAsBD, MathContext.DECIMAL32 );
-        }
-
-        public long avgAsLong( )
-        {
-            return avg().longValue();
-        }
-    }
-
     private class ReadLDAPTask implements Runnable
     {
         @Override
         public void run( )
         {
-            reportStatus.setCurrentProcess( ReportStatusInfo.ReportEngineProcess.SearchLDAP );
+            reportStatus.updateAndGet( reportStatusInfo -> reportStatusInfo.toBuilder()
+                    .currentProcess( ReportStatusInfo.ReportEngineProcess.SearchLDAP )
+                    .build() );
             try
             {
                 readUserListFromLdap();
@@ -409,15 +363,15 @@ public class ReportService implements PwmService
             }
             finally
             {
-                reportStatus.setCurrentProcess( ReportStatusInfo.ReportEngineProcess.None );
+                resetCurrentProcess();
             }
         }
 
         private void readUserListFromLdap( )
-                throws ChaiUnavailableException, ChaiOperationException, PwmUnrecoverableException, PwmOperationalException
+                throws PwmUnrecoverableException, PwmOperationalException
         {
             final Instant startTime = Instant.now();
-            LOGGER.trace( () -> "beginning ldap search process" );
+            LOGGER.trace( SessionLabel.REPORTING_SESSION_LABEL, () -> "beginning ldap search process" );
 
             resetJobStatus();
             clearWorkQueue();
@@ -429,8 +383,15 @@ public class ReportService implements PwmService
                     settings.getMaxSearchSize()
             );
 
+            LOGGER.trace( SessionLabel.REPORTING_SESSION_LABEL, () -> "completed ldap search process (" + TimeDuration.compactFromCurrent( startTime ) + ")" );
+
+            writeUsersToLocalDBQueue( memQueue );
+        }
 
-            LOGGER.trace( () -> "completed ldap search process, transferring search results to work queue" );
+        private void writeUsersToLocalDBQueue( final Iterator<UserIdentity> identityQueue )
+        {
+            final Instant startTime = Instant.now();
+            LOGGER.trace( SessionLabel.REPORTING_SESSION_LABEL, () -> "transferring search results to work queue" );
 
             final TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(
                     TransactionSizeCalculator.Settings.builder()
@@ -440,19 +401,20 @@ public class ReportService implements PwmService
                             .build()
             );
 
-            while ( status == STATUS.OPEN && !cancelFlag && memQueue.hasNext() )
+            while ( !cancelFlag && identityQueue.hasNext() )
             {
                 final Instant loopStart = Instant.now();
                 final List<String> bufferList = new ArrayList<>();
                 final int loopCount = transactionCalculator.getTransactionSize();
-                for ( int i = 0; i < loopCount && memQueue.hasNext(); i++ )
+                while ( !cancelFlag && identityQueue.hasNext() && bufferList.size() < loopCount )
                 {
-                    bufferList.add( memQueue.next().toDelimitedKey() );
+                    bufferList.add( identityQueue.next().toDelimitedKey() );
                 }
                 dnQueue.addAll( bufferList );
                 transactionCalculator.recordLastTransactionDuration( TimeDuration.fromCurrent( loopStart ) );
             }
-            LOGGER.trace( () -> "completed transfer of ldap search results to work queue in " + TimeDuration.fromCurrent( startTime ).asCompactString() );
+            LOGGER.trace( SessionLabel.REPORTING_SESSION_LABEL,
+                    () -> "completed transfer of ldap search results to work queue in " + TimeDuration.compactFromCurrent( startTime ) );
         }
     }
 
@@ -461,32 +423,38 @@ public class ReportService implements PwmService
         @Override
         public void run( )
         {
-            reportStatus.setCurrentProcess( ReportStatusInfo.ReportEngineProcess.ReadData );
+            reportStatus.updateAndGet( reportStatusInfo -> reportStatusInfo.toBuilder()
+                    .currentProcess( ReportStatusInfo.ReportEngineProcess.ReadData )
+                    .build() );
             try
             {
                 processWorkQueue();
+                if ( status == STATUS.OPEN )
+                {
+                    reportStatus.updateAndGet( reportStatusInfo -> reportStatusInfo.toBuilder()
+                            .reportComplete( true )
+                            .build() );
+                    writeReportStatus();
+                }
             }
-            catch ( Exception e )
+            catch ( PwmException e )
             {
-                if ( e instanceof PwmException )
+                if ( e.getErrorInformation().getError() == PwmError.ERROR_DIRECTORY_UNAVAILABLE )
                 {
-                    if ( ( ( PwmException ) e ).getErrorInformation().getError() == PwmError.ERROR_DIRECTORY_UNAVAILABLE )
+                    if ( executorService != null )
                     {
-                        if ( executorService != null )
-                        {
-                            LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, "directory unavailable error during background ReadData, will retry; error: " + e.getMessage() );
-                            pwmApplication.getPwmScheduler().scheduleJob( new ProcessWorkQueueTask(), executorService, TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
-                        }
-                    }
-                    else
-                    {
-                        LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, "error during background ReadData: " + e.getMessage() );
+                        LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, "directory unavailable error during background ReadData, will retry; error: " + e.getMessage() );
+                        pwmApplication.getPwmScheduler().scheduleJob( new ProcessWorkQueueTask(), executorService, TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
                     }
                 }
+                else
+                {
+                    LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, "error during background ReadData: " + e.getMessage() );
+                }
             }
             finally
             {
-                reportStatus.setCurrentProcess( ReportStatusInfo.ReportEngineProcess.None );
+                resetCurrentProcess();
             }
         }
 
@@ -537,8 +505,9 @@ public class ReportService implements PwmService
                         {
                             final Instant startUpdateTime = Instant.now();
                             updateCachedRecordFromLdap( userIdentity );
-                            reportStatus.setCount( reportStatus.getCount() + 1 );
-                            eventRateMeter.markEvents( 1 );
+                            reportStatus.updateAndGet( reportStatusInfo -> reportStatusInfo.toBuilder()
+                                    .count( reportStatusInfo.getCount() + 1 )
+                                    .build() );
                             final TimeDuration totalUpdateTime = TimeDuration.fromCurrent( startUpdateTime );
                             avgTracker.addSample( totalUpdateTime.asMillis() );
 
@@ -546,7 +515,9 @@ public class ReportService implements PwmService
                             {
                                 updateTimeLock.lock();
                                 final TimeDuration scaledTime = TimeDuration.of( totalUpdateTime.asMillis() / threadCount, TimeDuration.Unit.MILLISECONDS );
-                                reportStatus.setJobDuration( reportStatus.getJobDuration().add( scaledTime ) );
+                                reportStatus.updateAndGet( reportStatusInfo -> reportStatusInfo.toBuilder()
+                                        .jobDuration( reportStatusInfo.getJobDuration().add( scaledTime ) )
+                                        .build() );
                             }
                             finally
                             {
@@ -565,8 +536,10 @@ public class ReportService implements PwmService
                             final ErrorInformation errorInformation;
                             errorInformation = new ErrorInformation( PwmError.ERROR_REPORTING_ERROR, errorMsg );
                             LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, errorInformation.toDebugStr(), e );
-                            reportStatus.setLastError( errorInformation );
-                            reportStatus.setErrors( reportStatus.getErrors() + 1 );
+                            reportStatus.updateAndGet( reportStatusInfo -> reportStatusInfo.toBuilder()
+                                    .lastError( errorInformation )
+                                    .errors( reportStatusInfo.getErrors() + 1 )
+                                    .build() );
                         }
                         if ( pwmApplication.getConfig().isDevDebugMode() )
                         {
@@ -585,13 +558,20 @@ public class ReportService implements PwmService
 
                 if ( cancelFlag )
                 {
-                    reportStatus.setLastError( new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, "report cancelled by operator" ) );
+
+                    final ErrorInformation errorInformation = new ErrorInformation(
+                            PwmError.ERROR_SERVICE_NOT_AVAILABLE, "report cancelled by operator" );
+                    reportStatus.updateAndGet( reportStatusInfo -> reportStatusInfo.toBuilder()
+                            .lastError( errorInformation )
+                            .build() );
                 }
             }
             finally
             {
-                reportStatus.setFinishDate( Instant.now() );
-                saveTempData();
+                reportStatus.updateAndGet( reportStatusInfo -> reportStatusInfo.toBuilder()
+                        .finishDate( Instant.now() )
+                        .build() );
+                writeReportStatus();
             }
             LOGGER.debug( SessionLabel.REPORTING_SESSION_LABEL, () -> "update user cache process completed: " + JsonUtil.serialize( reportStatus ) );
         }
@@ -614,6 +594,7 @@ public class ReportService implements PwmService
 
             userCacheService.store( newUserCacheRecord );
             summaryData.update( newUserCacheRecord );
+            processRateMeter.markEvents( 1 );
 
             LOGGER.trace( SessionLabel.REPORTING_SESSION_LABEL, () -> "stored cache for " + userIdentity );
         }
@@ -632,16 +613,6 @@ public class ReportService implements PwmService
                 executorService.execute( new ReadLDAPTask() );
             }
         }
-
-        private void checkForOutdatedStoreData()
-        {
-            final Instant lastFinishDate = reportStatus.getFinishDate();
-            if ( lastFinishDate != null && TimeDuration.fromCurrent( lastFinishDate ).isLongerThan( settings.getMaxCacheAge() ) )
-            {
-                executorService.execute( new ClearTask() );
-            }
-        }
-
     }
 
     private class InitializationTask implements Runnable
@@ -652,8 +623,9 @@ public class ReportService implements PwmService
             try
             {
                 initTempData();
+                LOGGER.debug( SessionLabel.REPORTING_SESSION_LABEL, () -> "report service initialized: " + JsonUtil.serialize( reportStatus.get() ) );
             }
-            catch ( LocalDBException | PwmUnrecoverableException e )
+            catch ( PwmUnrecoverableException e )
             {
                 LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, "error during initialization: " + e.getMessage() );
                 status = STATUS.CLOSED;
@@ -668,13 +640,13 @@ public class ReportService implements PwmService
             }
         }
 
-
         private void initTempData( )
-                throws LocalDBException, PwmUnrecoverableException
+                throws PwmUnrecoverableException
         {
             try
             {
-                reportStatus = pwmApplication.readAppAttribute( PwmApplication.AppAttribute.REPORT_STATUS, ReportStatusInfo.class );
+                final ReportStatusInfo localReportStatus = pwmApplication.readAppAttribute( PwmApplication.AppAttribute.REPORT_STATUS, ReportStatusInfo.class );
+                reportStatus.set( localReportStatus );
             }
             catch ( Exception e )
             {
@@ -682,15 +654,14 @@ public class ReportService implements PwmService
             }
 
             boolean clearFlag = false;
-            if ( reportStatus == null )
+            if ( reportStatus.get() == null )
             {
                 clearFlag = true;
                 LOGGER.debug( SessionLabel.REPORTING_SESSION_LABEL, () -> "report service did not close cleanly, will clear data." );
             }
             else
             {
-                final String currentSettingCache = settings.getSettingsHash();
-                if ( reportStatus.getSettingsHash() != null && !reportStatus.getSettingsHash().equals( currentSettingCache ) )
+                if ( !Objects.equals( reportStatus.get().getSettingsHash(), settings.getSettingsHash() ) )
                 {
                     LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, "configuration has changed, will clear cached report data" );
                     clearFlag = true;
@@ -699,9 +670,22 @@ public class ReportService implements PwmService
 
             if ( clearFlag )
             {
-                reportStatus = ReportStatusInfo.builder().settingsHash( settings.getSettingsHash() ).build();
+                initReportStatus();
                 executeCommand( ReportCommand.Clear );
             }
+
+            startNextTask();
+        }
+
+        private void startNextTask()
+        {
+            checkForOutdatedStoreData();
+
+            if ( !reportStatus.get().isReportComplete() && !dnQueue.isEmpty() )
+            {
+                LOGGER.trace( SessionLabel.REPORTING_SESSION_LABEL, () -> "resuming report data processing" );
+                executorService.execute( new ProcessWorkQueueTask() );
+            }
         }
     }
 
@@ -718,6 +702,10 @@ public class ReportService implements PwmService
             {
                 LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, "error during clear operation: " + e.getMessage() );
             }
+            finally
+            {
+                resetCurrentProcess();
+            }
         }
 
         private void doClear( ) throws LocalDBException, PwmUnrecoverableException
@@ -730,8 +718,36 @@ public class ReportService implements PwmService
                 userCacheService.clear();
             }
             summaryData = ReportSummaryData.newSummaryData( settings.getTrackDays() );
-            reportStatus = ReportStatusInfo.builder().settingsHash( settings.getSettingsHash() ).build();
+            initReportStatus();
             LOGGER.debug( SessionLabel.REPORTING_SESSION_LABEL, () -> "finished clearing report " + TimeDuration.compactFromCurrent( startTime ) );
         }
     }
+
+    private void initReportStatus()
+            throws PwmUnrecoverableException
+    {
+        final String settingsHash = settings.getSettingsHash();
+        reportStatus.set( ReportStatusInfo.builder()
+                .settingsHash( settingsHash )
+                .currentProcess( ReportStatusInfo.ReportEngineProcess.None )
+                .build() );
+        writeReportStatus();
+    }
+
+    private void resetCurrentProcess()
+    {
+        reportStatus.updateAndGet( reportStatusInfo -> reportStatusInfo.toBuilder()
+                .currentProcess( ReportStatusInfo.ReportEngineProcess.None )
+                .build() );
+        writeReportStatus();
+    }
+
+    private void checkForOutdatedStoreData()
+    {
+        final Instant lastFinishDate = reportStatus.get().getFinishDate();
+        if ( lastFinishDate != null && TimeDuration.fromCurrent( lastFinishDate ).isLongerThan( settings.getMaxCacheAge() ) )
+        {
+            executorService.execute( new ClearTask() );
+        }
+    }
 }

server/src/main/java/password/pwm/svc/report/ReportStatusInfo.java

@@ -21,14 +21,14 @@
 package password.pwm.svc.report;
 
 import lombok.Builder;
-import lombok.Data;
+import lombok.Value;
 import password.pwm.error.ErrorInformation;
 import password.pwm.util.java.TimeDuration;
 
 import java.io.Serializable;
 import java.time.Instant;
 
-@Data
+@Value
 @Builder( toBuilder = true )
 public class ReportStatusInfo implements Serializable
 {
@@ -37,6 +37,7 @@ public class ReportStatusInfo implements Serializable
 
     private Instant startDate;
     private Instant finishDate;
+    private boolean reportComplete;
     private int count;
     private int errors;
     private ErrorInformation lastError;

server/src/main/java/password/pwm/svc/wordlist/WordlistSource.java

@@ -144,6 +144,11 @@ class WordlistSource
     {
         final Instant startTime = Instant.now();
 
+        if ( cancelFlag.getAsBoolean() )
+        {
+            return null;
+        }
+
         pwmLogger.debug( () -> "begin reading file info for " + this.getWordlistSourceType() + " wordlist" );
 
         final long bytes;

server/src/main/java/password/pwm/util/cli/commands/ResponseStatsCommand.java

@@ -21,7 +21,6 @@
 package password.pwm.util.cli.commands;
 
 import com.novell.ldapchai.cr.Challenge;
-import com.novell.ldapchai.exception.ChaiOperationException;
 import com.novell.ldapchai.exception.ChaiUnavailableException;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
@@ -159,7 +158,7 @@ public class ResponseStatsCommand extends AbstractCliCommand
     private static List<UserIdentity> readAllUsersFromLdap(
             final PwmApplication pwmApplication
     )
-            throws ChaiUnavailableException, ChaiOperationException, PwmUnrecoverableException, PwmOperationalException
+            throws PwmUnrecoverableException, PwmOperationalException
     {
         final List<UserIdentity> returnList = new ArrayList<>();
 

server/src/main/java/password/pwm/util/java/AverageTracker.java

@@ -0,0 +1,68 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2019 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.util.java;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.MathContext;
+import java.util.LinkedList;
+import java.util.Queue;
+
+public class AverageTracker
+{
+    private final int maxSamples;
+    private final Queue<BigInteger> samples = new LinkedList<>();
+
+    public AverageTracker( final int maxSamples )
+    {
+        this.maxSamples = maxSamples;
+    }
+
+    public void addSample( final long input )
+    {
+        samples.add( new BigInteger( Long.toString( input ) ) );
+        while ( samples.size() > maxSamples )
+        {
+            samples.remove();
+        }
+    }
+
+    public BigDecimal avg( )
+    {
+        if ( samples.isEmpty() )
+        {
+            throw new IllegalStateException( "unable to compute avg without samples" );
+        }
+
+        BigInteger total = BigInteger.ZERO;
+        for ( final BigInteger sample : samples )
+        {
+            total = total.add( sample );
+        }
+        final BigDecimal maxAsBD = new BigDecimal( Integer.toString( maxSamples ) );
+        return new BigDecimal( total ).divide( maxAsBD, MathContext.DECIMAL32 );
+    }
+
+    public long avgAsLong( )
+    {
+        return avg().longValue();
+    }
+}