process report enhancements

pom.xml

@@ -333,7 +333,7 @@
             <plugin>
                 <groupId>com.github.spotbugs</groupId>
                 <artifactId>spotbugs-maven-plugin</artifactId>
-                <version>4.7.2.0</version>
+                <version>4.7.2.1</version>
                 <dependencies>
                     <dependency>
                         <groupId>com.github.spotbugs</groupId>

server/src/main/java/password/pwm/AppProperty.java

@@ -329,8 +329,9 @@ public enum AppProperty
     RECAPTCHA_CLIENT_JS_URL                         ( "recaptcha.clientJsUrl" ),
     RECAPTCHA_CLIENT_IFRAME_URL                     ( "recaptcha.clientIframeUrl" ),
     RECAPTCHA_VALIDATE_URL                          ( "recaptcha.validateUrl" ),
+    REPORTING_LDAP_JOB_THREADS                      ( "reporting.ldap.recordJob.threads" ),
+    REPORTING_LDAP_JOB_TIMEOUT_MS                   ( "reporting.ldap.recordJob.timeoutMs" ),
     REPORTING_LDAP_SEARCH_TIMEOUT_MS                ( "reporting.ldap.searchTimeoutMs" ),
-    REPORTING_LDAP_SEARCH_THREADS                   ( "reporting.ldap.searchThreads" ),
     REPORTING_MAX_REPORT_AGE_SECONDS                ( "reporting.maxReportAgeSeconds" ),
     SECURITY_STRIP_INLINE_JAVASCRIPT                ( "security.html.stripInlineJavascript" ),
     SECURITY_HTTP_FORCE_REQUEST_SEQUENCING          ( "security.http.forceRequestSequencing" ),

server/src/main/java/password/pwm/http/JspUtility.java

@@ -37,6 +37,7 @@ import javax.servlet.jsp.PageContext;
 import java.text.NumberFormat;
 import java.time.Instant;
 import java.util.Locale;
+import java.util.function.Supplier;
 
 public abstract class JspUtility
 {
@@ -189,6 +190,26 @@ public abstract class JspUtility
         return StringUtil.escapeHtml( input );
     }
 
+    public static String friendlyWrite( final PageContext pageContext, final Supplier<String> input )
+    {
+        try
+        {
+            final String str = input.get();
+            if ( StringUtil.isEmpty( str ) )
+            {
+                return friendlyWriteNotApplicable( pageContext );
+            }
+            return StringUtil.escapeHtml( str );
+        }
+        catch ( final Exception e )
+        {
+            LOGGER.debug( () -> "error while performing JSP write: " + e.getMessage() );
+        }
+
+        return "";
+    }
+
+
     public static String friendlyWriteNotApplicable( final PageContext pageContext )
     {
         final PwmRequest pwmRequest = forRequest( pageContext.getRequest() );

server/src/main/java/password/pwm/http/servlet/admin/domain/AdminReportServlet.java

@@ -145,6 +145,8 @@ public class AdminReportServlet extends ControlledPwmServlet
 
         final ReportProcessRequest reportProcessRequest = ReportProcessRequest.builder()
                 .domainID( pwmRequest.getDomainID() )
+                .sessionLabel( pwmRequest.getLabel() )
+                .locale( pwmRequest.getLocale() )
                 .maximumRecords( pwmRequest.readParameterAsInt( "recordCount", 1000 ) )
                 .reportType( pwmRequest.readParameterAsEnum( "recordType", ReportProcessRequest.ReportType.class )
                         .orElse( ReportProcessRequest.ReportType.json ) )
@@ -153,10 +155,10 @@ public class AdminReportServlet extends ControlledPwmServlet
         try ( OutputStream outputStream = pwmRequest.getPwmResponse().getOutputStream() )
         {
             final ReportService reportService = pwmRequest.getPwmApplication().getReportService();
-            try ( ReportProcess reportProcess = reportService.createReportProcess( pwmRequest.getLocale(), pwmRequest.getLabel() ) )
+            try ( ReportProcess reportProcess = reportService.createReportProcess( reportProcessRequest ) )
             {
                 pwmRequest.getPwmSession().setReportProcess( reportProcess );
-                reportProcess.startReport( reportProcessRequest, outputStream );
+                reportProcess.startReport( outputStream );
             }
         }
         catch ( final PwmException e )

server/src/main/java/password/pwm/svc/PwmServiceEnum.java

@@ -60,7 +60,6 @@ public enum PwmServiceEnum
     TokenSystemService( password.pwm.svc.token.TokenSystemService.class, PwmSettingScope.SYSTEM ),
     HealthMonitor( HealthService.class, PwmSettingScope.SYSTEM ),
     DebugOutputService( password.pwm.health.DebugOutputService.class, PwmSettingScope.SYSTEM ),
-    ReportService( password.pwm.svc.report.ReportService.class, PwmSettingScope.SYSTEM, Flag.StartDuringRuntimeInstance ),
     SessionTrackService( password.pwm.svc.sessiontrack.SessionTrackService.class, PwmSettingScope.SYSTEM ),
     SessionStateSvc( password.pwm.http.state.SessionStateService.class, PwmSettingScope.SYSTEM ),
     TelemetryService( password.pwm.svc.telemetry.TelemetryService.class, PwmSettingScope.SYSTEM ),
@@ -77,7 +76,8 @@ public enum PwmServiceEnum
     UserHistoryService( password.pwm.svc.userhistory.UserHistoryService.class, PwmSettingScope.DOMAIN, Flag.StartDuringRuntimeInstance ),
     PeopleSearchService( password.pwm.http.servlet.peoplesearch.PeopleSearchService.class, PwmSettingScope.DOMAIN ),
     PwExpiryNotifyService( PwNotifyService.class, PwmSettingScope.DOMAIN ),
-    ResourceServletService( password.pwm.http.servlet.resource.ResourceServletService.class, PwmSettingScope.DOMAIN ),;
+    ResourceServletService( password.pwm.http.servlet.resource.ResourceServletService.class, PwmSettingScope.DOMAIN ),
+    ReportService( password.pwm.svc.report.ReportService.class, PwmSettingScope.DOMAIN  ),;
 
 
     private final Class<? extends PwmService> clazz;

server/src/main/java/password/pwm/svc/report/ReportCsvRecordWriter.java

@@ -21,7 +21,7 @@
 package password.pwm.svc.report;
 
 import org.apache.commons.csv.CSVPrinter;
-import password.pwm.PwmApplication;
+import password.pwm.PwmDomain;
 import password.pwm.config.SettingReader;
 import password.pwm.i18n.Display;
 import password.pwm.i18n.PwmDisplayBundle;
@@ -38,15 +38,15 @@ import java.util.Locale;
 class ReportCsvRecordWriter implements ReportRecordWriter
 {
     private final Locale locale;
-    private final PwmApplication pwmApplication;
+    private final PwmDomain pwmDomain;
     private final CSVPrinter csvPrinter;
 
-    ReportCsvRecordWriter( final OutputStream outputStream, final PwmApplication pwmApplication, final Locale locale )
+    ReportCsvRecordWriter( final OutputStream outputStream, final PwmDomain pwmDomain, final Locale locale )
             throws IOException
     {
         this.csvPrinter = PwmUtil.makeCsvPrinter( outputStream );
         this.locale = locale;
-        this.pwmApplication = pwmApplication;
+        this.pwmDomain = pwmDomain;
     }
 
     static void outputHeaderRow( final Locale locale, final CSVPrinter csvPrinter, final SettingReader config )
@@ -146,13 +146,13 @@ class ReportCsvRecordWriter implements ReportRecordWriter
     @Override
     public void outputHeader() throws IOException
     {
-        outputHeaderRow( locale, csvPrinter, pwmApplication.getConfig() );
+        outputHeaderRow( locale, csvPrinter, pwmDomain.getConfig() );
     }
 
     @Override
-    public void outputRecord( final UserReportRecord userReportRecord, final boolean lastRecord ) throws IOException
+    public void outputRecord( final UserReportRecord userReportRecord ) throws IOException
     {
-        final SettingReader settingReader = pwmApplication.getConfig();
+        final SettingReader settingReader = pwmDomain.getConfig();
         outputRecordRow( settingReader, locale, userReportRecord, csvPrinter );
     }
 

server/src/main/java/password/pwm/svc/report/ReportJsonRecordWriter.java

@@ -36,6 +36,8 @@ class ReportJsonRecordWriter implements ReportRecordWriter
     private final Writer writer;
     private final JsonProvider jsonFactory = JsonFactory.get();
 
+    private boolean firstRecordWritten = false;
+
     ReportJsonRecordWriter( final OutputStream outputStream )
             throws IOException
     {
@@ -55,10 +57,15 @@ class ReportJsonRecordWriter implements ReportRecordWriter
     }
 
     @Override
-    public void outputRecord( final UserReportRecord userReportRecord, final boolean lastRecord ) throws IOException
+    public void outputRecord( final UserReportRecord userReportRecord ) throws IOException
     {
         writer.write( ' ' );
 
+        if ( firstRecordWritten )
+        {
+            writer.write( ',' );
+        }
+
         final String jsonString = jsonFactory.serialize( userReportRecord, UserReportRecord.class, JsonProvider.Flag.PrettyPrint );
         final String indentedJson = " " + StringUtil.replaceAllChars( jsonString, character ->
         {
@@ -70,13 +77,9 @@ class ReportJsonRecordWriter implements ReportRecordWriter
         } );
 
         writer.write( indentedJson );
-
-        if ( !lastRecord )
-        {
-            writer.write( ',' );
-        }
-
         writer.write( '\n' );
+
+        firstRecordWritten = true;
     }
 
     @Override

server/src/main/java/password/pwm/svc/report/ReportProcess.java

@@ -20,35 +20,28 @@
 
 package password.pwm.svc.report;
 
-import org.apache.commons.csv.CSVPrinter;
+import lombok.Value;
 import org.jetbrains.annotations.NotNull;
 import password.pwm.AppAttribute;
-import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
 import password.pwm.PwmDomain;
 import password.pwm.bean.SessionLabel;
 import password.pwm.bean.UserIdentity;
-import password.pwm.config.AppConfig;
 import password.pwm.config.value.data.UserPermission;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
-import password.pwm.error.PwmInternalException;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.http.bean.DisplayElement;
-import password.pwm.ldap.UserInfoFactory;
 import password.pwm.ldap.permission.UserPermissionUtility;
-import password.pwm.user.UserInfo;
 import password.pwm.util.EventRateMeter;
+import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.ConditionalTaskExecutor;
 import password.pwm.util.java.FunctionalReentrantLock;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.PwmTimeUtil;
-import password.pwm.util.java.PwmUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.json.JsonFactory;
-import password.pwm.util.json.JsonProvider;
 import password.pwm.util.logging.PwmLogLevel;
 import password.pwm.util.logging.PwmLogger;
 
@@ -59,18 +52,17 @@ import java.math.RoundingMode;
 import java.time.Instant;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Queue;
-import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
-import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
@@ -83,84 +75,93 @@ public class ReportProcess implements AutoCloseable
 
     private static final FunctionalReentrantLock REPORT_ID_LOCK = new FunctionalReentrantLock();
 
-    private final PwmApplication pwmApplication;
-    private final Semaphore reportServiceSemaphore;
+    private final PwmDomain pwmDomain;
+    private final ReportService reportService;
+
     private final ReportSettings reportSettings;
-    private final Locale locale;
-    private final SessionLabel sessionLabel;
 
     private final ConditionalTaskExecutor debugOutputLogger;
     private final long reportId;
+    private final ReportProcessRequest reportProcessRequest;
+
     private final AtomicLong recordCounter = new AtomicLong();
     private final AtomicBoolean inProgress = new AtomicBoolean();
     private final AtomicBoolean cancelFlag = new AtomicBoolean();
     private final EventRateMeter processRateMeter = new EventRateMeter( TimeDuration.MINUTE.asDuration() );
-
+    private final List<String> recordErrorMessages = new ArrayList<>();
 
     private Instant startTime = Instant.now();
-    private ReportSummaryCalculator summaryData = ReportSummaryCalculator.newSummaryData( Collections.singletonList( 1 ) );
+    private ReportSummaryCalculator summaryCalculator = ReportSummaryCalculator.empty();
+    private ReportProcessResult result;
 
     ReportProcess(
-            @NotNull final PwmApplication pwmApplication,
-            @NotNull final Semaphore reportServiceSemaphore,
-            @NotNull final ReportSettings reportSettings,
-            final Locale locale,
-            @NotNull final SessionLabel sessionLabel
+            final PwmDomain pwmDomain,
+            final ReportService reportService,
+            final ReportProcessRequest reportProcessRequest,
+            final ReportSettings reportSettings
     )
     {
-        this.pwmApplication = Objects.requireNonNull( pwmApplication );
-        this.reportServiceSemaphore = Objects.requireNonNull( reportServiceSemaphore );
+        this.pwmDomain = Objects.requireNonNull( pwmDomain );
+        this.reportService = Objects.requireNonNull( reportService );
         this.reportSettings = Objects.requireNonNull( reportSettings );
-        this.locale = Objects.requireNonNullElse( locale, PwmConstants.DEFAULT_LOCALE );
-        this.sessionLabel = sessionLabel;
+        this.reportProcessRequest = reportProcessRequest;
 
         this.reportId = nextProcessId();
 
         this.debugOutputLogger = ConditionalTaskExecutor.forPeriodicTask(
-                () -> log( PwmLogLevel.TRACE, () -> " in progress: " + recordCounter.longValue() + " records exported at " + processRateMeter.prettyEps( locale ),
-                        TimeDuration.fromCurrent( startTime ) ),
+                this::logStatus,
                 TimeDuration.MINUTE.asDuration() );
     }
 
-    private void log( final PwmLogLevel level, final Supplier<String> message, final TimeDuration timeDuration )
+    void log( final PwmLogLevel level, final Supplier<String> message, final TimeDuration timeDuration )
     {
         final Supplier<String> wrappedMsg = () -> "report #" + reportId + " " + message.get();
-        LOGGER.log( level, sessionLabel, wrappedMsg, null, timeDuration );
+        LOGGER.log( level, reportProcessRequest.getSessionLabel(), wrappedMsg, null, timeDuration );
     }
 
-    static ReportProcess createReportProcess(
-            @NotNull final PwmApplication pwmApplication,
-            @NotNull final Semaphore reportServiceSemaphore,
-            @NotNull final ReportSettings reportSettings,
-            final Locale locale,
-            @NotNull final SessionLabel sessionLabel
-    )
+    private void logStatus()
     {
-        return new ReportProcess( pwmApplication, reportServiceSemaphore, reportSettings, locale, sessionLabel );
+        log( PwmLogLevel.TRACE,
+                () -> "in progress: " + recordCounter.longValue()
+                        + " records exported at " + processRateMeter.prettyEps( reportProcessRequest.getLocale() )
+                        + " records/second, duration: "
+                        + TimeDuration.fromCurrent( startTime ).asCompactString()
+                        + " jobDetails: "
+                        + JsonFactory.get().serialize( reportProcessRequest ),
+                null );
     }
 
-    public static void outputSummaryToCsv(
-            final AppConfig config,
-            final ReportSummaryCalculator reportSummaryData,
-            final OutputStream outputStream,
-            final Locale locale
-    )
-            throws IOException
+    boolean isCancelled()
     {
+        return cancelFlag.get();
+    }
 
-        final List<ReportSummaryCalculator.PresentationRow> outputList = reportSummaryData.asPresentableCollection( config, locale );
-        final CSVPrinter csvPrinter = PwmUtil.makeCsvPrinter( outputStream );
+    PwmDomain getPwmDomain()
+    {
+        return pwmDomain;
+    }
 
-        for ( final ReportSummaryCalculator.PresentationRow presentationRow : outputList )
-        {
-            csvPrinter.printRecord( presentationRow.toStringList() );
-        }
+    SessionLabel getSessionLabel()
+    {
+        return reportProcessRequest.getSessionLabel();
+    }
+
+    Optional<ReportProcessResult> getResult()
+    {
+        return Optional.ofNullable( result );
+    }
 
-        csvPrinter.flush();
+    static ReportProcess createReportProcess(
+            final PwmDomain pwmDomain,
+            final ReportService reportService,
+            final ReportProcessRequest reportProcessRequest,
+            final ReportSettings reportSettings
+    )
+    {
+        return new ReportProcess( pwmDomain, reportService, reportProcessRequest, reportSettings );
     }
 
     public void startReport(
-            @NotNull final ReportProcessRequest reportProcessRequest,
             @NotNull final OutputStream outputStream
     )
             throws PwmUnrecoverableException
@@ -173,13 +174,13 @@ public class ReportProcess implements AutoCloseable
             throw new PwmUnrecoverableException( PwmError.ERROR_INTERNAL, "report process #" + reportId + " cannot be started, report already in progress" );
         }
 
-        if ( !reportServiceSemaphore.tryAcquire() )
+        if ( cancelFlag.get() )
         {
-            throw new PwmUnrecoverableException( PwmError.ERROR_INTERNAL, "report process #" + reportId + " cannot be started, maximum concurrent reports already in progress." );
+            throw new PwmUnrecoverableException( PwmError.ERROR_INTERNAL, "report process #" + reportId + " cannot be started, report already cancelled" );
         }
 
         this.startTime = Instant.now();
-        this.summaryData = ReportSummaryCalculator.newSummaryData( reportSettings.getTrackDays() );
+        this.summaryCalculator = ReportSummaryCalculator.newSummaryData( reportSettings.getTrackDays() );
         this.recordCounter.set( 0 );
         this.processRateMeter.reset();
         this.inProgress.set( true );
@@ -195,13 +196,15 @@ public class ReportProcess implements AutoCloseable
         catch ( final PwmException e )
         {
             log( PwmLogLevel.DEBUG, () -> "error during report generation: " + e.getMessage(), null );
-            cancelFlag.set( true );
             throw new PwmUnrecoverableException( new ErrorInformation(  PwmError.ERROR_INTERNAL, e.getMessage() ) );
         }
         catch ( final IOException e )
         {
             log( PwmLogLevel.DEBUG, () -> "I/O error during report generation: " + e.getMessage(), null );
-            cancelFlag.set( true );
+        }
+        catch ( final Exception e )
+        {
+            log( PwmLogLevel.DEBUG, () -> "Job interrupted during report generation: " + e.getMessage(), null );
         }
         finally
         {
@@ -213,22 +216,25 @@ public class ReportProcess implements AutoCloseable
             final ReportProcessRequest reportProcessRequest,
             final ZipOutputStream zipOutputStream
     )
-            throws PwmUnrecoverableException, PwmOperationalException, IOException
+            throws PwmUnrecoverableException, PwmOperationalException, IOException, ExecutionException, InterruptedException
     {
         final ReportRecordWriter recordWriter = reportProcessRequest.getReportType() == ReportProcessRequest.ReportType.json
                 ? new ReportJsonRecordWriter( zipOutputStream )
-                : new ReportCsvRecordWriter( zipOutputStream, pwmApplication, locale );
+                : new ReportCsvRecordWriter( zipOutputStream, pwmDomain, reportProcessRequest.getLocale() );
 
-        final boolean recordLimitReached = processReport( reportProcessRequest, zipOutputStream, recordWriter );
+        result = executeDomainReport( reportProcessRequest, zipOutputStream, recordWriter );
+
+        checkCancel( zipOutputStream );
+        ReportProcessUtil.outputSummary( pwmDomain, summaryCalculator, reportProcessRequest.getLocale(), zipOutputStream );
 
         checkCancel( zipOutputStream );
-        outputSummary( zipOutputStream );
+        ReportProcessUtil.outputResult( zipOutputStream, result );
 
         checkCancel( zipOutputStream );
-        outputResult( reportProcessRequest, zipOutputStream, recordLimitReached );
+        ReportProcessUtil.outputErrors( zipOutputStream, recordErrorMessages );
 
         log( PwmLogLevel.TRACE, () -> "completed report generation with " + recordCounter.longValue() + " records at "
-                + processRateMeter.prettyEps( locale ), TimeDuration.fromCurrent( startTime ) );
+                + processRateMeter.prettyEps( reportProcessRequest.getLocale() ), TimeDuration.fromCurrent( startTime ) );
 
     }
 
@@ -241,183 +247,136 @@ public class ReportProcess implements AutoCloseable
         }
     }
 
-    private void outputResult(
-            final ReportProcessRequest request,
-            final ZipOutputStream zipOutputStream,
-            final boolean recordLimitReached
-    )
-            throws IOException
-    {
-        final ReportProcessResult result = new ReportProcessResult(
-                request,
-                this.recordCounter.get(),
-                startTime,
-                Instant.now(),
-                TimeDuration.fromCurrent( startTime ),
-                recordLimitReached );
-
-        final String jsonData = JsonFactory.get().serialize( result, ReportProcessResult.class, JsonProvider.Flag.PrettyPrint );
-
-        zipOutputStream.putNextEntry( new ZipEntry( "result.json" ) );
-        zipOutputStream.write( jsonData.getBytes( PwmConstants.DEFAULT_CHARSET ) );
-        zipOutputStream.closeEntry();
-    }
-
-    private void outputSummary(
-            final ZipOutputStream zipOutputStream
-    )
-            throws IOException
-    {
-        {
-            zipOutputStream.putNextEntry( new ZipEntry( "summary.json" ) );
-            outputJsonSummaryToZip( summaryData, zipOutputStream );
-            zipOutputStream.closeEntry();
-        }
-        {
-            zipOutputStream.putNextEntry( new ZipEntry( "summary.csv" ) );
-            outputSummaryToCsv( pwmApplication.getConfig(), summaryData, zipOutputStream, locale );
-            zipOutputStream.closeEntry();
-        }
-    }
-
-    private boolean processReport(
+    private ReportProcessResult executeDomainReport(
             final ReportProcessRequest reportProcessRequest,
             final ZipOutputStream zipOutputStream,
             final ReportRecordWriter recordWriter
     )
-            throws IOException, PwmUnrecoverableException, PwmOperationalException
+            throws IOException, PwmUnrecoverableException, PwmOperationalException, InterruptedException
     {
-        zipOutputStream.putNextEntry( new ZipEntry( recordWriter.getZipName() ) );
+        final Instant startTime = Instant.now();
 
+        zipOutputStream.putNextEntry( new ZipEntry( recordWriter.getZipName() ) );
         recordWriter.outputHeader();
 
-        int processCounter = 0;
+        final long jobTimeoutMs = reportSettings.getReportJobTimeout().asMillis();
+
+        int recordCounter = 0;
+        int errorCounter = 0;
         boolean recordLimitReached = false;
 
-        for (
-                final Iterator<PwmDomain> domainIterator = applicableDomains( reportProcessRequest ).iterator();
-                domainIterator.hasNext() && !cancelFlag.get() && !recordLimitReached;
-        )
-        {
-            final PwmDomain pwmDomain = domainIterator.next();
+        final CompletionWrapper<UserReportRecord> completion = createAndSubmitRecordReaderTasks( reportProcessRequest );
 
-            for (
-                    final Iterator<UserReportRecord> reportRecordQueue = executeUserRecordReadJobs( reportProcessRequest, pwmDomain );
-                    reportRecordQueue.hasNext() && !cancelFlag.get() && !recordLimitReached;
-            )
+        try
+        {
+            while ( recordCounter < completion.getItemCount() && !cancelFlag.get() && !recordLimitReached )
             {
-                processCounter++;
+                final Future<UserReportRecord> future = completion.getCompletionService().poll( jobTimeoutMs, TimeUnit.MILLISECONDS );
+                recordCounter++;
 
-                if ( processCounter >= reportProcessRequest.getMaximumRecords() )
+                try
                 {
-                    recordLimitReached = true;
+                    final UserReportRecord nextRecord = future.get();
+                    recordWriter.outputRecord( nextRecord );
+                    perRecordOutputTasks( nextRecord, zipOutputStream );
+                }
+                catch ( final Exception e )
+                {
+                    errorCounter++;
+
+                    final String msg = JavaHelper.readHostileExceptionMessage( e.getCause() );
+
+                    log( PwmLogLevel.TRACE, () -> msg, null );
+
+                    if ( recordErrorMessages.size() < 1000 )
+                    {
+                        recordErrorMessages.add( msg );
+                    }
                 }
 
-                final UserReportRecord userReportRecord = reportRecordQueue.next();
-                final boolean lastRecord = recordLimitReached || ( !reportRecordQueue.hasNext() && !domainIterator.hasNext() );
-                recordWriter.outputRecord( userReportRecord, lastRecord );
-                perRecordOutputTasks( userReportRecord, zipOutputStream );
+                if ( recordCounter >= reportProcessRequest.getMaximumRecords() )
+                {
+                    recordLimitReached = true;
+                }
             }
-        }
 
-        recordWriter.outputFooter();
-        recordWriter.close();
+            recordWriter.outputFooter();
+            recordWriter.close();
+        }
+        finally
+        {
+            completion.getExecutorService().shutdown();
+        }
 
-        zipOutputStream.closeEntry();
-        return recordLimitReached;
+        final Instant finishTime = Instant.now();
+        final TimeDuration duration = TimeDuration.between( startTime, finishTime );
+        return new ReportProcessResult(
+                reportProcessRequest,
+                recordCounter,
+                errorCounter,
+                startTime,
+                finishTime,
+                duration,
+                recordLimitReached,
+                pwmDomain.getPwmApplication().getInstanceID(),
+                Long.toString( reportId ) );
     }
 
-    private Iterator<UserReportRecord> executeUserRecordReadJobs(
-            final ReportProcessRequest reportProcessRequest,
-            final PwmDomain pwmDomain
+    private CompletionWrapper<UserReportRecord> createAndSubmitRecordReaderTasks(
+            final ReportProcessRequest reportProcessRequest
     )
             throws PwmUnrecoverableException, PwmOperationalException
     {
-        final Queue<UserIdentity> identityIterator = readUserListFromLdap( reportProcessRequest, pwmDomain );
-        final ExecutorService executor = pwmApplication.getReportService().getExecutor();
-        final Queue<Future<UserReportRecord>> returnQueue = new ArrayDeque<>();
-        while ( !identityIterator.isEmpty() )
-        {
-            returnQueue.add( executor.submit( new UserReportRecordReaderTask( identityIterator.poll() ) ) );
-        }
-        return new FutureUserReportRecordIterator( returnQueue );
-    }
+        final Queue<UserIdentity> identityIterator = readUserListFromLdap( reportProcessRequest );
 
-    private Collection<PwmDomain> applicableDomains( final ReportProcessRequest reportProcessRequest )
-    {
-        if ( reportProcessRequest.getDomainID() != null )
+        final ExecutorService executorService = PwmScheduler.makeMultiThreadExecutor(
+                reportSettings.getReportJobThreads(),
+                pwmDomain.getPwmApplication(),
+                getSessionLabel(),
+                ReportProcess.class,
+                "reportId-" + reportId );
+
+        final CompletionService<UserReportRecord> completionService = new ExecutorCompletionService<>( executorService );
+
+        int itemCount = 0;
+        while ( !identityIterator.isEmpty() )
         {
-            return Collections.singleton( pwmApplication.domains().get( reportProcessRequest.getDomainID() ) );
+            final UserIdentity nextIdentity = identityIterator.poll();
+            final UserReportRecordReaderTask task = new UserReportRecordReaderTask( this, nextIdentity );
+            completionService.submit( task );
+            itemCount++;
         }
 
-        return pwmApplication.domains().values();
+        return new CompletionWrapper<>( completionService, executorService, itemCount );
     }
 
     private void perRecordOutputTasks( final UserReportRecord userReportRecord, final ZipOutputStream zipOutputStream )
             throws IOException
     {
         checkCancel( zipOutputStream );
-        summaryData.update( userReportRecord );
+        summaryCalculator.update( userReportRecord );
         recordCounter.incrementAndGet();
         processRateMeter.markEvents( 1 );
         debugOutputLogger.conditionallyExecuteTask();
-
-        log( PwmLogLevel.TRACE, () -> "completed output of user " + UserIdentity.create(
-                        userReportRecord.getUserDN(),
-                        userReportRecord.getLdapProfile(),
-                        userReportRecord.getDomainID() ).toDisplayString(),
-                TimeDuration.fromCurrent( startTime ) );
-    }
-
-    private static void outputJsonSummaryToZip( final ReportSummaryCalculator reportSummary, final OutputStream outputStream )
-    {
-        try
-        {
-            final ReportSummaryData data = ReportSummaryData.fromCalculator( reportSummary );
-            final String json = JsonFactory.get().serialize( data, ReportSummaryData.class, JsonProvider.Flag.PrettyPrint );
-            outputStream.write( json.getBytes( PwmConstants.DEFAULT_CHARSET ) );
-        }
-        catch ( final IOException e )
-        {
-            throw new PwmInternalException( e.getMessage(), e );
-        }
-    }
-
-    private UserReportRecord readUserReportRecord(
-            final UserIdentity userIdentity
-    )
-    {
-        try
-        {
-            final UserInfo userInfo = UserInfoFactory.newUserInfoUsingProxyForOfflineUser(
-                    pwmApplication,
-                    sessionLabel,
-                    userIdentity );
-
-            return UserReportRecord.fromUserInfo( userInfo );
-        }
-        catch ( final PwmUnrecoverableException e )
-        {
-            throw PwmInternalException.fromPwmException( e );
-        }
     }
 
     Queue<UserIdentity> readUserListFromLdap(
-            final ReportProcessRequest reportProcessRequest,
-            final PwmDomain pwmDomain
+            final ReportProcessRequest reportProcessRequest
     )
             throws PwmUnrecoverableException, PwmOperationalException
     {
         final Instant loopStartTime = Instant.now();
         final int maxSearchSize = ( int ) JavaHelper.rangeCheck( 0, reportSettings.getMaxSearchSize(), reportProcessRequest.getMaximumRecords() );
-        log( PwmLogLevel.TRACE, () -> "beginning ldap search process for domain '" + pwmDomain.getDomainID() + "'", null );
 
+
+        final PwmDomain pwmDomain = getPwmDomain();
+        log( PwmLogLevel.TRACE, () -> "beginning ldap search process for domain '" + pwmDomain.getDomainID() + "'", null );
         final List<UserPermission> searchFilters = reportSettings.getSearchFilter().get( pwmDomain.getDomainID() );
 
         final List<UserIdentity> searchResults = UserPermissionUtility.discoverMatchingUsers(
                 pwmDomain,
                 searchFilters,
-                sessionLabel,
+                reportProcessRequest.getSessionLabel(),
                 maxSearchSize,
                 reportSettings.getSearchTimeout() );
 
@@ -469,73 +428,26 @@ public class ReportProcess implements AutoCloseable
             log( PwmLogLevel.TRACE, () -> "cancelling report process", null );
             cancelFlag.set( true );
         }
-        reportServiceSemaphore.release();
-    }
-
-    public class UserReportRecordReaderTask implements Callable<UserReportRecord>
-    {
-        private final UserIdentity userIdentity;
-
-        public UserReportRecordReaderTask( final UserIdentity userIdentity )
-        {
-            this.userIdentity = userIdentity;
-        }
 
-        @Override
-        public UserReportRecord call()
-        {
-            if ( cancelFlag.get() )
-            {
-                throw new RuntimeException( "report process job cancelled" );
-            }
-            return readUserReportRecord( userIdentity );
-        }
-    }
-
-    class FutureUserReportRecordIterator implements Iterator<UserReportRecord>
-    {
-        private final Queue<Future<UserReportRecord>> reportRecordQueue;
-
-        FutureUserReportRecordIterator( final Queue<Future<UserReportRecord>> reportRecordQueue )
-        {
-            this.reportRecordQueue = reportRecordQueue;
-        }
-
-        @Override
-        public boolean hasNext()
-        {
-            return !reportRecordQueue.isEmpty();
-        }
-
-        @Override
-        public UserReportRecord next()
-        {
-            try
-            {
-                final Future<UserReportRecord> future = reportRecordQueue.poll();
-                if ( future == null )
-                {
-                    throw new NoSuchMethodException();
-                }
-                return future.get();
-            }
-            catch ( final InterruptedException | ExecutionException | NoSuchMethodException e )
-            {
-                log( PwmLogLevel.TRACE, () -> "user report record job failure: " + e.getMessage(), null );
-                throw new RuntimeException( e );
-            }
-        }
+        reportService.closeReportProcess( this );
     }
 
     private long nextProcessId()
     {
         return REPORT_ID_LOCK.exec( () ->
         {
-            final long lastId = pwmApplication.readAppAttribute( AppAttribute.REPORT_COUNTER, Long.class ).orElse( 0L );
+            final long lastId = pwmDomain.getPwmApplication().readAppAttribute( AppAttribute.REPORT_COUNTER, Long.class ).orElse( 0L );
             final long nextId = JavaHelper.nextPositiveLong( lastId );
-            pwmApplication.writeAppAttribute( AppAttribute.REPORT_COUNTER, nextId );
+            pwmDomain.getPwmApplication().writeAppAttribute( AppAttribute.REPORT_COUNTER, nextId );
             return nextId;
         } );
     }
 
+    @Value
+    private static class CompletionWrapper<E>
+    {
+        private final CompletionService<E> completionService;
+        private final ExecutorService executorService;
+        private final int itemCount;
+    }
 }

server/src/main/java/password/pwm/svc/report/ReportProcessRequest.java

@@ -22,9 +22,12 @@ package password.pwm.svc.report;
 
 import lombok.Builder;
 import lombok.Value;
+import password.pwm.PwmConstants;
 import password.pwm.bean.DomainID;
+import password.pwm.bean.SessionLabel;
 
 import java.io.Serializable;
+import java.util.Locale;
 
 @Value
 @Builder
@@ -32,6 +35,11 @@ public class ReportProcessRequest implements Serializable
 {
     private final DomainID domainID;
 
+    @Builder.Default
+    private final Locale locale = PwmConstants.DEFAULT_LOCALE;
+
+    private final SessionLabel sessionLabel;
+
     @Builder.Default
     private long maximumRecords = Long.MAX_VALUE;
 

server/src/main/java/password/pwm/svc/report/ReportProcessResult.java

@@ -31,8 +31,11 @@ public class ReportProcessResult implements Serializable
 {
     private final ReportProcessRequest request;
     private final long recordCount;
+    private final long errorCount;
     private final Instant startTime;
     private final Instant finishTime;
     private final TimeDuration timeDuration;
     private final boolean recordLimitReached;
+    private final String instanceId;
+    private final String reportId;
 }

server/src/main/java/password/pwm/svc/report/ReportProcessUtil.java

@@ -0,0 +1,144 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.svc.report;
+
+import org.apache.commons.csv.CSVPrinter;
+import password.pwm.PwmConstants;
+import password.pwm.PwmDomain;
+import password.pwm.config.AppConfig;
+import password.pwm.error.PwmInternalException;
+import password.pwm.util.java.CollectionUtil;
+import password.pwm.util.java.PwmUtil;
+import password.pwm.util.json.JsonFactory;
+import password.pwm.util.json.JsonProvider;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.List;
+import java.util.Locale;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+public class ReportProcessUtil
+{
+    static void outputResult(
+            final ZipOutputStream zipOutputStream,
+            final ReportProcessResult result
+    )
+            throws IOException
+    {
+        final String jsonData = JsonFactory.get().serialize( result, ReportProcessResult.class, JsonProvider.Flag.PrettyPrint );
+
+        zipOutputStream.putNextEntry( new ZipEntry( "result.json" ) );
+        zipOutputStream.write( jsonData.getBytes( PwmConstants.DEFAULT_CHARSET ) );
+        zipOutputStream.closeEntry();
+    }
+
+    static void outputSummary(
+            final PwmDomain pwmDomain,
+            final ReportSummaryCalculator summaryData,
+            final Locale locale,
+            final ZipOutputStream zipOutputStream
+    )
+            throws IOException
+    {
+        zipOutputStream.putNextEntry( new ZipEntry( "summary.json" ) );
+        try
+        {
+            outputJsonSummaryToZip( summaryData, zipOutputStream );
+        }
+        finally
+        {
+            zipOutputStream.closeEntry();
+        }
+
+        zipOutputStream.putNextEntry( new ZipEntry( "summary.csv" ) );
+        try
+        {
+            outputSummaryToCsv( pwmDomain.getPwmApplication().getConfig(), summaryData, zipOutputStream, locale );
+        }
+        finally
+        {
+            zipOutputStream.closeEntry();
+        }
+    }
+
+    private static void outputJsonSummaryToZip( final ReportSummaryCalculator reportSummary, final OutputStream outputStream )
+    {
+        try
+        {
+            final ReportSummaryData data = ReportSummaryData.fromCalculator( reportSummary );
+            final String json = JsonFactory.get().serialize( data, ReportSummaryData.class, JsonProvider.Flag.PrettyPrint );
+            outputStream.write( json.getBytes( PwmConstants.DEFAULT_CHARSET ) );
+        }
+        catch ( final IOException e )
+        {
+            throw new PwmInternalException( e.getMessage(), e );
+        }
+    }
+
+    private static void outputSummaryToCsv(
+            final AppConfig config,
+            final ReportSummaryCalculator reportSummaryData,
+            final OutputStream outputStream,
+            final Locale locale
+    )
+            throws IOException
+    {
+
+        final List<ReportSummaryCalculator.PresentationRow> outputList = reportSummaryData.asPresentableCollection( config, locale );
+
+        final CSVPrinter csvPrinter = PwmUtil.makeCsvPrinter( outputStream );
+
+        for ( final ReportSummaryCalculator.PresentationRow presentationRow : outputList )
+        {
+            csvPrinter.printRecord( presentationRow.toStringList() );
+        }
+
+        csvPrinter.flush();
+    }
+
+    public static void outputErrors( final ZipOutputStream zipOutputStream, final List<String> recordErrorMessages )
+            throws IOException
+    {
+        if ( CollectionUtil.isEmpty( recordErrorMessages ) )
+        {
+            return;
+        }
+
+        zipOutputStream.putNextEntry( new ZipEntry( "errors.csv" ) );
+        try
+        {
+            final CSVPrinter csvPrinter = PwmUtil.makeCsvPrinter( zipOutputStream );
+
+            for ( final String errorMsg : recordErrorMessages )
+            {
+                csvPrinter.printRecord( List.of( errorMsg ) );
+            }
+
+            csvPrinter.flush();
+        }
+        finally
+        {
+            zipOutputStream.closeEntry();
+        }
+    }
+}

server/src/main/java/password/pwm/svc/report/ReportRecordWriter.java

@@ -29,7 +29,7 @@ interface ReportRecordWriter
     void outputHeader()
             throws IOException;
 
-    void outputRecord( UserReportRecord userReportRecord, boolean lastRecord )
+    void outputRecord( UserReportRecord userReportRecord )
             throws IOException;
 
     void outputFooter()

server/src/main/java/password/pwm/svc/report/ReportService.java

@@ -20,35 +20,39 @@
 
 package password.pwm.svc.report;
 
-import org.jetbrains.annotations.NotNull;
 import password.pwm.PwmApplication;
+import password.pwm.PwmConstants;
+import password.pwm.PwmDomain;
 import password.pwm.bean.DomainID;
-import password.pwm.bean.SessionLabel;
 import password.pwm.config.option.DataStorageMethod;
 import password.pwm.error.PwmException;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
-import password.pwm.util.PwmScheduler;
+import password.pwm.util.java.StatisticCounterBundle;
 
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Set;
 import java.util.WeakHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Semaphore;
 
 
 public class ReportService extends AbstractPwmService implements PwmService
 {
-    private PwmApplication pwmApplication;
+    private PwmDomain pwmDomain;
     private ReportSettings settings = ReportSettings.builder().build();
 
-    private final Semaphore activeReportSemaphore = new Semaphore( 5 );
     private final Set<ReportProcess> outstandingReportProcesses = Collections.newSetFromMap( new WeakHashMap<>() );
 
-    private ExecutorService threadPool;
+    private final StatisticCounterBundle<CounterStats> statisticCounterBundle = new StatisticCounterBundle<>( CounterStats.class );
+
+    private enum CounterStats
+    {
+        ReportsStarted,
+        ReportsCompleted,
+        RecordsRead,
+        RecordReadErrors,
+    }
 
     public ReportService( )
     {
@@ -59,9 +63,17 @@ public class ReportService extends AbstractPwmService implements PwmService
         return settings;
     }
 
-    public ExecutorService getExecutor()
+    void closeReportProcess( final ReportProcess reportProcess )
     {
-        return this.threadPool;
+        outstandingReportProcesses.remove( reportProcess );
+
+        statisticCounterBundle.increment( CounterStats.ReportsCompleted );
+
+        reportProcess.getResult().ifPresent( result ->
+        {
+            statisticCounterBundle.increment( CounterStats.RecordsRead, result.getRecordCount() );
+            statisticCounterBundle.increment( CounterStats.RecordReadErrors, result.getErrorCount() );
+        } );
     }
 
     public enum ReportCommand
@@ -74,21 +86,13 @@ public class ReportService extends AbstractPwmService implements PwmService
     protected STATUS postAbstractInit( final PwmApplication pwmApplication, final DomainID domainID )
             throws PwmException
     {
-        this.pwmApplication = pwmApplication;
+        this.pwmDomain = pwmApplication.domains().get( domainID );
         this.settings = ReportSettings.readSettingsFromConfig( this.getPwmApplication().getConfig() );
-        final int maxThreads = settings.getReportJobThreads();
-        this.threadPool = PwmScheduler.makeMultiThreadExecutor( maxThreads, pwmApplication, getSessionLabel(), ReportService.class );
         return STATUS.OPEN;
     }
 
     public void shutdownImpl( )
     {
-        if ( threadPool != null )
-        {
-            threadPool.shutdown();
-            threadPool = null;
-        }
-
         setStatus( STATUS.CLOSED );
 
         for ( final ReportProcess reportProcess : outstandingReportProcesses )
@@ -101,12 +105,12 @@ public class ReportService extends AbstractPwmService implements PwmService
     }
 
     public ReportProcess createReportProcess(
-            final Locale locale,
-            @NotNull final SessionLabel sessionLabel
+            final ReportProcessRequest request
     )
     {
-        final ReportProcess reportProcess = ReportProcess.createReportProcess( pwmApplication, activeReportSemaphore, settings, locale, sessionLabel );
+        final ReportProcess reportProcess = ReportProcess.createReportProcess( pwmDomain, this, request, settings );
         outstandingReportProcesses.add( reportProcess );
+        statisticCounterBundle.increment( CounterStats.ReportsStarted );
         return reportProcess;
     }
 
@@ -120,6 +124,8 @@ public class ReportService extends AbstractPwmService implements PwmService
     @Override
     public ServiceInfoBean serviceInfo( )
     {
-        return ServiceInfoBean.builder().storageMethod( DataStorageMethod.LDAP ).build();
+        return ServiceInfoBean.builder()
+                .debugProperties( statisticCounterBundle.debugStats( PwmConstants.DEFAULT_LOCALE ) )
+                .storageMethod( DataStorageMethod.LDAP ).build();
     }
 }

server/src/main/java/password/pwm/svc/report/ReportSettings.java

@@ -76,6 +76,10 @@ public class ReportSettings implements Serializable
     @Builder.Default
     private JobIntensity reportJobIntensity = JobIntensity.LOW;
 
+
+    @Builder.Default
+    private TimeDuration reportJobTimeout = TimeDuration.MINUTE;
+
     public enum JobIntensity
     {
         LOW,
@@ -96,7 +100,7 @@ public class ReportSettings implements Serializable
         builder.maxSearchSize ( ( int ) config.readSettingAsLong( PwmSetting.REPORTING_MAX_QUERY_SIZE ) );
         builder.dailyJobEnabled( config.readSettingAsBoolean( PwmSetting.REPORTING_ENABLE_DAILY_JOB ) );
         builder.searchTimeout( TimeDuration.of( Long.parseLong( config.readAppProperty( AppProperty.REPORTING_LDAP_SEARCH_TIMEOUT_MS ) ), TimeDuration.Unit.MILLISECONDS ) );
-
+        builder.reportJobTimeout( TimeDuration.of( Long.parseLong( config.readAppProperty( AppProperty.REPORTING_LDAP_JOB_TIMEOUT_MS ) ), TimeDuration.Unit.SECONDS ) );
 
         {
             int reportJobOffset = ( int ) config.readSettingAsLong( PwmSetting.REPORTING_JOB_TIME_OFFSET );
@@ -110,7 +114,7 @@ public class ReportSettings implements Serializable
 
         builder.trackDays( parseDayIntervalStr( config ) );
 
-        builder.reportJobThreads( Integer.parseInt( config.readAppProperty( AppProperty.REPORTING_LDAP_SEARCH_THREADS ) ) );
+        builder.reportJobThreads( Integer.parseInt( config.readAppProperty( AppProperty.REPORTING_LDAP_JOB_THREADS ) ) );
 
         builder.reportJobIntensity( config.readSettingAsEnum( PwmSetting.REPORTING_JOB_INTENSITY, JobIntensity.class ) );
 

server/src/main/java/password/pwm/svc/report/ReportSummaryCalculator.java

@@ -67,6 +67,11 @@ public class ReportSummaryCalculator
     {
     }
 
+    static ReportSummaryCalculator empty()
+    {
+        return ReportSummaryCalculator.newSummaryData( Collections.singletonList( 1 ) );
+    }
+
     static ReportSummaryCalculator newSummaryData( final List<Integer> trackedDays )
     {
         final ReportSummaryCalculator reportSummaryData = new ReportSummaryCalculator();

server/src/main/java/password/pwm/svc/report/UserReportRecordReaderTask.java

@@ -0,0 +1,85 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.svc.report;
+
+import password.pwm.bean.UserIdentity;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.ldap.UserInfoFactory;
+import password.pwm.user.UserInfo;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogLevel;
+
+import java.time.Instant;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CancellationException;
+
+public class UserReportRecordReaderTask implements Callable<UserReportRecord>
+{
+    private final ReportProcess reportProcess;
+    private final UserIdentity userIdentity;
+
+    public UserReportRecordReaderTask( final ReportProcess reportProcess, final UserIdentity userIdentity )
+    {
+        this.reportProcess = reportProcess;
+        this.userIdentity = userIdentity;
+    }
+
+    @Override
+    public UserReportRecord call()
+            throws PwmUnrecoverableException
+    {
+        if ( reportProcess.isCancelled() )
+        {
+            throw new CancellationException( "report process job cancelled" );
+        }
+
+        try
+        {
+            return readUserReportRecord( userIdentity );
+        }
+        catch ( final Exception e )
+        {
+            final String msg = "error while reading report record for user " + userIdentity.toDisplayString() + ": error " + e.getMessage();
+            throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_LDAP_DATA_ERROR, msg ), e );
+        }
+    }
+
+    private UserReportRecord readUserReportRecord(
+            final UserIdentity userIdentity
+    )
+            throws PwmUnrecoverableException
+    {
+        final Instant startTime = Instant.now();
+        final UserInfo userInfo = UserInfoFactory.newUserInfoUsingProxyForOfflineUser(
+                reportProcess.getPwmDomain().getPwmApplication(),
+                reportProcess.getSessionLabel(),
+                userIdentity );
+
+        final UserReportRecord record = UserReportRecord.fromUserInfo( userInfo );
+
+        reportProcess.log( PwmLogLevel.TRACE, () -> "completed output of user " + userIdentity.toDisplayString(),
+                TimeDuration.fromCurrent( startTime ) );
+
+        return record;
+    }
+}

server/src/main/java/password/pwm/util/PwmScheduler.java

@@ -289,7 +289,7 @@ public class PwmScheduler
     )
     {
         final ThreadPoolExecutor executor = new ThreadPoolExecutor(
-                1,
+                maxThreadCount,
                 maxThreadCount,
                 1, TimeUnit.SECONDS,
                 new LinkedBlockingQueue<>(),

server/src/main/java/password/pwm/util/cli/commands/UserReportCommand.java

@@ -72,11 +72,14 @@ public class UserReportCommand extends AbstractCliCommand
                 return;
             }
 
-            final ReportProcessRequest reportProcessRequest = ReportProcessRequest.builder().build();
+            final ReportProcessRequest reportProcessRequest = ReportProcessRequest.builder()
+                    .locale( PwmConstants.DEFAULT_LOCALE )
+                    .sessionLabel( SessionLabel.CLI_SESSION_LABEL )
+                    .build();
 
-            try ( ReportProcess reportProcess = reportService.createReportProcess( PwmConstants.DEFAULT_LOCALE, SessionLabel.CLI_SESSION_LABEL ) )
+            try ( ReportProcess reportProcess = reportService.createReportProcess( reportProcessRequest ) )
             {
-                reportProcess.startReport( reportProcessRequest, outputFileStream );
+                reportProcess.startReport( outputFileStream );
             }
         }
         catch ( final IOException | PwmUnrecoverableException e )

server/src/main/resources/password/pwm/AppProperty.properties

@@ -302,8 +302,9 @@ queue.sms.maxCount=100000
 queue.syslog.retryTimeoutMs=30000
 queue.syslog.maxAgeMs=2592000000
 queue.syslog.maxCount=100000
+reporting.ldap.recordJob.timeoutMs=60000
+reporting.ldap.recordJob.threads=30
 reporting.ldap.searchTimeoutMs=1800000
-reporting.ldap.searchThreads=30
 reporting.maxReportAgeSeconds=864000
 recaptcha.clientJsUrl=//www.recaptcha.net/recaptcha/api.js
 recaptcha.clientIframeUrl=//www.recaptcha.net/recaptcha/api/noscript

webapp/src/main/webapp/WEB-INF/jsp/admin-reporting.jsp

@@ -63,7 +63,7 @@
                             Maximum Record Count
                         </td>
                         <td>
-                            <input type="number" name="recordCount" id="recordCount" value="1000"/>
+                            <input type="number" name="recordCount" id="recordCount" value="1000" min="0" max="<%=Integer.MAX_VALUE%>"/>
                         </td>
                     </tr>
                 </table>

webapp/src/main/webapp/WEB-INF/jsp/admin-user-debug.jsp

@@ -344,9 +344,9 @@
                         <tr>
                             <td>ID</td>
                             <td><pwm:display key="<%=Display.Value_NotApplicable.toString()%>"/></td>
-                            <td><%=JspUtility.friendlyWrite(pageContext, configPolicy.getId().stringValue())%></td>
-                            <td><%=JspUtility.friendlyWrite(pageContext, ldapPolicy.getId().stringValue())%></td>
-                            <td><%=JspUtility.friendlyWrite(pageContext, userPolicy.getId().stringValue())%></td>
+                            <td><%=JspUtility.friendlyWrite(pageContext, () -> configPolicy.getId().stringValue())%></td>
+                            <td><%=JspUtility.friendlyWrite(pageContext, () -> ldapPolicy.getId().stringValue())%></td>
+                            <td><%=JspUtility.friendlyWrite(pageContext, () -> userPolicy.getId().stringValue())%></td>
                         </tr>
                         <tr>
                             <td>Display Name</td>