introduce hikari db load balancer

build/checkstyle-import.xml

@@ -135,6 +135,7 @@
 
     <!-- database -->
     <subpackage name="svc.db">
+        <allow pkg="com.zaxxer.hikari"/>
         <allow pkg="java.sql"/>
     </subpackage>
 

lib-util/src/main/java/password/pwm/util/java/CollectionUtil.java

@@ -99,7 +99,7 @@ public final class CollectionUtil
     {
         if ( CollectionUtil.isEmpty( source ) )
         {
-            return new EnumMap<K, V>( classOfT );
+            return new EnumMap<>( classOfT );
         }
 
         return source.entrySet().stream()
@@ -157,6 +157,27 @@ public final class CollectionUtil
         return enumMapToStringMap( inputMap, Enum::name );
     }
 
+    public static <E extends Enum<E>> List<String> enumSetToStringList( final Set<E> inputSet )
+    {
+        return enumSetToStringList( inputSet, Enum::name );
+    }
+
+    public static <E extends Enum<E>> List<String> enumSetToStringList(
+            final Set<E> inputSet,
+            final Function<E, String> keyToStringFunction
+    )
+    {
+        if ( CollectionUtil.isEmpty( inputSet ) )
+        {
+            return List.of();
+        }
+
+        return inputSet.stream()
+                .filter( Objects::nonNull )
+                .map( keyToStringFunction )
+                .toList();
+    }
+
     public static <K> boolean isEmpty( final Collection<K> collection )
     {
         return collection == null || collection.isEmpty();
@@ -183,8 +204,7 @@ public final class CollectionUtil
 
     public static <E> List<E> iteratorToList( final Iterator<E> iterator )
     {
-        return iteratorToStream( iterator )
-                .collect( Collectors.toUnmodifiableList() );
+        return iteratorToStream( iterator ).toList();
     }
 
     /**
@@ -217,7 +237,7 @@ public final class CollectionUtil
     public static <T, R> List<R> convertListType( final List<T> input, final Function<T, R> convertFunction )
 
     {
-        return stripNulls( input ).stream().map( convertFunction ).collect( Collectors.toUnmodifiableList() );
+        return stripNulls( input ).stream().map( convertFunction ).toList();
     }
 
     private static <K, V> boolean testMapEntryForNotNull( final Map.Entry<K, V> entry )

server/pom.xml

@@ -180,6 +180,11 @@
             <artifactId>xmlchai</artifactId>
             <version>0.1.3</version>
         </dependency>
+        <dependency>
+            <groupId>com.zaxxer</groupId>
+            <artifactId>HikariCP</artifactId>
+            <version>5.0.1</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.directory.api</groupId>
             <artifactId>api-all</artifactId>

server/src/main/java/password/pwm/PwmAboutProperty.java

@@ -102,13 +102,13 @@ public enum PwmAboutProperty
     java_gcName( "Java GC Name", pwmApplication -> readGcName() ),
 
     database_driverName( null,
-            pwmApplication -> pwmApplication.getDatabaseService().getConnectionDebugProperties().get( DatabaseService.DatabaseAboutProperty.driverName ) ),
+            pwmApplication -> pwmApplication.getDatabaseService().getConnectionDebugProperties().get( DatabaseService.DatabaseDebugProperty.driverName ) ),
     database_driverVersion( null,
-            pwmApplication -> pwmApplication.getDatabaseService().getConnectionDebugProperties().get( DatabaseService.DatabaseAboutProperty.driverVersion ) ),
+            pwmApplication -> pwmApplication.getDatabaseService().getConnectionDebugProperties().get( DatabaseService.DatabaseDebugProperty.driverVersion ) ),
     database_databaseProductName( null,
-            pwmApplication -> pwmApplication.getDatabaseService().getConnectionDebugProperties().get( DatabaseService.DatabaseAboutProperty.databaseProductName ) ),
+            pwmApplication -> pwmApplication.getDatabaseService().getConnectionDebugProperties().get( DatabaseService.DatabaseDebugProperty.databaseProductName ) ),
     database_databaseProductVersion( null,
-            pwmApplication -> pwmApplication.getDatabaseService().getConnectionDebugProperties().get( DatabaseService.DatabaseAboutProperty.databaseProductVersion ) ),;
+            pwmApplication -> pwmApplication.getDatabaseService().getConnectionDebugProperties().get( DatabaseService.DatabaseDebugProperty.databaseProductVersion ) ),;
 
     private final String label;
     private final transient Function<PwmApplication, String> value;

server/src/main/java/password/pwm/PwmConstants.java

@@ -20,7 +20,6 @@
 
 package password.pwm;
 
-import com.novell.ldapchai.ChaiConstant;
 import org.apache.commons.csv.CSVFormat;
 import password.pwm.util.java.StringUtil;
 
@@ -229,7 +228,7 @@ public abstract class PwmConstants
         final Map<String, String> returnMap = new TreeMap<>();
         try
         {
-            final Enumeration<URL> resources = ChaiConstant.class.getClassLoader().getResources( manifestFileName );
+            final Enumeration<URL> resources = PwmConstants.class.getClassLoader().getResources( manifestFileName );
             while ( resources.hasMoreElements() )
             {
                 try ( InputStream inputStream = resources.nextElement().openStream() )

server/src/main/java/password/pwm/health/ConfigurationChecker.java

@@ -71,6 +71,7 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.NoSuchElementException;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Supplier;
@@ -234,14 +235,14 @@ public class ConfigurationChecker implements HealthSupplier
                         appPropertyKey ) );
             }
 
-            if ( config.readSettingAsBoolean( PwmSetting.DISPLAY_SHOW_DETAILED_ERRORS ) )
+            if ( config.getAppConfig().readSettingAsBoolean( PwmSetting.DISPLAY_SHOW_DETAILED_ERRORS ) )
             {
                 records.add( HealthRecord.forMessage(
                         DomainID.systemId(),
                         HealthMessage.Config_ShowDetailedErrors,
                         PwmSetting.DISPLAY_SHOW_DETAILED_ERRORS.toMenuLocationDebug( null, locale ) ) );
             }
-            return Collections.unmodifiableList( records );
+            return List.copyOf( records );
         }
     }
 
@@ -255,9 +256,9 @@ public class ConfigurationChecker implements HealthSupplier
 
             final List<HealthRecord> records = new ArrayList<>();
             final String siteUrl = config.getAppConfig().readSettingAsString( PwmSetting.PWM_SITE_URL );
+            final String defaultSiteUrl = ( String) PwmSetting.PWM_SITE_URL.getDefaultValue( config.getTemplate() ).toNativeObject();
 
-            if ( siteUrl == null || siteUrl.isEmpty() || siteUrl.equals(
-                    PwmSetting.PWM_SITE_URL.getDefaultValue( config.getTemplate() ).toNativeObject() ) )
+            if ( StringUtil.isEmpty( siteUrl ) || Objects.equals( siteUrl, defaultSiteUrl ) )
             {
                 records.add( HealthRecord.forMessage(
                         config.getDomainID(),

server/src/main/java/password/pwm/health/DebugOutputService.java

@@ -28,7 +28,7 @@ import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
-import password.pwm.util.debug.DebugItemGenerator;
+import password.pwm.util.debug.DebugGenerator;
 import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.java.JavaHelper;
@@ -143,7 +143,7 @@ public class DebugOutputService extends AbstractPwmService implements PwmService
             }
 
             final int rotationCount = JavaHelper.silentParseInt( pwmApplication.getConfig().readAppProperty( AppProperty.HEALTH_SUPPORT_BUNDLE_FILE_WRITE_COUNT ), 10 );
-            final DebugItemGenerator debugItemGenerator = new DebugItemGenerator( pwmApplication, getSessionLabel() );
+            final DebugGenerator debugItemGenerator = new DebugGenerator( pwmApplication, getSessionLabel() );
 
             final Path supportPath = appPath.resolve( "support" );
 

server/src/main/java/password/pwm/http/servlet/ClientApiServlet.java

@@ -444,9 +444,13 @@ public class ClientApiServlet extends ControlledPwmServlet
         final RestStatisticsServer.OutputVersion1.JsonOutput jsonOutput = new RestStatisticsServer.OutputVersion1.JsonOutput();
         jsonOutput.EPS = RestStatisticsServer.OutputVersion1.addEpsStats( statisticsManager );
 
-        if ( statName != null && statName.length() > 0 )
+        if ( !StringUtil.isEmpty( statName ) )
         {
-            jsonOutput.nameData = RestStatisticsServer.OutputVersion1.doNameStat( statisticsManager, statName, days );
+            jsonOutput.nameData = RestStatisticsServer.OutputVersion1.doNameStat(
+                    pwmRequest.getPwmRequestContext(),
+                    statisticsManager,
+                    statName,
+                    days );
         }
         else
         {

server/src/main/java/password/pwm/http/servlet/admin/system/ConfigManagerServlet.java

@@ -55,7 +55,7 @@ import password.pwm.i18n.Admin;
 import password.pwm.i18n.Config;
 import password.pwm.i18n.Display;
 import password.pwm.ldap.LdapPermissionCalculator;
-import password.pwm.util.debug.DebugItemGenerator;
+import password.pwm.util.debug.DebugGenerator;
 import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.EnumUtil;
@@ -360,7 +360,7 @@ public class ConfigManagerServlet extends AbstractPwmServlet
     private void doGenerateSupportZip( final PwmRequest pwmRequest )
             throws IOException, PwmUnrecoverableException
     {
-        final DebugItemGenerator debugItemGenerator = new DebugItemGenerator( pwmRequest.getPwmApplication(), pwmRequest.getLabel() );
+        final DebugGenerator debugItemGenerator = new DebugGenerator( pwmRequest.getPwmApplication(), pwmRequest.getLabel() );
         final PwmResponse resp = pwmRequest.getPwmResponse();
         resp.markAsDownload( HttpContentType.zip, PwmConstants.PWM_APP_NAME + "-Support.zip" );
         try ( ZipOutputStream zipOutput = new ZipOutputStream( resp.getOutputStream(), PwmConstants.DEFAULT_CHARSET ) )

server/src/main/java/password/pwm/http/servlet/resource/ResourceServletService.java

@@ -59,6 +59,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Consumer;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
@@ -304,6 +305,19 @@ public class ResourceServletService extends AbstractPwmService implements PwmSer
             return;
         }
 
+        final Consumer<FileSystemUtility.FileSummaryInformation> consumer = fileSummaryInformation ->
+        {
+            try
+            {
+                checksumStream.write( fileSummaryInformation.sha512Hash().getBytes( StandardCharsets.UTF_8 ) );
+            }
+            catch ( final Exception e )
+            {
+                LOGGER.error( () -> "unable to generate resource path nonce: " + e.getMessage() );
+            }
+
+        };
+
         pwmDomain.getPwmApplication().getPwmEnvironment().getContextManager().locateWebInfFilePath().ifPresent( webInfPath ->
         {
             final Path basePath = webInfPath.getParent();
@@ -312,21 +326,8 @@ public class ResourceServletService extends AbstractPwmService implements PwmSer
                 final Path resourcePath = basePath.resolve( "public" ).resolve( "resources" );
                 if ( Files.exists( resourcePath ) )
                 {
-                    final List<FileSystemUtility.FileSummaryInformation> fileSummaryInformations =
-                            FileSystemUtility.readFileInformation( Collections.singletonList( resourcePath ) );
-                    {
-                        for ( final FileSystemUtility.FileSummaryInformation fileSummaryInformation : fileSummaryInformations  )
-                        {
-                            try
-                            {
-                                checksumStream.write( fileSummaryInformation.getSha512Hash().getBytes( StandardCharsets.UTF_8 ) );
-                            }
-                            catch ( final Exception e )
-                            {
-                                LOGGER.error( () -> "unable to generate resource path nonce: " + e.getMessage() );
-                            }
-                        }
-                    }
+                    FileSystemUtility.readFileInformation( Collections.singletonList( resourcePath ) )
+                            .forEach( consumer::accept );
                 }
             }
         } );

server/src/main/java/password/pwm/ldap/LdapDebugDataGenerator.java

@@ -59,14 +59,15 @@ public class LdapDebugDataGenerator
     )
 
     {
+        final DomainConfig nonObfuscatedDomainConf = pwmDomain.getConfig();
+
         final List<LdapDebugInfo> returnList = new ArrayList<>();
-        for ( final LdapProfile ldapProfile : domainConfig.getLdapProfiles().values() )
+        for ( final LdapProfile ldapProfile : nonObfuscatedDomainConf.getLdapProfiles().values() )
         {
             final List<LdapDebugServerInfo> ldapDebugServerInfos = new ArrayList<>();
 
             try
             {
-                final DomainConfig nonObfuscatedDomainConf = pwmDomain.getConfig();
                 final ChaiConfiguration profileChaiConf = LdapOperationsHelper.createChaiConfiguration(
                         nonObfuscatedDomainConf,
                         ldapProfile );
@@ -80,7 +81,7 @@ public class LdapDebugDataGenerator
                                 pwmDomain,
                                 sessionLabel,
                                 ldapProfile,
-                                domainConfig,
+                                nonObfuscatedDomainConf,
                                 ldapProfile.readSettingAsString( PwmSetting.LDAP_PROXY_USER_DN ),
                                 ldapProfile.readSettingAsPassword( PwmSetting.LDAP_PROXY_USER_PASSWORD )
                         );

server/src/main/java/password/pwm/ldap/LdapDomainService.java

@@ -24,8 +24,6 @@ import com.novell.ldapchai.provider.ChaiProvider;
 import com.novell.ldapchai.provider.ChaiProviderFactory;
 import com.novell.ldapchai.provider.ChaiSetting;
 import com.novell.ldapchai.provider.ProviderStatistics;
-import lombok.Builder;
-import lombok.Value;
 import password.pwm.DomainProperty;
 import password.pwm.PwmApplication;
 import password.pwm.PwmDomain;
@@ -338,30 +336,29 @@ public class LdapDomainService extends AbstractPwmService implements PwmService
         for ( final ChaiProvider chaiProvider : chaiProviderFactory.activeProviders() )
         {
             final String bindDN = chaiProvider.getChaiConfiguration().getSetting( ChaiSetting.BIND_DN );
-            final ConnectionInfo connectionInfo = ConnectionInfo.builder()
-                    .bindDN( bindDN )
-                    .active( chaiProvider.isConnected() )
-                    .operationCount( chaiProvider.getProviderStatistics().getIncrementorStatistic( ProviderStatistics.IncrementerStatistic.OPERATION_COUNT ) )
-                    .modifyCount( chaiProvider.getProviderStatistics().getIncrementorStatistic( ProviderStatistics.IncrementerStatistic.MODIFY_COUNT ) )
-                    .readCount( chaiProvider.getProviderStatistics().getIncrementorStatistic( ProviderStatistics.IncrementerStatistic.READ_COUNT ) )
-                    .searchCount( chaiProvider.getProviderStatistics().getIncrementorStatistic( ProviderStatistics.IncrementerStatistic.SEARCH_COUNT ) )
-                    .build();
+            final ConnectionInfo connectionInfo = new ConnectionInfo(
+                    bindDN,
+                    chaiProvider.isConnected(),
+                    chaiProvider.getProviderStatistics().getIncrementorStatistic( ProviderStatistics.IncrementerStatistic.OPERATION_COUNT ),
+                    chaiProvider.getProviderStatistics().getIncrementorStatistic( ProviderStatistics.IncrementerStatistic.MODIFY_COUNT ),
+                    chaiProvider.getProviderStatistics().getIncrementorStatistic( ProviderStatistics.IncrementerStatistic.READ_COUNT ),
+                    chaiProvider.getProviderStatistics().getIncrementorStatistic( ProviderStatistics.IncrementerStatistic.SEARCH_COUNT ) );
+
 
             returnData.put( bindDN, connectionInfo );
         }
         return List.copyOf( returnData.values() );
     }
 
-    @Value
-    @Builder
-    public static class ConnectionInfo
+    public record ConnectionInfo(
+            String bindDN,
+            boolean active,
+            long operationCount,
+            long modifyCount,
+            long readCount,
+            long searchCount
+    )
     {
-        private final String bindDN;
-        private final boolean active;
-        private final long operationCount;
-        private final long modifyCount;
-        private final long readCount;
-        private final long searchCount;
     }
 
     private Map<String, String> connectionDebugInfo( )

server/src/main/java/password/pwm/svc/db/DBConfiguration.java

@@ -20,9 +20,6 @@
 
 package password.pwm.svc.db;
 
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Value;
 import password.pwm.AppProperty;
 import password.pwm.config.AppConfig;
 import password.pwm.config.PwmSetting;
@@ -32,23 +29,22 @@ import password.pwm.util.PasswordData;
 
 import java.util.Map;
 
-@Value
-@AllArgsConstructor( access = AccessLevel.PRIVATE )
-public class DBConfiguration
+public record DBConfiguration(
+        String driverClassname,
+        String connectionString,
+        String username,
+        PasswordData password,
+        String columnTypeKey,
+        String columnTypeValue,
+        ImmutableByteArray jdbcDriver,
+        int maxConnections,
+        int connectionTimeout,
+        int keyColumnLength,
+        boolean failOnIndexCreation,
+        boolean traceLogging
+)
 {
-    private final String driverClassname;
-    private final String connectionString;
-    private final String username;
-    private final PasswordData password;
-    private final String columnTypeKey;
-    private final String columnTypeValue;
-    private final ImmutableByteArray jdbcDriver;
-    private final int maxConnections;
-    private final int connectionTimeout;
-    private final int keyColumnLength;
-    private final boolean failOnIndexCreation;
-
-    public ImmutableByteArray getJdbcDriver( )
+    public ImmutableByteArray getJdbcDriver()
     {
         return jdbcDriver;
     }
@@ -86,7 +82,8 @@ public class DBConfiguration
                 maxConnections,
                 connectionTimeout,
                 keyColumnLength,
-                haltOnIndexCreateError
+                haltOnIndexCreateError,
+                config.readSettingAsBoolean( PwmSetting.DATABASE_DEBUG_TRACE )
         );
     }
 }

server/src/main/java/password/pwm/svc/db/DatabaseAccessor.java

@@ -45,7 +45,6 @@ public interface DatabaseAccessor
     {
     }
 
-
     @DbOperation
     @DbModifyOperation
     boolean put(
@@ -92,6 +91,4 @@ public interface DatabaseAccessor
     @DbOperation
     int size( DatabaseTable table ) throws
             DatabaseException;
-
-    boolean isConnected( );
 }

server/src/main/java/password/pwm/svc/db/DatabaseAccessorImpl.java

@@ -20,10 +20,11 @@
 
 package password.pwm.svc.db;
 
+import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.ClosableIterator;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.logging.PwmLogger;
 
 import java.sql.Connection;
@@ -31,14 +32,11 @@ import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.AbstractMap;
-import java.util.HashSet;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReentrantLock;
 
 /**
  * @author Jason D. Rivard
@@ -48,48 +46,27 @@ class DatabaseAccessorImpl implements DatabaseAccessor
 
     private static final PwmLogger LOGGER = PwmLogger.forClass( DatabaseAccessorImpl.class, true );
 
-    private final Connection connection;
     private final DatabaseService databaseService;
-    private final DBConfiguration dbConfiguration;
 
     private final boolean traceLogEnabled;
 
-    private static final AtomicInteger ACCESSOR_COUNTER = new AtomicInteger( 0 );
-    private final int accessorNumber = ACCESSOR_COUNTER.getAndIncrement();
+    private static final AtomicLoopIntIncrementer ACCESSOR_COUNTER = new AtomicLoopIntIncrementer();
+    private static final AtomicLoopIntIncrementer ITERATOR_COUNTER = new AtomicLoopIntIncrementer();
 
-    private static final AtomicInteger ITERATOR_COUNTER = new AtomicInteger( 0 );
-    private final Set<DBIterator> outstandingIterators = ConcurrentHashMap.newKeySet();
+    private final int accessorNumber = ACCESSOR_COUNTER.next();
 
+    private final Set<DBIterator> outstandingIterators = ConcurrentHashMap.newKeySet();
     private final AtomicBoolean closed = new AtomicBoolean( false );
 
-    private final ReentrantLock lock = new ReentrantLock();
-
     DatabaseAccessorImpl(
             final DatabaseService databaseService,
-            final DBConfiguration dbConfiguration,
-            final Connection connection,
-            final boolean traceLogEnabled
+            final DBConfiguration dbConfiguration
     )
     {
-        this.connection = connection;
-        this.dbConfiguration = dbConfiguration;
-        this.traceLogEnabled = traceLogEnabled;
+        this.traceLogEnabled = dbConfiguration.traceLogging();
         this.databaseService = databaseService;
     }
 
-
-    private void processSqlException(
-            final DatabaseUtil.DebugInfo debugInfo,
-            final SQLException e
-    )
-            throws DatabaseException
-    {
-        DatabaseUtil.rollbackTransaction( connection );
-        final DatabaseException databaseException = DatabaseUtil.convertSqlException( debugInfo, e );
-        databaseService.setLastError( databaseException.getErrorInformation() );
-        throw databaseException;
-    }
-
     @Override
     public boolean put(
             final DatabaseTable table,
@@ -102,17 +79,9 @@ class DatabaseAccessorImpl implements DatabaseAccessor
 
         final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create( "put", table, key, value );
 
-        return execute( debugInfo, ( ) ->
+        final Boolean result = execute( debugInfo, connection ->
         {
-            boolean exists = false;
-            try
-            {
-                exists = containsImpl( table, key );
-            }
-            catch ( final SQLException e )
-            {
-                processSqlException( debugInfo, e );
-            }
+            final boolean exists = containsImpl( table, key, connection );
 
             if ( exists )
             {
@@ -121,18 +90,20 @@ class DatabaseAccessorImpl implements DatabaseAccessor
                         + DatabaseService.KEY_COLUMN + "=?";
 
                 // note the value/key are reversed for this statement
-                executeUpdate( sqlText, debugInfo, value, key );
+                executeStatementWithParams( sqlText, connection, value, key );
             }
             else
             {
                 final String sqlText = "INSERT INTO " + table
                         + "(" + DatabaseService.KEY_COLUMN + ", "
                         + DatabaseService.VALUE_COLUMN + ") VALUES(?,?)";
-                executeUpdate( sqlText, debugInfo, key, value );
+                executeStatementWithParams( sqlText, connection, key, value );
             }
 
             return !exists;
         } );
+
+        return result != null && result;
     }
 
     @Override
@@ -147,26 +118,20 @@ class DatabaseAccessorImpl implements DatabaseAccessor
 
         final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create( "putIfAbsent", table, key, value );
 
-        return execute( debugInfo, ( ) ->
+        final Boolean result = execute( debugInfo, connection ->
         {
-            boolean valueExists = false;
-            try
-            {
-                valueExists = DatabaseAccessorImpl.this.containsImpl( table, key );
-            }
-            catch ( final SQLException e )
-            {
-                DatabaseAccessorImpl.this.processSqlException( debugInfo, e );
-            }
+            final boolean valueExists = DatabaseAccessorImpl.this.containsImpl( table, key, connection );
 
             if ( !valueExists )
             {
                 final String insertSql = "INSERT INTO " + table.name() + "(" + DatabaseService.KEY_COLUMN + ", " + DatabaseService.VALUE_COLUMN + ") VALUES(?,?)";
-                DatabaseAccessorImpl.this.executeUpdate( insertSql, debugInfo, key, value );
+                executeStatementWithParams( insertSql, connection, key, value );
             }
 
             return !valueExists;
         } );
+
+        return result != null && result;
     }
 
 
@@ -181,19 +146,13 @@ class DatabaseAccessorImpl implements DatabaseAccessor
 
         final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create( "contains", table, key, null );
 
-        return execute( debugInfo, ( ) ->
+        final Boolean result = execute( debugInfo, connection ->
         {
-            boolean valueExists = false;
-            try
-            {
-                valueExists = containsImpl( table, key );
-            }
-            catch ( final SQLException e )
-            {
-                processSqlException( debugInfo, e );
-            }
-            return valueExists;
+            return containsImpl( table, key, connection );
         } );
+
+        return result != null && result;
+
     }
 
     @Override
@@ -207,7 +166,7 @@ class DatabaseAccessorImpl implements DatabaseAccessor
 
         final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create( "get", table, key, null );
 
-        return execute( debugInfo, ( ) ->
+        return execute( debugInfo, connection ->
         {
             final String sqlStatement = "SELECT * FROM " + table.name() + " WHERE " + DatabaseService.KEY_COLUMN + " = ?";
 
@@ -224,10 +183,6 @@ class DatabaseAccessorImpl implements DatabaseAccessor
                     }
                 }
             }
-            catch ( final SQLException e )
-            {
-                processSqlException( debugInfo, e );
-            }
             return Optional.empty();
         } );
     }
@@ -236,15 +191,7 @@ class DatabaseAccessorImpl implements DatabaseAccessor
     public ClosableIterator<Map.Entry<String, String>> iterator( final DatabaseTable table )
             throws DatabaseException
     {
-        try
-        {
-            lock.lock();
-            return new DBIterator( table );
-        }
-        finally
-        {
-            lock.unlock();
-        }
+        return new DBIterator( table );
     }
 
     @Override
@@ -258,12 +205,10 @@ class DatabaseAccessorImpl implements DatabaseAccessor
 
         final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create( "remove", table, key, null );
 
-        execute( debugInfo, ( ) ->
+        execute( debugInfo, connection ->
         {
-
-
             final String sqlText = "DELETE FROM " + table.name() + " WHERE " + DatabaseService.KEY_COLUMN + "=?";
-            executeUpdate( sqlText, debugInfo, key );
+            executeStatementWithParams( sqlText, connection, key );
 
             return null;
         } );
@@ -277,7 +222,7 @@ class DatabaseAccessorImpl implements DatabaseAccessor
 
         final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create( "size", table, null, null );
 
-        return execute( debugInfo, ( ) ->
+        final Integer result = execute( debugInfo, connection ->
         {
             final String sqlStatement = "SELECT COUNT(" + DatabaseService.KEY_COLUMN + ") FROM " + table.name();
 
@@ -291,48 +236,13 @@ class DatabaseAccessorImpl implements DatabaseAccessor
                     }
                 }
             }
-            catch ( final SQLException e )
-            {
-                processSqlException( debugInfo, e );
-            }
 
             return 0;
         } );
-    }
-
-    boolean isValid( )
-    {
-        preCheck();
-
-        if ( connection == null )
-        {
-            return false;
-        }
 
-        try
-        {
-            if ( connection.isClosed() )
-            {
-                return false;
-            }
-
-            final int connectionTimeout = dbConfiguration.getConnectionTimeout();
-
-            if ( !connection.isValid( connectionTimeout ) )
-            {
-                return false;
-            }
-
-        }
-        catch ( final SQLException e )
-        {
-            LOGGER.debug( () -> "error while checking connection validity: " + e.getMessage() );
-        }
-
-        return true;
+        return result == null ? 0 : result;
     }
 
-
     public class DBIterator implements ClosableIterator<Map.Entry<String, String>>
     {
         private final DatabaseTable table;
@@ -340,7 +250,7 @@ class DatabaseAccessorImpl implements DatabaseAccessor
         private PreparedStatement statement;
         private Map.Entry<String, String> nextValue;
         private boolean finished;
-        private final int counter = ITERATOR_COUNTER.getAndIncrement();
+        private final int counter = ITERATOR_COUNTER.next();
 
         DBIterator( final DatabaseTable table )
                 throws DatabaseException
@@ -350,14 +260,15 @@ class DatabaseAccessorImpl implements DatabaseAccessor
             getNextItem();
         }
 
-        private void init( ) throws DatabaseException
+        private void init( )
+                throws DatabaseException
         {
             final DatabaseUtil.DebugInfo debugInfo = DatabaseUtil.DebugInfo.create(
                     "iterator #" + counter + " open", table, null, null );
             traceBegin( debugInfo );
 
             final String sqlText = "SELECT * FROM " + table.name();
-            try
+            try ( Connection connection = databaseService.getConnection() )
             {
                 outstandingIterators.add( this );
                 statement = connection.prepareStatement( sqlText );
@@ -366,7 +277,7 @@ class DatabaseAccessorImpl implements DatabaseAccessor
             }
             catch ( final SQLException e )
             {
-                processSqlException( null, e );
+                throw DatabaseUtil.convertSqlException( debugInfo, e );
             }
 
             traceResult( debugInfo, null );
@@ -426,44 +337,36 @@ class DatabaseAccessorImpl implements DatabaseAccessor
                     "iterator #" + counter + " close", table, null, null );
             traceBegin( debugInfo );
 
-            lock.lock();
-            try
-            {
-                outstandingIterators.remove( this );
+            outstandingIterators.remove( this );
 
-                if ( resultSet != null )
+            if ( resultSet != null )
+            {
+                try
                 {
-                    try
-                    {
-                        resultSet.close();
-                        resultSet = null;
-                    }
-                    catch ( final SQLException e )
-                    {
-                        LOGGER.error( () -> "error closing inner resultSet in iterator: " + e.getMessage() );
-                    }
+                    resultSet.close();
+                    resultSet = null;
                 }
-
-                if ( statement != null )
+                catch ( final SQLException e )
                 {
-                    try
-                    {
-                        statement.close();
-                        statement = null;
-                    }
-                    catch ( final SQLException e )
-                    {
-                        LOGGER.error( () -> "error closing inner statement in iterator: " + e.getMessage() );
-                    }
+                    LOGGER.error( () -> "error closing inner resultSet in iterator: " + e.getMessage() );
                 }
-
-                finished = true;
             }
-            finally
+
+            if ( statement != null )
             {
-                lock.unlock();
+                try
+                {
+                    statement.close();
+                    statement = null;
+                }
+                catch ( final SQLException e )
+                {
+                    LOGGER.error( () -> "error closing inner statement in iterator: " + e.getMessage() );
+                }
             }
 
+            finished = true;
+
             traceResult( debugInfo, "outstandingIterators=" + outstandingIterators.size() );
         }
     }
@@ -499,7 +402,7 @@ class DatabaseAccessorImpl implements DatabaseAccessor
 
     private interface SqlFunction<T>
     {
-        T execute( ) throws DatabaseException;
+        T execute( Connection connection ) throws SQLException;
     }
 
     private <T> T execute( final DatabaseUtil.DebugInfo debugInfo, final SqlFunction<T> sqlFunction )
@@ -507,78 +410,35 @@ class DatabaseAccessorImpl implements DatabaseAccessor
     {
         traceBegin( debugInfo );
 
-        try
+        try ( Connection connection = databaseService.getConnection() )
         {
-            lock.lock();
-
             try
             {
-                final T result = sqlFunction.execute();
+                final T result = sqlFunction.execute( connection );
                 traceResult( debugInfo, result );
                 databaseService.updateStats( DatabaseService.OperationType.WRITE );
                 return result;
             }
-            finally
-            {
-                DatabaseUtil.commit( connection );
-            }
-
-        }
-        finally
-        {
-            lock.unlock();
-        }
-
-    }
-
-    Connection getConnection( )
-    {
-        return connection;
-    }
-
-    void close( )
-    {
-        closed.set( true );
-
-        try
-        {
-            lock.lock();
-            try
+            catch ( final SQLException sqlException )
             {
-                if ( !outstandingIterators.isEmpty() )
-                {
-                    LOGGER.warn( () -> "closing outstanding " + outstandingIterators.size() + " iterators" );
-                }
-                for ( final DBIterator iterator : new HashSet<>( outstandingIterators ) )
-                {
-                    iterator.close();
-                }
+                throw DatabaseUtil.convertSqlException( debugInfo, sqlException );
             }
-            catch ( final Exception e )
+            finally
             {
-                LOGGER.warn( () -> "error while closing connection: " + e.getMessage() );
+                DatabaseUtil.commit( connection );
             }
 
-            try
-            {
-                connection.close();
-            }
-            catch ( final SQLException e )
-            {
-                LOGGER.warn( () -> "error while closing connection: " + e.getMessage() );
-            }
         }
-        finally
+        catch ( final SQLException e )
         {
-            lock.unlock();
+            throw DatabaseUtil.convertSqlException( debugInfo, e );
         }
-
-        LOGGER.trace( () -> "closed accessor #" + accessorNumber );
     }
 
-    private boolean containsImpl( final DatabaseTable table, final String key )
+    private boolean containsImpl( final DatabaseTable table, final String key, final Connection connection )
             throws SQLException
     {
+
         final String sqlStatement = "SELECT COUNT(" + DatabaseService.KEY_COLUMN + ") FROM " + table.name()
                 + " WHERE " + DatabaseService.KEY_COLUMN + " = ?";
 
@@ -599,8 +459,8 @@ class DatabaseAccessorImpl implements DatabaseAccessor
         return false;
     }
 
-    private void executeUpdate( final String sqlStatement, final DatabaseUtil.DebugInfo debugInfo, final String... params )
-            throws DatabaseException
+    private void executeStatementWithParams( final String sqlStatement, final Connection connection, final String... params )
+            throws SQLException
     {
         try ( PreparedStatement statement = connection.prepareStatement( sqlStatement ) )
         {
@@ -610,10 +470,6 @@ class DatabaseAccessorImpl implements DatabaseAccessor
             }
             statement.executeUpdate();
         }
-        catch ( final SQLException e )
-        {
-            processSqlException( debugInfo, e );
-        }
     }
 
     private void preCheck( )
@@ -623,19 +479,4 @@ class DatabaseAccessorImpl implements DatabaseAccessor
             throw new IllegalStateException( "call to perform database operation but accessor has been closed" );
         }
     }
-
-    @Override
-    public boolean isConnected()
-    {
-        try
-        {
-            return connection.isValid( 5000 );
-        }
-        catch ( final SQLException e )
-        {
-            LOGGER.error( () -> "error while checking database connection: " + e.getMessage() );
-        }
-
-        return false;
-    }
 }

server/src/main/java/password/pwm/svc/db/DatabaseService.java

@@ -20,10 +20,12 @@
 
 package password.pwm.svc.db;
 
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import com.zaxxer.hikari.HikariPoolMXBean;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.bean.DomainID;
-import password.pwm.config.PwmSetting;
 import password.pwm.config.option.DataStorageMethod;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
@@ -35,9 +37,8 @@ import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.EpsStatistic;
 import password.pwm.svc.stats.StatisticsClient;
-import password.pwm.util.java.AtomicLoopIntIncrementer;
-import password.pwm.util.java.JavaHelper;
-import password.pwm.util.java.PwmTimeUtil;
+import password.pwm.util.java.CollectionUtil;
+import password.pwm.util.java.StatisticCounterBundle;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.json.JsonFactory;
@@ -45,18 +46,16 @@ import password.pwm.util.logging.PwmLogger;
 
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
-import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Properties;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 
 
 public class DatabaseService extends AbstractPwmService implements PwmService
@@ -70,21 +69,28 @@ public class DatabaseService extends AbstractPwmService implements PwmService
 
     private DBConfiguration dbConfiguration;
 
-    private ErrorInformation lastError;
+    private Map<DatabaseDebugProperty, String> initializedDebugData = Map.of();
+    private volatile boolean initialized = false;
 
-    private AtomicLoopIntIncrementer slotIncrementer;
-    private final Map<Integer, DatabaseAccessorImpl> accessors = new ConcurrentHashMap<>();
+    private HikariDataSource hikariDataSource;
 
-    private final Map<DatabaseAboutProperty, String> debugInfo = new LinkedHashMap<>();
+    private final StatisticCounterBundle<DebugStat> statsBundle = new StatisticCounterBundle<>( DebugStat.class );
 
-    private volatile boolean initialized = false;
+    enum DebugStat
+    {
+        issuedAccessors,
+    }
 
-    public enum DatabaseAboutProperty
+    public enum DatabaseDebugProperty
     {
         driverName,
         driverVersion,
         databaseProductName,
         databaseProductVersion,
+
+        idleConnections,
+        activeConnections,
+        totalConnections, maxConnections,
     }
 
     @Override
@@ -99,19 +105,15 @@ public class DatabaseService extends AbstractPwmService implements PwmService
     {
         this.dbConfiguration = DBConfiguration.fromConfiguration( getPwmApplication().getConfig() );
 
-        final TimeDuration watchdogFrequency = TimeDuration.of(
-                Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.DB_CONNECTIONS_WATCHDOG_FREQUENCY_SECONDS ) ),
-                TimeDuration.Unit.SECONDS );
-
         if ( !dbShouldOpen() )
         {
             initialized = true;
             return STATUS.CLOSED;
         }
 
-        scheduleFixedRateJob( new ConnectionMonitor(), watchdogFrequency, watchdogFrequency );
+        scheduleJob( new PwmDbInitializer() );
 
-        return dbInit();
+        return STATUS.OPEN;
     }
 
     private boolean dbShouldOpen()
@@ -131,82 +133,108 @@ public class DatabaseService extends AbstractPwmService implements PwmService
         return true;
     }
 
-    private STATUS dbInit( )
+    private void dbInit( )
     {
         if ( initialized )
         {
-            return STATUS.OPEN;
+            return;
         }
 
         final Instant startTime = Instant.now();
 
         try
         {
-            LOGGER.debug( getSessionLabel(), () -> "opening connection to database " + this.dbConfiguration.getConnectionString() );
-            slotIncrementer = AtomicLoopIntIncrementer.builder().ceiling( dbConfiguration.getMaxConnections() ).build();
+            JDBCDriverLoader.loadDriver( getPwmApplication(), dbConfiguration );
 
-            {
-                // make initial connection and establish schema
-                clearCurrentAccessors();
+            final String poolName = makePoolName( getPwmApplication() );
 
-                final Connection connection = openConnection( dbConfiguration );
-                updateDebugProperties( connection );
-                LOGGER.debug( getSessionLabel(), () -> "established initial connection to " + dbConfiguration.getConnectionString() + ", properties: "
-                        + JsonFactory.get().serializeMap( this.debugInfo ) );
+            hikariDataSource = new HikariDataSource( makeHikariConfig( dbConfiguration, poolName ) );
 
-                for ( final DatabaseTable table : DatabaseTable.values() )
-                {
-                    DatabaseUtil.initTable( connection, table, dbConfiguration );
-                }
+            LOGGER.debug( getSessionLabel(), () -> "opening connection to database "
+                    + this.dbConfiguration.connectionString() );
 
-                connection.close();
-            }
-
-            accessors.clear();
+            try ( Connection connection = getConnection() )
             {
-                // set up connection pool
-                final boolean traceLogging = getPwmApplication().getConfig().readSettingAsBoolean( PwmSetting.DATABASE_DEBUG_TRACE );
-                for ( int i = 0; i < dbConfiguration.getMaxConnections(); i++ )
-                {
-                    final Connection connection = openConnection( dbConfiguration );
-                    final DatabaseAccessorImpl accessor = new DatabaseAccessorImpl( this, this.dbConfiguration, connection, traceLogging );
-                    accessors.put( i, accessor );
-                }
+                initTables( connection );
+
+                initializedDebugData = initConnectionDebugData( connection );
             }
 
-            LOGGER.debug( getSessionLabel(), () -> "successfully connected to remote database (" + TimeDuration.compactFromCurrent( startTime ) + ")" );
+            postDbInitLogging();
 
+            LOGGER.debug( getSessionLabel(), () -> "successfully connected to remote database ("
+                    + TimeDuration.compactFromCurrent( startTime ) + ")" );
+
+            initialized = true;
+            setStartupError( null );
         }
         catch ( final Throwable t )
         {
             final String errorMsg = "exception initializing database service: " + t.getMessage();
             LOGGER.warn( getSessionLabel(), () -> errorMsg );
             initialized = false;
-            lastError = new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, errorMsg );
-            return STATUS.CLOSED;
+            setStartupError( new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, errorMsg ) );
         }
+    }
 
-        initialized = true;
-        return STATUS.OPEN;
+    private void postDbInitLogging()
+            throws DatabaseException
+
+    {
+        final DatabaseAccessor databaseAccessor = new DatabaseAccessorImpl(
+                this,
+                dbConfiguration );
+
+        for ( final DatabaseTable databaseTable : DatabaseTable.values() )
+        {
+            final int size = databaseAccessor.size( databaseTable );
+            LOGGER.trace( getSessionLabel(), () -> "opened table " + databaseTable.name() + " with "
+                    + size + " records" );
+        }
     }
 
-    @Override
-    public void shutdownImpl( )
+    private void initTables( final Connection connection )
+            throws DatabaseException
     {
-        setStatus( STATUS.CLOSED );
+        final Instant startTime = Instant.now();
+
+        LOGGER.trace( getSessionLabel(), () -> "beginning check for database table schema" );
+
+        for ( final DatabaseTable table : DatabaseTable.values() )
+        {
+            DatabaseUtil.initTable( getSessionLabel(), connection, table, dbConfiguration );
+        }
 
-        clearCurrentAccessors();
+        LOGGER.trace( getSessionLabel(), () -> "completed check for database table schema", TimeDuration.fromCurrent( startTime ) );
     }
 
-    private void clearCurrentAccessors( )
+    private Map<DatabaseDebugProperty, String> initConnectionDebugData( final Connection connection )
     {
-        for ( final DatabaseAccessorImpl accessor : accessors.values() )
+        try
+        {
+            final Map<DatabaseDebugProperty, String> returnObj = new EnumMap<>( DatabaseDebugProperty.class );
+            final DatabaseMetaData databaseMetaData = connection.getMetaData();
+            returnObj.put( DatabaseDebugProperty.driverName, databaseMetaData.getDriverName() );
+            returnObj.put( DatabaseDebugProperty.driverVersion, databaseMetaData.getDriverVersion() );
+            returnObj.put( DatabaseDebugProperty.databaseProductName, databaseMetaData.getDatabaseProductName() );
+            returnObj.put( DatabaseDebugProperty.databaseProductVersion, databaseMetaData.getDatabaseProductVersion() );
+            return returnObj;
+        }
+        catch ( final SQLException e )
         {
-            accessor.close();
+            LOGGER.error( getSessionLabel(), () -> "error reading jdbc meta data: " + e.getMessage() );
         }
-        accessors.clear();
+
+        return Map.of();
     }
 
+    @Override
+    public void shutdownImpl( )
+    {
+        setStatus( STATUS.CLOSED );
+    }
+
+
     @Override
     public List<HealthRecord> serviceHealthCheck( )
     {
@@ -244,32 +272,14 @@ public class DatabaseService extends AbstractPwmService implements PwmService
             return returnRecords;
         }
 
-        if ( lastError != null )
-        {
-            final TimeDuration errorAge = TimeDuration.fromCurrent( lastError.getDate() );
-            final long cautionDurationMS = Long.parseLong( getPwmApplication().getConfig().readAppProperty( AppProperty.HEALTH_DB_CAUTION_DURATION_MS ) );
-
-            if ( errorAge.isShorterThan( cautionDurationMS ) )
-            {
-                final String ageString = PwmTimeUtil.asLongString( errorAge );
-                final String errorDate = StringUtil.toIsoDate( lastError.getDate() );
-                final String errorMsg = lastError.toDebugStr();
-                returnRecords.add( HealthRecord.forMessage(
-                        DomainID.systemId(),
-                        HealthMessage.Database_RecentlyUnreachable,
-                        ageString,
-                        errorDate,
-                        errorMsg
-                ) );
-            }
-        }
+        // final long cautionDurationMS = Long.parseLong( getPwmApplication().getConfig().readAppProperty( AppProperty.HEALTH_DB_CAUTION_DURATION_MS ) );
 
         if ( returnRecords.isEmpty() )
         {
             returnRecords.add( HealthRecord.forMessage(
                     DomainID.systemId(),
                     HealthMessage.Database_OK,
-                    this.dbConfiguration.getConnectionString() ) );
+                    this.dbConfiguration.connectionString() ) );
         }
 
         return returnRecords;
@@ -285,9 +295,10 @@ public class DatabaseService extends AbstractPwmService implements PwmService
         }
         else
         {
-            if ( lastError != null )
+            final ErrorInformation startupError = getStartupError();
+            if ( startupError != null )
             {
-                errorMsg = "unable to initialize database: " + lastError.getDetailedErrorMsg();
+                errorMsg = "unable to initialize database: " + startupError.getDetailedErrorMsg();
             }
             else
             {
@@ -301,12 +312,9 @@ public class DatabaseService extends AbstractPwmService implements PwmService
     @Override
     public ServiceInfoBean serviceInfo( )
     {
-        final Map<String, String> debugProperties = new LinkedHashMap<>( debugInfo.size() );
-        for ( final Map.Entry<DatabaseAboutProperty, String> entry : debugInfo.entrySet() )
-        {
-            final DatabaseAboutProperty databaseAboutProperty = entry.getKey();
-            debugProperties.put( databaseAboutProperty.name(), entry.getValue() );
-        }
+        final Map<String, String> debugProperties = new LinkedHashMap<>(
+                CollectionUtil.enumMapToStringMap( makeDebugProperties() )
+        );
 
         if ( status() == STATUS.OPEN )
         {
@@ -319,6 +327,12 @@ public class DatabaseService extends AbstractPwmService implements PwmService
         return ServiceInfoBean.builder().debugProperties( debugProperties ).build();
     }
 
+    Connection getConnection()
+            throws SQLException
+    {
+        return hikariDataSource.getConnection();
+    }
+
     public DatabaseAccessor getAccessor( )
             throws PwmUnrecoverableException
     {
@@ -332,44 +346,10 @@ public class DatabaseService extends AbstractPwmService implements PwmService
             throw new PwmUnrecoverableException( makeUninitializedError() );
         }
 
-        return accessors.get( slotIncrementer.next() );
-    }
-
-    private Connection openConnection( final DBConfiguration dbConfiguration )
-            throws DatabaseException
-    {
-        final String connectionURL = dbConfiguration.getConnectionString();
-
-        try
-        {
-            LOGGER.debug( getSessionLabel(), () -> "initiating connecting to database " + connectionURL );
-            JDBCDriverLoader.loadDriver( getPwmApplication(), dbConfiguration );
-            final Properties connectionProperties = new Properties();
-            if ( dbConfiguration.getUsername() != null && !dbConfiguration.getUsername().isEmpty() )
-            {
-                connectionProperties.setProperty( "user", dbConfiguration.getUsername() );
-            }
-            if ( dbConfiguration.getPassword() != null )
-            {
-                connectionProperties.setProperty( "password", dbConfiguration.getPassword().getStringValue() );
-            }
-
-            final Connection connection = DriverManager.getConnection( connectionURL, connectionProperties );
-            LOGGER.debug( getSessionLabel(), () -> "connected to database " + connectionURL );
-
-            connection.setAutoCommit( false );
-            return connection;
-        }
-        catch ( final Throwable e )
-        {
-            final String errorMsg = "error connecting to database: " + JavaHelper.readHostileExceptionMessage( e );
-            final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, errorMsg );
-            LOGGER.error( getSessionLabel(),  errorInformation );
-            throw new DatabaseException( errorInformation );
-        }
+        statsBundle.increment( DebugStat.issuedAccessors );
+        return new DatabaseAccessorImpl( this, dbConfiguration );
     }
 
-
     enum OperationType
     {
         WRITE,
@@ -388,66 +368,98 @@ public class DatabaseService extends AbstractPwmService implements PwmService
         }
     }
 
-    public Map<DatabaseAboutProperty, String> getConnectionDebugProperties( )
+    public Map<DatabaseDebugProperty, String> getConnectionDebugProperties( )
     {
-        return Collections.unmodifiableMap( debugInfo );
+        return makeDebugProperties();
     }
 
-    private void updateDebugProperties( final Connection connection )
+    private Map<DatabaseDebugProperty, String> makeDebugProperties()
     {
-        if ( connection != null )
+        final Map<DatabaseDebugProperty, String> returnObj = new EnumMap<>( DatabaseDebugProperty.class );
+
+        try
         {
-            try
-            {
-                final Map<DatabaseAboutProperty, String> returnObj = new LinkedHashMap<>();
-                final DatabaseMetaData databaseMetaData = connection.getMetaData();
-                returnObj.put( DatabaseAboutProperty.driverName, databaseMetaData.getDriverName() );
-                returnObj.put( DatabaseAboutProperty.driverVersion, databaseMetaData.getDriverVersion() );
-                returnObj.put( DatabaseAboutProperty.databaseProductName, databaseMetaData.getDatabaseProductName() );
-                returnObj.put( DatabaseAboutProperty.databaseProductVersion, databaseMetaData.getDatabaseProductVersion() );
-                debugInfo.clear();
-                debugInfo.putAll( Collections.unmodifiableMap( returnObj ) );
-            }
-            catch ( final SQLException e )
+            if ( hikariDataSource != null )
             {
-                LOGGER.error( getSessionLabel(), () -> "error reading jdbc meta data: " + e.getMessage() );
+                final HikariPoolMXBean poolProxy = hikariDataSource.getHikariPoolMXBean();
+                if ( poolProxy != null )
+                {
+                    returnObj.put(
+                            DatabaseDebugProperty.idleConnections,
+                            String.valueOf( poolProxy.getIdleConnections() ) );
+                    returnObj.put(
+                            DatabaseDebugProperty.activeConnections,
+                            String.valueOf( poolProxy.getActiveConnections() ) );
+                    returnObj.put(
+                            DatabaseDebugProperty.totalConnections,
+                            String.valueOf( poolProxy.getTotalConnections() ) );
+                    returnObj.put(
+                            DatabaseDebugProperty.maxConnections,
+                            String.valueOf( hikariDataSource.getMaximumPoolSize() ) );
+                }
             }
         }
-    }
+        catch ( final Throwable t )
+        {
+            LOGGER.error( getSessionLabel(), () -> "error reading hikari mxBean during debug property generation: "
+                    + t.getMessage() );
+        }
 
-    void setLastError( final ErrorInformation lastError )
-    {
-        this.lastError = lastError;
+        returnObj.putAll( initializedDebugData );
+
+        return Map.copyOf( returnObj );
     }
 
-    private class ConnectionMonitor implements Runnable
+    private class PwmDbInitializer implements Runnable
     {
         @Override
         public void run( )
         {
             if ( initialized )
             {
-                boolean valid = true;
-                for ( final DatabaseAccessorImpl databaseAccessor : accessors.values() )
-                {
-                    if ( !databaseAccessor.isValid() )
-                    {
-                        valid = false;
-                        break;
-                    }
-                }
-                if ( !valid )
-                {
-                    LOGGER.warn( getSessionLabel(), () -> "database connection lost; will retry connect periodically" );
-                    initialized = false;
-                }
+                return;
+            }
 
+            try
+            {
+                dbInit();
+            }
+            catch ( final Throwable t )
+            {
+                LOGGER.error( getSessionLabel(), () -> "error during database initialization: " + t.getMessage() );
             }
 
             if ( !initialized )
             {
-                dbInit();
+                final TimeDuration watchdogFrequency = TimeDuration.of(
+                        Integer.parseInt( getPwmApplication().getConfig().readAppProperty( AppProperty.DB_CONNECTIONS_WATCHDOG_FREQUENCY_SECONDS ) ),
+                        TimeDuration.Unit.SECONDS );
+                scheduleJob( this, watchdogFrequency );
             }
         }
     }
+
+    private static String makePoolName( final PwmApplication pwmApplication )
+    {
+        return "pwm-" + pwmApplication.getInstanceID() + "-hikari-pool";
+    }
+
+
+    private static HikariConfig makeHikariConfig( final DBConfiguration dbConfiguration, final String poolName )
+            throws PwmUnrecoverableException
+    {
+        final HikariConfig config = new HikariConfig();
+        config.setJdbcUrl( dbConfiguration.connectionString() );
+        config.setUsername( dbConfiguration.username() );
+        config.setPassword( dbConfiguration.password().getStringValue() );
+        config.addDataSourceProperty( "registerMbeans", "true" );
+        config.setPoolName( poolName );
+
+        if ( dbConfiguration.maxConnections() > 0 )
+        {
+            config.setMaximumPoolSize( dbConfiguration.maxConnections() );
+        }
+
+        return config;
+    }
 }

server/src/main/java/password/pwm/svc/db/DatabaseUtil.java

@@ -23,6 +23,7 @@ package password.pwm.svc.db;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
+import password.pwm.bean.SessionLabel;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.util.logging.PwmLogger;
@@ -105,6 +106,7 @@ class DatabaseUtil
     }
 
     static void initTable(
+            final SessionLabel sessionLabel,
             final Connection connection,
             final DatabaseTable table,
             final DBConfiguration dbConfiguration
@@ -115,13 +117,13 @@ class DatabaseUtil
         try
         {
             checkIfTableExists( connection, table );
-            LOGGER.trace( () -> "table " + table + " appears to exist" );
+            LOGGER.trace( sessionLabel, () -> "table " + table + " appears to exist" );
             tableExists = true;
         }
         catch ( final DatabaseException e )
         {
             // assume error was due to table missing;
-            LOGGER.trace( () -> "error while checking for table: " + e.getMessage() + ", assuming due to table non-existence" );
+            LOGGER.trace( sessionLabel, () -> "error while checking for table: " + e.getMessage() + ", assuming due to table non-existence" );
         }
 
         if ( !tableExists )
@@ -139,9 +141,9 @@ class DatabaseUtil
     {
         {
             final String sqlString = "CREATE table " + table.toString() + " (" + "\n"
-                    + "  " + DatabaseService.KEY_COLUMN + " " + dbConfiguration.getColumnTypeKey() + "("
-                    + dbConfiguration.getKeyColumnLength() + ") NOT NULL PRIMARY KEY," + "\n"
-                    + "  " + DatabaseService.VALUE_COLUMN + " " + dbConfiguration.getColumnTypeValue() + " " + "\n"
+                    + "  " + DatabaseService.KEY_COLUMN + " " + dbConfiguration.columnTypeKey() + "("
+                    + dbConfiguration.keyColumnLength() + ") NOT NULL PRIMARY KEY," + "\n"
+                    + "  " + DatabaseService.VALUE_COLUMN + " " + dbConfiguration.columnTypeValue() + " " + "\n"
                     + ")" + "\n";
 
             LOGGER.trace( () -> "attempting to execute the following sql statement:\n " + sqlString );
@@ -187,7 +189,7 @@ class DatabaseUtil
             {
                 final String errorMsg = "error creating new index " + indexName + ": " + ex.getMessage();
                 final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, errorMsg );
-                if ( dbConfiguration.isFailOnIndexCreation() )
+                if ( dbConfiguration.failOnIndexCreation() )
                 {
                     throw new DatabaseException ( errorInformation );
                 }

server/src/main/java/password/pwm/svc/db/JDBCDriverLoader.java

@@ -127,12 +127,12 @@ public class JDBCDriverLoader
         public Driver loadDriver( final PwmApplication pwmApplication, final DBConfiguration dbConfiguration )
                 throws DatabaseException
         {
-            final String jdbcClassName = dbConfiguration.getDriverClassname();
+            final String jdbcClassName = dbConfiguration.driverClassname();
 
             try
             {
                 LOGGER.debug( () -> "loading JDBC database driver from classpath: " + jdbcClassName );
-                final Driver driver = DriverManager.getDriver( dbConfiguration.getConnectionString() );
+                final Driver driver = DriverManager.getDriver( dbConfiguration.connectionString() );
 
                 LOGGER.debug( () -> "successfully loaded JDBC database driver from classpath: " + jdbcClassName );
                 return driver;
@@ -179,7 +179,7 @@ public class JDBCDriverLoader
 
             try
             {
-                return DriverManager.getDriver( dbConfiguration.getConnectionString() );
+                return DriverManager.getDriver( dbConfiguration.connectionString() );
             }
             catch ( final Throwable e )
             {
@@ -226,9 +226,9 @@ public class JDBCDriverLoader
                     this.getClass().getClassLoader() );
 
             //Create object of loaded class
-            final Class<?> jdbcDriverClass = urlClassLoader.loadClass( dbConfiguration.getDriverClassname() );
+            final Class<?> jdbcDriverClass = urlClassLoader.loadClass( dbConfiguration.driverClassname() );
             final Driver driver = new DriverShim( ( Driver ) jdbcDriverClass.getDeclaredConstructor().newInstance() );
-            LOGGER.debug( () -> "successfully loaded JDBC database driver '" + dbConfiguration.getDriverClassname() + "' from application configuration" );
+            LOGGER.debug( () -> "successfully loaded JDBC database driver '" + dbConfiguration.driverClassname() + "' from application configuration" );
             DriverManager.registerDriver( driver );
             return true;
 

server/src/main/java/password/pwm/svc/stats/Statistic.java

@@ -32,6 +32,7 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.Locale;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.SortedSet;
@@ -213,6 +214,7 @@ public enum Statistic
 
     public static Optional<Statistic> forKey( final String key )
     {
-        return EnumUtil.readEnumFromPredicate( Statistic.class, loopValue -> loopValue.getKey().equals( key ) );
+        return EnumUtil.readEnumFromPredicate( Statistic.class, loopValue ->
+                Objects.equals( loopValue.getKey(), key ) || Objects.equals( loopValue.name(), key ) );
     }
 }

server/src/main/java/password/pwm/svc/stats/StatisticsBundle.java

@@ -20,9 +20,11 @@
 
 package password.pwm.svc.stats;
 
+import password.pwm.bean.SessionLabel;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.json.JsonFactory;
+import password.pwm.util.logging.PwmLogger;
 
 import java.math.BigInteger;
 import java.util.EnumMap;
@@ -34,6 +36,8 @@ import java.util.concurrent.locks.ReentrantLock;
 
 public class StatisticsBundle
 {
+    private static final PwmLogger LOGGER = PwmLogger.forClass( StatisticsBundle.class );
+
     private final Map<Statistic, LongAccumulator> incrementerMap = new EnumMap<>( Statistic.class );
     private final Map<AvgStatistic, AverageBean> avgMap = new EnumMap<>( AvgStatistic.class );
 
@@ -95,8 +99,17 @@ public class StatisticsBundle
             final String value = loadedMap.get( loopStat.name() );
             if ( StringUtil.notEmpty( value ) )
             {
-                final AverageBean avgBean = JsonFactory.get().deserialize( value, AverageBean.class );
-                bundle.avgMap.put( loopStat, avgBean );
+                try
+                {
+                    final AverageBean avgBean = JsonFactory.get().deserialize( value, AverageBean.class );
+                    bundle.avgMap.put( loopStat, avgBean );
+                }
+                catch ( final Exception e )
+                {
+                    LOGGER.error( SessionLabel.SYSTEM_LABEL, () -> "error reading stored stat bundle for key '"
+                            + inputString + "', storedValue: '" + value + " ', error: " + e.getMessage() );
+                }
+
             }
         }
 

server/src/main/java/password/pwm/svc/stats/StatisticsUtils.java

@@ -35,6 +35,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
+import java.util.Optional;
 import java.util.SortedSet;
 import java.util.TreeSet;
 import java.util.stream.Collectors;
@@ -80,37 +81,36 @@ public class StatisticsUtils
         for ( final StatisticsBundleKey loopKey : allKeys( statisticsService ) )
         {
             counter++;
-            final StatisticsBundle bundle = statisticsService.getStatBundleForKey( loopKey )
-                    .orElseThrow();
-
-            final List<String> lineOutput = new ArrayList<>( Statistic.asSet().size() );
-
-            lineOutput.add( loopKey.toString() );
-
-            if ( loopKey.getKeyType() == StatisticsBundleKey.KeyType.DAILY )
-            {
-                lineOutput.add( Integer.toString( loopKey.getYear() ) );
-                lineOutput.add( Integer.toString( loopKey.getDay() ) );
-            }
-            else
+            final Optional<StatisticsBundle> bundle = statisticsService.getStatBundleForKey( loopKey );
+            if ( bundle.isPresent() )
             {
-                lineOutput.add( "" );
-                lineOutput.add( "" );
+                final List<String> lineOutput = new ArrayList<>( Statistic.asSet().size() );
+
+                lineOutput.add( loopKey.toString() );
+
+                if ( loopKey.getKeyType() == StatisticsBundleKey.KeyType.DAILY )
+                {
+                    lineOutput.add( Integer.toString( loopKey.getYear() ) );
+                    lineOutput.add( Integer.toString( loopKey.getDay() ) );
+                }
+                else
+                {
+                    lineOutput.add( "" );
+                    lineOutput.add( "" );
+                }
+
+                lineOutput.addAll( EnumUtil.enumStream( Statistic.class )
+                        .map( stat -> bundle.get().getStatistic( stat ) )
+                        .collect( Collectors.toList() ) );
+
+                csvPrinter.printRecord( lineOutput );
             }
-
-            lineOutput.addAll( EnumUtil.enumStream( Statistic.class )
-                    .map( bundle::getStatistic )
-                    .collect( Collectors.toList() ) );
-
-            csvPrinter.printRecord( lineOutput );
         }
 
-        {
-            final int finalCounter = counter;
-            LOGGER.trace( sessionLabel, () -> "completed output stats to csv process; output "
-                    + finalCounter + " records in "
-                    + TimeDuration.compactFromCurrent( startTime ) );
-        }
+        final int finalCounter = counter;
+        LOGGER.trace( sessionLabel, () -> "completed output stats to csv process; output "
+                + finalCounter + " records in "
+                + TimeDuration.compactFromCurrent( startTime ) );
 
         return counter;
     }

server/src/main/java/password/pwm/util/debug/AboutItemGenerator.java

@@ -27,7 +27,7 @@ import password.pwm.util.json.JsonProvider;
 import java.io.OutputStream;
 import java.util.Map;
 
-class AboutItemGenerator implements AppItemGenerator
+final class AboutItemGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -36,10 +36,10 @@ class AboutItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
     {
-        final Map<PwmAboutProperty, String> infoBeanMap = PwmAboutProperty.makeInfoBean( debugItemInput.getPwmApplication() );
+        final Map<PwmAboutProperty, String> infoBeanMap = PwmAboutProperty.makeInfoBean( debugItemInput.pwmApplication() );
         final String jsonValue = JsonFactory.get().serializeMap( infoBeanMap, JsonProvider.Flag.PrettyPrint );
-        DebugItemGenerator.writeString( outputStream, jsonValue );
+        DebugGenerator.writeString( outputStream, jsonValue );
     }
 }

server/src/main/java/password/pwm/util/debug/AppDebugItemInput.java → server/src/main/java/password/pwm/util/debug/AppDebugItemRequest.java

@@ -20,18 +20,18 @@
 
 package password.pwm.util.debug;
 
-import lombok.Value;
 import password.pwm.PwmApplication;
 import password.pwm.bean.SessionLabel;
 import password.pwm.config.AppConfig;
 
 import java.util.Locale;
 
-@Value
-class AppDebugItemInput
+record AppDebugItemRequest(
+        PwmApplication pwmApplication,
+        SessionLabel sessionLabel,
+        AppConfig obfuscatedAppConfig,
+        Locale locale,
+        DebugGeneratorLogger logger
+)
 {
-    private final PwmApplication pwmApplication;
-    private final SessionLabel sessionLabel;
-    private final AppConfig obfuscatedAppConfig;
-    private final Locale locale;
 }

server/src/main/java/password/pwm/util/debug/AppItemGenerator.java

@@ -25,13 +25,30 @@ import password.pwm.error.PwmUnrecoverableException;
 import java.io.IOException;
 import java.io.OutputStream;
 
-interface AppItemGenerator
+sealed interface AppItemGenerator extends ItemGenerator permits
+        ConfigurationFileItemGenerator,
+        ConfigurationDebugJsonItemGenerator,
+        ConfigurationDebugTextItemGenerator,
+        AboutItemGenerator,
+        SystemEnvironmentItemGenerator,
+        AppPropertiesItemGenerator,
+        ServicesDebugItemGenerator,
+        HealthDebugItemGenerator,
+        ThreadDumpDebugItemGenerator,
+        FileInfoDebugItemGenerator,
+        IntruderDataGenerator,
+        LogDebugItemGenerator,
+        LogJsonItemGenerator,
+        LocalDBDebugGenerator,
+        SessionDataGenerator,
+        ClusterInfoDebugGenerator,
+        RootFileSystemDebugItemGenerator,
+        StatisticsDataDebugItemGenerator,
+        StatisticsEpsDataDebugItemGenerator,
+        BuildManifestDebugItemGenerator
 {
-
-    String getFilename();
-
     void outputItem(
-            AppDebugItemInput debugItemInput,
+            AppDebugItemRequest debugItemInput,
             OutputStream outputStream
     ) throws IOException, PwmUnrecoverableException;
 }

server/src/main/java/password/pwm/util/debug/AppPropertiesItemGenerator.java

@@ -30,7 +30,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.util.Map;
 
-class AppPropertiesItemGenerator implements AppItemGenerator
+final class AppPropertiesItemGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -39,13 +39,13 @@ class AppPropertiesItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
-        final AppConfig config = debugItemInput.getObfuscatedAppConfig();
+        final AppConfig config = debugItemInput.obfuscatedAppConfig();
         final Map<AppProperty, String> appPropertyMap = config.readAllAppProperties();
         final Map<String, String> stringMap = CollectionUtil.enumMapToStringMap( appPropertyMap, AppProperty::getKey );
         final String json = JsonFactory.get().serializeMap( stringMap, JsonProvider.Flag.PrettyPrint );
-        DebugItemGenerator.writeString( outputStream, json );
+        DebugGenerator.writeString( outputStream, json );
     }
 }

server/src/main/java/password/pwm/util/debug/BuildManifestDebugItemGenerator.java

@@ -27,7 +27,7 @@ import password.pwm.util.json.JsonProvider;
 import java.io.IOException;
 import java.io.OutputStream;
 
-class BuildManifestDebugItemGenerator implements AppItemGenerator
+final class BuildManifestDebugItemGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -36,10 +36,10 @@ class BuildManifestDebugItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
         final String json = JsonFactory.get().serializeMap( PwmConstants.BUILD_MANIFEST, JsonProvider.Flag.PrettyPrint );
-        DebugItemGenerator.writeString( outputStream, json );
+        DebugGenerator.writeString( outputStream, json );
     }
 }

server/src/main/java/password/pwm/util/debug/CacheServiceDebugItemGenerator.java

@@ -23,6 +23,7 @@ package password.pwm.util.debug;
 import password.pwm.PwmConstants;
 import password.pwm.PwmDomain;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.svc.PwmService;
 import password.pwm.svc.cache.CacheService;
 import password.pwm.util.json.JsonFactory;
 import password.pwm.util.json.JsonProvider;
@@ -32,7 +33,7 @@ import java.io.OutputStream;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
-class CacheServiceDebugItemGenerator implements DomainItemGenerator
+final class CacheServiceDebugItemGenerator implements DomainItemGenerator
 {
     @Override
     public String getFilename()
@@ -41,13 +42,23 @@ class CacheServiceDebugItemGenerator implements DomainItemGenerator
     }
 
     @Override
-    public void outputItem( final DomainDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final DomainDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException, PwmUnrecoverableException
     {
-        final PwmDomain pwmApplication = debugItemInput.getPwmDomain();
+        final PwmDomain pwmApplication = debugItemInput.pwmDomain();
         final CacheService cacheService = pwmApplication.getCacheService();
 
-        final Map<String, Object> debugOutput = new LinkedHashMap<>( cacheService.debugInfo() );
+        final Map<String, Object> debugOutput = new LinkedHashMap<>();
+
+        if ( cacheService != null && cacheService.status() == PwmService.STATUS.OPEN )
+        {
+            debugOutput.putAll( cacheService.debugInfo() );
+        }
+        else
+        {
+            debugOutput.put( "status", "closed" );
+        }
+
         outputStream.write( JsonFactory.get().serializeMap( debugOutput, JsonProvider.Flag.PrettyPrint ).getBytes( PwmConstants.DEFAULT_CHARSET ) );
     }
 }

server/src/main/java/password/pwm/util/debug/ClusterInfoDebugGenerator.java

@@ -34,7 +34,7 @@ import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
-class ClusterInfoDebugGenerator implements AppItemGenerator
+final class ClusterInfoDebugGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -43,10 +43,10 @@ class ClusterInfoDebugGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException, PwmUnrecoverableException
     {
-        final PwmApplication pwmApplication = debugItemInput.getPwmApplication();
+        final PwmApplication pwmApplication = debugItemInput.pwmApplication();
         final NodeService nodeService = pwmApplication.getNodeService();
 
         final Map<String, Object> debugOutput = new LinkedHashMap<>();

server/src/main/java/password/pwm/util/debug/ConfigurationDebugJsonItemGenerator.java

@@ -32,7 +32,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.util.TreeMap;
 
-class ConfigurationDebugJsonItemGenerator implements AppItemGenerator
+final class ConfigurationDebugJsonItemGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -41,10 +41,10 @@ class ConfigurationDebugJsonItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
-        final StoredConfiguration storedConfiguration = debugItemInput.getObfuscatedAppConfig().getStoredConfiguration();
+        final StoredConfiguration storedConfiguration = debugItemInput.obfuscatedAppConfig().getStoredConfiguration();
         final TreeMap<String, Object> outputObject = new TreeMap<>();
 
         CollectionUtil.iteratorToStream( storedConfiguration.keys() )

server/src/main/java/password/pwm/util/debug/ConfigurationDebugTextItemGenerator.java

@@ -30,7 +30,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.util.Locale;
 
-class ConfigurationDebugTextItemGenerator implements AppItemGenerator
+final class ConfigurationDebugTextItemGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -39,11 +39,11 @@ class ConfigurationDebugTextItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
         final Locale locale = PwmConstants.DEFAULT_LOCALE;
-        final StoredConfiguration storedConfiguration = debugItemInput.getObfuscatedAppConfig().getStoredConfiguration();
+        final StoredConfiguration storedConfiguration = debugItemInput.obfuscatedAppConfig().getStoredConfiguration();
 
         final String headerString = "Configuration Debug Output for "
                 + PwmConstants.PWM_APP_NAME + " "
@@ -51,12 +51,12 @@ class ConfigurationDebugTextItemGenerator implements AppItemGenerator
                 +  "Timestamp: " + StringUtil.toIsoDate( storedConfiguration.modifyTime() ) + "\n"
                 +  "This file is " + PwmConstants.DEFAULT_CHARSET.displayName() + " encoded\n"
                 + '\n';
-        DebugItemGenerator.writeString( outputStream, headerString );
+        DebugGenerator.writeString( outputStream, headerString );
 
         CollectionUtil.iteratorToStream( storedConfiguration.keys() )
                 .filter( k -> k.isRecordType( StoredConfigKey.RecordType.SETTING ) )
                 .map( storedConfigKey -> settingDebugOutput( locale, storedConfiguration, storedConfigKey ) )
-                .forEach( line -> DebugItemGenerator.writeString( outputStream, line ) );
+                .forEach( line -> DebugGenerator.writeString( outputStream, line ) );
 
     }
 

server/src/main/java/password/pwm/util/debug/ConfigurationFileItemGenerator.java

@@ -29,7 +29,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 
-class ConfigurationFileItemGenerator implements AppItemGenerator
+final class ConfigurationFileItemGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -38,10 +38,10 @@ class ConfigurationFileItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException, PwmUnrecoverableException
     {
-        final StoredConfiguration storedConfiguration = debugItemInput.getObfuscatedAppConfig().getStoredConfiguration();
+        final StoredConfiguration storedConfiguration = debugItemInput.obfuscatedAppConfig().getStoredConfiguration();
 
         // temporary output stream required because .toXml closes stream.
         final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();

server/src/main/java/password/pwm/util/debug/DashboardDataDebugItemGenerator.java

@@ -31,7 +31,7 @@ import password.pwm.util.json.JsonProvider;
 import java.io.IOException;
 import java.io.OutputStream;
 
-class DashboardDataDebugItemGenerator implements DomainItemGenerator
+final class DashboardDataDebugItemGenerator implements DomainItemGenerator
 {
     @Override
     public String getFilename()
@@ -40,15 +40,15 @@ class DashboardDataDebugItemGenerator implements DomainItemGenerator
     }
 
     @Override
-    public void outputItem( final DomainDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final DomainDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException, PwmUnrecoverableException
     {
-        final PwmDomain pwmDomain = debugItemInput.getPwmDomain();
+        final PwmDomain pwmDomain = debugItemInput.pwmDomain();
         final ContextManager contextManager = pwmDomain.getPwmApplication().getPwmEnvironment().getContextManager();
         final AppDashboardData appDashboardData = AppDashboardData.makeDashboardData(
                 pwmDomain,
                 contextManager,
-                debugItemInput.getLocale()
+                debugItemInput.locale()
         );
 
         outputStream.write( JsonFactory.get().serialize( appDashboardData, JsonProvider.Flag.PrettyPrint ).getBytes( PwmConstants.DEFAULT_CHARSET ) );

server/src/main/java/password/pwm/util/debug/DebugItemGenerator.java → server/src/main/java/password/pwm/util/debug/DebugGenerator.java

@@ -30,7 +30,6 @@ import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationUtil;
 import password.pwm.error.PwmInternalException;
 import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.util.java.DebugOutputBuilder;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
@@ -41,42 +40,13 @@ import java.nio.charset.StandardCharsets;
 import java.time.Instant;
 import java.util.List;
 import java.util.Locale;
+import java.util.function.Supplier;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
-public class DebugItemGenerator
+public class DebugGenerator
 {
-    private static final PwmLogger LOGGER = PwmLogger.forClass( DebugItemGenerator.class );
-
-    private static final List<Class<? extends AppItemGenerator>> APP_ITEM_GENERATORS = List.of(
-            ConfigurationFileItemGenerator.class,
-            ConfigurationDebugJsonItemGenerator.class,
-            ConfigurationDebugTextItemGenerator.class,
-            AboutItemGenerator.class,
-            SystemEnvironmentItemGenerator.class,
-            AppPropertiesItemGenerator.class,
-            ServicesDebugItemGenerator.class,
-            HealthDebugItemGenerator.class,
-            ThreadDumpDebugItemGenerator.class,
-            FileInfoDebugItemGenerator.class,
-            IntruderDataGenerator.class,
-            LogDebugItemGenerator.class,
-            LogJsonItemGenerator.class,
-            LocalDBDebugGenerator.class,
-            SessionDataGenerator.class,
-            ClusterInfoDebugGenerator.class,
-            RootFileSystemDebugItemGenerator.class,
-            StatisticsDataDebugItemGenerator.class,
-            StatisticsEpsDataDebugItemGenerator.class,
-            BuildManifestDebugItemGenerator.class );
-
-    private static final List<Class<? extends DomainItemGenerator>> DOMAIN_ITEM_GENERATORS = List.of(
-            LDAPPermissionItemGenerator.class,
-            CacheServiceDebugItemGenerator.class,
-            LdapDebugItemGenerator.class,
-            LdapRecentUserDebugGenerator.class,
-            DashboardDataDebugItemGenerator.class,
-            LdapConnectionsDebugItemGenerator.class );
+    private static final PwmLogger LOGGER = PwmLogger.forClass( DebugGenerator.class );
 
     private final PwmApplication pwmApplication;
     private final AppConfig obfuscatedAppConfig;
@@ -84,7 +54,7 @@ public class DebugItemGenerator
 
     private static final Locale LOCALE = PwmConstants.DEFAULT_LOCALE;
 
-    public DebugItemGenerator( final PwmApplication pwmApplication, final SessionLabel sessionLabel )
+    public DebugGenerator( final PwmApplication pwmApplication, final SessionLabel sessionLabel )
             throws PwmUnrecoverableException
     {
         this.pwmApplication = pwmApplication;
@@ -109,15 +79,16 @@ public class DebugItemGenerator
     {
         final String debugFileName = "zipDebugGeneration.csv";
         final Instant startTime = Instant.now();
-        final DebugOutputBuilder debugGeneratorLogFile = new DebugOutputBuilder();
-        debugGeneratorLogFile.appendLine( "beginning debug output" );
+        final DebugGeneratorLogger debugLogger = new DebugGeneratorLogger( sessionLabel );
+        debugLogger.appendLine( "beginning debug output" );
 
         {
-            final AppDebugItemInput debugItemInput = new AppDebugItemInput( pwmApplication, sessionLabel, this.obfuscatedAppConfig, LOCALE );
+            final AppDebugItemRequest debugItemInput = new AppDebugItemRequest( pwmApplication, sessionLabel, this.obfuscatedAppConfig, LOCALE, debugLogger );
             final String pathPrefix = getFilenameBase() + "/";
-            for ( final Class<? extends AppItemGenerator> serviceClass : APP_ITEM_GENERATORS )
+            final List<AppItemGenerator> appItemGenerators = JavaHelper.instancesOfSealedInterface( AppItemGenerator.class );
+            for ( final AppItemGenerator serviceClass : appItemGenerators )
             {
-                executeAppDebugItem( debugItemInput, serviceClass, debugGeneratorLogFile, zipOutput, pathPrefix );
+                executeAppDebugItem( debugItemInput, serviceClass, debugLogger, zipOutput, pathPrefix );
             }
         }
 
@@ -125,25 +96,26 @@ public class DebugItemGenerator
             for ( final PwmDomain pwmDomain : pwmApplication.domains().values() )
             {
                 final DomainConfig obfuscatedDomainConfig = this.obfuscatedAppConfig.getDomainConfigs().get( pwmDomain.getDomainID() );
-                final DomainDebugItemInput debugItemInput = new DomainDebugItemInput( pwmDomain, sessionLabel, obfuscatedDomainConfig, LOCALE );
+                final DomainDebugItemRequest debugItemInput = new DomainDebugItemRequest( pwmDomain, sessionLabel, obfuscatedDomainConfig, LOCALE, debugLogger );
                 final String pathPrefix = getFilenameBase() + "/" + pwmDomain.getDomainID() + "/";
-                for ( final Class<? extends DomainItemGenerator> serviceClass : DOMAIN_ITEM_GENERATORS )
+                final List<DomainItemGenerator> domainItemGenerator = JavaHelper.instancesOfSealedInterface( DomainItemGenerator.class );
+                for ( final DomainItemGenerator serviceClass : domainItemGenerator )
                 {
-                    executeDomainDebugItem( debugItemInput, serviceClass, debugGeneratorLogFile, zipOutput, pathPrefix );
+                    executeDomainDebugItem( debugItemInput, serviceClass, debugLogger, zipOutput, pathPrefix );
                 }
             }
         }
 
         {
             final String msg = "completed";
-            debugGeneratorLogFile.appendLine( msg );
+            debugLogger.appendLine( msg );
             LOGGER.trace( sessionLabel, () -> msg, TimeDuration.fromCurrent( startTime ) );
         }
 
         try
         {
             zipOutput.putNextEntry( new ZipEntry( getFilenameBase() + "/" + debugFileName ) );
-            zipOutput.write( debugGeneratorLogFile.toString().getBytes( PwmConstants.DEFAULT_CHARSET ) );
+            zipOutput.write( debugLogger.toString().getBytes( PwmConstants.DEFAULT_CHARSET ) );
             zipOutput.closeEntry();
         }
         catch ( final Exception e )
@@ -155,9 +127,9 @@ public class DebugItemGenerator
     }
 
     private void executeAppDebugItem(
-            final AppDebugItemInput debugItemInput,
-            final Class<? extends AppItemGenerator> serviceClass,
-            final DebugOutputBuilder debugGeneratorLogFile,
+            final AppDebugItemRequest debugItemInput,
+            final AppItemGenerator serviceClass,
+            final DebugGeneratorLogger debugGeneratorLogFile,
             final ZipOutputStream zipOutput,
             final String pathPrefix
     )
@@ -165,31 +137,29 @@ public class DebugItemGenerator
         try
         {
             final Instant itemStartTime = Instant.now();
-            LOGGER.trace( sessionLabel, () -> "beginning output of item " + serviceClass.getSimpleName() );
-            final AppItemGenerator newAppItemGeneratorItem = serviceClass.getDeclaredConstructor().newInstance();
-            zipOutput.putNextEntry( new ZipEntry( pathPrefix + newAppItemGeneratorItem.getFilename() ) );
-            newAppItemGeneratorItem.outputItem( debugItemInput, zipOutput );
+            LOGGER.trace( sessionLabel, () -> "beginning output of item " + serviceClass.getClass().getSimpleName() );
+            zipOutput.putNextEntry( new ZipEntry( pathPrefix + serviceClass.getFilename() ) );
+            serviceClass.outputItem( debugItemInput, zipOutput );
             zipOutput.closeEntry();
             zipOutput.flush();
-            final String finishMsg = "completed output of " + newAppItemGeneratorItem.getFilename()
-                    + " in " + TimeDuration.fromCurrent( itemStartTime ).asCompactString();
-            LOGGER.trace( sessionLabel, () -> finishMsg );
-            debugGeneratorLogFile.appendLine( finishMsg );
+            final Supplier<String> finishMsg = () -> "completed output of " + serviceClass.getFilename();
+            LOGGER.trace( sessionLabel, finishMsg, TimeDuration.fromCurrent( itemStartTime ) );
+            debugGeneratorLogFile.appendLine( finishMsg.get() );
         }
         catch ( final Throwable e )
         {
-            final String errorMsg = "unexpected error executing debug item output class '" + serviceClass.getName() + "', error: " + e;
-            LOGGER.error( sessionLabel, () -> errorMsg, e );
-            debugGeneratorLogFile.appendLine( errorMsg );
+            final Supplier<String> errorMsg = () -> "unexpected error executing debug item output class '" + serviceClass.getClass().getName() + "', error: " + e;
+            LOGGER.error( sessionLabel, errorMsg, e );
+            debugGeneratorLogFile.appendLine( errorMsg.get() );
             debugGeneratorLogFile.appendLine( JavaHelper.stackTraceToString( e ) );
         }
     }
 
 
     void executeDomainDebugItem(
-            final DomainDebugItemInput debugItemInput,
-            final Class<? extends DomainItemGenerator> serviceClass,
-            final DebugOutputBuilder debugGeneratorLogFile,
+            final DomainDebugItemRequest debugItemInput,
+            final DomainItemGenerator serviceClass,
+            final DebugGeneratorLogger debugGeneratorLogFile,
             final ZipOutputStream zipOutput,
             final String pathPrefix
     )
@@ -197,25 +167,22 @@ public class DebugItemGenerator
         try
         {
             final Instant itemStartTime = Instant.now();
-            LOGGER.trace( sessionLabel, () -> "beginning output of item " + serviceClass.getSimpleName() );
-            final DomainItemGenerator newAppItemGeneratorItem = serviceClass.getDeclaredConstructor().newInstance();
-            zipOutput.putNextEntry( new ZipEntry( pathPrefix + newAppItemGeneratorItem.getFilename() ) );
-            newAppItemGeneratorItem.outputItem( debugItemInput, zipOutput );
+            LOGGER.trace( sessionLabel, () -> "beginning output of item " + serviceClass.getClass().getSimpleName() );
+            zipOutput.putNextEntry( new ZipEntry( pathPrefix + serviceClass.getFilename() ) );
+            serviceClass.outputItem( debugItemInput, zipOutput );
             zipOutput.closeEntry();
             zipOutput.flush();
-            final String finishMsg = "completed output of " + newAppItemGeneratorItem.getFilename()
-                    + " in " + TimeDuration.fromCurrent( itemStartTime ).asCompactString();
-            LOGGER.trace( sessionLabel, () -> finishMsg );
-            debugGeneratorLogFile.appendLine( finishMsg );
+            final Supplier<String> finishMsg = () -> "completed output of " + serviceClass.getFilename();
+            LOGGER.trace( sessionLabel, finishMsg, TimeDuration.fromCurrent( itemStartTime ) );
+            debugGeneratorLogFile.appendLine( finishMsg.get() );
         }
         catch ( final Throwable e )
         {
-            final String errorMsg = "unexpected error executing debug item output class '" + serviceClass.getName() + "', error: " + e;
-            LOGGER.error( sessionLabel, () -> errorMsg, e );
-            debugGeneratorLogFile.appendLine( errorMsg );
+            final Supplier<String> errorMsg = () -> "unexpected error executing debug item output class '" + serviceClass.getClass().getName() + "', error: " + e;
+            LOGGER.error( sessionLabel, errorMsg, e );
+            debugGeneratorLogFile.appendLine( errorMsg.get() );
             debugGeneratorLogFile.appendLine( JavaHelper.stackTraceToString( e ) );
         }
-
     }
 
     static void writeString( final OutputStream outputStream, final String value )

server/src/main/java/password/pwm/util/debug/DebugGeneratorLogger.java

@@ -0,0 +1,66 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.util.debug;
+
+import password.pwm.bean.SessionLabel;
+import password.pwm.error.ErrorInformation;
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.logging.PwmLogger;
+
+import java.io.StringWriter;
+import java.time.Instant;
+import java.util.function.Supplier;
+
+public class DebugGeneratorLogger
+{
+    private final StringWriter writer = new StringWriter();
+    private final SessionLabel sessionLabel;
+
+    public DebugGeneratorLogger( final SessionLabel sessionLabel )
+    {
+        this.sessionLabel = sessionLabel;
+    }
+
+    public void appendLine( final CharSequence charSequence )
+    {
+        writer.append( StringUtil.toIsoDate( Instant.now() ) );
+        writer.append( ' ' );
+        writer.append( charSequence );
+        writer.append( '\n' );
+    }
+
+    public void error( final ItemGenerator source, final ErrorInformation errorInformation )
+    {
+        appendLine( errorInformation.toDebugStr() );
+        PwmLogger.forClass( source.getClass() ).error( sessionLabel, errorInformation );
+    }
+
+    public void error( final ItemGenerator source, final Supplier<String> message )
+    {
+        appendLine( message.get() );
+        PwmLogger.forClass( source.getClass() ).error( sessionLabel, message );
+    }
+
+    public String toString()
+    {
+        return writer.toString();
+    }
+}

server/src/main/java/password/pwm/util/debug/DomainDebugItemInput.java → server/src/main/java/password/pwm/util/debug/DomainDebugItemRequest.java

@@ -20,18 +20,18 @@
 
 package password.pwm.util.debug;
 
-import lombok.Value;
 import password.pwm.PwmDomain;
 import password.pwm.bean.SessionLabel;
 import password.pwm.config.DomainConfig;
 
 import java.util.Locale;
 
-@Value
-class DomainDebugItemInput
+record DomainDebugItemRequest(
+        PwmDomain pwmDomain,
+        SessionLabel sessionLabel,
+        DomainConfig obfuscatedDomainConfig,
+        Locale locale,
+        DebugGeneratorLogger logger
+)
 {
-    private final PwmDomain pwmDomain;
-    private final SessionLabel sessionLabel;
-    private final DomainConfig obfuscatedDomainConfig;
-    private final Locale locale;
 }

server/src/main/java/password/pwm/util/debug/DomainItemGenerator.java

@@ -25,13 +25,17 @@ import password.pwm.error.PwmUnrecoverableException;
 import java.io.IOException;
 import java.io.OutputStream;
 
-interface DomainItemGenerator
+sealed interface DomainItemGenerator extends ItemGenerator permits
+        LDAPPermissionItemGenerator,
+        CacheServiceDebugItemGenerator,
+        LdapDebugItemGenerator,
+        LdapRecentUserDebugGenerator,
+        DashboardDataDebugItemGenerator,
+        LdapConnectionsDebugItemGenerator
 {
-
-    String getFilename();
-
     void outputItem(
-            DomainDebugItemInput debugItemInput,
+            DomainDebugItemRequest debugItemInput,
             OutputStream outputStream
-    ) throws IOException, PwmUnrecoverableException;
+    )
+            throws IOException, PwmUnrecoverableException;
 }

server/src/main/java/password/pwm/util/debug/FileInfoDebugItemGenerator.java

@@ -22,22 +22,31 @@ package password.pwm.util.debug;
 
 import org.apache.commons.csv.CSVPrinter;
 import password.pwm.PwmApplication;
+import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.java.PwmUtil;
 import password.pwm.util.java.StringUtil;
-import password.pwm.util.logging.PwmLogger;
 
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Optional;
+import java.util.function.Consumer;
 
-class FileInfoDebugItemGenerator implements AppItemGenerator
+final class FileInfoDebugItemGenerator implements AppItemGenerator
 {
-    private static final PwmLogger LOGGER = PwmLogger.forClass( FileInfoDebugItemGenerator.class );
+    enum CsvHeaders
+    {
+        Filepath,
+        Filename,
+        ModifiedTimestamp,
+        Size,
+        Sha512Hash,
+    }
 
     @Override
     public String getFilename()
@@ -46,10 +55,10 @@ class FileInfoDebugItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
-        final PwmApplication pwmApplication = debugItemInput.getPwmApplication();
+        final PwmApplication pwmApplication = debugItemInput.pwmApplication();
         final Path applicationPath = pwmApplication.getPwmEnvironment().getApplicationPath();
         final List<Path> interestedFiles = new ArrayList<>();
 
@@ -70,7 +79,7 @@ class FileInfoDebugItemGenerator implements AppItemGenerator
             }
             catch ( final Exception e )
             {
-                LOGGER.error( debugItemInput.getSessionLabel(), () -> "unable to generate webInfPath fileMd5sums during zip debug building: " + e.getMessage() );
+                debugItemInput.logger().error( this, () -> "unable to generate webInfPath fileMd5sums during zip debug building: " + e.getMessage()  );
             }
         }
 
@@ -82,40 +91,41 @@ class FileInfoDebugItemGenerator implements AppItemGenerator
             }
             catch ( final Exception e )
             {
-                LOGGER.error( debugItemInput.getSessionLabel(), () -> "unable to generate appPath fileMd5sums during zip debug building: " + e.getMessage() );
+                debugItemInput.logger().error( this, () -> "unable to generate appPath fileMd5sums during zip debug building: " + e.getMessage() );
             }
         }
 
         final CSVPrinter csvPrinter = PwmUtil.makeCsvPrinter( outputStream );
         {
-            final List<String> headerRow = new ArrayList<>();
-            headerRow.add( "Filepath" );
-            headerRow.add( "Filename" );
-            headerRow.add( "Last Modified" );
-            headerRow.add( "Size" );
-            headerRow.add( "Sha512Hash" );
+            final List<String> headerRow = CollectionUtil.enumSetToStringList( EnumSet.allOf( CsvHeaders.class ) );
             csvPrinter.printComment( StringUtil.join( headerRow, "," ) );
         }
 
-        final List<FileSystemUtility.FileSummaryInformation> fileSummaries = FileSystemUtility.readFileInformation( interestedFiles );
-        for ( final FileSystemUtility.FileSummaryInformation fileSummaryInformation : fileSummaries )
+        final Consumer<FileSystemUtility.FileSummaryInformation> consumer = new Consumer<FileSystemUtility.FileSummaryInformation>()
         {
-            try
+            @Override
+            public void accept( final FileSystemUtility.FileSummaryInformation fileSummaryInformation )
             {
-                final List<String> dataRow = List.of(
-                        fileSummaryInformation.getFilepath(),
-                        fileSummaryInformation.getFilename(),
-                        StringUtil.toIsoDate( fileSummaryInformation.getModified() ),
-                        String.valueOf( fileSummaryInformation.getSize() ),
-                        fileSummaryInformation.getSha512Hash() );
+                try
+                {
+                    final List<String> dataRow = List.of(
+                            fileSummaryInformation.filepath(),
+                            fileSummaryInformation.filename(),
+                            StringUtil.toIsoDate( fileSummaryInformation.modified() ),
+                            String.valueOf( fileSummaryInformation.size() ),
+                            fileSummaryInformation.sha512Hash() );
 
-                csvPrinter.printRecord( dataRow );
-            }
-            catch ( final Exception e )
-            {
-                LOGGER.trace( () -> "error generating file summary info: " + e.getMessage() );
+                    csvPrinter.printRecord( dataRow );
+                }
+                catch ( final Exception e )
+                {
+                    debugItemInput.logger().error( FileInfoDebugItemGenerator.this,
+                            () -> "error generating file summary info: " + e.getMessage() );
+                }
             }
-        }
+        };
+
+        FileSystemUtility.readFileInformation( interestedFiles ).forEach( consumer );
         csvPrinter.flush();
     }
 }

server/src/main/java/password/pwm/util/debug/HealthDebugItemGenerator.java

@@ -34,7 +34,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Set;
 
-class HealthDebugItemGenerator implements AppItemGenerator
+final class HealthDebugItemGenerator implements AppItemGenerator
 {
     @Value
     private static class HealthDebugInfo
@@ -50,16 +50,16 @@ class HealthDebugItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
         final Locale locale = PwmConstants.DEFAULT_LOCALE;
-        final PwmApplication pwmApplication = debugItemInput.getPwmApplication();
+        final PwmApplication pwmApplication = debugItemInput.pwmApplication();
         final Set<HealthRecord> records = pwmApplication.getHealthMonitor().getHealthRecords();
 
         final List<HealthDebugInfo> outputInfos = new ArrayList<>();
         records.forEach( healthRecord -> outputInfos.add( new HealthDebugInfo( healthRecord, healthRecord.getDetail( locale,
-                debugItemInput.getObfuscatedAppConfig() ) ) ) );
+                debugItemInput.obfuscatedAppConfig() ) ) ) );
         final String recordJson = JsonFactory.get().serializeCollection( outputInfos, JsonProvider.Flag.PrettyPrint );
         outputStream.write( recordJson.getBytes( PwmConstants.DEFAULT_CHARSET ) );
     }

server/src/main/java/password/pwm/util/debug/IntruderDataGenerator.java

@@ -31,7 +31,7 @@ import password.pwm.util.json.JsonFactory;
 import java.io.IOException;
 import java.io.OutputStream;
 
-class IntruderDataGenerator implements AppItemGenerator
+final class IntruderDataGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -40,10 +40,10 @@ class IntruderDataGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException, PwmUnrecoverableException
     {
-        final PwmApplication pwmApplication = debugItemInput.getPwmApplication();
+        final PwmApplication pwmApplication = debugItemInput.pwmApplication();
         final IntruderSystemService service = pwmApplication.getIntruderSystemService();
 
         try ( ClosableIterator<PublicIntruderRecord> record = service.allRecordIterator() )

server/src/main/java/password/pwm/util/java/DebugOutputBuilder.java → server/src/main/java/password/pwm/util/debug/ItemGenerator.java

@@ -18,24 +18,9 @@
  * limitations under the License.
  */
 
-package password.pwm.util.java;
+package password.pwm.util.debug;
 
-import java.time.Instant;
-
-public class DebugOutputBuilder
+public interface ItemGenerator
 {
-    private final StringBuilder stringBuilder = new StringBuilder();
-
-    public void appendLine( final CharSequence charSequence )
-    {
-        stringBuilder.append( StringUtil.toIsoDate( Instant.now() ) );
-        stringBuilder.append( ' ' );
-        stringBuilder.append( charSequence );
-        stringBuilder.append( '\n' );
-    }
-
-    public String toString()
-    {
-        return stringBuilder.toString();
-    }
+    String getFilename();
 }

server/src/main/java/password/pwm/util/debug/LDAPPermissionItemGenerator.java

@@ -32,7 +32,7 @@ import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
 
-class LDAPPermissionItemGenerator implements DomainItemGenerator
+final class LDAPPermissionItemGenerator implements DomainItemGenerator
 {
     @Override
     public String getFilename()
@@ -41,11 +41,11 @@ class LDAPPermissionItemGenerator implements DomainItemGenerator
     }
 
     @Override
-    public void outputItem( final DomainDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final DomainDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException, PwmUnrecoverableException
     {
 
-        final DomainConfig domainConfig = debugItemInput.getObfuscatedDomainConfig();
+        final DomainConfig domainConfig = debugItemInput.obfuscatedDomainConfig();
         final LdapPermissionCalculator ldapPermissionCalculator = new LdapPermissionCalculator( domainConfig );
 
         final CSVPrinter csvPrinter = PwmUtil.makeCsvPrinter( outputStream );

server/src/main/java/password/pwm/util/debug/LdapConnectionsDebugItemGenerator.java

@@ -30,7 +30,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.util.List;
 
-class LdapConnectionsDebugItemGenerator implements DomainItemGenerator
+final class LdapConnectionsDebugItemGenerator implements DomainItemGenerator
 {
     @Override
     public String getFilename()
@@ -39,10 +39,10 @@ class LdapConnectionsDebugItemGenerator implements DomainItemGenerator
     }
 
     @Override
-    public void outputItem( final DomainDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final DomainDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
-        final PwmDomain pwmDomain = debugItemInput.getPwmDomain();
+        final PwmDomain pwmDomain = debugItemInput.pwmDomain();
         final List<LdapDomainService.ConnectionInfo> connectionInfos = pwmDomain.getLdapService().getConnectionInfos();
         final String jsonString = JsonFactory.get().serializeCollection( connectionInfos, JsonProvider.Flag.PrettyPrint );
         outputStream.write( jsonString.getBytes( PwmConstants.DEFAULT_CHARSET ) );

server/src/main/java/password/pwm/util/debug/LdapDebugItemGenerator.java

@@ -28,7 +28,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.util.List;
 
-class LdapDebugItemGenerator implements DomainItemGenerator
+final class LdapDebugItemGenerator implements DomainItemGenerator
 {
     @Override
     public String getFilename()
@@ -37,16 +37,16 @@ class LdapDebugItemGenerator implements DomainItemGenerator
     }
 
     @Override
-    public void outputItem( final DomainDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final DomainDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
         final List<LdapDebugDataGenerator.LdapDebugInfo> ldapDebugInfos = LdapDebugDataGenerator.makeLdapDebugInfos(
-                debugItemInput.getPwmDomain(),
-                debugItemInput.getSessionLabel(),
-                debugItemInput.getObfuscatedDomainConfig(),
-                debugItemInput.getLocale()
+                debugItemInput.pwmDomain(),
+                debugItemInput.sessionLabel(),
+                debugItemInput.obfuscatedDomainConfig(),
+                debugItemInput.locale()
         );
         final String jsonData = JsonFactory.get().serializeCollection( ldapDebugInfos, JsonProvider.Flag.PrettyPrint );
-        DebugItemGenerator.writeString( outputStream, jsonData );
+        DebugGenerator.writeString( outputStream, jsonData );
     }
 }

server/src/main/java/password/pwm/util/debug/LdapRecentUserDebugGenerator.java

@@ -35,7 +35,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
-class LdapRecentUserDebugGenerator implements DomainItemGenerator
+final class LdapRecentUserDebugGenerator implements DomainItemGenerator
 {
     @Override
     public String getFilename()
@@ -44,10 +44,10 @@ class LdapRecentUserDebugGenerator implements DomainItemGenerator
     }
 
     @Override
-    public void outputItem( final DomainDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final DomainDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException, PwmUnrecoverableException
     {
-        final PwmDomain pwmDomain = debugItemInput.getPwmDomain();
+        final PwmDomain pwmDomain = debugItemInput.pwmDomain();
         final List<UserIdentity> recentUsers = pwmDomain.getPwmApplication().getSessionTrackService().getRecentLogins();
         final List<UserDebugDataBean> recentDebugBeans = new ArrayList<>();
 
@@ -57,8 +57,8 @@ class LdapRecentUserDebugGenerator implements DomainItemGenerator
             {
                 final UserDebugDataBean dataBean = UserDebugDataReader.readUserDebugData(
                         pwmDomain,
-                        debugItemInput.getLocale(),
-                        debugItemInput.getSessionLabel(),
+                        debugItemInput.locale(),
+                        debugItemInput.sessionLabel(),
                         userIdentity
                 );
                 recentDebugBeans.add( dataBean );

server/src/main/java/password/pwm/util/debug/LocalDBDebugGenerator.java

@@ -30,7 +30,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.util.Map;
 
-class LocalDBDebugGenerator implements AppItemGenerator
+final class LocalDBDebugGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -39,10 +39,10 @@ class LocalDBDebugGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
-        final PwmApplication pwmApplication = debugItemInput.getPwmApplication();
+        final PwmApplication pwmApplication = debugItemInput.pwmApplication();
         final LocalDB localDB = pwmApplication.getLocalDB();
         final Map<String, Object> serializableMap = localDB.debugInfo();
         outputStream.write( JsonFactory.get().serializeMap( serializableMap, JsonProvider.Flag.PrettyPrint ).getBytes( PwmConstants.DEFAULT_CHARSET ) );

server/src/main/java/password/pwm/util/debug/LogDebugItemGenerator.java

@@ -30,17 +30,13 @@ import password.pwm.util.logging.LocalDBSearchQuery;
 import password.pwm.util.logging.LocalDBSearchResults;
 import password.pwm.util.logging.PwmLogEvent;
 import password.pwm.util.logging.PwmLogLevel;
-import password.pwm.util.logging.PwmLogger;
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.time.Instant;
 import java.util.function.Function;
 
-class LogDebugItemGenerator implements AppItemGenerator
+final class LogDebugItemGenerator implements AppItemGenerator
 {
-    private static final PwmLogger LOGGER = PwmLogger.forClass( LogDebugItemGenerator.class );
-
     static void outputLogs(
             final PwmApplication pwmApplication,
             final OutputStream outputStream,
@@ -75,13 +71,11 @@ class LogDebugItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
-        final Instant startTime = Instant.now();
         final Function<PwmLogEvent, String> logEventFormatter = PwmLogEvent::toLogString;
 
-        outputLogs( debugItemInput.getPwmApplication(), outputStream, logEventFormatter );
-        LOGGER.trace( () -> "debug log output completed in ", TimeDuration.fromCurrent( startTime ) );
+        outputLogs( debugItemInput.pwmApplication(), outputStream, logEventFormatter );
     }
 }

server/src/main/java/password/pwm/util/debug/LogJsonItemGenerator.java

@@ -22,6 +22,7 @@ package password.pwm.util.debug;
 
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.json.JsonFactory;
+import password.pwm.util.json.JsonProvider;
 import password.pwm.util.logging.PwmLogEvent;
 import password.pwm.util.logging.PwmLogger;
 
@@ -30,7 +31,7 @@ import java.io.OutputStream;
 import java.time.Instant;
 import java.util.function.Function;
 
-class LogJsonItemGenerator implements AppItemGenerator
+final class LogJsonItemGenerator implements AppItemGenerator
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( LogJsonItemGenerator.class );
 
@@ -41,13 +42,14 @@ class LogJsonItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
         final Instant startTime = Instant.now();
-        final Function<PwmLogEvent, String> logEventFormatter = JsonFactory.get()::serialize;
+        final JsonProvider jsonFactory = JsonFactory.get();
+        final Function<PwmLogEvent, String> logEventFormatter = pwmLogEvent -> jsonFactory.serialize( pwmLogEvent ) + "\n";
 
-        LogDebugItemGenerator.outputLogs( debugItemInput.getPwmApplication(), outputStream, logEventFormatter );
+        LogDebugItemGenerator.outputLogs( debugItemInput.pwmApplication(), outputStream, logEventFormatter );
         LOGGER.trace( () -> "debug json output completed in ", TimeDuration.fromCurrent( startTime ) );
     }
 }

server/src/main/java/password/pwm/util/debug/RootFileSystemDebugItemGenerator.java

@@ -20,8 +20,6 @@
 
 package password.pwm.util.debug;
 
-import lombok.Builder;
-import lombok.Value;
 import password.pwm.PwmConstants;
 import password.pwm.util.json.JsonFactory;
 import password.pwm.util.json.JsonProvider;
@@ -35,7 +33,7 @@ import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 
-class RootFileSystemDebugItemGenerator implements AppItemGenerator
+final class RootFileSystemDebugItemGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -44,49 +42,53 @@ class RootFileSystemDebugItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
-        final Collection<RootFileSystemInfo> rootInfos = RootFileSystemInfo.forAllRootFileSystems();
+        final Collection<RootFileSystemInfo> rootInfos = forAllRootFileSystems( debugItemInput );
         outputStream.write( JsonFactory.get().serializeCollection( rootInfos, JsonProvider.Flag.PrettyPrint ).getBytes( PwmConstants.DEFAULT_CHARSET ) );
     }
 
-    @Value
-    @Builder
-    private static class RootFileSystemInfo
+    private List<RootFileSystemInfo> forAllRootFileSystems( final AppDebugItemRequest appDebugItemInput )
     {
-        private String name;
-        private String type;
-        private long totalSpace;
-        private long freeSpace;
-        private long usableSpace;
+        final Iterator<FileStore> fileStoreIterator = FileSystems.getDefault().getFileStores().iterator();
 
-        static Collection<RootFileSystemInfo> forAllRootFileSystems()
-                throws IOException
+        final List<RootFileSystemInfo> returnList = new ArrayList<>();
+        while ( fileStoreIterator.hasNext() )
         {
-            final Iterator<FileStore> fileStoreIterator = FileSystems.getDefault().getFileStores().iterator();
-
-            final List<RootFileSystemInfo> returnList = new ArrayList<>();
-            while ( fileStoreIterator.hasNext() )
+            final FileStore fileStore = fileStoreIterator.next();
+            try
             {
-                final FileStore fileStore = fileStoreIterator.next();
                 final RootFileSystemInfo rootFileSystemInfo = RootFileSystemInfo.forRoot( fileStore );
                 returnList.add( rootFileSystemInfo );
             }
-
-            return List.copyOf( returnList );
+            catch ( final Throwable t )
+            {
+                final String errorMsg = "error during root filesystem debug reading: " + t.getMessage();
+                appDebugItemInput.logger().error( this, () -> errorMsg );
+            }
         }
 
+        return List.copyOf( returnList );
+    }
+
+    private record RootFileSystemInfo(
+            String name,
+            String type,
+            long totalSpace,
+            long freeSpace,
+            long usableSpace
+    )
+    {
         static RootFileSystemInfo forRoot( final FileStore fileRoot )
                 throws IOException
         {
-            return RootFileSystemInfo.builder()
-                    .name( fileRoot.name() )
-                    .type( fileRoot.type() )
-                    .totalSpace( fileRoot.getTotalSpace() )
-                    .freeSpace( fileRoot.getUnallocatedSpace() )
-                    .usableSpace( fileRoot.getUsableSpace() )
-                    .build();
+            return new RootFileSystemInfo(
+                     fileRoot.name(),
+                     fileRoot.type(),
+                     fileRoot.getTotalSpace(),
+                     fileRoot.getUnallocatedSpace(),
+                     fileRoot.getUsableSpace() );
         }
     }
 }

server/src/main/java/password/pwm/util/debug/ServicesDebugItemGenerator.java

@@ -31,7 +31,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.util.List;
 
-class ServicesDebugItemGenerator implements AppItemGenerator
+final class ServicesDebugItemGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -40,10 +40,10 @@ class ServicesDebugItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException, PwmUnrecoverableException
     {
-        final PwmApplication pwmApplication = debugItemInput.getPwmApplication();
+        final PwmApplication pwmApplication = debugItemInput.pwmApplication();
         final List<AppDashboardData.ServiceData> serviceDataList = AppDashboardData.makeServiceData( pwmApplication );
         final String recordJson = JsonFactory.get().serializeCollection( serviceDataList, JsonProvider.Flag.PrettyPrint );
         outputStream.write( recordJson.getBytes( PwmConstants.DEFAULT_CHARSET ) );

server/src/main/java/password/pwm/util/debug/SessionDataGenerator.java

@@ -25,7 +25,7 @@ import password.pwm.PwmApplication;
 import java.io.IOException;
 import java.io.OutputStream;
 
-class SessionDataGenerator implements AppItemGenerator
+final class SessionDataGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -34,10 +34,10 @@ class SessionDataGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
-        final PwmApplication pwmDomain = debugItemInput.getPwmApplication();
-        pwmDomain.getSessionTrackService().outputToCsv( debugItemInput.getLocale(), pwmDomain.getConfig(), outputStream );
+        final PwmApplication pwmDomain = debugItemInput.pwmApplication();
+        pwmDomain.getSessionTrackService().outputToCsv( debugItemInput.locale(), pwmDomain.getConfig(), outputStream );
     }
 }

server/src/main/java/password/pwm/util/debug/StatisticsDataDebugItemGenerator.java

@@ -27,7 +27,7 @@ import password.pwm.svc.stats.StatisticsUtils;
 import java.io.IOException;
 import java.io.OutputStream;
 
-class StatisticsDataDebugItemGenerator implements AppItemGenerator
+final class StatisticsDataDebugItemGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -36,16 +36,16 @@ class StatisticsDataDebugItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
-        final PwmApplication pwmApplication = debugItemInput.getPwmApplication();
+        final PwmApplication pwmApplication = debugItemInput.pwmApplication();
         final StatisticsService statsManager = pwmApplication.getStatisticsService();
         StatisticsUtils.outputStatsToCsv(
-                debugItemInput.getSessionLabel(),
+                debugItemInput.sessionLabel(),
                 statsManager,
                 outputStream,
-                debugItemInput.getLocale(),
+                debugItemInput.locale(),
                 StatisticsUtils.CsvOutputFlag.includeHeader );
     }
 }

server/src/main/java/password/pwm/util/debug/StatisticsEpsDataDebugItemGenerator.java

@@ -27,7 +27,6 @@ import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsService;
 import password.pwm.util.java.PwmUtil;
 import password.pwm.util.java.StringUtil;
-import password.pwm.util.logging.PwmLogger;
 
 import java.io.IOException;
 import java.io.OutputStream;
@@ -35,10 +34,8 @@ import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.List;
 
-class StatisticsEpsDataDebugItemGenerator implements AppItemGenerator
+final class StatisticsEpsDataDebugItemGenerator implements AppItemGenerator
 {
-    private static final PwmLogger LOGGER = PwmLogger.forClass( StatisticsDataDebugItemGenerator.class );
-
     @Override
     public String getFilename()
     {
@@ -46,11 +43,11 @@ class StatisticsEpsDataDebugItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
 
     {
-        final PwmApplication pwmDomain = debugItemInput.getPwmApplication();
+        final PwmApplication pwmDomain = debugItemInput.pwmApplication();
         final StatisticsService statsManager = pwmDomain.getStatisticsService();
         final CSVPrinter csvPrinter = PwmUtil.makeCsvPrinter( outputStream );
         {
@@ -69,14 +66,14 @@ class StatisticsEpsDataDebugItemGenerator implements AppItemGenerator
                     final List<String> dataRow = new ArrayList<>();
                     final BigDecimal value = statsManager.readEps( epsStatistic, epsDuration );
                     final String sValue = value.toPlainString();
-                    dataRow.add( epsStatistic.getLabel( debugItemInput.getLocale() ) );
+                    dataRow.add( epsStatistic.getLabel( debugItemInput.locale() ) );
                     dataRow.add( epsDuration.getTimeDuration().asCompactString() );
                     dataRow.add( sValue );
                     csvPrinter.printRecord( dataRow );
                 }
                 catch ( final Exception e )
                 {
-                    LOGGER.trace( () -> "error generating csv-stats summary info: " + e.getMessage() );
+                    debugItemInput.logger().error( this, () -> "error generating csv-stats summary info: " + e.getMessage() );
                 }
             }
         }

server/src/main/java/password/pwm/util/debug/SystemEnvironmentItemGenerator.java

@@ -26,7 +26,7 @@ import password.pwm.util.json.JsonProvider;
 import java.io.IOException;
 import java.io.OutputStream;
 
-class SystemEnvironmentItemGenerator implements AppItemGenerator
+final class SystemEnvironmentItemGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -35,10 +35,10 @@ class SystemEnvironmentItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
         final String json = JsonFactory.get().serializeMap( System.getenv(), JsonProvider.Flag.PrettyPrint );
-        DebugItemGenerator.writeString( outputStream, json );
+        DebugGenerator.writeString( outputStream, json );
     }
 }

server/src/main/java/password/pwm/util/debug/ThreadDumpDebugItemGenerator.java

@@ -27,7 +27,7 @@ import java.io.OutputStream;
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadInfo;
 
-class ThreadDumpDebugItemGenerator implements AppItemGenerator
+final class ThreadDumpDebugItemGenerator implements AppItemGenerator
 {
     @Override
     public String getFilename()
@@ -36,13 +36,13 @@ class ThreadDumpDebugItemGenerator implements AppItemGenerator
     }
 
     @Override
-    public void outputItem( final AppDebugItemInput debugItemInput, final OutputStream outputStream )
+    public void outputItem( final AppDebugItemRequest debugItemInput, final OutputStream outputStream )
             throws IOException
     {
         final ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true );
         for ( final ThreadInfo threadInfo : threads )
         {
-            DebugItemGenerator.writeString( outputStream,  JavaHelper.threadInfoToString( threadInfo ) + '\n' );
+            DebugGenerator.writeString( outputStream,  JavaHelper.threadInfoToString( threadInfo ) + '\n' );
         }
     }
 }

server/src/main/java/password/pwm/util/java/FileSystemUtility.java

@@ -20,7 +20,6 @@
 
 package password.pwm.util.java;
 
-import lombok.Value;
 import password.pwm.PwmConstants;
 import password.pwm.error.PwmException;
 import password.pwm.util.i18n.LocaleHelper;
@@ -32,50 +31,52 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public class FileSystemUtility
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( FileSystemUtility.class );
 
-    public static List<FileSummaryInformation> readFileInformation( final List<Path> rootFiles )
+    public static Stream<FileSummaryInformation> readFileInformation( final List<Path> rootFiles )
     {
-        final List<FileSummaryInformation> returnList = new ArrayList<>();
-        for ( final Path path : rootFiles )
-        {
-            returnList.addAll( readFileInformationHierarchy( path ) );
-        }
-        return List.copyOf( returnList );
+        return rootFiles.stream().flatMap( FileSystemUtility::readFileInformationHierarchy );
     }
 
-    public static List<FileSummaryInformation> readFileInformationHierarchy( final Path rootFile )
+    public static Stream<FileSummaryInformation> readFileInformationHierarchy( final Path rootFile )
     {
-        try
+        final Function<Path, Optional<FileSummaryInformation>> converter = path ->
         {
-            final List<Path> paths =  Files.walk( rootFile )
-                    .filter( Files::isRegularFile )
-                    .collect( Collectors.toList() );
-
-            final List<FileSummaryInformation> returnList = new ArrayList<>();
-            for ( final Path path : paths )
+            try
             {
-                returnList.add( FileSummaryInformation.fromFile( path ) );
+                return Optional.of( FileSummaryInformation.fromFile( path ) );
             }
-            return List.copyOf( returnList );
+            catch ( final IOException e )
+            {
+                LOGGER.trace( () -> "error during file summary load: " + e.getMessage() );
+                return Optional.empty();
+            }
+        };
 
+        try
+        {
+            return Files.walk( rootFile )
+                    .filter( Files::isRegularFile )
+                    .map( converter )
+                    .flatMap( Optional::stream );
         }
         catch ( final IOException e )
         {
             LOGGER.trace( () -> "error during file summary load: " + e.getMessage() );
         }
 
-        return Collections.emptyList();
+        return Stream.empty();
     }
 
     public static long getFileDirectorySize( final Path dir )
@@ -160,15 +161,14 @@ public class FileSystemUtility
         }
     }
 
-    @Value
-    public static class FileSummaryInformation
+    public record FileSummaryInformation(
+            String filename,
+            String filepath,
+            Instant modified,
+            long size,
+            String sha512Hash
+    )
     {
-        private final String filename;
-        private final String filepath;
-        private final Instant modified;
-        private final long size;
-        private final String sha512Hash;
-
         public static FileSummaryInformation fromFile( final Path file )
                 throws IOException
         {

server/src/main/java/password/pwm/util/logging/PwmLogEvent.java

@@ -63,8 +63,8 @@ public class PwmLogEvent implements Comparable<PwmLogEvent>
     private final String threadName;
 
     private static final Comparator<PwmLogEvent> COMPARATOR = Comparator.comparing(
-            PwmLogEvent::getTimestamp,
-            Comparator.nullsLast( Comparator.naturalOrder() ) )
+                    PwmLogEvent::getTimestamp,
+                    Comparator.nullsLast( Comparator.naturalOrder() ) )
             .thenComparing(
                     PwmLogEvent::getSessionID,
                     Comparator.nullsLast( Comparator.naturalOrder() ) )
@@ -164,15 +164,14 @@ public class PwmLogEvent implements Comparable<PwmLogEvent>
     {
         final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
         final CSVPrinter csvPrinter = PwmUtil.makeCsvPrinter( byteArrayOutputStream );
-
         final String throwableMessage = ( getLoggedThrowable() == null || getLoggedThrowable().getMessage() == null ) ? "" : getLoggedThrowable().getMessage();
 
         final List<String> dataRow = new ArrayList<>();
         dataRow.add( StringUtil.toIsoDate( getTimestamp() ) );
         dataRow.add( getLevel().name() );
-        dataRow.add( getSourceAddress( ) == null ? "" : getSourceAddress() );
+        dataRow.add( getSourceAddress() == null ? "" : getSourceAddress() );
         dataRow.add( getSessionID() == null ? "" : getSessionID() );
-        dataRow.add( getUsername( ) );
+        dataRow.add( getUsername() );
         dataRow.add( getTopic() );
         dataRow.add( getMessage() );
         dataRow.add( throwableMessage );

server/src/main/java/password/pwm/ws/server/rest/RestStatisticsServer.java

@@ -32,6 +32,7 @@ import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.http.HttpContentType;
 import password.pwm.http.HttpMethod;
 import password.pwm.http.PwmHttpRequestWrapper;
+import password.pwm.http.PwmRequestContext;
 import password.pwm.svc.stats.AvgStatistic;
 import password.pwm.svc.stats.EpsStatistic;
 import password.pwm.svc.stats.Statistic;
@@ -61,6 +62,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.NoSuchElementException;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
 
@@ -304,7 +306,7 @@ public class RestStatisticsServer extends RestServlet
 
                 if ( statName != null && statName.length() > 0 )
                 {
-                    jsonOutput.nameData = doNameStat( statisticsManager, statName, days );
+                    jsonOutput.nameData = doNameStat( restRequest.getPwmRestRequest(), statisticsManager, statName, days );
                 }
                 else
                 {
@@ -323,9 +325,19 @@ public class RestStatisticsServer extends RestServlet
             }
         }
 
-        public static Map<String, Object> doNameStat( final StatisticsService statisticsManager, final String statName, final String days )
+        public static Map<String, Object> doNameStat(
+                final PwmRequestContext pwmRequestContext,
+                final StatisticsService statisticsManager,
+                final String statName,
+                final String days
+        )
         {
-            final Statistic statistic = Statistic.forKey( statName ).orElseThrow();
+            final Statistic statistic = Statistic.forKey( statName ).orElseThrow( () ->
+            {
+                LOGGER.debug( pwmRequestContext.getSessionLabel(), () -> "request unknown statName '" + statName + "'" );
+                return new NoSuchElementException( "request for unknown statName" );
+            } );
+
             final int historyDays = StringUtil.convertStrToInt( days, 30 );
 
             return statisticsManager.getStatHistory( statistic, historyDays )

server/src/main/resources/password/pwm/AppProperty.properties

@@ -76,7 +76,7 @@ configEditor.settingFunction.timeoutMs=5000
 configGuide.idleTimeoutSeconds=3600
 configManager.zipDebug.maxLogBytes=50000000
 configManager.zipDebug.maxLogSeconds=120
-db.connections.max=5
+db.connections.max=-1
 db.connections.timeoutMs=30000
 db.connections.watchdogFrequencySeconds=30
 db.init.haltOnIndexCreateError=false

webapp/src/main/webapp/WEB-INF/jsp/admin-dashboard.jsp

@@ -45,6 +45,7 @@
 <!DOCTYPE html>
 <%@ page language="java" session="true" isThreadSafe="true" contentType="text/html" %>
 <%@ taglib uri="pwm" prefix="pwm" %>
+<% JspUtility.setFlag(pageContext, PwmRequestFlag.INCLUDE_DOJO); %>
 <% final AppDashboardData appDashboardData = (AppDashboardData)JspUtility.getAttribute(pageContext, PwmRequestAttribute.AppDashboardData); %>
 <% final PwmRequest dashboard_pwmRequest = JspUtility.getPwmRequest(pageContext); %>
 <% final PwmDomain dashboard_pwmDomain = dashboard_pwmRequest.getPwmDomain(); %>