Merge remote-tracking branch 'origin/master' into cofigmanager-ui

# Conflicts:
#	data-service/pom.xml
#	docker/pom.xml
#	onejar/pom.xml
#	pom.xml
#	rest-test-service/pom.xml
#	server/pom.xml
#	server/src/main/java/password/pwm/http/servlet/admin/AdminServlet.java
#	webapp/pom.xml
health message refactoring

server/src/main/java/password/pwm/AppProperty.java

@@ -200,6 +200,7 @@ public enum AppProperty
     HEALTH_SUPPORT_BUNDLE_FILE_WRITE_COUNT          ( "health.supportBundle.file.writeRetentionCount" ),
     HEALTH_DISK_MIN_FREE_WARNING                    ( "health.disk.minFreeWarning" ),
     HEALTH_CERTIFICATE_WARN_SECONDS                 ( "health.certificate.warnSeconds" ),
+    HEALTH_DB_CAUTION_DURATION_MS                   ( "health.database.cautionDurationMS" ),
     HEALTH_LDAP_CAUTION_DURATION_MS                 ( "health.ldap.cautionDurationMS" ),
     HEALTH_LDAP_PROXY_WARN_PW_EXPIRE_SECONDS        ( "health.ldap.proxy.pwExpireWarnSeconds" ),
     HEALTH_LDAP_USER_SEARCH_TERM                    ( "health.ldap.userSearch.searchTerm" ),

server/src/main/java/password/pwm/PwmApplication.java

@@ -101,13 +101,13 @@ import java.security.KeyStore;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 /**
@@ -277,7 +277,7 @@ public class PwmApplication
             StatisticsManager.incrementStat( this, Statistic.PWM_STARTUPS );
             LOGGER.debug( () -> "buildTime=" + PwmConstants.BUILD_TIME + ", javaLocale=" + Locale.getDefault() + ", DefaultLocale=" + PwmConstants.DEFAULT_LOCALE );
 
-            pwmScheduler.immediateExecuteInNewThread( this::postInitTasks );
+            pwmScheduler.immediateExecuteInNewThread( this::postInitTasks, this.getClass().getSimpleName() + " postInit tasks" );
         }
     }
 
@@ -297,32 +297,12 @@ public class PwmApplication
     {
         final Instant startTime = Instant.now();
 
-        try
-        {
-            outputConfigurationToLog( this );
-        }
-        catch ( final PwmException e )
-        {
-            LOGGER.error( () -> "error outputting log to debug: " + e.getMessage() );
-        }
+        getPwmScheduler().immediateExecuteInNewThread( UserAgentUtils::initializeCache, "initialize useragent cache" );
+        getPwmScheduler().immediateExecuteInNewThread( SettingDataMaker::initializeCache, "initialize PwmSetting metadata" );
 
-        if ( this.getConfig() != null )
-        {
-            final Map<AppProperty, String> nonDefaultProperties = getConfig().readAllNonDefaultAppProperties();
-            if ( nonDefaultProperties != null && !nonDefaultProperties.isEmpty() )
-            {
-                final Map<String, String> tempMap = new LinkedHashMap<>();
-                for ( final Map.Entry<AppProperty, String> entry : nonDefaultProperties.entrySet() )
-                {
-                    tempMap.put( entry.getKey().getKey(), entry.getValue() );
-                }
-                LOGGER.trace( () -> "non-default app properties read from configuration: " + JsonUtil.serializeMap( tempMap ) );
-            }
-            else
-            {
-                LOGGER.trace( () -> "no non-default app properties in configuration" );
-            }
-        }
+        outputConfigurationToLog( this );
+
+        outputNonDefaultPropertiesToLog( this );
 
         // send system audit event
         try
@@ -380,26 +360,6 @@ public class PwmApplication
 
         MBeanUtility.registerMBean( this );
 
-        try
-        {
-            UserAgentUtils.initializeCache();
-        }
-        catch ( final Exception e )
-        {
-            LOGGER.debug( () -> "error initializing UserAgentUtils: " + e.getMessage() );
-        }
-
-        try
-        {
-            final Instant itemStartTime = Instant.now();
-            SettingDataMaker.generateSettingData( this.getConfig().getStoredConfiguration(), null, PwmConstants.DEFAULT_LOCALE );
-            LOGGER.trace( () -> "completed setting data cache loading", () -> TimeDuration.fromCurrent( itemStartTime ) );
-        }
-        catch ( final Exception e )
-        {
-            LOGGER.debug( () -> "error initializing generateSettingData: " + e.getMessage() );
-        }
-
         {
             final ExecutorService executorService = PwmScheduler.makeSingleThreadExecutorService( this, PwmApplication.class );
             pwmScheduler.scheduleDailyZuluZeroStartJob( new DailySummaryJob( this ), executorService, TimeDuration.ZERO );
@@ -496,30 +456,47 @@ public class PwmApplication
     }
 
     private static void outputConfigurationToLog( final PwmApplication pwmApplication )
-            throws PwmUnrecoverableException
     {
-        if ( !LOGGER.isEnabled( PwmLogLevel.TRACE ) )
+        final Instant startTime = Instant.now();
+
+        final Function<Map.Entry<String, String>, String> valueFormatter = entry ->
         {
-            return;
-        }
+            final String spacedValue = entry.getValue().replace( "\n", "\n   " );
+            return " " + entry.getKey() + "\n   " + spacedValue + "\n";
+        };
 
         final StoredConfiguration storedConfiguration = pwmApplication.getConfig().getStoredConfiguration();
         final Map<String, String> debugStrings = StoredConfigurationUtil.makeDebugMap( storedConfiguration, storedConfiguration.modifiedItems(), PwmConstants.DEFAULT_LOCALE );
-        final List<Supplier<CharSequence>> outputStrings = new ArrayList<>();
 
-        for ( final Map.Entry<String, String> entry : debugStrings.entrySet() )
+        LOGGER.trace( () -> "--begin current configuration output--" );
+        debugStrings.entrySet().stream()
+                .map( valueFormatter )
+                .map( s -> ( Supplier<CharSequence> ) () -> s )
+                .forEach( LOGGER::trace );
+        LOGGER.trace( () -> "--end current configuration output--", () -> TimeDuration.fromCurrent( startTime ) );
+    }
+
+    private static void outputNonDefaultPropertiesToLog( final PwmApplication pwmApplication )
+    {
+        final Instant startTime = Instant.now();
+
+        final Map<AppProperty, String> nonDefaultProperties = pwmApplication.getConfig().readAllNonDefaultAppProperties();
+        if ( !JavaHelper.isEmpty( nonDefaultProperties ) )
         {
-            final String spacedValue = entry.getValue().replace( "\n", "\n   " );
-            final String output = " " + entry.getKey() + "\n   " + spacedValue + "\n";
-            outputStrings.add( () -> output );
+            LOGGER.trace( () -> "--begin non-default app properties output--" );
+            nonDefaultProperties.entrySet().stream()
+                    .map( entry -> "AppProperty: " + entry.getKey().getKey() + " -> " + entry.getValue() )
+                    .map( s -> ( Supplier<CharSequence> ) () -> s )
+                    .forEach( LOGGER::trace );
+            LOGGER.trace( () -> "--end non-default app properties output--", () -> TimeDuration.fromCurrent( startTime ) );
         }
+        else
+        {
+            LOGGER.trace( () -> "no non-default app properties in configuration" );
 
-        LOGGER.trace( () -> "--begin current configuration output--" );
-        outputStrings.forEach( LOGGER::trace );
-        LOGGER.trace( () -> "--end current configuration output--" );
+        }
     }
 
-
     public String getInstanceID( )
     {
         return instanceID;

server/src/main/java/password/pwm/config/Configuration.java

@@ -75,6 +75,7 @@ import java.lang.reflect.InvocationTargetException;
 import java.security.cert.X509Certificate;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.EnumMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -569,12 +570,12 @@ public class Configuration
 
     public Map<AppProperty, String> readAllNonDefaultAppProperties( )
     {
-        final LinkedHashMap<AppProperty, String> nonDefaultProperties = new LinkedHashMap<>();
+        final Map<AppProperty, String> nonDefaultProperties = new EnumMap<>( AppProperty.class );
         for ( final AppProperty loopProperty : AppProperty.values() )
         {
             final String configuredValue = readAppProperty( loopProperty );
             final String defaultValue = loopProperty.getDefaultValue();
-            if ( configuredValue != null && !configuredValue.equals( defaultValue ) )
+            if ( !Objects.equals(  configuredValue, defaultValue ) )
             {
                 nonDefaultProperties.put( loopProperty, configuredValue );
             }

server/src/main/java/password/pwm/config/PwmSettingMetaDataReader.java

@@ -117,8 +117,7 @@ public class PwmSettingMetaDataReader
 
     public String getExample( final PwmSettingTemplateSet template )
     {
-        return null;
-       // return PwmSetting.TemplateSetReference.referenceForTempleSet( examples.get(), template );
+         return PwmSetting.TemplateSetReference.referenceForTempleSet( examples.get(), template );
     }
 
     public boolean isRequired( )

server/src/main/java/password/pwm/config/stored/StoredConfigurationImpl.java

@@ -220,13 +220,7 @@ public class StoredConfigurationImpl implements StoredConfiguration
     public StoredValue readSetting( final PwmSetting setting, final String profileID )
     {
         final StoredConfigItemKey key = StoredConfigItemKey.fromSetting( setting, profileID );
-        final StoredValue storedValue = storedValues.get( key );
-        if ( storedValue == null )
-        {
-            return setting.getDefaultValue( getTemplateSet() );
-        }
-
-        return storedValue;
+        return StoredConfigurationUtil.getValueOrDefault( this, key );
     }
 
     @Override

server/src/main/java/password/pwm/config/stored/StoredConfigurationUtil.java

@@ -27,14 +27,17 @@ import password.pwm.config.PwmSetting;
 import password.pwm.config.PwmSettingCategory;
 import password.pwm.config.PwmSettingSyntax;
 import password.pwm.config.PwmSettingTemplateSet;
+import password.pwm.config.value.LocalizedStringArrayValue;
 import password.pwm.config.value.PasswordValue;
 import password.pwm.config.value.StoredValue;
+import password.pwm.config.value.StringValue;
 import password.pwm.config.value.ValueTypeConverter;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.PasswordData;
+import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.PwmExceptionLoggingConsumer;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
@@ -52,7 +55,6 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -366,44 +368,18 @@ public abstract class StoredConfigurationUtil
         LOGGER.debug( () -> "initialized new random security key" );
     }
 
-
     public static Map<String, String> makeDebugMap(
             final StoredConfiguration storedConfiguration,
             final Collection<StoredConfigItemKey> interestedItems,
             final Locale locale
     )
     {
-        final PwmSettingTemplateSet templateSet = storedConfiguration.getTemplateSet();
-
-        final Map<String, String> output = new TreeMap<>();
-
-        for ( final StoredConfigItemKey key : interestedItems )
-        {
-            if ( !StoredConfigItemKey.RecordType.PROPERTY.equals( key.getRecordType() ) )
-            {
-                final Optional<StoredValue> storedValue = storedConfiguration.readStoredValue( key );
-
-                String debugOutput = null;
-                if ( storedValue.isPresent() )
-                {
-                    debugOutput = storedValue.get().toDebugString( locale );
-                }
-                else
-                {
-                    if ( StoredConfigItemKey.RecordType.SETTING.equals( key.getRecordType() ) )
-                    {
-                        debugOutput = key.toPwmSetting().getDefaultValue( templateSet ).toDebugString( locale );
-                    }
-                }
-
-                if ( debugOutput != null )
-                {
-                    output.put( key.getLabel( locale ), debugOutput );
-                }
-            }
-        }
-
-        return Collections.unmodifiableMap( output );
+        return interestedItems.stream()
+                .filter( ( key ) -> !key.isRecordType( StoredConfigItemKey.RecordType.PROPERTY ) )
+                .collect( Collectors.toUnmodifiableMap(
+                        ( key ) -> key.getLabel( locale ),
+                        ( key ) -> StoredConfigurationUtil.getValueOrDefault( storedConfiguration, key ).toDebugString( locale )
+                ) );
     }
 
     public static Set<StoredConfigItemKey> allPossibleSettingKeysForConfiguration(
@@ -426,10 +402,10 @@ public abstract class StoredConfigurationUtil
             }
         };
 
-        return Collections.unmodifiableSet( PwmSetting.sortedValues().stream()
+        return PwmSetting.sortedValues().stream()
                 .parallel()
                 .flatMap( function )
-                .collect( Collectors.collectingAndThen( Collectors.toSet(), Collections::unmodifiableSet ) ) );
+                .collect( Collectors.toUnmodifiableSet() );
     }
 
     public static Set<StoredConfigItemKey> changedValues (
@@ -456,4 +432,39 @@ public abstract class StoredConfigurationUtil
 
         return Collections.unmodifiableSet( deltaReferences );
     }
+
+    public static StoredValue getValueOrDefault(
+            final StoredConfiguration storedConfiguration,
+            final StoredConfigItemKey key
+    )
+    {
+        final Optional<StoredValue> storedValue = storedConfiguration.readStoredValue( key );
+
+        if ( storedValue.isPresent() )
+        {
+            return storedValue.get();
+        }
+
+        switch ( key.getRecordType() )
+        {
+            case SETTING:
+            {
+                final PwmSettingTemplateSet templateSet = storedConfiguration.getTemplateSet();
+                return key.toPwmSetting().getDefaultValue( templateSet );
+            }
+
+            case LOCALE_BUNDLE:
+            {
+                return new LocalizedStringArrayValue( Collections.emptyMap() );
+            }
+
+            case PROPERTY:
+                return new StringValue( "" );
+
+            default:
+                JavaHelper.unhandledSwitchStatement( key );
+        }
+
+        throw new IllegalStateException();
+    }
 }

server/src/main/java/password/pwm/config/value/LocalizedStringArrayValue.java

@@ -42,7 +42,7 @@ public class LocalizedStringArrayValue extends AbstractValue implements StoredVa
 {
     private final Map<String, List<String>> values;
 
-    LocalizedStringArrayValue( final Map<String, List<String>> values )
+    public LocalizedStringArrayValue( final Map<String, List<String>> values )
     {
         this.values = values == null ? Collections.emptyMap() : Collections.unmodifiableMap( values );
     }

server/src/main/java/password/pwm/health/ConfigurationChecker.java

@@ -94,7 +94,10 @@ public class ConfigurationChecker implements HealthChecker
                 }
                 catch ( final PwmUnrecoverableException e )
                 {
-                    records.add( new HealthRecord( HealthStatus.WARN, HealthTopic.Configuration, e.getMessage() ) );
+                    records.add( HealthRecord.forMessage(
+                            HealthMessage.NewUser_PwTemplateBad,
+                             PwmSetting.NEWUSER_PASSWORD_POLICY_USER.toMenuLocationDebug( newUserProfile.getIdentifier(), PwmConstants.DEFAULT_LOCALE ),
+                             e.getMessage() ) );
                 }
             }
         }

server/src/main/java/password/pwm/health/DatabaseStatusChecker.java

@@ -50,7 +50,8 @@ public class DatabaseStatusChecker implements HealthChecker
     {
         if ( !config.hasDbConfigured() )
         {
-            return Collections.singletonList( new HealthRecord( HealthStatus.INFO, HealthTopic.Database, "Database not configured" ) );
+            return Collections.singletonList( HealthRecord.forMessage( HealthMessage.Database_Error,
+                            "Database not configured" ) );
         }
 
         PwmApplication runtimeInstance = null;

server/src/main/java/password/pwm/health/HealthMessage.java

@@ -33,7 +33,9 @@ public enum HealthMessage
     LDAP_Ad_History_Asn_Missing( HealthStatus.WARN, HealthTopic.LDAP ),
     LDAP_AD_Unsecure( HealthStatus.WARN, HealthTopic.LDAP ),
     LDAP_AD_StaticIP( HealthStatus.WARN, HealthTopic.LDAP ),
+    LDAP_AdminUserOk( HealthStatus.GOOD, HealthTopic.LDAP ),
     LDAP_ProxyTestSameUser( HealthStatus.WARN, HealthTopic.Configuration ),
+    LDAP_ProxyUserOk( HealthStatus.GOOD, HealthTopic.LDAP ),
     LDAP_ProxyUserPwExpired( HealthStatus.WARN, HealthTopic.LDAP ),
     LDAP_TestUserUnavailable( HealthStatus.CAUTION, HealthTopic.LDAP ),
     LDAP_TestUserUnexpected( HealthStatus.WARN, HealthTopic.LDAP ),
@@ -41,6 +43,7 @@ public enum HealthMessage
     LDAP_TestUserWritePwError( HealthStatus.WARN, HealthTopic.LDAP ),
     LDAP_TestUserReadPwError( HealthStatus.WARN, HealthTopic.LDAP ),
     LDAP_TestUserOK( HealthStatus.GOOD, HealthTopic.LDAP ),
+    Email_OK( HealthStatus.GOOD, HealthTopic.Email ),
     Email_SendFailure( HealthStatus.WARN, HealthTopic.Email ),
     Email_ConnectFailure( HealthStatus.WARN, HealthTopic.Email ),
     PwNotify_Failure( HealthStatus.WARN, HealthTopic.Application ),
@@ -70,6 +73,7 @@ public enum HealthMessage
     Config_DNValueValidity( HealthStatus.CONFIG, HealthTopic.Configuration ),
     Config_ProfileValueValidity( HealthStatus.CONFIG, HealthTopic.Configuration ),
     Config_SettingIssue( HealthStatus.CAUTION, HealthTopic.Configuration ),
+    Config_SettingOk( HealthStatus.GOOD, HealthTopic.Configuration ),
     Config_NoRecoveryEnabled( HealthStatus.CAUTION, HealthTopic.Configuration ),
     Config_Certificate( HealthStatus.WARN, HealthTopic.Configuration ),
     Config_InvalidSendMethod( HealthStatus.CAUTION, HealthTopic.Configuration ),
@@ -78,9 +82,11 @@ public enum HealthMessage
     Config_NoLdapProfiles( HealthStatus.CONFIG, HealthTopic.Configuration ),
     Config_ValueConflict( HealthStatus.CONFIG, HealthTopic.Configuration ),
 
+    Database_OK( HealthStatus.GOOD, HealthTopic.Database ),
+    Database_Error( HealthStatus.WARN, HealthTopic.Database ),
+    Database_RecentlyUnreachable( HealthStatus.CAUTION, HealthTopic.Database ),
     LDAP_VendorsNotSame( HealthStatus.CONFIG, HealthTopic.LDAP ),
     LDAP_OK( HealthStatus.GOOD, HealthTopic.LDAP ),
-    EMail_OK( HealthStatus.GOOD, HealthTopic.Email ),
     LDAP_RecentlyUnreachable( HealthStatus.CAUTION, HealthTopic.LDAP ),
     LDAP_SearchFailure( HealthStatus.WARN, HealthTopic.LDAP ),
     CryptoTokenWithNewUserVerification( HealthStatus.CAUTION, HealthTopic.Configuration ),
@@ -93,13 +99,14 @@ public enum HealthMessage
     LocalDB_NEW( HealthStatus.WARN, HealthTopic.LocalDB ),
     LocalDB_CLOSED( HealthStatus.WARN, HealthTopic.LocalDB ),
     LocalDB_LowDiskSpace( HealthStatus.WARN, HealthTopic.LocalDB ),
-    LocalDBLogger_NOTOPEN( HealthStatus.CAUTION, HealthTopic.LocalDB ),
+    LocalDBLogger_Closed( HealthStatus.CAUTION, HealthTopic.LocalDB ),
     LocalDBLogger_HighRecordCount( HealthStatus.CAUTION, HealthTopic.LocalDB ),
     LocalDBLogger_OldRecordPresent( HealthStatus.CAUTION, HealthTopic.LocalDB ),
     NewUser_PwTemplateBad( HealthStatus.CAUTION, HealthTopic.Configuration ),
     ServiceClosed( HealthStatus.CAUTION, HealthTopic.Application ),
     ServiceClosed_LocalDBUnavail( HealthStatus.CAUTION, HealthTopic.Application ),
     ServiceClosed_AppReadOnly( HealthStatus.CAUTION, HealthTopic.Application ),
+    ServiceError( HealthStatus.WARN, HealthTopic.Application ),
     SMS_SendFailure( HealthStatus.WARN, HealthTopic.SMS ),
     Wordlist_AutoImportFailure( HealthStatus.WARN, HealthTopic.Configuration ),
     Wordlist_ImportInProgress( HealthStatus.CAUTION, HealthTopic.Application ),;

server/src/main/java/password/pwm/health/HealthRecord.java

@@ -21,12 +21,12 @@
 package password.pwm.health;
 
 import lombok.EqualsAndHashCode;
+import org.jetbrains.annotations.NotNull;
 import password.pwm.config.Configuration;
 import password.pwm.ws.server.rest.bean.HealthData;
 
 import java.io.Serializable;
 import java.time.Instant;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
@@ -44,10 +44,6 @@ public class HealthRecord implements Serializable, Comparable<HealthRecord>
     private final HealthMessage message;
     private final List<String> fields;
 
-    // old fields
-    private final String oldTopic;
-    private final String oldDetail;
-
     private static final Comparator<HealthRecord> COMPARATOR = Comparator.comparing(
             HealthRecord::getStatus,
             Comparator.nullsLast( Comparator.naturalOrder() ) )
@@ -59,64 +55,32 @@ public class HealthRecord implements Serializable, Comparable<HealthRecord>
                     Comparator.nullsLast( Comparator.naturalOrder() ) );
 
 
-    @Deprecated
-    public HealthRecord(
-            final HealthStatus status,
-            final String topic,
-            final String detail
-    )
-    {
-        this.status = Objects.requireNonNull( status,  "status cannot be null" );
-
-        this.oldTopic = topic;
-        this.oldDetail = detail;
-
-        this.topic = null;
-        this.message = null;
-        this.fields = null;
-    }
-
-    @Deprecated
-    public HealthRecord(
-            final HealthStatus status,
-            final HealthTopic topic,
-            final String detail
-    )
-    {
-        this.status = Objects.requireNonNull( status,  "status cannot be null" );
-        this.oldTopic = null;
-        this.oldDetail = detail;
-
-        this.topic = topic;
-        this.message = null;
-        this.fields = null;
-    }
-
     private HealthRecord(
             final HealthStatus status,
             final HealthTopic topicEnum,
             final HealthMessage message,
-            final List<String> fields
+            final String... fields
     )
     {
         this.status = Objects.requireNonNull( status,  "status cannot be null" );
         this.topic = Objects.requireNonNull( topicEnum,  "topic cannot be null" );
         this.message = Objects.requireNonNull( message,  "message cannot be null" );
-        this.fields = fields == null ? Collections.emptyList() : Collections.unmodifiableList( new ArrayList<>( fields ) );
-
-        this.oldTopic = null;
-        this.oldDetail = null;
+        this.fields = fields == null ? Collections.emptyList() : List.copyOf( Arrays.asList( fields ) );
     }
 
     public static HealthRecord forMessage( final HealthMessage message )
     {
-        return new HealthRecord( message.getStatus(), message.getTopic(), message, Collections.emptyList() );
+        return new HealthRecord( message.getStatus(), message.getTopic(), message );
     }
 
     public static HealthRecord forMessage( final HealthMessage message, final String... fields )
     {
-        final List<String> fieldList = fields == null ? Collections.emptyList() : Arrays.asList( fields );
-        return new HealthRecord( message.getStatus(), message.getTopic(), message, fieldList );
+        return new HealthRecord( message.getStatus(), message.getTopic(), message, fields );
+    }
+
+    public static HealthRecord forMessage( final HealthMessage message, final HealthTopic healthTopic, final String... fields )
+    {
+        return new HealthRecord( message.getStatus(), healthTopic, message, fields );
     }
 
     public HealthStatus getStatus( )
@@ -126,10 +90,6 @@ public class HealthRecord implements Serializable, Comparable<HealthRecord>
 
     public String getTopic( final Locale locale, final Configuration config )
     {
-        if ( oldTopic != null )
-        {
-            return oldTopic;
-        }
         if ( topic != null )
         {
             return this.topic.getDescription( locale, config );
@@ -139,10 +99,6 @@ public class HealthRecord implements Serializable, Comparable<HealthRecord>
 
     public String getDetail( final Locale locale, final Configuration config )
     {
-        if ( oldDetail != null )
-        {
-            return oldDetail;
-        }
         if ( message != null )
         {
             return this.message.getDescription( locale, config, fields.toArray( new String[0] ) );
@@ -157,7 +113,7 @@ public class HealthRecord implements Serializable, Comparable<HealthRecord>
     }
 
     @Override
-    public int compareTo( final HealthRecord otherHealthRecord )
+    public int compareTo( @NotNull final HealthRecord otherHealthRecord )
     {
         return COMPARATOR.compare( this, otherHealthRecord );
     }

server/src/main/java/password/pwm/health/LDAPHealthChecker.java

@@ -55,7 +55,6 @@ import password.pwm.ldap.UserInfo;
 import password.pwm.ldap.UserInfoFactory;
 import password.pwm.ldap.search.SearchConfiguration;
 import password.pwm.util.PasswordData;
-import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
@@ -85,7 +84,6 @@ import java.util.Set;
 public class LDAPHealthChecker implements HealthChecker
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( LDAPHealthChecker.class );
-    private static final String TOPIC = "LDAP";
 
     @Override
     public List<HealthRecord> doHealthCheck( final PwmApplication pwmApplication )
@@ -387,9 +385,9 @@ public class LDAPHealthChecker implements HealthChecker
             }
             catch ( final PwmUnrecoverableException e )
             {
-                returnRecords.add( new HealthRecord(
-                        HealthStatus.WARN,
-                        makeLdapTopic( ldapProfile, config ),
+                returnRecords.add( HealthRecord.forMessage(
+                        HealthMessage.LDAP_TestUserError,
+                        PwmSetting.LDAP_TEST_USER_DN.toMenuLocationDebug( ldapProfile.getIdentifier(), PwmConstants.DEFAULT_LOCALE ),
                         "unable to read test user data: " + e.getMessage() ) );
                 return returnRecords;
             }
@@ -444,9 +442,9 @@ public class LDAPHealthChecker implements HealthChecker
             catch ( final Exception e )
             {
                 final String errorString = "error connecting to ldap server '" + loopURL + "': " + e.getMessage();
-                returnRecords.add( new HealthRecord(
-                        HealthStatus.WARN,
-                        makeLdapTopic( ldapProfile, config ),
+                returnRecords.add( HealthRecord.forMessage(
+                        HealthMessage.LDAP_No_Connection,
+                        loopURL,
                         errorString ) );
             }
             finally
@@ -486,11 +484,19 @@ public class LDAPHealthChecker implements HealthChecker
                 final PasswordData proxyPW = ldapProfile.readSettingAsPassword( PwmSetting.LDAP_PROXY_USER_PASSWORD );
                 if ( proxyDN == null || proxyDN.length() < 1 )
                 {
-                    return Collections.singletonList( new HealthRecord( HealthStatus.WARN, HealthTopic.LDAP, "Missing Proxy User DN" ) );
+                    final String menuLocationStr = PwmSetting.LDAP_PROXY_USER_DN.toMenuLocationDebug( ldapProfile.getIdentifier(), PwmConstants.DEFAULT_LOCALE );
+                    return Collections.singletonList( HealthRecord.forMessage(
+                            HealthMessage.LDAP_No_Connection,
+                             ldapProfile.getIdentifier(),
+                             "Missing Proxy User DN: " + menuLocationStr ) );
                 }
                 if ( proxyPW == null )
                 {
-                    return Collections.singletonList( new HealthRecord( HealthStatus.WARN, HealthTopic.LDAP, "Missing Proxy User Password" ) );
+                    final String menuLocationStr = PwmSetting.LDAP_PROXY_USER_PASSWORD.toMenuLocationDebug( ldapProfile.getIdentifier(), PwmConstants.DEFAULT_LOCALE );
+                    return Collections.singletonList( HealthRecord.forMessage(
+                            HealthMessage.LDAP_No_Connection,
+                            ldapProfile.getIdentifier(),
+                            "Missing Proxy User Password: " + menuLocationStr ) );
                 }
                 chaiProvider = LdapOperationsHelper.createChaiProvider( pwmApplication, SessionLabel.HEALTH_SESSION_LABEL, ldapProfile, config, proxyDN, proxyPW );
                 final ChaiUser adminEntry = chaiProvider.getEntryFactory().newChaiUser( proxyDN );
@@ -533,10 +539,11 @@ public class LDAPHealthChecker implements HealthChecker
                     }
                     errorString.append( ")" );
                 }
-                returnRecords.add( new HealthRecord(
-                        HealthStatus.WARN,
-                        makeLdapTopic( ldapProfile, config ),
+                returnRecords.add( HealthRecord.forMessage(
+                        HealthMessage.LDAP_No_Connection,
+                        ldapProfile.getIdentifier(),
                         errorString.toString() ) );
+
                 pwmApplication.getLdapConnectionService().setLastLdapFailure( ldapProfile,
                         new ErrorInformation( PwmError.ERROR_DIRECTORY_UNAVAILABLE, errorString.toString() ) );
                 return returnRecords;
@@ -566,14 +573,24 @@ public class LDAPHealthChecker implements HealthChecker
 
                         if ( objectClasses == null || objectClasses.isEmpty() )
                         {
-                            final String errorString = "ldap context setting '" + loopContext + "' is not valid";
-                            returnRecords.add( new HealthRecord( HealthStatus.WARN, makeLdapTopic( ldapProfile, config ), errorString ) );
+                            final String errorString = "ldap context setting '"
+                                    + PwmSetting.LDAP_CONTEXTLESS_ROOT.toMenuLocationDebug( ldapProfile.getIdentifier(), PwmConstants.DEFAULT_LOCALE )
+                                    + "' value '" + loopContext + "' is not valid";
+                            returnRecords.add( HealthRecord.forMessage(
+                                    HealthMessage.LDAP_No_Connection,
+                                    ldapProfile.getIdentifier(),
+                                    errorString ) );
                         }
                     }
                     catch ( final Exception e )
                     {
-                        final String errorString = "ldap root context '" + loopContext + "' is not valid: " + e.getMessage();
-                        returnRecords.add( new HealthRecord( HealthStatus.WARN, makeLdapTopic( ldapProfile, config ), errorString ) );
+                        final String errorString = "ldap context setting '"
+                                + PwmSetting.LDAP_CONTEXTLESS_ROOT.toMenuLocationDebug( ldapProfile.getIdentifier(), PwmConstants.DEFAULT_LOCALE )
+                                + "' value '" + loopContext + "' is not valid: " + e.getMessage();
+                        returnRecords.add( HealthRecord.forMessage(
+                                HealthMessage.LDAP_No_Connection,
+                                ldapProfile.getIdentifier(),
+                                errorString ) );
                     }
                 }
             }
@@ -650,26 +667,6 @@ public class LDAPHealthChecker implements HealthChecker
         return false;
     }
 
-    private static String makeLdapTopic(
-            final LdapProfile ldapProfile,
-            final Configuration configuration
-    )
-    {
-        return makeLdapTopic( ldapProfile.getIdentifier(), configuration );
-    }
-
-    private static String makeLdapTopic(
-            final String profileID,
-            final Configuration configuration
-    )
-    {
-        if ( configuration.getLdapProfiles().isEmpty() || configuration.getLdapProfiles().size() < 2 )
-        {
-            return TOPIC;
-        }
-        return TOPIC + "-" + profileID;
-    }
-
     private List<HealthRecord> checkVendorSameness( final PwmApplication pwmApplication )
     {
         final Map<HealthMonitor.HealthMonitorFlag, Serializable> healthProperties = pwmApplication.getHealthMonitor().getHealthProperties();
@@ -916,7 +913,7 @@ public class LDAPHealthChecker implements HealthChecker
                         HealthRecord.forMessage(
                                 HealthMessage.NewUser_PwTemplateBad,
                                 PwmSetting.NEWUSER_PASSWORD_POLICY_USER.toMenuLocationDebug( newUserProfile.getIdentifier(), locale ),
-                                LocaleHelper.valueNotApplicable( locale )
+                                "Value missing"
                         )
                 );
             }
@@ -944,7 +941,8 @@ public class LDAPHealthChecker implements HealthChecker
                         return Collections.singletonList(
                                 HealthRecord.forMessage(
                                         HealthMessage.NewUser_PwTemplateBad,
-                                        PwmSetting.NEWUSER_PASSWORD_POLICY_USER.toMenuLocationDebug( newUserProfile.getIdentifier(), locale )
+                                        PwmSetting.NEWUSER_PASSWORD_POLICY_USER.toMenuLocationDebug( newUserProfile.getIdentifier(), locale ),
+                                        "userDN value is not valid"
                                 )
                         );
                     }

server/src/main/java/password/pwm/http/servlet/configeditor/ConfigEditorServlet.java

@@ -49,8 +49,6 @@ import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.DatabaseStatusChecker;
 import password.pwm.health.HealthRecord;
-import password.pwm.health.HealthStatus;
-import password.pwm.health.HealthTopic;
 import password.pwm.health.LDAPHealthChecker;
 import password.pwm.http.HttpMethod;
 import password.pwm.http.JspUrl;
@@ -606,12 +604,13 @@ public class ConfigEditorServlet extends ControlledPwmServlet
         final ConfigManagerBean configManagerBean = getBean( pwmRequest );
         LOGGER.debug( pwmRequest, () -> "beginning restSmsHealthCheck" );
 
-        final List<HealthRecord> returnRecords = new ArrayList<>();
         final Configuration config = new Configuration( configManagerBean.getStoredConfiguration() );
+        final StringBuilder output = new StringBuilder();
+        output.append( "beginning SMS send process:\n" );
 
         if ( !SmsQueueManager.smsIsConfigured( config ) )
         {
-            returnRecords.add( new HealthRecord( HealthStatus.INFO, HealthTopic.SMS, "SMS not configured" ) );
+            output.append( "SMS not configured." );
         }
         else
         {
@@ -625,17 +624,16 @@ public class ConfigEditorServlet extends ControlledPwmServlet
                         pwmRequest.getLabel(),
                         testSmsItem
                 );
-                returnRecords.add( new HealthRecord( HealthStatus.INFO, HealthTopic.SMS, "message sent" ) );
-                returnRecords.add( new HealthRecord( HealthStatus.INFO, HealthTopic.SMS, "response body: \n" + StringUtil.escapeHtml( responseBody ) ) );
+                output.append( "message sent:\n" );
+                output.append( "response body: \n" ).append( StringUtil.escapeHtml( responseBody ) );
             }
             catch ( final PwmException e )
             {
-                returnRecords.add( new HealthRecord( HealthStatus.WARN, HealthTopic.SMS, "unable to send message: " + e.getMessage() ) );
+                output.append( "unable to send message: " ).append( StringUtil.escapeHtml( e.getMessage() ) );
             }
         }
 
-        final HealthData healthData = HealthRecord.asHealthDataBean( config, pwmRequest.getLocale(), returnRecords );
-        final RestResultBean restResultBean = RestResultBean.withData( healthData );
+        final RestResultBean restResultBean = RestResultBean.withData( output.toString() );
         pwmRequest.outputJsonResult( restResultBean );
         LOGGER.debug( pwmRequest, () -> "completed restSmsHealthCheck in " + TimeDuration.fromCurrent( startTime ).asCompactString() );
         return ProcessStatus.Halt;
@@ -656,7 +654,8 @@ public class ConfigEditorServlet extends ControlledPwmServlet
         final Map<String, String> params = pwmRequest.readBodyAsJsonStringMap();
         final EmailItemBean testEmailItem = new EmailItemBean( params.get( "to" ), params.get( "from" ), params.get( "subject" ), params.get( "body" ), null );
 
-        final List<HealthRecord> returnRecords = new ArrayList<>();
+        final StringBuilder output = new StringBuilder();
+        output.append( "beginning EMail send process:\n" );
 
         final Configuration testConfiguration = new Configuration( configManagerBean.getStoredConfiguration() );
 
@@ -671,22 +670,20 @@ public class ConfigEditorServlet extends ControlledPwmServlet
                 try
                 {
                     EmailService.sendEmailSynchronous( emailServer.get(), testConfiguration, testEmailItem, macroRequest );
-                    returnRecords.add( new HealthRecord( HealthStatus.INFO, HealthTopic.Email, "message sent" ) );
+                   output.append( "message delivered" );
                 }
                 catch ( final PwmException e )
                 {
-                    returnRecords.add( new HealthRecord( HealthStatus.WARN, HealthTopic.Email, JavaHelper.readHostileExceptionMessage( e ) ) );
+                    output.append( "error: " + StringUtil.escapeHtml( JavaHelper.readHostileExceptionMessage( e ) ) );
                 }
             }
         }
-
-        if ( returnRecords.isEmpty() )
+        else
         {
-            returnRecords.add( new HealthRecord( HealthStatus.WARN, HealthTopic.Email, "smtp service is not configured." ) );
+            output.append( "smtp service is not configured." );
         }
 
-        final HealthData healthData = HealthRecord.asHealthDataBean( testConfiguration, pwmRequest.getLocale(), returnRecords );
-        final RestResultBean restResultBean = RestResultBean.withData( healthData );
+        final RestResultBean restResultBean = RestResultBean.withData( output.toString() );
         pwmRequest.outputJsonResult( restResultBean );
         LOGGER.debug( pwmRequest, () -> "completed restEmailHealthCheck in " + TimeDuration.fromCurrent( startTime ).asCompactString() );
         return ProcessStatus.Halt;

server/src/main/java/password/pwm/http/servlet/configeditor/data/SettingDataMaker.java

@@ -20,11 +20,13 @@
 
 package password.pwm.http.servlet.configeditor.data;
 
+import password.pwm.PwmConstants;
 import password.pwm.bean.SessionLabel;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.PwmSettingCategory;
 import password.pwm.config.PwmSettingTemplateSet;
 import password.pwm.config.stored.StoredConfiguration;
+import password.pwm.config.stored.StoredConfigurationFactory;
 import password.pwm.config.value.ValueTypeConverter;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.i18n.PwmLocaleBundle;
@@ -43,6 +45,18 @@ public class SettingDataMaker
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( SettingDataMaker.class );
 
+    public static void initializeCache()
+    {
+            try
+            {
+                SettingDataMaker.generateSettingData( StoredConfigurationFactory.newConfig(), null, PwmConstants.DEFAULT_LOCALE );
+            }
+            catch ( final Exception e )
+            {
+                LOGGER.debug( () -> "error initializing generateSettingData: " + e.getMessage() );
+            }
+    }
+
     public static SettingData generateSettingData(
             final StoredConfiguration storedConfiguration,
             final SessionLabel sessionLabel,

server/src/main/java/password/pwm/http/servlet/configguide/ConfigGuideForm.java

@@ -53,6 +53,8 @@ public class ConfigGuideForm
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( ConfigGuideForm.class );
 
+    static final String LDAP_PROFILE_NAME = "default";
+
     public static Map<ConfigGuideFormField, String> defaultForm( )
     {
         final Map<ConfigGuideFormField, String> defaultLdapForm = new EnumMap<>( ConfigGuideFormField.class );
@@ -86,8 +88,6 @@ public class ConfigGuideForm
         }
     }
 
-    private static final String LDAP_PROFILE_NAME = "default";
-
     public static StoredConfiguration generateStoredConfig(
             final ConfigGuideBean configGuideBean
     )

server/src/main/java/password/pwm/http/servlet/configguide/ConfigGuideServlet.java

@@ -28,7 +28,6 @@ import password.pwm.PwmApplicationMode;
 import password.pwm.PwmConstants;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
-import password.pwm.config.value.StoredValue;
 import password.pwm.config.profile.LdapProfile;
 import password.pwm.config.stored.ConfigurationProperty;
 import password.pwm.config.stored.StoredConfiguration;
@@ -36,6 +35,7 @@ import password.pwm.config.stored.StoredConfigurationFactory;
 import password.pwm.config.stored.StoredConfigurationModifier;
 import password.pwm.config.stored.StoredConfigurationUtil;
 import password.pwm.config.value.FileValue;
+import password.pwm.config.value.StoredValue;
 import password.pwm.config.value.ValueFactory;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
@@ -46,8 +46,6 @@ import password.pwm.health.DatabaseStatusChecker;
 import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthMonitor;
 import password.pwm.health.HealthRecord;
-import password.pwm.health.HealthStatus;
-import password.pwm.health.HealthTopic;
 import password.pwm.health.LDAPHealthChecker;
 import password.pwm.http.ContextManager;
 import password.pwm.http.HttpMethod;
@@ -249,11 +247,16 @@ public class ConfigGuideServlet extends ControlledPwmServlet
                 try
                 {
                     ConfigGuideUtils.checkLdapServer( configGuideBean );
-                    records.add( password.pwm.health.HealthRecord.forMessage( HealthMessage.LDAP_OK ) );
+                    records.add( HealthRecord.forMessage( HealthMessage.LDAP_OK ) );
                 }
                 catch ( final Exception e )
                 {
-                    records.add( new HealthRecord( HealthStatus.WARN, HealthTopic.LDAP, "Can not connect to remote server: " + e.getMessage() ) );
+                    final String ldapUrl = ldapProfile.readSettingAsStringArray( PwmSetting.LDAP_SERVER_URLS )
+                            .stream().findFirst().orElse( "" );
+                    records.add( HealthRecord.forMessage(
+                            HealthMessage.LDAP_No_Connection,
+                            ldapUrl,
+                            e.getMessage() ) );
                 }
             }
             break;
@@ -274,7 +277,8 @@ public class ConfigGuideServlet extends ControlledPwmServlet
                 records.addAll( ldapHealthChecker.checkBasicLdapConnectivity( tempApplication, tempConfiguration, ldapProfile, true ) );
                 if ( records.isEmpty() )
                 {
-                    records.add( new HealthRecord( HealthStatus.GOOD, HealthTopic.LDAP, "LDAP Contextless Login Root validated" ) );
+                    records.add( HealthRecord.forMessage( HealthMessage.Config_SettingOk,
+                            PwmSetting.LDAP_CONTEXTLESS_ROOT.getLabel( pwmRequest.getLocale() ) ) );
                 }
             }
             break;
@@ -295,7 +299,10 @@ public class ConfigGuideServlet extends ControlledPwmServlet
                 }
                 else
                 {
-                    records.add( new HealthRecord( HealthStatus.CAUTION, HealthTopic.LDAP, "No test user specified" ) );
+                    records.add(
+                            HealthRecord.forMessage(
+                                    HealthMessage.Config_AddTestUser,
+                                    PwmSetting.LDAP_TEST_USER_DN.toMenuLocationDebug( ConfigGuideForm.LDAP_PROFILE_NAME, pwmRequest.getLocale() ) ) );
                 }
             }
             break;

server/src/main/java/password/pwm/http/servlet/configguide/ConfigGuideUtils.java

@@ -42,9 +42,8 @@ import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
-import password.pwm.health.HealthStatus;
-import password.pwm.health.HealthTopic;
 import password.pwm.http.ContextManager;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmRequestAttribute;
@@ -302,22 +301,24 @@ public class ConfigGuideUtils
                 final UserIdentity foundIdentity = results.iterator().next();
                 if ( foundIdentity.canonicalEquals( adminIdentity, tempApplication ) )
                 {
-                    records.add( new HealthRecord( HealthStatus.GOOD, HealthTopic.LDAP, "Admin user validated." ) );
+                    records.add( HealthRecord.forMessage( HealthMessage.LDAP_AdminUserOk ) );
                 }
             }
         }
-        catch ( final PwmException e )
-        {
-            records.add( new HealthRecord( HealthStatus.WARN, HealthTopic.LDAP, "Error during admin user validation: " + e.getErrorInformation().toDebugStr() ) );
-        }
         catch ( final Exception e )
         {
-            records.add( new HealthRecord( HealthStatus.WARN, HealthTopic.LDAP, "Error during admin user validation: " + e.getMessage() ) );
+            records.add( HealthRecord.forMessage(
+                    HealthMessage.Config_SettingIssue,
+                    PwmSetting.LDAP_PROXY_USER_DN.getLabel( pwmRequest.getLocale() ),
+                    e.getMessage() ) );
         }
 
         if ( records.isEmpty() )
         {
-            records.add( new HealthRecord( HealthStatus.WARN, HealthTopic.LDAP, "Admin user not found." ) );
+            records.add( HealthRecord.forMessage(
+                    HealthMessage.Config_SettingIssue,
+                    PwmSetting.LDAP_PROXY_USER_DN.getLabel( pwmRequest.getLocale() ),
+                    "User not found" ) );
         }
 
         return records;

server/src/main/java/password/pwm/svc/event/AuditService.java

@@ -36,9 +36,8 @@ import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
-import password.pwm.health.HealthStatus;
-import password.pwm.health.HealthTopic;
 import password.pwm.http.PwmSession;
 import password.pwm.i18n.Display;
 import password.pwm.ldap.UserInfo;
@@ -231,7 +230,10 @@ public class AuditService implements PwmService
 
         if ( lastError != null )
         {
-            healthRecords.add( new HealthRecord( HealthStatus.WARN, HealthTopic.Audit, lastError.toDebugStr() ) );
+            healthRecords.add( HealthRecord.forMessage(
+                    HealthMessage.ServiceClosed,
+                    this.getClass().getSimpleName(),
+                    lastError.toDebugStr() ) );
         }
 
         return Collections.unmodifiableList( healthRecords );

server/src/main/java/password/pwm/svc/event/SyslogAuditService.java

@@ -45,8 +45,8 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmOperationalException;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
-import password.pwm.health.HealthStatus;
 import password.pwm.health.HealthTopic;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsManager;
@@ -242,7 +242,10 @@ public class SyslogAuditService
             final ErrorInformation errorInformation = lastError;
             if ( TimeDuration.fromCurrent( errorInformation.getDate() ).isShorterThan( WARNING_WINDOW_MS ) )
             {
-                healthRecords.add( new HealthRecord( HealthStatus.WARN, HealthTopic.Audit,
+                healthRecords.add( HealthRecord.forMessage(
+                        HealthMessage.ServiceError,
+                        HealthTopic.Audit,
+                        this.getClass().getSimpleName(),
                         errorInformation.toUserStr( PwmConstants.DEFAULT_LOCALE, configuration ) ) );
             }
         }

server/src/main/java/password/pwm/svc/intruder/IntruderManager.java

@@ -34,9 +34,8 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
-import password.pwm.health.HealthStatus;
-import password.pwm.health.HealthTopic;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.PwmSession;
 import password.pwm.ldap.UserInfo;
@@ -52,9 +51,9 @@ import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.DataStore;
 import password.pwm.util.DataStoreFactory;
 import password.pwm.util.PwmScheduler;
-import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.db.DatabaseDataStore;
 import password.pwm.util.db.DatabaseTable;
+import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.ClosableIterator;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
@@ -285,7 +284,10 @@ public class IntruderManager implements PwmService
     {
         if ( startupError != null && status != STATUS.OPEN )
         {
-            return Collections.singletonList( new HealthRecord( HealthStatus.WARN, HealthTopic.Application, "unable to start: " + startupError.toDebugStr() ) );
+            return Collections.singletonList( HealthRecord.forMessage(
+                    HealthMessage.ServiceClosed,
+                    this.getClass().getSimpleName(),
+                    startupError.toDebugStr() ) );
         }
         return Collections.emptyList();
     }

server/src/main/java/password/pwm/svc/sessiontrack/UserAgentUtils.java

@@ -90,7 +90,7 @@ public class UserAgentUtils
         return null;
     }
 
-    public static void initializeCache() throws PwmUnrecoverableException
+    public static void initializeCache()
     {
         final Instant startTime = Instant.now();
         CACHED_PARSER.get();

server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java

@@ -30,8 +30,6 @@ import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
-import password.pwm.health.HealthStatus;
-import password.pwm.health.HealthTopic;
 import password.pwm.svc.PwmService;
 import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
@@ -309,7 +307,10 @@ abstract class AbstractWordlist implements Wordlist, PwmService
 
         if ( lastError != null )
         {
-            final HealthRecord healthRecord = new HealthRecord( HealthStatus.WARN, HealthTopic.Application, this.getClass().getName() + " error: " + lastError.toDebugStr() );
+            final HealthRecord healthRecord = HealthRecord.forMessage(
+                    HealthMessage.ServiceClosed,
+                    this.getClass().getSimpleName(),
+                    lastError.toDebugStr() );
             returnList.add( healthRecord );
         }
         return Collections.unmodifiableList( returnList );

server/src/main/java/password/pwm/util/PwmScheduler.java

@@ -23,6 +23,7 @@ package password.pwm.util;
 import org.jetbrains.annotations.NotNull;
 import password.pwm.PwmApplication;
 import password.pwm.PwmConstants;
+import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
@@ -45,6 +46,7 @@ import java.util.concurrent.TimeUnit;
 public class PwmScheduler
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwmScheduler.class );
+    private static final AtomicLoopIntIncrementer THREAD_ID_COUNTER = new AtomicLoopIntIncrementer();
     private final ScheduledExecutorService applicationExecutorService;
     private final String instanceID;
 
@@ -60,11 +62,17 @@ public class PwmScheduler
     }
 
     public Future immediateExecuteInNewThread(
-            final Runnable runnable
+            final Runnable runnable,
+            final String threadName
     )
     {
         Objects.requireNonNull( runnable );
 
+        final Instant itemStartTime = Instant.now();
+        final String name = "runtime thread #" + THREAD_ID_COUNTER.next() + " " + threadName;
+
+        LOGGER.trace( () -> "started " + name );
+
         final ScheduledExecutorService executor = makeSingleThreadExecutorService( instanceID, runnable.getClass() );
 
         if ( applicationExecutorService.isShutdown() )
@@ -72,7 +80,13 @@ public class PwmScheduler
             return null;
         }
 
-        final WrappedRunner wrappedRunner = new WrappedRunner( runnable, executor, WrappedRunner.Flag.ShutdownExecutorAfterExecution );
+        final Runnable logOutputWrapper = () ->
+        {
+            runnable.run();
+            LOGGER.trace( () -> "completed " + name, () -> TimeDuration.fromCurrent( itemStartTime ) );
+        };
+
+        final WrappedRunner wrappedRunner =  new WrappedRunner( logOutputWrapper, executor, WrappedRunner.Flag.ShutdownExecutorAfterExecution );
         applicationExecutorService.submit( wrappedRunner );
         return wrappedRunner.getFuture();
     }
@@ -84,8 +98,8 @@ public class PwmScheduler
     )
     {
         final TimeDuration delayTillNextZulu = TimeDuration.fromCurrent( nextZuluZeroTime() );
-        final TimeDuration delayTillNextOFfiset = delayTillNextZulu.add( offset );
-        scheduleFixedRateJob( runnable, executorService, delayTillNextOFfiset, TimeDuration.DAY );
+        final TimeDuration delayTillNextOffset = delayTillNextZulu.add( offset );
+        scheduleFixedRateJob( runnable, executorService, delayTillNextOffset, TimeDuration.DAY );
     }
 
     public Future scheduleJob(

server/src/main/java/password/pwm/util/db/DatabaseService.java

@@ -23,7 +23,6 @@ package password.pwm.util.db;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.PwmApplicationMode;
-import password.pwm.PwmConstants;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.option.DataStorageMethod;
@@ -31,9 +30,8 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
-import password.pwm.health.HealthStatus;
-import password.pwm.health.HealthTopic;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.EpsStatistic;
 import password.pwm.svc.stats.StatisticsManager;
@@ -233,7 +231,10 @@ public class DatabaseService implements PwmService
 
         if ( !initialized )
         {
-            returnRecords.add( new HealthRecord( HealthStatus.WARN, HealthTopic.Database, makeUninitializedError().getDetailedErrorMsg() ) );
+            returnRecords.add( HealthRecord.forMessage(
+                    HealthMessage.ServiceClosed,
+                    this.getClass().getSimpleName(),
+                    makeUninitializedError().getDetailedErrorMsg() ) );
             return returnRecords;
         }
 
@@ -246,26 +247,35 @@ public class DatabaseService implements PwmService
         }
         catch ( final PwmException e )
         {
-            returnRecords.add( new HealthRecord( HealthStatus.WARN, HealthTopic.Database, "Error writing to database: " + e.getErrorInformation().toDebugStr() ) );
+            returnRecords.add( HealthRecord.forMessage(
+                    HealthMessage.ServiceClosed,
+                    this.getClass().getSimpleName(),
+                    "error writing to database: " + e.getMessage() ) );
             return returnRecords;
         }
 
         if ( lastError != null )
         {
             final TimeDuration errorAge = TimeDuration.fromCurrent( lastError.getDate() );
+            final long cautionDurationMS = Long.parseLong( pwmApplication.getConfig().readAppProperty( AppProperty.HEALTH_DB_CAUTION_DURATION_MS ) );
 
-            if ( errorAge.isShorterThan( TimeDuration.HOUR ) )
+            if ( errorAge.isShorterThan( cautionDurationMS ) )
             {
-                final String msg = "Database server was recently unavailable ("
-                        + errorAge.asLongString( PwmConstants.DEFAULT_LOCALE )
-                        + " ago at " + lastError.getDate().toString() + "): " + lastError.toDebugStr();
-                returnRecords.add( new HealthRecord( HealthStatus.CAUTION, HealthTopic.Database, msg ) );
+                final String ageString = errorAge.asLongString();
+                final String errorDate = JavaHelper.toIsoDate( lastError.getDate() );
+                final String errorMsg = lastError.toDebugStr();
+                returnRecords.add( HealthRecord.forMessage(
+                        HealthMessage.Database_RecentlyUnreachable,
+                        ageString,
+                        errorDate,
+                        errorMsg
+                ) );
             }
         }
 
         if ( returnRecords.isEmpty() )
         {
-            returnRecords.add( new HealthRecord( HealthStatus.GOOD, HealthTopic.Database, "Database connection to " + this.dbConfiguration.getConnectionString() + " okay" ) );
+            returnRecords.add( HealthRecord.forMessage( HealthMessage.Database_OK, this.dbConfiguration.getConnectionString() ) );
         }
 
         return returnRecords;

server/src/main/java/password/pwm/util/logging/LocalDBLogger.java

@@ -488,7 +488,7 @@ public class LocalDBLogger implements PwmService
 
         if ( status != STATUS.OPEN )
         {
-            healthRecords.add( HealthRecord.forMessage( HealthMessage.LocalDBLogger_NOTOPEN, status.toString() ) );
+            healthRecords.add( HealthRecord.forMessage( HealthMessage.LocalDBLogger_Closed, status.toString() ) );
             return healthRecords;
         }
 

server/src/main/resources/password/pwm/AppProperty.properties

@@ -103,6 +103,7 @@ health.supportBundle.file.writeIntervalSeconds=0
 health.supportBundle.file.writeRetentionCount=10
 health.certificate.warnSeconds=2592000
 health.disk.minFreeWarning=500000000
+health.database.cautionDurationMS=10800000
 health.ldap.cautionDurationMS=10800000
 health.ldap.proxy.pwExpireWarnSeconds=2592000
 health.ldap.userSearch.warnMS=500

server/src/main/resources/password/pwm/i18n/Health.properties

@@ -21,9 +21,10 @@
 
 HealthMessage_NoData=Health data is not currently available.  Please check again in a moment.
 HealthMessage_LDAP_OK=All configured LDAP servers are reachable
-HealthMessage_EMail_OK=All configured Email servers are reachable
 HealthMessage_LDAP_No_Connection=Unable to connect to LDAP server %1%, error: %2%
+HealthMessage_LDAP_AdminUserOk=The admin user has been validated.
 HealthMessage_LDAP_ProxyTestSameUser=%1% setting is the same value as the %2% setting
+HealthMessage_LDAP_ProxyUserOk=The proxy user for the %1 profile is okay.
 HealthMessage_LDAP_ProxyUserPwExpired=Proxy user %1% password will expire within %2%.  The proxy user password should never expire. 
 HealthMessage_LDAP_SearchFailure=Error while searching for users: %1%
 HealthMessage_LDAP_TestUserUnavailable=LDAP unavailable error while testing ldap test user %1%, error: %2%
@@ -34,6 +35,7 @@ HealthMessage_LDAP_TestUserReadPwError=Setting %1% is enabled, however unable to
 HealthMessage_LDAP_TestUserOK=LDAP test user account is functioning normally for profile %1%
 HealthMessage_LDAP_AD_Unsecure=%1% is not configured as a secure connection.  Active Directory requires a secure connection to allow password changes.
 HealthMessage_LDAP_AD_StaticIP=%1% should be configured using a dns hostname instead of an IP address.  Active Directory can sometimes have errors when using an IP address for configuration.
+HealthMessage_Email_OK=All configured Email servers are reachable
 HealthMessage_Email_SendFailure=Unable to send email due to error: %1%
 HealthMessage_Email_ConnectFailure=Unable to connect to SMTP email profile '%1%', error: %2%
 HealthMessage_PwNotify_Failure=Error while sending password notification emails: %1%
@@ -48,7 +50,7 @@ HealthMessage_Config_NoSiteURL=The site URL is not configured, please configure
 HealthMessage_Config_LDAPWireTrace=The %1% setting is enabled and should be disabled for proper security
 HealthMessage_Config_PromiscuousLDAP=%1% setting should be set to false for proper security
 HealthMessage_Config_ShowDetailedErrors=%1% setting should be set to false for proper security
-HealthMessage_Config_AddTestUser=%1% setting should be set to verify proper operation
+HealthMessage_Config_AddTestUser=A test user is not configured for setting %1%.  Configuring a test user allows @PwmAppName@ to verify the configuration and health of the LDAP directory.
 HealthMessage_Config_ParseError=%1% error parsing setting %2%: %3%
 HealthMessage_Config_UsingLocalDBResponseStorage=The setting %1% is configured to store user data in the LocalDB.  This should never be used in a production environment.
 HealthMessage_Config_WeakPassword=%1% strength of password is weak (%2%/100); increase password length/complexity for proper security
@@ -62,6 +64,7 @@ HealthMessage_Config_MissingProxyPassword=Missing proxy user password for profil
 HealthMessage_Config_PasswordPolicyProblem=Password policy '%1%' configuration anomaly: %2%
 HealthMessage_Config_UserPermissionValidity=User Permission configuration for setting %1% issue: %2%.  This may cause unexpected issues.
 HealthMessage_Config_SettingIssue=Setting %1% has issue: %2%.
+HealthMessage_Config_SettingOk=Setting %1% has been validated.
 HealthMessage_Config_DNValueValidity=LDAP DN configuration setting %1% issue: %2%.  This may cause unexpected issues.
 HealthMessage_Config_ProfileValueValidity=LDAP Profile for setting %1% has invalid value '%2%'.  This may cause unexpected issues.
 HealthMessage_Config_Certificate=Certificate for setting %1% issue: %2%
@@ -69,6 +72,9 @@ HealthMessage_Config_InvalidSendMethod=The send method '%1%' is no longer availa
 HealthMessage_Config_DeprecatedJSForm=The javascript form option in the form setting %1% has been deprecated and will be removed in a future version.  Use the setting %2% instead.
 HealthMessage_Config_InvalidLdapProfile=The configured LDAP profile is invalid or inactive for setting %1%.
 HealthMessage_Config_NoLdapProfiles=There are no enabled LDAP Profiles configured.
+HealthMessage_Database_OK=Database connection to %1% okay
+HealthMessage_Database_Error=Database connection error: %1%
+HealthMessage_Database_RecentlyUnreachable=Database connection was recently unavailable (%2% ago at %3%): %4%
 HealthMessage_LDAP_VendorsNotSame=LDAP directories of different vendor types are in use.  This configuration may cause undesirable side effects and is not supported.  %1%
 HealthMessage_LDAP_Ad_History_Asn_Missing=%1% is enabled, but the server at %2% does not support this feature.  Check to be sure it is upgraded to Windows Server 2008 R2 SP1 or greater.  Password changes against this server may fail until this is resolved.
 HealthMessage_LDAP_RecentlyUnreachable=LDAP profile %1% was recently unavailable (%2% ago at %3%): %4%
@@ -83,13 +89,14 @@ HealthMessage_LocalDB_OK=LocalDB and related services are operating correctly
 HealthMessage_LocalDB_BAD=LocalDB is not online. Startup error: %1%
 HealthMessage_LocalDB_NEW=LocalDB status is NEW (loading) state, until LocalDB loads, statistics, online logging, word lists and other features are disabled
 HealthMessage_LocalDB_CLOSED=LocalDB is CLOSED, statistics, online logging, word lists and other features are disabled.  Check logs to troubleshoot
-HealthMessage_LocalDBLogger_NOTOPEN=LocalDBLogger is not open, status is %1%
+HealthMessage_LocalDBLogger_Closed=LocalDBLogger is not open, status is %1%
 HealthMessage_LocalDBLogger_HighRecordCount=LocalDBLogger event log record count of %1% records, is more than the configured maximum of %2%.  Excess records are being purged.
 HealthMessage_LocalDBLogger_OldRecordPresent=Oldest LocalDBLogger event log record is %1%, configured maximum is %2%.  Excess records are being purged.
-HealthMessage_NewUser_PwTemplateBad=The setting %1% is set to a LDAP DN value that is invalid
-HealthMessage_ServiceClosed=unable to start %1% service   %2%
+HealthMessage_NewUser_PwTemplateBad=The setting %1% is set to a LDAP DN value that is not functioning properly: %2%
+HealthMessage_ServiceClosed=Unable to start %1% service:  %2%
 HealthMessage_ServiceClosed_LocalDBUnavail=unable to start %1% service, LocalDB is not available
 HealthMessage_ServiceClosed_AppReadOnly=unable to start %1% service, application is in read-only mode
+HealthMessage_ServiceError=Error operating service %1% service: %2%
 HealthMessage_SMS_SendFailure=Error while sending SMS messages: %1%
 HealthMessage_Wordlist_AutoImportFailure=Configured word list (%1%) failed to import due to error: %2% at timestamp %3%
 HealthMessage_Wordlist_ImportInProgress=Wordlist import is in progress.  Application performance may be degraded during import.  %1%