convert log4j to logback

build/checkstyle-import.xml

@@ -52,8 +52,9 @@
     <!-- graylog2 -->
     <allow pkg="org.graylog2"/>
 
-    <!-- log4j -->
-    <allow pkg="org.apache.log4j"/>
+    <!-- logback -->
+    <allow pkg="org.slf4j"/>
+    <allow pkg="ch.qos.logback"/>
 
     <!-- testing -->
     <allow pkg="org.junit"/>

lib-util/src/main/java/password/pwm/util/java/StringUtil.java

@@ -389,20 +389,13 @@ public abstract class StringUtil
             return input;
         }
 
-        final StringBuilder sb = new StringBuilder( input );
-        while ( sb.length() < length )
-        {
-            if ( right )
-            {
-                sb.append( appendChar );
-            }
-            else
-            {
-                sb.insert( 0, appendChar );
-            }
-        }
+        final char[] charArray = new char[ length - input.length() ];
+        Arrays.fill( charArray, appendChar );
+        final String paddingString = new String( charArray );
 
-        return sb.toString();
+        return right
+                ? input + paddingString
+                : paddingString + input;
     }
 
     public static List<String> splitAndTrim( final String input, final String separator )

lib-util/src/test/java/password/pwm/util/java/StringUtilTest.java

@@ -310,4 +310,24 @@ public class StringUtilTest
         final byte[] b64array = StringUtil.base64Decode( B64_TEST_GZIP, StringUtil.Base64Options.GZIP );
         Assertions.assertArrayEquals( makeB64inputByteArray(), b64array );
     }
+
+    @Test
+    public void padRightTest()
+    {
+        Assertions.assertEquals( "TEST   ", StringUtil.padRight( "TEST", 7, ' ' ) );
+        Assertions.assertEquals( "TEST888", StringUtil.padRight( "TEST", 7, '8' ) );
+        Assertions.assertEquals( "TEST", StringUtil.padRight( "TEST", 4, ' ' ) );
+        Assertions.assertEquals( "TEST", StringUtil.padRight( "TEST", 3, ' ' ) );
+        Assertions.assertEquals( "TEST", StringUtil.padRight( "TEST", -3, ' ' ) );
+    }
+
+    @Test
+    public void padLeftTest()
+    {
+        Assertions.assertEquals( "   TEST", StringUtil.padLeft( "TEST", 7, ' ' ) );
+        Assertions.assertEquals( "888TEST", StringUtil.padLeft( "TEST", 7, '8' ) );
+        Assertions.assertEquals( "TEST", StringUtil.padLeft( "TEST", 4, ' ' ) );
+        Assertions.assertEquals( "TEST", StringUtil.padLeft( "TEST", 3, ' ' ) );
+        Assertions.assertEquals( "TEST", StringUtil.padLeft( "TEST", -3, ' ' ) );
+    }
 }

server/pom.xml

@@ -220,9 +220,19 @@
             <version>0.9.60</version>
         </dependency>
         <dependency>
-            <groupId>ch.qos.reload4j</groupId>
-            <artifactId>reload4j</artifactId>
-            <version>1.2.22</version>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>1.4.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>2.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>log4j-over-slf4j</artifactId>
+            <version>2.0.0</version>
         </dependency>
         <dependency>
             <groupId>org.jasig.cas.client</groupId>

server/src/main/java/password/pwm/AppProperty.java

@@ -244,8 +244,11 @@ public enum AppProperty
     LDAP_SEARCH_PARALLEL_FACTOR                     ( "ldap.search.parallel.factor" ),
     LDAP_SEARCH_PARALLEL_THREAD_MAX                 ( "ldap.search.parallel.threadMax" ),
     LDAP_ORACLE_POST_TEMPPW_USE_CURRENT_TIME        ( "ldap.oracle.postTempPasswordUseCurrentTime" ),
-    LOGGING_OUTPUT_CONFIGURATION                    ( "logging.outputConfiguration" ),
-    LOGGING_PATTERN                                 ( "logging.pattern" ),
+    LOGGING_OUTPUT_CONFIGURATION                    ( "logging.output.configuration.enable" ),
+    LOGGING_OUTPUT_HEALTHCHECK                      ( "logging.output.healthCheck.enable" ),
+    LOGGING_OUTPUT_RUNTIME                          ( "logging.output.runtime.enable" ),
+    LOGGING_OUTPUT_MODE                             ( "logging.logOutputMode" ),
+    LOGGING_PACKAGE_LIST                            ( "logging.packageList" ),
     LOGGING_EXTRA_PERIODIC_THREAD_DUMP_INTERVAL     ( "logging.extra.periodicThreadDumpIntervalSeconds" ),
     LOGGING_FILE_MAX_SIZE                           ( "logging.file.maxSize" ),
     LOGGING_FILE_MAX_ROLLOVER                       ( "logging.file.maxRollover" ),

server/src/main/java/password/pwm/PwmApplication.java

@@ -437,6 +437,11 @@ public class PwmApplication
 
         LOGGER.info( sessionLabel, () -> PwmConstants.PWM_APP_NAME + " " + PwmConstants.SERVLET_VERSION
                 + " closed for bidness, cya!", TimeDuration.fromCurrent( startTime ) );
+
+        if ( !pwmEnvironment.isInternalRuntimeInstance() )
+        {
+            PwmLogManager.disableLogging();
+        }
     }
 
     public String getInstanceID( )

server/src/main/java/password/pwm/PwmApplicationUtil.java

@@ -21,7 +21,6 @@
 package password.pwm;
 
 import password.pwm.bean.DomainID;
-import password.pwm.config.PwmSetting;
 import password.pwm.config.stored.StoredConfigKey;
 import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationUtil;
@@ -39,6 +38,7 @@ import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBFactory;
 import password.pwm.util.logging.PwmLogLevel;
 import password.pwm.util.logging.PwmLogManager;
+import password.pwm.util.logging.PwmLogSettings;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.secure.HttpsServerCertificateManager;
 import password.pwm.util.secure.PwmRandom;
@@ -101,49 +101,30 @@ class PwmApplicationUtil
     {
         final PwmEnvironment pwmEnvironment = pwmApplication.getPwmEnvironment();
 
-        if ( !pwmEnvironment.isInternalRuntimeInstance() && !pwmEnvironment.getFlags().contains( PwmEnvironment.ApplicationFlag.CommandLineInstance ) )
+        if ( pwmEnvironment.isInternalRuntimeInstance() || pwmEnvironment.getFlags().contains( PwmEnvironment.ApplicationFlag.CommandLineInstance ) )
         {
-            final String log4jFileName = pwmEnvironment.getConfig().readSettingAsString( PwmSetting.EVENTS_JAVA_LOG4JCONFIG_FILE );
-            final File log4jFile = FileSystemUtility.figureFilepath( log4jFileName, pwmEnvironment.getApplicationPath() );
-            final String consoleLevel;
-            final String fileLevel;
-
-            switch ( pwmApplication.getApplicationMode() )
-            {
-                case ERROR:
-                case NEW:
-                    consoleLevel = PwmLogLevel.TRACE.toString();
-                    fileLevel = PwmLogLevel.TRACE.toString();
-                    break;
-
-                default:
-                    consoleLevel = pwmEnvironment.getConfig().readSettingAsString( PwmSetting.EVENTS_JAVA_STDOUT_LEVEL );
-                    fileLevel = pwmEnvironment.getConfig().readSettingAsString( PwmSetting.EVENTS_FILE_LEVEL );
-                    break;
-            }
-
-            PwmLogManager.initializeLogger(
-                    pwmApplication,
-                    pwmApplication.getConfig(),
-                    log4jFile,
-                    consoleLevel,
-                    pwmEnvironment.getApplicationPath(),
-                    fileLevel );
+            return;
+        }
 
-            switch ( pwmApplication.getApplicationMode() )
-            {
-                case RUNNING:
-                    break;
+        final PwmLogSettings pwmLogSettings;
+        switch ( pwmApplication.getApplicationMode() )
+        {
+            case ERROR:
+            case NEW:
+                pwmLogSettings = PwmLogSettings.defaultSettings();
+                break;
+
+            default:
+                pwmLogSettings = PwmLogSettings.fromAppConfig( pwmApplication.getConfig() );
+                break;
+        }
 
-                case ERROR:
-                    LOGGER.fatal( pwmApplication.getSessionLabel(), () -> "starting up in ERROR mode! Check log or health check information for cause" );
-                    break;
+        PwmLogManager.initializeLogging(
+                pwmApplication,
+                pwmApplication.getConfig(),
+                pwmEnvironment.getApplicationPath(),
+                pwmLogSettings );
 
-                default:
-                    LOGGER.trace( pwmApplication.getSessionLabel(), () -> "setting log level to TRACE because application mode is " + pwmApplication.getApplicationMode() );
-                    break;
-            }
-        }
     }
 
     static String fetchInstanceID(
@@ -311,7 +292,7 @@ class PwmApplicationUtil
         LOGGER.trace( pwmApplication.getSessionLabel(), () -> "--begin current configuration output for domainID '" + domainID + "'--" );
         debugStrings.entrySet().stream()
                 .map( valueFormatter )
-                .map( s -> ( Supplier<CharSequence> ) () -> s )
+                .map( s -> ( Supplier<String> ) () -> s )
                 .forEach( s -> LOGGER.trace( pwmApplication.getSessionLabel(), s ) );
 
         final long itemCount = debugStrings.size();
@@ -351,7 +332,7 @@ class PwmApplicationUtil
         {
             final String separator = " -> ";
             input.entrySet().stream()
-                    .map( entry -> ( Supplier<CharSequence> ) () -> entry.getKey() + separator + entry.getValue() )
+                    .map( entry -> ( Supplier<String> ) () -> entry.getKey() + separator + entry.getValue() )
                     .forEach( s -> LOGGER.trace( pwmApplication.getSessionLabel(), s ) );
         }
         else
@@ -364,7 +345,7 @@ class PwmApplicationUtil
 
     private static boolean checkIfOutputDumpingEnabled( final PwmApplication pwmApplication )
     {
-        return LOGGER.isEnabled( PwmLogLevel.TRACE )
+        return LOGGER.isInterestingLevel( PwmLogLevel.TRACE )
                 && !pwmApplication.getPwmEnvironment().isInternalRuntimeInstance()
                 && Boolean.parseBoolean( pwmApplication.getConfig().readAppProperty( AppProperty.LOGGING_OUTPUT_CONFIGURATION ) );
     }

server/src/main/java/password/pwm/bean/SessionLabel.java

@@ -34,9 +34,13 @@ public class SessionLabel implements Serializable
 {
     private static final String SYSTEM_LABEL_SESSION_ID = "#";
     private static final String RUNTIME_LABEL_SESSION_ID = "#";
+    private static final String HEALTH_LABEL_SESSION_ID = "H";
+    private static final String RUNTIME_USERNAME = "internal";
+    private static final String HEALTH_USERNAME = "health";
 
     public static final SessionLabel SYSTEM_LABEL = SessionLabel.builder().sessionID( SYSTEM_LABEL_SESSION_ID ).username( PwmConstants.PWM_APP_NAME ).build();
-    public static final SessionLabel RUNTIME_LABEL = SessionLabel.builder().sessionID( RUNTIME_LABEL_SESSION_ID ).username( "internal" ).build();
+    public static final SessionLabel RUNTIME_LABEL = SessionLabel.builder().sessionID( RUNTIME_LABEL_SESSION_ID ).username( RUNTIME_USERNAME ).build();
+    public static final SessionLabel HEALTH_LABEL = SessionLabel.builder().sessionID( HEALTH_LABEL_SESSION_ID ).username( HEALTH_USERNAME ).build();
     public static final SessionLabel TEST_SESSION_LABEL = SessionLabel.builder().sessionID( SYSTEM_LABEL_SESSION_ID ).username( "test" ).build();
     public static final SessionLabel CLI_SESSION_LABEL = SessionLabel.builder().sessionID( SYSTEM_LABEL_SESSION_ID ).username( "cli" ).build();
     public static final SessionLabel CONTEXT_SESSION_LABEL = SessionLabel.builder().sessionID( SYSTEM_LABEL_SESSION_ID ).username( "context" ).build();
@@ -97,4 +101,13 @@ public class SessionLabel implements Serializable
         return sb.toString();
     }
 
+    public boolean isRuntime()
+    {
+        return RUNTIME_LABEL.equals( this );
+    }
+
+    public boolean isHealth()
+    {
+        return HEALTH_LABEL.equals( this );
+    }
 }

server/src/main/java/password/pwm/bean/UserIdentity.java

@@ -21,10 +21,10 @@
 package password.pwm.bean;
 
 import com.novell.ldapchai.ChaiUser;
-import com.novell.ldapchai.exception.ChaiException;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import org.jetbrains.annotations.NotNull;
 import password.pwm.PwmApplication;
+import password.pwm.PwmDomain;
 import password.pwm.config.AppConfig;
 import password.pwm.config.profile.LdapProfile;
 import password.pwm.error.ErrorInformation;
@@ -234,17 +234,9 @@ public class UserIdentity implements Serializable, Comparable<UserIdentity>
         }
 
         final ChaiUser chaiUser = pwmApplication.domains().get( this.getDomainID() ).getProxiedChaiUser( sessionLabel, this );
-        final String userDN;
-        try
-        {
-            userDN = chaiUser.readCanonicalDN();
-        }
-        catch ( final ChaiException e )
-        {
-            throw PwmUnrecoverableException.fromChaiException( e );
-        }
-        final UserIdentity canonicalziedIdentity = create( userDN, this.getLdapProfileID(), this.getDomainID() );
-        canonicalziedIdentity.canonical = true;
-        return canonicalziedIdentity;
+        final LdapProfile ldapProfile = getLdapProfile( pwmApplication.getConfig() );
+        final PwmDomain domain = pwmApplication.domains().get( domainID );
+        final String userDN = ldapProfile.readCanonicalDN( sessionLabel, domain, chaiUser.getEntryDN() );
+        return create( userDN, this.getLdapProfileID(), this.getDomainID(), Flag.PreCanonicalized );
     }
 }

server/src/main/java/password/pwm/config/AppConfig.java

@@ -46,7 +46,6 @@ import password.pwm.util.java.CollectorUtil;
 import password.pwm.util.java.LazySupplier;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
-import password.pwm.util.logging.PwmLogLevel;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.secure.PwmRandom;
 import password.pwm.util.secure.PwmSecurityKey;
@@ -223,11 +222,6 @@ public class AppConfig implements SettingReader
         return settingReader.readSettingAsStringArray( pwmSetting );
     }
 
-    public PwmLogLevel getEventLogLocalDBLevel()
-    {
-        return readSettingAsEnum( PwmSetting.EVENTS_LOCALDB_LOG_LEVEL, PwmLogLevel.class );
-    }
-
     public boolean isDevDebugMode()
     {
         return Boolean.parseBoolean( readAppProperty( AppProperty.LOGGING_DEV_OUTPUT ) );

server/src/main/java/password/pwm/config/PwmSetting.java

@@ -703,8 +703,6 @@ public enum PwmSetting
             "events.pwmDB.maxAge", PwmSettingSyntax.DURATION, PwmSettingCategory.LOGGING ),
     EVENTS_ALERT_DAILY_SUMMARY(
             "events.alert.dailySummary.enable", PwmSettingSyntax.BOOLEAN, PwmSettingCategory.LOGGING ),
-    EVENTS_JAVA_LOG4JCONFIG_FILE(
-            "events.java.log4jconfigFile", PwmSettingSyntax.STRING, PwmSettingCategory.LOGGING ),
 
     PASSWORD_STRENGTH_METER_TYPE(
             "password.strengthMeter.type", PwmSettingSyntax.SELECT, PwmSettingCategory.LOGGING ),
@@ -1277,6 +1275,11 @@ public enum PwmSetting
 
     // deprecated.
 
+
+    // deprecated 2022-09-04
+    EVENTS_JAVA_LOG4JCONFIG_FILE(
+            "events.java.log4jconfigFile", PwmSettingSyntax.STRING, PwmSettingCategory.LOGGING ),
+
     // deprecated 2022-07-25
     SEEDLIST_FILENAME(
             "pwm.seedlist.location", PwmSettingSyntax.STRING, PwmSettingCategory.WORDLISTS ),

server/src/main/java/password/pwm/config/profile/LdapProfile.java

@@ -59,6 +59,10 @@ public class LdapProfile extends AbstractProfile implements Profile
     private List<String> rootContextSupplier;
     private Map<String, String> selectableContexts;
 
+    private Optional<UserIdentity> cachedTestUser;
+    private UserIdentity cachedProxyUser;
+
+
     protected LdapProfile( final DomainID domainID, final ProfileID identifier, final StoredConfiguration storedValueMap )
     {
         super( domainID, identifier, storedValueMap );
@@ -211,17 +215,27 @@ public class LdapProfile extends AbstractProfile implements Profile
     public Optional<UserIdentity> getTestUser( final SessionLabel sessionLabel, final PwmDomain pwmDomain )
             throws PwmUnrecoverableException
     {
-        return readUserIdentity( sessionLabel, pwmDomain, PwmSetting.LDAP_TEST_USER_DN );
+        if ( cachedTestUser == null )
+        {
+            cachedTestUser = readUserIdentity( sessionLabel, pwmDomain, PwmSetting.LDAP_TEST_USER_DN );
+        }
+
+        return cachedTestUser;
     }
 
     public UserIdentity getProxyUser( final SessionLabel sessionLabel, final PwmDomain pwmDomain )
             throws PwmUnrecoverableException
     {
-        return readUserIdentity( sessionLabel, pwmDomain, PwmSetting.LDAP_PROXY_USER_DN )
-                .orElseThrow( () ->
-                        new PwmUnrecoverableException( new ErrorInformation(
-                                PwmError.CONFIG_FORMAT_ERROR,
-                                "ldap proxy user is not defined" ) ) );
+        if ( cachedProxyUser == null )
+        {
+            cachedProxyUser = readUserIdentity( sessionLabel, pwmDomain, PwmSetting.LDAP_PROXY_USER_DN )
+                    .orElseThrow( () ->
+                            new PwmUnrecoverableException( new ErrorInformation(
+                                    PwmError.CONFIG_FORMAT_ERROR,
+                                    "ldap proxy user is not defined" ) ) );
+        }
+
+        return cachedProxyUser;
     }
 
     private Optional<UserIdentity> readUserIdentity(

server/src/main/java/password/pwm/error/PwmError.java

@@ -25,10 +25,13 @@ import password.pwm.config.SettingReader;
 import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.JavaHelper;
 
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.Locale;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * @author Jason D. Rivard
@@ -195,7 +198,7 @@ public enum PwmError
     ERROR_BAD_CURRENT_PASSWORD(
             5038, "Error_BadCurrentPassword", Collections.emptySet() ),
     ERROR_CLOSING(
-            5039, "Error_Closing", Collections.emptySet() ),
+            5039, "Error_Closing", Collections.emptySet(), ErrorFlag.AuditIgnored ),
     ERROR_MISSING_GUID(
             5040, "Error_Missing_GUID", Collections.emptySet() ),
     ERROR_TOKEN_EXPIRED(
@@ -358,6 +361,18 @@ public enum PwmError
     {
         Permanent,
         Trivial,
+        AuditIgnored,
+    }
+
+    private static final Set<PwmError> AUDIT_IGNORED_ERRORS;
+
+    static
+    {
+        final Set<PwmError> set = EnumSet.noneOf( PwmError.class );
+        set.addAll( Arrays.stream( values() )
+                .filter( PwmError::isAuditIgnored )
+                .collect( Collectors.toSet() ) );
+        AUDIT_IGNORED_ERRORS = Collections.unmodifiableSet( set );
     }
 
     private final int errorCode;
@@ -365,6 +380,7 @@ public enum PwmError
     private final Set<ChaiError> chaiErrorCode;
     private final boolean errorIsPermanent;
     private final boolean trivial;
+    private final boolean auditIgnored;
 
     PwmError(
             final int errorCode,
@@ -377,6 +393,7 @@ public enum PwmError
         this.errorCode = errorCode;
         this.errorIsPermanent = JavaHelper.enumArrayContainsValue( errorFlags, ErrorFlag.Permanent );
         this.trivial = JavaHelper.enumArrayContainsValue( errorFlags, ErrorFlag.Trivial );
+        this.auditIgnored = JavaHelper.enumArrayContainsValue( errorFlags, ErrorFlag.AuditIgnored );
         this.chaiErrorCode = chaiErrorCode == null ? Collections.emptySet() : Set.copyOf( chaiErrorCode );
     }
 
@@ -406,6 +423,16 @@ public enum PwmError
         return errorIsPermanent;
     }
 
+    public boolean isAuditIgnored()
+    {
+        return auditIgnored;
+    }
+
+    public static Set<PwmError> auditIgnoredErrors()
+    {
+        return AUDIT_IGNORED_ERRORS;
+    }
+
     public String getResourceKey( )
     {
         return resourceKey;

server/src/main/java/password/pwm/health/HealthService.java

@@ -228,7 +228,7 @@ public class HealthService extends AbstractPwmService implements PwmService
         LOGGER.trace( getSessionLabel(), () -> "beginning health check execution #" + counter  );
         final List<HealthRecord> tempResults = new ArrayList<>();
 
-        for ( final Supplier<List<HealthRecord>> loopSupplier : gatherSuppliers( getPwmApplication(), getSessionLabel() ) )
+        for ( final Supplier<List<HealthRecord>> loopSupplier : gatherSuppliers( getPwmApplication(), SessionLabel.HEALTH_LABEL ) )
         {
             try
             {
@@ -321,9 +321,7 @@ public class HealthService extends AbstractPwmService implements PwmService
         {
             try
             {
-                final Instant startTime = Instant.now();
                 doHealthChecks();
-                LOGGER.trace( getSessionLabel(), () -> "completed health check dredge", TimeDuration.fromCurrent( startTime ) );
             }
             catch ( final Throwable e )
             {
@@ -431,7 +429,7 @@ public class HealthService extends AbstractPwmService implements PwmService
         @Override
         public void run()
         {
-            if ( !LOGGER.isEnabled( PwmLogLevel.TRACE ) )
+            if ( !LOGGER.isInterestingLevel( PwmLogLevel.TRACE ) )
             {
                 return;
             }

server/src/main/java/password/pwm/http/PwmRequest.java

@@ -323,7 +323,7 @@ public class PwmRequest extends PwmHttpRequestWrapper
     public void debugHttpRequestToLog( final String extraText, final TimeDuration timeDuration )
             throws PwmUnrecoverableException
     {
-        if ( LOGGER.isEnabled( PwmLogLevel.TRACE ) )
+        if ( LOGGER.isInterestingLevel( PwmLogLevel.TRACE ) )
         {
             final String moreExtraText = ( StringUtil.isEmpty( extraText ) ? "" : extraText + " " )
                     + "request=" + this.getPwmRequestID() + ", domain=" + this.getDomainID().stringValue();

server/src/main/java/password/pwm/http/servlet/AbstractPwmServlet.java

@@ -244,7 +244,7 @@ public abstract class AbstractPwmServlet extends HttpServlet implements PwmServl
 
             case ERROR_INTERNAL:
             default:
-                final Supplier<CharSequence> msg = () -> "unexpected error: " + e.getErrorInformation().toDebugStr();
+                final Supplier<String> msg = () -> "unexpected error: " + e.getErrorInformation().toDebugStr();
                 final PwmLogLevel level = e.getError().isTrivial() ? PwmLogLevel.TRACE : PwmLogLevel.ERROR;
                 LOGGER.log( level, pwmRequest.getLabel(), msg );
                 StatisticsClient.incrementStat( pwmRequest, Statistic.PWM_UNKNOWN_ERRORS );

server/src/main/java/password/pwm/http/servlet/admin/AdminServlet.java

@@ -832,6 +832,7 @@ public class AdminServlet extends ControlledPwmServlet
     {
         plain,
         csv,
+        json,
     }
 
     public enum LogDownloadCompression
@@ -918,6 +919,20 @@ public class AdminServlet extends ControlledPwmServlet
             }
             break;
 
+            case json:
+            {
+                final JsonProvider jsonProvider = JsonFactory.get();
+                pwmRequest.getPwmResponse().markAsDownload(
+                        compressedContentType != null ? compressedContentType : HttpContentType.json,
+                        "debug.json" + compressFileNameSuffix );
+                while ( searchResults.hasNext() )
+                {
+                    writer.write( jsonProvider.serialize( searchResults.next(), PwmLogEvent.class ) );
+                    writer.write( "\n" );
+                }
+            }
+            break;
+
             default:
                 MiscUtil.unhandledSwitchStatement( logDownloadType );
 

server/src/main/java/password/pwm/http/servlet/admin/AppDashboardData.java

@@ -305,7 +305,7 @@ public class AppDashboardData implements Serializable
                 returnData.add( new ServiceData(
                         guid,
                         domainID,
-                        pwmService.getClass().getSimpleName(),
+                        pwmService.name(),
                         pwmService.status(),
                         storageMethods,
                         pwmService.healthCheck(),

server/src/main/java/password/pwm/http/state/CryptoCookieLoginImpl.java

@@ -78,7 +78,7 @@ class CryptoCookieLoginImpl implements SessionLoginProvider
                     COOKIE_PATH
             );
 
-            if ( LOGGER.isEnabled( PwmLogLevel.TRACE ) )
+            if ( LOGGER.isInterestingLevel( PwmLogLevel.TRACE ) )
             {
                 final String debugTxt = loginInfoBean.toDebugString();
                 LOGGER.trace( pwmRequest, () -> "wrote LoginInfoBean=" + debugTxt );
@@ -213,7 +213,7 @@ class CryptoCookieLoginImpl implements SessionLoginProvider
             }
         }
 
-        if ( pwmRequest.getAppConfig().isDevDebugMode() && LOGGER.isEnabled( PwmLogLevel.TRACE ) )
+        if ( pwmRequest.getAppConfig().isDevDebugMode() && LOGGER.isInterestingLevel( PwmLogLevel.TRACE ) )
         {
             final String debugTxt = remoteLoginCookie.toDebugString();
             LOGGER.trace( pwmRequest, () -> "imported LoginInfoBean=" + debugTxt );

server/src/main/java/password/pwm/http/tag/DisplayTag.java

@@ -142,14 +142,12 @@ public class DisplayTag extends PwmAbstractTag
         }
         catch ( final PwmUnrecoverableException e )
         {
-            {
-                LOGGER.debug( pwmRequest, () -> "error while executing jsp display tag: " + e.getMessage() );
-                return EVAL_PAGE;
-            }
+            LOGGER.error( pwmRequest, () -> "error while executing jsp display tag: " + e.getMessage() );
+            return EVAL_PAGE;
         }
         catch ( final Exception e )
         {
-            LOGGER.debug( pwmRequest, () -> "error while executing jsp display tag: " + e.getMessage(), e );
+            LOGGER.error( pwmRequest, () -> "error while executing jsp display tag: " + e.getMessage(), e );
             throw new JspTagException( e.getMessage(), e );
         }
         return EVAL_PAGE;

server/src/main/java/password/pwm/ldap/schema/EdirSchemaExtender.java

@@ -348,7 +348,7 @@ public class EdirSchemaExtender implements SchemaExtender
         return returnObj;
     }
 
-    private void logActivity( final CharSequence charSequence )
+    private void logActivity( final String charSequence )
     {
         LOGGER.info( () -> charSequence );
         activityLog.append( charSequence ).append( '\n' );

server/src/main/java/password/pwm/svc/PwmServiceManager.java

@@ -156,8 +156,8 @@ public class PwmServiceManager
             newServiceInstance.init( pwmApplication, domainID );
             final TimeDuration startupDuration = TimeDuration.fromCurrent( startTime );
             LOGGER.debug( sessionLabel, () -> "completed initialization of " + debugSvcType()
-                    + " " + newServiceInstance.name() + " in " + startupDuration.asCompactString()
-                    + ", status=" + newServiceInstance.status() );
+                    + " " + newServiceInstance.name()
+                    + ", status=" + newServiceInstance.status(), startupDuration );
         }
         catch ( final PwmException e )
         {

server/src/main/java/password/pwm/svc/cache/CacheService.java

@@ -29,9 +29,9 @@ import password.pwm.health.HealthRecord;
 import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
 import password.pwm.util.java.ConditionalTaskExecutor;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.StatisticCounterBundle;
 import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 import password.pwm.util.logging.PwmLogger;
 
 import java.io.Serializable;
@@ -177,15 +177,18 @@ public class CacheService extends AbstractPwmService implements PwmService
 
     private void outputTraceInfo( )
     {
-        final StringBuilder traceOutput = new StringBuilder();
-        if ( memoryCacheStore != null )
+        LOGGER.trace( getSessionLabel(), () ->
         {
-            final StatisticCounterBundle<CacheStore.DebugKey> info = memoryCacheStore.getCacheStoreInfo();
-            traceOutput.append( "memCache=" );
-            traceOutput.append( JsonFactory.get().serializeMap( info.debugStats() ) );
-            traceOutput.append( ", histogram=" );
-            traceOutput.append( JsonFactory.get().serializeMap( memoryCacheStore.storedClassHistogram( "" ) ) );
-        }
-        LOGGER.trace( getSessionLabel(), () -> traceOutput );
+            final StringBuilder traceOutput = new StringBuilder();
+            if ( memoryCacheStore != null )
+            {
+                final StatisticCounterBundle<CacheStore.DebugKey> info = memoryCacheStore.getCacheStoreInfo();
+                traceOutput.append( "memCache=" );
+                traceOutput.append( JsonFactory.get().serializeMap( info.debugStats() ) );
+                traceOutput.append( ", histogram=" );
+                traceOutput.append( JsonFactory.get().serializeMap( memoryCacheStore.storedClassHistogram( "" ) ) );
+            }
+            return traceOutput.toString();
+        } );
     }
 }

server/src/main/java/password/pwm/svc/db/DatabaseService.java

@@ -121,7 +121,7 @@ public class DatabaseService extends AbstractPwmService implements PwmService
         if ( !dbConfiguration.isEnabled() )
         {
             setStatus( STATUS.CLOSED );
-            LOGGER.debug( () -> "skipping database connection open, no connection parameters configured" );
+            LOGGER.debug( getSessionLabel(), () -> "skipping database connection open, no connection parameters configured" );
             initialized = true;
             return STATUS.CLOSED;
         }
@@ -130,7 +130,7 @@ public class DatabaseService extends AbstractPwmService implements PwmService
 
         try
         {
-            LOGGER.debug( () -> "opening connection to database " + this.dbConfiguration.getConnectionString() );
+            LOGGER.debug( getSessionLabel(), () -> "opening connection to database " + this.dbConfiguration.getConnectionString() );
             slotIncrementer = AtomicLoopIntIncrementer.builder().ceiling( dbConfiguration.getMaxConnections() ).build();
 
             {
@@ -139,7 +139,7 @@ public class DatabaseService extends AbstractPwmService implements PwmService
 
                 final Connection connection = openConnection( dbConfiguration );
                 updateDebugProperties( connection );
-                LOGGER.debug( () -> "established initial connection to " + dbConfiguration.getConnectionString() + ", properties: "
+                LOGGER.debug( getSessionLabel(), () -> "established initial connection to " + dbConfiguration.getConnectionString() + ", properties: "
                         + JsonFactory.get().serializeMap( this.debugInfo ) );
 
                 for ( final DatabaseTable table : DatabaseTable.values() )
@@ -162,13 +162,13 @@ public class DatabaseService extends AbstractPwmService implements PwmService
                 }
             }
 
-            LOGGER.debug( () -> "successfully connected to remote database (" + TimeDuration.compactFromCurrent( startTime ) + ")" );
+            LOGGER.debug( getSessionLabel(), () -> "successfully connected to remote database (" + TimeDuration.compactFromCurrent( startTime ) + ")" );
 
         }
         catch ( final Throwable t )
         {
             final String errorMsg = "exception initializing database service: " + t.getMessage();
-            LOGGER.warn( () -> errorMsg );
+            LOGGER.warn( getSessionLabel(), () -> errorMsg );
             initialized = false;
             lastError = new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, errorMsg );
             return STATUS.CLOSED;
@@ -191,7 +191,7 @@ public class DatabaseService extends AbstractPwmService implements PwmService
         }
         catch ( final Exception e )
         {
-            LOGGER.debug( () -> "error while de-registering driver: " + e.getMessage() );
+            LOGGER.debug( getSessionLabel(), () -> "error while de-registering driver: " + e.getMessage() );
         }
 
         if ( jdbcDriverLoader != null )
@@ -349,7 +349,7 @@ public class DatabaseService extends AbstractPwmService implements PwmService
 
         try
         {
-            LOGGER.debug( () -> "initiating connecting to database " + connectionURL );
+            LOGGER.debug( getSessionLabel(), () -> "initiating connecting to database " + connectionURL );
             final Properties connectionProperties = new Properties();
             if ( dbConfiguration.getUsername() != null && !dbConfiguration.getUsername().isEmpty() )
             {
@@ -361,7 +361,7 @@ public class DatabaseService extends AbstractPwmService implements PwmService
             }
 
             final Connection connection = driver.connect( connectionURL, connectionProperties );
-            LOGGER.debug( () -> "connected to database " + connectionURL );
+            LOGGER.debug( getSessionLabel(), () -> "connected to database " + connectionURL );
 
             connection.setAutoCommit( false );
             return connection;
@@ -370,7 +370,7 @@ public class DatabaseService extends AbstractPwmService implements PwmService
         {
             final String errorMsg = "error connecting to database: " + JavaHelper.readHostileExceptionMessage( e );
             final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, errorMsg );
-            LOGGER.error( errorInformation );
+            LOGGER.error( getSessionLabel(),  errorInformation );
             throw new DatabaseException( errorInformation );
         }
     }
@@ -416,7 +416,7 @@ public class DatabaseService extends AbstractPwmService implements PwmService
             }
             catch ( final SQLException e )
             {
-                LOGGER.error( () -> "error reading jdbc meta data: " + e.getMessage() );
+                LOGGER.error( getSessionLabel(), () -> "error reading jdbc meta data: " + e.getMessage() );
             }
         }
     }
@@ -444,7 +444,7 @@ public class DatabaseService extends AbstractPwmService implements PwmService
                 }
                 if ( !valid )
                 {
-                    LOGGER.warn( () -> "database connection lost; will retry connect periodically" );
+                    LOGGER.warn( getSessionLabel(), () -> "database connection lost; will retry connect periodically" );
                     initialized = false;
                 }
 

server/src/main/java/password/pwm/svc/httpclient/ApachePwmHttpClient.java

@@ -269,7 +269,7 @@ public class ApachePwmHttpClient implements AutoCloseable, PwmHttpClientProvider
             throws IOException, PwmUnrecoverableException
     {
         final Instant startTime = Instant.now();
-        if ( LOGGER.isEnabled( PwmLogLevel.TRACE ) )
+        if ( LOGGER.isInterestingLevel( PwmLogLevel.TRACE ) )
         {
             final String sslDebugText;
             if ( clientRequest.isHttps() )

server/src/main/java/password/pwm/svc/httpclient/JavaPwmHttpClient.java

@@ -191,7 +191,7 @@ public class JavaPwmHttpClient implements PwmHttpClientProvider
 
     private void logRequest( final PwmHttpClientRequest clientRequest, final SessionLabel sessionLabel ) throws PwmUnrecoverableException
     {
-        if ( LOGGER.isEnabled( PwmLogLevel.TRACE ) )
+        if ( LOGGER.isInterestingLevel( PwmLogLevel.TRACE ) )
         {
             final String sslDebugText;
             if ( clientRequest.isHttps() )

server/src/main/java/password/pwm/svc/node/LDAPNodeDataService.java

@@ -31,6 +31,7 @@ import password.pwm.config.profile.LdapProfile;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.java.LazySupplier;
 import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
@@ -48,6 +49,7 @@ class LDAPNodeDataService implements NodeDataServiceProvider
     private final PwmDomain pwmDomain;
     private static final String VALUE_PREFIX = "0006#.#.#";
     private final NodeService nodeService;
+    private final LazySupplier.CheckedSupplier<LDAPHelper, PwmUnrecoverableException> ldapHelperSupplier;
 
     LDAPNodeDataService( final NodeService nodeService, final PwmDomain pwmDomain )
             throws PwmUnrecoverableException
@@ -65,6 +67,8 @@ class LDAPNodeDataService implements NodeDataServiceProvider
                     + " is configured";
             throw PwmUnrecoverableException.newException( PwmError.ERROR_NODE_SERVICE_ERROR, msg );
         }
+
+        ldapHelperSupplier = LazySupplier.checked( () -> LDAPHelper.createLDAPHelper( nodeService, pwmDomain ) );
     }
 
     @Override
@@ -72,8 +76,7 @@ class LDAPNodeDataService implements NodeDataServiceProvider
     {
         final Map<String, StoredNodeData> returnData = new LinkedHashMap<>(  );
 
-        final LDAPHelper ldapHelper = LDAPHelper.createLDAPHelper( nodeService, pwmDomain );
-
+        final LDAPHelper ldapHelper = ldapHelperSupplier.call();
         try
         {
             final Set<String> values = ldapHelper.getChaiUser().readMultiStringAttribute( ldapHelper.getAttr() );
@@ -102,7 +105,7 @@ class LDAPNodeDataService implements NodeDataServiceProvider
         final Map<String, StoredNodeData> currentServerData = readStoredData();
         final StoredNodeData removeNode = currentServerData.get( storedNodeData.getInstanceID() );
 
-        final LDAPHelper ldapHelper = LDAPHelper.createLDAPHelper( nodeService, pwmDomain );
+        final LDAPHelper ldapHelper = ldapHelperSupplier.call();
 
         final String newRawValue = VALUE_PREFIX + JsonFactory.get().serialize( storedNodeData );
 
@@ -129,7 +132,7 @@ class LDAPNodeDataService implements NodeDataServiceProvider
     @Override
     public int purgeOutdatedNodes( final TimeDuration maxNodeAge ) throws PwmUnrecoverableException
     {
-        final LDAPHelper ldapHelper = LDAPHelper.createLDAPHelper( nodeService, pwmDomain );
+        final LDAPHelper ldapHelper = ldapHelperSupplier.call();
 
         int nodesPurged = 0;
 
@@ -154,7 +157,7 @@ class LDAPNodeDataService implements NodeDataServiceProvider
                 catch ( final ChaiException e )
                 {
                     throw new PwmUnrecoverableException( PwmError.ERROR_LDAP_DATA_ERROR, "error purging node service data "
-                          + ldapHelper.debugInfo() + ", error: " + e.getMessage() );
+                            + ldapHelper.debugInfo() + ", error: " + e.getMessage() );
                 }
             }
         }
@@ -182,11 +185,11 @@ class LDAPNodeDataService implements NodeDataServiceProvider
 
             userIdentity = pwmDomain.getConfig().getDefaultLdapProfile().getTestUser( nodeService.getSessionLabel(), pwmDomain )
                     .orElseThrow( () ->
-            {
-                final String errorMsg = "a test user is not configured for ldap profile '" + ldapProfileID + "'";
-                final ErrorInformation errorInformation = new ErrorInformation( PwmError.CONFIG_FORMAT_ERROR, errorMsg );
-                return new PwmUnrecoverableException( errorInformation );
-            } );
+                    {
+                        final String errorMsg = "a test user is not configured for ldap profile '" + ldapProfileID + "'";
+                        final ErrorInformation errorInformation = new ErrorInformation( PwmError.CONFIG_FORMAT_ERROR, errorMsg );
+                        return new PwmUnrecoverableException( errorInformation );
+                    } );
 
             chaiUser = pwmDomain.getProxiedChaiUser( nodeService.getSessionLabel(), userIdentity );
 

server/src/main/java/password/pwm/svc/otp/OtpService.java

@@ -350,7 +350,7 @@ public class OtpService extends AbstractPwmService implements PwmService
 
         {
             final OTPUserRecord finalOtpConfig = otpConfig;
-            final Supplier<CharSequence> msg = () -> finalOtpConfig == null
+            final Supplier<String> msg = () -> finalOtpConfig == null
                     ? "no otp record found for user " + userIdentity.toDisplayString()
                     : "loaded otp record for user " + userIdentity.toDisplayString()
                     + " [recordType=" + finalOtpConfig.getType() + ", identifier=" + finalOtpConfig.getIdentifier() + ", timestamp="

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyService.java

@@ -164,7 +164,7 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
         try
         {
             nextExecutionTime = figureNextJobExecutionTime();
-            LOGGER.debug( getSessionLabel(), () -> "scheduled next job execution at " + nextExecutionTime.toString() );
+            LOGGER.debug( getSessionLabel(), () -> "scheduled next job execution at " + StringUtil.toIsoDate( nextExecutionTime ) );
         }
         catch ( final Exception e )
         {

server/src/main/java/password/pwm/util/OnejarHelper.java

@@ -118,6 +118,7 @@ public class OnejarHelper
                 .applicationPath( applicationPath )
                 .applicationMode( PwmApplicationMode.READ_ONLY )
                 .configurationFile( configFile )
+                .internalRuntimeInstance( true )
                 .flags( Collections.singleton( PwmEnvironment.ApplicationFlag.CommandLineInstance ) )
                 .build();
 

server/src/main/java/password/pwm/util/SampleDataGenerator.java

@@ -21,8 +21,6 @@
 package password.pwm.util;
 
 import com.novell.ldapchai.cr.Answer;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
 import password.pwm.PwmApplication;
 import password.pwm.PwmApplicationMode;
 import password.pwm.PwmConstants;
@@ -207,7 +205,6 @@ public class SampleDataGenerator
     private static PwmApplication makeSamplePwmApp( final AppConfig appConfig )
             throws PwmUnrecoverableException
     {
-        Logger.getRootLogger().setLevel( Level.OFF );
         final PwmEnvironment pwmEnvironment = PwmEnvironment.builder()
                 .config( appConfig )
                 .applicationPath( null )

server/src/main/java/password/pwm/util/cli/MainClass.java

@@ -20,10 +20,6 @@
 
 package password.pwm.util.cli;
 
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.EnhancedPatternLayout;
-import org.apache.log4j.Layout;
-import org.apache.log4j.Logger;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.PwmApplicationMode;
@@ -68,8 +64,6 @@ import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.java.MiscUtil;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBFactory;
-import password.pwm.util.logging.PwmLogLevel;
-import password.pwm.util.logging.PwmLogManager;
 import password.pwm.util.logging.PwmLogger;
 
 import java.io.File;
@@ -319,8 +313,6 @@ public class MainClass
         mainOptions = MainOptions.parseMainCommandLineOptions( args, new OutputStreamWriter( System.out, PwmConstants.DEFAULT_CHARSET ) );
         final List<String> workingArgs = mainOptions.getRemainingArguments();
 
-        initLog4j( mainOptions.getPwmLogLevel() );
-
         final String commandStr = workingArgs == null || workingArgs.size() < 1 ? null : workingArgs.iterator().next();
 
         boolean commandExceuted = false;
@@ -407,28 +399,6 @@ public class MainClass
         }
     }
 
-    private static void initLog4j( final PwmLogLevel logLevel )
-    {
-        if ( logLevel == null )
-        {
-            PwmLogger.disableAllLogging();
-            return;
-        }
-
-        final Layout patternLayout = new EnhancedPatternLayout( LOGGING_PATTERN );
-        final ConsoleAppender consoleAppender = new ConsoleAppender( patternLayout );
-        for ( final String logPackage : PwmLogManager.LOGGING_PACKAGES )
-        {
-            if ( logPackage != null )
-            {
-                final Logger logger = Logger.getLogger( logPackage );
-                logger.addAppender( consoleAppender );
-                logger.setLevel( logLevel.getLog4jLevel() );
-            }
-        }
-        PwmLogger.markInitialized();
-    }
-
     private static LocalDB loadPwmDB(
             final AppConfig config,
             final boolean readonly,

server/src/main/java/password/pwm/util/json/GsonJsonAdaptors.java

@@ -69,6 +69,7 @@ class GsonJsonAdaptors
         gsonBuilder.registerTypeAdapter( ProfileID.class, new ProfileIDTypeAdaptor() );
         gsonBuilder.registerTypeAdapter( LongAdder.class, new LongAdderTypeAdaptor() );
         gsonBuilder.registerTypeAdapter( TimeDuration.class, new TimeDurationAdaptor() );
+        gsonBuilder.registerTypeAdapter( Duration.class, new DurationAdaptor() );
         return gsonBuilder;
     }
 
@@ -308,4 +309,20 @@ class GsonJsonAdaptors
             return new JsonPrimitive( src.asDuration().toString() );
         }
     }
+
+    private static class DurationAdaptor implements JsonSerializer<Duration>, JsonDeserializer<Duration>
+    {
+        @Override
+        public Duration deserialize( final JsonElement json, final Type typeOfT, final JsonDeserializationContext context ) throws JsonParseException
+        {
+            final String stringValue = json.getAsString();
+            return Duration.parse( stringValue );
+        }
+
+        @Override
+        public JsonElement serialize( final Duration src, final Type typeOfSrc, final JsonSerializationContext context )
+        {
+            return new JsonPrimitive( src.toString() );
+        }
+    }
 }

server/src/main/java/password/pwm/util/json/MoshiJsonAdaptors.java

@@ -68,6 +68,7 @@ class MoshiJsonAdaptors
         moshiBuilder.add( BigInteger.class, applyFlagsToAdapter( new BigIntegerTypeAdaptor(), flags ) );
         moshiBuilder.add( Locale.class, applyFlagsToAdapter( new LocaleTypeAdaptor(), flags ) );
         moshiBuilder.add( TimeDuration.class, applyFlagsToAdapter( new TimeDurationAdaptor(), flags ) );
+        moshiBuilder.add( Duration.class, applyFlagsToAdapter( new DurationAdaptor(), flags ) );
     }
 
     static <T> JsonAdapter<T> applyFlagsToAdapter( final JsonAdapter<T> adapter, final JsonProvider.Flag... flags )
@@ -431,4 +432,31 @@ class MoshiJsonAdaptors
             writer.value( value.asDuration().toString() );
         }
     }
+
+    private static class DurationAdaptor extends JsonAdapter<Duration>
+    {
+        @Nullable
+        @Override
+        public Duration fromJson( final JsonReader reader ) throws IOException
+        {
+            final String strValue = reader.nextString();
+            if ( StringUtil.isEmpty( strValue ) )
+            {
+                return null;
+            }
+            return Duration.parse( strValue );
+        }
+
+        @Override
+        public void toJson( final JsonWriter writer, @Nullable final Duration value ) throws IOException
+        {
+            if ( value == null )
+            {
+                writer.nullValue();
+                return;
+            }
+
+            writer.value( value.toString() );
+        }
+    }
 }

server/src/main/java/password/pwm/util/localdb/LocalDBFactory.java

@@ -103,6 +103,20 @@ public class LocalDBFactory
                 }
             }
 
+            logInstanceCreation( dbDirectory, readonly, startTime, localDB );
+
+            return localDB;
+        }
+        finally
+        {
+            CREATION_LOCK.unlock();
+        }
+    }
+
+    private static void logInstanceCreation( final File dbDirectory, final boolean readonly, final Instant startTime, final LocalDB localDB )
+    {
+        LOGGER.info( () ->
+        {
             final StringBuilder debugText = new StringBuilder();
             debugText.append( "LocalDB open" );
 
@@ -121,14 +135,8 @@ public class LocalDBFactory
                     debugText.append( ", " ).append( StringUtil.formatDiskSize( freeSpace ) ).append( " free" );
                 }
             }
-            LOGGER.info( () -> debugText, TimeDuration.fromCurrent( startTime ) );
-
-            return localDB;
-        }
-        finally
-        {
-            CREATION_LOCK.unlock();
-        }
+            return debugText.toString();
+        }, TimeDuration.fromCurrent( startTime ) );
     }
 
     private static LocalDBProvider createInstance( final String className )

server/src/main/java/password/pwm/util/localdb/LocalDBStoredQueue.java

@@ -1090,7 +1090,7 @@ public class LocalDBStoredQueue implements Queue<String>, Deque<String>
                 return;
             }
 
-            final Supplier<CharSequence> debugOutput = () ->
+            final Supplier<String> debugOutput = () ->
             {
                 final StringBuilder sb = new StringBuilder();
                 try

server/src/main/java/password/pwm/util/logging/LocalDBLog4jAppender.java

@@ -1,74 +0,0 @@
-/*
- * Password Management Servlets (PWM)
- * http://www.pwm-project.org
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2021 The PWM Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package password.pwm.util.logging;
-
-import org.apache.log4j.AppenderSkeleton;
-import org.apache.log4j.Level;
-import org.apache.log4j.spi.LoggingEvent;
-import org.apache.log4j.spi.ThrowableInformation;
-
-import java.time.Instant;
-
-public class LocalDBLog4jAppender extends AppenderSkeleton
-{
-
-    private LocalDBLogger localDBLogger;
-
-    public LocalDBLog4jAppender( final LocalDBLogger localDBLogger )
-    {
-        this.localDBLogger = localDBLogger;
-    }
-
-    @Override
-    protected void append( final LoggingEvent loggingEvent )
-    {
-        final Object message = loggingEvent.getMessage();
-        final ThrowableInformation throwableInformation = loggingEvent.getThrowableInformation();
-        final Level level = loggingEvent.getLevel();
-        final Instant timeStamp = Instant.ofEpochMilli( loggingEvent.getTimeStamp() );
-        final String sourceLogger = loggingEvent.getLogger().getName();
-
-        if ( localDBLogger != null )
-        {
-            final PwmLogEvent logEvent = PwmLogEvent.createPwmLogEvent(
-                    timeStamp,
-                    sourceLogger,
-                    message.toString(),
-                    null,
-                    throwableInformation == null ? null : throwableInformation.getThrowable(),
-                    PwmLogLevel.fromLog4jLevel( level )
-            );
-
-            localDBLogger.writeEvent( logEvent );
-        }
-    }
-
-    @Override
-    public void close( )
-    {
-    }
-
-    @Override
-    public boolean requiresLayout( )
-    {
-        return false;
-    }
-}

server/src/main/java/password/pwm/util/logging/LocalDBLogbackAppender.java

@@ -0,0 +1,51 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.util.logging;
+
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.UnsynchronizedAppenderBase;
+import org.slf4j.Marker;
+
+import java.util.List;
+
+public class LocalDBLogbackAppender extends UnsynchronizedAppenderBase<ILoggingEvent>
+{
+    private final LocalDBLogger localDBLogger;
+
+    public LocalDBLogbackAppender( final LocalDBLogger localDBLogger )
+    {
+        this.localDBLogger = localDBLogger;
+    }
+
+    @Override
+    protected void append( final ILoggingEvent event )
+    {
+        final List<Marker> markerList = event.getMarkerList();
+        if ( markerList != null && markerList.contains( PwmLogbackMarker.singleton() ) )
+        {
+            // event was generated by PWM, which has already sent the event to the localDB
+            return;
+        }
+
+        final PwmLogMessage logEvent = PwmLogMessage.fromLogbackEvent( event );
+        localDBLogger.writeEvent( logEvent );
+    }
+}

server/src/main/java/password/pwm/util/logging/LocalDBLogger.java

@@ -42,7 +42,6 @@ import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBException;
 import password.pwm.util.localdb.LocalDBStoredQueue;
 
-import java.io.IOException;
 import java.text.NumberFormat;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -73,11 +72,12 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
 
     private final LocalDBLoggerSettings settings;
     private final LocalDBStoredQueue localDBListQueue;
-    private final Queue<PwmLogEvent> tempMemoryEventQueue;
+    private final Queue<PwmLogMessage> tempMemoryEventQueue;
     private final ScheduledExecutorService cleanerService;
     private final ScheduledExecutorService writerService;
     private final AtomicBoolean cleanOnWriteFlag = new AtomicBoolean( false );
     private final AtomicBoolean flushScheduled = new AtomicBoolean( true );
+    private final PwmLogLevel minimumLevel;
 
     private final StatisticCounterBundle<CounterStat> stats = new StatisticCounterBundle<>( CounterStat.class );
     private final StatisticAverageBundle<AverageStat> averages = new StatisticAverageBundle<>( AverageStat.class );
@@ -109,12 +109,15 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
     public LocalDBLogger(
             final PwmApplication pwmApplication,
             final LocalDB localDB,
+            final PwmLogLevel minimumLevel,
             final LocalDBLoggerSettings settings
     )
             throws LocalDBException
     {
         Objects.requireNonNull( localDB, "localDB can not be null" );
 
+        this.minimumLevel = Objects.requireNonNull( minimumLevel );
+
         this.settings = settings == null
                 ? LocalDBLoggerSettings.builder().build().applyValueChecks()
                 : settings.applyValueChecks();
@@ -195,7 +198,6 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
 
     private void scheduleNextFlush()
     {
-
         if ( tempMemoryEventQueue.size() > settings.getMaxBufferSize() / 2 )
         {
             writerService.schedule( new FlushTask(), 0, TimeUnit.SECONDS );
@@ -385,6 +387,7 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
         {
             LOGGER.trace( () -> "invalid regex syntax for " + searchParameters.getUsername() + ", reverting to plaintext search" );
         }
+
         if ( pattern != null )
         {
             final Matcher matcher = pattern.matcher( event.getUsername() == null ? "" : event.getUsername() );
@@ -402,27 +405,35 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
             }
         }
 
-        if ( eventMatchesParams && ( searchParameters.getText() != null && searchParameters.getText().length() > 0 ) )
         {
-            final String eventMessage = event.getMessage();
-            final String textLowercase = searchParameters.getText().toLowerCase();
-            boolean isAMatch = false;
-            if ( eventMessage != null && eventMessage.length() > 0 )
+            final String searchParamText = searchParameters.getText();
+            if ( eventMatchesParams && !StringUtil.isEmpty( searchParamText ) )
             {
-                if ( eventMessage.toLowerCase().contains( textLowercase ) )
-                {
-                    isAMatch = true;
-                }
-                else if ( event.getTopic() != null && event.getTopic().length() > 0 )
+                final String eventMessage = event.getMessage();
+                if ( eventMessage != null && eventMessage.length() > 0 )
                 {
-                    if ( event.getTopic().toLowerCase().contains( textLowercase ) )
+                    final String textLowercase = searchParamText.toLowerCase();
+
+                    boolean isAMatch = false;
+                    if ( eventMessage.toLowerCase().contains( textLowercase ) )
                     {
                         isAMatch = true;
                     }
-                }
-                if ( !isAMatch )
-                {
-                    eventMatchesParams = false;
+                    else if ( event.getTopic() != null && event.getTopic().toLowerCase().contains( textLowercase ) )
+                    {
+                        isAMatch = true;
+                    }
+                    else if ( event.getTopic() != null && event.getTopic().length() > 0 )
+                    {
+                        if ( event.getTopic().toLowerCase().contains( textLowercase ) )
+                        {
+                            isAMatch = true;
+                        }
+                    }
+                    if ( !isAMatch )
+                    {
+                        eventMatchesParams = false;
+                    }
                 }
             }
         }
@@ -431,14 +442,14 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
         {
             if ( searchParameters.getEventType() == EventType.System )
             {
-                if ( event.getUsername() != null && event.getUsername().length() > 0 )
+                if ( !StringUtil.isEmpty( event.getUsername() ) )
                 {
                     eventMatchesParams = false;
                 }
             }
             else if ( searchParameters.getEventType() == EventType.User )
             {
-                if ( event.getUsername() == null || event.getUsername().length() < 1 )
+                if ( StringUtil.isEmpty( event.getUsername() ) )
                 {
                     eventMatchesParams = false;
                 }
@@ -448,28 +459,34 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
         return eventMatchesParams;
     }
 
-    public void writeEvent( final PwmLogEvent event )
+    public void writeEvent( final PwmLogMessage event )
     {
-        if ( status() == STATUS.OPEN )
+        if ( ignoreLogEvent( event ) )
         {
-            if ( settings.getMaxEvents() > 0 )
-            {
-                scheduleNextFlush();
+            return;
+        }
 
-                final Instant startTime = Instant.now();
-                while ( !tempMemoryEventQueue.offer( event ) )
-                {
-                    if ( TimeDuration.fromCurrent( startTime ).isLongerThan( settings.getMaxBufferWaitTime() ) )
-                    {
-                        LOGGER.warn( () -> "discarded event after waiting max buffer wait time of " + settings.getMaxBufferWaitTime().asCompactString() );
-                        return;
-                    }
-                    TimeDuration.of( 100, TimeDuration.Unit.MILLISECONDS ).pause();
-                }
+        scheduleNextFlush();
+
+        final Instant startTime = Instant.now();
+        while ( !tempMemoryEventQueue.offer( event ) )
+        {
+            if ( TimeDuration.fromCurrent( startTime ).isLongerThan( settings.getMaxBufferWaitTime() ) )
+            {
+                LOGGER.warn( () -> "discarded event after waiting max buffer wait time of " + settings.getMaxBufferWaitTime().asCompactString() );
+                return;
             }
+            TimeDuration.of( 100, TimeDuration.Unit.MILLISECONDS ).pause();
         }
     }
 
+    private boolean ignoreLogEvent( final PwmLogMessage event )
+    {
+        return status() != STATUS.OPEN
+                || settings.getMaxEvents() <= 0
+                ||         !event.getLevel().isGreaterOrSameAs( minimumLevel );
+    }
+
     private void flushEvents( )
     {
         if ( tempMemoryEventQueue.isEmpty() )
@@ -481,16 +498,9 @@ public class LocalDBLogger extends AbstractPwmService implements PwmService
         final List<String> localBuffer = new ArrayList<>( Math.min( tempMemoryEventQueue.size(), settings.getMaxBufferSize() ) );
         while ( localBuffer.size() < ( settings.getMaxBufferSize() ) - 1 && !tempMemoryEventQueue.isEmpty() )
         {
-            final PwmLogEvent pwmLogEvent = tempMemoryEventQueue.poll();
-            try
-            {
-                localBuffer.add( pwmLogEvent.toEncodedString() );
-                eldestEntry = pwmLogEvent.getTimestamp();
-            }
-            catch ( final IOException e )
-            {
-                LOGGER.warn( () -> "error flushing events to localDB: " + e.getMessage(), e );
-            }
+            final PwmLogMessage pwmLogEvent = tempMemoryEventQueue.poll();
+            localBuffer.add( pwmLogEvent.toLogEvent().toEncodedString() );
+            eldestEntry = pwmLogEvent.getTimestamp();
         }
 
         try

server/src/main/java/password/pwm/util/logging/PwmLogEvent.java

@@ -20,27 +20,34 @@
 
 package password.pwm.util.logging;
 
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Value;
 import org.apache.commons.csv.CSVPrinter;
 import password.pwm.PwmConstants;
 import password.pwm.bean.SessionLabel;
-import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.MiscUtil;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.Serializable;
+import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
 
 @Value
+@AllArgsConstructor( access = AccessLevel.PRIVATE )
+@Builder( access = AccessLevel.PRIVATE, toBuilder = true )
 public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
 {
-    private static final int MAX_MESSAGE_LENGTH = 50_000;
+    private static final int MAX_MESSAGE_LENGTH = 10_000;
+    private static final int MAX_FIELD_LENGTH = 256;
 
     private final Instant timestamp;
     private final String sessionID;
@@ -52,6 +59,7 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
     private final String username;
     private final String domain;
     private final String sourceAddress;
+    private final Duration duration;
 
     private static final Comparator<PwmLogEvent> COMPARATOR = Comparator.comparing(
             PwmLogEvent::getTimestamp,
@@ -78,7 +86,8 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
             final String message,
             final SessionLabel sessionLabel,
             final LoggedThrowable loggedThrowable,
-            final PwmLogLevel level
+            final PwmLogLevel level,
+            final Duration duration
     )
     {
         if ( timestamp == null )
@@ -91,20 +100,18 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
             throw new IllegalArgumentException( "level may not be null" );
         }
 
-        // safety truncate
-        final int maxFieldLength = 256;
-
         this.timestamp = timestamp;
-        this.topic = StringUtil.truncate( topic, maxFieldLength );
+        this.topic = StringUtil.truncate( topic, MAX_FIELD_LENGTH );
         this.message = StringUtil.truncate( message, MAX_MESSAGE_LENGTH, " [truncated message]" );
         this.loggedThrowable = loggedThrowable;
         this.level = level;
 
-        this.sessionID = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getSessionID(), 256 );
-        this.requestID = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getRequestID(), 256 );
-        this.username = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getUsername(), 256 );
-        this.domain = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getDomain(), 256 );
-        this.sourceAddress = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getSourceAddress(), 256 );
+        this.sessionID = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getSessionID(), MAX_FIELD_LENGTH );
+        this.requestID = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getRequestID(), MAX_FIELD_LENGTH );
+        this.username = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getUsername(), MAX_FIELD_LENGTH );
+        this.domain = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getDomain(), MAX_FIELD_LENGTH );
+        this.sourceAddress = sessionLabel == null ? "" : StringUtil.truncate( sessionLabel.getSourceAddress(), MAX_FIELD_LENGTH );
+        this.duration = duration == null ? Duration.ZERO : duration;
     }
 
     public static PwmLogEvent createPwmLogEvent(
@@ -113,40 +120,29 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
             final String message,
             final SessionLabel sessionLabel,
             final Throwable throwable,
-            final PwmLogLevel level
+            final PwmLogLevel level,
+            final Duration duration
     )
     {
-        return new PwmLogEvent( timestamp, topic, message, sessionLabel, LoggedThrowable.fromThrowable( throwable ), level );
+        return new PwmLogEvent( timestamp, topic, message, sessionLabel, LoggedThrowable.fromThrowable( throwable ), level, duration );
     }
 
     String getEnhancedMessage( )
     {
-        final StringBuilder output = new StringBuilder();
-        output.append( getDebugLabel() );
-        output.append( message );
-
-        final String srcAddrString = getSourceAddress();
-        if ( StringUtil.notEmpty( srcAddrString ) )
-        {
-            final String srcStr = " [" + srcAddrString + "]";
 
-            final int firstCR = output.indexOf( "\n" );
-            if ( firstCR == -1 )
-            {
-                output.append( srcStr );
-            }
-            else
-            {
-                output.insert( firstCR, srcStr );
-            }
-        }
-
-        if ( this.getLoggedThrowable() != null && this.getLoggedThrowable().getStackTrace() != null )
-        {
-            output.append( JavaHelper.throwableToString( getLoggedThrowable().toThrowable() ) );
-        }
+        final SessionLabel sessionLabel = SessionLabel.builder()
+                .sessionID( getSessionID() )
+                .requestID( getRequestID() )
+                .username( getUsername() )
+                .sourceAddress( getSourceAddress() )
+                .domain( getDomain() )
+                .build();
 
-        return output.toString();
+        return PwmLogUtil.createEnhancedMessage(
+                sessionLabel,
+                message,
+                loggedThrowable == null ? null : loggedThrowable.toThrowable(),
+                duration == null ? null : TimeDuration.fromDuration( duration ) );
     }
 
     @Override
@@ -156,30 +152,10 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
     }
 
     String toEncodedString( )
-            throws IOException
     {
         return JsonFactory.get().serialize( this, PwmLogEvent.class );
     }
 
-    Throwable getThrowable()
-    {
-        return getLoggedThrowable() == null
-                ? null
-                : getLoggedThrowable().toThrowable();
-    }
-
-    private String getDebugLabel( )
-    {
-        return SessionLabel.builder()
-                .sessionID( getSessionID() )
-                .requestID( getRequestID() )
-                .username( getUsername() )
-                .sourceAddress( getSourceAddress() )
-                .domain( getDomain() )
-                .build()
-                .toDebugLabel();
-    }
-
     public String toLogString( )
     {
         return toLogString( true );
@@ -189,6 +165,9 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
     {
         final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
         final CSVPrinter csvPrinter = MiscUtil.makeCsvPrinter( byteArrayOutputStream );
+
+        final String throwableMessage = ( getLoggedThrowable() == null || getLoggedThrowable().getMessage() == null ) ? "" : getLoggedThrowable().getMessage();
+
         final List<String> dataRow = new ArrayList<>();
         dataRow.add( StringUtil.toIsoDate( getTimestamp() ) );
         dataRow.add( getLevel().name() );
@@ -197,11 +176,12 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
         dataRow.add( getUsername( ) );
         dataRow.add( getTopic() );
         dataRow.add( getMessage() );
-        dataRow.add( getLoggedThrowable() == null && getLoggedThrowable().getMessage() != null ? "" : getLoggedThrowable().getMessage() );
+        dataRow.add( throwableMessage );
+
         csvPrinter.printRecord( dataRow );
         csvPrinter.flush();
-        return byteArrayOutputStream.toString( PwmConstants.DEFAULT_CHARSET.name() );
 
+        return byteArrayOutputStream.toString( PwmConstants.DEFAULT_CHARSET.name() );
     }
 
     public String toLogString( final boolean includeTimeStamp )
@@ -251,4 +231,8 @@ public class PwmLogEvent implements Serializable, Comparable<PwmLogEvent>
         return output.toString();
     }
 
+    PwmLogEvent stripThrowable()
+    {
+        return this.toBuilder().loggedThrowable( null ).build();
+    }
 }

server/src/main/java/password/pwm/util/logging/PwmLogLevel.java

@@ -20,40 +20,75 @@
 
 package password.pwm.util.logging;
 
-import org.apache.log4j.Level;
+import org.slf4j.event.Level;
+import password.pwm.util.java.CollectionUtil;
 import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.StringUtil;
+
+import java.util.Collection;
 
 public enum PwmLogLevel
 {
-    TRACE( Level.TRACE ),
-    DEBUG( Level.DEBUG ),
-    INFO( Level.INFO ),
-    WARN( Level.WARN ),
-    ERROR( Level.ERROR ),
-    FATAL( Level.FATAL ),;
+    TRACE( ch.qos.logback.classic.Level.TRACE, org.slf4j.event.Level.TRACE ),
+    DEBUG( ch.qos.logback.classic.Level.DEBUG, org.slf4j.event.Level.DEBUG ),
+    INFO( ch.qos.logback.classic.Level.INFO, org.slf4j.event.Level.INFO ),
+    WARN( ch.qos.logback.classic.Level.WARN, org.slf4j.event.Level.WARN ),
+    ERROR( ch.qos.logback.classic.Level.ERROR, org.slf4j.event.Level.ERROR ),
+    FATAL( ch.qos.logback.classic.Level.ERROR, org.slf4j.event.Level.ERROR ),
+    NONE( ch.qos.logback.classic.Level.OFF, org.slf4j.event.Level.ERROR ),;
+
+    private final ch.qos.logback.classic.Level logbackLevel;
+    private final org.slf4j.event.Level slf4jLevel;
+
+    PwmLogLevel( final ch.qos.logback.classic.Level logbackLevel, final org.slf4j.event.Level slf4jLevel )
+    {
+        this.logbackLevel = logbackLevel;
+        this.slf4jLevel = slf4jLevel;
+    }
 
-    private final int log4jLevel;
+    public ch.qos.logback.classic.Level getLogbackLevel( )
+    {
+        return logbackLevel;
+    }
 
-    PwmLogLevel( final Level log4jLevel )
+    public Level getSlf4jLevel()
     {
-        this.log4jLevel = log4jLevel.toInt();
+        return slf4jLevel;
     }
 
-    public Level getLog4jLevel( )
+    public boolean isGreaterOrSameAs( final PwmLogLevel logLevel )
     {
-        return Level.toLevel( log4jLevel );
+        return logLevel != null && this.compareTo( logLevel ) >= 0;
     }
 
-    public static PwmLogLevel fromLog4jLevel( final Level level )
+    public static PwmLogLevel fromLogbackLevel( final ch.qos.logback.classic.Level level )
     {
-        final int log4jIntLevel = level == null
-                ? Level.TRACE.toInt()
-                : level.toInt();
+        if ( level == null )
+        {
+            return TRACE;
+        }
 
         return JavaHelper.readEnumFromPredicate(
                 PwmLogLevel.class,
-                pwmLogLevel -> pwmLogLevel.log4jLevel == log4jIntLevel
-        )
+                pwmLogLevel -> pwmLogLevel.logbackLevel == level
+        ).orElse( TRACE );
+    }
+
+    public static PwmLogLevel fromString( final String stringLogLevel )
+    {
+        return JavaHelper.readEnumFromPredicate(
+                        PwmLogLevel.class,
+                        pwmLogLevel -> StringUtil.nullSafeEqualsIgnoreCase( stringLogLevel, pwmLogLevel.name() ) )
                 .orElse( TRACE );
     }
+
+    public static PwmLogLevel lowestLevel( final Collection<PwmLogLevel> logLevels )
+    {
+        if ( CollectionUtil.isEmpty( logLevels ) )
+        {
+            return TRACE;
+        }
+
+        return CollectionUtil.copyToEnumSet( logLevels, PwmLogLevel.class ).iterator().next();
+    }
 }

server/src/main/java/password/pwm/util/logging/PwmLogManager.java

@@ -20,151 +20,248 @@
 
 package password.pwm.util.logging;
 
-import com.novell.ldapchai.ChaiUser;
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.Layout;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
-import org.apache.log4j.RollingFileAppender;
-import org.apache.log4j.xml.DOMConfigurator;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
+import ch.qos.logback.classic.filter.ThresholdFilter;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.Appender;
+import ch.qos.logback.core.ConsoleAppender;
+import ch.qos.logback.core.encoder.EncoderBase;
+import ch.qos.logback.core.encoder.LayoutWrappingEncoder;
+import ch.qos.logback.core.filter.Filter;
+import ch.qos.logback.core.joran.spi.ConfigurationWatchList;
+import ch.qos.logback.core.joran.util.ConfigurationWatchListUtil;
+import ch.qos.logback.core.rolling.FixedWindowRollingPolicy;
+import ch.qos.logback.core.rolling.RollingFileAppender;
+import ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy;
+import ch.qos.logback.core.util.FileSize;
+import org.slf4j.LoggerFactory;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.PwmApplicationMode;
 import password.pwm.PwmConstants;
 import password.pwm.config.AppConfig;
-import password.pwm.config.stored.StoredConfigurationFactory;
 import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBException;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Objects;
 
 public class PwmLogManager
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwmLogManager.class );
 
-    public static final List<String> LOGGING_PACKAGES = List.of(
-            PwmApplication.class.getPackage().getName(),
-            ChaiUser.class.getPackage().getName(),
-            "org.jasig.cas.client" );
+    private static final String LOGGER_NAME_LOCALDB = "pwmLocalDBLogger";
+    private static final String LOGGER_NAME_FILE = "pwmFileLogger";
+    private static final String LOGGER_NAME_CONSOLE = "pwmConsoleLogger";
 
-    public static void deinitializeLogger( )
+    private static PwmApplication pwmApplication;
+    private static LocalDBLogger localDBLogger;
+    private static PwmLogSettings pwmLogSettings = PwmLogSettings.defaultSettings();
+    private static PwmLogLevel lowestLogLevelConfigured = PwmLogLevel.TRACE;
+
+    public static void disableLogging( )
     {
-        // clear all existing package loggers
-        for ( final String logPackage : LOGGING_PACKAGES )
+        if ( logbackConfigFileExists() )
         {
-            if ( logPackage != null )
-            {
-                final Logger logger = Logger.getLogger( logPackage );
-                logger.setAdditivity( false );
-                logger.removeAllAppenders();
-                logger.setLevel( Level.TRACE );
-            }
+            LOGGER.info( () -> "skipping " + PwmConstants.PWM_APP_NAME
+                    + " logback configuration, logback is already configured" );
+
+            return;
         }
 
-        PwmLogger.setLocalDBLogger( null, null );
-        PwmLogger.setPwmApplication( null );
-        PwmLogger.setFileAppender( null );
+        final LoggerContext logCtx = getLoggerContext();
+
+        final Logger rootLogger = logCtx.getLogger( Logger.ROOT_LOGGER_NAME );
+        for ( final Iterator<Appender<ILoggingEvent>> iter = rootLogger.iteratorForAppenders(); iter.hasNext(); )
+        {
+            final Appender<ILoggingEvent> appender = iter.next();
+            rootLogger.detachAppender( appender );
+            appender.stop();
+        }
+
+        logCtx.reset();
+        logCtx.stop();
+
+        PwmLogManager.localDBLogger = null;
+        PwmLogManager.pwmApplication = null;
+        PwmLogManager.pwmLogSettings = PwmLogSettings.defaultSettings();
+        lowestLogLevelConfigured = PwmLogLevel.TRACE;
     }
 
-    public static void initializeLogger(
+    public static void initializeLogging(
             final PwmApplication pwmApplication,
             final AppConfig config,
-            final File log4jConfigFile,
-            final String consoleLogLevel,
             final File pwmApplicationPath,
-            final String fileLogLevel
+            final PwmLogSettings pwmLogSettings
     )
     {
-        PwmLogger.setPwmApplication( pwmApplication );
-
-        // try to configure using the log4j config file (if it exists)
-        if ( log4jConfigFile != null )
+        if ( pwmApplicationPath != null )
         {
-            try
+            final File logbackXmlInAppPath = new File( pwmApplicationPath.getPath() + File.separator + "logback.xml" );
+            if ( logbackXmlInAppPath.exists() )
             {
-                if ( !log4jConfigFile.exists() )
+                if ( PwmLogUtil.initLogbackFromXmlFile( logbackXmlInAppPath ) )
                 {
-                    throw new Exception( "file not found: " + log4jConfigFile.getAbsolutePath() );
+                    LOGGER.info( () -> "used appPath logback xml file '" + logbackXmlInAppPath.getPath()
+                            + "' to configure logging system, will ignore configured logging settings " );
                 }
-                DOMConfigurator.configure( log4jConfigFile.getAbsolutePath() );
-                LOGGER.debug( () -> "successfully initialized log4j using file " + log4jConfigFile.getAbsolutePath() );
-                return;
-            }
-            catch ( final Exception e )
-            {
-                LOGGER.error( () -> "error loading log4jconfig file '" + log4jConfigFile + "' error: " + e.getMessage() );
             }
         }
 
-        deinitializeLogger();
+        if ( logbackConfigFileExists() )
+        {
+            LOGGER.info( () -> "skipping " + PwmConstants.PWM_APP_NAME
+                    + " logback configuration, logback is already configured" );
+
+            return;
+        }
+
+        disableLogging();
+
+        // all initialize lines start here (above line disables everything !!)
+
+        PwmLogManager.pwmLogSettings = pwmLogSettings;
 
-        initConsoleLogger( config, consoleLogLevel );
+        PwmLogManager.pwmApplication = pwmApplication;
 
-        initFileLogger( config, fileLogLevel, pwmApplicationPath );
+        lowestLogLevelConfigured = pwmLogSettings.calculateLowestLevel();
 
-        // disable jersey warnings.
-        java.util.logging.Logger.getLogger( "org.glassfish.jersey" ).setLevel( java.util.logging.Level.SEVERE );
+        initConsoleLogger( config, pwmLogSettings.getStdoutLevel() );
+
+        initFileLogger( config, pwmLogSettings.getFileLevel(), pwmApplicationPath );
     }
 
-    public static void preInitConsoleLogLevel( final String pwmLogLevel )
+    static PwmLogLevel getLowestLogLevelConfigured()
     {
-        try
-        {
-            initConsoleLogger( AppConfig.forStoredConfig( StoredConfigurationFactory.newConfig() ), pwmLogLevel );
-        }
-        catch ( final Exception e )
-        {
-            final String msg = "error pre-initializing logger: " + e.getMessage();
-            System.err.println( msg );
-        }
+        return lowestLogLevelConfigured;
+    }
+
+    static PwmLogSettings getPwmLogSettings()
+    {
+        return pwmLogSettings;
+    }
+
+    static LocalDBLogger getLocalDbLogger()
+    {
+        return localDBLogger;
+    }
+
+    static PwmApplication getPwmApplication()
+    {
+        return pwmApplication;
     }
 
     private static void initConsoleLogger(
             final AppConfig config,
-            final String consoleLogLevel
+            final PwmLogLevel consoleLogLevel
     )
     {
-        final Layout patternLayout = new PatternLayout( config.readAppProperty( AppProperty.LOGGING_PATTERN ) );
-        // configure console logging
-        if ( consoleLogLevel != null && consoleLogLevel.length() > 0 && !"Off".equals( consoleLogLevel ) )
+        final LoggerContext logCtx = getLoggerContext();
+
+        final ConsoleAppender<ILoggingEvent> logConsoleAppender = new ConsoleAppender<>();
+        logConsoleAppender.setContext( logCtx );
+        logConsoleAppender.setName( LOGGER_NAME_CONSOLE );
+        logConsoleAppender.setEncoder( makePatternLayoutEncoder( config ) );
+        logConsoleAppender.addFilter( makeLevelFilter( consoleLogLevel ) );
+        logConsoleAppender.start();
+
+        attachAppender( logConsoleAppender );
+    }
+
+    static boolean logbackConfigFileExists()
+    {
+        final LoggerContext context = getLoggerContext();
+        final ConfigurationWatchList configurationWatchList = ConfigurationWatchListUtil.getConfigurationWatchList( context );
+
+        if ( configurationWatchList != null )
         {
-            final ConsoleAppender consoleAppender = new ConsoleAppender( patternLayout );
-            final Level level = Level.toLevel( consoleLogLevel );
-            consoleAppender.setThreshold( level );
-            for ( final String logPackage : LOGGING_PACKAGES )
-            {
-                if ( logPackage != null )
+            final List<File> watchList = ConfigurationWatchListUtil.getConfigurationWatchList( context ).getCopyOfFileWatchList();
+            return !watchList.isEmpty();
+        }
+
+        return false;
+    }
+
+    static LoggerContext getLoggerContext()
+    {
+        return ( LoggerContext ) LoggerFactory.getILoggerFactory();
+    }
+
+    private static void attachAppender( final Appender<ILoggingEvent> appender )
+    {
+        final LoggerContext logCtx = getLoggerContext();
+
+        PwmLogManager.getPwmLogSettings().getLoggingPackages().stream()
+                .filter( Objects::nonNull )
+                .map( logCtx::getLogger )
+                .forEach( logger ->
                 {
-                    final Logger logger = Logger.getLogger( logPackage );
                     logger.setLevel( Level.TRACE );
-                    logger.addAppender( consoleAppender );
-                }
-            }
-            LOGGER.debug( () -> "successfully initialized default console log4j config at log level " + level.toString() );
+                    logger.detachAppender( appender.getName() );
+                    logger.addAppender( appender );
+                } );
+    }
+
+    private static Filter<ILoggingEvent> makeLevelFilter( final PwmLogLevel pwmLogLevel )
+    {
+        final ThresholdFilter levelFilter = new ThresholdFilter();
+        levelFilter.setContext( getLoggerContext() );
+        levelFilter.setLevel( pwmLogLevel.getLogbackLevel().levelStr );
+        levelFilter.start();
+        return levelFilter;
+    }
+
+    private static EncoderBase<ILoggingEvent> makePatternLayoutEncoder( final AppConfig appConfig )
+    {
+        final String loggingPatternStr;
+        if ( appConfig == null )
+        {
+            loggingPatternStr = "no-pattern %-12date{YYYY-MM-dd HH:mm:ss.SSS} %-5level – %msg%n";
         }
         else
         {
-            LOGGER.debug( () -> "skipping stdout log4j initialization due to blank setting for log level" );
+            if ( pwmLogSettings.getLogOutputMode() == PwmLogSettings.LogOutputMode.json )
+            {
+                loggingPatternStr = "%msg%n";
+            }
+            else
+            {
+                final PwmLogbackPattern pwmLogbackPattern = new PwmLogbackPattern();
+                pwmLogbackPattern.setContext( getLoggerContext() );
+                pwmLogbackPattern.start();
+                final LayoutWrappingEncoder<ILoggingEvent> layoutWrappingEncoder = new LayoutWrappingEncoder<>();
+                layoutWrappingEncoder.setLayout( pwmLogbackPattern );
+                layoutWrappingEncoder.setContext( getLoggerContext() );
+                layoutWrappingEncoder.start();
+                return layoutWrappingEncoder;
+            }
         }
+
+        final PatternLayoutEncoder logEncoder = new PatternLayoutEncoder();
+        logEncoder.setContext( getLoggerContext() );
+        logEncoder.setPattern( loggingPatternStr );
+        logEncoder.start();
+        return logEncoder;
     }
 
     private static void initFileLogger(
             final AppConfig config,
-            final String fileLogLevel,
+            final PwmLogLevel fileLogLevel,
             final File pwmApplicationPath
     )
     {
-        final Layout patternLayout = new PatternLayout( config.readAppProperty( AppProperty.LOGGING_PATTERN ) );
-
         // configure file logging
         final String logDirectorySetting = config.readAppProperty( AppProperty.LOGGING_FILE_PATH );
         final File logDirectory = FileSystemUtility.figureFilepath( logDirectorySetting, pwmApplicationPath );
 
-        if ( logDirectory != null && fileLogLevel != null && fileLogLevel.length() > 0 && !"Off".equals( fileLogLevel ) )
+        if ( logDirectory != null && fileLogLevel != null )
         {
             try
             {
@@ -181,27 +278,40 @@ public class PwmLogManager
                 }
 
                 final String fileName = logDirectory.getAbsolutePath() + File.separator + PwmConstants.PWM_APP_NAME + ".log";
-                final RollingFileAppender fileAppender = new RollingFileAppender( patternLayout, fileName, true );
-                final Level level = Level.toLevel( fileLogLevel );
-                fileAppender.setThreshold( level );
-                fileAppender.setEncoding( PwmConstants.DEFAULT_CHARSET.name() );
-                fileAppender.setMaxBackupIndex( Integer.parseInt( config.readAppProperty( AppProperty.LOGGING_FILE_MAX_ROLLOVER ) ) );
-                fileAppender.setMaxFileSize( config.readAppProperty( AppProperty.LOGGING_FILE_MAX_SIZE ) );
+                final String fileNamePattern = logDirectory.getAbsolutePath() + File.separator + PwmConstants.PWM_APP_NAME + ".log.%i.zip";
 
-                PwmLogger.setFileAppender( fileAppender );
+                final LoggerContext logCtx = getLoggerContext();
 
-                for ( final String logPackage : LOGGING_PACKAGES )
-                {
-                    if ( logPackage != null )
-                    {
-                        //if (!logPackage.equals(PwmApplication.class.getPackage())) {
-                        final Logger logger = Logger.getLogger( logPackage );
-                        logger.setLevel( Level.TRACE );
-                        logger.addAppender( fileAppender );
-                        //}
-                    }
-                }
-                LOGGER.debug( () -> "successfully initialized default file log4j config at log level " + level.toString() );
+                final FixedWindowRollingPolicy rollingPolicy = new FixedWindowRollingPolicy();
+                rollingPolicy.setContext( logCtx );
+                rollingPolicy.setMinIndex( 1 );
+                rollingPolicy.setMaxIndex( Integer.parseInt( config.readAppProperty( AppProperty.LOGGING_FILE_MAX_ROLLOVER ) )  );
+                rollingPolicy.setFileNamePattern( fileNamePattern );
+
+                final SizeBasedTriggeringPolicy<ILoggingEvent> sizeBasedTriggeringPolicy = new SizeBasedTriggeringPolicy<>();
+                sizeBasedTriggeringPolicy.setMaxFileSize( FileSize.valueOf( config.readAppProperty( AppProperty.LOGGING_FILE_MAX_SIZE ) ) );
+                sizeBasedTriggeringPolicy.setContext( logCtx );
+                sizeBasedTriggeringPolicy.start();
+
+                final RollingFileAppender<ILoggingEvent> fileAppender = new RollingFileAppender<>();
+                fileAppender.setContext( logCtx );
+                fileAppender.setName( LOGGER_NAME_FILE );
+                fileAppender.setEncoder( makePatternLayoutEncoder( config ) );
+                fileAppender.setFile( fileName );
+                fileAppender.setPrudent( false );
+                fileAppender.addFilter( makeLevelFilter( fileLogLevel ) );
+                fileAppender.setRollingPolicy( rollingPolicy );
+                fileAppender.setTriggeringPolicy( sizeBasedTriggeringPolicy );
+
+                rollingPolicy.setParent( fileAppender );
+
+                rollingPolicy.start();
+                fileAppender.start();
+
+                attachAppender( fileAppender );
+
+                LOGGER.debug( () -> "successfully initialized default file log4j config at log level "
+                        + fileLogLevel );
             }
             catch ( final IOException e )
             {
@@ -214,7 +324,7 @@ public class PwmLogManager
     {
         final LocalDB localDB = pwmApplication.getLocalDB();
 
-        if ( pwmApplication.getApplicationMode() == PwmApplicationMode.READ_ONLY )
+        if ( pwmApplication.getApplicationMode() == PwmApplicationMode.READ_ONLY || pwmApplication.getPwmEnvironment().isInternalRuntimeInstance() )
         {
             LOGGER.trace( () -> "skipping initialization of LocalDBLogger due to read-only mode" );
             return null;
@@ -222,13 +332,13 @@ public class PwmLogManager
 
         // initialize the localDBLogger
         final LocalDBLogger localDBLogger;
-        final PwmLogLevel localDBLogLevel = pwmApplication.getConfig().getEventLogLocalDBLevel();
+        final PwmLogSettings pwmLogSettings = PwmLogSettings.fromAppConfig( pwmApplication.getConfig() );
         try
         {
-            localDBLogger = initLocalDBLogger( localDB, pwmApplication );
+            localDBLogger = initLocalDBLogger( localDB, pwmApplication, pwmLogSettings.getLocalDbLevel() );
             if ( localDBLogger != null )
             {
-                PwmLogger.setLocalDBLogger( localDBLogLevel, localDBLogger );
+                PwmLogManager.localDBLogger = localDBLogger;
             }
         }
         catch ( final Exception e )
@@ -240,17 +350,13 @@ public class PwmLogManager
         // add appender for other packages;
         try
         {
-            final LocalDBLog4jAppender localDBLog4jAppender = new LocalDBLog4jAppender( localDBLogger );
-            localDBLog4jAppender.setThreshold( localDBLogLevel.getLog4jLevel() );
-            for ( final String logPackage : LOGGING_PACKAGES )
-            {
-                if ( logPackage != null && !logPackage.equals( PwmApplication.class.getPackage().getName() ) )
-                {
-                    final Logger logger = Logger.getLogger( logPackage );
-                    logger.addAppender( localDBLog4jAppender );
-                    logger.setLevel( Level.TRACE );
-                }
-            }
+            final LocalDBLogbackAppender localDBLogbackAppender = new LocalDBLogbackAppender( localDBLogger );
+            localDBLogbackAppender.setContext( getLoggerContext() );
+            localDBLogbackAppender.setName( LOGGER_NAME_LOCALDB );
+            localDBLogbackAppender.addFilter( makeLevelFilter( pwmLogSettings.getLocalDbLevel() ) );
+            localDBLogbackAppender.start();
+
+            attachAppender( localDBLogbackAppender );
         }
         catch ( final Exception e )
         {
@@ -261,14 +367,15 @@ public class PwmLogManager
     }
 
     static LocalDBLogger initLocalDBLogger(
-            final LocalDB pwmDB,
-            final PwmApplication pwmApplication
+            final LocalDB localDB,
+            final PwmApplication pwmApplication,
+            final PwmLogLevel level
     )
     {
         try
         {
             final LocalDBLoggerSettings settings = LocalDBLoggerSettings.fromConfiguration( pwmApplication.getConfig() );
-            return new LocalDBLogger( pwmApplication, pwmDB, settings );
+            return new LocalDBLogger( pwmApplication, localDB, level, settings );
         }
         catch ( final LocalDBException e )
         {

server/src/main/java/password/pwm/util/logging/PwmLogMessage.java

@@ -0,0 +1,116 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.util.logging;
+
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.classic.spi.ThrowableProxy;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Value;
+import password.pwm.bean.SessionLabel;
+import password.pwm.util.java.LazySupplier;
+import password.pwm.util.java.TimeDuration;
+
+import java.time.Instant;
+import java.util.function.Supplier;
+
+/**
+ * In memory non-intended-for-serialization logging event.  See {@link PwmLogEvent} for a stored version;
+ */
+@Value
+@AllArgsConstructor( access = AccessLevel.PRIVATE )
+public class PwmLogMessage
+{
+    private Instant timestamp;
+    private String topic;
+    private PwmLogLevel level;
+    private SessionLabel sessionLabel;
+    private Supplier<? extends CharSequence> message;
+    private TimeDuration duration;
+    private Throwable throwable;
+
+    private final Supplier<String> enhancedMessage = LazySupplier.create( () -> PwmLogMessage.createEnhancedMessage( this ) );
+
+    public static PwmLogMessage create(
+            final Instant timestamp,
+            final String topic,
+            final PwmLogLevel level,
+            final SessionLabel sessionLabel,
+            final Supplier<? extends CharSequence> message,
+            final TimeDuration duration,
+            final Throwable throwable
+    )
+    {
+        return new PwmLogMessage( timestamp, topic, level, sessionLabel, message, duration, throwable );
+    }
+
+    String messageToString()
+    {
+        return message == null || message.get() == null ? "" : message.get().toString();
+    }
+
+    PwmLogEvent toLogEvent()
+    {
+        return PwmLogEvent.createPwmLogEvent(
+                timestamp,
+                topic,
+                messageToString(),
+                sessionLabel,
+                throwable,
+                level,
+                duration == null ? null : duration.asDuration() );
+    }
+
+    String getEnhancedMessage()
+    {
+        return enhancedMessage.get();
+    }
+
+    private static String createEnhancedMessage( final PwmLogMessage pwmLogMessage )
+    {
+        return PwmLogUtil.createEnhancedMessage(
+                pwmLogMessage.getSessionLabel(),
+                pwmLogMessage.messageToString(),
+                pwmLogMessage.getThrowable(),
+                pwmLogMessage.getDuration() );
+    }
+
+    static PwmLogMessage fromLogbackEvent( final ILoggingEvent event  )
+    {
+        final Supplier<? extends CharSequence> message = ( Supplier<CharSequence> ) event::getMessage;
+        final Throwable throwableInformation = event.getThrowableProxy() instanceof ThrowableProxy
+                ? ( ( ThrowableProxy ) event.getThrowableProxy() ).getThrowable()
+                : null;
+        final PwmLogLevel level = PwmLogLevel.fromLogbackLevel( event.getLevel() );
+        final Instant timeStamp = Instant.ofEpochMilli( event.getTimeStamp() );
+        final String sourceLogger = event.getLoggerName();
+
+        return PwmLogMessage.create(
+                timeStamp,
+                sourceLogger,
+                level,
+                null,
+                message,
+                TimeDuration.ZERO,
+                throwableInformation
+        );
+    }
+}

server/src/main/java/password/pwm/util/logging/PwmLogSettings.java

@@ -0,0 +1,112 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.util.logging;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.Value;
+import password.pwm.AppProperty;
+import password.pwm.config.AppConfig;
+import password.pwm.config.PwmSetting;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.StringUtil;
+
+import java.util.List;
+import java.util.function.Function;
+
+@Value
+@Builder( access = AccessLevel.PRIVATE )
+public class PwmLogSettings
+{
+    @Builder.Default
+    private PwmLogLevel localDbLevel = PwmLogLevel.TRACE;
+
+    @Builder.Default
+    private PwmLogLevel stdoutLevel = PwmLogLevel.TRACE;
+
+    @Builder.Default
+    private PwmLogLevel fileLevel = PwmLogLevel.TRACE;
+
+    @Builder.Default
+    private boolean healthLoggingEnabled = false;
+
+    @Builder.Default
+    private boolean runtimeLoggingEnabled = false;
+
+    @Builder.Default
+    private LogOutputMode logOutputMode = LogOutputMode.traditional;
+
+    @Builder.Default
+    private List<String> loggingPackages = List.of( "ROOT" );
+
+    enum LogOutputMode
+    {
+        traditional( new PwmLogUtil.TraditionalMsgFunction() ),
+        json( new PwmLogUtil.JsonMsgFunction() ),;
+
+        private final Function<PwmLogMessage, String> messageSupplier;
+
+        LogOutputMode( final Function<PwmLogMessage, String> messageSupplier )
+        {
+            this.messageSupplier = messageSupplier;
+        }
+
+        public Function<PwmLogMessage, String> getMessageSupplier()
+        {
+            return messageSupplier;
+        }
+    }
+
+    public static PwmLogSettings defaultSettings()
+    {
+        return PwmLogSettings.builder().build();
+    }
+
+    public static PwmLogSettings fromAppConfig( final AppConfig appConfig )
+    {
+        final LogOutputMode logOutputMode =
+                JavaHelper.readEnumFromString( LogOutputMode.class, appConfig.readAppProperty( AppProperty.LOGGING_OUTPUT_MODE ) )
+                        .orElse( LogOutputMode.traditional );
+
+        final boolean healthLoggingEnabled = Boolean.parseBoolean( appConfig.readAppProperty( AppProperty.LOGGING_OUTPUT_HEALTHCHECK ) );
+
+        final boolean runtimeLoggingEnabled = Boolean.parseBoolean( appConfig.readAppProperty( AppProperty.LOGGING_OUTPUT_RUNTIME ) );
+
+        final List<String> loggingPackages = List.copyOf(
+                StringUtil.splitAndTrim(
+                        appConfig.readAppProperty( AppProperty.LOGGING_PACKAGE_LIST ), "," ) );
+
+        return PwmLogSettings.builder()
+                .localDbLevel( appConfig.readSettingAsEnum( PwmSetting.EVENTS_LOCALDB_LOG_LEVEL, PwmLogLevel.class ) )
+                .fileLevel( appConfig.readSettingAsEnum( PwmSetting.EVENTS_FILE_LEVEL, PwmLogLevel.class ) )
+                .stdoutLevel( appConfig.readSettingAsEnum( PwmSetting.EVENTS_JAVA_STDOUT_LEVEL, PwmLogLevel.class ) )
+                .healthLoggingEnabled( healthLoggingEnabled )
+                .runtimeLoggingEnabled( runtimeLoggingEnabled )
+                .logOutputMode( logOutputMode )
+                .loggingPackages( loggingPackages )
+                .build();
+    }
+
+    PwmLogLevel calculateLowestLevel()
+    {
+        return PwmLogLevel.lowestLevel( List.of( localDbLevel, stdoutLevel, fileLevel ) );
+    }
+}

server/src/main/java/password/pwm/util/logging/PwmLogUtil.java

@@ -0,0 +1,254 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.util.logging;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.joran.JoranConfigurator;
+import ch.qos.logback.core.joran.spi.JoranException;
+import ch.qos.logback.core.util.StatusPrinter;
+import lombok.Builder;
+import lombok.Value;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import password.pwm.PwmApplication;
+import password.pwm.PwmConstants;
+import password.pwm.bean.LoginInfoBean;
+import password.pwm.bean.SessionLabel;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.svc.event.AuditEvent;
+import password.pwm.svc.event.AuditServiceClient;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.json.JsonFactory;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+class PwmLogUtil
+{
+    static void pushMessageToSlf4j( final Logger slf4jLogger, final PwmLogMessage logMessage )
+    {
+        if ( !slf4jLogger.isEnabledForLevel( logMessage.getLevel().getSlf4jLevel() ) )
+        {
+            return;
+        }
+
+        final PwmLogSettings.LogOutputMode logOutputMode = PwmLogManager.getPwmLogSettings().getLogOutputMode();
+        final Supplier<String> messageSupplier = () -> logOutputMode.getMessageSupplier().apply( logMessage );
+
+        slf4jLogger.makeLoggingEventBuilder( logMessage.getLevel().getSlf4jLevel() )
+                .setCause( logMessage.getThrowable() )
+                .addMarker( PwmLogbackMarker.singleton() )
+                .log( messageSupplier );
+    }
+
+    static void captureFilteredLogEventsToAudit(
+            final PwmApplication pwmApplication,
+            final PwmLogMessage logEvent
+    )
+    {
+        if ( logEvent.getLevel() != PwmLogLevel.FATAL )
+        {
+            return;
+        }
+
+        final boolean ignoreEvent = PwmError.auditIgnoredErrors().stream()
+                .anyMatch( error -> logEvent.getEnhancedMessage().contains( String.valueOf( error.getErrorCode() ) ) );
+
+        if ( !ignoreEvent )
+        {
+            final LogToAuditMessageInfo.LogToAuditMessageInfoBuilder messageInfoBuilder = LogToAuditMessageInfo.builder()
+                    .level( logEvent.getLevel().toString() )
+                    .errorMsg( logEvent.getEnhancedMessage() )
+                    .topic( logEvent.getTopic() );
+
+            final SessionLabel sessionLabel = logEvent.getSessionLabel();
+            if ( sessionLabel != null )
+            {
+
+                messageInfoBuilder.actor( sessionLabel.getUsername() );
+                messageInfoBuilder.source( sessionLabel.getSourceAddress() );
+            }
+
+            final LogToAuditMessageInfo messageInfo = messageInfoBuilder.build();
+            final String messageInfoStr = JsonFactory.get().serialize( messageInfo );
+            AuditServiceClient.submitSystemEvent( pwmApplication, SessionLabel.SYSTEM_LABEL, AuditEvent.FATAL_EVENT, messageInfoStr );
+        }
+    }
+
+    /**
+     * Init logback system directly from a known XML file.
+     *
+     * @param file file to load XML config from
+     * @return true if successfully loaded and initialized loggers
+     * @see <a href=" https://logback.qos.ch/manual/configuration.html#joranDirectly">Logback Docs</a>
+     */
+    public static boolean initLogbackFromXmlFile( final File file )
+    {
+        if ( !file.exists() )
+        {
+            return false;
+        }
+
+        final LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
+
+        try
+        {
+            final JoranConfigurator configurator = new JoranConfigurator();
+
+            configurator.setContext( context );
+
+            context.reset();
+            configurator.doConfigure( file );
+
+            return true;
+        }
+        catch ( final JoranException je )
+        {
+            context.reset();
+            // StatusPrinter will handle this
+        }
+
+        StatusPrinter.printInCaseOfErrorsOrWarnings( context );
+
+        return false;
+    }
+
+    static boolean ignorableLogEvent( final PwmLogMessage pwmLogMessage )
+    {
+        final SessionLabel sessionLabel = pwmLogMessage.getSessionLabel();
+
+        if ( sessionLabel != null )
+        {
+            if ( sessionLabel.isRuntime() )
+            {
+                return !PwmLogManager.getPwmLogSettings().isRuntimeLoggingEnabled();
+            }
+
+            if ( sessionLabel.isHealth() )
+            {
+                return !PwmLogManager.getPwmLogSettings().isHealthLoggingEnabled();
+            }
+        }
+
+        return false;
+    }
+
+    public static String removeUserDataFromString( final LoginInfoBean loginInfoBean, final String input )
+            throws PwmUnrecoverableException
+    {
+        if ( input == null || loginInfoBean == null )
+        {
+            return input;
+        }
+
+        String returnString = input;
+        if ( loginInfoBean.getUserCurrentPassword() != null )
+        {
+            final String pwdStringValue = loginInfoBean.getUserCurrentPassword().getStringValue();
+            if ( pwdStringValue != null && !pwdStringValue.isEmpty() && returnString.contains( pwdStringValue ) )
+            {
+                returnString = returnString.replace( pwdStringValue, PwmConstants.LOG_REMOVED_VALUE_REPLACEMENT );
+            }
+        }
+
+        return returnString;
+    }
+
+    static String createEnhancedMessage(
+            final SessionLabel sessionLabel,
+            final String message,
+            final Throwable throwable,
+            final TimeDuration duration
+    )
+    {
+        final StringBuilder output = new StringBuilder();
+        if ( sessionLabel != null )
+        {
+            output.append( sessionLabel.toDebugLabel() );
+        }
+
+        output.append( message );
+
+        final String srcAddrString = sessionLabel == null ? null : sessionLabel.getSourceAddress();
+        if ( StringUtil.notEmpty( srcAddrString ) )
+        {
+            final String srcStr = " [" + srcAddrString + "]";
+
+            final int firstCR = output.indexOf( "\n" );
+            if ( firstCR == -1 )
+            {
+                output.append( srcStr );
+            }
+            else
+            {
+                output.insert( firstCR, srcStr );
+            }
+        }
+
+        if ( throwable != null )
+        {
+            output.append( JavaHelper.throwableToString( throwable ) );
+        }
+
+        if ( duration != null )
+        {
+            output.append( " (" ).append( duration.asCompactString() ).append( ")" );
+        }
+
+        return output.toString();
+    }
+
+
+    @Value
+    @Builder
+    private static class LogToAuditMessageInfo implements Serializable
+    {
+        private final String level;
+        private final String actor;
+        private final String source;
+        private final String topic;
+        private final String errorMsg;
+    }
+
+    static class TraditionalMsgFunction implements Function<PwmLogMessage, String>
+    {
+        @Override
+        public String apply( final PwmLogMessage pwmLogMessage )
+        {
+            return pwmLogMessage.getEnhancedMessage();
+        }
+    }
+
+    static class JsonMsgFunction implements Function<PwmLogMessage, String>
+    {
+        @Override
+        public String apply( final PwmLogMessage pwmLogMessage )
+        {
+            return pwmLogMessage.toLogEvent().toEncodedString();
+        }
+    }
+
+}

server/src/main/java/password/pwm/util/logging/PwmLogbackMarker.java

@@ -0,0 +1,103 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.util.logging;
+
+import org.slf4j.Marker;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+/**
+ * Simple marker instance used to indicate an event was generated by a PwmLogger instance.
+ */
+public class PwmLogbackMarker implements Marker
+{
+    private static final PwmLogbackMarker SINGLETON = new PwmLogbackMarker();
+
+    static Marker singleton()
+    {
+        return SINGLETON;
+    }
+
+    private PwmLogbackMarker()
+    {
+    }
+
+    @Override
+    public String getName()
+    {
+        return PwmLogbackMarker.class.getName();
+    }
+
+    @Override
+    public void add( final Marker reference )
+    {
+
+    }
+
+    @Override
+    public boolean remove( final Marker reference )
+    {
+        return false;
+    }
+
+    @Override
+    public boolean hasChildren()
+    {
+        return false;
+    }
+
+    @Override
+    public boolean hasReferences()
+    {
+        return false;
+    }
+
+    @Override
+    public Iterator<Marker> iterator()
+    {
+        return Collections.emptyIterator();
+    }
+
+    @Override
+    public boolean contains( final Marker other )
+    {
+        return false;
+    }
+
+    @Override
+    public boolean contains( final String name )
+    {
+        return false;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return 0;
+    }
+
+    @Override
+    public boolean equals( final Object obj )
+    {
+        return obj instanceof PwmLogbackMarker;
+    }
+}

server/src/main/java/password/pwm/util/logging/PwmLogbackPattern.java

@@ -0,0 +1,83 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.util.logging;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.LayoutBase;
+import password.pwm.util.java.StringUtil;
+
+import java.time.Instant;
+
+/**
+ * Logback output pattern to produce output similar to original log4j1 PWM logging
+ * output.
+ */
+public class PwmLogbackPattern extends LayoutBase<ILoggingEvent>
+{
+    public String doLayout( final ILoggingEvent event )
+    {
+        if ( event == null )
+        {
+            return "[null logback event]";
+        }
+
+        return StringUtil.toIsoDate( Instant.ofEpochMilli( event.getTimeStamp() ) )
+                + ", "
+                + printLevel( event.getLevel() )
+                + ", "
+                + printLoggerName( event.getLoggerName() )
+                + ", "
+                + event.getFormattedMessage()
+                + "\n";
+    }
+
+    private static String printLevel( final Level pwmLogLevel )
+    {
+        return StringUtil.padRight( pwmLogLevel.toString(), 5, ' ' );
+    }
+
+    /**
+     * Creates output similar to "c{2}" pattern from legacy Log4j1 which was
+     * original logging API used in PWM.
+     * @param loggerName name of logger (typically fully-qualified class and package name)
+     * @return formatted class name with top level package name.
+     */
+    private static String printLoggerName( final String loggerName )
+    {
+        if ( StringUtil.isEmpty( loggerName ) )
+        {
+            return "";
+        }
+
+        final int firstDot = loggerName.lastIndexOf( '.' );
+        if ( firstDot > 0 )
+        {
+            final int secondDot = loggerName.lastIndexOf( '.', firstDot - 1 );
+            if ( secondDot > -1 )
+            {
+                return loggerName.substring( secondDot + 1 );
+            }
+        }
+
+        return loggerName;
+    }
+}

server/src/main/java/password/pwm/util/logging/PwmLogger.java

@@ -20,27 +20,18 @@
 
 package password.pwm.util.logging;
 
-import org.apache.log4j.Logger;
-import org.apache.log4j.RollingFileAppender;
-import org.apache.log4j.varia.NullAppender;
+import org.slf4j.ILoggerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
-import password.pwm.bean.LoginInfoBean;
 import password.pwm.bean.SessionLabel;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.http.PwmRequest;
-import password.pwm.svc.event.AuditEvent;
-import password.pwm.svc.event.AuditServiceClient;
-import password.pwm.util.java.MiscUtil;
-import password.pwm.util.json.JsonFactory;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 
-import java.io.IOException;
 import java.time.Instant;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.function.Supplier;
 
 /**
@@ -48,41 +39,10 @@ import java.util.function.Supplier;
  */
 public class PwmLogger
 {
-    private static LocalDBLogger localDBLogger;
-    private static PwmLogLevel minimumDbLogLevel;
-    private static PwmApplication pwmApplication;
-    private static RollingFileAppender fileAppender;
-    private static boolean initialized;
-
     private final String name;
-    private final org.apache.log4j.Logger log4jLogger;
+    private final Logger logbackLogger;
     private final boolean localDBDisabled;
 
-    public static void markInitialized( )
-    {
-        initialized = true;
-    }
-
-    static void setPwmApplication( final PwmApplication pwmApplication )
-    {
-        PwmLogger.pwmApplication = pwmApplication;
-        if ( pwmApplication != null )
-        {
-            initialized = true;
-        }
-    }
-
-    static void setLocalDBLogger( final PwmLogLevel minimumDbLogLevel, final LocalDBLogger localDBLogger )
-    {
-        PwmLogger.minimumDbLogLevel = minimumDbLogLevel;
-        PwmLogger.localDBLogger = localDBLogger;
-    }
-
-    static void setFileAppender( final RollingFileAppender rollingFileAppender )
-    {
-        PwmLogger.fileAppender = rollingFileAppender;
-    }
-
     public static PwmLogger forClass( final Class<?> className )
     {
         return new PwmLogger( className.getName(), false );
@@ -110,7 +70,9 @@ public class PwmLogger
     {
         this.name = name;
         this.localDBDisabled = localDBDisabled;
-        log4jLogger = org.apache.log4j.Logger.getLogger( name );
+
+        final ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory();
+        logbackLogger = loggerFactory.getLogger( name );
     }
 
     public String getName( )
@@ -118,231 +80,70 @@ public class PwmLogger
         return name;
     }
 
-    private void doPwmRequestLogEvent(
-            final PwmLogLevel level,
-            final PwmRequest pwmRequest,
-            final Supplier<CharSequence> message,
-            final Throwable e,
-            final TimeDuration timeDuration
-    )
-    {
-        if ( !isEnabled( level ) )
-        {
-            return;
-        }
-
-        final SessionLabel sessionLabel = pwmRequest != null ? pwmRequest.getLabel() : null;
-
-        Supplier<CharSequence> cleanedMessage = message;
-
-        if ( pwmRequest != null && message != null )
-        {
-            try
-            {
-                final CharSequence cleanedString = pwmRequest.hasSession()
-                        ? PwmLogger.removeUserDataFromString( pwmRequest.getPwmSession().getLoginInfoBean(), message.get() )
-                        : message.get();
-                final CharSequence printableString = StringUtil.cleanNonPrintableCharacters( cleanedString );
-                cleanedMessage = () -> printableString;
-            }
-            catch ( final PwmUnrecoverableException e1 )
-            {
-                /* can't be logged */
-            }
-        }
-
-        doLogEvent( level, sessionLabel, cleanedMessage, e, timeDuration );
-    }
-
-    private void doLogEvent(
-            final PwmLogLevel level,
-            final SessionLabel sessionLabel,
-            final Supplier<CharSequence> message,
-            final Throwable e
-    )
+    public void log( final PwmLogLevel level, final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message )
     {
-        doLogEvent( level, sessionLabel, message, e, null );
+        doLogEvent( level, sessionLabel, message, null, null );
     }
 
-    private void doLogEvent(
-            final PwmLogLevel level,
-            final SessionLabel sessionLabel,
-            final Supplier<CharSequence> message,
-            final Throwable e,
-            final TimeDuration timeDuration
-    )
+    public void trace( final Supplier<? extends CharSequence> message )
     {
-        if ( !isEnabled( level ) )
-        {
-            return;
-        }
-
-        final PwmLogLevel effectiveLevel = level == null ? PwmLogLevel.TRACE : level;
-        final String topic = log4jLogger.getName();
-        final CharSequence effectiveMessage = formatEffectiveMessage( message, timeDuration );
-        final PwmLogEvent logEvent = PwmLogEvent.createPwmLogEvent( Instant.now(), topic, effectiveMessage.toString(), sessionLabel,
-                e, effectiveLevel );
-        doLogEvent( logEvent );
+        doLogEvent( PwmLogLevel.TRACE, null, message, null, null );
     }
 
-    private CharSequence formatEffectiveMessage( final Supplier<CharSequence> message, final TimeDuration timeDuration )
-    {
-        final CharSequence output = message == null
-                ? ""
-                : message.get();
-
-        if ( timeDuration != null )
-        {
-            return output + " (" + timeDuration.asCompactString() + ")";
-        }
-
-        return output;
-    }
-
-    private void doLogEvent( final PwmLogEvent logEvent )
-    {
-        pushMessageToLog4j( logEvent );
-
-        try
-        {
-
-            if ( !localDBDisabled && localDBLogger != null && minimumDbLogLevel != null )
-            {
-                if ( logEvent.getLevel().compareTo( minimumDbLogLevel ) >= 0 )
-                {
-                    localDBLogger.writeEvent( logEvent );
-                }
-            }
-
-            if ( logEvent.getLevel() == PwmLogLevel.FATAL )
-            {
-                if ( !logEvent.getMessage().contains( "5039" ) )
-                {
-                    final Map<String, String> messageInfo = new HashMap<>();
-                    messageInfo.put( "level", logEvent.getLevel() == null ? "null" : logEvent.getLevel().toString() );
-                    messageInfo.put( "actor", logEvent.getUsername() );
-                    messageInfo.put( "source", logEvent.getSourceAddress() );
-                    messageInfo.put( "topic", logEvent.getTopic() );
-                    messageInfo.put( "errorMessage", logEvent.getMessage() );
-
-                    final String messageInfoStr = JsonFactory.get().serializeMap( messageInfo );
-                    AuditServiceClient.submitSystemEvent( pwmApplication, SessionLabel.SYSTEM_LABEL, AuditEvent.FATAL_EVENT, messageInfoStr );
-                }
-            }
-        }
-        catch ( final Exception e2 )
-        {
-            //nothing can be done about it now;
-        }
-    }
-
-    private void pushMessageToLog4j( final PwmLogEvent logEvent )
-    {
-        final String wrappedMessage = logEvent.getEnhancedMessage();
-        final Throwable throwable = logEvent.getLoggedThrowable() == null ? null : logEvent.getLoggedThrowable().toThrowable();
-        final PwmLogLevel level = logEvent.getLevel();
-
-        if ( initialized )
-        {
-            switch ( level )
-            {
-                case DEBUG:
-                    log4jLogger.debug( wrappedMessage, throwable );
-                    break;
-                case ERROR:
-                    log4jLogger.error( wrappedMessage, throwable );
-                    break;
-                case INFO:
-                    log4jLogger.info( wrappedMessage, throwable );
-                    break;
-                case TRACE:
-                    log4jLogger.trace( wrappedMessage, throwable );
-                    break;
-                case WARN:
-                    log4jLogger.warn( wrappedMessage, throwable );
-                    break;
-                case FATAL:
-                    log4jLogger.fatal( wrappedMessage, throwable );
-                    break;
-
-                default:
-                    MiscUtil.unhandledSwitchStatement( level );
-            }
-        }
-        else
-        {
-            System.err.println( logEvent.toLogString() );
-        }
-    }
-
-    private static Supplier<CharSequence> convertErrorInformation( final ErrorInformation info )
-    {
-        return info::toDebugStr;
-    }
-
-    public void log( final PwmLogLevel level, final SessionLabel sessionLabel, final Supplier<CharSequence> message )
-    {
-        doLogEvent( level, sessionLabel, message, null );
-    }
-
-    public void trace( final Supplier<CharSequence> message )
-    {
-        doLogEvent( PwmLogLevel.TRACE, null, message, null );
-    }
-
-    public void trace( final Supplier<CharSequence> message, final TimeDuration timeDuration )
+    public void trace( final Supplier<? extends CharSequence> message, final TimeDuration timeDuration )
     {
         doLogEvent( PwmLogLevel.TRACE, null, message, null, timeDuration );
     }
 
-    public void trace( final PwmRequest pwmRequest, final Supplier<CharSequence> message )
+    public void trace( final PwmRequest pwmRequest, final Supplier<? extends CharSequence> message )
     {
         doPwmRequestLogEvent( PwmLogLevel.TRACE, pwmRequest, message, null, null );
     }
 
-    public void trace( final PwmRequest pwmRequest, final Supplier<CharSequence> message, final TimeDuration timeDuration )
+    public void trace( final PwmRequest pwmRequest, final Supplier<? extends CharSequence> message, final TimeDuration timeDuration )
     {
         doPwmRequestLogEvent( PwmLogLevel.TRACE, pwmRequest, message, null, timeDuration );
     }
 
-    public void trace( final SessionLabel sessionLabel, final Supplier<CharSequence> message )
+    public void trace( final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message )
     {
-        doLogEvent( PwmLogLevel.TRACE, sessionLabel, message, null );
+        doLogEvent( PwmLogLevel.TRACE, sessionLabel, message, null, null );
     }
 
-    public void traceDevDebug( final SessionLabel sessionLabel, final Supplier<CharSequence> message )
+    public void traceDevDebug( final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message )
     {
+        final PwmApplication pwmApplication = PwmLogManager.getPwmApplication();
         if ( pwmApplication == null || !pwmApplication.getConfig().isDevDebugMode() )
         {
             return;
         }
 
-        doLogEvent( PwmLogLevel.TRACE, sessionLabel, message, null );
+        doLogEvent( PwmLogLevel.TRACE, sessionLabel, message, null, null );
     }
 
-    public void trace( final SessionLabel sessionLabel, final Supplier<CharSequence> message, final TimeDuration timeDuration )
+    public void trace( final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message, final TimeDuration timeDuration )
     {
         doLogEvent( PwmLogLevel.TRACE, sessionLabel, message, null, timeDuration );
     }
 
-    public void trace( final Supplier<CharSequence> message, final Throwable exception )
+    public void trace( final Supplier<? extends CharSequence> message, final Throwable exception )
     {
-        doLogEvent( PwmLogLevel.TRACE, null, message, exception );
+        doLogEvent( PwmLogLevel.TRACE, null, message, exception, null );
     }
 
-    public void debug( final Supplier<CharSequence> message )
+    public void debug( final Supplier<? extends CharSequence> message )
     {
-        doLogEvent( PwmLogLevel.DEBUG, null, message, null );
+        doLogEvent( PwmLogLevel.DEBUG, null, message, null, null );
     }
 
-    public void debug( final PwmRequest pwmRequest, final Supplier<CharSequence> message )
+    public void debug( final Supplier<? extends CharSequence> message, final TimeDuration timeDuration )
     {
-        doPwmRequestLogEvent( PwmLogLevel.DEBUG, pwmRequest, message, null, null );
+        doLogEvent( PwmLogLevel.DEBUG, null, message, null, timeDuration );
     }
 
-    public void debug( final PwmRequest pwmRequest, final Supplier<CharSequence> message, final Throwable exception )
+    public void debug( final PwmRequest pwmRequest, final Supplier<? extends CharSequence> message )
     {
-        doPwmRequestLogEvent( PwmLogLevel.DEBUG, pwmRequest, message, exception, null );
+        doPwmRequestLogEvent( PwmLogLevel.DEBUG, pwmRequest, message, null, null );
     }
 
     public void debug( final PwmRequest pwmRequest, final ErrorInformation errorInformation )
@@ -350,32 +151,37 @@ public class PwmLogger
         doPwmRequestLogEvent( PwmLogLevel.DEBUG, pwmRequest, convertErrorInformation( errorInformation ), null, null );
     }
 
-    public void debug( final SessionLabel sessionLabel, final Supplier<CharSequence> message )
+    public void debug( final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message )
     {
-        doLogEvent( PwmLogLevel.DEBUG, sessionLabel, message, null );
+        doLogEvent( PwmLogLevel.DEBUG, sessionLabel, message, null, null );
     }
 
-    public void debug( final SessionLabel sessionLabel, final Supplier<CharSequence> message, final TimeDuration timeDuration )
+    public void debug( final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message, final TimeDuration timeDuration )
     {
         doLogEvent( PwmLogLevel.DEBUG, sessionLabel, message, null, timeDuration );
     }
 
     public void debug( final SessionLabel sessionLabel, final ErrorInformation errorInformation )
     {
-        doLogEvent( PwmLogLevel.DEBUG, sessionLabel, convertErrorInformation( errorInformation ), null );
+        doLogEvent( PwmLogLevel.DEBUG, sessionLabel, convertErrorInformation( errorInformation ), null, null );
+    }
+
+    public void debug( final Supplier<? extends CharSequence> message, final Throwable exception )
+    {
+        doPwmRequestLogEvent( PwmLogLevel.DEBUG, null, message, exception, null );
     }
 
-    public void info( final Supplier<CharSequence> message )
+    public void info( final Supplier<? extends CharSequence> message )
     {
-        doLogEvent( PwmLogLevel.INFO, null, message, null );
+        doLogEvent( PwmLogLevel.INFO, null, message, null, null );
     }
 
-    public void info( final Supplier<CharSequence> message, final TimeDuration timeDuration )
+    public void info( final Supplier<? extends CharSequence> message, final TimeDuration timeDuration )
     {
         doLogEvent( PwmLogLevel.INFO, null, message, null, timeDuration );
     }
 
-    public void info( final PwmRequest pwmRequest, final Supplier<CharSequence> message )
+    public void info( final PwmRequest pwmRequest, final Supplier<? extends CharSequence> message )
     {
         doPwmRequestLogEvent( PwmLogLevel.INFO, pwmRequest, message, null, null );
     }
@@ -385,32 +191,32 @@ public class PwmLogger
         doPwmRequestLogEvent( PwmLogLevel.INFO, pwmRequest, convertErrorInformation( errorInformation ), null, null );
     }
 
-    public void info( final SessionLabel sessionLabel, final Supplier<CharSequence> message )
+    public void info( final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message )
     {
-        doLogEvent( PwmLogLevel.INFO, sessionLabel, message, null );
+        doLogEvent( PwmLogLevel.INFO, sessionLabel, message, null, null );
     }
 
-    public void info( final SessionLabel sessionLabel, final Supplier<CharSequence> message, final TimeDuration timeDuration )
+    public void info( final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message, final TimeDuration timeDuration )
     {
         doLogEvent( PwmLogLevel.INFO, sessionLabel, message, null, timeDuration );
     }
 
-    public void info( final Supplier<CharSequence> message, final Throwable exception )
+    public void info( final Supplier<? extends CharSequence> message, final Throwable exception )
     {
-        doLogEvent( PwmLogLevel.INFO, null, message, exception );
+        doLogEvent( PwmLogLevel.INFO, null, message, exception, null );
     }
 
-    public void error( final Supplier<CharSequence> message )
+    public void error( final Supplier<? extends CharSequence> message )
     {
-        doLogEvent( PwmLogLevel.ERROR, null, message, null );
+        doLogEvent( PwmLogLevel.ERROR, null, message, null, null );
     }
 
-    public void error( final PwmRequest pwmRequest, final Supplier<CharSequence> message, final Throwable exception )
+    public void error( final PwmRequest pwmRequest, final Supplier<? extends CharSequence> message, final Throwable exception )
     {
         doPwmRequestLogEvent( PwmLogLevel.ERROR, pwmRequest, message, exception, null );
     }
 
-    public void error( final PwmRequest pwmRequest, final Supplier<CharSequence> message )
+    public void error( final PwmRequest pwmRequest, final Supplier<? extends CharSequence> message )
     {
         doPwmRequestLogEvent( PwmLogLevel.ERROR, pwmRequest, message, null, null );
     }
@@ -425,191 +231,181 @@ public class PwmLogger
         doPwmRequestLogEvent( PwmLogLevel.ERROR, null, convertErrorInformation( errorInformation ), null, null );
     }
 
-    public void error( final SessionLabel sessionLabel, final Supplier<CharSequence> message )
+    public void error( final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message )
     {
-        doLogEvent( PwmLogLevel.ERROR, sessionLabel, message, null );
+        doLogEvent( PwmLogLevel.ERROR, sessionLabel, message, null, null );
     }
 
-    public void error( final SessionLabel sessionLabel, final Supplier<CharSequence> message, final TimeDuration timeDurationSupplier )
+    public void error( final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message, final TimeDuration timeDurationSupplier )
     {
         doLogEvent( PwmLogLevel.ERROR, sessionLabel, message, null, timeDurationSupplier );
     }
 
     public void error( final SessionLabel sessionLabel, final ErrorInformation errorInformation )
     {
-        doLogEvent( PwmLogLevel.ERROR, sessionLabel, convertErrorInformation( errorInformation ), null );
+        doLogEvent( PwmLogLevel.ERROR, sessionLabel, convertErrorInformation( errorInformation ), null, null );
     }
 
     public void error( final SessionLabel sessionLabel, final ErrorInformation errorInformation, final Throwable exception )
     {
-        doLogEvent( PwmLogLevel.ERROR, sessionLabel, convertErrorInformation( errorInformation ), exception );
+        doLogEvent( PwmLogLevel.ERROR, sessionLabel, convertErrorInformation( errorInformation ), exception, null );
     }
 
-    public void error( final Supplier<CharSequence> message, final Throwable exception )
+    public void error( final Supplier<? extends CharSequence> message, final Throwable exception )
     {
-        doLogEvent( PwmLogLevel.ERROR, null, message, exception );
+        doLogEvent( PwmLogLevel.ERROR, null, message, exception, null );
     }
 
-    public void error( final SessionLabel sessionLabel, final Supplier<CharSequence> message, final Throwable exception )
+    public void error( final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message, final Throwable exception )
     {
-        doLogEvent( PwmLogLevel.ERROR, sessionLabel, message, exception );
+        doLogEvent( PwmLogLevel.ERROR, sessionLabel, message, exception, null );
     }
 
-    public void warn( final Supplier<CharSequence> message )
+    public void warn( final Supplier<? extends CharSequence> message )
     {
-        doLogEvent( PwmLogLevel.WARN, null, message, null );
+        doLogEvent( PwmLogLevel.WARN, null, message, null, null );
     }
 
-    public void warn( final PwmRequest pwmRequest, final Supplier<CharSequence> message )
+    public void warn( final PwmRequest pwmRequest, final Supplier<? extends CharSequence> message )
     {
         doPwmRequestLogEvent( PwmLogLevel.WARN, pwmRequest, message, null, null );
     }
 
-    public void warn( final SessionLabel sessionLabel, final Supplier<CharSequence> message, final Throwable exception )
+    public void warn( final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message, final Throwable exception )
     {
-        doLogEvent( PwmLogLevel.WARN, sessionLabel, message, exception );
+        doLogEvent( PwmLogLevel.WARN, sessionLabel, message, exception, null );
     }
 
-    public void warn( final PwmRequest pwmRequest, final Supplier<CharSequence> message, final Throwable exception )
+    public void warn( final PwmRequest pwmRequest, final Supplier<? extends CharSequence> message, final Throwable exception )
     {
         doPwmRequestLogEvent( PwmLogLevel.WARN, pwmRequest, message, exception, null );
     }
 
     public void warn( final SessionLabel sessionLabel, final ErrorInformation errorInformation, final Throwable exception )
     {
-        doLogEvent( PwmLogLevel.ERROR, sessionLabel, convertErrorInformation( errorInformation ), exception );
+        doLogEvent( PwmLogLevel.ERROR, sessionLabel, convertErrorInformation( errorInformation ), exception, null );
     }
 
-    public void warn( final SessionLabel sessionLabel, final Supplier<CharSequence> message )
+    public void warn( final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message )
     {
-        doLogEvent( PwmLogLevel.WARN, sessionLabel, message, null );
+        doLogEvent( PwmLogLevel.WARN, sessionLabel, message, null, null );
     }
 
-    public void warn( final Supplier<CharSequence> message, final Throwable exception )
+    public void warn( final Supplier<? extends CharSequence> message, final Throwable exception )
     {
-        doLogEvent( PwmLogLevel.WARN, null, message, exception );
+        doLogEvent( PwmLogLevel.WARN, null, message, exception, null );
     }
 
-    public void fatal( final Supplier<CharSequence> message )
+    public void fatal( final Supplier<? extends CharSequence> message )
     {
-        doLogEvent( PwmLogLevel.FATAL, null, message, null );
+        doLogEvent( PwmLogLevel.FATAL, null, message, null, null );
     }
 
-    public void fatal( final SessionLabel sessionLabel, final Supplier<CharSequence> message )
+    public void fatal( final SessionLabel sessionLabel, final Supplier<? extends CharSequence> message )
     {
-        doLogEvent( PwmLogLevel.FATAL, sessionLabel, message, null );
+        doLogEvent( PwmLogLevel.FATAL, sessionLabel, message, null, null );
     }
 
-    public void fatal( final Supplier<CharSequence> message, final Throwable exception )
+    public void fatal( final Supplier<? extends CharSequence> message, final Throwable exception )
     {
-        doLogEvent( PwmLogLevel.FATAL, null, message, exception );
+        doLogEvent( PwmLogLevel.FATAL, null, message, exception, null );
     }
 
     public Appendable asAppendable( final PwmLogLevel pwmLogLevel, final SessionLabel sessionLabel )
     {
-        return new PwmLoggerAppendable( pwmLogLevel, sessionLabel );
+        return new PwmLoggerAppendable( this, pwmLogLevel, sessionLabel );
     }
 
-    private class PwmLoggerAppendable implements Appendable
+    private void doPwmRequestLogEvent(
+            final PwmLogLevel level,
+            final PwmRequest pwmRequest,
+            final Supplier<? extends CharSequence> message,
+            final Throwable e,
+            final TimeDuration timeDuration
+    )
     {
-        private final PwmLogLevel logLevel;
-        private final SessionLabel sessionLabel;
-
-        private final StringBuilder buffer = new StringBuilder();
-
-        private PwmLoggerAppendable(
-                final PwmLogLevel logLevel,
-                final SessionLabel sessionLabel
-        )
+        if ( !isInterestingLevel( level ) )
         {
-            this.logLevel = logLevel;
-            this.sessionLabel = sessionLabel;
+            return;
         }
 
-        @Override
-        public Appendable append( final CharSequence csq )
-                throws IOException
-        {
+        final SessionLabel sessionLabel = pwmRequest != null ? pwmRequest.getLabel() : null;
 
-            doAppend( csq );
-            return this;
-        }
+        Supplier<? extends CharSequence> cleanedMessage = message;
 
-        @Override
-        public Appendable append(
-                final CharSequence csq,
-                final int start,
-                final int end
-        )
-                throws IOException
+        if ( pwmRequest != null && message != null )
         {
-            doAppend( csq.subSequence( start, end ) );
-            return this;
+            try
+            {
+                final String cleanedString = PwmLogUtil.removeUserDataFromString( pwmRequest.getPwmSession().getLoginInfoBean(), message.get().toString() );
+                final String printableString = StringUtil.cleanNonPrintableCharacters( cleanedString ).toString();
+                cleanedMessage = () -> printableString;
+            }
+            catch ( final PwmUnrecoverableException e1 )
+            {
+                /* can't be logged */
+            }
         }
 
-        @Override
-        public Appendable append( final char c )
-                throws IOException
-        {
-            doAppend( String.valueOf( c ) );
-            return this;
-        }
+        doLogEvent( level, sessionLabel, cleanedMessage, e, timeDuration );
+    }
 
-        private synchronized void doAppend( final CharSequence charSequence )
-        {
-            buffer.append( charSequence );
+    private void doLogEvent(
+            final PwmLogLevel level,
+            final SessionLabel sessionLabel,
+            final Supplier<? extends CharSequence> message,
+            final Throwable e,
+            final TimeDuration timeDuration
+    )
+    {
+        final PwmLogMessage pwmLogMessage = PwmLogMessage.create(
+                Instant.now(),
+                logbackLogger.getName(),
+                level,
+                sessionLabel,
+                message,
+                timeDuration,
+                e );
 
-            int length = buffer.indexOf( "\n" );
-            while ( length > 0 )
-            {
-                final String line = buffer.substring( 0, length );
-                buffer.delete( 0, length + 1 );
-                doLogEvent( logLevel, sessionLabel, () -> line, null );
-                length = buffer.indexOf( "\n" );
-            }
-        }
+        doLogEvent( pwmLogMessage );
     }
 
-    public static CharSequence removeUserDataFromString( final LoginInfoBean loginInfoBean, final CharSequence input )
-            throws PwmUnrecoverableException
+    void doLogEvent( final PwmLogMessage pwmLogMessage )
     {
-        if ( input == null || loginInfoBean == null )
+        if ( PwmLogUtil.ignorableLogEvent( pwmLogMessage ) )
         {
-            return input;
+            return;
         }
 
-        String returnString = input.toString();
-        if ( loginInfoBean.getUserCurrentPassword() != null )
+        PwmLogUtil.pushMessageToSlf4j( logbackLogger, pwmLogMessage );
+
+        try
         {
-            final String pwdStringValue = loginInfoBean.getUserCurrentPassword().getStringValue();
-            if ( pwdStringValue != null && !pwdStringValue.isEmpty() && returnString.contains( pwdStringValue ) )
+            final LocalDBLogger localDBLogger = PwmLogManager.getLocalDbLogger();
+            if ( !localDBDisabled && localDBLogger != null )
             {
-                returnString = returnString.replace( pwdStringValue, PwmConstants.LOG_REMOVED_VALUE_REPLACEMENT );
+                localDBLogger.writeEvent( pwmLogMessage );
             }
-        }
 
-        return returnString;
+            final PwmApplication pwmApplication = PwmLogManager.getPwmApplication();
+            PwmLogUtil.captureFilteredLogEventsToAudit( pwmApplication, pwmLogMessage );
+        }
+        catch ( final Exception e2 )
+        {
+            //nothing can be done about it now;
+        }
     }
 
-    public boolean isEnabled( final PwmLogLevel pwmLogLevel )
+
+    private static Supplier<? extends CharSequence> convertErrorInformation( final ErrorInformation info )
     {
-        return (
-                log4jLogger != null
-                        && log4jLogger.isEnabledFor( pwmLogLevel.getLog4jLevel() )
-        )
-                ||
-                (
-                        localDBLogger != null
-                                && minimumDbLogLevel != null
-                                && minimumDbLogLevel.compareTo( pwmLogLevel ) <= 0
-                );
+        return info::toDebugStr;
     }
 
-    public static void disableAllLogging()
+    public boolean isInterestingLevel( final PwmLogLevel pwmLogLevel )
     {
-        Logger.getRootLogger().removeAllAppenders();
-        Logger.getRootLogger().addAppender( new NullAppender() );
-        PwmLogger.markInitialized();
+        final PwmLogLevel lowestLevelConfigured = PwmLogManager.getLowestLogLevelConfigured();
+        return pwmLogLevel.isGreaterOrSameAs( lowestLevelConfigured );
     }
 }
 

server/src/main/java/password/pwm/util/logging/PwmLoggerAppendable.java

@@ -0,0 +1,97 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.util.logging;
+
+import password.pwm.bean.SessionLabel;
+
+import java.io.IOException;
+import java.time.Instant;
+
+class PwmLoggerAppendable implements Appendable
+{
+    private final PwmLogger pwmLogger;
+    private final PwmLogLevel logLevel;
+    private final SessionLabel sessionLabel;
+
+    private final StringBuilder buffer = new StringBuilder();
+
+    PwmLoggerAppendable( final PwmLogger pwmLogger,
+                         final PwmLogLevel logLevel,
+                         final SessionLabel sessionLabel
+    )
+    {
+        this.pwmLogger = pwmLogger;
+        this.logLevel = logLevel;
+        this.sessionLabel = sessionLabel;
+    }
+
+    @Override
+    public Appendable append( final CharSequence csq )
+            throws IOException
+    {
+
+        doAppend( csq );
+        return this;
+    }
+
+    @Override
+    public Appendable append(
+            final CharSequence csq,
+            final int start,
+            final int end
+    )
+            throws IOException
+    {
+        doAppend( csq.subSequence( start, end ) );
+        return this;
+    }
+
+    @Override
+    public Appendable append( final char c )
+            throws IOException
+    {
+        doAppend( String.valueOf( c ) );
+        return this;
+    }
+
+    private synchronized void doAppend( final CharSequence charSequence )
+    {
+        buffer.append( charSequence );
+
+        final PwmLogMessage pwmLogMessage = PwmLogMessage.create(
+                Instant.now(),
+                pwmLogger.getName(),
+                logLevel,
+                sessionLabel,
+                () -> charSequence,
+                null,
+                null );
+
+        int length = buffer.indexOf( "\n" );
+        while ( length > 0 )
+        {
+            final String line = buffer.substring( 0, length );
+            buffer.delete( 0, +length + 1 );
+            pwmLogger.doLogEvent( pwmLogMessage );
+            length = buffer.indexOf( "\n" );
+        }
+    }
+}

server/src/main/java/password/pwm/util/password/RandomPasswordGenerator.java

@@ -135,7 +135,7 @@ public class RandomPasswordGenerator
         }
         else
         {
-            if ( LOGGER.isEnabled( PwmLogLevel.ERROR ) )
+            if ( LOGGER.isInterestingLevel( PwmLogLevel.ERROR ) )
             {
                 final PwmPasswordRuleValidator pwmPasswordRuleValidator = PwmPasswordRuleValidator.create( sessionLabel, pwmDomain, randomGenPolicy );
                 final int errors = pwmPasswordRuleValidator.internalPwmPolicyValidator( mutatorResult.getPassword(), null, null ).size();

server/src/main/java/password/pwm/ws/server/RestServlet.java

@@ -198,7 +198,7 @@ public abstract class RestServlet extends HttpServlet
     {
         try
         {
-            if ( LOGGER.isEnabled( PwmLogLevel.TRACE ) )
+            if ( LOGGER.isInterestingLevel( PwmLogLevel.TRACE ) )
             {
                 final PwmHttpRequestWrapper httpRequestWrapper = new PwmHttpRequestWrapper( req, pwmApplication.getConfig() );
                 final String debutTxt = httpRequestWrapper.debugHttpRequestToString( null, true );

server/src/main/resources/password/pwm/AppProperty.properties

@@ -230,8 +230,11 @@ macro.ldapAttr.maxLength=100
 logging.cspReport.enable=true
 logging.devOutput.enable=false
 logging.extra.periodicThreadDumpIntervalSeconds=0
-logging.outputConfiguration=true
-logging.pattern=%d{yyyy-MM-dd'T'HH:mm:ss'Z'}, %-5p, %c{2}, %m%n
+logging.logOutputMode=traditional
+logging.output.configuration.enable=true
+logging.output.healthCheck.enable=false
+logging.output.runtime.enable=false
+logging.packageList=password.pwm,com.novell.ldapchai,org.jasig.cas.client
 logging.file.maxSize=10MB
 logging.file.maxRollover=50
 logging.file.path=logs

server/src/test/java/password/pwm/config/stored/ConfigurationCleanerTest.java

@@ -36,7 +36,6 @@ import password.pwm.config.profile.ForgottenPasswordProfile;
 import password.pwm.config.profile.PeopleSearchProfile;
 import password.pwm.config.profile.PwmPasswordPolicy;
 import password.pwm.config.value.data.UserPermission;
-import password.pwm.util.logging.PwmLogger;
 
 import java.io.InputStream;
 import java.util.List;
@@ -87,8 +86,7 @@ public class ConfigurationCleanerTest
 
     @Test
     public void testDeprecatedPublicHealthStatsWebService()
-    {        PwmLogger.disableAllLogging();
-
+    {
         {
             final Set<WebServiceUsage> usages = domainConfig.readSettingAsOptionList( PwmSetting.WEBSERVICES_PUBLIC_ENABLE, WebServiceUsage.class );
             Assertions.assertEquals( 2, usages.size() );

server/src/test/java/password/pwm/util/json/JsonProviderTest.java

@@ -46,6 +46,7 @@ import java.io.IOException;
 import java.io.Serializable;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
+import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Date;
@@ -349,7 +350,8 @@ public class JsonProviderTest
                         "message",
                         SessionLabel.TEST_SESSION_LABEL,
                         throwable,
-                        PwmLogLevel.TRACE
+                        PwmLogLevel.TRACE,
+                        Duration.ZERO
                 );
             }
         }

server/src/test/java/password/pwm/util/localdb/LocalDBLoggerExtendedTest.java

@@ -26,6 +26,7 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 import password.pwm.AppProperty;
+import password.pwm.bean.SessionLabel;
 import password.pwm.config.AppConfig;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.stored.StoredConfigurationFactory;
@@ -37,8 +38,8 @@ import password.pwm.util.java.TimeDuration;
 import password.pwm.util.json.JsonFactory;
 import password.pwm.util.logging.LocalDBLogger;
 import password.pwm.util.logging.LocalDBLoggerSettings;
-import password.pwm.util.logging.PwmLogEvent;
 import password.pwm.util.logging.PwmLogLevel;
+import password.pwm.util.logging.PwmLogMessage;
 import password.pwm.util.secure.PwmRandom;
 
 import java.io.File;
@@ -59,6 +60,7 @@ import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
 
 public class LocalDBLoggerExtendedTest
 {
@@ -83,7 +85,6 @@ public class LocalDBLoggerExtendedTest
     @BeforeEach
     public void setUp() throws Exception
     {
-        TestHelper.setupLogging();
         final File localDBPath = FileSystemUtility.createDirectory( temporaryFolder, "test-localdb-logger-test" );
 
         config = AppConfig.forStoredConfig( StoredConfigurationFactory.newConfig() );
@@ -106,7 +107,7 @@ public class LocalDBLoggerExtendedTest
                     .maxAge( TimeDuration.of( 1, TimeDuration.Unit.MINUTES ) )
                     .flags( Collections.emptySet() )
                     .build();
-            localDBLogger = new LocalDBLogger( null, localDB, settings );
+            localDBLogger = new LocalDBLogger( null, localDB, PwmLogLevel.TRACE, settings );
         }
 
         settings = Settings.builder()
@@ -163,8 +164,8 @@ public class LocalDBLoggerExtendedTest
             final RandomValueMaker randomValueMaker = new RandomValueMaker( settings.valueLength );
             while ( TimeDuration.fromCurrent( startTime ).isShorterThan( settings.testDuration ) )
             {
-                final Collection<PwmLogEvent> events = makeEvents( randomValueMaker );
-                for ( final PwmLogEvent logEvent : events )
+                final Collection<PwmLogMessage> events = makeEvents( randomValueMaker );
+                for ( final PwmLogMessage logEvent : events )
                 {
                     localDBLogger.writeEvent( logEvent );
                     eventRateMeter.markEvents( 1 );
@@ -174,20 +175,21 @@ public class LocalDBLoggerExtendedTest
         }
     }
 
-    private Collection<PwmLogEvent> makeEvents( final RandomValueMaker randomValueMaker )
+    private Collection<PwmLogMessage> makeEvents( final RandomValueMaker randomValueMaker )
     {
         final int count = settings.batchSize;
-        final Collection<PwmLogEvent> events = new ArrayList<>();
+        final Collection<PwmLogMessage> events = new ArrayList<>();
         for ( int i = 0; i < count; i++ )
         {
-            final String description = randomValueMaker.next();
-            final PwmLogEvent event = PwmLogEvent.createPwmLogEvent(
+            final Supplier<? extends CharSequence> description = ( Supplier<CharSequence> ) randomValueMaker::next;
+            final PwmLogMessage event = PwmLogMessage.create(
                     Instant.now(),
                     LocalDBLogger.class.getName(),
+                    PwmLogLevel.TRACE,
+                    SessionLabel.TEST_SESSION_LABEL,
                     description,
-                    null,
-                    null,
-                    PwmLogLevel.TRACE );
+                    TimeDuration.ZERO,
+                    null );
             events.add( event );
         }
 

server/src/test/java/password/pwm/util/localdb/LocalDBStoredQueueExtendedTest.java

@@ -48,8 +48,6 @@ public class LocalDBStoredQueueExtendedTest
     @BeforeAll
     public static void setUp() throws Exception
     {
-
-        TestHelper.setupLogging();
         final File fileLocation = FileSystemUtility.createDirectory( temporaryFolder, "localdb-storedqueue-test" );
         localDB = LocalDBFactory.getInstance( fileLocation, false, null, null );
         storedQueue = LocalDBStoredQueue.createLocalDBStoredQueue( localDB, LocalDB.DB.TEMP, ENABLE_DEBUG_OUTPUT );

server/src/test/java/password/pwm/util/localdb/TestHelper.java

@@ -20,15 +20,8 @@
 
 package password.pwm.util.localdb;
 
-import com.novell.ldapchai.ChaiUser;
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.Layout;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
 import password.pwm.PwmApplication;
 import password.pwm.PwmApplicationMode;
-import password.pwm.PwmDomain;
 import password.pwm.PwmEnvironment;
 import password.pwm.bean.DomainID;
 import password.pwm.config.AppConfig;
@@ -45,21 +38,6 @@ import java.io.File;
 
 public class TestHelper
 {
-    public static void setupLogging()
-    {
-        final String pwmPackageName = PwmDomain.class.getPackage().getName();
-        final Logger pwmPackageLogger = Logger.getLogger( pwmPackageName );
-        final String chaiPackageName = ChaiUser.class.getPackage().getName();
-        final Logger chaiPackageLogger = Logger.getLogger( chaiPackageName );
-        final Layout patternLayout = new PatternLayout( "%d{yyyy-MM-dd HH:mm:ss}, %-5p, %c{2}, %m%n" );
-        final ConsoleAppender consoleAppender = new ConsoleAppender( patternLayout );
-        final Level level = Level.TRACE;
-        pwmPackageLogger.addAppender( consoleAppender );
-        pwmPackageLogger.setLevel( level );
-        chaiPackageLogger.addAppender( consoleAppender );
-        chaiPackageLogger.setLevel( level );
-    }
-
     public static PwmApplication makeTestPwmApplication( final File tempFolder )
             throws PwmUnrecoverableException
     {
@@ -74,7 +52,6 @@ public class TestHelper
     public static PwmApplication makeTestPwmApplication( final File tempFolder, final AppConfig appConfig )
             throws PwmUnrecoverableException
     {
-        Logger.getRootLogger().setLevel( Level.OFF );
         final PwmEnvironment pwmEnvironment = PwmEnvironment.builder()
                 .config( appConfig )
                 .applicationPath( tempFolder )

server/src/test/java/password/pwm/util/logging/PwmLogLevelTest.java

@@ -0,0 +1,46 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.util.logging;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+class PwmLogLevelTest
+{
+
+    @Test
+    void isLessOrSameAs()
+    {
+        Assertions.assertFalse( PwmLogLevel.DEBUG.isGreaterOrSameAs( PwmLogLevel.ERROR ) );
+        Assertions.assertTrue( PwmLogLevel.DEBUG.isGreaterOrSameAs( PwmLogLevel.DEBUG ) );
+        Assertions.assertTrue( PwmLogLevel.DEBUG.isGreaterOrSameAs( PwmLogLevel.TRACE ) );
+    }
+
+    @Test
+    void lowestLevel()
+    {
+        Assertions.assertEquals( PwmLogLevel.DEBUG, PwmLogLevel.lowestLevel( List.of( PwmLogLevel.DEBUG ) ) );
+        Assertions.assertEquals( PwmLogLevel.DEBUG, PwmLogLevel.lowestLevel( List.of( PwmLogLevel.DEBUG, PwmLogLevel.DEBUG ) ) );
+        Assertions.assertEquals( PwmLogLevel.DEBUG, PwmLogLevel.lowestLevel( List.of( PwmLogLevel.DEBUG, PwmLogLevel.DEBUG ) ) );
+    }
+}

server/src/test/java/password/pwm/util/logging/PwmLogbackPatternTest.java

@@ -0,0 +1,63 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2021 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.util.logging;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.spi.LoggingEvent;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.Method;
+import java.time.Instant;
+
+class PwmLogbackPatternTest
+{
+    @Test
+    public void doLayoutTest()
+    {
+        final Logger logger = new LoggerContext().getLogger( PwmLogbackPatternTest.class );
+        final String classFqdn = PwmLogbackPatternTest.class.getName();
+        final String message = "the message!";
+        final LoggingEvent event = new LoggingEvent( classFqdn, logger, Level.DEBUG, message, null, new Object[0] );
+        event.setInstant( Instant.EPOCH );
+
+        final PwmLogbackPattern pattern = new PwmLogbackPattern();
+        final String layout = pattern.doLayout( event );
+
+        Assertions.assertEquals( "1970-01-01T00:00:00Z, DEBUG, logging.PwmLogbackPatternTest, the message!\n", layout );
+    }
+
+    @Test
+    public void printLoggerNameTest()
+            throws Exception
+    {
+        final Method method = PwmLogbackPattern.class.getDeclaredMethod( "printLoggerName", String.class );
+        method.setAccessible( true );
+
+        Assertions.assertEquals( "second.third", method.invoke( null, "zeroth.first.second.third" ) );
+        Assertions.assertEquals( "second.third", method.invoke( null, "first.second.third" ) );
+        Assertions.assertEquals( "second.third", method.invoke( null, ".second.third" ) );
+        Assertions.assertEquals( "second.third", method.invoke( null, "second.third" ) );
+        Assertions.assertEquals( "third", method.invoke( null, "third" ) );
+    }
+}

data-service/src/main/webapp/WEB-INF/classes/logback.xml → server/src/test/resources/logback-test.xml

@@ -19,13 +19,16 @@
   ~ limitations under the License.
   -->
 
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{yyyy-MM-dd'T'HH:mm:ss'Z'}, %-5level, %logger{36}, %m%n</pattern>
-        </encoder>
+<configuration debug="true">
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <layout class="ch.qos.logback.classic.PatternLayout">
+            <Pattern>testlog %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
+        </layout>
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>OFF</level>
+        </filter>
     </appender>
-    <root level="trace">
-        <appender-ref ref="STDOUT" />
+    <root level="OFF">
+        <appender-ref ref="CONSOLE"/>
     </root>
 </configuration>

webapp/src/main/webapp/WEB-INF/jsp/admin-logview.jsp

@@ -64,6 +64,7 @@
                             <select id="downloadType" name="downloadType">
                                 <option value="plain">Plain</option>
                                 <option value="csv">CSV</option>
+                                <option value="json">JSON</option>
                             </select>
                         </td>
                         <td>

webapp/src/main/webapp/WEB-INF/jsp/configmanager.jsp

@@ -31,6 +31,7 @@
 <%@ page import="password.pwm.http.tag.conditional.PwmIfTest" %>
 <%@ page import="password.pwm.http.tag.value.PwmValue" %>
 <%@ page import="password.pwm.http.PwmRequestAttribute" %>
+<%@ page import="password.pwm.util.logging.PwmLogSettings" %>
 
 <!DOCTYPE html>
 
@@ -245,16 +246,8 @@
 <pwm:script>
     <script type="text/javascript">
         PWM_GLOBAL['startupFunctions'].push(function(){
-            require(["dojo/parser","dijit/TitlePane","dojo/domReady!","dojox/form/Uploader"],function(dojoParser){
-                dojoParser.parse();
-            });
-            PWM_VAR['config_localDBLogLevel'] = '<%=pwmRequest.getAppConfig().getEventLogLocalDBLevel()%>'
-
-            require(["dojo/domReady!"],function(){
-                PWM_ADMIN.showAppHealth('healthBody', {showRefresh: true, showTimestamp: true});
-            });
+            PWM_VAR['config_localDBLogLevel'] = '<%=PwmLogSettings.fromAppConfig( pwmRequest.getAppConfig() ).getLocalDbLevel()%>'
         });
-
     </script>
 </pwm:script>
 <pwm:script-ref url="/public/resources/js/configmanager.js"/>

webapp/src/main/webapp/WEB-INF/jsp/fragment/log-viewer.jsp

@@ -107,17 +107,17 @@
         </form>
     </div>
     <br/>
-    <div id="div-noResultsMessage" class="hidden" style="min-height: 200px">
+    <div id="div-noResultsMessage" class="nodisplay" style="min-height: 200px">
         <p style="text-align:center;" >
             No events matched the search settings. Please refine your search query and try again.
         </p>
     </div>
     <div style="margin: 20px; min-height: 200px">
-        <div id="wrapper-logViewerGrid" class="hidden">
+        <div id="wrapper-logViewerGrid" class="nodisplay">
             <div id="logViewerGrid" >
             </div>
         </div>
-        <div id="wrapper-lineViewer" class="hidden">
+        <div id="wrapper-lineViewer" class="nodisplay">
             <div id="lineViewer" class="logViewer">
 
             </div>

webapp/src/main/webapp/WEB-INF/log4jconfig-sample.xml

@@ -1,68 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Password Management Servlets (PWM)
-  ~ http://www.pwm-project.org
-  ~
-  ~ Copyright (c) 2006-2009 Novell, Inc.
-  ~ Copyright (c) 2009-2021 The PWM Project
-  ~
-  ~ Licensed under the Apache License, Version 2.0 (the "License");
-  ~ you may not use this file except in compliance with the License.
-  ~ You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<!--
-   NOTICE: THIS FILE IS NOT USED BY DEFAULT!
-   You must edit the application configuration to specify this file for the log4j configuration if you
-   would like to use this file to control the LOG4J system.
--->
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/" debug="false">
-    <appender name="applicaton-stdout" class="org.apache.log4j.ConsoleAppender">
-        <layout class="org.apache.log4j.PatternLayout">
-            <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss}, %-5p, %c{2}, %m%n"/>
-        </layout>
-    </appender>
-    <appender name="applicaton-filelogger" class="org.apache.log4j.DailyRollingFileAppender">
-        <param name="File" value="PWMlog.csv"/>
-        <param name="Append" value="false"/>
-        <layout class="org.apache.log4j.PatternLayout">
-            <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss}, %c{2}, %m%n"/>
-        </layout>
-    </appender>
-
-    <!-- this appender logs information about the application itself -->
-    <category name="password.pwm">
-        <priority value="trace"/>
-        <!-- possible priority values are "trace,debug,info,error,warn,fatal" -->
-
-        <appender-ref ref="applicaton-stdout"/>
-        <!-- uncomment this to log to file <appender-ref ref="applicaton-filelogger"/>-->
-    </category>
-
-    <!-- ldapchai logs information about ldap connections -->
-    <category name="com.novell.ldapchai">
-        <priority value="info"/>
-        <!-- possible priority values are "trace,debug,error,warn,fatal" -->
-
-        <appender-ref ref="applicaton-stdout"/>
-        <!-- uncomment this to log to file <appender-ref ref="applicaton-filelogger"/>-->
-    </category>
-
-    <category name="org.jasig.cas.client">
-        <priority value="info"/>
-        <!-- possible priority values are "trace,debug,error,warn,fatal" -->
-
-        <appender-ref ref="applicaton-stdout"/>
-        <!-- uncomment this to log to file <appender-ref ref="applicaton-filelogger"/>-->
-    </category>
-
-</log4j:configuration>

webapp/src/main/webapp/public/resources/js/admin.js

@@ -513,23 +513,23 @@ PWM_ADMIN.refreshLogData = function() {
 
         var records = data['data']['records'];
         if (PWM_MAIN.JSLibrary.isEmpty(records)) {
-            PWM_MAIN.removeCssClass('div-noResultsMessage', 'hidden');
-            PWM_MAIN.addCssClass('wrapper-logViewerGrid', 'hidden');
-            PWM_MAIN.addCssClass('wrapper-lineViewer', 'hidden');
+            PWM_MAIN.removeCssClass('div-noResultsMessage', 'nodisplay');
+            PWM_MAIN.addCssClass('wrapper-logViewerGrid', 'nodisplay');
+            PWM_MAIN.addCssClass('wrapper-lineViewer', 'nodisplay');
 
         } else {
             if (data['data']['display'] === 'grid') {
-                PWM_MAIN.addCssClass('div-noResultsMessage', 'hidden');
-                PWM_MAIN.removeCssClass('wrapper-logViewerGrid', 'hidden');
-                PWM_MAIN.addCssClass('wrapper-lineViewer', 'hidden');
+                PWM_MAIN.addCssClass('div-noResultsMessage', 'nodisplay');
+                PWM_MAIN.removeCssClass('wrapper-logViewerGrid', 'nodisplay');
+                PWM_MAIN.addCssClass('wrapper-lineViewer', 'nodisplay');
                 var grid = PWM_VAR['logViewerGrid'];
                 grid.refresh();
                 grid.renderArray(records);
                 grid.set("timestamp", { attribute : 'createTime', ascending: false, descending: true });
             } else {
-                PWM_MAIN.addCssClass('div-noResultsMessage', 'hidden');
-                PWM_MAIN.addCssClass('wrapper-logViewerGrid', 'hidden');
-                PWM_MAIN.removeCssClass('wrapper-lineViewer', 'hidden');
+                PWM_MAIN.addCssClass('div-noResultsMessage', 'nodisplay');
+                PWM_MAIN.addCssClass('wrapper-logViewerGrid', 'nodisplay');
+                PWM_MAIN.removeCssClass('wrapper-lineViewer', 'nodisplay');
                 var textOutput = '';
 
                 for (var iterator in records) {