restart enhancements

server/src/main/java/password/pwm/FileLocker.java

@@ -0,0 +1,158 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2020 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm;
+
+import password.pwm.config.Configuration;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogger;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.io.StringWriter;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.time.Instant;
+import java.util.Properties;
+
+class FileLocker
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( FileLocker.class );
+
+    private final PwmEnvironment pwmEnvironment;
+    private FileLock lock;
+    private final File lockfile;
+
+    FileLocker( final PwmEnvironment pwmEnvironment )
+    {
+        this.pwmEnvironment = pwmEnvironment;
+        final String lockfileName = pwmEnvironment.getConfig().readAppProperty( AppProperty.APPLICATION_FILELOCK_FILENAME );
+        lockfile = new File( pwmEnvironment.getApplicationPath(), lockfileName );
+    }
+
+    private boolean lockingAllowed( )
+    {
+        return !pwmEnvironment.isInternalRuntimeInstance() && !pwmEnvironment.getFlags().contains( PwmEnvironment.ApplicationFlag.NoFileLock );
+    }
+
+    public boolean isLocked( )
+    {
+        return !lockingAllowed() || lock != null && lock.isValid();
+    }
+
+    public void attemptFileLock( )
+    {
+        if ( lockingAllowed() && !isLocked() )
+        {
+            try
+            {
+                final RandomAccessFile file = new RandomAccessFile( lockfile, "rw" );
+                final FileChannel f = file.getChannel();
+                lock = f.tryLock();
+                if ( lock != null )
+                {
+                    LOGGER.debug( () -> "obtained file lock on file " + lockfile.getAbsolutePath() + " lock is valid=" + lock.isValid() );
+                    writeLockFileContents( file );
+                }
+                else
+                {
+                    LOGGER.debug( () -> "unable to obtain file lock on file " + lockfile.getAbsolutePath() );
+                }
+            }
+            catch ( final Exception e )
+            {
+                LOGGER.error( () -> "unable to obtain file lock on file " + lockfile.getAbsolutePath() + " due to error: " + e.getMessage() );
+            }
+        }
+    }
+
+    void writeLockFileContents( final RandomAccessFile file )
+    {
+        try
+        {
+            final Properties props = new Properties();
+            props.put( "timestamp", JavaHelper.toIsoDate( Instant.now() ) );
+            props.put( "applicationPath", pwmEnvironment.getApplicationPath() == null ? "n/a" : pwmEnvironment.getApplicationPath().getAbsolutePath() );
+            props.put( "configurationFile", pwmEnvironment.getConfigurationFile() == null ? "n/a" : pwmEnvironment.getConfigurationFile().getAbsolutePath() );
+            final String comment = PwmConstants.PWM_APP_NAME + " file lock";
+            final StringWriter stringWriter = new StringWriter();
+            props.store( stringWriter, comment );
+            file.write( stringWriter.getBuffer().toString().getBytes( PwmConstants.DEFAULT_CHARSET ) );
+        }
+        catch ( final IOException e )
+        {
+            LOGGER.error( () -> "unable to write contents of application lock file: " + e.getMessage() );
+        }
+        // do not close FileWriter, otherwise lock is released.
+    }
+
+    public void releaseFileLock( )
+    {
+        if ( lock != null && lock.isValid() )
+        {
+            try
+            {
+                lock.release();
+            }
+            catch ( final IOException e )
+            {
+                LOGGER.error( () -> "error releasing file lock: " + e.getMessage() );
+            }
+
+            LOGGER.debug( () -> "released file lock on file " + lockfile.getAbsolutePath() );
+        }
+    }
+
+    public void waitForFileLock( )
+            throws PwmUnrecoverableException
+    {
+        final Configuration configuration = pwmEnvironment.getConfig();
+        final int maxWaitSeconds = pwmEnvironment.getFlags().contains( PwmEnvironment.ApplicationFlag.CommandLineInstance )
+                ? 1
+                : Integer.parseInt( configuration.readAppProperty( AppProperty.APPLICATION_FILELOCK_WAIT_SECONDS ) );
+        final Instant startTime = Instant.now();
+        final TimeDuration attemptInterval = TimeDuration.of( 5021, TimeDuration.Unit.MILLISECONDS );
+
+        while ( !isLocked() && TimeDuration.fromCurrent( startTime ).isShorterThan( maxWaitSeconds, TimeDuration.Unit.SECONDS ) )
+        {
+            attemptFileLock();
+
+            if ( !isLocked() )
+            {
+                LOGGER.debug( () -> "can't establish application file lock after "
+                        + TimeDuration.fromCurrent( startTime ).asCompactString()
+                        + ", will retry;" );
+                attemptInterval.pause();
+            }
+        }
+
+        if ( !isLocked() )
+        {
+            final String errorMsg = "unable to obtain application path file lock";
+            final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_STARTUP_ERROR, errorMsg );
+            throw new PwmUnrecoverableException( errorInformation );
+        }
+    }
+}

server/src/main/java/password/pwm/PwmApplication.java

@@ -41,6 +41,7 @@ import password.pwm.http.state.SessionStateService;
 import password.pwm.ldap.LdapConnectionService;
 import password.pwm.ldap.search.UserSearchEngine;
 import password.pwm.svc.PwmService;
+import password.pwm.svc.PwmServiceEnum;
 import password.pwm.svc.PwmServiceManager;
 import password.pwm.svc.cache.CacheService;
 import password.pwm.svc.email.EmailService;
@@ -114,7 +115,6 @@ import java.util.function.Supplier;
  */
 public class PwmApplication
 {
-
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwmApplication.class );
     private static final String DEFAULT_INSTANCE_ID = "-1";
 
@@ -129,10 +129,12 @@ public class PwmApplication
     private ErrorInformation lastLocalDBFailure;
     private final AtomicInteger inprogressRequests = new AtomicInteger( 0 );
 
-    private final PwmEnvironment pwmEnvironment;
+    private PwmEnvironment pwmEnvironment;
 
     private final PwmServiceManager pwmServiceManager = new PwmServiceManager( this );
 
+    private FileLocker fileLocker;
+
     private PwmScheduler pwmScheduler;
 
     private PwmApplication( final PwmEnvironment pwmEnvironment, final LocalDB localDB )
@@ -210,7 +212,8 @@ public class PwmApplication
         // get file lock
         if ( !pwmEnvironment.isInternalRuntimeInstance() )
         {
-            pwmEnvironment.waitForFileLock();
+            fileLocker = new FileLocker( pwmEnvironment );
+            fileLocker.waitForFileLock();
         }
 
         // clear temp dir
@@ -283,6 +286,15 @@ public class PwmApplication
         }
     }
 
+    public void reInit( final Configuration config )
+            throws PwmException
+    {
+        fileLocker.releaseFileLock();
+        this.pwmEnvironment = pwmEnvironment.toBuilder().config( config ).build();
+        pwmServiceManager.reInitialize( this );
+        runtimeNonce = PwmRandom.getInstance().randomUUID().toString();
+    }
+
     private void postInitTasks( )
     {
         final Instant startTime = Instant.now();
@@ -507,12 +519,12 @@ public class PwmApplication
 
     public SharedHistoryManager getSharedHistoryManager( )
     {
-        return ( SharedHistoryManager ) pwmServiceManager.getService( SharedHistoryManager.class );
+        return ( SharedHistoryManager ) pwmServiceManager.getService( PwmServiceEnum.SharedHistoryManager );
     }
 
     public IntruderManager getIntruderManager( )
     {
-        return ( IntruderManager ) pwmServiceManager.getService( IntruderManager.class );
+        return ( IntruderManager ) pwmServiceManager.getService( PwmServiceEnum.IntruderManager );
     }
 
     public ChaiUser getProxiedChaiUser( final UserIdentity userIdentity )
@@ -542,12 +554,12 @@ public class PwmApplication
 
     public HealthMonitor getHealthMonitor( )
     {
-        return ( HealthMonitor ) pwmServiceManager.getService( HealthMonitor.class );
+        return ( HealthMonitor ) pwmServiceManager.getService( PwmServiceEnum.HealthMonitor );
     }
 
     public HttpClientService getHttpClientService()
     {
-        return ( HttpClientService ) pwmServiceManager.getService( HttpClientService.class );
+        return ( HttpClientService ) pwmServiceManager.getService( PwmServiceEnum.HttpClientService );
     }
 
     public List<PwmService> getPwmServices( )
@@ -561,52 +573,52 @@ public class PwmApplication
 
     public WordlistService getWordlistService( )
     {
-        return ( WordlistService ) pwmServiceManager.getService( WordlistService.class );
+        return ( WordlistService ) pwmServiceManager.getService( PwmServiceEnum.WordlistManager );
     }
 
     public SeedlistService getSeedlistManager( )
     {
-        return ( SeedlistService ) pwmServiceManager.getService( SeedlistService.class );
+        return ( SeedlistService ) pwmServiceManager.getService( PwmServiceEnum.SeedlistManager );
     }
 
     public ReportService getReportService( )
     {
-        return ( ReportService ) pwmServiceManager.getService( ReportService.class );
+        return ( ReportService ) pwmServiceManager.getService( PwmServiceEnum.ReportService );
     }
 
     public EmailService getEmailQueue( )
     {
-        return ( EmailService ) pwmServiceManager.getService( EmailService.class );
+        return ( EmailService ) pwmServiceManager.getService( PwmServiceEnum.EmailQueueManager );
     }
 
     public AuditService getAuditManager( )
     {
-        return ( AuditService ) pwmServiceManager.getService( AuditService.class );
+        return ( AuditService ) pwmServiceManager.getService( PwmServiceEnum.AuditService );
     }
 
     public SmsQueueManager getSmsQueue( )
     {
-        return ( SmsQueueManager ) pwmServiceManager.getService( SmsQueueManager.class );
+        return ( SmsQueueManager ) pwmServiceManager.getService( PwmServiceEnum.SmsQueueManager );
     }
 
     public PwNotifyService getPwNotifyService( )
     {
-        return ( PwNotifyService ) pwmServiceManager.getService( PwNotifyService.class );
+        return ( PwNotifyService ) pwmServiceManager.getService( PwmServiceEnum.PwExpiryNotifyService );
     }
 
     public UrlShortenerService getUrlShortener( )
     {
-        return ( UrlShortenerService ) pwmServiceManager.getService( UrlShortenerService.class );
+        return ( UrlShortenerService ) pwmServiceManager.getService( PwmServiceEnum.UrlShortenerService );
     }
 
     public UserSearchEngine getUserSearchEngine( )
     {
-        return ( UserSearchEngine ) pwmServiceManager.getService( UserSearchEngine.class );
+        return ( UserSearchEngine ) pwmServiceManager.getService( PwmServiceEnum.UserSearchEngine );
     }
 
     public NodeService getClusterService( )
     {
-        return ( NodeService ) pwmServiceManager.getService( NodeService.class );
+        return ( NodeService ) pwmServiceManager.getService( PwmServiceEnum.ClusterService );
     }
 
     public ErrorInformation getLastLocalDBFailure( )
@@ -616,27 +628,27 @@ public class PwmApplication
 
     public TokenService getTokenService( )
     {
-        return ( TokenService ) pwmServiceManager.getService( TokenService.class );
+        return ( TokenService ) pwmServiceManager.getService( PwmServiceEnum.TokenService );
     }
 
     public LdapConnectionService getLdapConnectionService( )
     {
-        return ( LdapConnectionService ) pwmServiceManager.getService( LdapConnectionService.class );
+        return ( LdapConnectionService ) pwmServiceManager.getService( PwmServiceEnum.LdapConnectionService );
     }
 
     public SessionTrackService getSessionTrackService( )
     {
-        return ( SessionTrackService ) pwmServiceManager.getService( SessionTrackService.class );
+        return ( SessionTrackService ) pwmServiceManager.getService( PwmServiceEnum.SessionTrackService );
     }
 
     public ResourceServletService getResourceServletService( )
     {
-        return ( ResourceServletService ) pwmServiceManager.getService( ResourceServletService.class );
+        return ( ResourceServletService ) pwmServiceManager.getService( PwmServiceEnum.ResourceServletService );
     }
 
     public PeopleSearchService getPeopleSearchService( )
     {
-        return ( PeopleSearchService ) pwmServiceManager.getService( PeopleSearchService.class );
+        return ( PeopleSearchService ) pwmServiceManager.getService( PwmServiceEnum.PeopleSearchService );
     }
 
     public Configuration getConfig( )
@@ -658,7 +670,7 @@ public class PwmApplication
 
     public DatabaseService getDatabaseService( )
     {
-        return ( DatabaseService ) pwmServiceManager.getService( DatabaseService.class );
+        return ( DatabaseService ) pwmServiceManager.getService( PwmServiceEnum.DatabaseService );
     }
 
 
@@ -726,33 +738,33 @@ public class PwmApplication
 
     public StatisticsManager getStatisticsManager( )
     {
-        return ( StatisticsManager ) pwmServiceManager.getService( StatisticsManager.class );
+        return ( StatisticsManager ) pwmServiceManager.getService( PwmServiceEnum.StatisticsManager );
     }
 
     public OtpService getOtpService( )
     {
-        return ( OtpService ) pwmServiceManager.getService( OtpService.class );
+        return ( OtpService ) pwmServiceManager.getService( PwmServiceEnum.OtpService );
     }
 
     public CrService getCrService( )
     {
-        return ( CrService ) pwmServiceManager.getService( CrService.class );
+        return ( CrService ) pwmServiceManager.getService( PwmServiceEnum.CrService );
     }
 
     public SessionStateService getSessionStateService( )
     {
-        return ( SessionStateService ) pwmServiceManager.getService( SessionStateService.class );
+        return ( SessionStateService ) pwmServiceManager.getService( PwmServiceEnum.SessionStateSvc );
     }
 
 
     public CacheService getCacheService( )
     {
-        return ( CacheService ) pwmServiceManager.getService( CacheService.class );
+        return ( CacheService ) pwmServiceManager.getService( PwmServiceEnum.CacheService );
     }
 
     public SecureService getSecureService( )
     {
-        return ( SecureService ) pwmServiceManager.getService( SecureService.class );
+        return ( SecureService ) pwmServiceManager.getService( PwmServiceEnum.SecureService );
     }
 
     public void sendSmsUsingQueue(
@@ -849,7 +861,10 @@ public class PwmApplication
             localDB = null;
         }
 
-        pwmEnvironment.releaseFileLock();
+        if ( fileLocker != null )
+        {
+            fileLocker.releaseFileLock();
+        }
 
         LOGGER.info( () -> PwmConstants.PWM_APP_NAME + " " + PwmConstants.SERVLET_VERSION + " closed for bidness, cya!" );
     }
@@ -871,7 +886,8 @@ public class PwmApplication
 
     private static class Initializer
     {
-        public static LocalDB initializeLocalDB( final PwmApplication pwmApplication, final PwmEnvironment pwmEnvironment ) throws PwmUnrecoverableException
+        public static LocalDB initializeLocalDB( final PwmApplication pwmApplication, final PwmEnvironment pwmEnvironment )
+                throws PwmUnrecoverableException
         {
             final File databaseDirectory;
 

server/src/main/java/password/pwm/PwmEnvironment.java

@@ -20,6 +20,9 @@
 
 package password.pwm;
 
+import lombok.Builder;
+import lombok.Singular;
+import lombok.Value;
 import password.pwm.config.Configuration;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
@@ -27,41 +30,39 @@ import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.http.ContextManager;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
-import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.io.StringWriter;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileLock;
-import java.time.Instant;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
+import java.util.EnumMap;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 
+@Value
+@Builder( toBuilder = true )
 public class PwmEnvironment
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwmEnvironment.class );
 
-    // data elements
-    private final PwmApplicationMode applicationMode;
-    private final Configuration config;
-    private final File applicationPath;
-    private final boolean internalRuntimeInstance;
-    private final File configurationFile;
-    private final ContextManager contextManager;
-    private final Collection<ApplicationFlag> flags;
-    private final Map<ApplicationParameter, String> parameters;
+    @lombok.Builder.Default
+    private PwmApplicationMode applicationMode = PwmApplicationMode.ERROR;
 
-    private final FileLocker fileLocker;
+    private Configuration config;
+    private File applicationPath;
+    private boolean internalRuntimeInstance;
+    private File configurationFile;
+    private ContextManager contextManager;
+
+    @Singular
+    private Set<ApplicationFlag> flags;
+
+    @Singular
+    private Map<ApplicationParameter, String> parameters;
 
     public enum ApplicationParameter
     {
@@ -143,76 +144,6 @@ public class PwmEnvironment
         }
     }
 
-    @SuppressWarnings( "checkstyle:ParameterNumber" )
-    private PwmEnvironment(
-            final PwmApplicationMode applicationMode,
-            final Configuration config,
-            final File applicationPath,
-            final boolean internalRuntimeInstance,
-            final File configurationFile,
-            final ContextManager contextManager,
-            final Collection<ApplicationFlag> flags,
-            final Map<ApplicationParameter, String> parameters
-    )
-    {
-        this.applicationMode = applicationMode == null ? PwmApplicationMode.ERROR : applicationMode;
-        this.config = config;
-        this.applicationPath = applicationPath;
-        this.internalRuntimeInstance = internalRuntimeInstance;
-        this.configurationFile = configurationFile;
-        this.contextManager = contextManager;
-        this.flags = flags == null ? Collections.emptySet() : Collections.unmodifiableSet( new HashSet<>( flags ) );
-        this.parameters = parameters == null ? Collections.emptyMap() : Collections.unmodifiableMap( parameters );
-
-        this.fileLocker = new FileLocker();
-
-        verify();
-    }
-
-    public PwmApplicationMode getApplicationMode( )
-    {
-        return applicationMode;
-    }
-
-    public Configuration getConfig( )
-    {
-        return config;
-    }
-
-    public File getApplicationPath( )
-    {
-        return applicationPath;
-    }
-
-    public boolean isInternalRuntimeInstance( )
-    {
-        return internalRuntimeInstance;
-    }
-
-    public File getConfigurationFile( )
-    {
-        return configurationFile;
-    }
-
-    public ContextManager getContextManager( )
-    {
-        return contextManager;
-    }
-
-    public Collection<ApplicationFlag> getFlags( )
-    {
-        return flags;
-    }
-
-    public Map<ApplicationParameter, String> getParameters( )
-    {
-        return parameters;
-    }
-
-    private void verify( )
-    {
-
-    }
 
     public void verifyIfApplicationPathIsSetProperly( )
             throws PwmUnrecoverableException
@@ -241,12 +172,12 @@ public class PwmEnvironment
     )
             throws PwmUnrecoverableException
     {
-        return new Builder( this )
-                .setApplicationMode( PwmApplicationMode.NEW )
-                .setInternalRuntimeInstance( true )
-                .setConfigurationFile( null )
-                .setConfig( configuration )
-                .createPwmEnvironment();
+        return this.toBuilder()
+                .applicationMode( PwmApplicationMode.NEW )
+                .internalRuntimeInstance( true )
+                .configurationFile( null )
+                .config( configuration )
+                .build();
     }
 
 
@@ -302,14 +233,14 @@ public class PwmEnvironment
 
     public static class ParseHelper
     {
-        public static Collection<ApplicationFlag> readApplicationFlagsFromSystem( final String contextName )
+        public static Set<ApplicationFlag> readApplicationFlagsFromSystem( final String contextName )
         {
             final String rawValue = readValueFromSystem( EnvironmentParameter.applicationFlags, contextName );
             if ( rawValue != null )
             {
                 return parseApplicationFlagValueParameter( rawValue );
             }
-            return Collections.emptyList();
+            return Collections.emptySet();
         }
 
         public static Map<ApplicationParameter, String> readApplicationParmsFromSystem( final String contextName )
@@ -347,37 +278,25 @@ public class PwmEnvironment
             return null;
         }
 
-        public static Collection<ApplicationFlag> parseApplicationFlagValueParameter( final String input )
+        public static Set<ApplicationFlag> parseApplicationFlagValueParameter( final String input )
         {
             if ( input == null )
             {
-                return Collections.emptyList();
+                return Collections.emptySet();
             }
 
             try
             {
                 final List<String> jsonValues = JsonUtil.deserializeStringList( input );
-                final List<ApplicationFlag> returnFlags = new ArrayList<>();
-                for ( final String value : jsonValues )
-                {
-                    final ApplicationFlag flag = ApplicationFlag.forString( value );
-                    if ( value != null )
-                    {
-                        returnFlags.add( flag );
-                    }
-                    else
-                    {
-                        LOGGER.warn( () -> "unknown " + EnvironmentParameter.applicationFlags.toString() + " value: " + input );
-                    }
-                }
-                return Collections.unmodifiableList( returnFlags );
+                final Set<ApplicationFlag> returnFlags = JavaHelper.readEnumSetFromStringCollection( ApplicationFlag.class, jsonValues );
+                return Collections.unmodifiableSet( returnFlags );
             }
             catch ( final Exception e )
             {
                 //
             }
 
-            final List<ApplicationFlag> returnFlags = new ArrayList<>();
+            final Set<ApplicationFlag> returnFlags = EnumSet.noneOf( ApplicationFlag.class );
             for ( final String value : input.split( "," ) )
             {
                 final ApplicationFlag flag = ApplicationFlag.forString( value );
@@ -413,7 +332,7 @@ public class PwmEnvironment
 
             try
             {
-                final Map<ApplicationParameter, String> returnParams = new HashMap<>();
+                final Map<ApplicationParameter, String> returnParams = new EnumMap<>( ApplicationParameter.class );
                 for ( final Object key : propValues.keySet() )
                 {
 
@@ -438,235 +357,14 @@ public class PwmEnvironment
         }
     }
 
-
-    public static class Builder
+    public static PwmApplicationMode checkForTrial( final PwmApplicationMode mode )
     {
-        private PwmApplicationMode applicationMode;
-        private Configuration config;
-        private File applicationPath;
-        private boolean internalRuntimeInstance;
-        private File configurationFile;
-        private ContextManager contextManager;
-        private Collection<ApplicationFlag> flags = new HashSet<>();
-        private Map<ApplicationParameter, String> params = new HashMap<>();
-
-        public Builder( final PwmEnvironment pwmEnvironment )
-        {
-            this.applicationMode = pwmEnvironment.applicationMode;
-            this.config = pwmEnvironment.config;
-            this.applicationPath = pwmEnvironment.applicationPath;
-            this.internalRuntimeInstance = pwmEnvironment.internalRuntimeInstance;
-            this.configurationFile = pwmEnvironment.configurationFile;
-            this.contextManager = pwmEnvironment.contextManager;
-            this.flags = pwmEnvironment.flags;
-            this.params = pwmEnvironment.parameters;
-        }
-
-        public Builder( final Configuration config, final File applicationPath )
-        {
-            this.config = config;
-            this.applicationPath = applicationPath;
-        }
-
-        public Builder setApplicationMode( final PwmApplicationMode applicationMode )
-        {
-            if ( PwmConstants.TRIAL_MODE && applicationMode == PwmApplicationMode.RUNNING )
-            {
-                LOGGER.info( () -> "application is in trial mode" );
-                this.applicationMode = PwmApplicationMode.CONFIGURATION;
-            }
-            else
-            {
-                this.applicationMode = applicationMode;
-            }
-            return this;
-        }
-
-        public Builder setInternalRuntimeInstance( final boolean internalRuntimeInstance )
+        if ( PwmConstants.TRIAL_MODE && mode == PwmApplicationMode.RUNNING )
         {
-            this.internalRuntimeInstance = internalRuntimeInstance;
-            return this;
+            LOGGER.info( () -> "application is in trial mode" );
+            return PwmApplicationMode.CONFIGURATION;
         }
 
-        public Builder setConfigurationFile( final File configurationFile )
-        {
-            this.configurationFile = configurationFile;
-            return this;
-        }
-
-        public Builder setContextManager( final ContextManager contextManager )
-        {
-            this.contextManager = contextManager;
-            return this;
-        }
-
-        public Builder setFlags( final Collection<ApplicationFlag> flags )
-        {
-            this.flags.clear();
-            if ( flags != null )
-            {
-                this.flags.addAll( flags );
-            }
-            return this;
-        }
-
-        public Builder setParams( final Map<ApplicationParameter, String> params )
-        {
-            this.params.clear();
-            if ( params != null )
-            {
-                this.params.putAll( params );
-            }
-            return this;
-        }
-
-        public Builder setConfig( final Configuration config )
-        {
-            this.config = config;
-            return this;
-        }
-
-        public PwmEnvironment createPwmEnvironment( )
-        {
-            return new PwmEnvironment(
-                    applicationMode,
-                    config,
-                    applicationPath,
-                    internalRuntimeInstance,
-                    configurationFile,
-                    contextManager,
-                    flags,
-                    params
-            );
-        }
-    }
-
-    public void attemptFileLock( )
-    {
-        fileLocker.attemptFileLock();
-    }
-
-    public void releaseFileLock( )
-    {
-        fileLocker.releaseFileLock();
-    }
-
-    public boolean isFileLocked( )
-    {
-        return fileLocker.isLocked();
-    }
-
-    public void waitForFileLock( ) throws PwmUnrecoverableException
-    {
-        final int maxWaitSeconds = this.getFlags().contains( ApplicationFlag.CommandLineInstance )
-                ? 1
-                : Integer.parseInt( getConfig().readAppProperty( AppProperty.APPLICATION_FILELOCK_WAIT_SECONDS ) );
-        final Instant startTime = Instant.now();
-        final TimeDuration attemptInterval = TimeDuration.of( 5021, TimeDuration.Unit.MILLISECONDS );
-
-        while ( !this.isFileLocked() && TimeDuration.fromCurrent( startTime ).isShorterThan( maxWaitSeconds, TimeDuration.Unit.SECONDS ) )
-        {
-            attemptFileLock();
-
-            if ( !isFileLocked() )
-            {
-                LOGGER.debug( () -> "can't establish application file lock after "
-                        + TimeDuration.fromCurrent( startTime ).asCompactString()
-                        + ", will retry;" );
-                attemptInterval.pause();
-            }
-        }
-
-        if ( !isFileLocked() )
-        {
-            final String errorMsg = "unable to obtain application path file lock";
-            final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_STARTUP_ERROR, errorMsg );
-            throw new PwmUnrecoverableException( errorInformation );
-        }
-    }
-
-    private class FileLocker
-    {
-        private FileLock lock;
-        private final File lockfile;
-
-        FileLocker( )
-        {
-            final String lockfileName = config.readAppProperty( AppProperty.APPLICATION_FILELOCK_FILENAME );
-            lockfile = new File( getApplicationPath(), lockfileName );
-        }
-
-        private boolean lockingAllowed( )
-        {
-            return !isInternalRuntimeInstance() && !getFlags().contains( ApplicationFlag.NoFileLock );
-        }
-
-        public boolean isLocked( )
-        {
-            return !lockingAllowed() || lock != null && lock.isValid();
-        }
-
-        public void attemptFileLock( )
-        {
-            if ( lockingAllowed() && !isLocked() )
-            {
-                try
-                {
-                    final RandomAccessFile file = new RandomAccessFile( lockfile, "rw" );
-                    final FileChannel f = file.getChannel();
-                    lock = f.tryLock();
-                    if ( lock != null )
-                    {
-                        LOGGER.debug( () -> "obtained file lock on file " + lockfile.getAbsolutePath() + " lock is valid=" + lock.isValid() );
-                        writeLockFileContents( file );
-                    }
-                    else
-                    {
-                        LOGGER.debug( () -> "unable to obtain file lock on file " + lockfile.getAbsolutePath() );
-                    }
-                }
-                catch ( final Exception e )
-                {
-                    LOGGER.error( () -> "unable to obtain file lock on file " + lockfile.getAbsolutePath() + " due to error: " + e.getMessage() );
-                }
-            }
-        }
-
-        void writeLockFileContents( final RandomAccessFile file )
-        {
-            try
-            {
-                final Properties props = new Properties();
-                props.put( "timestamp", JavaHelper.toIsoDate( Instant.now() ) );
-                props.put( "applicationPath", PwmEnvironment.this.getApplicationPath() == null ? "n/a" : PwmEnvironment.this.getApplicationPath().getAbsolutePath() );
-                props.put( "configurationFile", PwmEnvironment.this.getConfigurationFile() == null ? "n/a" : PwmEnvironment.this.getConfigurationFile().getAbsolutePath() );
-                final String comment = PwmConstants.PWM_APP_NAME + " file lock";
-                final StringWriter stringWriter = new StringWriter();
-                props.store( stringWriter, comment );
-                file.write( stringWriter.getBuffer().toString().getBytes( PwmConstants.DEFAULT_CHARSET ) );
-            }
-            catch ( final IOException e )
-            {
-                LOGGER.error( () -> "unable to write contents of application lock file: " + e.getMessage() );
-            }
-            // do not close FileWriter, otherwise lock is released.
-        }
-
-        public void releaseFileLock( )
-        {
-            if ( lock != null && lock.isValid() )
-            {
-                try
-                {
-                    lock.release();
-                }
-                catch ( final IOException e )
-                {
-                    LOGGER.error( () -> "error releasing file lock: " + e.getMessage() );
-                }
-
-                LOGGER.debug( () -> "released file lock on file " + lockfile.getAbsolutePath() );
-            }
-        }
+        return mode;
     }
 }

server/src/main/java/password/pwm/PwmEnvironmentUtils.java

@@ -0,0 +1,31 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2020 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm;
+
+import password.pwm.util.logging.PwmLogger;
+
+public class PwmEnvironmentUtils
+{
+
+    private static final PwmLogger LOGGER = PwmLogger.forClass( PwmEnvironmentUtils.class );
+
+
+}

server/src/main/java/password/pwm/health/HealthMonitor.java

@@ -81,9 +81,9 @@ public class HealthMonitor implements PwmService
     private ExecutorService supportZipWriterService;
     private HealthMonitorSettings settings;
 
-    private Map<HealthMonitorFlag, Serializable> healthProperties = new ConcurrentHashMap<>();
+    private final Map<HealthMonitorFlag, Serializable> healthProperties = new ConcurrentHashMap<>();
 
-    private STATUS status = STATUS.NEW;
+    private STATUS status = STATUS.CLOSED;
     private PwmApplication pwmApplication;
     private volatile HealthData healthData = emptyHealthData();
 
@@ -99,7 +99,6 @@ public class HealthMonitor implements PwmService
 
     public void init( final PwmApplication pwmApplication ) throws PwmException
     {
-        status = STATUS.OPENING;
         this.pwmApplication = pwmApplication;
         settings = HealthMonitorSettings.fromConfiguration( pwmApplication.getConfig() );
 
@@ -127,6 +126,14 @@ public class HealthMonitor implements PwmService
         status = STATUS.OPEN;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     public Instant getLastHealthCheckTime( )
     {
         if ( status != STATUS.OPEN )

server/src/main/java/password/pwm/http/ContextManager.java

@@ -28,12 +28,12 @@ import password.pwm.PwmEnvironment;
 import password.pwm.bean.SessionLabel;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
-import password.pwm.config.value.StoredValue;
 import password.pwm.config.profile.LdapProfile;
 import password.pwm.config.stored.ConfigurationProperty;
 import password.pwm.config.stored.ConfigurationReader;
 import password.pwm.config.stored.StoredConfiguration;
 import password.pwm.config.stored.StoredConfigurationModifier;
+import password.pwm.config.value.StoredValue;
 import password.pwm.config.value.X509CertificateValue;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
@@ -46,7 +46,6 @@ import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.logging.PwmLogManager;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.secure.X509Utils;
 
@@ -266,21 +265,20 @@ public class ContextManager implements Serializable
         final Collection<PwmEnvironment.ApplicationFlag> applicationFlags = parameterReader.readApplicationFlags();
         final Map<PwmEnvironment.ApplicationParameter, String> applicationParams = parameterReader.readApplicationParams( applicationPath );
 
-        if ( applicationParams != null && applicationParams.containsKey( PwmEnvironment.ApplicationParameter.InitConsoleLogLevel ) )
-        {
-            final String logLevel = applicationParams.get( PwmEnvironment.ApplicationParameter.InitConsoleLogLevel );
-            PwmLogManager.preInitConsoleLogLevel( logLevel );
-        }
+        mode = PwmEnvironment.checkForTrial( mode );
 
         try
         {
-            final PwmEnvironment pwmEnvironment = new PwmEnvironment.Builder( configuration, applicationPath )
-                    .setApplicationMode( mode )
-                    .setConfigurationFile( configurationFile )
-                    .setContextManager( this )
-                    .setFlags( applicationFlags )
-                    .setParams( applicationParams )
-                    .createPwmEnvironment();
+            final PwmEnvironment pwmEnvironment = PwmEnvironment.builder()
+                    .config( configuration )
+                    .applicationPath( applicationPath )
+                    .applicationMode( mode )
+                    .configurationFile( configurationFile )
+                    .contextManager( this )
+                    .flags( applicationFlags )
+                    .parameters( applicationParams )
+                    .build();
+
             pwmApplication = PwmApplication.createPwmApplication( pwmEnvironment, preExistingLocalDB );
         }
         catch ( final Exception e )
@@ -491,7 +489,6 @@ public class ContextManager implements Serializable
 
         private void doRestart( )
         {
-            final boolean reloadLocalDB = Boolean.parseBoolean( pwmApplication.getConfig().readAppProperty( AppProperty.LOCALDB_RELOAD_WHEN_APP_RESTARTED ) );
             final Instant startTime = Instant.now();
 
             if ( restartInProgressFlag.get() )
@@ -507,53 +504,25 @@ public class ContextManager implements Serializable
                 return;
             }
 
-            final PwmApplication oldPwmApplication = pwmApplication;
-            final LocalDB oldLocalDB = reloadLocalDB
-                    ? null
-                    : pwmApplication.getLocalDB();
-
-            pwmApplication = null;
-
             try
             {
                 restartInProgressFlag.set( true );
-
-                waitForRequestsToComplete( oldPwmApplication );
+                waitForRequestsToComplete( pwmApplication );
 
                 {
                     final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
                     LOGGER.info( SESSION_LABEL, () -> "beginning application restart (" + timeDuration.asCompactString() + "), restart count=" + restartCount.incrementAndGet() );
                 }
 
-                final Instant shutdownStartTime = Instant.now();
                 try
                 {
-                    try
-                    {
-                        // prevent restart watcher from detecting in-progress restart in a loop
-                        taskMaster.shutdown();
-
-                        oldPwmApplication.shutdown( oldLocalDB != null );
-                    }
-                    catch ( final Exception e )
-                    {
-                        LOGGER.error( SESSION_LABEL, () -> "unexpected error attempting to close application: " + e.getMessage() );
-                    }
+                        newReInit();
                 }
                 catch ( final Exception e )
                 {
                     LOGGER.fatal( () -> "unexpected error during shutdown: " + e.getMessage(), e );
                 }
 
-                {
-                    final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
-                    final TimeDuration shutdownDuration = TimeDuration.fromCurrent( shutdownStartTime );
-                    LOGGER.info( SESSION_LABEL, () -> "application restart; shutdown completed, ("
-                            + shutdownDuration.asCompactString()
-                            + ") now starting new application instance ("
-                            + timeDuration.asCompactString() + ")" );
-                }
-                initialize( oldLocalDB );
 
                 {
                     final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
@@ -659,7 +628,7 @@ public class ContextManager implements Serializable
             );
         }
 
-        Collection<PwmEnvironment.ApplicationFlag> readApplicationFlags( )
+        Set<PwmEnvironment.ApplicationFlag> readApplicationFlags( )
         {
             final String contextAppFlagsValue = readEnvironmentParameter( PwmEnvironment.EnvironmentParameter.applicationFlags );
 
@@ -797,4 +766,13 @@ public class ContextManager implements Serializable
             }
         }
     }
+
+    private void newReInit() throws PwmException
+    {
+        final File configurationFile = locateConfigurationFile( applicationPath, PwmConstants.DEFAULT_CONFIG_FILE_FILENAME );
+
+        configReader = new ConfigurationReader( configurationFile );
+        final Configuration configuration = configReader.getConfiguration();
+        pwmApplication.reInit( configuration );
+    }
 }

server/src/main/java/password/pwm/http/servlet/peoplesearch/PeopleSearchService.java

@@ -59,7 +59,14 @@ public class PeopleSearchService implements PwmService
                 new ArrayBlockingQueue<>( 5000 ),
                 threadFactory
         );
+    }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
     }
 
     @Override

server/src/main/java/password/pwm/http/servlet/resource/ResourceServletService.java

@@ -60,9 +60,9 @@ public class ResourceServletService implements PwmService
 
     private ResourceServletConfiguration resourceServletConfiguration;
     private Cache<CacheKey, CacheEntry> cache;
-    private MovingAverage cacheHitRatio = new MovingAverage( 60 * 60 * 1000 );
+    private final MovingAverage cacheHitRatio = new MovingAverage( 60 * 60 * 1000 );
     private String resourceNonce;
-    private STATUS status = STATUS.NEW;
+    private STATUS status = STATUS.CLOSED;
 
     private PwmApplication pwmApplication;
 
@@ -81,6 +81,13 @@ public class ResourceServletService implements PwmService
         return cacheHitRatio;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
 
     public long bytesInCache( )
     {
@@ -119,7 +126,6 @@ public class ResourceServletService implements PwmService
     public void init( final PwmApplication pwmApplication ) throws PwmException
     {
         this.pwmApplication = pwmApplication;
-        status = STATUS.OPENING;
         try
         {
             this.resourceServletConfiguration = ResourceServletConfiguration.createResourceServletConfiguration( pwmApplication );

server/src/main/java/password/pwm/http/state/SessionStateService.java

@@ -111,9 +111,16 @@ public class SessionStateService implements PwmService
     }
 
     @Override
-    public void close( )
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
     {
+        close();
+        init( pwmApplication );
+    }
 
+    @Override
+    public void close( )
+    {
     }
 
     @Override

server/src/main/java/password/pwm/ldap/LdapConnectionService.java

@@ -83,7 +83,7 @@ public class LdapConnectionService implements PwmService
     private ExecutorService executorService;
     private AtomicLoopIntIncrementer slotIncrementer;
 
-    private volatile STATUS status = STATUS.NEW;
+    private volatile STATUS status = STATUS.CLOSED;
 
     private boolean useThreadLocal;
     private final AtomicLoopIntIncrementer statCreatedProxies = new AtomicLoopIntIncrementer();
@@ -163,6 +163,13 @@ public class LdapConnectionService implements PwmService
         executorService.shutdown();
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication ) throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     public List<HealthRecord> healthCheck( )
     {
         return null;

server/src/main/java/password/pwm/ldap/search/UserSearchEngine.java

@@ -46,6 +46,7 @@ import password.pwm.util.java.AtomicLoopIntIncrementer;
 import password.pwm.util.java.ConditionalTaskExecutor;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.StatisticIntCounterMap;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogLevel;
@@ -71,18 +72,24 @@ import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
+
 
 public class UserSearchEngine implements PwmService
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( UserSearchEngine.class );
 
-    private final AtomicInteger searchCounter = new AtomicInteger( 0 );
-    private final AtomicInteger foregroundJobCounter = new AtomicInteger( 0 );
-    private final AtomicInteger backgroundJobCounter = new AtomicInteger( 0 );
-    private final AtomicInteger rejectionJobCounter = new AtomicInteger( 0 );
-    private final AtomicInteger canceledJobCounter = new AtomicInteger( 0 );
-    private final AtomicInteger jobTimeoutCounter = new AtomicInteger( 0 );
+    private final StatisticIntCounterMap<SearchStatistic> counters = new StatisticIntCounterMap<>( SearchStatistic.class );
+    private final AtomicLoopIntIncrementer searchIdCounter = new AtomicLoopIntIncrementer();
+
+    enum SearchStatistic
+    {
+        searchCounter,
+        foregroundJobCounter,
+        backgroundJobCounter,
+        backgroundRejectionJobCounter,
+        backgroundCanceledJobCounter,
+        backgroundJobTimeoutCounter,
+    }
 
     private PwmApplication pwmApplication;
 
@@ -111,6 +118,15 @@ public class UserSearchEngine implements PwmService
         this.periodicDebugOutput();
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
+
     @Override
     public void close( )
     {
@@ -322,7 +338,8 @@ public class UserSearchEngine implements PwmService
 
         final List<String> errors = new ArrayList<>();
 
-        final int searchID = searchCounter.getAndIncrement();
+        counters.increment( SearchStatistic.searchCounter );
+        final int searchID = searchIdCounter.next();
         final long profileRetryDelayMS = Long.parseLong( pwmApplication.getConfig().readAppProperty( AppProperty.LDAP_PROFILE_RETRY_DELAY ) );
         final AtomicLoopIntIncrementer jobIncrementer = AtomicLoopIntIncrementer.builder().build();
 
@@ -643,12 +660,12 @@ public class UserSearchEngine implements PwmService
                 {
                     executor.submit( jobInfo.getFutureTask() );
                     submittedToExecutor = true;
-                    backgroundJobCounter.incrementAndGet();
+                    counters.increment( SearchStatistic.backgroundJobCounter );
                 }
                 catch ( final RejectedExecutionException e )
                 {
                     // executor is full, so revert to running locally
-                    rejectionJobCounter.incrementAndGet();
+                    counters.increment( SearchStatistic.backgroundRejectionJobCounter );
                 }
             }
 
@@ -657,7 +674,7 @@ public class UserSearchEngine implements PwmService
                 try
                 {
                     jobInfo.getFutureTask().run();
-                    foregroundJobCounter.incrementAndGet();
+                    counters.increment( SearchStatistic.foregroundJobCounter );
                 }
                 catch ( final Throwable t )
                 {
@@ -691,7 +708,7 @@ public class UserSearchEngine implements PwmService
                 final FutureTask<Map<UserIdentity, Map<String, String>>> futureTask = jobInfo.getFutureTask();
                 if ( !futureTask.isDone() )
                 {
-                    canceledJobCounter.incrementAndGet();
+                    counters.increment( SearchStatistic.backgroundCanceledJobCounter );
                 }
                 jobInfo.getFutureTask().cancel( false );
             }
@@ -734,9 +751,7 @@ public class UserSearchEngine implements PwmService
     private Map<String, String> debugProperties( )
     {
         final Map<String, String> properties = new TreeMap<>();
-        properties.put( "searchCount", this.searchCounter.toString() );
-        properties.put( "backgroundJobCounter", Integer.toString( this.backgroundJobCounter.get() ) );
-        properties.put( "foregroundJobCounter", Integer.toString( this.foregroundJobCounter.get() ) );
+        properties.putAll( counters.debugStats() );
         properties.put( "jvmThreadCount", Integer.toString( Thread.activeCount() ) );
         if ( executor == null )
         {
@@ -750,9 +765,6 @@ public class UserSearchEngine implements PwmService
             properties.put( "background-largestPoolSize", Integer.toString( executor.getLargestPoolSize() ) );
             properties.put( "background-poolSize", Integer.toString( executor.getPoolSize() ) );
             properties.put( "background-queue-size", Integer.toString( executor.getQueue().size() ) );
-            properties.put( "background-rejectionJobCounter", Integer.toString( rejectionJobCounter.get() ) );
-            properties.put( "background-canceledJobCounter", Integer.toString( canceledJobCounter.get() ) );
-            properties.put( "background-jobTimeoutCounter", Integer.toString( jobTimeoutCounter.get() ) );
         }
         return Collections.unmodifiableMap( properties );
     }

server/src/main/java/password/pwm/svc/PwmService.java

@@ -42,16 +42,16 @@ public interface PwmService
 
     enum STATUS
     {
-        NEW,
-        OPENING,
         OPEN,
-        CLOSED
+        CLOSED,
     }
 
     STATUS status( );
 
     void init( PwmApplication pwmApplication ) throws PwmException;
 
+    void reInit( PwmApplication pwmApplication ) throws PwmException;
+
     void close( );
 
     List<HealthRecord> healthCheck( );

server/src/main/java/password/pwm/svc/PwmServiceEnum.java

@@ -30,6 +30,7 @@ import password.pwm.util.java.JavaHelper;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 public enum PwmServiceEnum
 {
@@ -63,7 +64,7 @@ public enum PwmServiceEnum
     PwExpiryNotifyService( PwNotifyService.class ),;
 
     private final Class<? extends PwmService> clazz;
-    private final Flag[] flags;
+    private final Set<Flag> flags;
 
     private enum Flag
     {
@@ -73,12 +74,12 @@ public enum PwmServiceEnum
     PwmServiceEnum( final Class<? extends PwmService> clazz, final Flag... flags )
     {
         this.clazz = clazz;
-        this.flags = flags;
+        this.flags = JavaHelper.enumSetFromArray( flags );
     }
 
     public boolean isInternalRuntime( )
     {
-        return JavaHelper.enumArrayContainsValue( flags, Flag.StartDuringRuntimeInstance );
+        return this.flags.contains( Flag.StartDuringRuntimeInstance );
     }
 
     static List<Class<? extends PwmService>> allClasses( )
@@ -95,4 +96,9 @@ public enum PwmServiceEnum
     {
         return clazz;
     }
+
+    public String serviceName()
+    {
+        return "[" + getPwmServiceClass().getSimpleName() + "]";
+    }
 }

server/src/main/java/password/pwm/svc/PwmServiceManager.java

@@ -31,6 +31,7 @@ import password.pwm.util.logging.PwmLogger;
 
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -38,12 +39,10 @@ import java.util.Map;
 
 public class PwmServiceManager
 {
-
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwmServiceManager.class );
 
-
-    private final PwmApplication pwmApplication;
-    private final Map<Class<? extends PwmService>, PwmService> runningServices = new HashMap<>();
+    private PwmApplication pwmApplication;
+    private final Map<PwmServiceEnum, PwmService> runningServices = new HashMap<>();
     private boolean initialized;
 
     public PwmServiceManager( final PwmApplication pwmApplication )
@@ -51,7 +50,7 @@ public class PwmServiceManager
         this.pwmApplication = pwmApplication;
     }
 
-    public PwmService getService( final Class<? extends PwmService> serviceClass )
+    public PwmService getService( final PwmServiceEnum serviceClass )
     {
         return runningServices.get( serviceClass );
     }
@@ -77,17 +76,35 @@ public class PwmServiceManager
             {
                 final Class<? extends PwmService> serviceClass = serviceClassEnum.getPwmServiceClass();
                 final PwmService newServiceInstance = initService( serviceClass );
-                runningServices.put( serviceClass, newServiceInstance );
+                runningServices.put( serviceClassEnum, newServiceInstance );
                 serviceCounter++;
             }
         }
 
         initialized = true;
 
-        final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
         {
             final int finalServiceCounter = serviceCounter;
-            LOGGER.trace( () -> "started " + finalServiceCounter + " services in " + timeDuration.asCompactString() );
+            LOGGER.trace( () -> "started " + finalServiceCounter + " services", () -> TimeDuration.fromCurrent( startTime ) );
+        }
+    }
+
+    public void reInitialize( final PwmApplication pwmApplication )
+    {
+        final Instant startTime = Instant.now();
+        this.pwmApplication = pwmApplication;
+        LOGGER.trace( () -> "beginning service restart process" );
+
+        int counter = 0;
+        for ( final Map.Entry<PwmServiceEnum, PwmService> entry : runningServices.entrySet() )
+        {
+            counter++;
+            reInitService( entry.getKey(), entry.getValue() );
+        }
+
+        {
+            final int totalRestarts = counter;
+            LOGGER.trace( () -> "completed restart of " + totalRestarts + " services", () -> TimeDuration.fromCurrent( startTime ) );
         }
     }
 
@@ -99,8 +116,7 @@ public class PwmServiceManager
         final String serviceName = serviceClass.getName();
         try
         {
-            final Object newInstance = serviceClass.newInstance();
-            newServiceInstance = ( PwmService ) newInstance;
+            newServiceInstance = serviceClass.getDeclaredConstructor().newInstance();
         }
         catch ( final Exception e )
         {
@@ -135,6 +151,24 @@ public class PwmServiceManager
         return newServiceInstance;
     }
 
+    private void reInitService( final PwmServiceEnum serviceEnum, final PwmService newServiceInstance )
+    {
+        final Instant startTime = Instant.now();
+        final String serviceName = serviceEnum.serviceName();
+
+        try
+        {
+            LOGGER.debug( () -> "re-initializing service " + serviceName );
+            newServiceInstance.init( pwmApplication );
+            final TimeDuration startupDuration = TimeDuration.fromCurrent( startTime );
+            LOGGER.debug( () -> "completed initialization of service " + serviceName + " in " + startupDuration.asCompactString() + ", status=" + newServiceInstance.status() );
+        }
+        catch ( final PwmException e )
+        {
+            LOGGER.warn( () -> "error instantiating service class '" + serviceName + "', service will remain unavailable, error: " + e.getMessage() );
+        }
+    }
+
     public void shutdownAllServices( )
     {
         if ( !initialized )
@@ -146,13 +180,13 @@ public class PwmServiceManager
         final Instant startTime = Instant.now();
 
 
-        final List<Class<? extends PwmService>> reverseServiceList = new ArrayList<>( PwmServiceEnum.allClasses() );
+        final List<PwmServiceEnum> reverseServiceList = Arrays.asList( PwmServiceEnum.values() );
         Collections.reverse( reverseServiceList );
-        for ( final Class<? extends PwmService> serviceClass : reverseServiceList )
+        for ( final PwmServiceEnum pwmServiceEnum : reverseServiceList )
         {
-            if ( runningServices.containsKey( serviceClass ) )
+            if ( runningServices.containsKey( pwmServiceEnum ) )
             {
-                shutDownService( serviceClass );
+                shutDownService( pwmServiceEnum, runningServices.get( pwmServiceEnum ) );
             }
         }
         initialized = false;
@@ -160,17 +194,17 @@ public class PwmServiceManager
         LOGGER.trace( () -> "closed all services in ", () -> TimeDuration.fromCurrent( startTime ) );
     }
 
-    private void shutDownService( final Class<? extends PwmService> serviceClass )
+    private void shutDownService( final PwmServiceEnum pwmServiceEnum, final PwmService loopService )
     {
 
-        LOGGER.trace( () -> "closing service " + serviceClass.getName() );
-        final PwmService loopService = runningServices.get( serviceClass );
+        LOGGER.trace( () -> "closing service " + pwmServiceEnum.serviceName() );
+
         try
         {
             final Instant startTime = Instant.now();
             loopService.close();
             final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
-            LOGGER.trace( () -> "successfully closed service " + serviceClass.getName() + " (" + timeDuration.asCompactString() + ")" );
+            LOGGER.trace( () -> "successfully closed service " + pwmServiceEnum.serviceName() + " (" + timeDuration.asCompactString() + ")" );
         }
         catch ( final Exception e )
         {

server/src/main/java/password/pwm/svc/cache/CacheService.java

@@ -49,7 +49,7 @@ public class CacheService implements PwmService
 
     private MemoryCacheStore memoryCacheStore;
 
-    private STATUS status = STATUS.NEW;
+    private STATUS status = STATUS.CLOSED;
 
     private ConditionalTaskExecutor traceDebugOutputter;
 
@@ -85,7 +85,6 @@ public class CacheService implements PwmService
             return;
         }
 
-        status = STATUS.OPENING;
         final int maxMemItems = Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.CACHE_MEMORY_MAX_ITEMS ) );
         memoryCacheStore = new MemoryCacheStore( maxMemItems );
         this.traceDebugOutputter = new ConditionalTaskExecutor(
@@ -95,6 +94,14 @@ public class CacheService implements PwmService
         status = STATUS.OPEN;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/svc/email/EmailService.java

@@ -58,6 +58,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
 
 /**
  * @author Jason D. Rivard
@@ -75,21 +76,21 @@ public class EmailService implements PwmService
     private final AtomicReference<ErrorInformation> lastSendError = new AtomicReference<>();
 
     private final ConditionalTaskExecutor statsLogger = ConditionalTaskExecutor.forPeriodicTask( this::logStats, TimeDuration.MINUTE );
+    private final ReentrantLock submitLock = new ReentrantLock();
 
-    private PwmService.STATUS status = STATUS.NEW;
+    private PwmService.STATUS status = STATUS.CLOSED;
 
     public void init( final PwmApplication pwmApplication )
             throws PwmException
     {
-        status = STATUS.OPENING;
         this.pwmApplication = pwmApplication;
         this.emailServiceSettings = EmailServiceSettings.fromConfiguration( pwmApplication.getConfig() );
         LOGGER.trace( () -> "initializing with settings: " + JsonUtil.serialize( emailServiceSettings ) );
 
-        final List<EmailServer> servers = new ArrayList<>();
+        final List<EmailServer> servers;
         try
         {
-            servers.addAll( EmailServerUtil.makeEmailServersMap( pwmApplication.getConfig() ) );
+            servers = new ArrayList<>( EmailServerUtil.makeEmailServersMap( pwmApplication.getConfig() ) );
         }
         catch ( final PwmUnrecoverableException e )
         {
@@ -108,7 +109,7 @@ public class EmailService implements PwmService
 
         if ( pwmApplication.getLocalDB() == null || pwmApplication.getLocalDB().status() != LocalDB.Status.OPEN )
         {
-            LOGGER.warn( () -> "localdb is not open, EmailService will remain closed" );
+            LOGGER.warn( () -> "localDB is not open, EmailService will remain closed" );
             status = STATUS.CLOSED;
             return;
         }
@@ -132,6 +133,22 @@ public class EmailService implements PwmService
         statsLogger.conditionallyExecuteTask();
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        submitLock.lock();
+        try
+        {
+            close();
+            init( pwmApplication );
+        }
+        finally
+        {
+            submitLock.unlock();
+        }
+    }
+
     public void close( )
     {
         status = STATUS.CLOSED;
@@ -352,46 +369,54 @@ public class EmailService implements PwmService
 
         checkIfServiceIsOpen();
 
-        final EmailItemBean finalBean;
+        submitLock.lock();
+        try
         {
-            EmailItemBean workingItemBean = emailItem;
-            if ( ( emailItem.getTo() == null || emailItem.getTo().isEmpty() ) && userInfo != null )
+            final EmailItemBean finalBean;
             {
-                final String toAddress = userInfo.getUserEmailAddress();
-                workingItemBean = EmailServerUtil.newEmailToAddress( workingItemBean, toAddress );
-            }
+                EmailItemBean workingItemBean = emailItem;
+                if ( ( emailItem.getTo() == null || emailItem.getTo().isEmpty() ) && userInfo != null )
+                {
+                    final String toAddress = userInfo.getUserEmailAddress();
+                    workingItemBean = EmailServerUtil.newEmailToAddress( workingItemBean, toAddress );
+                }
 
-            if ( macroMachine != null )
-            {
-                workingItemBean = EmailServerUtil.applyMacrosToEmail( workingItemBean, macroMachine );
-            }
+                if ( macroMachine != null )
+                {
+                    workingItemBean = EmailServerUtil.applyMacrosToEmail( workingItemBean, macroMachine );
+                }
 
-            if ( StringUtil.isEmpty( workingItemBean.getTo() ) )
-            {
-                LOGGER.error( () -> "no destination address available for email, skipping; email: " + emailItem.toDebugString() );
-            }
+                if ( StringUtil.isEmpty( workingItemBean.getTo() ) )
+                {
+                    LOGGER.error( () -> "no destination address available for email, skipping; email: " + emailItem.toDebugString() );
+                }
 
-            if ( !determineIfItemCanBeDelivered( emailItem ) )
-            {
-                return;
+                if ( !determineIfItemCanBeDelivered( emailItem ) )
+                {
+                    return;
+                }
+                finalBean = workingItemBean;
             }
-            finalBean = workingItemBean;
-        }
 
-        try
-        {
-            if ( immediate )
+            try
             {
-                workQueueProcessor.submitImmediate( finalBean );
+                if ( immediate )
+                {
+                    workQueueProcessor.submitImmediate( finalBean );
+                }
+                else
+                {
+                    workQueueProcessor.submit( finalBean );
+                }
             }
-            else
+            catch ( final PwmOperationalException e )
             {
-                workQueueProcessor.submit( finalBean );
+                LOGGER.warn( () -> "unable to add email to queue: " + e.getMessage() );
             }
         }
-        catch ( final PwmOperationalException e )
+        finally
         {
-            LOGGER.warn( () -> "unable to add email to queue: " + e.getMessage() );
+            submitLock.unlock();
         }
 
         statsLogger.conditionallyExecuteTask();

server/src/main/java/password/pwm/svc/event/AuditService.java

@@ -68,7 +68,7 @@ public class AuditService implements PwmService
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( AuditService.class );
 
-    private STATUS status = STATUS.NEW;
+    private STATUS status = STATUS.CLOSED;
     private AuditSettings settings;
     private ServiceInfoBean serviceInfo = new ServiceInfoBean( Collections.emptyList() );
 
@@ -90,7 +90,6 @@ public class AuditService implements PwmService
 
     public void init( final PwmApplication pwmApplication ) throws PwmException
     {
-        this.status = STATUS.OPENING;
         this.pwmApplication = pwmApplication;
 
         final Instant startTime = Instant.now();
@@ -198,6 +197,14 @@ public class AuditService implements PwmService
         this.status = STATUS.OPEN;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/svc/event/LocalDbAuditVault.java

@@ -50,7 +50,7 @@ public class LocalDbAuditVault implements AuditVault
     private int maxBulkRemovals = 105;
 
     private ExecutorService executorService;
-    private volatile PwmService.STATUS status = PwmService.STATUS.NEW;
+    private volatile PwmService.STATUS status = PwmService.STATUS.CLOSED;
 
 
     public LocalDbAuditVault(

server/src/main/java/password/pwm/svc/httpclient/HttpClientService.java

@@ -76,6 +76,14 @@ public class HttpClientService implements PwmService
         this.pwmApplication = pwmApplication;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     @Override
     public void close()
     {

server/src/main/java/password/pwm/svc/intruder/IntruderManager.java

@@ -81,7 +81,7 @@ public class IntruderManager implements PwmService
     private static final PwmLogger LOGGER = PwmLogger.forClass( IntruderManager.class );
 
     private PwmApplication pwmApplication;
-    private STATUS status = STATUS.NEW;
+    private STATUS status = STATUS.CLOSED;
     private ErrorInformation startupError;
     private Timer timer;
 
@@ -109,7 +109,6 @@ public class IntruderManager implements PwmService
     {
         this.pwmApplication = pwmApplication;
         final Configuration config = pwmApplication.getConfig();
-        status = STATUS.OPENING;
         if ( pwmApplication.getLocalDB() == null || pwmApplication.getLocalDB().status() != LocalDB.Status.OPEN )
         {
             final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, "unable to start IntruderManager, LocalDB unavailable" );
@@ -204,6 +203,14 @@ public class IntruderManager implements PwmService
         }
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     private void initializeRecordManagers( final Configuration config, final RecordStore recordStore )
     {
         {

server/src/main/java/password/pwm/svc/node/NodeService.java

@@ -45,7 +45,7 @@ public class NodeService implements PwmService
     private static final PwmLogger LOGGER = PwmLogger.forClass( NodeService.class );
 
     private PwmApplication pwmApplication;
-    private STATUS status = STATUS.NEW;
+    private STATUS status = STATUS.CLOSED;
     private NodeMachine nodeMachine;
     private DataStorageMethod dataStore;
     private ErrorInformation startupError;
@@ -60,7 +60,6 @@ public class NodeService implements PwmService
     @Override
     public void init( final PwmApplication pwmApplication ) throws PwmException
     {
-        status = STATUS.OPENING;
         this.pwmApplication = pwmApplication;
 
         final boolean serviceEnabled = pwmApplication.getConfig().readSettingAsBoolean( PwmSetting.CLUSTER_ENABLED );
@@ -122,6 +121,14 @@ public class NodeService implements PwmService
         status = STATUS.CLOSED;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyService.java

@@ -154,6 +154,14 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
         }
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     public Instant getNextExecutionTime( )
     {
         return nextExecutionTime;

server/src/main/java/password/pwm/svc/report/ReportService.java

@@ -72,7 +72,7 @@ public class ReportService implements PwmService
     private final AverageTracker avgTracker = new AverageTracker( 100 );
 
     private PwmApplication pwmApplication;
-    private STATUS status = STATUS.NEW;
+    private STATUS status = STATUS.CLOSED;
     private volatile boolean cancelFlag = false;
     private ReportSummaryData summaryData = ReportSummaryData.newSummaryData( null );
     private ExecutorService executorService;
@@ -107,7 +107,6 @@ public class ReportService implements PwmService
     public void init( final PwmApplication pwmApplication )
             throws PwmException
     {
-        status = STATUS.OPENING;
         this.pwmApplication = pwmApplication;
 
         if ( pwmApplication.getApplicationMode() == PwmApplicationMode.READ_ONLY )
@@ -148,6 +147,14 @@ public class ReportService implements PwmService
         status = STATUS.OPEN;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/svc/report/UserCacheService.java

@@ -146,12 +146,18 @@ public class UserCacheService implements PwmService
 
     public void init( final PwmApplication pwmApplication ) throws PwmException
     {
-        status = STATUS.OPENING;
         this.pwmApplication = pwmApplication;
         this.cacheStore = new CacheStoreWrapper( pwmApplication.getLocalDB() );
         status = STATUS.OPEN;
     }
 
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     public void close( )
     {
         status = STATUS.CLOSED;

server/src/main/java/password/pwm/svc/sessiontrack/SessionTrackService.java

@@ -80,6 +80,14 @@ public class SessionTrackService implements PwmService
         this.pwmApplication = pwmApplication;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/svc/shorturl/UrlShortenerService.java

@@ -25,6 +25,7 @@ import password.pwm.PwmApplication;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.option.DataStorageMethod;
+import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.PwmService;
@@ -48,7 +49,7 @@ public class UrlShortenerService implements PwmService
 
     private PwmApplication pwmApplication;
     private BasicUrlShortener theShortener = null;
-    private STATUS status = PwmService.STATUS.NEW;
+    private STATUS status = STATUS.CLOSED;
 
     public UrlShortenerService( )
     {
@@ -97,6 +98,14 @@ public class UrlShortenerService implements PwmService
         status = PwmService.STATUS.OPEN;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     public STATUS status( )
     {
         return status;

server/src/main/java/password/pwm/svc/stats/StatisticsManager.java

@@ -83,7 +83,7 @@ public class StatisticsManager implements PwmService
 
     private PwmApplication pwmApplication;
 
-    private STATUS status = STATUS.NEW;
+    private STATUS status = STATUS.CLOSED;
 
 
     private final Map<String, StatisticsBundle> cachedStoredStats = new LinkedHashMap<String, StatisticsBundle>()
@@ -231,7 +231,6 @@ public class StatisticsManager implements PwmService
 
     public void init( final PwmApplication pwmApplication ) throws PwmException
     {
-        status = STATUS.OPENING;
         this.localDB = pwmApplication.getLocalDB();
         this.pwmApplication = pwmApplication;
 
@@ -298,6 +297,14 @@ public class StatisticsManager implements PwmService
         status = STATUS.OPEN;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     private void writeDbValues( )
     {
         if ( localDB != null && status == STATUS.OPEN )

server/src/main/java/password/pwm/svc/telemetry/TelemetryService.java

@@ -80,7 +80,7 @@ public class TelemetryService implements PwmService
     private ErrorInformation lastError;
     private TelemetrySender sender;
 
-    private STATUS status = STATUS.NEW;
+    private STATUS status = STATUS.CLOSED;
 
 
     @Override
@@ -92,7 +92,6 @@ public class TelemetryService implements PwmService
     @Override
     public void init( final PwmApplication pwmApplication ) throws PwmException
     {
-        status = STATUS.OPENING;
         this.pwmApplication = pwmApplication;
 
         if ( pwmApplication.getApplicationMode() != PwmApplicationMode.RUNNING )
@@ -150,6 +149,14 @@ public class TelemetryService implements PwmService
         status = STATUS.OPEN;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     private void initSender( ) throws PwmUnrecoverableException
     {
         if ( StringUtil.isEmpty( settings.getSenderImplementation() ) )

server/src/main/java/password/pwm/svc/token/TokenService.java

@@ -99,7 +99,7 @@ public class TokenService implements PwmService
     private TokenMachine tokenMachine;
 
     private ServiceInfoBean serviceInfo = new ServiceInfoBean( Collections.emptyList() );
-    private volatile STATUS status = STATUS.NEW;
+    private volatile STATUS status = STATUS.CLOSED;
 
     private ErrorInformation errorInformation = null;
 
@@ -133,7 +133,6 @@ public class TokenService implements PwmService
             throws PwmException
     {
         LOGGER.trace( () -> "opening" );
-        status = STATUS.OPENING;
 
         this.pwmApplication = pwmApplication;
         this.configuration = pwmApplication.getConfig();
@@ -209,6 +208,14 @@ public class TokenService implements PwmService
         LOGGER.debug( () -> "open" );
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     public boolean supportsName( )
     {
         return tokenMachine.supportsName();

server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java

@@ -63,7 +63,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
     private ExecutorService executorService;
     private Set<WordType> wordTypesCache = null;
 
-    private volatile STATUS wlStatus = STATUS.NEW;
+    private volatile STATUS wlStatus = STATUS.CLOSED;
 
     private volatile ErrorInformation lastError;
     private volatile ErrorInformation autoImportError;
@@ -91,6 +91,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
         if ( this.wordlistConfiguration.isTestMode() )
         {
             startTestInstance( type );
+            wlStatus = STATUS.OPEN;
             return;
         }
         else
@@ -128,6 +129,14 @@ abstract class AbstractWordlist implements Wordlist, PwmService
         getLogger().trace( () -> "opening with configuration: " + JsonUtil.serialize( wordlistConfiguration ) );
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     private void startTestInstance( final WordlistType wordlistType )
     {
         this.wordlistBucket = new MemoryWordlistBucket( pwmApplication, wordlistConfiguration, wordlistType );

server/src/main/java/password/pwm/svc/wordlist/SharedHistoryManager.java

@@ -65,7 +65,7 @@ public class SharedHistoryManager implements PwmService
     private static final LocalDB.DB META_DB = LocalDB.DB.SHAREDHISTORY_META;
     private static final LocalDB.DB WORDS_DB = LocalDB.DB.SHAREDHISTORY_WORDS;
 
-    private volatile PwmService.STATUS status = STATUS.NEW;
+    private volatile PwmService.STATUS status = STATUS.CLOSED;
 
     private ExecutorService executorService;
 
@@ -89,6 +89,14 @@ public class SharedHistoryManager implements PwmService
         localDB = null;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     public boolean containsWord( final String word )
     {
         if ( status != STATUS.OPEN )
@@ -190,7 +198,6 @@ public class SharedHistoryManager implements PwmService
 
     private void init( final PwmApplication pwmApplication, final long maxAgeMs )
     {
-        status = STATUS.OPENING;
         final Instant startTime = Instant.now();
 
         try

server/src/main/java/password/pwm/util/OnejarHelper.java

@@ -112,12 +112,15 @@ public class OnejarHelper
         final File configFile = new File( applicationPath + File.separator + PwmConstants.DEFAULT_CONFIG_FILE_FILENAME );
         final ConfigurationReader configReader = new ConfigurationReader( configFile );
         final Configuration config = configReader.getConfiguration();
-        final PwmEnvironment pwmEnvironment = new PwmEnvironment.Builder( config, applicationPath )
-                .setApplicationMode( PwmApplicationMode.READ_ONLY )
-                .setConfigurationFile( configFile )
-                .setFlags( Collections.singleton( PwmEnvironment.ApplicationFlag.CommandLineInstance ) )
-                .setInternalRuntimeInstance( true )
-                .createPwmEnvironment();
+        final PwmEnvironment pwmEnvironment = PwmEnvironment.builder()
+                .config( config )
+                .applicationPath( applicationPath )
+                .applicationMode( PwmApplicationMode.READ_ONLY )
+                .configurationFile( configFile )
+                .flags( Collections.singleton( PwmEnvironment.ApplicationFlag.CommandLineInstance ) )
+                .internalRuntimeInstance( true )
+                .build();
+
         return PwmApplication.createPwmApplication( pwmEnvironment );
     }
 }

server/src/main/java/password/pwm/util/cli/MainClass.java

@@ -487,11 +487,14 @@ public class MainClass
             applicationFlags.addAll( flags );
         }
         applicationFlags.add( PwmEnvironment.ApplicationFlag.CommandLineInstance );
-        final PwmEnvironment pwmEnvironment = new PwmEnvironment.Builder( config, applicationPath )
-                .setApplicationMode( mode )
-                .setConfigurationFile( configurationFile )
-                .setFlags( applicationFlags )
-                .createPwmEnvironment();
+        final PwmEnvironment pwmEnvironment = PwmEnvironment.builder()
+                .config( config )
+                .applicationPath( applicationPath )
+                .applicationMode( mode )
+                .configurationFile( configurationFile )
+                .flags( applicationFlags )
+                .build();
+
         final PwmApplication pwmApplication = PwmApplication.createPwmApplication( pwmEnvironment );
         final PwmApplicationMode runningMode = pwmApplication.getApplicationMode();
 

server/src/main/java/password/pwm/util/db/DatabaseService.java

@@ -77,7 +77,7 @@ public class DatabaseService implements PwmService
     private ErrorInformation lastError;
     private PwmApplication pwmApplication;
 
-    private STATUS status = STATUS.NEW;
+    private STATUS status = STATUS.CLOSED;
 
     private AtomicLoopIntIncrementer slotIncrementer;
     private final Map<Integer, DatabaseAccessorImpl> accessors = new ConcurrentHashMap<>();
@@ -125,7 +125,6 @@ public class DatabaseService implements PwmService
         }
 
         final Instant startTime = Instant.now();
-        status = STATUS.OPENING;
 
         try
         {
@@ -181,11 +180,18 @@ public class DatabaseService implements PwmService
             final String errorMsg = "exception initializing database service: " + t.getMessage();
             LOGGER.warn( () -> errorMsg );
             initialized = false;
-            final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, errorMsg );
-            lastError = errorInformation;
+            lastError = new ErrorInformation( PwmError.ERROR_DB_UNAVAILABLE, errorMsg );
         }
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/util/java/JavaHelper.java

@@ -161,6 +161,13 @@ public class JavaHelper
         return Collections.unmodifiableSet( returnList );
     }
 
+    public static <E extends Enum<E>> Set<E> enumSetFromArray( final E[] arrayValues )
+    {
+        return arrayValues == null || arrayValues.length == 0
+                ? Collections.emptySet()
+                : Collections.unmodifiableSet( EnumSet.copyOf( Arrays.asList( arrayValues ) ) );
+    }
+
     public static <E extends Enum<E>> Map<String, String> enumMapToStringMap( final Map<E, String> inputMap )
     {
         return Collections.unmodifiableMap( inputMap.entrySet().stream()

server/src/main/java/password/pwm/util/localdb/LocalDBService.java

@@ -55,6 +55,14 @@ public class LocalDBService implements PwmService
         this.pwmApplication = pwmApplication;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/util/logging/LocalDBLogger.java

@@ -71,7 +71,7 @@ public class LocalDBLogger implements PwmService
     private final ScheduledExecutorService writerService;
     private final AtomicBoolean cleanOnWriteFlag = new AtomicBoolean( false );
 
-    private volatile STATUS status = STATUS.NEW;
+    private volatile STATUS status = STATUS.CLOSED;
     private boolean hasShownReadError = false;
 
     private static final String STORAGE_FORMAT_VERSION = "4";
@@ -86,7 +86,6 @@ public class LocalDBLogger implements PwmService
         Objects.requireNonNull( localDB, "localDB can not be null" );
 
         final Instant startTime = Instant.now();
-        status = STATUS.OPENING;
 
         this.settings = settings == null
                 ? LocalDBLoggerSettings.builder().build().applyValueChecks()
@@ -140,6 +139,7 @@ public class LocalDBLogger implements PwmService
 
         cleanOnWriteFlag.set( eventQueue.size() >= this.settings.getMaxEvents() );
 
+        status = STATUS.OPEN;
         LOGGER.info( () -> "open, " + debugStats(), () -> TimeDuration.fromCurrent( startTime ) );
     }
 
@@ -516,6 +516,11 @@ public class LocalDBLogger implements PwmService
     {
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication ) throws PwmException
+    {
+
+    }
 
     public String sizeToDebugString( )
     {

server/src/main/java/password/pwm/util/operations/CrService.java

@@ -106,6 +106,14 @@ public class CrService implements PwmService
         operatorMap.put( DataStorageMethod.NMAS, new NMASCrOperator( pwmApplication ) );
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/util/operations/OtpService.java

@@ -162,6 +162,14 @@ public class OtpService implements PwmService
         return otpCorrect;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     private List<String> createRawRecoveryCodes( final int numRecoveryCodes, final SessionLabel sessionLabel )
             throws PwmUnrecoverableException
     {

server/src/main/java/password/pwm/util/queue/SmsQueueManager.java

@@ -102,7 +102,7 @@ public class SmsQueueManager implements PwmService
 
     private WorkQueueProcessor<SmsItemBean> workQueueProcessor;
     private PwmApplication pwmApplication;
-    private STATUS status = STATUS.NEW;
+    private STATUS status = STATUS.CLOSED;
     private ErrorInformation lastError;
 
     public SmsQueueManager( )
@@ -114,7 +114,6 @@ public class SmsQueueManager implements PwmService
     )
             throws PwmException
     {
-        status = STATUS.OPENING;
         this.pwmApplication = pwmApplication;
         if ( pwmApplication.getLocalDB() == null || pwmApplication.getLocalDB().status() != LocalDB.Status.OPEN )
         {
@@ -141,6 +140,14 @@ public class SmsQueueManager implements PwmService
         status = STATUS.OPEN;
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     private class SmsItemProcessor implements WorkQueueProcessor.ItemProcessor<SmsItemBean>
     {
         @Override

server/src/main/java/password/pwm/util/secure/SecureService.java

@@ -74,6 +74,14 @@ public class SecureService implements PwmService
         }
     }
 
+    @Override
+    public void reInit( final PwmApplication pwmApplication )
+            throws PwmException
+    {
+        close();
+        init( pwmApplication );
+    }
+
     @Override
     public void close( )
     {

server/src/test/java/password/pwm/util/localdb/TestHelper.java

@@ -71,10 +71,13 @@ public class TestHelper
             throws PwmUnrecoverableException
     {
         Logger.getRootLogger().setLevel( Level.OFF );
-        final PwmEnvironment pwmEnvironment = new PwmEnvironment.Builder( configuration, tempFolder )
-                .setApplicationMode( PwmApplicationMode.READ_ONLY )
-                .setInternalRuntimeInstance( true )
-                .createPwmEnvironment();
+        final PwmEnvironment pwmEnvironment = PwmEnvironment.builder()
+                .config( configuration )
+                .applicationPath( tempFolder )
+                .applicationMode( PwmApplicationMode.READ_ONLY )
+                .internalRuntimeInstance( true )
+                .build();
+
         return PwmApplication.createPwmApplication( pwmEnvironment );
     }
 }

webapp/src/main/webapp/public/resources/js/configeditor.js

@@ -259,7 +259,7 @@ PWM_CFGEDIT.saveConfiguration = function() {
                     PWM_MAIN.showErrorDialog(data);
                 } else {
                     console.log('save completed');
-                    PWM_MAIN.showWaitDialog({title:'Save complete, restarting application...',loadFunction:function(){
+                    PWM_MAIN.showWaitDialog({title:'Save complete, applying configuration...',loadFunction:function(){
                             PWM_CONFIG.waitForRestart({location:'/'});
                         }});
                 }