restart improvements

server/src/main/java/password/pwm/PwmAboutProperty.java

@@ -73,7 +73,7 @@ public enum PwmAboutProperty
     app_ldapProfileCount( null, pwmApplication -> Integer.toString( pwmApplication.getConfig().getLdapProfiles().size() ) ),
     app_ldapConnectionCount( null, pwmApplication -> Integer.toString( pwmApplication.getLdapConnectionService().connectionCount() ) ),
     app_activeSessionCount( "Active Session Count", pwmApplication -> Integer.toString( pwmApplication.getSessionTrackService().sessionCount() ) ),
-    app_activeRequestCount( "Active Request Count", pwmApplication -> Integer.toString( pwmApplication.getInprogressRequests().get() ) ),
+    app_activeRequestCount( "Active Request Count", pwmApplication -> Integer.toString( pwmApplication.getActiveServletRequests().get() ) ),
 
     build_Time( "Build Time", pwmApplication -> PwmConstants.BUILD_TIME ),
     build_Number( "Build Number", pwmApplication -> PwmConstants.BUILD_NUMBER ),

server/src/main/java/password/pwm/PwmApplication.java

@@ -117,30 +117,24 @@ public class PwmApplication
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( PwmApplication.class );
     private static final String DEFAULT_INSTANCE_ID = "-1";
-
-    private String instanceID = DEFAULT_INSTANCE_ID;
-    private String runtimeNonce = PwmRandom.getInstance().randomUUID().toString();
-
-    private LocalDB localDB;
-    private LocalDBLogger localDBLogger;
-
+    
     private final Instant startupTime = Instant.now();
+    private final AtomicInteger activeServletRequests = new AtomicInteger( 0 );
+    private final PwmServiceManager pwmServiceManager = new PwmServiceManager();
+
     private Instant installTime = Instant.now();
     private ErrorInformation lastLocalDBFailure;
-    private final AtomicInteger inprogressRequests = new AtomicInteger( 0 );
-
     private PwmEnvironment pwmEnvironment;
-
-    private final PwmServiceManager pwmServiceManager = new PwmServiceManager( this );
-
     private FileLocker fileLocker;
-
     private PwmScheduler pwmScheduler;
+    private String instanceID = DEFAULT_INSTANCE_ID;
+    private String runtimeNonce = PwmRandom.getInstance().randomUUID().toString();
+    private LocalDB localDB;
+    private LocalDBLogger localDBLogger;
 
-    private PwmApplication( final PwmEnvironment pwmEnvironment, final LocalDB localDB )
+    private PwmApplication( final PwmEnvironment pwmEnvironment )
             throws PwmUnrecoverableException
     {
-        this.localDB = localDB;
         this.pwmEnvironment = pwmEnvironment;
         pwmEnvironment.verifyIfApplicationPathIsSetProperly();
 
@@ -157,12 +151,7 @@ public class PwmApplication
 
     public static PwmApplication createPwmApplication( final PwmEnvironment pwmEnvironment ) throws PwmUnrecoverableException
     {
-        return new PwmApplication( pwmEnvironment, null );
-    }
-
-    public static PwmApplication createPwmApplication( final PwmEnvironment pwmEnvironment, final LocalDB localDB ) throws PwmUnrecoverableException
-    {
-        return new PwmApplication( pwmEnvironment, localDB );
+        return new PwmApplication( pwmEnvironment );
     }
 
     private void initialize( )
@@ -239,7 +228,7 @@ public class PwmApplication
 
         if ( !pwmEnvironment.isInternalRuntimeInstance() )
         {
-            if ( getApplicationMode() == PwmApplicationMode.ERROR || getApplicationMode() == PwmApplicationMode.NEW )
+            if ( getApplicationMode() == PwmApplicationMode.ERROR )
             {
                 LOGGER.warn( () -> "skipping LocalDB open due to application mode " + getApplicationMode() );
             }
@@ -270,7 +259,7 @@ public class PwmApplication
 
         pwmScheduler = new PwmScheduler( getInstanceID() );
 
-        pwmServiceManager.initAllServices();
+        pwmServiceManager.initAllServices( this );
 
         final boolean skipPostInit = pwmEnvironment.isInternalRuntimeInstance()
                 || pwmEnvironment.getFlags().contains( PwmEnvironment.ApplicationFlag.CommandLineInstance );
@@ -286,13 +275,16 @@ public class PwmApplication
         }
     }
 
-    public void reInit( final Configuration config )
+    public void reInit( final PwmEnvironment pwmEnvironment )
             throws PwmException
     {
-        fileLocker.releaseFileLock();
-        this.pwmEnvironment = pwmEnvironment.toBuilder().config( config ).build();
-        pwmServiceManager.reInitialize( this );
+        final Instant startTime = Instant.now();
+        LOGGER.debug( () -> "beginning application restart" );
+        shutdown( true );
+        this.pwmEnvironment = pwmEnvironment;
+        initialize();
         runtimeNonce = PwmRandom.getInstance().randomUUID().toString();
+        LOGGER.debug( () -> "completed application restart", () -> TimeDuration.fromCurrent( startTime ) );
     }
 
     private void postInitTasks( )
@@ -799,10 +791,10 @@ public class PwmApplication
 
     public void shutdown( )
     {
-        shutdown( true );
+        shutdown( false );
     }
 
-    public void shutdown( final boolean leaveLocalDbOpen )
+    public void shutdown( final boolean keepServicesRunning )
     {
         pwmScheduler.shutdown();
 
@@ -828,7 +820,10 @@ public class PwmApplication
 
         MBeanUtility.unregisterMBean( this );
 
-        pwmServiceManager.shutdownAllServices();
+        if ( !keepServicesRunning )
+        {
+            pwmServiceManager.shutdownAllServices();
+        }
 
         if ( localDBLogger != null )
         {
@@ -843,7 +838,7 @@ public class PwmApplication
             localDBLogger = null;
         }
 
-        if ( leaveLocalDbOpen )
+        if ( keepServicesRunning )
         {
             LOGGER.trace( () -> "skipping close of LocalDB (restart request)" );
         }
@@ -1045,9 +1040,9 @@ public class PwmApplication
         return false;
     }
 
-    public AtomicInteger getInprogressRequests( )
+    public AtomicInteger getActiveServletRequests( )
     {
-        return inprogressRequests;
+        return activeServletRequests;
     }
 
     public PwmScheduler getPwmScheduler()

server/src/main/java/password/pwm/health/HealthMonitor.java

@@ -128,14 +128,6 @@ public class HealthMonitor implements PwmService
         status = STATUS.OPEN;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     public Instant getLastHealthCheckTime( )
     {
         if ( status != STATUS.OPEN )

server/src/main/java/password/pwm/health/LDAPHealthChecker.java

@@ -85,7 +85,6 @@ import java.util.Set;
 
 public class LDAPHealthChecker implements HealthChecker
 {
-
     private static final PwmLogger LOGGER = PwmLogger.forClass( LDAPHealthChecker.class );
     private static final String TOPIC = "LDAP";
 

server/src/main/java/password/pwm/http/ContextManager.java

@@ -45,7 +45,6 @@ import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
-import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.secure.X509Utils;
 
@@ -83,7 +82,9 @@ public class ContextManager implements Serializable
 
     private static final TimeDuration RESTART_DELAY = TimeDuration.of( 5, TimeDuration.Unit.SECONDS );
 
-    private transient ServletContext servletContext;
+    private final transient ServletContext servletContext;
+    private final String contextPath;
+
     private transient ScheduledExecutorService taskMaster;
 
     private transient volatile PwmApplication pwmApplication;
@@ -94,7 +95,6 @@ public class ContextManager implements Serializable
     private TimeDuration readApplicationLockMaxWait = TimeDuration.of( 10, TimeDuration.Unit.SECONDS );
     private final AtomicBoolean restartInProgressFlag = new AtomicBoolean();
 
-    private String contextPath;
     private File applicationPath;
 
     private static final String UNSPECIFIED_VALUE = "unspecified";
@@ -105,7 +105,6 @@ public class ContextManager implements Serializable
         this.contextPath = servletContext.getContextPath();
     }
 
-
     public static PwmApplication getPwmApplication( final ServletRequest request ) throws PwmUnrecoverableException
     {
         final PwmApplication appInRequest = ( PwmApplication ) request.getAttribute( PwmConstants.REQUEST_ATTR_PWM_APPLICATION );
@@ -197,11 +196,6 @@ public class ContextManager implements Serializable
     }
 
     public void initialize( )
-    {
-        this.initialize( null );
-    }
-
-    private void initialize( final LocalDB preExistingLocalDB )
     {
         final Instant startTime = Instant.now();
 
@@ -217,7 +211,6 @@ public class ContextManager implements Serializable
         Configuration configuration = null;
         PwmApplicationMode mode = PwmApplicationMode.ERROR;
 
-
         final ParameterReader parameterReader = new ParameterReader( servletContext );
         {
             final String applicationPathStr = parameterReader.readApplicationPath();
@@ -279,7 +272,14 @@ public class ContextManager implements Serializable
                     .parameters( applicationParams )
                     .build();
 
-            pwmApplication = PwmApplication.createPwmApplication( pwmEnvironment, preExistingLocalDB );
+            if ( pwmApplication == null )
+            {
+                pwmApplication = PwmApplication.createPwmApplication( pwmEnvironment );
+            }
+            else
+            {
+                pwmApplication.reInit( pwmEnvironment );
+            }
         }
         catch ( final Exception e )
         {
@@ -516,7 +516,7 @@ public class ContextManager implements Serializable
 
                 try
                 {
-                        newReInit();
+                        reInitialize();
                 }
                 catch ( final Exception e )
                 {
@@ -541,7 +541,7 @@ public class ContextManager implements Serializable
             final TimeDuration maxRequestWaitTime = TimeDuration.of(
                     Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.APPLICATION_RESTART_MAX_REQUEST_WAIT_MS ) ),
                     TimeDuration.Unit.MILLISECONDS );
-            final int startingRequestInProgress = pwmApplication.getInprogressRequests().get();
+            final int startingRequestInProgress = pwmApplication.getActiveServletRequests().get();
 
             if ( startingRequestInProgress == 0 )
             {
@@ -550,10 +550,10 @@ public class ContextManager implements Serializable
 
             LOGGER.trace( SESSION_LABEL, () -> "waiting up to " + maxRequestWaitTime.asCompactString()
                     + " for " + startingRequestInProgress  + " requests to complete." );
-            maxRequestWaitTime.pause( TimeDuration.of( 10, TimeDuration.Unit.MILLISECONDS ), () -> pwmApplication.getInprogressRequests().get() == 0
+            maxRequestWaitTime.pause( TimeDuration.of( 10, TimeDuration.Unit.MILLISECONDS ), () -> pwmApplication.getActiveServletRequests().get() == 0
             );
 
-            final int requestsInProgress = pwmApplication.getInprogressRequests().get();
+            final int requestsInProgress = pwmApplication.getActiveServletRequests().get();
             final TimeDuration waitTime = TimeDuration.fromCurrent( startTime  );
             LOGGER.trace( SESSION_LABEL, () -> "after " + waitTime.asCompactString() + ", " + requestsInProgress
                     + " requests in progress, proceeding with restart" );
@@ -767,12 +767,8 @@ public class ContextManager implements Serializable
         }
     }
 
-    private void newReInit() throws PwmException
+    private void reInitialize() throws PwmException
     {
-        final File configurationFile = locateConfigurationFile( applicationPath, PwmConstants.DEFAULT_CONFIG_FILE_FILENAME );
-
-        configReader = new ConfigurationReader( configurationFile );
-        final Configuration configuration = configReader.getConfiguration();
-        pwmApplication.reInit( configuration );
+        initialize();
     }
 }

server/src/main/java/password/pwm/http/filter/RequestInitializationFilter.java

@@ -162,12 +162,12 @@ public class RequestInitializationFilter implements Filter
 
         try
         {
-            localPwmApplication.getInprogressRequests().incrementAndGet();
+            localPwmApplication.getActiveServletRequests().incrementAndGet();
             initializeServletRequest( req, resp, filterChain );
         }
         finally
         {
-            localPwmApplication.getInprogressRequests().decrementAndGet();
+            localPwmApplication.getActiveServletRequests().decrementAndGet();
         }
     }
 

server/src/main/java/password/pwm/http/servlet/admin/AppDashboardData.java

@@ -165,7 +165,7 @@ public class AppDashboardData implements Serializable
 
         builder.ldapConnectionCount( ldapConnectionCount( pwmApplication ) );
         builder.sessionCount( pwmApplication.getSessionTrackService().sessionCount() );
-        builder.requestsInProgress( pwmApplication.getInprogressRequests().get() );
+        builder.requestsInProgress( pwmApplication.getActiveServletRequests().get() );
 
         LOGGER.trace( () -> "AppDashboardData bean created in ", () -> TimeDuration.fromCurrent( startTime ) );
         return builder.build();

server/src/main/java/password/pwm/http/servlet/peoplesearch/PeopleSearchService.java

@@ -61,14 +61,6 @@ public class PeopleSearchService implements PwmService
         );
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     @Override
     public void close()
     {

server/src/main/java/password/pwm/http/servlet/resource/ResourceServletService.java

@@ -81,14 +81,6 @@ public class ResourceServletService implements PwmService
         return cacheHitRatio;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     public long bytesInCache( )
     {
         final Map<CacheKey, CacheEntry> cacheCopy = new HashMap<>( cache.asMap() );

server/src/main/java/password/pwm/http/state/SessionStateService.java

@@ -110,14 +110,6 @@ public class SessionStateService implements PwmService
         LOGGER.trace( () -> "initialized " + sessionBeanProvider.getClass().getName() + " provider" );
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/ldap/LdapConnectionService.java

@@ -76,11 +76,11 @@ public class LdapConnectionService implements PwmService
     private final Set<ThreadLocalContainer> threadLocalContainers = Collections.synchronizedSet( Collections.newSetFromMap( new WeakHashMap<>() ) );
     private final ReentrantLock reentrantLock = new ReentrantLock();
     private final ConditionalTaskExecutor debugLogger = ConditionalTaskExecutor.forPeriodicTask( this::conditionallyLogDebugInfo, TimeDuration.MINUTE );
-    private final ChaiProviderFactory chaiProviderFactory = ChaiProviderFactory.newProviderFactory();
     private final Map<String, Map<Integer, ChaiProvider>> proxyChaiProviders = new HashMap<>();
 
     private PwmApplication pwmApplication;
     private ExecutorService executorService;
+    private ChaiProviderFactory chaiProviderFactory;
     private AtomicLoopIntIncrementer slotIncrementer;
 
     private volatile STATUS status = STATUS.CLOSED;
@@ -116,6 +116,7 @@ public class LdapConnectionService implements PwmService
             throws PwmException
     {
         this.pwmApplication = pwmApplication;
+        this.chaiProviderFactory = ChaiProviderFactory.newProviderFactory();
 
         useThreadLocal = Boolean.parseBoolean( pwmApplication.getConfig().readAppProperty( AppProperty.LDAP_PROXY_USE_THREAD_LOCAL ) );
         LOGGER.trace( () -> "threadLocal enabled: " + useThreadLocal );
@@ -158,18 +159,12 @@ public class LdapConnectionService implements PwmService
         }
 
         proxyChaiProviders.clear();
-
+        lastLdapErrors.clear();
         iterateThreadLocals( container -> container.getProviderMap().clear() );
+        threadLocalContainers.clear();
         executorService.shutdown();
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication ) throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     public List<HealthRecord> healthCheck( )
     {
         return null;

server/src/main/java/password/pwm/ldap/search/UserSearchEngine.java

@@ -118,15 +118,6 @@ public class UserSearchEngine implements PwmService
         this.periodicDebugOutput();
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
-
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/svc/PwmService.java

@@ -39,7 +39,6 @@ import java.util.Map;
  */
 public interface PwmService
 {
-
     enum STATUS
     {
         OPEN,
@@ -50,8 +49,6 @@ public interface PwmService
 
     void init( PwmApplication pwmApplication ) throws PwmException;
 
-    void reInit( PwmApplication pwmApplication ) throws PwmException;
-
     void close( );
 
     List<HealthRecord> healthCheck( );

server/src/main/java/password/pwm/svc/PwmServiceManager.java

@@ -26,6 +26,7 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.java.StatisticIntCounterMap;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
@@ -45,9 +46,15 @@ public class PwmServiceManager
     private final Map<PwmServiceEnum, PwmService> runningServices = new HashMap<>();
     private boolean initialized;
 
-    public PwmServiceManager( final PwmApplication pwmApplication )
+    public PwmServiceManager(  )
     {
-        this.pwmApplication = pwmApplication;
+    }
+
+    private enum InitializationStats
+    {
+        starts,
+        stops,
+        restarts,
     }
 
     public PwmService getService( final PwmServiceEnum serviceClass )
@@ -55,67 +62,67 @@ public class PwmServiceManager
         return runningServices.get( serviceClass );
     }
 
-    public void initAllServices( )
+    public void initAllServices( final PwmApplication pwmApplication )
             throws PwmUnrecoverableException
     {
+        this.pwmApplication = pwmApplication;
         final Instant startTime = Instant.now();
 
         final boolean internalRuntimeInstance = pwmApplication.getPwmEnvironment().isInternalRuntimeInstance()
                 || pwmApplication.getPwmEnvironment().getFlags().contains( PwmEnvironment.ApplicationFlag.CommandLineInstance );
 
-        int serviceCounter = 0;
+        final String logVerb = initialized ? "restart" : "start";
+        final StatisticIntCounterMap<InitializationStats> statCounter = new StatisticIntCounterMap<>( InitializationStats.class );
+        LOGGER.trace( () -> "beginning service " + logVerb + " process" );
 
         for ( final PwmServiceEnum serviceClassEnum : PwmServiceEnum.values() )
         {
-            boolean startService = true;
+            boolean serviceShouldBeRunning = true;
             if ( internalRuntimeInstance && !serviceClassEnum.isInternalRuntime() )
             {
-                startService = false;
+                serviceShouldBeRunning = false;
             }
-            if ( startService )
+
+            if ( serviceShouldBeRunning )
             {
-                final Class<? extends PwmService> serviceClass = serviceClassEnum.getPwmServiceClass();
-                final PwmService newServiceInstance = initService( serviceClass );
+                if ( runningServices.containsKey( serviceClassEnum ) )
+                {
+                    shutDownService( serviceClassEnum, runningServices.get( serviceClassEnum ) );
+                    statCounter.increment( InitializationStats.restarts );
+                }
+                else
+                {
+                    statCounter.increment( InitializationStats.starts );
+                }
+
+                final PwmService newServiceInstance = initService( serviceClassEnum );
                 runningServices.put( serviceClassEnum, newServiceInstance );
-                serviceCounter++;
+            }
+            else
+            {
+                if ( runningServices.containsKey( serviceClassEnum ) )
+                {
+                    shutDownService( serviceClassEnum, runningServices.get( serviceClassEnum ) );
+                    statCounter.increment( InitializationStats.stops );
+                }
             }
         }
 
         initialized = true;
 
-        {
-            final int finalServiceCounter = serviceCounter;
-            LOGGER.trace( () -> "started " + finalServiceCounter + " services", () -> TimeDuration.fromCurrent( startTime ) );
-        }
+        LOGGER.trace( () -> logVerb + "ed services, " + statCounter.debugStats(), () -> TimeDuration.fromCurrent( startTime ) );
     }
 
-    public void reInitialize( final PwmApplication pwmApplication )
-    {
-        final Instant startTime = Instant.now();
-        this.pwmApplication = pwmApplication;
-        LOGGER.trace( () -> "beginning service restart process" );
-
-        int counter = 0;
-        for ( final Map.Entry<PwmServiceEnum, PwmService> entry : runningServices.entrySet() )
-        {
-            counter++;
-            reInitService( entry.getKey(), entry.getValue() );
-        }
-
-        {
-            final int totalRestarts = counter;
-            LOGGER.trace( () -> "completed restart of " + totalRestarts + " services", () -> TimeDuration.fromCurrent( startTime ) );
-        }
-    }
-
-    private PwmService initService( final Class<? extends PwmService> serviceClass )
+    private PwmService initService( final PwmServiceEnum pwmServiceEnum )
             throws PwmUnrecoverableException
     {
         final Instant startTime = Instant.now();
         final PwmService newServiceInstance;
-        final String serviceName = serviceClass.getName();
+
+        final String serviceName = pwmServiceEnum.serviceName();
         try
         {
+            final Class<? extends PwmService> serviceClass = pwmServiceEnum.getPwmServiceClass();
             newServiceInstance = serviceClass.getDeclaredConstructor().newInstance();
         }
         catch ( final Exception e )
@@ -151,24 +158,6 @@ public class PwmServiceManager
         return newServiceInstance;
     }
 
-    private void reInitService( final PwmServiceEnum serviceEnum, final PwmService newServiceInstance )
-    {
-        final Instant startTime = Instant.now();
-        final String serviceName = serviceEnum.serviceName();
-
-        try
-        {
-            LOGGER.debug( () -> "re-initializing service " + serviceName );
-            newServiceInstance.init( pwmApplication );
-            final TimeDuration startupDuration = TimeDuration.fromCurrent( startTime );
-            LOGGER.debug( () -> "completed initialization of service " + serviceName + " in " + startupDuration.asCompactString() + ", status=" + newServiceInstance.status() );
-        }
-        catch ( final PwmException e )
-        {
-            LOGGER.warn( () -> "error instantiating service class '" + serviceName + "', service will remain unavailable, error: " + e.getMessage() );
-        }
-    }
-
     public void shutdownAllServices( )
     {
         if ( !initialized )
@@ -194,7 +183,7 @@ public class PwmServiceManager
         LOGGER.trace( () -> "closed all services in ", () -> TimeDuration.fromCurrent( startTime ) );
     }
 
-    private void shutDownService( final PwmServiceEnum pwmServiceEnum, final PwmService loopService )
+    private void shutDownService( final PwmServiceEnum pwmServiceEnum, final PwmService serviceInstance )
     {
 
         LOGGER.trace( () -> "closing service " + pwmServiceEnum.serviceName() );
@@ -202,13 +191,13 @@ public class PwmServiceManager
         try
         {
             final Instant startTime = Instant.now();
-            loopService.close();
+            serviceInstance.close();
             final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
             LOGGER.trace( () -> "successfully closed service " + pwmServiceEnum.serviceName() + " (" + timeDuration.asCompactString() + ")" );
         }
         catch ( final Exception e )
         {
-            LOGGER.error( () -> "error closing " + loopService.getClass().getSimpleName() + ": " + e.getMessage(), e );
+            LOGGER.error( () -> "error closing " + pwmServiceEnum.serviceName() + ": " + e.getMessage(), e );
         }
     }
 

server/src/main/java/password/pwm/svc/cache/CacheService.java

@@ -94,14 +94,6 @@ public class CacheService implements PwmService
         status = STATUS.OPEN;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/svc/email/EmailServerUtil.java

@@ -435,7 +435,10 @@ public class EmailServerUtil
             try
             {
                 final Transport transport = EmailServerUtil.makeSmtpTransport( emailServer );
-                transport.isConnected();
+                if ( !transport.isConnected() )
+                {
+                    records.add( HealthRecord.forMessage( HealthMessage.Email_ConnectFailure, emailServer.getId(), "unable to connect" ) );
+                }
                 transport.close();
             }
             catch ( final Exception e )

server/src/main/java/password/pwm/svc/email/EmailService.java

@@ -133,22 +133,6 @@ public class EmailService implements PwmService
         statsLogger.conditionallyExecuteTask();
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        submitLock.lock();
-        try
-        {
-            close();
-            init( pwmApplication );
-        }
-        finally
-        {
-            submitLock.unlock();
-        }
-    }
-
     public void close( )
     {
         status = STATUS.CLOSED;

server/src/main/java/password/pwm/svc/event/AuditService.java

@@ -197,14 +197,6 @@ public class AuditService implements PwmService
         this.status = STATUS.OPEN;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/svc/httpclient/HttpClientService.java

@@ -76,14 +76,6 @@ public class HttpClientService implements PwmService
         this.pwmApplication = pwmApplication;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     @Override
     public void close()
     {

server/src/main/java/password/pwm/svc/intruder/IntruderManager.java

@@ -203,14 +203,6 @@ public class IntruderManager implements PwmService
         }
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     private void initializeRecordManagers( final Configuration config, final RecordStore recordStore )
     {
         {

server/src/main/java/password/pwm/svc/node/NodeService.java

@@ -121,14 +121,6 @@ public class NodeService implements PwmService
         status = STATUS.CLOSED;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/svc/pwnotify/PwNotifyService.java

@@ -154,14 +154,6 @@ public class PwNotifyService extends AbstractPwmService implements PwmService
         }
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     public Instant getNextExecutionTime( )
     {
         return nextExecutionTime;

server/src/main/java/password/pwm/svc/report/ReportService.java

@@ -147,14 +147,6 @@ public class ReportService implements PwmService
         status = STATUS.OPEN;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/svc/report/UserCacheService.java

@@ -151,13 +151,6 @@ public class UserCacheService implements PwmService
         status = STATUS.OPEN;
     }
 
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     public void close( )
     {
         status = STATUS.CLOSED;

server/src/main/java/password/pwm/svc/sessiontrack/SessionTrackService.java

@@ -80,14 +80,6 @@ public class SessionTrackService implements PwmService
         this.pwmApplication = pwmApplication;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/svc/shorturl/UrlShortenerService.java

@@ -25,7 +25,6 @@ import password.pwm.PwmApplication;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.option.DataStorageMethod;
-import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
 import password.pwm.svc.PwmService;
@@ -98,14 +97,6 @@ public class UrlShortenerService implements PwmService
         status = PwmService.STATUS.OPEN;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     public STATUS status( )
     {
         return status;

server/src/main/java/password/pwm/svc/stats/StatisticsManager.java

@@ -297,14 +297,6 @@ public class StatisticsManager implements PwmService
         status = STATUS.OPEN;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     private void writeDbValues( )
     {
         if ( localDB != null && status == STATUS.OPEN )

server/src/main/java/password/pwm/svc/telemetry/TelemetryService.java

@@ -149,14 +149,6 @@ public class TelemetryService implements PwmService
         status = STATUS.OPEN;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     private void initSender( ) throws PwmUnrecoverableException
     {
         if ( StringUtil.isEmpty( settings.getSenderImplementation() ) )

server/src/main/java/password/pwm/svc/token/TokenService.java

@@ -208,14 +208,6 @@ public class TokenService implements PwmService
         LOGGER.debug( () -> "open" );
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     public boolean supportsName( )
     {
         return tokenMachine.supportsName();

server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java

@@ -56,7 +56,7 @@ import java.util.function.BooleanSupplier;
 abstract class AbstractWordlist implements Wordlist, PwmService
 {
     static final TimeDuration DEBUG_OUTPUT_FREQUENCY = TimeDuration.MINUTE;
-    private static final TimeDuration BUCKECT_CHECK_LOG_WARNING_TIMEOUT = TimeDuration.of( 100, TimeDuration.Unit.MILLISECONDS );
+    private static final TimeDuration BUCKET_CHECK_LOG_WARNING_TIMEOUT = TimeDuration.of( 100, TimeDuration.Unit.MILLISECONDS );
 
     private WordlistConfiguration wordlistConfiguration;
     private WordlistBucket wordlistBucket;
@@ -119,6 +119,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
             this.wordlistBucket = new LocalDBWordlistBucket( pwmApplication, wordlistConfiguration, type );
         }
 
+        inhibitBackgroundImportFlag.set( false );
         executorService = PwmScheduler.makeBackgroundExecutor( pwmApplication, this.getClass() );
 
         if ( !pwmApplication.getPwmEnvironment().isInternalRuntimeInstance() )
@@ -129,14 +130,6 @@ abstract class AbstractWordlist implements Wordlist, PwmService
         getLogger().trace( () -> "opening with configuration: " + JsonUtil.serialize( wordlistConfiguration ) );
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     private void startTestInstance( final WordlistType wordlistType )
     {
         this.wordlistBucket = new MemoryWordlistBucket( pwmApplication, wordlistConfiguration, wordlistType );
@@ -210,7 +203,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
         final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
         getStatistics().getWordCheckTimeMS().update( timeDuration.asMillis() );
 
-        if ( timeDuration.isLongerThan( BUCKECT_CHECK_LOG_WARNING_TIMEOUT ) )
+        if ( timeDuration.isLongerThan( BUCKET_CHECK_LOG_WARNING_TIMEOUT ) )
         {
             getLogger().debug( () -> "wordlist search time for wordlist permutations was greater then 100ms: " + timeDuration.asCompactString() );
         }
@@ -265,7 +258,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
 
     public synchronized void close( )
     {
-        final TimeDuration closeWaitTime = TimeDuration.of( 5, TimeDuration.Unit.SECONDS );
+        final TimeDuration closeWaitTime = TimeDuration.of( 1, TimeDuration.Unit.MINUTES );
 
         wlStatus = STATUS.CLOSED;
         inhibitBackgroundImportFlag.set( true );
@@ -392,7 +385,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
                     wordlistZipReader,
                     WordlistSourceType.User,
                     AbstractWordlist.this,
-                    () -> wlStatus != STATUS.OPEN
+                    makeProcessCancelSupplier()
             );
             wordlistImporter.run();
             getLogger().debug( () -> "completed direct user-supplied wordlist import" );
@@ -432,7 +425,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
                 if ( !inhibitBackgroundImportFlag.get() )
                 {
                     activity = Wordlist.Activity.ReadingWordlistFile;
-                    final BooleanSupplier cancelFlag = inhibitBackgroundImportFlag::get;
+                    final BooleanSupplier cancelFlag = makeProcessCancelSupplier( );
                     backgroundImportRunning.set( true );
                     final WordlistInspector wordlistInspector = new WordlistInspector( pwmApplication, AbstractWordlist.this, cancelFlag );
                     wordlistInspector.run();
@@ -515,4 +508,10 @@ abstract class AbstractWordlist implements Wordlist, PwmService
 
         return "";
     }
+
+    private BooleanSupplier makeProcessCancelSupplier( )
+    {
+        return () -> inhibitBackgroundImportFlag.get()
+                        || !STATUS.OPEN.equals( wlStatus );
+    }
 }

server/src/main/java/password/pwm/svc/wordlist/SharedHistoryManager.java

@@ -89,14 +89,6 @@ public class SharedHistoryManager implements PwmService
         localDB = null;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     public boolean containsWord( final String word )
     {
         if ( status != STATUS.OPEN )

server/src/main/java/password/pwm/svc/wordlist/WordlistImporter.java

@@ -225,7 +225,7 @@ class WordlistImporter implements Runnable
 
             if ( cancelFlag.getAsBoolean() )
             {
-                getLogger().warn( () -> "pausing import" );
+                getLogger().debug( () -> "pausing import" );
             }
             else
             {

server/src/main/java/password/pwm/svc/wordlist/WordlistInspector.java

@@ -75,6 +75,11 @@ class WordlistInspector implements Runnable
 
         if ( checkIfClearIsNeeded( existingStatus, autoImportUrlConfigured ) )
         {
+            if ( cancelFlag.getAsBoolean() )
+            {
+                return;
+            }
+
             rootWordlist.clearImpl( Wordlist.Activity.ReadingWordlistFile );
         }
 

server/src/main/java/password/pwm/svc/wordlist/WordlistSource.java

@@ -20,6 +20,7 @@
 
 package password.pwm.svc.wordlist;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.input.CountingInputStream;
 import org.apache.commons.io.output.NullOutputStream;
 import password.pwm.AppProperty;
@@ -42,10 +43,12 @@ import java.io.InputStream;
 import java.net.URL;
 import java.security.DigestInputStream;
 import java.time.Instant;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BooleanSupplier;
 
 class WordlistSource
 {
+    private static final AtomicInteger CLOSE_COUNTER = new AtomicInteger();
 
     private final WordlistSourceType wordlistSourceType;
     private final StreamProvider streamProvider;
@@ -153,23 +156,23 @@ class WordlistSource
 
         final long bytes;
         final String hash;
-        try (
-                InputStream inputStream = this.streamProvider.getInputStream();
-                DigestInputStream checksumInputStream = pwmApplication.getSecureService().digestInputStream( WordlistConfiguration.HASH_ALGORITHM, inputStream );
-                CountingInputStream countingInputStream = new CountingInputStream( checksumInputStream );
-        )
+
+        InputStream inputStream = null;
+        DigestInputStream checksumInputStream = null;
+        CountingInputStream countingInputStream = null;
+
+        try
         {
-            final ConditionalTaskExecutor debugOutputter = new ConditionalTaskExecutor(
-                    () -> pwmLogger.debug( () -> "continuing reading file info for " + getWordlistSourceType() + " wordlist"
-                            + " " + StringUtil.formatDiskSizeforDebug( countingInputStream.getByteCount() )
-                            + " (" + TimeDuration.compactFromCurrent( startTime ) + ")" ),
-                    new ConditionalTaskExecutor.TimeDurationPredicate( AbstractWordlist.DEBUG_OUTPUT_FREQUENCY )
-            );
+            inputStream = this.streamProvider.getInputStream();
+            checksumInputStream = pwmApplication.getSecureService().digestInputStream( WordlistConfiguration.HASH_ALGORITHM, inputStream );
+            countingInputStream = new CountingInputStream( checksumInputStream );
+            final ConditionalTaskExecutor debugOutputter = makeDebugLoggerExecutor( pwmLogger, startTime, countingInputStream );
 
             JavaHelper.copyWhilePredicate(
                     countingInputStream,
                     new NullOutputStream(),
-                    WordlistConfiguration.STREAM_BUFFER_SIZE, o -> !cancelFlag.getAsBoolean(),
+                    WordlistConfiguration.STREAM_BUFFER_SIZE,
+                    o -> !cancelFlag.getAsBoolean(),
                     debugOutputter );
 
             bytes = countingInputStream.getByteCount();
@@ -183,6 +186,10 @@ class WordlistSource
             );
             throw new PwmUnrecoverableException( errorInformation );
         }
+        finally
+        {
+            closeStreams( pwmLogger, countingInputStream, checksumInputStream, inputStream );
+        }
 
         if ( cancelFlag.getAsBoolean() )
         {
@@ -195,6 +202,42 @@ class WordlistSource
                 + " " + StringUtil.formatDiskSizeforDebug( bytes )
                 + ", " + JsonUtil.serialize( wordlistSourceInfo )
                 + " (" + TimeDuration.compactFromCurrent( startTime ) + ")" );
+
         return wordlistSourceInfo;
     }
+
+    private ConditionalTaskExecutor makeDebugLoggerExecutor(
+            final PwmLogger pwmLogger,
+            final Instant startTime,
+            final CountingInputStream countingInputStream
+            )
+    {
+        return new ConditionalTaskExecutor(
+                () -> pwmLogger.debug( () -> "continuing reading file info for " + getWordlistSourceType() + " wordlist"
+                                + " " + StringUtil.formatDiskSizeforDebug( countingInputStream.getByteCount() ),
+                        () -> TimeDuration.fromCurrent( startTime ) ),
+                new ConditionalTaskExecutor.TimeDurationPredicate( AbstractWordlist.DEBUG_OUTPUT_FREQUENCY )
+        );
+    }
+
+    private void closeStreams( final PwmLogger pwmLogger, final InputStream... inputStreams )
+    {
+        // use a thread to close the io tasks as the close will block until the tcp socket is closed
+        final Thread closerThread = new Thread( () ->
+        {
+            final int counter = CLOSE_COUNTER.incrementAndGet();
+
+            final Instant startClose = Instant.now();
+            pwmLogger.trace( () -> "beginning close of remote wordlist read process [" + counter + "]" );
+            for ( final InputStream inputStream : inputStreams )
+            {
+                IOUtils.closeQuietly( inputStream );
+            }
+            pwmLogger.trace( () -> "completed close of remote wordlist read process [" + counter + "]",
+                    () -> TimeDuration.fromCurrent( startClose ) );
+        } );
+        closerThread.setDaemon( true );
+        closerThread.setName( Thread.currentThread().getName() + "-import-close-io" );
+        closerThread.start();
+    }
 }

server/src/main/java/password/pwm/util/db/DatabaseService.java

@@ -184,14 +184,6 @@ public class DatabaseService implements PwmService
         }
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/util/java/JavaHelper.java

@@ -313,20 +313,21 @@ public class JavaHelper
             throws IOException
     {
         final byte[] buffer = new byte[ bufferSize ];
-        long bytesCopied;
+        int bytesCopied;
         long totalCopied = 0;
         do
         {
-            bytesCopied = IOUtils.copyLarge( input, output, 0, bufferSize, buffer );
+            bytesCopied = input.read( buffer );
             if ( bytesCopied > 0 )
             {
+                output.write( buffer, 0, bytesCopied );
                 totalCopied += bytesCopied;
             }
             if ( conditionalTaskExecutor != null )
             {
                 conditionalTaskExecutor.conditionallyExecuteTask();
             }
-            if ( !predicate.test( bytesCopied ) )
+            if ( !predicate.test( totalCopied ) )
             {
                 return totalCopied;
             }

server/src/main/java/password/pwm/util/java/StatisticIntCounterMap.java

@@ -20,6 +20,8 @@
 
 package password.pwm.util.java;
 
+import com.novell.ldapchai.util.StringHelper;
+
 import java.util.Collections;
 import java.util.EnumMap;
 import java.util.EnumSet;
@@ -59,4 +61,9 @@ public class StatisticIntCounterMap<K extends Enum<K>>
                         ( entry ) -> String.valueOf( entry.getValue().get() ) ) ) );
     }
 
+    public String debugString()
+    {
+        return StringHelper.stringMapToString( debugStats(), null );
+    }
+
 }

server/src/main/java/password/pwm/util/localdb/LocalDBService.java

@@ -55,14 +55,6 @@ public class LocalDBService implements PwmService
         this.pwmApplication = pwmApplication;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/util/logging/LocalDBLogger.java

@@ -516,12 +516,6 @@ public class LocalDBLogger implements PwmService
     {
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication ) throws PwmException
-    {
-
-    }
-
     public String sizeToDebugString( )
     {
         final int storedEvents = this.getStoredEventCount();

server/src/main/java/password/pwm/util/logging/PwmLogManager.java

@@ -113,7 +113,6 @@ public class PwmLogManager
         java.util.logging.Logger.getLogger( "org.glassfish.jersey" ).setLevel( java.util.logging.Level.SEVERE );
     }
 
-
     public static void preInitConsoleLogLevel( final String pwmLogLevel )
     {
         try

server/src/main/java/password/pwm/util/operations/CrService.java

@@ -106,14 +106,6 @@ public class CrService implements PwmService
         operatorMap.put( DataStorageMethod.NMAS, new NMASCrOperator( pwmApplication ) );
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     @Override
     public void close( )
     {

server/src/main/java/password/pwm/util/operations/OtpService.java

@@ -162,14 +162,6 @@ public class OtpService implements PwmService
         return otpCorrect;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     private List<String> createRawRecoveryCodes( final int numRecoveryCodes, final SessionLabel sessionLabel )
             throws PwmUnrecoverableException
     {

server/src/main/java/password/pwm/util/queue/SmsQueueManager.java

@@ -140,14 +140,6 @@ public class SmsQueueManager implements PwmService
         status = STATUS.OPEN;
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     private class SmsItemProcessor implements WorkQueueProcessor.ItemProcessor<SmsItemBean>
     {
         @Override

server/src/main/java/password/pwm/util/secure/SecureService.java

@@ -74,14 +74,6 @@ public class SecureService implements PwmService
         }
     }
 
-    @Override
-    public void reInit( final PwmApplication pwmApplication )
-            throws PwmException
-    {
-        close();
-        init( pwmApplication );
-    }
-
     @Override
     public void close( )
     {