contextmanager restart improvements

server/src/main/java/password/pwm/http/ContextManager.java

@@ -67,14 +67,14 @@ public class ContextManager implements Serializable
     private transient ServletContext servletContext;
     private transient ScheduledExecutorService taskMaster;
 
-    private transient PwmApplication pwmApplication;
+    private transient volatile PwmApplication pwmApplication;
     private transient ConfigurationReader configReader;
     private ErrorInformation startupErrorInformation;
 
-    private AtomicInteger restartCount = new AtomicInteger( 0 );
-    private TimeDuration readApplicationLockMaxWait = TimeDuration.of( 5, TimeDuration.Unit.SECONDS );
+    private final AtomicInteger restartCount = new AtomicInteger( 0 );
+    private TimeDuration readApplicationLockMaxWait = TimeDuration.SECONDS_30;
     private final String instanceGuid;
-    private final Lock reloadLock = new ReentrantLock();
+    private final Lock restartLock = new ReentrantLock();
 
     private String contextPath;
 
@@ -140,14 +140,26 @@ public class ContextManager implements Serializable
     public PwmApplication getPwmApplication( )
             throws PwmUnrecoverableException
     {
-        if ( pwmApplication != null )
+        PwmApplication localApplication = this.pwmApplication;
+
+        if ( localApplication == null )
         {
             try
             {
-                final boolean hasLock = reloadLock.tryLock( readApplicationLockMaxWait.asMillis(), TimeUnit.MILLISECONDS );
+                final Instant startTime = Instant.now();
+                final boolean hasLock = restartLock.tryLock( readApplicationLockMaxWait.asMillis(), TimeUnit.MILLISECONDS );
                 if ( hasLock )
                 {
-                    return pwmApplication;
+                    localApplication = this.pwmApplication;
+                    if ( localApplication == null )
+                    {
+                        LOGGER.trace( () -> "could not read pwmApplication after waiting " + TimeDuration.compactFromCurrent( startTime ) );
+                    }
+                    else
+                    {
+                        LOGGER.trace( () -> "waited " + TimeDuration.compactFromCurrent( startTime )
+                                + " to read pwmApplication due to restart in progress" );
+                    }
                 }
             }
             catch ( InterruptedException e )
@@ -156,10 +168,15 @@ public class ContextManager implements Serializable
             }
             finally
             {
-                reloadLock.unlock();
+                restartLock.unlock();
             }
         }
 
+        if ( localApplication != null )
+        {
+            return localApplication;
+        }
+
         final ErrorInformation errorInformation;
         if ( startupErrorInformation != null )
         {
@@ -400,10 +417,10 @@ public class ContextManager implements Serializable
 
         public void run( )
         {
-            doReinitialize();
+            doRestart();
         }
 
-        private void doReinitialize( )
+        private void doRestart( )
         {
             final Instant startTime = Instant.now();
 
@@ -415,11 +432,13 @@ public class ContextManager implements Serializable
                 return;
             }
 
-            reloadLock.lock();
+            restartLock.lock();
+            final PwmApplication oldPwmApplication = pwmApplication;
+            pwmApplication = null;
+
             try
             {
-
-                waitForRequestsToComplete( pwmApplication );
+                waitForRequestsToComplete( oldPwmApplication );
 
                 {
                     final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
@@ -429,7 +448,17 @@ public class ContextManager implements Serializable
                 final Instant shutdownStartTime = Instant.now();
                 try
                 {
-                    shutdown();
+                    try
+                    {
+                        // prevent restart watcher from detecting in-progress restart in a loop
+                        taskMaster.shutdown();
+
+                        oldPwmApplication.shutdown();
+                    }
+                    catch ( Exception e )
+                    {
+                        LOGGER.error( "unexpected error attempting to close application: " + e.getMessage() );
+                    }
                 }
                 catch ( Exception e )
                 {
@@ -453,7 +482,7 @@ public class ContextManager implements Serializable
             }
             finally
             {
-                reloadLock.unlock();
+                restartLock.unlock();
             }
         }
 

server/src/main/java/password/pwm/http/filter/RequestInitializationFilter.java

@@ -103,86 +103,76 @@ public class RequestInitializationFilter implements Filter
         final PwmApplicationMode mode = PwmApplicationMode.determineMode( req );
         final PwmURL pwmURL = new PwmURL( req );
 
-        PwmApplication testPwmApplicationLoad = null;
+        PwmApplication localPwmApplication = null;
         try
         {
-            testPwmApplicationLoad = ContextManager.getPwmApplication( req );
+            localPwmApplication = ContextManager.getPwmApplication( req );
         }
         catch ( PwmException e )
         {
+            LOGGER.trace( () -> "unable to load pwmApplication: " + e.getMessage() );
         }
 
-        if ( testPwmApplicationLoad != null && mode == PwmApplicationMode.RUNNING )
+        if ( localPwmApplication != null && mode == PwmApplicationMode.RUNNING )
         {
-            if ( testPwmApplicationLoad.getStatisticsManager() != null )
+            if ( localPwmApplication.getStatisticsManager() != null )
             {
-                testPwmApplicationLoad.getStatisticsManager().updateEps( EpsStatistic.REQUESTS, 1 );
+                localPwmApplication.getStatisticsManager().updateEps( EpsStatistic.REQUESTS, 1 );
             }
         }
 
-        try
+        if ( localPwmApplication == null && pwmURL.isResourceURL() )
         {
-            if ( testPwmApplicationLoad == null && pwmURL.isResourceURL() )
-            {
-                filterChain.doFilter( req, resp );
-                return;
-            }
+            filterChain.doFilter( req, resp );
+            return;
+        }
 
-            if ( testPwmApplicationLoad != null )
-            {
-                testPwmApplicationLoad.getInprogressRequests().incrementAndGet();
-            }
+        if ( pwmURL.isRestService() )
+        {
+            filterChain.doFilter( req, resp );
+            return;
+        }
 
-            if ( pwmURL.isRestService() )
-            {
-                filterChain.doFilter( req, resp );
-            }
-            else
+        if ( mode == PwmApplicationMode.ERROR || localPwmApplication == null )
+        {
+            try
             {
-                if ( mode == PwmApplicationMode.ERROR )
+                final ContextManager contextManager = ContextManager.getContextManager( req.getServletContext() );
+                if ( contextManager != null )
                 {
-                    try
-                    {
-                        final ContextManager contextManager = ContextManager.getContextManager( req.getServletContext() );
-                        if ( contextManager != null )
-                        {
-                            final ErrorInformation startupError = contextManager.getStartupErrorInformation();
-                            servletRequest.setAttribute( PwmRequestAttribute.PwmErrorInfo.toString(), startupError );
-                        }
-                    }
-                    catch ( Exception e )
-                    {
-                        if ( pwmURL.isResourceURL() )
-                        {
-                            filterChain.doFilter( servletRequest, servletResponse );
-                            return;
-                        }
-
-                        LOGGER.error( "error while trying to detect application status: " + e.getMessage() );
-                    }
-
-                    LOGGER.error( "unable to satisfy incoming request, application is not available" );
-                    resp.setStatus( 500 );
-                    final String url = JspUrl.APP_UNAVAILABLE.getPath();
-                    servletRequest.getServletContext().getRequestDispatcher( url ).forward( servletRequest, servletResponse );
+                    final ErrorInformation startupError = contextManager.getStartupErrorInformation();
+                    servletRequest.setAttribute( PwmRequestAttribute.PwmErrorInfo.toString(), startupError );
                 }
-                else
+            }
+            catch ( Exception e )
+            {
+                if ( pwmURL.isResourceURL() )
                 {
-                    initializeServletRequest( req, resp, filterChain );
+                    filterChain.doFilter( servletRequest, servletResponse );
+                    return;
                 }
+
+                LOGGER.error( "error while trying to detect application status: " + e.getMessage() );
             }
+
+            LOGGER.error( "unable to satisfy incoming request, application is not available" );
+            resp.setStatus( 500 );
+            final String url = JspUrl.APP_UNAVAILABLE.getPath();
+            servletRequest.getServletContext().getRequestDispatcher( url ).forward( servletRequest, servletResponse );
+            return;
+        }
+
+        try
+        {
+            localPwmApplication.getInprogressRequests().incrementAndGet();
+            initializeServletRequest( req, resp, filterChain );
         }
         finally
         {
-            if ( testPwmApplicationLoad != null )
-            {
-                testPwmApplicationLoad.getInprogressRequests().decrementAndGet();
-            }
+            localPwmApplication.getInprogressRequests().decrementAndGet();
         }
     }
 
-
-
     private void initializeServletRequest(
             final HttpServletRequest req,
             final HttpServletResponse resp,

server/src/main/resources/password/pwm/AppProperty.properties

@@ -26,7 +26,7 @@
 activateUser.token.autoSelectSingleDestination=false
 application.fileLock.filename=applicationPath.lock
 application.fileLock.waitSeconds=120
-application.readAppLock.maxWaitMs=5000
+application.readAppLock.maxWaitMs=30000
 application.restart.maxRequestWaitMs=3000
 application.wordlistRetryImportSeconds=600
 audit.events.emailFrom=Audit Event Notification <@DefaultEmailFromAddress@>