improve restart handling

build/checkstyle-import.xml

@@ -60,6 +60,8 @@
     <!-- gson -->
     <allow pkg="com.google.gson"/>
 
+    <allow pkg="javax.management"/>
+
     <!-- to be removed/scoped -->
     <allow pkg="org.apache.http"/>
     <allow pkg="org.apache.commons"/>

pom.xml

@@ -240,7 +240,7 @@
                     <dependency>
                         <groupId>com.github.spotbugs</groupId>
                         <artifactId>spotbugs</artifactId>
-                        <version>3.1.6</version>
+                        <version>3.1.7</version>
                     </dependency>
                 </dependencies>
                 <configuration>
@@ -272,7 +272,7 @@
         <dependency>
             <groupId>com.github.spotbugs</groupId>
             <artifactId>spotbugs-annotations</artifactId>
-            <version>3.1.6</version>
+            <version>3.1.7</version>
             <scope>provided</scope>
         </dependency>
     </dependencies>

server/src/main/java/password/pwm/AppProperty.java

@@ -35,6 +35,8 @@ public enum AppProperty
     ACTIVATE_USER_TOKEN_AUTO_SELECT_DEST            ( "activateUser.token.autoSelectSingleDestination" ),
     APPLICATION_FILELOCK_FILENAME                   ( "application.fileLock.filename" ),
     APPLICATION_FILELOCK_WAIT_SECONDS               ( "application.fileLock.waitSeconds" ),
+    APPLICATION_READ_APP_LOCK_MAX_WAIT_MS           ( "application.readAppLock.maxWaitMs" ),
+    APPLICATION_RESTART_MAX_REQUEST_WAIT_MS         ( "application.restart.maxRequestWaitMs" ),
     APPLICATION_WORDLIST_RETRY_SECONDS              ( "application.wordlistRetryImportSeconds" ),
     AUDIT_EVENTS_EMAILFROM                          ( "audit.events.emailFrom" ),
     AUDIT_EVENTS_EMAILSUBJECT                       ( "audit.events.emailSubject" ),

server/src/main/java/password/pwm/PwmApplication.java

@@ -60,6 +60,7 @@ import password.pwm.svc.token.TokenService;
 import password.pwm.svc.wordlist.SeedlistManager;
 import password.pwm.svc.wordlist.SharedHistoryManager;
 import password.pwm.svc.wordlist.WordlistManager;
+import password.pwm.util.MBeanUtility;
 import password.pwm.util.PasswordData;
 import password.pwm.util.cli.commands.ExportHttpsTomcatConfigCommand;
 import password.pwm.util.db.DatabaseAccessor;
@@ -96,6 +97,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * A repository for objects common to the servlet context.  A singleton
@@ -152,6 +154,7 @@ public class PwmApplication
     private final Instant startupTime = Instant.now();
     private Instant installTime = Instant.now();
     private ErrorInformation lastLocalDBFailure;
+    private final AtomicInteger inprogressRequests = new AtomicInteger( 0 );
 
     private final PwmEnvironment pwmEnvironment;
 
@@ -396,6 +399,8 @@ public class PwmApplication
             }
         }
 
+        MBeanUtility.registerMBean( this );
+
         LOGGER.trace( "completed post init tasks in " + TimeDuration.fromCurrent( startTime ).asCompactString() );
     }
 
@@ -775,6 +780,8 @@ public class PwmApplication
             }
         }
 
+        MBeanUtility.unregisterMBean( this );
+
         pwmServiceManager.shutdownAllServices();
 
         if ( localDBLogger != null )
@@ -984,6 +991,11 @@ public class PwmApplication
         }
         return false;
     }
+
+    public AtomicInteger getInprogressRequests( )
+    {
+        return inprogressRequests;
+    }
 }
 
 

server/src/main/java/password/pwm/PwmConstants.java

@@ -116,6 +116,7 @@ public abstract class PwmConstants
 
     public static final String REQUEST_ATTR_FORGOTTEN_PW_USERINFO_CACHE = "ForgottenPw-UserInfoCache";
     public static final String REQUEST_ATTR_FORGOTTEN_PW_AVAIL_TOKEN_DEST_CACHE = "ForgottenPw-AvailableTokenDestCache";
+    public static final String REQUEST_ATTR_PWM_APPLICATION = "PwmApplication";
 
     public static final PwmHashAlgorithm SETTING_CHECKSUM_HASH_METHOD = PwmHashAlgorithm.SHA256;
 

server/src/main/java/password/pwm/http/ContextManager.java

@@ -41,7 +41,7 @@ import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.secure.PwmRandom;
 
 import javax.servlet.ServletContext;
-import javax.servlet.http.HttpServletRequest;
+import javax.servlet.ServletRequest;
 import javax.servlet.http.HttpSession;
 import java.io.File;
 import java.io.InputStream;
@@ -56,6 +56,8 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 
 public class ContextManager implements Serializable
 {
@@ -70,7 +72,9 @@ public class ContextManager implements Serializable
     private ErrorInformation startupErrorInformation;
 
     private AtomicInteger restartCount = new AtomicInteger( 0 );
+    private TimeDuration readApplicationLockMaxWait = new TimeDuration( 5, TimeUnit.SECONDS );
     private final String instanceGuid;
+    private final Lock reloadLock = new ReentrantLock();
 
     private String contextPath;
 
@@ -84,9 +88,19 @@ public class ContextManager implements Serializable
     }
 
 
-    public static PwmApplication getPwmApplication( final HttpServletRequest request ) throws PwmUnrecoverableException
+    public static PwmApplication getPwmApplication( final ServletRequest request ) throws PwmUnrecoverableException
     {
-        return getPwmApplication( request.getServletContext() );
+        final PwmApplication appInRequest = ( PwmApplication ) request.getAttribute( PwmConstants.REQUEST_ATTR_PWM_APPLICATION );
+        if ( appInRequest != null )
+        {
+            return appInRequest;
+        }
+
+        final PwmApplication pwmApplication = getPwmApplication( request.getServletContext() );
+        request.setAttribute( PwmConstants.REQUEST_ATTR_PWM_APPLICATION, pwmApplication );
+        return pwmApplication;
+
+
     }
 
     public static PwmApplication getPwmApplication( final HttpSession session ) throws PwmUnrecoverableException
@@ -126,20 +140,37 @@ public class ContextManager implements Serializable
     public PwmApplication getPwmApplication( )
             throws PwmUnrecoverableException
     {
-        if ( pwmApplication == null )
+        if ( pwmApplication != null )
         {
-            final ErrorInformation errorInformation;
-            if ( startupErrorInformation != null )
+            try
             {
-                errorInformation = startupErrorInformation;
+                final boolean hasLock = reloadLock.tryLock( readApplicationLockMaxWait.getTotalMilliseconds(), TimeUnit.MICROSECONDS );
+                if ( hasLock )
+                {
+                    return pwmApplication;
+                }
             }
-            else
+            catch ( InterruptedException e )
             {
-                errorInformation = new ErrorInformation( PwmError.ERROR_APP_UNAVAILABLE, "application is not yet available, please try again in a moment." );
+                LOGGER.warn( "getPwmApplication restartLock unexpectedly interrupted" );
+            }
+            finally
+            {
+                reloadLock.unlock();
             }
-            throw new PwmUnrecoverableException( errorInformation );
         }
-        return pwmApplication;
+
+        final ErrorInformation errorInformation;
+        if ( startupErrorInformation != null )
+        {
+            errorInformation = startupErrorInformation;
+        }
+        else
+        {
+            final String msg = "application is not yet available, please try again in a moment.";
+            errorInformation = new ErrorInformation( PwmError.ERROR_APP_UNAVAILABLE, msg );
+        }
+        throw new PwmUnrecoverableException( errorInformation );
     }
 
     public void initialize( )
@@ -232,6 +263,11 @@ public class ContextManager implements Serializable
             {
                 reloadOnChange = Boolean.parseBoolean( pwmApplication.getConfig().readAppProperty( AppProperty.CONFIG_RELOAD_ON_CHANGE ) );
                 fileScanFrequencyMs = Long.parseLong( pwmApplication.getConfig().readAppProperty( AppProperty.CONFIG_FILE_SCAN_FREQUENCY ) );
+
+                this.readApplicationLockMaxWait = new TimeDuration(
+                        Long.parseLong( pwmApplication.getConfig().readAppProperty( AppProperty.APPLICATION_READ_APP_LOCK_MAX_WAIT_MS ) ),
+                        TimeUnit.MILLISECONDS
+                );
             }
             if ( reloadOnChange )
             {
@@ -375,31 +411,73 @@ public class ContextManager implements Serializable
                 return;
             }
 
-            {
-                final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
-                LOGGER.info( "beginning application restart (" + timeDuration.asCompactString() + "), restart count=" + restartCount.incrementAndGet() );
-            }
-
+            reloadLock.lock();
             try
             {
-                shutdown();
+
+                waitForRequestsToComplete( pwmApplication );
+
+                {
+                    final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
+                    LOGGER.info( "beginning application restart (" + timeDuration.asCompactString() + "), restart count=" + restartCount.incrementAndGet() );
+                }
+
+                final Instant shutdownStartTime = Instant.now();
+                try
+                {
+                    shutdown();
+                }
+                catch ( Exception e )
+                {
+                    LOGGER.fatal( "unexpected error during shutdown: " + e.getMessage(), e );
+                }
+
+                {
+                    final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
+                    final TimeDuration shutdownDuration = TimeDuration.fromCurrent( shutdownStartTime );
+                    LOGGER.info( "application restart; shutdown completed, ("
+                            + shutdownDuration.asCompactString()
+                            + ") now starting new application instance ("
+                            + timeDuration.asCompactString() + ")" );
+                }
+                initialize();
+
+                {
+                    final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
+                    LOGGER.info( "application restart completed (" + timeDuration.asCompactString() + ")" );
+                }
             }
-            catch ( Exception e )
+            finally
             {
-                LOGGER.fatal( "unexpected error during shutdown: " + e.getMessage(), e );
+                reloadLock.unlock();
             }
+        }
 
-            {
-                final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
-                LOGGER.info( "application restart; shutdown completed, now starting new application instance ("
-                        + timeDuration.asCompactString() + ")" );
-            }
-            initialize();
+        private void waitForRequestsToComplete( final PwmApplication pwmApplication )
+        {
+            final Instant startTime = Instant.now();
+            final TimeDuration maxRequestWaitTime = TimeDuration.of(
+                    Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.APPLICATION_RESTART_MAX_REQUEST_WAIT_MS ) ),
+                    TimeUnit.SECONDS );
+            final int startingRequetsInProgress = pwmApplication.getInprogressRequests().get();
 
+            if ( startingRequetsInProgress == 0 )
             {
-                final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
-                LOGGER.info( "application restart completed (" + timeDuration.asCompactString() + ")" );
+                return;
             }
+
+            LOGGER.trace( "waiting up to " + maxRequestWaitTime.asCompactString()
+                    + " for " + startingRequetsInProgress  + " requests to complete." );
+            JavaHelper.pause(
+                    maxRequestWaitTime.getTotalMilliseconds(),
+                    10,
+                    o -> pwmApplication.getInprogressRequests().get() == 0
+            );
+
+            final int requestsInPrgoress = pwmApplication.getInprogressRequests().get();
+            final TimeDuration waitTime = TimeDuration.fromCurrent( startTime  );
+            LOGGER.trace( "after " + waitTime.asCompactString() + ", " + requestsInPrgoress
+                    + " requests in progress, proceeding with restart" );
         }
     }
 

server/src/main/java/password/pwm/http/HttpEventManager.java

@@ -62,7 +62,7 @@ public class HttpEventManager implements
             final PwmApplication pwmApplication = contextManager.getPwmApplication();
             httpSession.setAttribute( PwmConstants.SESSION_ATTR_PWM_APP_NONCE, pwmApplication.getRuntimeNonce() );
 
-            if ( pwmApplication != null && pwmApplication.getStatisticsManager() != null )
+            if ( pwmApplication.getStatisticsManager() != null )
             {
                 pwmApplication.getStatisticsManager().updateEps( EpsStatistic.SESSIONS, 1 );
             }
@@ -87,7 +87,7 @@ public class HttpEventManager implements
                 {
                     pwmSession.unauthenticateUser( null );
                 }
-                final PwmApplication pwmApplication = ContextManager.getPwmApplication( httpSession );
+                final PwmApplication pwmApplication = ContextManager.getPwmApplication( httpSession.getServletContext() );
                 if ( pwmApplication != null )
                 {
                     pwmApplication.getSessionTrackService().removeSessionData( pwmSession );
@@ -169,7 +169,7 @@ public class HttpEventManager implements
             final HttpSession httpSession = event.getSession();
             final PwmSession pwmSession = PwmSessionWrapper.readPwmSession( httpSession );
             LOGGER.trace( pwmSession.getLabel(), "activating (de-passivating) session" );
-            final PwmApplication pwmApplication = ContextManager.getPwmApplication( httpSession );
+            final PwmApplication pwmApplication = ContextManager.getPwmApplication( httpSession.getServletContext() );
             if ( pwmApplication != null )
             {
                 pwmApplication.getSessionTrackService().addSessionData( pwmSession );

server/src/main/java/password/pwm/http/PwmSession.java

@@ -377,7 +377,13 @@ public class PwmSession implements Serializable
 
         if ( nonce == null || nonce.length() != length )
         {
-            nonce = pwmRequest.getPwmApplication().getSecureService().pwmRandom().alphaNumericString( length );
+            // random value
+            final String random = pwmRequest.getPwmApplication().getSecureService().pwmRandom().alphaNumericString( length );
+
+            // timestamp component for uniqueness
+            final String prefix = Long.toString( System.currentTimeMillis(), Character.MAX_RADIX );
+
+            nonce = random + prefix;
         }
 
         final PasswordData configSecret = pwmRequest.getConfig().readSettingAsPassword( PwmSetting.PWM_SECURITY_KEY );

server/src/main/java/password/pwm/http/filter/RequestInitializationFilter.java

@@ -120,51 +120,69 @@ public class RequestInitializationFilter implements Filter
             }
         }
 
-        if ( testPwmApplicationLoad == null && pwmURL.isResourceURL() )
-        {
-            filterChain.doFilter( req, resp );
-        }
-        else if ( pwmURL.isRestService() )
-        {
-            filterChain.doFilter( req, resp );
-        }
-        else
+        try
         {
-            if ( mode == PwmApplicationMode.ERROR )
+            if ( testPwmApplicationLoad == null && pwmURL.isResourceURL() )
+            {
+                filterChain.doFilter( req, resp );
+                return;
+            }
+
+            if ( testPwmApplicationLoad != null )
+            {
+                testPwmApplicationLoad.getInprogressRequests().incrementAndGet();
+            }
+
+            if ( pwmURL.isRestService() )
+            {
+                filterChain.doFilter( req, resp );
+            }
+            else
             {
-                try
+                if ( mode == PwmApplicationMode.ERROR )
                 {
-                    final ContextManager contextManager = ContextManager.getContextManager( req.getServletContext() );
-                    if ( contextManager != null )
+                    try
                     {
-                        final ErrorInformation startupError = contextManager.getStartupErrorInformation();
-                        servletRequest.setAttribute( PwmRequestAttribute.PwmErrorInfo.toString(), startupError );
+                        final ContextManager contextManager = ContextManager.getContextManager( req.getServletContext() );
+                        if ( contextManager != null )
+                        {
+                            final ErrorInformation startupError = contextManager.getStartupErrorInformation();
+                            servletRequest.setAttribute( PwmRequestAttribute.PwmErrorInfo.toString(), startupError );
+                        }
                     }
-                }
-                catch ( Exception e )
-                {
-                    if ( pwmURL.isResourceURL() )
+                    catch ( Exception e )
                     {
-                        filterChain.doFilter( servletRequest, servletResponse );
-                        return;
+                        if ( pwmURL.isResourceURL() )
+                        {
+                            filterChain.doFilter( servletRequest, servletResponse );
+                            return;
+                        }
+
+                        LOGGER.error( "error while trying to detect application status: " + e.getMessage() );
                     }
 
-                    LOGGER.error( "error while trying to detect application status: " + e.getMessage() );
+                    LOGGER.error( "unable to satisfy incoming request, application is not available" );
+                    resp.setStatus( 500 );
+                    final String url = JspUrl.APP_UNAVAILABLE.getPath();
+                    servletRequest.getServletContext().getRequestDispatcher( url ).forward( servletRequest, servletResponse );
+                }
+                else
+                {
+                    initializeServletRequest( req, resp, filterChain );
                 }
-
-                LOGGER.error( "unable to satisfy incoming request, application is not available" );
-                resp.setStatus( 500 );
-                final String url = JspUrl.APP_UNAVAILABLE.getPath();
-                servletRequest.getServletContext().getRequestDispatcher( url ).forward( servletRequest, servletResponse );
             }
-            else
+        }
+        finally
+        {
+            if ( testPwmApplicationLoad != null )
             {
-                initializeServletRequest( req, resp, filterChain );
+                testPwmApplicationLoad.getInprogressRequests().decrementAndGet();
             }
         }
     }
 
 
+
     private void initializeServletRequest(
             final HttpServletRequest req,
             final HttpServletResponse resp,

server/src/main/java/password/pwm/http/servlet/admin/AppDashboardData.java

@@ -115,9 +115,10 @@ public class AppDashboardData implements Serializable
     private String nodeSummary;
     private int ldapConnectionCount;
     private int sessionCount;
+    private int requestsInProgress;
 
 
-    static AppDashboardData makeDashboardData(
+    public static AppDashboardData makeDashboardData(
             final PwmApplication pwmApplication,
             final ContextManager contextManager,
             final Locale locale,
@@ -157,6 +158,7 @@ public class AppDashboardData implements Serializable
 
         builder.ldapConnectionCount( ldapConnectionCount( pwmApplication ) );
         builder.sessionCount( pwmApplication.getSessionTrackService().sessionCount() );
+        builder.requestsInProgress( pwmApplication.getInprogressRequests().get() );
 
         LOGGER.trace( "AppDashboardData bean created in " + TimeDuration.compactFromCurrent( startTime ) );
         return builder.build();

server/src/main/java/password/pwm/http/servlet/configmanager/DebugItemGenerator.java

@@ -33,7 +33,9 @@ import password.pwm.config.stored.StoredConfigurationImpl;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthMonitor;
 import password.pwm.health.HealthRecord;
+import password.pwm.http.ContextManager;
 import password.pwm.http.PwmRequest;
+import password.pwm.http.servlet.admin.AppDashboardData;
 import password.pwm.http.servlet.admin.UserDebugDataBean;
 import password.pwm.http.servlet.admin.UserDebugDataReader;
 import password.pwm.ldap.LdapDebugDataGenerator;
@@ -89,6 +91,7 @@ public class DebugItemGenerator
             ConfigurationDebugJsonItemGenerator.class,
             ConfigurationDebugTextItemGenerator.class,
             AboutItemGenerator.class,
+            DashboardDataDebugItemGenerator.class,
             SystemEnvironmentItemGenerator.class,
             AppPropertiesItemGenerator.class,
             ServicesDebugItemGenerator.class,
@@ -710,9 +713,34 @@ public class DebugItemGenerator
             final Map<String, Serializable> debugOutput = new LinkedHashMap<>( cacheService.debugInfo() );
             outputStream.write( JsonUtil.serializeMap( debugOutput, JsonUtil.Flag.PrettyPrint ).getBytes( PwmConstants.DEFAULT_CHARSET ) );
         }
-
     }
 
+    static class DashboardDataDebugItemGenerator implements Generator
+    {
+        @Override
+        public String getFilename( )
+        {
+            return "dashboard-data.json";
+        }
+
+        @Override
+        public void outputItem(
+                final PwmApplication pwmApplication,
+                final PwmRequest pwmRequest,
+                final OutputStream outputStream
+        )
+                throws Exception
+        {
+            final ContextManager contextManager = ContextManager.getContextManager( pwmRequest );
+            final AppDashboardData appDashboardData = AppDashboardData.makeDashboardData(
+                    pwmApplication,
+                    contextManager,
+                    pwmRequest.getLocale()
+            );
+
+            outputStream.write( JsonUtil.serialize( appDashboardData, JsonUtil.Flag.PrettyPrint ).getBytes( PwmConstants.DEFAULT_CHARSET ) );
+        }
+    }
 
     interface Generator
     {

server/src/main/java/password/pwm/http/tag/ErrorMessageTag.java

@@ -55,7 +55,7 @@ public class ErrorMessageTag extends PwmAbstractTag
             PwmApplication pwmApplication = null;
             try
             {
-                pwmApplication = ContextManager.getPwmApplication( pageContext.getSession() );
+                pwmApplication = ContextManager.getPwmApplication( pageContext.getRequest() );
             }
             catch ( PwmException e )
             { /* noop */ }

server/src/main/java/password/pwm/svc/PwmServiceManager.java

@@ -61,10 +61,13 @@ public class PwmServiceManager
     public void initAllServices( )
             throws PwmUnrecoverableException
     {
+        final Instant startTime = Instant.now();
 
         final boolean internalRuntimeInstance = pwmApplication.getPwmEnvironment().isInternalRuntimeInstance()
                 || pwmApplication.getPwmEnvironment().getFlags().contains( PwmEnvironment.ApplicationFlag.CommandLineInstance );
 
+        int serviceCounter = 0;
+
         for ( final PwmServiceEnum serviceClassEnum : PwmServiceEnum.values() )
         {
             boolean startService = true;
@@ -77,9 +80,14 @@ public class PwmServiceManager
                 final Class<? extends PwmService> serviceClass = serviceClassEnum.getPwmServiceClass();
                 final PwmService newServiceInstance = initService( serviceClass );
                 runningServices.put( serviceClass, newServiceInstance );
+                serviceCounter++;
             }
         }
+
         initialized = true;
+
+        final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
+        LOGGER.trace( "started " + serviceCounter + " services in " + timeDuration.asCompactString() );
     }
 
     private PwmService initService( final Class<? extends PwmService> serviceClass )

server/src/main/java/password/pwm/svc/cluster/ClusterMachine.java

@@ -72,7 +72,7 @@ class ClusterMachine
 
         this.executorService.scheduleAtFixedRate(
                 new HeartbeatProcess(),
-                1,
+                intervalSeconds,
                 intervalSeconds,
                 TimeUnit.SECONDS
         );

server/src/main/java/password/pwm/util/MBeanUtility.java

@@ -0,0 +1,118 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+
+package password.pwm.util;
+
+import lombok.Getter;
+import password.pwm.PwmAboutProperty;
+import password.pwm.PwmApplication;
+import password.pwm.PwmConstants;
+import password.pwm.util.logging.PwmLogger;
+
+import javax.management.Attribute;
+import javax.management.AttributeList;
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import java.lang.management.ManagementFactory;
+import java.util.HashMap;
+import java.util.Map;
+
+public class MBeanUtility
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( MBeanUtility.class );
+
+    private MBeanUtility( )
+    {
+    }
+
+    public static void registerMBean( final PwmApplication pwmApplication )
+    {
+        try
+        {
+            final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+            final ObjectName name = figureMBeanName( pwmApplication );
+            final Map<PwmAboutProperty, String> aboutMap = PwmAboutProperty.makeInfoBean( pwmApplication );
+            final Map<String, String> outputMap = new HashMap<>(  );
+            final AttributeList attributeList = new AttributeList(  );
+            for ( final Map.Entry<PwmAboutProperty, String> entry : aboutMap.entrySet() )
+            {
+                outputMap.put( entry.getKey().name(), entry.getValue() );
+                attributeList.add( new Attribute( entry.getKey().name(), entry.getValue() ) );
+            }
+            final PwmAbout mbean = new PwmAbout( outputMap );
+            mbs.registerMBean( mbean, name );
+            mbs.setAttributes( name, attributeList );
+        }
+        catch ( Exception e )
+        {
+            LOGGER.error( "error registering mbean: " + e.getMessage() );
+        }
+    }
+
+    public static void unregisterMBean( final PwmApplication pwmApplication )
+    {
+        try
+        {
+            final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+            mbs.unregisterMBean( figureMBeanName( pwmApplication ) );
+        }
+        catch ( Exception e )
+        {
+            LOGGER.error( "error unregistering mbean: " + e.getMessage() );
+        }
+    }
+
+    private static ObjectName figureMBeanName( final PwmApplication pwmApplication )
+            throws MalformedObjectNameException
+    {
+        final String context;
+        if ( pwmApplication.getPwmEnvironment() != null && pwmApplication.getPwmEnvironment().getContextManager() != null )
+        {
+            context = "-" + pwmApplication.getPwmEnvironment().getContextManager().getContextPath();
+        }
+        else
+        {
+            context = "";
+        }
+        final String mbeanName = "password.pwm:type=About" + PwmConstants.PWM_APP_NAME.toUpperCase() + context;
+        return new ObjectName( mbeanName );
+    }
+
+
+    public interface PwmAboutMXBean
+    {
+        Map<String, String> getAboutInfoMap();
+    }
+
+    @Getter
+    public static class PwmAbout implements PwmAboutMXBean
+    {
+        final Map<String, String> aboutInfoMap;
+
+        public PwmAbout( final Map<String, String> aboutInfoMap )
+        {
+            this.aboutInfoMap = aboutInfoMap;
+        }
+    }
+}

server/src/main/java/password/pwm/util/secure/PwmRandom.java

@@ -31,14 +31,15 @@ import java.util.stream.LongStream;
 public class PwmRandom extends SecureRandom
 {
 
-    private final SecureRandom internalRand = new SecureRandom();
+    private final SecureRandom internalRand;
 
-    private static final PwmRandom SINGLETON = new PwmRandom();
+    private static final PwmRandom SINGLETON = new PwmRandom( new SecureRandom( ) );
 
     private static final String ALPHANUMERIC_STRING = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
 
-    private PwmRandom( )
+    public PwmRandom( final SecureRandom internalRand )
     {
+        this.internalRand = internalRand;
     }
 
     public static PwmRandom getInstance( )

server/src/main/resources/password/pwm/AppProperty.properties

@@ -26,6 +26,8 @@
 activateUser.token.autoSelectSingleDestination=false
 application.fileLock.filename=applicationPath.lock
 application.fileLock.waitSeconds=120
+application.readAppLock.maxWaitMs=5000
+application.restart.maxRequestWaitMs=3000
 application.wordlistRetryImportSeconds=600
 audit.events.emailFrom=Audit Event Notification <@DefaultEmailFromAddress@>
 audit.events.emailSubject=@PwmAppName@ - Audit Event - %EVENT%

server/src/main/resources/password/pwm/i18n/Admin.properties

@@ -327,6 +327,7 @@ Title_DataAnalysis=Data Analysis
 Title_Dashboard=Dashboard
 Title_LogViewer=Log Viewer
 Title_TokenLookup=Token Search
+Title_RequestsInProgress=Requests in Progress
 Title_URLReference=URL Reference
 MenuItem_ConfigEditor=Configuration Editor
 MenuItem_ConfigManager=Configuration Manager

server/src/test/java/password/pwm/http/client/PwmHttpClientTest.java

@@ -75,7 +75,7 @@ public class PwmHttpClientTest {
         wm.stubFor(get(urlEqualTo("/simpleHello"))
             .willReturn(aResponse()
                 .withHeader("Content-Type", "text/plain")
-                .withBody("Hello from the local mock server")));
+                .withBody("PwmAbout from the local mock server")));
 
         // Obtain the HTTP client from PWM
         HttpClient httpClient = PwmHttpClient.getHttpClient(configuration);
@@ -89,7 +89,7 @@ public class PwmHttpClientTest {
         assertThat(responseStatusCode).isEqualTo(200);
 
         String responseContent = IOUtils.toString(response.getEntity().getContent());
-        assertThat(responseContent).startsWith("Hello");
+        assertThat(responseContent).startsWith("PwmAbout");
 
         // Verify the HTTP server got called as expected
         wm.verify(getRequestedFor(urlEqualTo("/simpleHello"))
@@ -105,7 +105,7 @@ public class PwmHttpClientTest {
         wm.stubFor(get(urlEqualTo("/simpleHello"))
             .willReturn(aResponse()
                 .withHeader("Content-Type", "text/plain")
-                .withBody("Hello from the local mock server")));
+                .withBody("PwmAbout from the local mock server")));
 
         HttpClient httpClient = PwmHttpClient.getHttpClient(configuration);
 
@@ -124,7 +124,7 @@ public class PwmHttpClientTest {
         wm.stubFor(get(urlEqualTo("/simpleHello"))
             .willReturn(aResponse()
                 .withHeader("Content-Type", "text/plain")
-                .withBody("Hello from the local mock server")));
+                .withBody("PwmAbout from the local mock server")));
 
         // Stub out some mock object behavior
         when(configuration.readAppProperty(AppProperty.SECURITY_HTTP_PROMISCUOUS_ENABLE)).thenReturn("true");
@@ -139,7 +139,7 @@ public class PwmHttpClientTest {
         assertThat(responseStatusCode).isEqualTo(200);
 
         String responseContent = IOUtils.toString(response.getEntity().getContent());
-        assertThat(responseContent).startsWith("Hello");
+        assertThat(responseContent).startsWith("PwmAbout");
     }
 
     /**
@@ -151,7 +151,7 @@ public class PwmHttpClientTest {
         wm.stubFor(get(urlEqualTo("/simpleHello"))
             .willReturn(aResponse()
                 .withHeader("Content-Type", "text/plain")
-                .withBody("Hello from the local mock server")));
+                .withBody("PwmAbout from the local mock server")));
 
         PwmHttpClientConfiguration pwmHttpClientConfiguration = PwmHttpClientConfiguration.builder()
                 .certificates(getWireMockSelfSignedCertificate())
@@ -167,7 +167,7 @@ public class PwmHttpClientTest {
         assertThat(responseStatusCode).isEqualTo(200);
 
         String responseContent = IOUtils.toString(response.getEntity().getContent());
-        assertThat(responseContent).startsWith("Hello");
+        assertThat(responseContent).startsWith("PwmAbout");
     }
 
     /**
@@ -179,7 +179,7 @@ public class PwmHttpClientTest {
         wm.stubFor(get(urlEqualTo("/simpleHello"))
             .willReturn(aResponse()
                 .withHeader("Content-Type", "text/plain")
-                .withBody("Hello from the local mock server")));
+                .withBody("PwmAbout from the local mock server")));
 
         // Stub out some mock object behavior
         when(configuration.readSettingAsString(PwmSetting.HTTP_PROXY_URL)).thenReturn(String.format("http://localhost:%d/simpleHello", wm.port()));
@@ -195,7 +195,7 @@ public class PwmHttpClientTest {
         assertThat(responseStatusCode).isEqualTo(200);
 
         String responseContent = IOUtils.toString(response.getEntity().getContent());
-        assertThat(responseContent).startsWith("Hello");
+        assertThat(responseContent).startsWith("PwmAbout");
     }
 
     private List<X509Certificate> getWireMockSelfSignedCertificate() {

webapp/src/main/webapp/WEB-INF/jsp/admin-dashboard.jsp

@@ -69,12 +69,23 @@
                         </td>
                         <td class="key">
                             <pwm:display key="Title_LDAPConnections" bundle="Admin"/>
-
                         </td>
                         <td id="LDAPConnectionCount">
                             <%= appDashboardData.getLdapConnectionCount() %>
                         </td>
                     </tr>
+                    <tr>
+                        <td class="key">
+                            <pwm:display key="Title_RequestsInProgress" bundle="Admin"/>
+                        </td>
+                        <td id="RequestsInProgress">
+                            <%= appDashboardData.getRequestsInProgress() %>
+                        </td>
+                        <td>
+                        </td>
+                        <td>
+                        </td>
+                    </tr>
                 </table>
                 <table class="nomargin">
                     <tr>

webapp/src/main/webapp/WEB-INF/jsp/admin-logview.jsp

@@ -107,11 +107,11 @@
                     <p>
                         This page shows the debug log
                         history. This records shown here are stored in the LocalDB.  The LocalDB contains <%=JspUtility.friendlyWrite( pageContext, localDBLogger.getStoredEventCount() )%> events. The oldest event stored in the LocalDB is from
-                        <span class="timestamp"><%= JspUtility.friendlyWrite( pageContext, ContextManager.getPwmApplication(session).getLocalDBLogger().getTailDate() ) %></span>.
+                        <span class="timestamp"><%= JspUtility.friendlyWrite( pageContext, JspUtility.getPwmRequest( pageContext ).getPwmApplication().getLocalDBLogger().getTailDate() ) %></span>.
                     </p>
                     <p>
                         The LocalDB is configured to capture events of level
-                        <i><%=ContextManager.getPwmApplication(session).getConfig().readSettingAsString(PwmSetting.EVENTS_LOCALDB_LOG_LEVEL)%>
+                        <i><%=JspUtility.getPwmRequest( pageContext ).getConfig().readSettingAsString(PwmSetting.EVENTS_LOCALDB_LOG_LEVEL)%>
                         </i> and higher.
                     </p>
                 </div>

webapp/src/main/webapp/WEB-INF/jsp/fragment/form.jsp

@@ -253,7 +253,7 @@
             <input type="<pwm:value name="passwordFieldType"/>" name="password1" id="password1" class="changepasswordfield passwordfield" style="margin-left:5px"/>
         </td>
         <td class="noborder">
-            <% if (ContextManager.getPwmApplication(session).getConfig() != null && ContextManager.getPwmApplication(session).getConfig().readSettingAsBoolean(PwmSetting.PASSWORD_SHOW_STRENGTH_METER)) { %>
+            <% if (JspUtility.getPwmRequest( pageContext ).getConfig() != null && JspUtility.getPwmRequest( pageContext ).getConfig().readSettingAsBoolean(PwmSetting.PASSWORD_SHOW_STRENGTH_METER)) { %>
             <div id="strengthBox" style="visibility:hidden;">
                 <div id="strengthLabel">
                     <pwm:display key="Display_StrengthMeter"/>

webapp/src/main/webapp/WEB-INF/jsp/newuser-profilechoice.jsp

@@ -31,7 +31,7 @@
 <%@ page language="java" session="true" isThreadSafe="true" contentType="text/html" %>
 <%@ taglib uri="pwm" prefix="pwm" %>
 <%
-    final PwmRequest pwmRequest = PwmRequest.forRequest(request, response);
+    final PwmRequest pwmRequest = JspUtility.getPwmRequest( pageContext );
     final Map<String,String> newUserProfiles = (Map)pwmRequest.getAttribute(PwmRequestAttribute.NewUser_VisibleProfiles);
 %>
 <html lang="<pwm:value name="<%=PwmValue.localeCode%>"/>" dir="<pwm:value name="<%=PwmValue.localeDir%>"/>">
@@ -70,7 +70,7 @@
         </table>
         <br/>
         <div class="buttonbar">
-            <% if (ContextManager.getPwmApplication(session).getConfig().readSettingAsBoolean(password.pwm.config.PwmSetting.DISPLAY_CANCEL_BUTTON)) { %>
+            <% if (JspUtility.getPwmRequest( pageContext ).getConfig().readSettingAsBoolean(password.pwm.config.PwmSetting.DISPLAY_CANCEL_BUTTON)) { %>
             <form action="<pwm:url url='<%=PwmServletDefinition.PublicCommand.servletUrl()%>' addContext="true"/>" method="get"
                   enctype="application/x-www-form-urlencoded" name="search" class="pwm-form">
                 <button class="btn" type="submit" name="submitBtn">