update servlet dependencies to 4.0

introduce ziplet library to replace built-in gzip http compression
fix rest servlet headers

server/pom.xml

@@ -589,7 +589,7 @@
         <dependency>
             <groupId>com.github.tomakehurst</groupId>
             <artifactId>wiremock</artifactId>
-            <version>1.58</version>
+            <version>2.14.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -603,13 +603,13 @@
         <dependency>
             <groupId>javax.servlet</groupId>
             <artifactId>javax.servlet-api</artifactId>
-            <version>3.0.1</version>
+            <version>4.0.0</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>javax.servlet.jsp</groupId>
             <artifactId>jsp-api</artifactId>
-            <version>2.2</version>
+            <version>2.2.1-b03</version>
             <scope>provided</scope>
         </dependency>
         <!-- / container dependencies -->
@@ -777,6 +777,11 @@
             <artifactId>zxcvbn</artifactId>
             <version>1.2.3</version>
         </dependency>
+        <dependency>
+            <groupId>com.github.ziplet</groupId>
+            <artifactId>ziplet</artifactId>
+            <version>2.3.0</version>
+        </dependency>
 
 
 

server/src/build/checkstyle-import.xml

@@ -133,5 +133,9 @@
         <allow pkg="org.apache.http"/>
     </subpackage>
 
+    <subpackage name="http.filter">
+        <allow pkg="com.github.ziplet.filter.compression"/>
+    </subpackage>
+
 
 </import-control>

server/src/main/java/password/pwm/http/PwmResponse.java

@@ -235,7 +235,7 @@ public class PwmResponse extends PwmHttpResponseWrapper {
         final HttpServletResponse resp = pwmRequest.getPwmResponse().getHttpServletResponse();
         resp.setStatus(redirectType.getCode()); // http "other" redirect
         resp.setHeader(HttpHeader.Location.getHttpName(), url);
-        LOGGER.trace("sending " + redirectType.getCode() + " redirect to " + url);
+        LOGGER.trace(pwmRequest, "sending " + redirectType.getCode() + " redirect to " + url);
     }
 
     private void preCommitActions() {

server/src/main/java/password/pwm/http/filter/GZIPFilter.java

@@ -22,11 +22,11 @@
 
 package password.pwm.http.filter;
 
+import com.github.ziplet.filter.compression.CompressingFilter;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.http.ContextManager;
-import password.pwm.http.HttpHeader;
 import password.pwm.http.PwmURL;
 import password.pwm.util.logging.PwmLogger;
 
@@ -34,17 +34,10 @@ import javax.servlet.Filter;
 import javax.servlet.FilterChain;
 import javax.servlet.FilterConfig;
 import javax.servlet.ServletException;
-import javax.servlet.ServletOutputStream;
 import javax.servlet.ServletRequest;
 import javax.servlet.ServletResponse;
 import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.servlet.http.HttpServletResponseWrapper;
 import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.zip.GZIPOutputStream;
 
 /**
  * GZip Filter Wrapper.  This filter must be invoked _before_ a PwmRequest object is instantiated, else
@@ -53,171 +46,51 @@ import java.util.zip.GZIPOutputStream;
 public class GZIPFilter implements Filter {
     private static final PwmLogger LOGGER = PwmLogger.forClass(GZIPFilter.class);
 
+    private final CompressingFilter compressingFilter = new CompressingFilter();
+    private boolean enabled = false;
+
     public void init(final FilterConfig filterConfig)
             throws ServletException
     {
+        final PwmApplication pwmApplication;
+        try {
+            pwmApplication = ContextManager.getPwmApplication( filterConfig.getServletContext() );
+            enabled = Boolean.parseBoolean(pwmApplication.getConfig().readAppProperty(AppProperty.HTTP_ENABLE_GZIP));
+        } catch (PwmUnrecoverableException e) {
+            LOGGER.warn( "unable to load application configuration, defaulting to disabled" );
+        }
+
+        compressingFilter.init( filterConfig );
     }
 
     public void destroy()
     {
+        compressingFilter.destroy();
     }
 
     @Override
     public void doFilter(final ServletRequest servletRequest, final ServletResponse servletResponse, final FilterChain filterChain)
             throws IOException, ServletException
     {
-        final String acceptEncoding = ((HttpServletRequest)servletRequest).getHeader(HttpHeader.Accept_Encoding.getHttpName());
-        if (acceptEncoding != null && acceptEncoding.contains("gzip") && isEnabled(servletRequest)) {
-            final GZIPHttpServletResponseWrapper gzipResponse = new GZIPHttpServletResponseWrapper((HttpServletResponse)servletResponse);
-            gzipResponse.addHeader(HttpHeader.Content_Encoding.getHttpName(), "gzip");
-            filterChain.doFilter(servletRequest, gzipResponse);
-            gzipResponse.finish();
-
+        if ( enabled && interestInRequest( servletRequest )) {
+            compressingFilter.doFilter( servletRequest, servletResponse, filterChain );
         } else {
             filterChain.doFilter(servletRequest, servletResponse);
         }
     }
 
-    private boolean isEnabled(final ServletRequest servletRequest) {
-
+    private boolean interestInRequest( final ServletRequest servletRequest) {
         try {
             final PwmURL pwmURL = new PwmURL((HttpServletRequest) servletRequest);
-            if (pwmURL.isResourceURL() || pwmURL.isRestService()) {
+
+            // resource servlet does its own gzip compression with fancy server-side caching
+            if (pwmURL.isResourceURL()) {
                 return false;
             }
         } catch (Exception e) {
             LOGGER.error("unable to parse request url, defaulting to non-gzip: " + e.getMessage());
         }
 
-        final PwmApplication pwmApplication;
-        try {
-            pwmApplication = ContextManager.getPwmApplication((HttpServletRequest) servletRequest);
-            return Boolean.parseBoolean(pwmApplication.getConfig().readAppProperty(AppProperty.HTTP_ENABLE_GZIP));
-        } catch (PwmUnrecoverableException e) {
-            //LOGGER.trace("unable to read http-gzip app-property, defaulting to non-gzip: " + e.getMessage());
-        }
-        return false;
-    }
-
-
-    public static class GZIPHttpServletResponseWrapper extends HttpServletResponseWrapper {
-        private ServletResponseGZIPOutputStream gzipStream;
-        private ServletOutputStream outputStream;
-        private PrintWriter printWriter;
-
-        public GZIPHttpServletResponseWrapper(final HttpServletResponse response) throws IOException {
-            super(response);
-        }
-
-        public void finish() throws IOException {
-            if (printWriter != null) {
-                printWriter.close();
-            }
-            if (outputStream != null) {
-                outputStream.close();
-            }
-            if (gzipStream != null) {
-                gzipStream.close();
-            }
-        }
-
-        @Override
-        public void flushBuffer() throws IOException {
-            if (printWriter != null) {
-                printWriter.flush();
-            }
-            if (outputStream != null) {
-                outputStream.flush();
-            }
-            super.flushBuffer();
-        }
-
-        @Override
-        public ServletOutputStream getOutputStream() throws IOException {
-            if (printWriter != null) {
-                throw new IllegalStateException("getWriter() has previously been invoked, can not call getOutputStream()");
-            }
-            if (outputStream == null) {
-                initGzip();
-                outputStream = gzipStream;
-            }
-            return outputStream;
-        }
-
-        @Override
-        public PrintWriter getWriter() throws IOException {
-            if (outputStream != null) {
-                throw new IllegalStateException("getOutputStream() has previously been invoked, can not call getWriter()");
-            }
-            if (printWriter == null) {
-                initGzip();
-                printWriter = new PrintWriter(new OutputStreamWriter(gzipStream, getResponse().getCharacterEncoding()));
-            }
-            return printWriter;
-        }
-
-        @Override
-        public void setContentLength(final int len) {
-        }
-
-        private void initGzip() throws IOException {
-            gzipStream = new ServletResponseGZIPOutputStream(getResponse().getOutputStream());
-        }
-    }
-
-    public static class ServletResponseGZIPOutputStream extends ServletOutputStream {
-        private final AtomicBoolean open = new AtomicBoolean(true);
-        private GZIPOutputStream gzipStream;
-
-        public ServletResponseGZIPOutputStream(final ServletOutputStream output) throws IOException {
-            gzipStream = new GZIPOutputStream(output);
-        }
-
-        @Override
-        public void close() throws IOException {
-            if (open.compareAndSet(true, false)) {
-                gzipStream.close();
-            }
-        }
-
-        @Override
-        public void flush() throws IOException {
-            gzipStream.flush();
-        }
-
-        @Override
-        public void write(final byte[] b) throws IOException {
-            write(b, 0, b.length);
-        }
-
-        @Override
-        public void write(final byte[] b, final int off, final int len) throws IOException {
-            if (!open.get()) {
-                throw new IOException("Stream closed!");
-            }
-            gzipStream.write(b, off, len);
-        }
-
-        @Override
-        public void write(final int b) throws IOException {
-            if (!open.get()) {
-                throw new IOException("Stream closed!");
-            }
-            gzipStream.write(b);
-        }
-
-        /*
-        // servlet 3.1 method
-        public void setWriteListener(WriteListener writeListener)
-        {
-            servletOutputStream.setWriteListener(writeListener);
-        }
-
-        // servlet 3.1 method
-        public boolean isReady()
-        {
-            return servletOutputStream.isReady();
-        }
-        */
+        return true;
     }
 }

server/src/main/java/password/pwm/http/filter/RequestInitializationFilter.java

@@ -155,7 +155,7 @@ public class RequestInitializationFilter implements Filter {
         } catch (Throwable e) {
             LOGGER.error("can't load application: " + e.getMessage(),e);
             if (!(new PwmURL(req).isResourceURL())) {
-                respondWithUnavalailbleError(req, resp);
+                respondWithUnavailableError(req, resp);
                 return;
             }
             return;
@@ -191,7 +191,7 @@ public class RequestInitializationFilter implements Filter {
                 LOGGER.error(logMsg,e);
             }
             if (!(new PwmURL(req).isResourceURL())) {
-                respondWithUnavalailbleError(req, resp);
+                respondWithUnavailableError(req, resp);
                 return;
             }
             return;
@@ -200,7 +200,7 @@ public class RequestInitializationFilter implements Filter {
         filterChain.doFilter(req, resp);
     }
 
-    private void respondWithUnavalailbleError(final HttpServletRequest req, final HttpServletResponse resp)
+    private void respondWithUnavailableError( final HttpServletRequest req, final HttpServletResponse resp)
             throws ServletException, IOException
     {
         ErrorInformation errorInformation = new ErrorInformation(PwmError.ERROR_APP_UNAVAILABLE);
@@ -307,84 +307,86 @@ public class RequestInitializationFilter implements Filter {
             return;
         }
 
+        final boolean includeXSessionID = Boolean.parseBoolean(config.readAppProperty(AppProperty.HTTP_HEADER_SEND_XSESSIONID));
+        if (includeXSessionID && pwmSession != null) {
+            resp.setHeader(HttpHeader.XSessionID, pwmSession.getSessionStateBean().getSessionID());
+        }
+
+        final boolean includeContentLanguage = Boolean.parseBoolean(config.readAppProperty(AppProperty.HTTP_HEADER_SEND_CONTENT_LANGUAGE));
+        if (includeContentLanguage) {
+            resp.setHeader(HttpHeader.Content_Language, pwmRequest.getLocale().toLanguageTag());
+        }
+
+        addStaticResponseHeaders( pwmApplication, resp.getHttpServletResponse() );
+
+
+        if (pwmSession != null) {
+            final String contentPolicy;
+            if (pwmRequest.getURL().isConfigGuideURL() || pwmRequest.getURL().isConfigManagerURL()) {
+                contentPolicy = config.readAppProperty(AppProperty.SECURITY_HTTP_CONFIG_CSP_HEADER);
+            } else {
+                contentPolicy = config.readSettingAsString(PwmSetting.SECURITY_CSP_HEADER);
+            }
+
+            if (contentPolicy != null && !contentPolicy.isEmpty()) {
+                final String nonce = pwmRequest.getCspNonce();
+                final String expandedPolicy = contentPolicy.replace("%NONCE%", nonce);
+                resp.setHeader(HttpHeader.ContentSecurityPolicy, expandedPolicy);
+            }
+        }
+    }
+
+    public static void addStaticResponseHeaders(final PwmApplication pwmApplication, final HttpServletResponse resp) throws PwmUnrecoverableException
+    {
+        final Configuration config = pwmApplication.getConfig();
+
         final String serverHeader = config.readAppProperty(AppProperty.HTTP_HEADER_SERVER);
         final boolean includeXInstance = Boolean.parseBoolean(config.readAppProperty(AppProperty.HTTP_HEADER_SEND_XINSTANCE));
-        final boolean includeXSessionID = Boolean.parseBoolean(config.readAppProperty(AppProperty.HTTP_HEADER_SEND_XSESSIONID));
         final boolean includeXVersion = Boolean.parseBoolean(config.readAppProperty(AppProperty.HTTP_HEADER_SEND_XVERSION));
         final boolean includeXContentTypeOptions = Boolean.parseBoolean(config.readAppProperty(AppProperty.HTTP_HEADER_SEND_XCONTENTTYPEOPTIONS));
         final boolean includeXXSSProtection = Boolean.parseBoolean(config.readAppProperty(AppProperty.HTTP_HEADER_SEND_XXSSPROTECTION));
+        final boolean includeXFrameDeny = config.readSettingAsBoolean(PwmSetting.SECURITY_PREVENT_FRAMING);
+        final boolean includeXAmb = Boolean.parseBoolean(config.readAppProperty(AppProperty.HTTP_HEADER_SEND_XAMB));
 
-        final boolean sendNoise = Boolean.parseBoolean(config.readAppProperty(AppProperty.HTTP_HEADER_SEND_XNOISE));
-
-        if (sendNoise) {
-            final int noiseLength = Integer.parseInt(config.readAppProperty(AppProperty.HTTP_HEADER_NOISE_LENGTH));
-            resp.setHeader(
-                    HttpHeader.XNoise,
-                    PwmRandom.getInstance().alphaNumericString(PwmRandom.getInstance().nextInt(noiseLength)+11)
-            );
+        {
+            final String noiseHeader = makeNoiseHeader( config );
+            if (noiseHeader != null) {
+                resp.setHeader( HttpHeader.XNoise.getHttpName(), noiseHeader );
+            }
         }
 
         if (includeXVersion) {
-            resp.setHeader(HttpHeader.XVersion, PwmConstants.SERVLET_VERSION);
+            resp.setHeader(HttpHeader.XVersion.getHttpName(), PwmConstants.SERVLET_VERSION);
         }
 
         if (includeXContentTypeOptions) {
-            resp.setHeader(HttpHeader.XContentTypeOptions, "nosniff");
+            resp.setHeader(HttpHeader.XContentTypeOptions.getHttpName(), "nosniff");
         }
 
         if (includeXXSSProtection) {
-            resp.setHeader(HttpHeader.XXSSProtection, "1");
+            resp.setHeader(HttpHeader.XXSSProtection.getHttpName(), "1");
         }
 
         if (includeXInstance) {
-            resp.setHeader(HttpHeader.XInstance, String.valueOf(pwmApplication.getInstanceID()));
-        }
-
-        if (includeXSessionID && pwmSession != null) {
-            resp.setHeader(HttpHeader.XSessionID, pwmSession.getSessionStateBean().getSessionID());
+            resp.setHeader(HttpHeader.XInstance.getHttpName(), String.valueOf(pwmApplication.getInstanceID()));
         }
 
         if (serverHeader != null && !serverHeader.isEmpty()) {
             final String value = MacroMachine.forNonUserSpecific(pwmApplication, null).expandMacros(serverHeader);
-            resp.setHeader(HttpHeader.Server, value);
+            resp.setHeader(HttpHeader.Server.getHttpName(), value);
         }
 
-        // ----- non-resource urls only for the following operations -----
-
-        final boolean includeXFrameDeny = config.readSettingAsBoolean(PwmSetting.SECURITY_PREVENT_FRAMING);
-        final boolean includeXAmb = Boolean.parseBoolean(config.readAppProperty(AppProperty.HTTP_HEADER_SEND_XAMB));
-        final boolean includeContentLanguage = Boolean.parseBoolean(config.readAppProperty(AppProperty.HTTP_HEADER_SEND_CONTENT_LANGUAGE));
-
         if (includeXFrameDeny) {
-            resp.setHeader(HttpHeader.XFrameOptions, "DENY");
+            resp.setHeader(HttpHeader.XFrameOptions.getHttpName(), "DENY");
         }
 
         if (includeXAmb) {
-            resp.setHeader(HttpHeader.XAmb, PwmConstants.X_AMB_HEADER.get(
+            resp.setHeader(HttpHeader.XAmb.getHttpName(), PwmConstants.X_AMB_HEADER.get(
                     PwmRandom.getInstance().nextInt(PwmConstants.X_AMB_HEADER.size())
             ));
         }
 
-        if (includeContentLanguage) {
-            resp.setHeader(HttpHeader.Content_Language, pwmRequest.getLocale().toLanguageTag());
-        }
-
-        resp.setHeader(HttpHeader.Cache_Control, "no-cache, no-store, must-revalidate, proxy-revalidate");
-
-        if (pwmSession != null) {
-            final String contentPolicy;
-            if (pwmRequest.getURL().isConfigGuideURL() || pwmRequest.getURL().isConfigManagerURL()) {
-                contentPolicy = config.readAppProperty(AppProperty.SECURITY_HTTP_CONFIG_CSP_HEADER);
-            } else {
-                contentPolicy = config.readSettingAsString(PwmSetting.SECURITY_CSP_HEADER);
-            }
-
-            if (contentPolicy != null && !contentPolicy.isEmpty()) {
-                final String nonce = pwmRequest.getCspNonce();
-                final String expandedPolicy = contentPolicy.replace("%NONCE%", nonce);
-                resp.setHeader(HttpHeader.ContentSecurityPolicy, expandedPolicy);
-            }
-        }
+        resp.setHeader(HttpHeader.Cache_Control.getHttpName(), "no-cache, no-store, must-revalidate, proxy-revalidate");
     }
 
 
@@ -685,4 +687,15 @@ public class RequestInitializationFilter implements Filter {
         return StringUtil.mapToString(values);
     }
 
+    private static String makeNoiseHeader(final Configuration configuration) {
+        final boolean sendNoise = Boolean.parseBoolean(configuration.readAppProperty(AppProperty.HTTP_HEADER_SEND_XNOISE));
+
+        if (sendNoise) {
+            final int noiseLength = Integer.parseInt(configuration.readAppProperty(AppProperty.HTTP_HEADER_NOISE_LENGTH));
+            return PwmRandom.getInstance().alphaNumericString(PwmRandom.getInstance().nextInt(noiseLength)+11);
+        }
+
+        return null;
+    }
+
 }

server/src/main/java/password/pwm/ws/server/RestServlet.java

@@ -104,6 +104,8 @@ public abstract class RestServlet extends HttpServlet{
 
             final RestRequest restRequest = RestRequest.forRequest(pwmApplication, restAuthentication, sessionLabel, req);
 
+            RequestInitializationFilter.addStaticResponseHeaders( pwmApplication, resp );
+
             preCheck(restRequest);
 
             preCheckRequest(restRequest);
@@ -372,5 +374,4 @@ public abstract class RestServlet extends HttpServlet{
             throw new PwmUnrecoverableException(e.getErrorInformation());
         }
     }
-
 }