Forráskód Böngészése

improve resource domain service

Jason Rivard 4 éve
szülő
commit
b8c4a14d57
33 módosított fájl, 583 hozzáadás és 364 törlés
  1. 24 0
      server/src/main/java/password/pwm/config/value/AbstractValue.java
  2. 4 17
      server/src/main/java/password/pwm/config/value/FileValue.java
  3. 2 3
      server/src/main/java/password/pwm/config/value/X509CertificateValue.java
  4. 2 1
      server/src/main/java/password/pwm/http/HttpHeader.java
  5. 2 1
      server/src/main/java/password/pwm/http/HttpMethod.java
  6. 2 1
      server/src/main/java/password/pwm/http/servlet/resource/CacheKey.java
  7. 3 10
      server/src/main/java/password/pwm/http/servlet/resource/ConfigSettingFileResource.java
  8. 2 3
      server/src/main/java/password/pwm/http/servlet/resource/FileResource.java
  9. 4 9
      server/src/main/java/password/pwm/http/servlet/resource/MemoryFileResource.java
  10. 3 8
      server/src/main/java/password/pwm/http/servlet/resource/RealFileResource.java
  11. 216 139
      server/src/main/java/password/pwm/http/servlet/resource/ResourceFileRequest.java
  12. 65 35
      server/src/main/java/password/pwm/http/servlet/resource/ResourceFileServlet.java
  13. 2 1
      server/src/main/java/password/pwm/http/servlet/resource/ResourceServletConfiguration.java
  14. 34 8
      server/src/main/java/password/pwm/http/servlet/resource/ResourceServletService.java
  15. 3 8
      server/src/main/java/password/pwm/http/servlet/resource/ZipFileResource.java
  16. 17 10
      server/src/main/java/password/pwm/svc/httpclient/ApachePwmHttpClient.java
  17. 6 6
      server/src/main/java/password/pwm/svc/httpclient/HttpClientService.java
  18. 1 1
      server/src/main/java/password/pwm/svc/httpclient/JavaPwmHttpClient.java
  19. 1 6
      server/src/main/java/password/pwm/svc/httpclient/PwmHttpClient.java
  20. 31 0
      server/src/main/java/password/pwm/svc/httpclient/PwmHttpClientProvider.java
  21. 1 1
      server/src/main/java/password/pwm/svc/httpclient/PwmHttpClientResponse.java
  22. 21 9
      server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java
  23. 5 0
      server/src/main/java/password/pwm/svc/wordlist/WordlistConfiguration.java
  24. 1 1
      server/src/main/java/password/pwm/svc/wordlist/WordlistInspector.java
  25. 58 5
      server/src/main/java/password/pwm/svc/wordlist/WordlistSource.java
  26. 7 1
      server/src/main/java/password/pwm/svc/wordlist/WordlistStatistics.java
  27. 2 1
      server/src/main/java/password/pwm/util/java/ConditionalTaskExecutor.java
  28. 2 2
      server/src/main/java/password/pwm/util/java/MovingAverage.java
  29. 47 72
      server/src/main/java/password/pwm/util/java/StringUtil.java
  30. 3 3
      server/src/main/java/password/pwm/util/localdb/LocalDBFactory.java
  31. 3 1
      server/src/main/java/password/pwm/util/logging/PwmLogger.java
  32. 1 1
      server/src/main/resources/password/pwm/AppProperty.properties
  33. 8 0
      server/src/test/java/password/pwm/util/java/StringUtilTest.java

+ 24 - 0
server/src/main/java/password/pwm/config/value/AbstractValue.java

@@ -20,11 +20,14 @@
 
 package password.pwm.config.value;
 
+import password.pwm.PwmConstants;
 import password.pwm.config.stored.StoredConfigXmlConstants;
 import password.pwm.config.stored.XmlOutputProcessData;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.http.bean.ImmutableByteArray;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.LazySupplier;
+import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.XmlDocument;
 import password.pwm.util.java.XmlElement;
 import password.pwm.util.java.XmlFactory;
@@ -85,6 +88,27 @@ public abstract class AbstractValue implements StoredValue
         return valueHashSupplier.get();
     }
 
+    protected static String b64encode( final ImmutableByteArray immutableByteArray )
+            throws PwmUnrecoverableException
+    {
+        final String input = StringUtil.base64Encode( immutableByteArray.copyOf(), StringUtil.Base64Options.GZIP );
+        return "\n" + StringUtil.insertRepeatedLineBreaks( input, PwmConstants.XML_OUTPUT_LINE_WRAP_LENGTH ) + "\n";
+    }
+
+    protected static ImmutableByteArray b64decode( final String b64EncodedContents )
+    {
+        try
+        {
+            final CharSequence whitespaceStripped = StringUtil.stripAllWhitespace( b64EncodedContents );
+            final byte[] output = StringUtil.base64Decode( whitespaceStripped, StringUtil.Base64Options.GZIP );
+            return ImmutableByteArray.of( output );
+        }
+        catch ( final Exception e )
+        {
+            throw new IllegalStateException( e );
+        }
+    }
+    
     static String valueHashComputer( final StoredValue storedValue )
     {
         try

+ 4 - 17
server/src/main/java/password/pwm/config/value/FileValue.java

@@ -74,11 +74,12 @@ public class FileValue extends AbstractValue implements StoredValue
         private static final long serialVersionUID = 1L;
 
         private final String b64EncodedContents;
-        private final transient Supplier<ImmutableByteArray> byteContents = new LazySupplier<>( this::convertToBytes );
+        private final transient Supplier<ImmutableByteArray> byteContents;
 
         private FileContent( final String b64EncodedContents )
         {
             this.b64EncodedContents = b64EncodedContents;
+            this.byteContents = new LazySupplier<>( () -> b64decode( b64EncodedContents ) );
         }
 
         public static FileContent fromEncodedString( final String input )
@@ -91,9 +92,7 @@ public class FileValue extends AbstractValue implements StoredValue
         public static FileContent fromBytes( final ImmutableByteArray contents )
                 throws PwmUnrecoverableException
         {
-            final String input = StringUtil.base64Encode( contents.copyOf(), StringUtil.Base64Options.GZIP );
-            final String encodedLineBreaks = StringUtil.insertRepeatedLineBreaks( input, PwmConstants.XML_OUTPUT_LINE_WRAP_LENGTH );
-            return new FileContent( encodedLineBreaks );
+            return new FileContent( b64encode( contents ) );
         }
 
         String toEncodedString( )
@@ -118,19 +117,7 @@ public class FileValue extends AbstractValue implements StoredValue
             return byteContents.get();
         }
 
-        private ImmutableByteArray convertToBytes( )
-        {
-            try
-            {
-                final String whitespaceStripped = StringUtil.stripAllWhitespace( b64EncodedContents );
-                final byte[] output = StringUtil.base64Decode( whitespaceStripped, StringUtil.Base64Options.GZIP );
-                return ImmutableByteArray.of( output );
-            }
-            catch ( final Exception e )
-            {
-                throw new IllegalStateException( e );
-            }
-        }
+        
     }
 
     public static FileValue newFileValue( final String filename, final String fileMimeType, final ImmutableByteArray contents )

+ 2 - 3
server/src/main/java/password/pwm/config/value/X509CertificateValue.java

@@ -97,10 +97,9 @@ public class X509CertificateValue extends AbstractValue implements StoredValue
         {
             throw new NullPointerException( "certificates cannot be null" );
         }
-        this.b64certificates = Collections.unmodifiableList(
-                b64certificates.stream()
+        this.b64certificates = b64certificates.stream()
                 .map( StringUtil::stripAllWhitespace )
-                .collect( Collectors.toList() ) );
+                .collect( Collectors.toUnmodifiableList() );
         this.certs = new LazySupplier<>( () -> X509Utils.certificatesFromBase64s( b64certificates ) );
     }
 

+ 2 - 1
server/src/main/java/password/pwm/http/HttpHeader.java

@@ -43,6 +43,7 @@ public enum HttpHeader
     ETag( "ETag" ),
     Expires( "Expires" ),
     If_None_Match( "If-None-Match" ),
+    Last_Modified( "Last-Modified" ),
     Location( "Location" ),
     Origin( "Origin" ),
     Referer( "Referer" ),
@@ -89,6 +90,6 @@ public enum HttpHeader
 
     public static Optional<HttpHeader> forHttpHeader( final String header )
     {
-        return JavaHelper.readEnumFromPredicate( HttpHeader.class, loopHeader -> header.equals( loopHeader.getHttpName() ) );
+        return JavaHelper.readEnumFromPredicate( HttpHeader.class, loopHeader -> header.equalsIgnoreCase( loopHeader.getHttpName() ) );
     }
 }

+ 2 - 1
server/src/main/java/password/pwm/http/HttpMethod.java

@@ -28,7 +28,8 @@ public enum HttpMethod
     GET( true, false ),
     DELETE( false, true ),
     PUT( false, true ),
-    PATCH( false, true ),;
+    PATCH( false, true ),
+    HEAD( true, false );
 
     private final boolean idempotent;
     private final boolean hasBody;

+ 2 - 1
server/src/main/java/password/pwm/http/servlet/resource/CacheKey.java

@@ -23,6 +23,7 @@ package password.pwm.http.servlet.resource;
 import lombok.Value;
 
 import java.io.Serializable;
+import java.time.Instant;
 import java.util.Objects;
 
 @Value
@@ -30,7 +31,7 @@ final class CacheKey implements Serializable
 {
     private final String fileName;
     private final boolean acceptsGzip;
-    private final long fileModificationTimestamp;
+    private final Instant fileModificationTimestamp;
 
     static CacheKey createCacheKey( final FileResource file, final boolean acceptsGzip )
     {

+ 3 - 10
server/src/main/java/password/pwm/http/servlet/resource/ConfigSettingFileResource.java

@@ -23,11 +23,11 @@ package password.pwm.http.servlet.resource;
 import password.pwm.PwmConstants;
 import password.pwm.config.DomainConfig;
 import password.pwm.config.PwmSetting;
-import password.pwm.util.java.StringUtil;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Instant;
 
 public class ConfigSettingFileResource implements FileResource
 {
@@ -40,7 +40,6 @@ public class ConfigSettingFileResource implements FileResource
         this.requestURI = requestURI;
     }
 
-
     @Override
     public InputStream getInputStream()
             throws IOException
@@ -55,15 +54,9 @@ public class ConfigSettingFileResource implements FileResource
     }
 
     @Override
-    public long lastModified()
-    {
-        return 0;
-    }
-
-    @Override
-    public boolean exists()
+    public Instant lastModified()
     {
-        return StringUtil.notEmpty( bodyText );
+        return Instant.EPOCH;
     }
 
     @Override

+ 2 - 3
server/src/main/java/password/pwm/http/servlet/resource/FileResource.java

@@ -22,6 +22,7 @@ package password.pwm.http.servlet.resource;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Instant;
 
 interface FileResource
 {
@@ -29,9 +30,7 @@ interface FileResource
 
     long length( );
 
-    long lastModified( );
-
-    boolean exists( );
+    Instant lastModified( );
 
     String getName( );
 }

+ 4 - 9
server/src/main/java/password/pwm/http/servlet/resource/MemoryFileResource.java

@@ -24,14 +24,15 @@ import password.pwm.http.bean.ImmutableByteArray;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Instant;
 
 class MemoryFileResource implements FileResource
 {
     private final String name;
     private final ImmutableByteArray contents;
-    private final long lastModified;
+    private final Instant lastModified;
 
-    MemoryFileResource( final String name, final ImmutableByteArray contents, final long lastModified )
+    MemoryFileResource( final String name, final ImmutableByteArray contents, final Instant lastModified )
     {
         this.name = name;
         this.contents = contents;
@@ -51,17 +52,11 @@ class MemoryFileResource implements FileResource
     }
 
     @Override
-    public long lastModified( )
+    public Instant lastModified( )
     {
         return lastModified;
     }
 
-    @Override
-    public boolean exists( )
-    {
-        return true;
-    }
-
     @Override
     public String getName( )
     {

+ 3 - 8
server/src/main/java/password/pwm/http/servlet/resource/RealFileResource.java

@@ -24,6 +24,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Instant;
 
 class RealFileResource implements FileResource
 {
@@ -47,15 +48,9 @@ class RealFileResource implements FileResource
     }
 
     @Override
-    public long lastModified( )
+    public Instant lastModified( )
     {
-        return realFile.lastModified();
-    }
-
-    @Override
-    public boolean exists( )
-    {
-        return realFile.exists();
+        return Instant.ofEpochMilli( realFile.lastModified() );
     }
 
     @Override

+ 216 - 139
server/src/main/java/password/pwm/http/servlet/resource/ResourceFileRequest.java

@@ -20,12 +20,14 @@
 
 package password.pwm.http.servlet.resource;
 
+import lombok.Value;
 import org.webjars.WebJarAssetLocator;
 import password.pwm.config.DomainConfig;
 import password.pwm.config.PwmSetting;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.http.HttpContentType;
 import password.pwm.http.HttpHeader;
 import password.pwm.http.PwmHttpRequestWrapper;
 import password.pwm.util.java.StringUtil;
@@ -34,13 +36,15 @@ import password.pwm.util.logging.PwmLogger;
 import javax.servlet.ServletContext;
 import javax.servlet.http.HttpServletRequest;
 import java.io.File;
-import java.io.IOException;
 import java.io.InputStream;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.regex.Matcher;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
@@ -58,17 +62,22 @@ class ResourceFileRequest
     private final DomainConfig domainConfig;
     private final ResourceServletConfiguration resourceServletConfiguration;
 
-    private FileResource fileResource;
+    private final Optional<FileResource> fileResource;
 
     ResourceFileRequest(
             final DomainConfig domainConfig,
             final ResourceServletConfiguration resourceServletConfiguration,
             final HttpServletRequest httpServletRequest
     )
+            throws PwmUnrecoverableException
     {
         this.domainConfig = domainConfig;
         this.resourceServletConfiguration = resourceServletConfiguration;
         this.httpServletRequest = httpServletRequest;
+
+        final String resourcePathUri = this.getRequestURI();
+        final ServletContext servletContext = this.getHttpServletRequest().getServletContext();
+        fileResource = resolveRequestedResource( domainConfig, servletContext, resourcePathUri, resourceServletConfiguration );
     }
 
     HttpServletRequest getHttpServletRequest()
@@ -94,35 +103,36 @@ class ResourceFileRequest
         return acceptsCompression ? rawContentType : rawContentType + ";charset=UTF-8";
     }
 
-    FileResource getRequestedFileResource()
+    Optional<FileResource> getRequestedFileResource()
             throws PwmUnrecoverableException
     {
-        if ( fileResource == null )
-        {
-            final String resourcePathUri = this.getRequestURI();
-            final ServletContext servletContext = this.getHttpServletRequest().getServletContext();
-            fileResource = resolveRequestedResource( domainConfig, servletContext, resourcePathUri, resourceServletConfiguration );
-        }
         return fileResource;
     }
 
     private String getRawMimeType()
             throws PwmUnrecoverableException
     {
-        final String filename = getRequestedFileResource().getName();
-        final String contentType = this.httpServletRequest.getServletContext().getMimeType( filename );
-        if ( contentType == null )
+        if ( fileResource.isPresent() )
         {
-            if ( filename.endsWith( ".woff2" ) )
+            final String filename = fileResource.get().getName();
+            final String contentType = this.httpServletRequest.getServletContext().getMimeType( filename );
+            if ( contentType == null )
             {
-                return "font/woff2";
+                if ( filename.endsWith( ".woff2" ) )
+                {
+                    return "font/woff2";
+                }
             }
+
+            // If content type is unknown, then set the default value.
+            // For all content types, see: http://www.w3schools.com/media/media_mimeref.asp
+            // To add new content types, add new mime-mapping entry in web.xml.
+            return contentType == null
+                    ? HttpContentType.octetstream.getMimeType()
+                    : contentType;
         }
 
-        // If content type is unknown, then set the default value.
-        // For all content types, see: http://www.w3schools.com/media/media_mimeref.asp
-        // To add new content types, add new mime-mapping entry in web.xml.
-        return contentType == null ? "application/octet-stream" : contentType;
+        return HttpContentType.octetstream.getMimeType();
     }
 
     boolean allowsCompression()
@@ -168,15 +178,11 @@ class ResourceFileRequest
         return requestURI.substring( httpServletRequest.getContextPath().length() );
     }
 
-    static FileResource resolveRequestedResource(
+    static String deriveEffectiveURI(
             final DomainConfig domainConfig,
-            final ServletContext servletContext,
-            final String inputResourcePathUri,
-            final ResourceServletConfiguration resourceServletConfiguration
+            final String inputResourcePathUri
     )
-            throws PwmUnrecoverableException
     {
-
         // URL-decode the file name (might contain spaces and on) and prepare file object.
         String effectiveUri = StringUtil.urlDecode( inputResourcePathUri );
 
@@ -195,14 +201,69 @@ class ResourceFileRequest
             }
         }
 
+        return effectiveUri;
+    }
+
+    static Optional<FileResource> resolveRequestedResource(
+            final DomainConfig domainConfig,
+            final ServletContext servletContext,
+            final String inputResourcePathUri,
+            final ResourceServletConfiguration resourceServletConfiguration
+    )
+            throws PwmUnrecoverableException
+    {
+        final String effectiveUri = deriveEffectiveURI( domainConfig, inputResourcePathUri );
+
         if ( !effectiveUri.startsWith( ResourceFileServlet.RESOURCE_PATH ) )
         {
-            final String filenameFinal = effectiveUri;
-            LOGGER.warn( () -> "illegal url request to " + filenameFinal );
+            LOGGER.warn( () -> "illegal url request to " + effectiveUri );
             throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, "illegal url request" ) );
         }
 
+        final ResourceRequestContext context = new ResourceRequestContext( effectiveUri, domainConfig, resourceServletConfiguration, servletContext );
+        for ( final ResourceUriResolver resourceUriResolver : RESOLVERS )
+        {
+            final Optional<FileResource> resource = resourceUriResolver.resolveUri( context );
+            if ( resource.isPresent() )
+            {
+                return resource;
+            }
+        }
+
+        return Optional.empty();
+    }
+
+    @Value
+    private static class ResourceRequestContext
+    {
+        private final String uri;
+        private final DomainConfig domainConfig;
+        private final ResourceServletConfiguration resourceServletConfiguration;
+        private final ServletContext servletContext;
+    }
+
+    private interface ResourceUriResolver
+    {
+        Optional<FileResource> resolveUri( ResourceRequestContext resourceRequestContext )
+                throws PwmUnrecoverableException;
+    }
+
+    private static final List<ResourceUriResolver> RESOLVERS = List.of(
+            new ThemeUriResolver(),
+            new BuiltInZipFileUriResolver(),
+            new WebJarUriResolver(),
+            new RealFileUriResolver(),
+            new CustomZipFileUriResolver()
+    );
+
+    private static class ThemeUriResolver implements ResourceUriResolver
+    {
+        @Override
+        public Optional<FileResource> resolveUri( final ResourceRequestContext resourceRequestContext )
         {
+            final String effectiveUri = resourceRequestContext.getUri();
+            final DomainConfig domainConfig = resourceRequestContext.getDomainConfig();
+
             final String embedThemeUrl = ResourceFileServlet.RESOURCE_PATH
                     + ResourceFileServlet.THEME_CSS_PATH.replace( ResourceFileServlet.TOKEN_THEME, ResourceFileServlet.EMBED_THEME );
             final String embedThemeMobileUrl = ResourceFileServlet.RESOURCE_PATH
@@ -210,24 +271,26 @@ class ResourceFileRequest
 
             if ( effectiveUri.equalsIgnoreCase( embedThemeUrl ) )
             {
-                return new ConfigSettingFileResource( PwmSetting.DISPLAY_CSS_EMBED, domainConfig, effectiveUri );
+                return Optional.of( new ConfigSettingFileResource( PwmSetting.DISPLAY_CSS_EMBED, domainConfig, effectiveUri ) );
             }
             else if ( effectiveUri.equalsIgnoreCase( embedThemeMobileUrl ) )
             {
-                return new ConfigSettingFileResource( PwmSetting.DISPLAY_CSS_MOBILE_EMBED, domainConfig, effectiveUri );
+                return Optional.of( new ConfigSettingFileResource( PwmSetting.DISPLAY_CSS_MOBILE_EMBED, domainConfig, effectiveUri ) );
             }
-        }
 
-
-        {
-            final FileResource resource = handleWebjarURIs( servletContext, effectiveUri );
-            if ( resource != null )
-            {
-                return resource;
-            }
+            return Optional.empty();
         }
+    }
 
+    private static class BuiltInZipFileUriResolver implements ResourceUriResolver
+    {
+        @Override
+        public Optional<FileResource> resolveUri( final ResourceRequestContext resourceRequestContext )
+                throws PwmUnrecoverableException
         {
+            final String effectiveUri = resourceRequestContext.getUri();
+            final ResourceServletConfiguration resourceServletConfiguration = resourceRequestContext.getResourceServletConfiguration();
+
             // check files system zip files.
             final Map<String, ZipFile> zipResources = resourceServletConfiguration.getZipResources();
             for ( final Map.Entry<String, ZipFile> entry : zipResources.entrySet() )
@@ -240,160 +303,174 @@ class ResourceFileRequest
                     final ZipEntry zipEntry = zipFile.getEntry( zipSubPath );
                     if ( zipEntry != null )
                     {
-                        return new ZipFileResource( zipFile, zipEntry );
+                        return Optional.of( new ZipFileResource( zipFile, zipEntry ) );
                     }
                 }
                 if ( effectiveUri.startsWith( zipResources.get( path ).getName() ) )
                 {
-                    final String filenameFinal = effectiveUri;
-                    LOGGER.warn( () -> "illegal url request to " + filenameFinal + " zip resource" );
+                    LOGGER.warn( () -> "illegal url request to " + effectiveUri + " zip resource" );
                     throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, "illegal url request" ) );
                 }
             }
-        }
-
-        // convert to file.
-        final String filePath = servletContext.getRealPath( effectiveUri );
-        final File file = new File( filePath );
 
-        // figure top-most path allowed by request
-        final String parentDirectoryPath = servletContext.getRealPath( ResourceFileServlet.RESOURCE_PATH );
-        final File parentDirectory = new File( parentDirectoryPath );
-
-        FileResource fileSystemResource = null;
-        {
-            //verify the requested page is a child of the servlet resource path.
-            int recursions = 0;
-            File recurseFile = file.getParentFile();
-            while ( recurseFile != null && recursions < 100 )
-            {
-                if ( parentDirectory.equals( recurseFile ) )
-                {
-                    fileSystemResource = new RealFileResource( file );
-                    break;
-                }
-                recurseFile = recurseFile.getParentFile();
-                recursions++;
-            }
+            return Optional.empty();
         }
+    }
 
-        if ( fileSystemResource == null )
+    private static class WebJarUriResolver implements ResourceUriResolver
+    {
+        @Override
+        public Optional<FileResource> resolveUri( final ResourceRequestContext resourceRequestContext )
         {
-            LOGGER.warn( () -> "attempt to access file outside of servlet path " + file.getAbsolutePath() );
-            throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, "illegal file path request" ) );
-        }
+            final String effectiveUri = resourceRequestContext.getUri();
+            final ServletContext servletContext = resourceRequestContext.getServletContext();
 
-        if ( !fileSystemResource.exists() )
-        {
-            // check custom (configuration defined) zip file bundles
-            final Map<String, FileResource> customResources = resourceServletConfiguration.getCustomFileBundle();
-            for ( final Map.Entry<String, FileResource> entry : customResources.entrySet() )
+            if ( effectiveUri.startsWith( ResourceFileServlet.WEBJAR_BASE_URL_PATH ) )
             {
-                final String customFileName = entry.getKey();
-                final String testName = ResourceFileServlet.RESOURCE_PATH + "/" + customFileName;
-                if ( testName.equals( effectiveUri ) )
+                // This allows us to override a webjar file, if needed.  Mostly helpful during development.
+                final File file = new File( servletContext.getRealPath( effectiveUri ) );
+                if ( file.exists() )
                 {
-                    return entry.getValue();
+                    return Optional.of( new RealFileResource( file ) );
                 }
-            }
-        }
-
-        return fileSystemResource;
-    }
-
-    private static FileResource handleWebjarURIs(
-            final ServletContext servletContext,
-            final String resourcePathUri
-    )
-            throws PwmUnrecoverableException
-    {
-        if ( resourcePathUri.startsWith( ResourceFileServlet.WEBJAR_BASE_URL_PATH ) )
-        {
-            // This allows us to override a webjar file, if needed.  Mostly helpful during development.
-            final File file = new File( servletContext.getRealPath( resourcePathUri ) );
-            if ( file.exists() )
-            {
-                return new RealFileResource( file );
-            }
 
-            final String remainingPath = resourcePathUri.substring( ResourceFileServlet.WEBJAR_BASE_URL_PATH.length() );
+                final String remainingPath = effectiveUri.substring( ResourceFileServlet.WEBJAR_BASE_URL_PATH.length() );
 
-            final String webJarName;
-            final String webJarPath;
-            {
-                final int slashIndex = remainingPath.indexOf( "/" );
-                if ( slashIndex < 0 )
+                final String webJarName;
+                final String webJarPath;
                 {
-                    return null;
+                    final int slashIndex = remainingPath.indexOf( "/" );
+                    if ( slashIndex < 0 )
+                    {
+                        return Optional.empty();
+                    }
+                    webJarName = remainingPath.substring( 0, slashIndex );
+                    webJarPath = remainingPath.substring( slashIndex + 1 );
                 }
-                webJarName = remainingPath.substring( 0, slashIndex );
-                webJarPath = remainingPath.substring( slashIndex + 1 );
-            }
-
-            final String versionString = WEB_JAR_VERSION_MAP.get( webJarName );
-            if ( versionString == null )
-            {
-                return null;
-            }
 
-            final String fullPath = ResourceFileServlet.WEBJAR_BASE_FILE_PATH + "/" + webJarName + "/" + versionString + "/" + webJarPath;
-            if ( WEB_JAR_ASSET_LIST.contains( fullPath ) )
-            {
-                final ClassLoader classLoader = servletContext.getClassLoader();
-                final InputStream inputStream = classLoader.getResourceAsStream( fullPath );
+                final String versionString = WEB_JAR_VERSION_MAP.get( webJarName );
+                if ( versionString == null )
+                {
+                    return Optional.empty();
+                }
 
-                if ( inputStream != null )
+                final String fullPath = ResourceFileServlet.WEBJAR_BASE_FILE_PATH + "/" + webJarName + "/" + versionString + "/" + webJarPath;
+                if ( WEB_JAR_ASSET_LIST.contains( fullPath ) )
                 {
-                    return new InputStreamFileResource( inputStream, fullPath );
+                    final ClassLoader classLoader = servletContext.getClassLoader();
+                    final InputStream inputStream = classLoader.getResourceAsStream( fullPath );
+
+                    if ( inputStream != null )
+                    {
+                        return Optional.of( new InputStreamFileResource( inputStream, fullPath ) );
+                    }
                 }
             }
-        }
 
-        return null;
+            return Optional.empty();
+        }
     }
 
+    @Value
     private static class InputStreamFileResource implements FileResource
     {
         private final InputStream inputStream;
-        private final String fullPath;
-
-        InputStreamFileResource( final InputStream inputStream, final String fullPath )
-        {
-            this.inputStream = inputStream;
-            this.fullPath = fullPath;
-        }
+        private final String name;
 
         @Override
-        public InputStream getInputStream( ) throws IOException
+        public long length( )
         {
-            return inputStream;
+            return 0;
         }
 
         @Override
-        public long length( )
+        public Instant lastModified( )
         {
-            return 0;
+            return Instant.EPOCH;
         }
+    }
+
 
+    private static class RealFileUriResolver implements ResourceUriResolver
+    {
         @Override
-        public long lastModified( )
+        public Optional<FileResource> resolveUri( final ResourceRequestContext resourceRequestContext )
+                throws PwmUnrecoverableException
         {
-            return 0;
+            final String effectiveUri = resourceRequestContext.getUri();
+            final ServletContext servletContext = resourceRequestContext.getServletContext();
+
+            // convert to file.
+            final String filePath = servletContext.getRealPath( effectiveUri );
+            final File file = new File( filePath );
+
+            if ( file.exists() )
+            {
+                verifyPath( file, servletContext );
+
+                return Optional.of( new RealFileResource( file ) );
+            }
+
+            return Optional.empty();
         }
 
-        @Override
-        public boolean exists( )
+        private void verifyPath(
+                final File file,
+                final ServletContext servletContext
+        )
+                throws PwmUnrecoverableException
         {
-            return true;
+            // figure top-most path allowed by request
+            final String parentDirectoryPath = servletContext.getRealPath( ResourceFileServlet.RESOURCE_PATH );
+            final File parentDirectory = new File( parentDirectoryPath );
+
+            {
+                //verify the requested page is a child of the servlet resource path.
+                int recursions = 0;
+                File recurseFile = file.getParentFile();
+                while ( recurseFile != null && recursions < 100 )
+                {
+                    if ( parentDirectory.equals( recurseFile ) )
+                    {
+                        return;
+                    }
+                    recurseFile = recurseFile.getParentFile();
+                    recursions++;
+                }
+            }
+
+            LOGGER.warn( () -> "attempt to access file outside of servlet path " + file.getAbsolutePath() );
+            throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, "illegal file path request" ) );
         }
+    }
 
+    private static class CustomZipFileUriResolver implements ResourceUriResolver
+    {
         @Override
-        public String getName( )
+        public Optional<FileResource> resolveUri( final ResourceRequestContext resourceRequestContext )
+                throws PwmUnrecoverableException
         {
-            return fullPath;
+            final String effectiveUri = resourceRequestContext.getUri();
+            final ResourceServletConfiguration resourceServletConfiguration = resourceRequestContext.getResourceServletConfiguration();
+
+            // check custom (configuration defined) zip file bundles
+            final Map<String, FileResource> customResources = resourceServletConfiguration.getCustomFileBundle();
+            for ( final Map.Entry<String, FileResource> entry : customResources.entrySet() )
+            {
+                final String customFileName = entry.getKey();
+                final String testName = ResourceFileServlet.RESOURCE_PATH + "/" + customFileName;
+                if ( testName.equals( effectiveUri ) )
+                {
+                    return Optional.of( entry.getValue() );
+                }
+            }
+
+            return Optional.empty();
         }
     }
 
+
+
     /**
      * Returns true if the given accept header accepts the given value.
      *

+ 65 - 35
server/src/main/java/password/pwm/http/servlet/resource/ResourceFileServlet.java

@@ -21,8 +21,8 @@
 package password.pwm.http.servlet.resource;
 
 import com.github.benmanes.caffeine.cache.Cache;
-import password.pwm.PwmDomain;
 import password.pwm.PwmConstants;
+import password.pwm.PwmDomain;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
@@ -34,7 +34,6 @@ import password.pwm.http.servlet.PwmServlet;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsClient;
 import password.pwm.util.java.JavaHelper;
-import password.pwm.util.java.MovingAverage;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
@@ -49,8 +48,10 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.time.Instant;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Supplier;
 import java.util.zip.GZIPOutputStream;
 
@@ -119,20 +120,22 @@ public class ResourceFileServlet extends HttpServlet implements PwmServlet
 
         final ResourceFileRequest resourceFileRequest = new ResourceFileRequest( null, ResourceServletConfiguration.defaultConfiguration(), req );
 
-        final FileResource file = resourceFileRequest.getRequestedFileResource();
+        final Optional<FileResource> file = resourceFileRequest.getRequestedFileResource();
 
-        if ( file == null || !file.exists() )
+        if ( file.isEmpty() )
         {
             resp.sendError( HttpServletResponse.SC_NOT_FOUND );
             return;
         }
 
-        handleUncachedResponse( resp, file, false );
+        handleUncachedResponse( resp, file.get(), false );
     }
 
     protected void processAction( final PwmRequest pwmRequest )
             throws IOException, PwmUnrecoverableException
     {
+        final Instant startTime = Instant.now();
+
         if ( pwmRequest.getMethod() != HttpMethod.GET )
         {
             throw new PwmUnrecoverableException( new ErrorInformation(
@@ -148,36 +151,15 @@ public class ResourceFileServlet extends HttpServlet implements PwmServlet
         final String requestURI = resourceFileRequest.getRequestURI();
 
         final FileResource file;
-        try
         {
-            file = resourceFileRequest.getRequestedFileResource();
-        }
-        catch ( final PwmUnrecoverableException e )
-        {
-            pwmRequest.getPwmResponse().getHttpServletResponse().sendError( HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage() );
-            try
-            {
-                pwmRequest.debugHttpRequestToLog( "returning HTTP 500 status", null );
-            }
-            catch ( final PwmUnrecoverableException e2 )
-            {
-                /* noop */
-            }
-            return;
-        }
+            final Optional<FileResource> resolvedFile = doResolve( resourceService, resourceFileRequest, pwmRequest );
 
-        if ( file == null || !file.exists() )
-        {
-            pwmRequest.getPwmResponse().getHttpServletResponse().sendError( HttpServletResponse.SC_NOT_FOUND );
-            try
-            {
-                pwmRequest.debugHttpRequestToLog( "returning HTTP 404 status", null );
-            }
-            catch ( final PwmUnrecoverableException e )
+            if ( resolvedFile.isEmpty() )
             {
-                /* noop */
+                return;
             }
-            return;
+
+            file = resolvedFile.get();
         }
 
         // Get content type by file name and set default GZIP support and content disposition.
@@ -213,9 +195,11 @@ public class ResourceFileServlet extends HttpServlet implements PwmServlet
 
             pwmRequest.debugHttpRequestToLog( debugText, () -> TimeDuration.fromCurrent( pwmRequest.getRequestStartTime() ) );
 
-            final MovingAverage cacheHitRatio = resourceService.getCacheHitRatio();
             StatisticsClient.incrementStat( pwmDomain, Statistic.HTTP_RESOURCE_REQUESTS );
-            cacheHitRatio.update( fromCache ? 1 : 0 );
+            resourceService.getAverageStats().update( ResourceServletService.AverageStat.cacheHitRatio, fromCache ? 1 : 0 );
+            resourceService.getAverageStats().update( ResourceServletService.AverageStat.avgResponseTimeMS, TimeDuration.fromCurrent( startTime ) );
+            resourceService.getCountingStats().increment( ResourceServletService.CountingStat.requestsServed );
+            resourceService.getCountingStats().increment( ResourceServletService.CountingStat.bytesServed, file.length() );
         }
         catch ( final Exception e )
         {
@@ -232,6 +216,52 @@ public class ResourceFileServlet extends HttpServlet implements PwmServlet
         }
     }
 
+    private Optional<FileResource> doResolve(
+            final ResourceServletService resourceService,
+            final ResourceFileRequest resourceFileRequest,
+            final PwmRequest pwmRequest
+
+    )
+            throws IOException
+    {
+        try
+        {
+            final Optional<FileResource> resolvedFile = resourceFileRequest.getRequestedFileResource();
+
+            if ( resolvedFile.isEmpty() )
+            {
+                pwmRequest.getPwmResponse().getHttpServletResponse().sendError( HttpServletResponse.SC_NOT_FOUND );
+                resourceService.getCountingStats().increment( ResourceServletService.CountingStat.requestsNotFound );
+
+                try
+                {
+                    pwmRequest.debugHttpRequestToLog( "returning HTTP 404 status", null );
+                }
+                catch ( final PwmUnrecoverableException e )
+                {
+                    /* noop */
+                }
+                return Optional.empty();
+            }
+
+            return Optional.of( resolvedFile.get() );
+        }
+        catch ( final PwmUnrecoverableException e )
+        {
+            pwmRequest.getPwmResponse().getHttpServletResponse().sendError( HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage() );
+            try
+            {
+                pwmRequest.debugHttpRequestToLog( "returning HTTP 500 status", null );
+            }
+            catch ( final PwmUnrecoverableException e2 )
+            {
+                /* noop */
+            }
+        }
+
+        return Optional.empty();
+    }
+
     private String makeDebugText( final boolean fromCache, final boolean acceptsGzip, final boolean uncacheable )
     {
         if ( uncacheable )
@@ -278,7 +308,7 @@ public class ResourceFileServlet extends HttpServlet implements PwmServlet
     )
             throws UncacheableResourceException, IOException, PwmUnrecoverableException
     {
-        final FileResource file = resourceFileRequest.getRequestedFileResource();
+        final FileResource file = resourceFileRequest.getRequestedFileResource().orElseThrow();
 
         if ( file.length() > resourceFileRequest.getResourceServletConfiguration().getMaxCacheBytes() )
         {
@@ -293,7 +323,7 @@ public class ResourceFileServlet extends HttpServlet implements PwmServlet
             final Map<String, String> headers = new HashMap<>();
             final ByteArrayOutputStream tempOutputStream = new ByteArrayOutputStream();
 
-            try ( InputStream input = resourceFileRequest.getRequestedFileResource().getInputStream() )
+            try ( InputStream input = file.getInputStream() )
             {
                 if ( resourceFileRequest.allowsCompression() )
                 {

+ 2 - 1
server/src/main/java/password/pwm/http/servlet/resource/ResourceServletConfiguration.java

@@ -40,6 +40,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -195,7 +196,7 @@ class ResourceServletConfiguration
             if ( !entry.isDirectory() )
             {
                 final String name = entry.getName();
-                final long lastModified = entry.getTime();
+                final Instant lastModified = Instant.ofEpochMilli( entry.getTime() );
                 final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
                 IOUtils.copy( stream, byteArrayOutputStream );
                 final ImmutableByteArray contents = ImmutableByteArray.of( byteArrayOutputStream.toByteArray() );

+ 34 - 8
server/src/main/java/password/pwm/http/servlet/resource/ResourceServletService.java

@@ -36,8 +36,9 @@ import password.pwm.svc.AbstractPwmService;
 import password.pwm.svc.PwmService;
 import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.java.JavaHelper;
-import password.pwm.util.java.MovingAverage;
 import password.pwm.util.java.Percent;
+import password.pwm.util.java.StatisticAverageBundle;
+import password.pwm.util.java.StatisticCounterBundle;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.secure.ChecksumOutputStream;
@@ -64,11 +65,26 @@ public class ResourceServletService extends AbstractPwmService implements PwmSer
 
     private ResourceServletConfiguration resourceServletConfiguration;
     private Cache<CacheKey, CacheEntry> cache;
-    private final MovingAverage cacheHitRatio = new MovingAverage( 60 * 60 * 1000 );
     private String resourceNonce = "";
 
     private PwmDomain pwmDomain;
 
+    private final StatisticAverageBundle<AverageStat> averageStats = new StatisticAverageBundle<>( AverageStat.class );
+    private final StatisticCounterBundle<CountingStat> countingStats = new StatisticCounterBundle<>( CountingStat.class );
+
+    enum AverageStat
+    {
+        cacheHitRatio,
+        avgResponseTimeMS,
+    }
+
+    enum CountingStat
+    {
+        requestsServed,
+        requestsNotFound,
+        bytesServed,
+    }
+
     public String getResourceNonce( )
     {
         return resourceNonce;
@@ -79,9 +95,14 @@ public class ResourceServletService extends AbstractPwmService implements PwmSer
         return cache;
     }
 
-    public MovingAverage getCacheHitRatio( )
+    StatisticAverageBundle<AverageStat> getAverageStats()
+    {
+        return averageStats;
+    }
+
+    StatisticCounterBundle<CountingStat> getCountingStats()
     {
-        return cacheHitRatio;
+        return countingStats;
     }
 
     public long bytesInCache( )
@@ -106,7 +127,7 @@ public class ResourceServletService extends AbstractPwmService implements PwmSer
 
     public Percent cacheHitRatio( )
     {
-        final BigDecimal numerator = BigDecimal.valueOf( getCacheHitRatio().getAverage() );
+        final BigDecimal numerator = BigDecimal.valueOf( averageStats.getAverage( AverageStat.cacheHitRatio ) );
         final BigDecimal denominator = BigDecimal.ONE;
         return Percent.of( numerator, denominator );
     }
@@ -168,7 +189,12 @@ public class ResourceServletService extends AbstractPwmService implements PwmSer
     @Override
     public ServiceInfoBean serviceInfo( )
     {
-        return null;
+        final Map<String, String> debugInfo = new HashMap<>();
+        debugInfo.putAll( averageStats.debugStats() );
+        debugInfo.putAll( countingStats.debugStats() );
+        return ServiceInfoBean.builder()
+                .debugProperties( debugInfo )
+                .build();
     }
 
     ResourceServletConfiguration getResourceServletConfiguration( )
@@ -223,12 +249,12 @@ public class ResourceServletService extends AbstractPwmService implements PwmSer
         for ( final String testUrl : testUrls )
         {
             final String themePathUrl = ResourceFileServlet.RESOURCE_PATH + testUrl.replace( ResourceFileServlet.TOKEN_THEME, themeName );
-            final FileResource resolvedFile = ResourceFileRequest.resolveRequestedResource(
+            final Optional<FileResource> resolvedFile = ResourceFileRequest.resolveRequestedResource(
                     pwmRequest.getDomainConfig(),
                     servletContext,
                     themePathUrl,
                     getResourceServletConfiguration() );
-            if ( resolvedFile != null && resolvedFile.exists() )
+            if ( resolvedFile.isPresent() )
             {
                 LOGGER.debug( pwmRequest, () -> "check for theme validity of '" + themeName + "' returned true" );
                 return true;

+ 3 - 8
server/src/main/java/password/pwm/http/servlet/resource/ZipFileResource.java

@@ -22,6 +22,7 @@ package password.pwm.http.servlet.resource;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Instant;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
@@ -53,15 +54,9 @@ class ZipFileResource implements FileResource
     }
 
     @Override
-    public long lastModified( )
+    public Instant lastModified( )
     {
-        return zipEntry.getTime();
-    }
-
-    @Override
-    public boolean exists( )
-    {
-        return zipEntry != null && zipFile != null;
+        return Instant.ofEpochMilli( zipEntry.getTime() );
     }
 
     @Override

+ 17 - 10
server/src/main/java/password/pwm/svc/httpclient/ApachePwmHttpClient.java

@@ -34,6 +34,7 @@ import org.apache.http.client.CredentialsProvider;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpHead;
 import org.apache.http.client.methods.HttpPatch;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
@@ -100,7 +101,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
-public class ApachePwmHttpClient implements AutoCloseable, PwmHttpClient
+public class ApachePwmHttpClient implements AutoCloseable, PwmHttpClientProvider
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( ApachePwmHttpClient.class );
 
@@ -383,15 +384,18 @@ public class ApachePwmHttpClient implements AutoCloseable, PwmHttpClient
 
         final Optional<HttpContentType> optionalHttpContentType = contentTypeForEntity( httpResponse.getEntity() );
 
-        if ( optionalHttpContentType.isPresent() && optionalHttpContentType.get().getDataType() ==  HttpEntityDataType.ByteArray )
+        if ( httpResponse.getEntity() != null )
         {
-            httpClientResponseBuilder.binaryBody( readBinaryEntityBody( httpResponse.getEntity() ) );
-            httpClientResponseBuilder.dataType( HttpEntityDataType.ByteArray );
-        }
-        else
-        {
-            httpClientResponseBuilder.body( EntityUtils.toString( httpResponse.getEntity() ) );
-            httpClientResponseBuilder.dataType( HttpEntityDataType.String );
+            if ( optionalHttpContentType.isPresent() && optionalHttpContentType.get().getDataType() == HttpEntityDataType.ByteArray )
+            {
+                httpClientResponseBuilder.binaryBody( readBinaryEntityBody( httpResponse.getEntity() ) );
+                httpClientResponseBuilder.dataType( HttpEntityDataType.ByteArray );
+            }
+            else
+            {
+                httpClientResponseBuilder.body( EntityUtils.toString( httpResponse.getEntity() ) );
+                httpClientResponseBuilder.dataType( HttpEntityDataType.String );
+            }
         }
 
         final Map<String, String> responseHeaders = new LinkedHashMap<>();
@@ -464,6 +468,10 @@ public class ApachePwmHttpClient implements AutoCloseable, PwmHttpClient
                 httpRequest = new HttpDelete( clientRequest.getUrl() );
                 break;
 
+            case HEAD:
+                httpRequest = new HttpHead( clientRequest.getUrl() );
+                break;
+
             default:
                 throw new IllegalStateException( "http method not yet implemented" );
         }
@@ -477,7 +485,6 @@ public class ApachePwmHttpClient implements AutoCloseable, PwmHttpClient
             }
         }
 
-
         httpClientService.getStats().increment( HttpClientService.StatsKey.requests );
         httpClientService.getStats().increment( HttpClientService.StatsKey.requestBytes, clientRequest.size() );
         StatisticsClient.incrementStat( pwmApplication, Statistic.HTTP_CLIENT_REQUESTS );

+ 6 - 6
server/src/main/java/password/pwm/svc/httpclient/HttpClientService.java

@@ -48,10 +48,10 @@ public class HttpClientService extends AbstractPwmService implements PwmService
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( HttpClientService.class );
 
-    private Class<PwmHttpClient> httpClientClass;
+    private Class<PwmHttpClientProvider> httpClientClass;
 
-    private final Map<PwmHttpClientConfiguration, ThreadLocal<PwmHttpClient>> clients = new ConcurrentHashMap<>(  );
-    private final Map<PwmHttpClient, Object> issuedClients = Collections.synchronizedMap( new WeakHashMap<>(  ) );
+    private final Map<PwmHttpClientConfiguration, ThreadLocal<PwmHttpClientProvider>> clients = new ConcurrentHashMap<>(  );
+    private final Map<PwmHttpClientProvider, Object> issuedClients = Collections.synchronizedMap( new WeakHashMap<>(  ) );
 
     private final StatisticCounterBundle<StatsKey> stats = new StatisticCounterBundle<>( StatsKey.class );
 
@@ -81,7 +81,7 @@ public class HttpClientService extends AbstractPwmService implements PwmService
         final String implClassName = pwmApplication.getConfig().readAppProperty( AppProperty.HTTP_CLIENT_IMPLEMENTATION );
         try
         {
-            this.httpClientClass = ( Class<PwmHttpClient> ) this.getClass().getClassLoader().loadClass( implClassName );
+            this.httpClientClass = ( Class<PwmHttpClientProvider> ) this.getClass().getClassLoader().loadClass( implClassName );
         }
         catch ( final ClassNotFoundException e )
         {
@@ -121,7 +121,7 @@ public class HttpClientService extends AbstractPwmService implements PwmService
     {
         Objects.requireNonNull( pwmHttpClientConfiguration );
 
-        final ThreadLocal<PwmHttpClient> threadLocal = clients.computeIfAbsent(
+        final ThreadLocal<PwmHttpClientProvider> threadLocal = clients.computeIfAbsent(
                 pwmHttpClientConfiguration,
                 clientConfig -> new ThreadLocal<>() );
 
@@ -134,7 +134,7 @@ public class HttpClientService extends AbstractPwmService implements PwmService
 
         try
         {
-            final PwmHttpClient newClient = httpClientClass.getDeclaredConstructor().newInstance();
+            final PwmHttpClientProvider newClient = httpClientClass.getDeclaredConstructor().newInstance();
             newClient.init( getPwmApplication(), this, pwmHttpClientConfiguration );
             issuedClients.put( newClient, null );
             threadLocal.set( newClient );

+ 1 - 1
server/src/main/java/password/pwm/svc/httpclient/JavaPwmHttpClient.java

@@ -64,7 +64,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
-public class JavaPwmHttpClient implements PwmHttpClient
+public class JavaPwmHttpClient implements PwmHttpClientProvider
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( JavaPwmHttpClient.class );
 

+ 1 - 6
server/src/main/java/password/pwm/svc/httpclient/PwmHttpClient.java

@@ -20,7 +20,6 @@
 
 package password.pwm.svc.httpclient;
 
-import password.pwm.PwmApplication;
 import password.pwm.bean.SessionLabel;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.secure.CertificateReadingTrustManager;
@@ -33,12 +32,8 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-public interface PwmHttpClient
+public interface PwmHttpClient extends AutoCloseable
 {
-
-    void init( PwmApplication pwmApplication, HttpClientService httpClientService, PwmHttpClientConfiguration pwmHttpClientConfiguration )
-            throws PwmUnrecoverableException;
-
     void close()
             throws Exception;
 

+ 31 - 0
server/src/main/java/password/pwm/svc/httpclient/PwmHttpClientProvider.java

@@ -0,0 +1,31 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2020 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.svc.httpclient;
+
+import password.pwm.PwmApplication;
+import password.pwm.error.PwmUnrecoverableException;
+
+public interface PwmHttpClientProvider extends PwmHttpClient
+{
+    void init( PwmApplication pwmApplication, HttpClientService httpClientService, PwmHttpClientConfiguration pwmHttpClientConfiguration )
+            throws PwmUnrecoverableException;
+
+}

+ 1 - 1
server/src/main/java/password/pwm/svc/httpclient/PwmHttpClientResponse.java

@@ -55,7 +55,7 @@ public class PwmHttpClientResponse implements Serializable, PwmHttpClientMessage
     public long size()
     {
         long size = 0;
-        size += statusPhrase.length();
+        size += statusPhrase == null ? 0 : statusPhrase.length();
         size += body == null ? 0 : body.length();
         size += binaryBody == null ? 0 : binaryBody.size();
         if ( headers != null )

+ 21 - 9
server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java

@@ -150,6 +150,8 @@ abstract class AbstractWordlist extends AbstractPwmService implements Wordlist,
 
     boolean containsWord( final Set<WordType> wordTypes, final String word ) throws PwmUnrecoverableException
     {
+        final Instant startTime = Instant.now();
+
         final Optional<String> testWord = WordlistUtil.normalizeWordLength( word, wordlistConfiguration );
 
         if ( testWord.isEmpty() )
@@ -173,6 +175,18 @@ abstract class AbstractWordlist extends AbstractPwmService implements Wordlist,
             }
         }
 
+        getStatistics().getAverageStats().update( WordlistStatistics.AverageStat.avgWordCheckLength, word.length() );
+        getStatistics().getAverageStats().update( WordlistStatistics.AverageStat.wordCheckTimeMS, TimeDuration.fromCurrent( startTime ) );
+        getStatistics().getCounterStats().increment( WordlistStatistics.CounterStat.wordChecks );
+        if ( result )
+        {
+            getStatistics().getCounterStats().increment( WordlistStatistics.CounterStat.wordHits );
+        }
+        else
+        {
+            getStatistics().getCounterStats().increment( WordlistStatistics.CounterStat.wordMisses );
+        }
+
         return result;
     }
 
@@ -223,26 +237,24 @@ abstract class AbstractWordlist extends AbstractPwmService implements Wordlist,
     private boolean realBucketCheck( final String word, final WordType wordType )
             throws PwmUnrecoverableException
     {
-        getStatistics().getCounterStats().increment( WordlistStatistics.CounterStat.wordChecks );
-
         final Instant startTime = Instant.now();
-        final boolean isContainsWord = wordlistBucket.containsWord( word );
-
-        final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
-        getStatistics().getAverageStats().update( WordlistStatistics.AverageStat.wordCheckTimeMS,  timeDuration.asMillis() );
+        final boolean results = wordlistBucket.containsWord( word );
 
         statsOutput.conditionallyExecuteTask();
 
-        if ( isContainsWord )
+        getStatistics().getAverageStats().update( WordlistStatistics.AverageStat.chunkCheckTimeMS, TimeDuration.fromCurrent( startTime ) );
+        getStatistics().getCounterStats().increment( WordlistStatistics.CounterStat.chunkChecks );
+        if ( results )
         {
             getStatistics().getWordTypeHits().get( wordType ).increment();
+            getStatistics().getCounterStats().increment( WordlistStatistics.CounterStat.chunkHits );
         }
         else
         {
-            getStatistics().getCounterStats().increment( WordlistStatistics.CounterStat.misses );
+            getStatistics().getCounterStats().increment( WordlistStatistics.CounterStat.chunkMisses );
         }
 
-        return isContainsWord;
+        return results;
     }
 
     String randomSeed() throws PwmUnrecoverableException

+ 5 - 0
server/src/main/java/password/pwm/svc/wordlist/WordlistConfiguration.java

@@ -178,6 +178,11 @@ public class WordlistConfiguration implements Serializable
         }
     } );
 
+    public boolean isAutoImportUrlConfigured()
+    {
+        return StringUtil.notEmpty( getAutoImportUrl() );
+    }
+
     String configHash( )
     {
         return configHash.get();

+ 1 - 1
server/src/main/java/password/pwm/svc/wordlist/WordlistInspector.java

@@ -80,7 +80,7 @@ class WordlistInspector implements Runnable
         cancelCheck();
 
         rootWordlist.setActivity( Wordlist.Activity.ReadingWordlistFile );
-        final boolean autoImportUrlConfigured = StringUtil.notEmpty( rootWordlist.getConfiguration().getAutoImportUrl() );
+        final boolean autoImportUrlConfigured = rootWordlist.getConfiguration().isAutoImportUrlConfigured();
         WordlistStatus existingStatus = rootWordlist.readWordlistStatus();
 
         if ( checkIfClearIsNeeded( existingStatus, autoImportUrlConfigured ) )

+ 58 - 5
server/src/main/java/password/pwm/svc/wordlist/WordlistSource.java

@@ -27,8 +27,12 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.http.ContextManager;
+import password.pwm.http.HttpHeader;
+import password.pwm.http.HttpMethod;
 import password.pwm.svc.httpclient.PwmHttpClient;
 import password.pwm.svc.httpclient.PwmHttpClientConfiguration;
+import password.pwm.svc.httpclient.PwmHttpClientRequest;
+import password.pwm.svc.httpclient.PwmHttpClientResponse;
 import password.pwm.util.java.ConditionalTaskExecutor;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
@@ -41,13 +45,22 @@ import java.io.InputStream;
 import java.net.URL;
 import java.security.DigestInputStream;
 import java.time.Instant;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BooleanSupplier;
 
 class WordlistSource
 {
+    private static final Set<HttpHeader> HTTP_INTERESTED_HEADERS = Set.of(
+            HttpHeader.ETag,
+            HttpHeader.ContentLength,
+            HttpHeader.Last_Modified );
+
     private static final AtomicInteger CLOSE_COUNTER = new AtomicInteger();
 
     private final WordlistSourceType wordlistSourceType;
@@ -138,6 +151,39 @@ class WordlistSource
         return new WordlistZipReader( this.streamProvider.getInputStream() );
     }
 
+    Map<HttpHeader, String> readRemoteHeaders(
+            final PwmApplication pwmApplication,
+            final PwmLogger pwmLogger
+    )
+            throws PwmUnrecoverableException
+    {
+        final Instant startTime = Instant.now();
+
+        final PwmHttpClient pwmHttpClient = pwmApplication.getHttpClientService().getPwmHttpClient();
+        final PwmHttpClientRequest request = PwmHttpClientRequest.builder()
+                .method( HttpMethod.HEAD )
+                .url( importUrl )
+                .build();
+        final PwmHttpClientResponse response = pwmHttpClient.makeRequest( request, null );
+        final Map<HttpHeader, String> returnResponses = new EnumMap<>( HttpHeader.class );
+        for ( final Map.Entry<String, String> entry : response.getHeaders().entrySet() )
+        {
+            final String headerStrName = entry.getKey();
+            HttpHeader.forHttpHeader( headerStrName ).ifPresent( header ->
+            {
+                if ( HTTP_INTERESTED_HEADERS.contains( header ) )
+                {
+                    returnResponses.put( header, entry.getValue() );
+                }
+            } );
+        }
+
+        final Map<HttpHeader, String> finalReturnResponses =  Collections.unmodifiableMap( returnResponses );
+        pwmLogger.debug( () -> "read remote header info for " + this.getWordlistSourceType() + " wordlist: "
+                + JsonUtil.serializeMap( finalReturnResponses ), () -> TimeDuration.fromCurrent( startTime ) );
+        return finalReturnResponses;
+    }
+
     WordlistSourceInfo readRemoteWordlistInfo(
             final PwmApplication pwmApplication,
             final BooleanSupplier cancelFlag,
@@ -169,11 +215,17 @@ class WordlistSource
             zipInputStream = new WordlistZipReader( checksumInputStream );
             final ConditionalTaskExecutor debugOutputter = makeDebugLoggerExecutor( pwmLogger, startTime, zipInputStream );
 
+            int counter = 0;
             String nextLine;
             do
             {
+                counter++;
                 nextLine = zipInputStream.nextLine();
-                debugOutputter.conditionallyExecuteTask();
+
+                if ( counter % 10000 == 0 )
+                {
+                    debugOutputter.conditionallyExecuteTask();
+                }
 
                 if ( cancelFlag.getAsBoolean() )
                 {
@@ -181,10 +233,6 @@ class WordlistSource
                 }
             }
             while ( nextLine != null );
-
-            bytes = zipInputStream.getByteCount();
-            hash = JavaHelper.byteArrayToHexString( checksumInputStream.getMessageDigest().digest() );
-            lines = zipInputStream.getLineCount();
         }
         catch ( final IOException e )
         {
@@ -200,6 +248,10 @@ class WordlistSource
             IOUtils.closeQuietly( zipInputStream );
         }
 
+        bytes = zipInputStream.getByteCount();
+        hash = JavaHelper.byteArrayToHexString( checksumInputStream.getMessageDigest().digest() );
+        lines = zipInputStream.getLineCount();
+
         if ( cancelFlag.getAsBoolean() )
         {
             throw new CancellationException();
@@ -264,6 +316,7 @@ class WordlistSource
             pwmLogger.trace( () -> "completed close of remote wordlist read process [" + counter + "]",
                     () -> TimeDuration.fromCurrent( startClose ) );
         } );
+
         closerThread.setDaemon( true );
         closerThread.setName( Thread.currentThread().getName() + "-import-close-io" );
         closerThread.start();

+ 7 - 1
server/src/main/java/password/pwm/svc/wordlist/WordlistStatistics.java

@@ -40,12 +40,18 @@ class WordlistStatistics
     enum CounterStat
     {
         wordChecks,
-        misses,
+        wordHits,
+        wordMisses,
+        chunkChecks,
+        chunkHits,
+        chunkMisses,
     }
 
     enum AverageStat
     {
+        avgWordCheckLength,
         wordCheckTimeMS,
+        chunkCheckTimeMS,
         chunksPerWordCheck,
     }
 

+ 2 - 1
server/src/main/java/password/pwm/util/java/ConditionalTaskExecutor.java

@@ -26,6 +26,7 @@ import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.BooleanSupplier;
 
@@ -43,7 +44,7 @@ public class ConditionalTaskExecutor
 
     private final Runnable task;
     private final BooleanSupplier predicate;
-    private final ReentrantLock lock = new ReentrantLock();
+    private final Lock lock = new ReentrantLock();
 
     /**
      * Execute the task if the conditional has been met.  Exceptions when running the task will be logged but not returned.

+ 2 - 2
server/src/main/java/password/pwm/util/java/MovingAverage.java

@@ -54,8 +54,8 @@ public class MovingAverage implements Serializable
     private final Lock lock = new ReentrantLock();
     private final long windowMillis;
 
-    private long lastMillis;
-    private double average;
+    private volatile long lastMillis;
+    private volatile double average;
 
 
     /**

+ 47 - 72
server/src/main/java/password/pwm/util/java/StringUtil.java

@@ -300,17 +300,22 @@ public abstract class StringUtil
         return new String( base32.encode( input ), PwmConstants.DEFAULT_CHARSET );
     }
 
-    public static byte[] base64Decode( final String input, final StringUtil.Base64Options... options )
+    public static byte[] base64Decode( final CharSequence input, final StringUtil.Base64Options... options )
             throws IOException
     {
+        if ( StringUtil.isEmpty( input ) )
+        {
+            return new byte[0];
+        }
+
         final byte[] decodedBytes;
         if ( JavaHelper.enumArrayContainsValue( options, Base64Options.URL_SAFE ) )
         {
-            decodedBytes = java.util.Base64.getUrlDecoder().decode( input );
+            decodedBytes = java.util.Base64.getUrlDecoder().decode( input.toString() );
         }
         else
         {
-            decodedBytes = java.util.Base64.getMimeDecoder().decode( input );
+            decodedBytes = java.util.Base64.getMimeDecoder().decode( input.toString() );
         }
 
         if ( JavaHelper.enumArrayContainsValue( options, Base64Options.GZIP ) )
@@ -572,63 +577,28 @@ public abstract class StringUtil
         }
     }
 
-    public static String stripAllWhitespace( final String input )
+    public static String stripAllWhitespace( final CharSequence input )
     {
         return stripAllChars( input, Character::isWhitespace );
     }
 
-    public static String stripAllChars( final String input, final Predicate<Character> characterPredicate )
+    public static CharSequence cleanNonPrintableCharacters( final CharSequence input )
     {
-        if ( isEmpty( input ) )
-        {
-            return "";
-        }
-
-        if ( characterPredicate == null )
-        {
-            return input;
-        }
-
-        // count of valid output chars
-        int copiedChars = 0;
-
-        // loop through input chars and stop if stripped char is found
-        while ( copiedChars < input.length() )
-        {
-            if ( !characterPredicate.test( input.charAt( copiedChars ) ) )
-            {
-                copiedChars++;
-            }
-            else
-            {
-                break;
-            }
-        }
-
-        // return input string if we made it through input without detecting stripped char
-        if ( copiedChars >= input.length() )
-        {
-            return input;
-        }
-
-        // creating sb with input gives good length value and handles copy of chars so far...
-        final StringBuilder sb = new StringBuilder( input );
+        final Predicate<Character> nonPrintableCharPredicate = character ->
+                ( Character.isISOControl( character ) && !Character.isWhitespace( character ) )
+                        || !Character.isDefined( character );
 
-        // loop through remaining chars and copy one by one
-        for ( int loopIndex = copiedChars; loopIndex < input.length(); loopIndex++ )
-        {
-            final char loopChar = input.charAt( loopIndex );
-            if ( !characterPredicate.test( loopChar ) )
-            {
-                sb.setCharAt( copiedChars, loopChar );
-                copiedChars++;
-            }
-        }
+        return replaceAllChars( input,
+                character -> nonPrintableCharPredicate.test( character ) ? Optional.of( "?" ) : Optional.empty() );
+    }
 
-        return sb.substring( 0, copiedChars );
+    public static String stripAllChars( final CharSequence input, final Predicate<Character> characterPredicate )
+    {
+        return replaceAllChars( input,
+                character -> ( characterPredicate.test( character ) ) ? Optional.of( "" ) : Optional.empty() );
     }
 
-    public static String replaceAllChars( final String input, final Function<Character, Optional<String>> replacementFunction )
+    public static String replaceAllChars( final CharSequence input, final Function<Character, Optional<String>> replacementFunction )
     {
         if ( isEmpty( input ) )
         {
@@ -637,39 +607,43 @@ public abstract class StringUtil
 
         if ( replacementFunction == null )
         {
-            return input;
+            return input.toString();
         }
 
-        // count of valid output chars
-        int copiedChars = 0;
+        final int inputLength = input.length();
 
-        // loop through input chars and stop if replacement char is needed
-        while ( copiedChars < input.length() )
+        // count of valid output chars
+        int index = 0;
         {
-            final Character indexChar = input.charAt( copiedChars );
-            final Optional<String> replacementStr = replacementFunction.apply( indexChar );
-            if ( replacementStr.isEmpty() )
+            // loop through input chars and stop if replacement char is needed ( but no actual coppying yet )
+            while ( index < inputLength )
             {
-                copiedChars++;
+                final Character indexChar = input.charAt( index );
+                final Optional<String> replacementStr = replacementFunction.apply( indexChar );
+                if ( replacementStr.isEmpty() )
+                {
+                    index++;
+                }
+                else
+                {
+                    break;
+                }
             }
-            else
+
+            // return input string if we made it through input without detecting replacement char
+            if ( index >= inputLength )
             {
-                break;
+                return input.toString();
             }
         }
 
-        // return input string if we made it through input without detecting replacement char
-        if ( copiedChars >= input.length() )
-        {
-            return input;
-        }
-
-        final StringBuilder sb = new StringBuilder( input.substring( 0, copiedChars ) );
+        // create the destination builder
+        final StringBuilder sb = new StringBuilder( input.subSequence( 0, index ) );
 
         // loop through remaining chars and copy one by one
-        for ( int loopIndex = copiedChars; loopIndex < input.length(); loopIndex++ )
+        while ( index < inputLength )
         {
-            final char loopChar = input.charAt( loopIndex );
+            final char loopChar = input.charAt( index );
             final Optional<String> replacementStr = replacementFunction.apply( loopChar );
             if ( replacementStr.isPresent() )
             {
@@ -679,6 +653,7 @@ public abstract class StringUtil
             {
                 sb.append( loopChar );
             }
+            index++;
         }
 
         return sb.toString();
@@ -781,7 +756,7 @@ public abstract class StringUtil
             Map.entry( ']', "%5D" )
     );
 
-    public static String urlPathEncode( final String input )
+    public static CharSequence urlPathEncode( final CharSequence input )
     {
         return replaceAllChars( input,
                 character -> Optional.ofNullable( URL_PATH_ENCODING_REPLACEMENTS.getOrDefault( character, null ) ) );

+ 3 - 3
server/src/main/java/password/pwm/util/localdb/LocalDBFactory.java

@@ -31,6 +31,7 @@ import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
 import java.io.File;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -62,7 +63,7 @@ public class LocalDBFactory
                     ? pwmEnvironment.getConfig()
                     : appConfig;
 
-            final long startTime = System.currentTimeMillis();
+            final Instant startTime = Instant.now();
 
             final String className;
             final Map<String, String> initParameters;
@@ -88,7 +89,6 @@ public class LocalDBFactory
             final LocalDB localDB = new LocalDBAdaptor( dbProvider );
 
             initInstance( dbProvider, dbDirectory, initParameters, className, parameters );
-            final TimeDuration openTime = TimeDuration.of( System.currentTimeMillis() - startTime, TimeDuration.Unit.MILLISECONDS );
 
             if ( !readonly )
             {
@@ -121,7 +121,7 @@ public class LocalDBFactory
                     debugText.append( ", " ).append( StringUtil.formatDiskSize( freeSpace ) ).append( " free" );
                 }
             }
-            LOGGER.info( () -> debugText, () -> openTime );
+            LOGGER.info( () -> debugText, () -> TimeDuration.fromCurrent( startTime ) );
 
             return localDB;
         }

+ 3 - 1
server/src/main/java/password/pwm/util/logging/PwmLogger.java

@@ -34,6 +34,7 @@ import password.pwm.svc.event.AuditEvent;
 import password.pwm.svc.event.AuditServiceClient;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 
 import java.io.IOException;
@@ -138,7 +139,8 @@ public class PwmLogger
             try
             {
                 final CharSequence cleanedString = PwmLogger.removeUserDataFromString( pwmRequest.getPwmSession().getLoginInfoBean(), message.get() );
-                cleanedMessage = () -> cleanedString;
+                final CharSequence printableString = StringUtil.cleanNonPrintableCharacters( cleanedString );
+                cleanedMessage = () -> printableString;
             }
             catch ( final PwmUnrecoverableException e1 )
             {

+ 1 - 1
server/src/main/resources/password/pwm/AppProperty.properties

@@ -354,7 +354,7 @@ wordlist.builtin.path=/WEB-INF/wordlist.zip
 wordlist.maxCharLength=64
 wordlist.minCharLength=2
 wordlist.warmup.count=1000
-wordlist.bucketCheckLogWarningTimeoutMs=100
+wordlist.bucketCheckLogWarningTimeoutMs=1000
 wordlist.import.autoImportRecheckSeconds=432000
 wordlist.import.durationGoalMS=200
 wordlist.import.minTransactions=1

+ 8 - 0
server/src/test/java/password/pwm/util/java/StringUtilTest.java

@@ -171,7 +171,15 @@ public class StringUtilTest
         final String linebreaks = StringUtil.insertRepeatedLineBreaks( original, 80 );
         final String stripped = StringUtil.stripAllWhitespace( linebreaks );
         Assert.assertEquals( original, stripped );
+    }
 
+    @Test
+    @SuppressWarnings( "AvoidEscapedUnicodeCharacters" )
+    public void stripNonPrintableCharactersTet()
+    {
+        final String input = "0�\u0000\u0000\u0000\u0007\u0002\u0001\u0001\u0002\u0002�\u007F";
+        final String expected = "0�?????????�?";
+        Assert.assertEquals( expected, StringUtil.cleanNonPrintableCharacters( input ) );
     }
 
     @Test