Merge remote-tracking branch 'origin/master' into orgchart-enhancements

client/.gitignore

@@ -10,3 +10,5 @@
 
 # Generated log files
 *.log
+
+/package-lock.json

docker/pom.xml

@@ -25,7 +25,7 @@
             <plugin>
                 <groupId>com.google.cloud.tools</groupId>
                 <artifactId>jib-maven-plugin</artifactId>
-                <version>0.9.10</version>
+                <version>0.9.13</version>
                 <executions>
                     <execution>
                         <id>make-docker-image</id>
@@ -34,24 +34,23 @@
                             <goal>buildTar</goal>
                         </goals>
                         <configuration>
+                            <from>
+                                <image>adoptopenjdk/openjdk11</image>
+                            </from>
                             <to>
                                 <image>${dockerImageTag}</image>
                             </to>
                             <container>
-                                <jvmFlags>
-                                    <jvmFlag>-Xms1g</jvmFlag>
-                                    <jvmFlag>-Xmx1g</jvmFlag>
-                                </jvmFlags>
-                                <mainClass>password.pwm.onejar.OnejarMain</mainClass>
-                                <args>
+                                <entrypoint>
+                                    <arg>java</arg>
+                                    <arg>-jar</arg>
+                                    <arg>/app/libs/pwm-onejar-${project.version}.jar</arg>
                                     <arg>-applicationPath</arg>
                                     <arg>/config</arg>
-                                </args>
+                                </entrypoint>
                                 <format>docker</format>
                                 <ports>8443</ports>
                             </container>
-                            <!--<useCurrentTimestamp>true</useCurrentTimestamp>-->
-                            <allowInsecureRegistries>true</allowInsecureRegistries>
                         </configuration>
                     </execution>
                 </executions>

onejar/src/main/java/password/pwm/onejar/TomcatOnejarRunner.java

@@ -22,7 +22,6 @@
 
 package password.pwm.onejar;
 
-import org.apache.catalina.LifecycleException;
 import org.apache.catalina.connector.Connector;
 import org.apache.catalina.startup.Tomcat;
 import org.apache.catalina.util.ServerInfo;
@@ -142,16 +141,13 @@ public class TomcatOnejarRunner
         final String warPath = onejarConfig.getWarFolder().getAbsolutePath();
         tomcat.addWebapp( "/" + onejarConfig.getContext(), warPath );
 
-
         try
         {
-            tomcat.start();
-
             tomcat.setConnector( makeConnector( onejarConfig ) );
-
+            tomcat.start();
             out( "tomcat started in " + Duration.between( Instant.now(), startTime ).toString() );
         }
-        catch ( LifecycleException e )
+        catch ( Exception e )
         {
             throw new OnejarException( "unable to start tomcat: " + e.getMessage() );
         }
@@ -185,6 +181,7 @@ public class TomcatOnejarRunner
 
 
     private Connector makeConnector( final OnejarConfig onejarConfig )
+            throws Exception
     {
         final Connector connector = new Connector( "HTTP/1.1" );
         connector.setPort( onejarConfig.getPort() );
@@ -201,10 +198,20 @@ public class TomcatOnejarRunner
         connector.setAttribute( "keyAlias", OnejarMain.KEYSTORE_ALIAS );
         connector.setAttribute( "clientAuth", "false" );
 
+        final Properties tlsProperties = readConfiguredTlsProperties( onejarConfig );
+        if ( tlsProperties != null )
+        {
+            for ( final String key : tlsProperties.stringPropertyNames() )
+            {
+                final String value = tlsProperties.getProperty( key );
+                connector.setAttribute( key, value );
+            }
+        }
+
         return connector;
     }
 
-     static String getVersion( ) throws OnejarException
+    static String getVersion( ) throws OnejarException
     {
         try
         {
@@ -250,31 +257,21 @@ public class TomcatOnejarRunner
     void generatePwmKeystore( final OnejarConfig onejarConfig )
             throws IOException, ClassNotFoundException, IllegalAccessException, NoSuchMethodException, InvocationTargetException
     {
-        final File warPath = onejarConfig.getWarFolder();
-        final String keystoreFile = onejarConfig.getKeystoreFile().getAbsolutePath();
-        final File webInfPath = new File( warPath.getAbsolutePath() + File.separator + "WEB-INF" + File.separator + "lib" );
-        final File[] jarFiles = webInfPath.listFiles();
-        final List<URL> jarURLList = new ArrayList<>();
-        if ( jarFiles != null )
+        try ( URLClassLoader classLoader = warClassLoaderFromConfig( onejarConfig ) )
         {
-            for ( final File jarFile : jarFiles )
-            {
-                jarURLList.add( jarFile.toURI().toURL() );
-            }
+            final Class pwmMainClass = classLoader.loadClass( "password.pwm.util.cli.MainClass" );
+            final String keystoreFile = onejarConfig.getKeystoreFile().getAbsolutePath();
+            final Method mainMethod = pwmMainClass.getMethod( "main", String[].class );
+            final String[] arguments = new String[] {
+                    "-applicationPath=" + onejarConfig.getApplicationPath().getAbsolutePath(),
+                    "ExportHttpsKeyStore",
+                    keystoreFile,
+                    OnejarMain.KEYSTORE_ALIAS,
+                    onejarConfig.getKeystorePass(),
+            };
+
+            mainMethod.invoke( null, ( Object ) arguments );
         }
-        final URLClassLoader classLoader = URLClassLoader.newInstance( jarURLList.toArray( new URL[ jarURLList.size() ] ) );
-        final Class pwmMainClass = classLoader.loadClass( "password.pwm.util.cli.MainClass" );
-        final Method mainMethod = pwmMainClass.getMethod( "main", String[].class );
-        final String[] arguments = new String[] {
-                "-applicationPath=" + onejarConfig.getApplicationPath().getAbsolutePath(),
-                "ExportHttpsKeyStore",
-                keystoreFile,
-                OnejarMain.KEYSTORE_ALIAS,
-                onejarConfig.getKeystorePass(),
-        };
-
-        mainMethod.invoke( null, ( Object ) arguments );
-        classLoader.close();
     }
 
     void setupEnv( final OnejarConfig onejarConfig )
@@ -315,4 +312,37 @@ public class TomcatOnejarRunner
             }
         }
     }
+
+    Properties readConfiguredTlsProperties( final OnejarConfig onejarConfig )
+            throws Exception
+    {
+        out( "beginning read of tlsProperties " );
+        try ( URLClassLoader classLoader = warClassLoaderFromConfig( onejarConfig ) )
+        {
+            final Class pwmMainClass = classLoader.loadClass( "password.pwm.util.cli.commands.ExportHttpsTomcatConfigCommand" );
+            final Method readMethod = pwmMainClass.getMethod( "readAsProperties", String.class );
+            final String arguments = onejarConfig.getApplicationPath().getAbsolutePath();
+            final Object returnObjValue = readMethod.invoke( null, ( Object ) arguments );
+            final Properties returnProps = ( Properties ) returnObjValue;
+            out( "completed read of tlsProperties " );
+            return returnProps;
+        }
+    }
+
+    URLClassLoader warClassLoaderFromConfig( final OnejarConfig onejarConfig )
+            throws IOException
+    {
+        final File warPath = onejarConfig.getWarFolder();
+        final File webInfPath = new File( warPath.getAbsolutePath() + File.separator + "WEB-INF" + File.separator + "lib" );
+        final File[] jarFiles = webInfPath.listFiles();
+        final List<URL> jarURLList = new ArrayList<>();
+        if ( jarFiles != null )
+        {
+            for ( final File jarFile : jarFiles )
+            {
+                jarURLList.add( jarFile.toURI().toURL() );
+            }
+        }
+        return URLClassLoader.newInstance( jarURLList.toArray( new URL[ jarURLList.size() ] ) );
+    }
 }

server/src/main/java/password/pwm/AppProperty.java

@@ -345,6 +345,11 @@ public enum AppProperty
     WORDLIST_BUILTIN_PATH                           ( "wordlist.builtin.path" ),
     WORDLIST_CHAR_LENGTH_MAX                        ( "wordlist.maxCharLength" ),
     WORDLIST_CHAR_LENGTH_MIN                        ( "wordlist.minCharLength" ),
+    WORDLIST_IMPORT_AUTO_IMPORT_RECHECK_SECONDS     ( "wordlist.import.autoImportRecheckSeconds" ),
+    WORDLIST_IMPORT_DURATION_GOAL_MS                ( "wordlist.import.durationGoalMS" ),
+    WORDLIST_IMPORT_MIN_TRANSACTIONS                ( "wordlist.import.minTransactions" ),
+    WORDLIST_IMPORT_MAX_TRANSACTIONS                ( "wordlist.import.maxTransactions" ),
+    WORDLIST_INSPECTOR_FREQUENCY_SECONDS            ( "wordlist.inspector.frequencySeconds" ),
     WS_REST_CLIENT_PWRULE_HALTONERROR               ( "ws.restClient.pwRule.haltOnError" ),
     WS_REST_SERVER_SIGNING_FORM_TIMEOUT_SECONDS     ( "ws.restServer.signing.form.timeoutSeconds" ),
     WS_REST_SERVER_STATISTICS_DEFAULT_HISTORY       ( "ws.restServer.statistics.defaultHistoryDays" ),

server/src/main/java/password/pwm/PwmAboutProperty.java

@@ -55,9 +55,9 @@ public enum PwmAboutProperty
     app_mode_manageHttps( null, pwmApplication -> Boolean.toString( pwmApplication.getPwmEnvironment().getFlags().contains( PwmEnvironment.ApplicationFlag.ManageHttps ) ) ),
     app_applicationPath( null, pwmApplication -> pwmApplication.getPwmEnvironment().getApplicationPath().getAbsolutePath() ),
     app_environmentFlags( null, pwmApplication -> StringUtil.collectionToString( pwmApplication.getPwmEnvironment().getFlags() ) ),
-    app_wordlistSize( null, pwmApplication -> Integer.toString( pwmApplication.getWordlistManager().size() ) ),
-    app_seedlistSize( null, pwmApplication -> Integer.toString( pwmApplication.getSeedlistManager().size() ) ),
-    app_sharedHistorySize( null, pwmApplication -> Integer.toString( pwmApplication.getSharedHistoryManager().size() ) ),
+    app_wordlistSize( null, pwmApplication -> Long.toString( pwmApplication.getWordlistManager().size() ) ),
+    app_seedlistSize( null, pwmApplication -> Long.toString( pwmApplication.getSeedlistManager().size() ) ),
+    app_sharedHistorySize( null, pwmApplication -> Long.toString( pwmApplication.getSharedHistoryManager().size() ) ),
     app_sharedHistoryOldestTime( null, pwmApplication -> format( pwmApplication.getSharedHistoryManager().getOldestEntryTime() ) ),
     app_emailQueueSize( null, pwmApplication -> Integer.toString( pwmApplication.getEmailQueue().queueSize() ) ),
     app_emailQueueOldestTime( null, pwmApplication -> format( Date.from( pwmApplication.getEmailQueue().eldestItem() ) ) ),

server/src/main/java/password/pwm/PwmApplication.java

@@ -59,9 +59,9 @@ import password.pwm.svc.shorturl.UrlShortenerService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.svc.token.TokenService;
-import password.pwm.svc.wordlist.SeedlistManager;
+import password.pwm.svc.wordlist.SeedlistService;
 import password.pwm.svc.wordlist.SharedHistoryManager;
-import password.pwm.svc.wordlist.WordlistManager;
+import password.pwm.svc.wordlist.WordlistService;
 import password.pwm.util.MBeanUtility;
 import password.pwm.util.PasswordData;
 import password.pwm.util.cli.commands.ExportHttpsTomcatConfigCommand;
@@ -99,6 +99,10 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 /**
@@ -162,6 +166,8 @@ public class PwmApplication
 
     private final PwmServiceManager pwmServiceManager = new PwmServiceManager( this );
 
+    private ScheduledExecutorService applicationExecutorService;
+
     public PwmApplication( final PwmEnvironment pwmEnvironment )
             throws PwmUnrecoverableException
     {
@@ -278,6 +284,8 @@ public class PwmApplication
         LOGGER.debug( "application environment flags: " + JsonUtil.serializeCollection( pwmEnvironment.getFlags() ) );
         LOGGER.debug( "application environment parameters: " + JsonUtil.serializeMap( pwmEnvironment.getParameters() ) );
 
+        applicationExecutorService = JavaHelper.makeSingleThreadExecutorService( this, this.getClass() );
+
         pwmServiceManager.initAllServices();
 
         final boolean skipPostInit = pwmEnvironment.isInternalRuntimeInstance()
@@ -290,10 +298,7 @@ public class PwmApplication
             StatisticsManager.incrementStat( this, Statistic.PWM_STARTUPS );
             LOGGER.debug( "buildTime=" + PwmConstants.BUILD_TIME + ", javaLocale=" + Locale.getDefault() + ", DefaultLocale=" + PwmConstants.DEFAULT_LOCALE );
 
-            final Thread postInitThread = new Thread( ( ) -> postInitTasks() );
-            postInitThread.setDaemon( true );
-            postInitThread.setName( JavaHelper.makeThreadName( this, PwmApplication.class ) );
-            postInitThread.start();
+            applicationExecutorService.execute( () -> postInitTasks() );
         }
     }
 
@@ -552,14 +557,14 @@ public class PwmApplication
         return Collections.unmodifiableList( pwmServices );
     }
 
-    public WordlistManager getWordlistManager( )
+    public WordlistService getWordlistManager( )
     {
-        return ( WordlistManager ) pwmServiceManager.getService( WordlistManager.class );
+        return ( WordlistService ) pwmServiceManager.getService( WordlistService.class );
     }
 
-    public SeedlistManager getSeedlistManager( )
+    public SeedlistService getSeedlistManager( )
     {
-        return ( SeedlistManager ) pwmServiceManager.getService( SeedlistManager.class );
+        return ( SeedlistService ) pwmServiceManager.getService( SeedlistService.class );
     }
 
     public ReportService getReportService( )
@@ -778,6 +783,8 @@ public class PwmApplication
 
     public void shutdown( )
     {
+        applicationExecutorService.shutdown();
+
         LOGGER.warn( "shutting down" );
         {
             // send system audit event
@@ -1014,6 +1021,64 @@ public class PwmApplication
     {
         return inprogressRequests;
     }
+
+    public ScheduledFuture scheduleFutureJob(
+            final Runnable runnable,
+            final Executor executor,
+            final TimeDuration delay
+    )
+    {
+        return applicationExecutorService.schedule(  new WrappedRunner( runnable, executor ), delay.asMillis(), TimeUnit.MILLISECONDS );
+    }
+
+    public void scheduleFixedRateJob(
+            final Runnable runnable,
+            final Executor executor,
+            final TimeDuration initialDelay,
+            final TimeDuration frequency
+    )
+    {
+        if ( initialDelay != null )
+        {
+            applicationExecutorService.schedule( new WrappedRunner( runnable, executor ), initialDelay.asMillis(), TimeUnit.MILLISECONDS );
+        }
+
+        final Runnable jobWithNextScheduler = () ->
+        {
+            new WrappedRunner( runnable, executor ).run();
+            if ( !applicationExecutorService.isShutdown() )
+            {
+                scheduleFixedRateJob( runnable, executor, null, frequency );
+            }
+        };
+
+        applicationExecutorService.schedule(  jobWithNextScheduler, frequency.asMillis(), TimeUnit.MILLISECONDS );
+    }
+
+    private static class WrappedRunner implements Runnable
+    {
+        private final Runnable runnable;
+        private final Executor executor;
+
+        WrappedRunner( final Runnable runnable, final Executor executor )
+        {
+            this.runnable = runnable;
+            this.executor = executor;
+        }
+
+        @Override
+        public void run()
+        {
+            try
+            {
+                executor.execute( runnable );
+            }
+            catch ( Throwable t )
+            {
+                LOGGER.error( "unexpected error running scheduled job: " + t.getMessage(), t );
+            }
+        }
+    }
 }
 
 

server/src/main/java/password/pwm/config/option/TLSVersion.java

@@ -28,7 +28,8 @@ public enum TLSVersion
     SSL_3_0( "SSLv3" ),
     TLS_1_0( "TLSv1" ),
     TLS_1_1( "TLSv1.1" ),
-    TLS_1_2( "TLSv1.2" ),;
+    TLS_1_2( "TLSv1.2" ),
+    TLS_1_3( "TLSv1.3" ),;
 
     private final String tomcatValueName;
 

server/src/main/java/password/pwm/error/PwmError.java

@@ -305,6 +305,8 @@ public enum PwmError
             5092, "Error_FileTooLarge", null ),
     ERROR_CLUSTER_SERVICE_ERROR(
             5093, "Error_ClusterServiceError", null ),
+    ERROR_WORDLIST_IMPORT_ERROR(
+            5094, "Error_WordlistImportError", null ),
 
     ERROR_REMOTE_ERROR_VALUE(
             6000, "Error_RemoteErrorValue", null, ErrorFlag.Permanent ),

server/src/main/java/password/pwm/health/HealthMonitor.java

@@ -40,10 +40,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
 
 public class HealthMonitor implements PwmService
 {
@@ -67,7 +65,7 @@ public class HealthMonitor implements PwmService
         HEALTH_CHECKERS = records;
     }
 
-    private ScheduledExecutorService executorService;
+    private ExecutorService executorService;
     private HealthMonitorSettings settings;
 
     private volatile Instant lastHealthCheckTime = Instant.ofEpochMilli( 0 );
@@ -152,14 +150,8 @@ public class HealthMonitor implements PwmService
         }
 
 
-        executorService = Executors.newSingleThreadScheduledExecutor(
-                JavaHelper.makePwmThreadFactory(
-                        JavaHelper.makeThreadName( pwmApplication, this.getClass() ) + "-",
-                        true
-                ) );
-
-
-        executorService.scheduleAtFixedRate( new ScheduledUpdater(), 0, settings.getNominalCheckInterval().asMillis(), TimeUnit.MILLISECONDS );
+        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
+        pwmApplication.scheduleFixedRateJob( new ScheduledUpdater(), executorService, TimeDuration.SECONDS_10, settings.getNominalCheckInterval() );
 
         status = STATUS.OPEN;
     }
@@ -177,7 +169,7 @@ public class HealthMonitor implements PwmService
             final boolean recordsAreStale = TimeDuration.fromCurrent( lastHealthCheckTime ).isLongerThan( settings.getMaximumRecordAge() );
             if ( timeliness == CheckTimeliness.Immediate || ( timeliness == CheckTimeliness.CurrentButNotAncient && recordsAreStale ) )
             {
-                final ScheduledFuture updateTask = executorService.schedule( new ImmediateUpdater(), 0, TimeUnit.NANOSECONDS );
+                final ScheduledFuture updateTask = pwmApplication.scheduleFutureJob( new ImmediateUpdater(), executorService, TimeDuration.ZERO );
                 final Instant beginWaitTime = Instant.now();
                 while ( !updateTask.isDone() && TimeDuration.fromCurrent( beginWaitTime ).isShorterThan( settings.getMaximumForceCheckWait() ) )
                 {

server/src/main/java/password/pwm/http/servlet/admin/AppDashboardData.java

@@ -288,12 +288,14 @@ public class AppDashboardData implements Serializable
                 "Word List Dictionary Size",
                 numberFormat.format( pwmApplication.getWordlistManager().size() )
         ) );
+
         localDbInfo.add( new DisplayElement(
                 "seedlistSize",
                 DisplayElement.Type.number,
                 "Seed List Dictionary Size",
                 numberFormat.format( pwmApplication.getSeedlistManager().size() )
         ) );
+
         localDbInfo.add( new DisplayElement(
                 "sharedHistorySize",
                 DisplayElement.Type.number,

server/src/main/java/password/pwm/http/servlet/admin/ReportStatusBean.java

@@ -130,7 +130,7 @@ public class ReportStatusBean implements Serializable
             {
                 presentableMap.add( new DisplayElement( "lastError", DisplayElement.Type.string, "Last Error", reportInfo.getLastError().toDebugStr() ) );
             }
-            final int totalRecords = reportService.getTotalRecords();
+            final long totalRecords = reportService.getTotalRecords();
             presentableMap.add( new DisplayElement( "recordsInCache", DisplayElement.Type.string, "Records in Cache", numberFormat.format( totalRecords ) ) );
             if ( totalRecords > 0 )
             {

server/src/main/java/password/pwm/http/servlet/configmanager/ConfigManagerWordlistServlet.java

@@ -36,12 +36,16 @@ import password.pwm.http.JspUrl;
 import password.pwm.http.PwmRequest;
 import password.pwm.http.bean.DisplayElement;
 import password.pwm.http.servlet.AbstractPwmServlet;
+import password.pwm.i18n.Display;
 import password.pwm.i18n.Message;
-import password.pwm.svc.wordlist.StoredWordlistDataBean;
 import password.pwm.svc.wordlist.Wordlist;
 import password.pwm.svc.wordlist.WordlistConfiguration;
+import password.pwm.svc.wordlist.WordlistSourceType;
+import password.pwm.svc.wordlist.WordlistStatus;
 import password.pwm.svc.wordlist.WordlistType;
+import password.pwm.util.LocaleHelper;
 import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.StringUtil;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.ws.server.RestResultBean;
 
@@ -206,7 +210,8 @@ public class ConfigManagerWordlistServlet extends AbstractPwmServlet
         for ( final WordlistType wordlistType : WordlistType.values() )
         {
             final Wordlist wordlist = wordlistType.forType( pwmRequest.getPwmApplication() );
-            final StoredWordlistDataBean storedWordlistDataBean = wordlist.readMetadata();
+            final WordlistStatus wordlistStatus = wordlist.readWordlistStatus();
+            final Wordlist.Activity activity = wordlist.getActivity();
             final WordlistConfiguration wordlistConfiguration = wordlistType.forType( pwmRequest.getPwmApplication() ).getConfiguration();
 
             final WordlistDataBean.WordlistDataBeanBuilder builder = WordlistDataBean.builder();
@@ -215,20 +220,20 @@ public class ConfigManagerWordlistServlet extends AbstractPwmServlet
                 presentableValues.add( new DisplayElement(
                         wordlistType.name() + "_populationStatus",
                         DisplayElement.Type.string,
-                        "Population Status",
-                        storedWordlistDataBean.isCompleted() ? "Completed" : "In-Progress" ) );
+                        "Import Status",
+                        activity.getLabel() ) );
                 presentableValues.add( new DisplayElement(
                         wordlistType.name() + "_listSource",
                         DisplayElement.Type.string, "List Source",
-                        storedWordlistDataBean.getSource() == null
-                                ? StoredWordlistDataBean.Source.BuiltIn.getLabel()
-                                : storedWordlistDataBean.getSource().getLabel() ) );
+                        wordlistStatus.getSourceType() == null
+                                ? LocaleHelper.getLocalizedMessage( Display.Value_NotApplicable, pwmRequest )
+                                : wordlistStatus.getSourceType().getLabel() ) );
                 if ( wordlistConfiguration.getAutoImportUrl() != null )
                 {
                     presentableValues.add( new DisplayElement(
                             wordlistType.name() + "_sourceURL",
                             DisplayElement.Type.string,
-                            "Configured Source URL",
+                            "Configured SourceType URL",
                             wordlistConfiguration.getAutoImportUrl() ) );
                 }
 
@@ -236,24 +241,27 @@ public class ConfigManagerWordlistServlet extends AbstractPwmServlet
                         wordlistType.name() + "_wordCount",
                         DisplayElement.Type.number,
                         "Word Count",
-                        Integer.toString( storedWordlistDataBean.getSize() ) ) );
+                        Long.toString( wordlist.size() ) ) );
 
-                if ( storedWordlistDataBean.isCompleted() )
+                if ( wordlistStatus.isCompleted() )
                 {
 
-                    if ( StoredWordlistDataBean.Source.BuiltIn != storedWordlistDataBean.getSource() )
+                    if ( WordlistSourceType.BuiltIn != wordlistStatus.getSourceType() )
                     {
                         presentableValues.add( new DisplayElement(
                                 wordlistType.name() + "_populationTimestamp",
-                                DisplayElement.Type.string,
+                                DisplayElement.Type.timestamp,
                                 "Population Timestamp",
-                                JavaHelper.toIsoDate( storedWordlistDataBean.getStoreDate() ) ) );
+                                JavaHelper.toIsoDate( wordlistStatus.getStoreDate() ) ) );
+                    }
+                    if ( wordlistStatus.getRemoteInfo() != null && !StringUtil.isEmpty( wordlistStatus.getRemoteInfo().getChecksum() ) )
+                    {
+                        presentableValues.add( new DisplayElement(
+                                wordlistType.name() + "_sha256Hash",
+                                DisplayElement.Type.string,
+                                "SHA-256 Checksum Hash",
+                                StringUtil.truncate( wordlistStatus.getRemoteInfo().getChecksum(), 32 ) + "..." ) );
                     }
-                    presentableValues.add( new DisplayElement(
-                            wordlistType.name() + "_sha1Hash",
-                            DisplayElement.Type.string,
-                            "SHA1 Checksum Hash",
-                            storedWordlistDataBean.getRemoteInfo().getChecksum() ) );
                 }
                 if ( wordlist.getAutoImportError() != null )
                 {
@@ -264,20 +272,20 @@ public class ConfigManagerWordlistServlet extends AbstractPwmServlet
                             wordlist.getAutoImportError().getDetailedErrorMsg() ) );
                     presentableValues.add( new DisplayElement(
                             wordlistType.name() + "_lastImportAttempt",
-                            DisplayElement.Type.string,
+                            DisplayElement.Type.timestamp,
                             "Last Import Attempt",
                             JavaHelper.toIsoDate( wordlist.getAutoImportError().getDate() ) ) );
                 }
                 builder.presentableData( Collections.unmodifiableList( presentableValues ) );
             }
 
-            if ( storedWordlistDataBean.isCompleted() )
+            if ( wordlistStatus.isCompleted() )
             {
                 if ( wordlistConfiguration.getAutoImportUrl() == null )
                 {
                     builder.allowUpload( true );
                 }
-                if ( wordlistConfiguration.getAutoImportUrl() != null || storedWordlistDataBean.getSource() != StoredWordlistDataBean.Source.BuiltIn )
+                if ( wordlistConfiguration.getAutoImportUrl() != null || wordlistStatus.getSourceType() != WordlistSourceType.BuiltIn )
                 {
                     builder.allowClear( true );
                 }

server/src/main/java/password/pwm/svc/PwmServiceEnum.java

@@ -24,6 +24,8 @@ package password.pwm.svc;
 
 import password.pwm.svc.email.EmailService;
 import password.pwm.svc.pwnotify.PwNotifyService;
+import password.pwm.svc.wordlist.SeedlistService;
+import password.pwm.svc.wordlist.WordlistService;
 import password.pwm.util.java.JavaHelper;
 
 import java.util.ArrayList;
@@ -39,8 +41,8 @@ public enum PwmServiceEnum
     SharedHistoryManager( password.pwm.svc.wordlist.SharedHistoryManager.class ),
     AuditService( password.pwm.svc.event.AuditService.class ),
     StatisticsManager( password.pwm.svc.stats.StatisticsManager.class, Flag.StartDuringRuntimeInstance ),
-    WordlistManager( password.pwm.svc.wordlist.WordlistManager.class ),
-    SeedlistManager( password.pwm.svc.wordlist.SeedlistManager.class ),
+    WordlistManager( WordlistService.class ),
+    SeedlistManager( SeedlistService.class ),
     EmailQueueManager( EmailService.class ),
     SmsQueueManager( password.pwm.util.queue.SmsQueueManager.class ),
     UrlShortenerService( password.pwm.svc.shorturl.UrlShortenerService.class ),

server/src/main/java/password/pwm/svc/cluster/ClusterMachine.java

@@ -38,15 +38,14 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutorService;
 
 class ClusterMachine
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( ClusterMachine.class );
 
     private final PwmApplication pwmApplication;
-    private final ScheduledExecutorService executorService;
+    private final ExecutorService executorService;
     private final ClusterDataServiceProvider clusterDataServiceProvider;
 
     private ErrorInformation lastError;
@@ -66,16 +65,9 @@ class ClusterMachine
         this.clusterDataServiceProvider = clusterDataServiceProvider;
         this.settings = clusterSettings;
 
-        this.executorService = JavaHelper.makeSingleThreadExecutorService( pwmApplication, ClusterMachine.class );
+        this.executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, ClusterMachine.class );
 
-        final long intervalSeconds = settings.getHeartbeatInterval().as( TimeDuration.Unit.SECONDS );
-
-        this.executorService.scheduleAtFixedRate(
-                new HeartbeatProcess(),
-                intervalSeconds,
-                intervalSeconds,
-                TimeUnit.SECONDS
-        );
+        pwmApplication.scheduleFixedRateJob( new HeartbeatProcess(), executorService, settings.getHeartbeatInterval(), settings.getHeartbeatInterval() );
     }
 
     public void close( )

server/src/main/java/password/pwm/svc/event/LocalDbAuditVault.java

@@ -39,9 +39,7 @@ import password.pwm.util.logging.PwmLogger;
 import java.time.Instant;
 import java.util.Iterator;
 import java.util.Map;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutorService;
 
 public class LocalDbAuditVault implements AuditVault
 {
@@ -53,7 +51,7 @@ public class LocalDbAuditVault implements AuditVault
 
     private int maxBulkRemovals = 105;
 
-    private ScheduledExecutorService executorService;
+    private ExecutorService executorService;
     private volatile PwmService.STATUS status = PwmService.STATUS.NEW;
 
 
@@ -76,14 +74,11 @@ public class LocalDbAuditVault implements AuditVault
 
         readOldestRecord();
 
-        executorService = Executors.newSingleThreadScheduledExecutor(
-                JavaHelper.makePwmThreadFactory(
-                        JavaHelper.makeThreadName( pwmApplication, this.getClass() ) + "-",
-                        true
-                ) );
+        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
 
         status = PwmService.STATUS.OPEN;
-        executorService.scheduleWithFixedDelay( new TrimmerThread(), 0, 10, TimeUnit.MINUTES );
+        final TimeDuration jobFrequency = TimeDuration.of( 10, TimeDuration.Unit.MINUTES );
+        pwmApplication.scheduleFixedRateJob( new TrimmerThread(), executorService, TimeDuration.SECONDS_10, jobFrequency );
     }
 
     public void close( )
@@ -226,11 +221,11 @@ public class LocalDbAuditVault implements AuditVault
 
         // keep transaction duration around 100ms if possible.
         final TransactionSizeCalculator transactionSizeCalculator = new TransactionSizeCalculator(
-                new TransactionSizeCalculator.SettingsBuilder()
-                        .setDurationGoal( TimeDuration.of( 101, TimeDuration.Unit.MILLISECONDS ) )
-                        .setMaxTransactions( 5003 )
-                        .setMinTransactions( 3 )
-                        .createSettings()
+                TransactionSizeCalculator.Settings.builder()
+                        .durationGoal( TimeDuration.of( 101, TimeDuration.Unit.MILLISECONDS ) )
+                        .maxTransactions( 5003 )
+                        .minTransactions( 3 )
+                        .build()
         );
 
         @Override

server/src/main/java/password/pwm/svc/report/ReportService.java

@@ -64,9 +64,7 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Queue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -81,7 +79,7 @@ public class ReportService implements PwmService
     private boolean cancelFlag = false;
     private ReportStatusInfo reportStatus = new ReportStatusInfo( "" );
     private ReportSummaryData summaryData = ReportSummaryData.newSummaryData( null );
-    private ScheduledExecutorService executorService;
+    private ExecutorService executorService;
 
     private UserCacheService userCacheService;
     private ReportSettings settings = new ReportSettings();
@@ -146,12 +144,7 @@ public class ReportService implements PwmService
 
         dnQueue = LocalDBStoredQueue.createLocalDBStoredQueue( pwmApplication, pwmApplication.getLocalDB(), LocalDB.DB.REPORT_QUEUE );
 
-        executorService = Executors.newSingleThreadScheduledExecutor(
-                JavaHelper.makePwmThreadFactory(
-                        JavaHelper.makeThreadName( pwmApplication, this.getClass() ) + "-",
-                        true
-                ) );
-
+        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
 
         final String startupMsg = "report service started";
         LOGGER.debug( startupMsg );
@@ -249,7 +242,7 @@ public class ReportService implements PwmService
         return eventRateMeter.readEventRate();
     }
 
-    public int getTotalRecords( )
+    public long getTotalRecords( )
     {
         return userCacheService.size();
     }
@@ -420,7 +413,7 @@ public class ReportService implements PwmService
                         if ( executorService != null )
                         {
                             LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, "directory unavailable error during background SearchLDAP, will retry; error: " + e.getMessage() );
-                            executorService.schedule( new ReadLDAPTask(), 10, TimeUnit.MINUTES );
+                            pwmApplication.scheduleFutureJob( new ReadLDAPTask(), executorService, TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
                         }
                     }
                     else
@@ -455,11 +448,11 @@ public class ReportService implements PwmService
             LOGGER.trace( "completed ldap search process, transferring search results to work queue" );
 
             final TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(
-                    new TransactionSizeCalculator.SettingsBuilder()
-                            .setDurationGoal( TimeDuration.SECOND )
-                            .setMinTransactions( 10 )
-                            .setMaxTransactions( 100 * 1000 )
-                            .createSettings()
+                    TransactionSizeCalculator.Settings.builder()
+                            .durationGoal( TimeDuration.SECOND )
+                            .minTransactions( 10 )
+                            .maxTransactions( 100 * 1000 )
+                            .build()
             );
 
             while ( status == STATUS.OPEN && !cancelFlag && memQueue.hasNext() )
@@ -497,7 +490,7 @@ public class ReportService implements PwmService
                         if ( executorService != null )
                         {
                             LOGGER.error( SessionLabel.REPORTING_SESSION_LABEL, "directory unavailable error during background ReadData, will retry; error: " + e.getMessage() );
-                            executorService.schedule( new ProcessWorkQueueTask(), 10, TimeUnit.MINUTES );
+                            pwmApplication.scheduleFutureJob( new ProcessWorkQueueTask(), executorService, TimeDuration.of( 10, TimeDuration.Unit.MINUTES ) );
                         }
                     }
                     else
@@ -675,7 +668,7 @@ public class ReportService implements PwmService
 
             try ( ClosableIterator<UserCacheRecord> iterator = iterator() )
             {
-                final int totalRecords = userCacheService.size();
+                final long totalRecords = userCacheService.size();
                 LOGGER.debug( SessionLabel.REPORTING_SESSION_LABEL, "beginning cache review process of " + totalRecords + " records" );
                 Instant lastLogOutputTime = Instant.now();
 
@@ -739,7 +732,8 @@ public class ReportService implements PwmService
             {
                 final Instant nextZuluZeroTime = JavaHelper.nextZuluZeroTime();
                 final long secondsUntilNextDredge = settings.getJobOffsetSeconds() + TimeDuration.fromCurrent( nextZuluZeroTime ).as( TimeDuration.Unit.SECONDS );
-                executorService.scheduleAtFixedRate( new DailyJobExecuteTask(), secondsUntilNextDredge, TimeDuration.DAY.as( TimeDuration.Unit.SECONDS ), TimeUnit.SECONDS );
+                final TimeDuration initialDelay = TimeDuration.of( secondsUntilNextDredge, TimeDuration.Unit.SECONDS );
+                pwmApplication.scheduleFixedRateJob( new ProcessWorkQueueTask(), executorService, initialDelay, TimeDuration.DAY );
                 LOGGER.debug( "scheduled daily execution, next task will be at " + nextZuluZeroTime.toString() );
             }
             executorService.submit( new RolloverTask() );

server/src/main/java/password/pwm/svc/report/UserCacheService.java

@@ -181,7 +181,7 @@ public class UserCacheService implements PwmService
         return new ServiceInfoBean( Collections.singletonList( DataStorageMethod.LOCALDB ) );
     }
 
-    public int size( )
+    public long size( )
     {
         return cacheStore.size();
     }
@@ -275,7 +275,7 @@ public class UserCacheService implements PwmService
             localDB.truncate( DB );
         }
 
-        private int size( )
+        private long size( )
         {
             try
             {

server/src/main/java/password/pwm/svc/stats/StatisticsManager.java

@@ -55,8 +55,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.TimeZone;
 import java.util.TimerTask;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutorService;
 
 public class StatisticsManager implements PwmService
 {
@@ -82,7 +81,7 @@ public class StatisticsManager implements PwmService
     private DailyKey currentDailyKey = new DailyKey( new Date() );
     private DailyKey initialDailyKey = new DailyKey( new Date() );
 
-    private ScheduledExecutorService executorService;
+    private ExecutorService executorService;
 
     private final StatisticsBundle statsCurrent = new StatisticsBundle();
     private StatisticsBundle statsDaily = new StatisticsBundle();
@@ -331,10 +330,10 @@ public class StatisticsManager implements PwmService
 
         {
             // setup a timer to roll over at 0 Zula and one to write current stats every 10 seconds
-            executorService = JavaHelper.makeSingleThreadExecutorService( pwmApplication, this.getClass() );
-            executorService.scheduleAtFixedRate( new FlushTask(), 10 * 1000, DB_WRITE_FREQUENCY.asMillis(), TimeUnit.MICROSECONDS );
+            executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
+            pwmApplication.scheduleFixedRateJob( new FlushTask(), executorService, DB_WRITE_FREQUENCY, DB_WRITE_FREQUENCY );
             final TimeDuration delayTillNextZulu = TimeDuration.fromCurrent( JavaHelper.nextZuluZeroTime() );
-            executorService.scheduleAtFixedRate( new NightlyTask(), delayTillNextZulu.asMillis(), TimeUnit.DAYS.toMillis( 1 ), TimeUnit.MILLISECONDS );
+            pwmApplication.scheduleFixedRateJob( new NightlyTask(), executorService, delayTillNextZulu, TimeDuration.DAY );
         }
 
         status = STATUS.OPEN;

server/src/main/java/password/pwm/svc/telemetry/TelemetryService.java

@@ -66,14 +66,13 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutorService;
 
 public class TelemetryService implements PwmService
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( TelemetryService.class );
 
-    private ScheduledExecutorService executorService;
+    private ExecutorService executorService;
     private PwmApplication pwmApplication;
     private Settings settings;
 
@@ -144,7 +143,7 @@ public class TelemetryService implements PwmService
             LOGGER.trace( SessionLabel.TELEMETRY_SESSION_LABEL, "last publish time was " + JavaHelper.toIsoDate( lastPublishTime ) );
         }
 
-        executorService = JavaHelper.makeSingleThreadExecutorService( pwmApplication, TelemetryService.class );
+        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, TelemetryService.class );
 
         scheduleNextJob();
 
@@ -215,10 +214,7 @@ public class TelemetryService implements PwmService
     private void scheduleNextJob( )
     {
         final TimeDuration durationUntilNextPublish = durationUntilNextPublish();
-        executorService.schedule(
-                new PublishJob(),
-                durationUntilNextPublish.asMillis(),
-                TimeUnit.MILLISECONDS );
+        pwmApplication.scheduleFutureJob( new PublishJob(), executorService, durationUntilNextPublish );
         LOGGER.trace( SessionLabel.TELEMETRY_SESSION_LABEL, "next publish time: " + durationUntilNextPublish().asCompactString() );
     }
 

server/src/main/java/password/pwm/svc/token/CryptoTokenMachine.java

@@ -70,7 +70,7 @@ class CryptoTokenMachine implements TokenMachine
     {
     }
 
-    public int size( ) throws PwmOperationalException, PwmUnrecoverableException
+    public long size( ) throws PwmOperationalException, PwmUnrecoverableException
     {
         return 0;
     }

server/src/main/java/password/pwm/svc/token/DataStoreTokenMachine.java

@@ -179,7 +179,7 @@ public class DataStoreTokenMachine implements TokenMachine
         dataStore.remove( storedHash );
     }
 
-    public int size( ) throws PwmOperationalException, PwmUnrecoverableException
+    public long size( ) throws PwmOperationalException, PwmUnrecoverableException
     {
         return dataStore.size();
     }

server/src/main/java/password/pwm/svc/token/LdapTokenMachine.java

@@ -167,7 +167,7 @@ class LdapTokenMachine implements TokenMachine
         }
     }
 
-    public int size( ) throws PwmOperationalException
+    public long size( ) throws PwmOperationalException
     {
         return -1;
     }

server/src/main/java/password/pwm/svc/token/TokenMachine.java

@@ -40,7 +40,7 @@ interface TokenMachine
     void removeToken( TokenKey tokenKey )
             throws PwmOperationalException, PwmUnrecoverableException;
 
-    int size( )
+    long size( )
             throws PwmOperationalException, PwmUnrecoverableException;
 
     void cleanup( )

server/src/main/java/password/pwm/svc/token/TokenService.java

@@ -76,9 +76,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.TimerTask;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutorService;
 
 /**
  * This PWM service is responsible for reading/writing tokens used for forgotten password,
@@ -92,7 +90,7 @@ public class TokenService implements PwmService
 
     private static final PwmLogger LOGGER = PwmLogger.forClass( TokenService.class );
 
-    private ScheduledExecutorService executorService;
+    private ExecutorService executorService;
 
     private PwmApplication pwmApplication;
     private Configuration configuration;
@@ -194,22 +192,17 @@ public class TokenService implements PwmService
             return;
         }
 
-        executorService = Executors.newSingleThreadScheduledExecutor(
-                JavaHelper.makePwmThreadFactory(
-                        JavaHelper.makeThreadName( pwmApplication, this.getClass() ) + "-",
-                        true
-                ) );
+        verifyPwModifyTime = Boolean.parseBoolean( configuration.readAppProperty( AppProperty.TOKEN_VERIFY_PW_MODIFY_TIME ) );
 
-        final TimerTask cleanerTask = new CleanerTask();
+        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
 
         {
             final int cleanerFrequencySeconds = Integer.parseInt( configuration.readAppProperty( AppProperty.TOKEN_CLEANER_INTERVAL_SECONDS ) );
             final TimeDuration cleanerFrequency = TimeDuration.of( cleanerFrequencySeconds, TimeDuration.Unit.SECONDS );
-            executorService.scheduleAtFixedRate( cleanerTask, 10, cleanerFrequencySeconds, TimeUnit.SECONDS );
+            pwmApplication.scheduleFixedRateJob( new CleanerTask(), executorService, TimeDuration.MINUTE, cleanerFrequency );
             LOGGER.trace( "token cleanup will occur every " + cleanerFrequency.asCompactString() );
         }
 
-        verifyPwModifyTime = Boolean.parseBoolean( configuration.readAppProperty( AppProperty.TOKEN_VERIFY_PW_MODIFY_TIME ) );
 
         status = STATUS.OPEN;
         LOGGER.debug( "open" );
@@ -424,7 +417,7 @@ public class TokenService implements PwmService
         }
     }
 
-    public int size( ) throws PwmUnrecoverableException
+    public long size( )
     {
         if ( status != STATUS.OPEN )
         {

server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java

@@ -22,12 +22,9 @@
 
 package password.pwm.svc.wordlist;
 
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.output.NullOutputStream;
-import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
+import password.pwm.PwmApplicationMode;
 import password.pwm.PwmConstants;
-import password.pwm.config.PwmSetting;
 import password.pwm.config.option.DataStorageMethod;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
@@ -37,253 +34,149 @@ import password.pwm.health.HealthMessage;
 import password.pwm.health.HealthRecord;
 import password.pwm.health.HealthStatus;
 import password.pwm.health.HealthTopic;
-import password.pwm.http.ContextManager;
-import password.pwm.http.client.PwmHttpClient;
-import password.pwm.http.client.PwmHttpClientConfiguration;
 import password.pwm.svc.PwmService;
 import password.pwm.util.java.JavaHelper;
-import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.TimeDuration;
-import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBException;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.secure.ChecksumInputStream;
 import password.pwm.util.secure.PwmHashAlgorithm;
-import password.pwm.util.secure.X509Utils;
 
-import java.io.FileNotFoundException;
-import java.io.IOException;
 import java.io.InputStream;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BooleanSupplier;
 
 abstract class AbstractWordlist implements Wordlist, PwmService
 {
+    static final PwmHashAlgorithm CHECKSUM_HASH_ALG = PwmHashAlgorithm.SHA256;
+    static final TimeDuration DEBUG_OUTPUT_FREQUENCY = TimeDuration.MINUTE;
 
-    static final PwmHashAlgorithm CHECKSUM_HASH_ALG = PwmHashAlgorithm.SHA1;
+    private WordlistConfiguration wordlistConfiguration;
+    private WordlistBucket wordklistBucket;
+    private ExecutorService executorService;
 
-    protected WordlistConfiguration wordlistConfiguration;
+    private volatile STATUS wlStatus = STATUS.NEW;
 
-    protected volatile STATUS wlStatus = STATUS.NEW;
-    protected LocalDB localDB;
-
-    protected PwmLogger logger = PwmLogger.forClass( AbstractWordlist.class );
-    protected String debugLabel = "Generic Word List";
-
-    protected boolean debugTrace;
-
-    private ErrorInformation lastError;
-    private ErrorInformation autoImportError;
+    private volatile ErrorInformation lastError;
+    private volatile ErrorInformation autoImportError;
 
     private PwmApplication pwmApplication;
-    protected Populator populator;
-
-    private ScheduledExecutorService executorService;
-    private PopulationManager populationManager = new PopulationManager();
+    private final AtomicBoolean inhibitBackgroundImportFlag = new AtomicBoolean( false );
+    private final AtomicBoolean backgroundImportRunning = new AtomicBoolean( false );
 
+    private volatile Activity activity = Wordlist.Activity.Idle;
 
-    protected AbstractWordlist( )
+    AbstractWordlist( )
     {
     }
 
-    public void init( final PwmApplication pwmApplication ) throws PwmException
+    public void init(
+            final PwmApplication pwmApplication,
+            final WordlistType type
+    )
+            throws PwmException
     {
         this.pwmApplication = pwmApplication;
-        this.localDB = pwmApplication.getLocalDB();
-        if ( pwmApplication.getConfig().isDevDebugMode() )
-        {
-            debugTrace = true;
-        }
-
-        executorService = Executors.newSingleThreadScheduledExecutor(
-                JavaHelper.makePwmThreadFactory(
-                        JavaHelper.makeThreadName( pwmApplication, this.getClass() ) + "-",
-                        true
-                ) );
-    }
-
-    @Override
-    public WordlistConfiguration getConfiguration( )
-    {
-        return wordlistConfiguration;
-    }
-
-    @Override
-    public ErrorInformation getAutoImportError( )
-    {
-        return autoImportError;
-    }
-
-    protected final void backgroundStartup( )
-    {
-        executorService.schedule( ( ) ->
-        {
-            try
-            {
-                startup();
-            }
-            catch ( Exception e )
-            {
-                logger.warn( "error during startup: " + e.getMessage() );
-            }
-        }, 0, TimeUnit.MILLISECONDS );
-    }
-
+        this.wordlistConfiguration = WordlistConfiguration.fromConfiguration( pwmApplication.getConfig(), type );
 
-    protected final void startup( )
-    {
-        wlStatus = STATUS.OPENING;
-
-        if ( localDB == null )
+        if ( pwmApplication.getApplicationMode() != PwmApplicationMode.RUNNING
+                || pwmApplication.getLocalDB() == null
+        )
         {
-            final String errorMsg = "LocalDB is not available, " + debugLabel + " will remain closed";
-            logger.warn( errorMsg );
-            lastError = new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, errorMsg );
-            close();
+            wlStatus = STATUS.CLOSED;
             return;
         }
 
-        try
+        this.wordklistBucket = new WordlistBucket( pwmApplication, wordlistConfiguration, type );
+
+        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
+
+        if ( pwmApplication.getLocalDB() != null )
         {
-            populationManager.checkPopulation();
+            wlStatus = STATUS.OPEN;
         }
-        catch ( Exception e )
+        else
         {
-            final String errorMsg = "unexpected error while examining wordlist db: " + e.getMessage();
-            if ( ( e instanceof PwmUnrecoverableException ) || ( e instanceof NullPointerException ) || ( e instanceof LocalDBException ) )
-            {
-                logger.warn( errorMsg );
-            }
-            else
-            {
-                logger.warn( errorMsg, e );
-            }
+            wlStatus = STATUS.CLOSED;
+            final String errorMsg = "LocalDB is not available, will remain closed";
+            getLogger().warn( errorMsg );
             lastError = new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, errorMsg );
-            populator = null;
-            close();
-            return;
         }
 
-        wlStatus = STATUS.OPEN;
+        pwmApplication.scheduleFixedRateJob( new InspectorJob(), executorService, TimeDuration.SECOND, wordlistConfiguration.getInspectorFrequency() );
     }
 
-    String normalizeWord( final String input )
+    boolean containsWord( final String word ) throws PwmUnrecoverableException
     {
-        if ( input == null )
+        try
         {
-            return null;
+            return wordklistBucket.containsWord( word );
         }
-
-        String word = input.trim();
-
-        if ( word.length() < wordlistConfiguration.getMinSize() )
+        catch ( LocalDBException e )
         {
-            return null;
+            throw PwmUnrecoverableException.newException( PwmError.ERROR_LOCALDB_UNAVAILABLE, e.getMessage() );
         }
+    }
 
-        if ( word.length() > wordlistConfiguration.getMaxSize() )
-        {
-            word = word.substring( 0, wordlistConfiguration.getMaxSize() );
-        }
+    String randomSeed() throws PwmUnrecoverableException
+    {
+        return getWordlistBucket().randomSeed();
+    }
 
-        if ( !wordlistConfiguration.isCaseSensitive() )
-        {
-            word = word.toLowerCase();
-        }
+    @Override
+    public WordlistConfiguration getConfiguration( )
+    {
+        return wordlistConfiguration;
+    }
 
-        return word.length() > 0 ? word : null;
+    @Override
+    public ErrorInformation getAutoImportError( )
+    {
+        return autoImportError;
     }
 
-    public boolean containsWord( final String word )
+    public long size( )
     {
         if ( wlStatus != STATUS.OPEN )
         {
-            return false;
-        }
-
-        final String testWord = normalizeWord( word );
-
-        if ( testWord == null || testWord.length() < 1 )
-        {
-            return false;
-        }
-
-
-        final Set<String> testWords = chunkWord( testWord, this.wordlistConfiguration.getCheckSize() );
-
-        final Instant startTime = Instant.now();
-        try
-        {
-            boolean result = false;
-            for ( final String t : testWords )
-            {
-                if ( !result )
-                {
-                    // stop checking once found
-                    if ( localDB.contains( getWordlistDB(), t ) )
-                    {
-                        result = true;
-                    }
-                }
-            }
-            final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
-            if ( timeDuration.isLongerThan( 100 ) )
-            {
-                logger.debug( "wordlist search time for " + testWords.size() + " wordlist permutations was greater then 100ms: " + timeDuration.asCompactString() );
-            }
-            return result;
+            return 0;
         }
-        catch ( Exception e )
-        {
-            logger.error( "database error checking for word: " + e.getMessage() );
-        }
-
-        return false;
-    }
 
-    public int size( )
-    {
         try
         {
-            if ( populator != null && wlStatus != STATUS.CLOSED )
-            {
-                return localDB.size( this.getWordlistDB() );
-            }
+            return wordklistBucket.size();
         }
         catch ( LocalDBException e )
         {
-            logger.debug( "error reading wordlist size: " + e.getMessage() );
+            getLogger().error( "error reading size: " + e.getMessage() );
         }
 
-        return readMetadata().getSize();
+        return -1;
     }
 
     public synchronized void close( )
     {
+        final TimeDuration closeWaitTime = TimeDuration.of( 5, TimeDuration.Unit.SECONDS );
+
         wlStatus = STATUS.CLOSED;
-        if ( populator != null )
+        inhibitBackgroundImportFlag.set( true );
+        if ( executorService != null )
         {
-            try
-            {
-                populator.cancel();
-                populator = null;
-            }
-            catch ( PwmUnrecoverableException e )
+            executorService.shutdown();
+
+            if ( backgroundImportRunning.get() )
             {
-                logger.error( "wordlist populator failed to exit" );
+                JavaHelper.pause( closeWaitTime.asMillis(), 50, o -> !backgroundImportRunning.get() );
+                if ( backgroundImportRunning.get() )
+                {
+                    getLogger().warn( "background thread still running after waiting " + closeWaitTime.asCompactString() );
+                }
             }
         }
-
-        executorService.shutdown();
-        localDB = null;
     }
 
     public STATUS status( )
@@ -291,28 +184,6 @@ abstract class AbstractWordlist implements Wordlist, PwmService
         return wlStatus;
     }
 
-    public String getDebugStatus( )
-    {
-        if ( wlStatus == STATUS.OPENING && populator != null )
-        {
-            return populator.makeStatString();
-        }
-        else
-        {
-            return wlStatus.toString();
-        }
-    }
-
-    protected abstract Map<String, String> getWriteTxnForValue( String value );
-
-    protected abstract PwmApplication.AppAttribute getMetaDataAppAttribute( );
-
-    protected abstract AppProperty getBuiltInWordlistLocationProperty( );
-
-    protected abstract LocalDB.DB getWordlistDB( );
-
-    protected abstract PwmSetting getWordlistFileSetting( );
-
     public List<HealthRecord> healthCheck( )
     {
         final List<HealthRecord> returnList = new ArrayList<>();
@@ -320,22 +191,16 @@ abstract class AbstractWordlist implements Wordlist, PwmService
         if ( autoImportError != null )
         {
             final HealthRecord healthRecord = HealthRecord.forMessage( HealthMessage.Wordlist_AutoImportFailure,
-                    this.getWordlistFileSetting().toMenuLocationDebug( null, PwmConstants.DEFAULT_LOCALE ),
+                    wordlistConfiguration.getWordlistFilenameSetting().toMenuLocationDebug( null, PwmConstants.DEFAULT_LOCALE ),
                     autoImportError.getDetailedErrorMsg(),
                     JavaHelper.toIsoDate( autoImportError.getDate() )
             );
             returnList.add( healthRecord );
         }
 
-        if ( wlStatus == STATUS.OPENING )
-        {
-            final HealthRecord healthRecord = new HealthRecord( HealthStatus.CAUTION, HealthTopic.Application, this.debugLabel + " is not yet open: " + this.getDebugStatus() );
-            returnList.add( healthRecord );
-        }
-
         if ( lastError != null )
         {
-            final HealthRecord healthRecord = new HealthRecord( HealthStatus.WARN, HealthTopic.Application, this.debugLabel + " error: " + lastError.toDebugStr() );
+            final HealthRecord healthRecord = new HealthRecord( HealthStatus.WARN, HealthTopic.Application, this.getClass().getName() + " error: " + lastError.toDebugStr() );
             returnList.add( healthRecord );
         }
         return Collections.unmodifiableList( returnList );
@@ -353,354 +218,162 @@ abstract class AbstractWordlist implements Wordlist, PwmService
         }
     }
 
-    protected Set<String> chunkWord( final String input, final int size )
+    public WordlistStatus readWordlistStatus( )
     {
-        int checkSize = size == 0 || size > input.length() ? input.length() : size;
-        final TreeSet<String> testWords = new TreeSet<>();
-        while ( checkSize <= input.length() )
+        if ( wlStatus == STATUS.CLOSED )
         {
-            for ( int i = 0; i + checkSize <= input.length(); i++ )
-            {
-                final String loopWord = input.substring( i, i + checkSize );
-                testWords.add( loopWord );
-            }
-            checkSize++;
+            return WordlistStatus.builder().build();
         }
 
-        return testWords;
-    }
-
-    protected String readAutoImportUrl( )
-    {
-        final String inputUrl = pwmApplication.getConfig().readSettingAsString( getWordlistFileSetting() );
-
-        if ( inputUrl == null || inputUrl.isEmpty() )
-        {
-            return null;
-        }
-
-        if ( !inputUrl.startsWith( "http:" ) && !inputUrl.startsWith( "https:" ) && !inputUrl.startsWith( "file:" ) )
-        {
-            logger.debug( "assuming configured auto-import url is a file url; derived url is " + inputUrl );
-            return "file:" + inputUrl;
-        }
-
-        return inputUrl;
-    }
-
-    public StoredWordlistDataBean readMetadata( )
-    {
-        final StoredWordlistDataBean storedValue = pwmApplication.readAppAttribute( getMetaDataAppAttribute(), StoredWordlistDataBean.class );
+        final PwmApplication.AppAttribute appAttribute = wordlistConfiguration.getMetaDataAppAttribute();
+        final WordlistStatus storedValue = pwmApplication.readAppAttribute( appAttribute, WordlistStatus.class );
         if ( storedValue != null )
         {
             return storedValue;
         }
-        return StoredWordlistDataBean.builder().build();
+        return WordlistStatus.builder().build();
     }
 
-    void writeMetadata( final StoredWordlistDataBean metadataBean )
+    void writeWordlistStatus( final WordlistStatus metadataBean )
     {
-        pwmApplication.writeAppAttribute( getMetaDataAppAttribute(), metadataBean );
-        logger.trace( "updated stored state: " + JsonUtil.serialize( metadataBean ) );
+        final PwmApplication.AppAttribute appAttribute = wordlistConfiguration.getMetaDataAppAttribute();
+        pwmApplication.writeAppAttribute( appAttribute, metadataBean );
+        //getLogger().trace( "updated stored state: " + JsonUtil.serialize( metadataBean ) );
     }
 
     @Override
-    public void populate( final InputStream inputStream )
-            throws IOException, PwmUnrecoverableException
+    public void clear( ) throws PwmUnrecoverableException
     {
-        try
+        if ( wlStatus != STATUS.OPEN )
         {
-            populationManager.populateImpl( null, inputStream, StoredWordlistDataBean.Source.User );
+            throw new PwmUnrecoverableException( PwmError.ERROR_SERVICE_NOT_AVAILABLE.toInfo() );
         }
-        finally
+
+        cancelBackgroundAndRunImmediate( () ->
         {
-            if ( !readMetadata().isCompleted() )
+            try
             {
-                logger.debug( "beginning population using builtin source in background thread" );
-                final Thread t = new Thread( ( ) ->
-                {
-                    try
-                    {
-                        populationManager.populateBuiltIn();
-                    }
-                    catch ( Exception e )
-                    {
-                        logger.warn( "unexpected error during builtin source population process: " + e.getMessage(), e );
-                    }
-                    populator = null;
-                }, JavaHelper.makeThreadName( pwmApplication, WordlistManager.class ) );
-                t.setDaemon( true );
-                t.start();
+                clearImpl( Activity.Idle );
+                executorService.execute( new InspectorJob() );
             }
-        }
-    }
-
-    @Override
-    public void clear( ) throws IOException, PwmUnrecoverableException
-    {
-        if ( wlStatus == STATUS.OPEN )
-        {
-            executorService.schedule( ( ) ->
+            catch ( LocalDBException e )
             {
-                try
-                {
-                    writeMetadata( StoredWordlistDataBean.builder().build() );
-                    populationManager.checkPopulation();
-                }
-                catch ( Exception e )
-                {
-                    logger.error( "error during clear operation: " + e.getMessage() );
-                }
-            }, 0, TimeUnit.MILLISECONDS );
-        }
+                throw new PwmUnrecoverableException( e.getErrorInformation() );
+            }
+        } );
     }
 
-    private class PopulationManager
+    void clearImpl( final Activity postCleanActivity ) throws LocalDBException
     {
-        protected void checkPopulation( )
-                throws Exception
-        {
-            final boolean autoImportUrlConfigured = wordlistConfiguration.getAutoImportUrl() != null;
-
-            if ( autoImportUrlConfigured )
-            {
-                final StoredWordlistDataBean.RemoteWordlistInfo remoteInfo = readRemoteWordlistInfo( autoImportInputStream() );
-                final StoredWordlistDataBean.RemoteWordlistInfo localInfo = readMetadata().getRemoteInfo();
-                if ( remoteInfo != null )
-                {
-                    if ( !remoteInfo.equals( localInfo ) )
-                    {
-                        logger.debug( "auto-import url remote hash does not equal currently stored hash, will start auto-import" );
-                        populateAutoImport( remoteInfo );
-                    }
-                    else if ( remoteInfo.getBytes() > readMetadata().getBytes() )
-                    {
-                        logger.debug( "auto-import did not previously complete, will continue previous import" );
-                        populateAutoImport( remoteInfo );
-                    }
-                }
-
-                if ( autoImportError != null )
-                {
-                    final int retrySeconds = Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.APPLICATION_WORDLIST_RETRY_SECONDS ) );
-                    logger.error( "auto-import of remote wordlist failed, will retry in " + ( TimeDuration.of( retrySeconds, TimeDuration.Unit.SECONDS ).asCompactString() ) );
-                    executorService.schedule( ( ) ->
-                    {
-                        try
-                        {
-                            logger.debug( "attempting wordlist remote import" );
-                            checkPopulation();
-                        }
-                        catch ( Exception e )
-                        {
-                            logger.error( "error during auto-import retry: " + e.getMessage() );
-                        }
-
-                    }, retrySeconds, TimeUnit.SECONDS );
-                }
-            }
-            else
-            {
-                if ( readMetadata().getSource() == StoredWordlistDataBean.Source.AutoImport )
-                {
-                    logger.trace( "source list is from auto-import, but not currently configured for auto-import; clearing stored data" );
+        final Instant startTime = Instant.now();
+        getLogger().trace( "clearing stored wordlist" );
+        activity = Wordlist.Activity.Clearing;
+        writeWordlistStatus( WordlistStatus.builder().build() );
+        getWordlistBucket().clear();
+        getLogger().debug( "cleared stored wordlist (" + TimeDuration.compactFromCurrent( startTime ) + ")" );
+        setActivity( postCleanActivity );
+    }
 
-                    // clear previous auto-import wl
-                    writeMetadata( StoredWordlistDataBean.builder().build() );
-                }
-            }
 
-            if ( wlStatus != STATUS.CLOSED )
-            {
-                boolean needsBuiltinPopulating = false;
-                if ( !readMetadata().isCompleted() )
-                {
-                    needsBuiltinPopulating = true;
-                    logger.debug( "wordlist stored in database does not have a completed load status, will load built-in wordlist" );
-                }
-                else if ( StoredWordlistDataBean.Source.BuiltIn == readMetadata().getSource() )
-                {
-                    final StoredWordlistDataBean.RemoteWordlistInfo builtInWordlistHash = getBuiltInWordlistHash();
-                    if ( !builtInWordlistHash.equals( readMetadata().getRemoteInfo() ) )
-                    {
-                        logger.debug( "wordlist stored in database does not have match checksum with built-in wordlist file, will load built-in wordlist" );
-                        needsBuiltinPopulating = true;
-                    }
-                }
+    void setAutoImportError( final ErrorInformation autoImportError )
+    {
+        this.autoImportError = autoImportError;
+    }
 
-                if ( !needsBuiltinPopulating )
-                {
-                    return;
-                }
+    abstract PwmLogger getLogger();
 
-                this.populateBuiltIn();
-            }
-        }
+    WordlistBucket getWordlistBucket()
+    {
+        return wordklistBucket;
+    }
 
-        protected void populateBuiltIn( )
-                throws IOException, PwmUnrecoverableException
+    @Override
+    public void populate( final InputStream inputStream ) throws PwmUnrecoverableException
+    {
+        if ( wlStatus != STATUS.OPEN )
         {
-            final StoredWordlistDataBean.RemoteWordlistInfo remoteWordlistInfo = readRemoteWordlistInfo( getBuiltInWordlist() );
-            populateImpl( remoteWordlistInfo, getBuiltInWordlist(), StoredWordlistDataBean.Source.BuiltIn );
+            throw new PwmUnrecoverableException( PwmError.ERROR_SERVICE_NOT_AVAILABLE.toInfo() );
         }
 
-        private void populateImpl(
-                final StoredWordlistDataBean.RemoteWordlistInfo remoteWordlistInfo,
-                final InputStream inputStream,
-                final StoredWordlistDataBean.Source source
-        )
-                throws IOException, PwmUnrecoverableException
+        cancelBackgroundAndRunImmediate( () ->
         {
-            if ( inputStream == null )
-            {
-                throw new NullPointerException( "input stream can not be null for populateImpl()" );
-            }
-
-            if ( wlStatus == STATUS.CLOSED )
-            {
-                return;
-            }
-
-            wlStatus = STATUS.OPENING;
+            setActivity( Activity.Importing );
+            getLogger().debug( "beginning direct user-supplied wordlist import" );
+            setAutoImportError( null );
+            final WordlistZipReader wordlistZipReader = new WordlistZipReader( inputStream );
+            final WordlistImporter wordlistImporter = new WordlistImporter(
+                    null,
+                    wordlistZipReader,
+                    WordlistSourceType.User,
+                    AbstractWordlist.this,
+                    () -> wlStatus != STATUS.OPEN
+            );
+            wordlistImporter.run();
+            getLogger().debug( "completed direct user-supplied wordlist import" );
+        } );
 
-            try
-            {
-                if ( populator != null )
-                {
-                    populator.cancel();
-
-                    final int maxWaitMs = 1000 * 30;
-                    final Instant startWaitTime = Instant.now();
-                    while ( populator.isRunning() && TimeDuration.fromCurrent( startWaitTime ).isShorterThan( maxWaitMs ) )
-                    {
-                        JavaHelper.pause( 1000 );
-                    }
-                    if ( populator.isRunning() && TimeDuration.fromCurrent( startWaitTime ).isShorterThan( maxWaitMs ) )
-                    {
-                        throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_INTERNAL, "unable to abort populator" ) );
-                    }
-                }
+        setActivity( Activity.Idle );
+        executorService.execute( new InspectorJob() );
+    }
 
-                if ( remoteWordlistInfo == null || !remoteWordlistInfo.equals( readMetadata().getRemoteInfo() ) )
-                {
-                    // reset the wordlist metadata
-                    final StoredWordlistDataBean storedWordlistDataBean = StoredWordlistDataBean.builder()
-                            .source( source )
-                            .remoteInfo( remoteWordlistInfo )
-                            .build();
-                    writeMetadata( storedWordlistDataBean );
-                }
+    private interface PwmCallable
+    {
+        void call() throws PwmUnrecoverableException;
+    }
 
-                populator = new Populator( remoteWordlistInfo, inputStream, source, AbstractWordlist.this, pwmApplication );
-                populator.populate();
-            }
-            catch ( Exception e )
-            {
-                final ErrorInformation populationError;
-                populationError = e instanceof PwmException
-                        ? ( ( PwmException ) e ).getErrorInformation()
-                        : new ErrorInformation( PwmError.ERROR_INTERNAL, e.getMessage() );
-                logger.error( "error during wordlist population: " + populationError.toDebugStr() );
-                throw new PwmUnrecoverableException( populationError );
-            }
-            finally
+    private void cancelBackgroundAndRunImmediate( final PwmCallable runnable ) throws PwmUnrecoverableException
+    {
+        inhibitBackgroundImportFlag.set( true );
+        try
+        {
+            JavaHelper.pause( 10_000, 100, o -> !backgroundImportRunning.get() );
+            if ( backgroundImportRunning.get() )
             {
-                populator = null;
-                IOUtils.closeQuietly( inputStream );
+                throw PwmUnrecoverableException.newException( PwmError.ERROR_WORDLIST_IMPORT_ERROR, "unable to cancel background operation in progress" );
             }
 
-            wlStatus = STATUS.OPEN;
+            runnable.call();
         }
-
-        protected InputStream getBuiltInWordlist( ) throws FileNotFoundException, PwmUnrecoverableException
+        finally
         {
-            final ContextManager contextManager = pwmApplication.getPwmEnvironment().getContextManager();
-            if ( contextManager != null )
-            {
-                final String wordlistFilename = pwmApplication.getConfig().readAppProperty( getBuiltInWordlistLocationProperty() );
-                return contextManager.getResourceAsStream( wordlistFilename );
-            }
-            throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, "unable to locate builtin wordlist file" ) );
+            inhibitBackgroundImportFlag.set( false );
         }
 
-        protected StoredWordlistDataBean.RemoteWordlistInfo getBuiltInWordlistHash( ) throws IOException, PwmUnrecoverableException
-        {
-
-            try ( InputStream inputStream = getBuiltInWordlist() )
-            {
-                return readRemoteWordlistInfo( inputStream );
-            }
-        }
+    }
 
-        public boolean populateAutoImport( final StoredWordlistDataBean.RemoteWordlistInfo remoteWordlistInfo )
-                throws IOException, PwmUnrecoverableException
+    class InspectorJob implements Runnable
+    {
+        @Override
+        public void run()
         {
-            autoImportError = null;
-            InputStream inputStream = null;
             try
             {
-                inputStream = autoImportInputStream();
-                populateImpl( remoteWordlistInfo, inputStream, StoredWordlistDataBean.Source.AutoImport );
-                return true;
-            }
-            catch ( Exception e )
-            {
-                final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_INTERNAL, "error during remote wordlist import: " + e.getMessage() );
-                logger.error( errorInformation );
-                autoImportError = errorInformation;
-            }
-            finally
-            {
-                IOUtils.closeQuietly( inputStream );
-            }
-            return false;
-        }
+                if ( !inhibitBackgroundImportFlag.get() )
+                {
+                    activity = Wordlist.Activity.ReadingWordlistFile;
+                    final BooleanSupplier cancelFlag = inhibitBackgroundImportFlag::get;
+                    backgroundImportRunning.set( true );
+                    final WordlistInspector wordlistInspector = new WordlistInspector( pwmApplication, AbstractWordlist.this, cancelFlag );
+                    wordlistInspector.run();
+                    activity = Wordlist.Activity.Idle;
+                }
 
-        StoredWordlistDataBean.RemoteWordlistInfo readRemoteWordlistInfo( final InputStream inputStream )
-        {
-            try
-            {
-                final Instant startTime = Instant.now();
-                logger.debug( "beginning read of auto-import remote url data" );
-
-                final ChecksumInputStream checksumInputStream = new ChecksumInputStream( CHECKSUM_HASH_ALG, inputStream );
-                final long bytes = JavaHelper.copyWhilePredicate( checksumInputStream, new NullOutputStream(), o -> wlStatus != STATUS.CLOSED );
-                final String hash = JavaHelper.binaryArrayToHex( checksumInputStream.closeAndFinalChecksum() );
-
-                final StoredWordlistDataBean.RemoteWordlistInfo remoteWordlistInfo = new StoredWordlistDataBean.RemoteWordlistInfo(
-                        hash,
-                        bytes );
-
-                logger.debug( "completed read of wordlist data: "
-                        + JsonUtil.serialize( remoteWordlistInfo )
-                        + " (" + TimeDuration.fromCurrent( startTime ).asCompactString() + ")" );
-                autoImportError = null;
-                return remoteWordlistInfo;
-            }
-            catch ( Exception e )
-            {
-                final ErrorInformation errorInformation = new ErrorInformation(
-                        PwmError.ERROR_INTERNAL,
-                        "error reading from remote wordlist auto-import url: " + JavaHelper.readHostileExceptionMessage( e )
-                );
-                logger.error( errorInformation );
-                autoImportError = errorInformation;
             }
             finally
             {
-                IOUtils.closeQuietly( inputStream );
+                backgroundImportRunning.set( false );
             }
-            return null;
         }
+    }
 
-        private InputStream autoImportInputStream( ) throws IOException, PwmUnrecoverableException
-        {
-            final boolean promiscuous = Boolean.parseBoolean( pwmApplication.getConfig().readAppProperty( AppProperty.HTTP_CLIENT_PROMISCUOUS_WORDLIST_ENABLE ) );
-            final PwmHttpClientConfiguration pwmHttpClientConfiguration = PwmHttpClientConfiguration.builder()
-                    .trustManager( promiscuous ? new X509Utils.PromiscuousTrustManager() : null )
-                    .build();
-            final PwmHttpClient client = new PwmHttpClient( pwmApplication, null, pwmHttpClientConfiguration );
-            return client.streamForUrl( wordlistConfiguration.getAutoImportUrl() );
-        }
+    @Override
+    public Activity getActivity()
+    {
+        return activity;
+    }
+
+    void setActivity( final Activity activity )
+    {
+        this.activity = activity;
     }
 }

server/src/main/java/password/pwm/svc/wordlist/Populator.java

@@ -1,344 +0,0 @@
-/*
- * Password Management Servlets (PWM)
- * http://www.pwm-project.org
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2018 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.svc.wordlist;
-
-import org.apache.commons.io.IOUtils;
-import password.pwm.PwmApplication;
-import password.pwm.error.ErrorInformation;
-import password.pwm.error.PwmError;
-import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.svc.stats.EventRateMeter;
-import password.pwm.util.TransactionSizeCalculator;
-import password.pwm.util.java.ConditionalTaskExecutor;
-import password.pwm.util.java.JavaHelper;
-import password.pwm.util.java.Percent;
-import password.pwm.util.java.StringUtil;
-import password.pwm.util.java.TimeDuration;
-import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.localdb.LocalDBException;
-import password.pwm.util.logging.PwmLogger;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.text.DecimalFormat;
-import java.text.NumberFormat;
-import java.time.Instant;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * @author Jason D. Rivard
- */
-class Populator
-{
-    private static final PwmLogger LOGGER = PwmLogger.forClass( Populator.class );
-
-    // words truncated to this length, prevents massive words if the input
-    private static final int MAX_LINE_LENGTH = 64;
-
-    private static final TimeDuration DEBUG_OUTPUT_FREQUENCY = TimeDuration.SECONDS_30;
-
-    // words tarting with this prefix are ignored.
-    private static final String COMMENT_PREFIX = "!#comment:";
-
-    private static final NumberFormat PERCENT_FORMAT = DecimalFormat.getPercentInstance();
-
-    private final WordlistZipReader zipFileReader;
-    private final StoredWordlistDataBean.Source source;
-
-    private final AtomicBoolean running = new AtomicBoolean( false );
-    private final AtomicBoolean abortFlag = new AtomicBoolean( false );
-
-    private TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(
-            new TransactionSizeCalculator.SettingsBuilder()
-                    .setDurationGoal( TimeDuration.of( 600, TimeDuration.Unit.MILLISECONDS ) )
-                    .setMinTransactions( 10 )
-                    .setMaxTransactions( 350 * 1000 )
-                    .createSettings()
-    );
-
-    private final Map<String, String> bufferedWords = new TreeMap<>();
-
-    private final LocalDB localDB;
-
-    private final AbstractWordlist rootWordlist;
-
-    private final StoredWordlistDataBean.RemoteWordlistInfo remoteWordlistInfo;
-
-    private final Instant startTime = Instant.now();
-
-    private final AtomicLong bytesSkipped = new AtomicLong( 0 );
-
-    private final EventRateMeter.MovingAverage byteRateMeter
-            = new EventRateMeter.MovingAverage( TimeDuration.of( 5, TimeDuration.Unit.MINUTES ) );
-
-
-    static
-    {
-        PERCENT_FORMAT.setMinimumFractionDigits( 2 );
-    }
-
-    Populator(
-            final StoredWordlistDataBean.RemoteWordlistInfo remoteWordlistInfo,
-            final InputStream inputStream,
-            final StoredWordlistDataBean.Source source,
-            final AbstractWordlist rootWordlist,
-            final PwmApplication pwmApplication
-    )
-            throws Exception
-    {
-        this.remoteWordlistInfo = remoteWordlistInfo;
-        this.source = source;
-        this.zipFileReader = new WordlistZipReader( inputStream );
-        this.localDB = pwmApplication.getLocalDB();
-        this.rootWordlist = rootWordlist;
-    }
-
-    private void init( ) throws IOException, LocalDBException
-    {
-        if ( abortFlag.get() )
-        {
-            return;
-        }
-
-        final long previousBytesRead = rootWordlist.readMetadata().getBytes();
-
-        if ( previousBytesRead == 0 )
-        {
-            LOGGER.debug( "clearing stored wordlist" );
-            localDB.truncate( rootWordlist.getWordlistDB() );
-        }
-
-        if ( previousBytesRead > 0 )
-        {
-            final Instant startSkip = Instant.now();
-            LOGGER.debug( "skipping forward " + previousBytesRead + " bytes in stream that have been previously imported" );
-            while ( !abortFlag.get() && bytesSkipped.get() < previousBytesRead )
-            {
-                zipFileReader.nextLine();
-                bytesSkipped.set( zipFileReader.getByteCount() );
-            }
-            LOGGER.debug( "skipped forward " + previousBytesRead + " bytes in stream (" + TimeDuration.fromCurrent( startSkip ).asCompactString() + ")" );
-        }
-    }
-
-    String makeStatString( )
-    {
-        if ( !running.get() )
-        {
-            return "not running";
-        }
-
-        return rootWordlist.debugLabel + " " + StringUtil.mapToString( makeStatValues() );
-    }
-
-    private Map<String, String> makeStatValues()
-    {
-        final Map<String, String> stats = new LinkedHashMap<>();
-        stats.put( "LinesRead", Long.toString( zipFileReader.getLineCount() ) );
-        stats.put( "BytesRead", Long.toString( zipFileReader.getByteCount() ) );
-        stats.put( "BufferSize", Integer.toString( transactionCalculator.getTransactionSize() ) );
-
-        final long elapsedSeconds = TimeDuration.fromCurrent( startTime ).as( TimeDuration.Unit.SECONDS );
-
-        if ( bytesSkipped.get() > 0 )
-        {
-            stats.put( "BytesSkipped", Long.toString( bytesSkipped.get() ) );
-        }
-
-        if ( elapsedSeconds > 10 )
-        {
-            stats.put( "BytesPerSecond", Double.toString( byteRateMeter.getAverage() * 1000 ) );
-        }
-
-        if ( remoteWordlistInfo != null )
-        {
-            final Percent percent = new Percent( zipFileReader.getByteCount(), remoteWordlistInfo.getBytes() );
-            stats.put( "PercentComplete", percent.pretty( 2 ) );
-        }
-
-        stats.put( "ImportTime", TimeDuration.fromCurrent( startTime ).asCompactString() );
-
-        return Collections.unmodifiableMap( stats );
-    }
-
-    @SuppressWarnings( "checkstyle:InnerAssignment" )
-    void populate( ) throws IOException, LocalDBException, PwmUnrecoverableException
-    {
-        final ConditionalTaskExecutor metaUpdater = new ConditionalTaskExecutor(
-                () -> rootWordlist.writeMetadata( StoredWordlistDataBean.builder()
-                        .source( source )
-                        .size( rootWordlist.size() )
-                        .storeDate( Instant.now() )
-                        .remoteInfo( remoteWordlistInfo )
-                        .bytes( zipFileReader.getByteCount() )
-                        .build() ),
-                new ConditionalTaskExecutor.TimeDurationPredicate( TimeDuration.SECONDS_10 )
-        );
-
-        final ConditionalTaskExecutor debugOutputter = new ConditionalTaskExecutor(
-                () -> LOGGER.debug( makeStatString() ),
-                new ConditionalTaskExecutor.TimeDurationPredicate( DEBUG_OUTPUT_FREQUENCY )
-        );
-
-        try
-        {
-            debugOutputter.conditionallyExecuteTask();
-            running.set( true );
-            init();
-
-            String line;
-
-            long lastBytes = zipFileReader.getByteCount();
-
-            while ( !abortFlag.get() && ( line = zipFileReader.nextLine() ) != null )
-            {
-                addLine( line );
-
-                debugOutputter.conditionallyExecuteTask();
-                final long cycleBytes = zipFileReader.getByteCount() - lastBytes;
-                lastBytes = zipFileReader.getByteCount();
-                byteRateMeter.update( cycleBytes );
-
-                if ( bufferedWords.size() > transactionCalculator.getTransactionSize() )
-                {
-                    flushBuffer();
-                    metaUpdater.conditionallyExecuteTask();
-                }
-
-            }
-
-            if ( abortFlag.get() )
-            {
-                LOGGER.warn( "pausing " + rootWordlist.debugLabel + " population" );
-            }
-            else
-            {
-                populationComplete();
-            }
-        }
-        finally
-        {
-            running.set( false );
-            IOUtils.closeQuietly( zipFileReader );
-        }
-    }
-
-    private void addLine( final String word )
-            throws IOException
-    {
-        // check for word suitability
-        String normalizedWord = rootWordlist.normalizeWord( word );
-
-        if ( normalizedWord == null || normalizedWord.length() < 1 || normalizedWord.startsWith( COMMENT_PREFIX ) )
-        {
-            return;
-        }
-
-        if ( normalizedWord.length() > MAX_LINE_LENGTH )
-        {
-            normalizedWord = normalizedWord.substring( 0, MAX_LINE_LENGTH );
-        }
-
-        final Map<String, String> wordTxn = rootWordlist.getWriteTxnForValue( normalizedWord );
-        bufferedWords.putAll( wordTxn );
-    }
-
-    private void flushBuffer( )
-            throws LocalDBException
-    {
-        final long startTime = System.currentTimeMillis();
-
-        //add the elements
-        localDB.putAll( rootWordlist.getWordlistDB(), bufferedWords );
-
-        if ( abortFlag.get() )
-        {
-            return;
-        }
-
-        //mark how long the buffer close took
-        final long commitTime = System.currentTimeMillis() - startTime;
-        transactionCalculator.recordLastTransactionDuration( commitTime );
-
-        //clear the buffers.
-        bufferedWords.clear();
-    }
-
-    private void populationComplete( )
-            throws LocalDBException, PwmUnrecoverableException, IOException
-    {
-        flushBuffer();
-        LOGGER.info( makeStatString() );
-        LOGGER.trace( "beginning wordlist size query" );
-        final int wordlistSize = localDB.size( rootWordlist.getWordlistDB() );
-        if ( wordlistSize < 1 )
-        {
-            throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_INTERNAL, rootWordlist.debugLabel + " population completed, but no words stored" ) );
-        }
-
-        final StringBuilder sb = new StringBuilder();
-        sb.append( rootWordlist.debugLabel );
-        sb.append( " population complete, added " ).append( wordlistSize );
-        sb.append( " total words in " ).append( TimeDuration.fromCurrent( startTime ).asCompactString() );
-        {
-            final StoredWordlistDataBean storedWordlistDataBean = StoredWordlistDataBean.builder()
-                    .remoteInfo( remoteWordlistInfo )
-                    .size( wordlistSize )
-                    .storeDate( Instant.now() )
-                    .source( source )
-                    .completed( !abortFlag.get() )
-                    .bytes( zipFileReader.getByteCount() )
-                    .build();
-            rootWordlist.writeMetadata( storedWordlistDataBean );
-        }
-        LOGGER.info( sb.toString() );
-    }
-
-    public void cancel( ) throws PwmUnrecoverableException
-    {
-        LOGGER.debug( "cancelling in-progress population" );
-        abortFlag.set( true );
-
-        final int maxWaitMs = 1000 * 30;
-        final Instant startWaitTime = Instant.now();
-        while ( isRunning() && TimeDuration.fromCurrent( startWaitTime ).isShorterThan( maxWaitMs ) )
-        {
-            JavaHelper.pause( 1000 );
-        }
-        if ( isRunning() && TimeDuration.fromCurrent( startWaitTime ).isShorterThan( maxWaitMs ) )
-        {
-            throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_INTERNAL, "unable to abort in progress population" ) );
-        }
-
-    }
-
-    public boolean isRunning( )
-    {
-        return running.get();
-    }
-}

server/src/main/java/password/pwm/svc/wordlist/SeedlistManager.java

@@ -1,124 +0,0 @@
-/*
- * Password Management Servlets (PWM)
- * http://www.pwm-project.org
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2018 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.svc.wordlist;
-
-import password.pwm.AppProperty;
-import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
-import password.pwm.config.PwmSetting;
-import password.pwm.error.PwmException;
-import password.pwm.util.java.TimeDuration;
-import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.secure.PwmRandom;
-
-import java.util.Collections;
-import java.util.Map;
-
-public class SeedlistManager extends AbstractWordlist implements Wordlist
-{
-
-    private int initialPopulationCounter = 0;
-
-    public SeedlistManager( )
-    {
-        logger = PwmLogger.forClass( SeedlistManager.class );
-    }
-
-    public String randomSeed( )
-    {
-        if ( wlStatus != STATUS.OPEN )
-        {
-            return null;
-        }
-        final long startTime = System.currentTimeMillis();
-        String returnValue = null;
-        try
-        {
-            final int seedCount = size();
-            if ( seedCount > 1000 )
-            {
-                final int randomKey = PwmRandom.getInstance().nextInt( size() );
-                final Object obj = localDB.get( getWordlistDB(), String.valueOf( randomKey ) );
-                if ( obj != null )
-                {
-                    returnValue = obj.toString();
-                }
-            }
-        }
-        catch ( Exception e )
-        {
-            logger.warn( "error while generating random word: " + e.getMessage() );
-        }
-
-        if ( debugTrace )
-        {
-            logger.trace( "getRandomSeed fetch time: " + TimeDuration.fromCurrent( startTime ).asCompactString() );
-        }
-        return returnValue;
-    }
-
-    protected Map<String, String> getWriteTxnForValue( final String value )
-    {
-        final Map<String, String> txItem = Collections.singletonMap( String.valueOf( initialPopulationCounter ), value );
-        initialPopulationCounter++;
-        return txItem;
-    }
-
-    public void init( final PwmApplication pwmApplication ) throws PwmException
-    {
-        super.init( pwmApplication );
-        final String seedlistUrl = readAutoImportUrl();
-
-        final int minSize = Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.WORDLIST_CHAR_LENGTH_MIN ) );
-        final int maxSize = Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.WORDLIST_CHAR_LENGTH_MAX ) );
-
-        this.wordlistConfiguration = new WordlistConfiguration( true, 0, seedlistUrl, minSize, maxSize );
-        this.debugLabel = PwmConstants.PWM_APP_NAME + "-Seedist";
-        backgroundStartup();
-    }
-
-    @Override
-    protected PwmApplication.AppAttribute getMetaDataAppAttribute( )
-    {
-        return PwmApplication.AppAttribute.SEEDLIST_METADATA;
-    }
-
-    @Override
-    protected LocalDB.DB getWordlistDB( )
-    {
-        return LocalDB.DB.SEEDLIST_WORDS;
-    }
-
-    @Override
-    protected AppProperty getBuiltInWordlistLocationProperty( )
-    {
-        return AppProperty.SEEDLIST_BUILTIN_PATH;
-    }
-
-    @Override
-    protected PwmSetting getWordlistFileSetting( )
-    {
-        return PwmSetting.SEEDLIST_FILENAME;
-    }
-}

server/src/main/java/password/pwm/svc/wordlist/SeedlistService.java

@@ -0,0 +1,54 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import password.pwm.PwmApplication;
+import password.pwm.error.PwmException;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.logging.PwmLogger;
+
+public class SeedlistService extends AbstractWordlist implements Wordlist
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( SeedlistService.class );
+
+    public SeedlistService()
+    {
+    }
+
+    public void init( final PwmApplication pwmApplication ) throws PwmException
+    {
+        super.init( pwmApplication, WordlistType.SEEDLIST );
+    }
+
+    @Override
+    PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
+    @Override
+    public String randomSeed() throws PwmUnrecoverableException
+    {
+        return super.randomSeed();
+    }
+}

server/src/main/java/password/pwm/svc/wordlist/SharedHistoryManager.java

@@ -45,8 +45,8 @@ import java.security.NoSuchAlgorithmException;
 import java.time.Instant;
 import java.util.Collections;
 import java.util.List;
-import java.util.Timer;
 import java.util.TimerTask;
+import java.util.concurrent.ExecutorService;
 
 
 public class SharedHistoryManager implements PwmService
@@ -68,7 +68,7 @@ public class SharedHistoryManager implements PwmService
 
     private volatile PwmService.STATUS status = STATUS.NEW;
 
-    private volatile Timer cleanerTimer = null;
+    private ExecutorService executorService;
 
     private LocalDB localDB;
     private String salt;
@@ -83,9 +83,9 @@ public class SharedHistoryManager implements PwmService
     public void close( )
     {
         status = STATUS.CLOSED;
-        if ( cleanerTimer != null )
+        if ( executorService != null )
         {
-            cleanerTimer.cancel();
+            executorService.shutdown();
         }
         localDB = null;
     }
@@ -145,7 +145,7 @@ public class SharedHistoryManager implements PwmService
         return null;
     }
 
-    public int size( )
+    public long size( )
     {
         if ( localDB != null )
         {
@@ -229,7 +229,7 @@ public class SharedHistoryManager implements PwmService
 
         try
         {
-            final int size = localDB.size( WORDS_DB );
+            final long size = localDB.size( WORDS_DB );
             final StringBuilder sb = new StringBuilder();
             sb.append( "open with " ).append( size ).append( " words (" );
             sb.append( TimeDuration.compactFromCurrent( startTime ) ).append( ")" );
@@ -251,11 +251,11 @@ public class SharedHistoryManager implements PwmService
         {
             long frequencyMs = maxAgeMs > MAX_CLEANER_FREQUENCY ? MAX_CLEANER_FREQUENCY : maxAgeMs;
             frequencyMs = frequencyMs < MIN_CLEANER_FREQUENCY ? MIN_CLEANER_FREQUENCY : frequencyMs;
+            final TimeDuration frequency = TimeDuration.of( frequencyMs, TimeDuration.Unit.MILLISECONDS );
 
-            LOGGER.debug( "scheduling cleaner task to run once every " + TimeDuration.of( frequencyMs, TimeDuration.Unit.MILLISECONDS ).asCompactString() );
-            final String threadName = JavaHelper.makeThreadName( pwmApplication, this.getClass() ) + " timer";
-            cleanerTimer = new Timer( threadName, true );
-            cleanerTimer.schedule( new CleanerTask(), 1000, frequencyMs );
+            LOGGER.debug( "scheduling cleaner task to run once every " + frequency.asCompactString() );
+            executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
+            pwmApplication.scheduleFixedRateJob( new CleanerTask(), executorService, null, frequency );
         }
     }
 
@@ -372,7 +372,7 @@ public class SharedHistoryManager implements PwmService
             }
 
             final long startTime = System.currentTimeMillis();
-            final int initialSize = size();
+            final long initialSize = size();
             int removeCount = 0;
             long localOldestEntry = System.currentTimeMillis();
 

server/src/main/java/password/pwm/svc/wordlist/Wordlist.java

@@ -33,12 +33,9 @@ import java.io.InputStream;
 public interface Wordlist extends PwmService
 {
 
-    boolean containsWord( String word );
-
-    int size( );
-
-    StoredWordlistDataBean readMetadata( );
+    long size( );
 
+    WordlistStatus readWordlistStatus( );
 
     void populate( InputStream inputStream )
             throws IOException, PwmUnrecoverableException;
@@ -49,4 +46,26 @@ public interface Wordlist extends PwmService
     WordlistConfiguration getConfiguration( );
 
     ErrorInformation getAutoImportError( );
+
+    AbstractWordlist.Activity getActivity();
+
+    enum Activity
+    {
+        Idle( "Idle" ),
+        ReadingWordlistFile( "Reading Wordlist File" ),
+        Clearing( "Clearing Stored Wordlist" ),
+        Importing( "Importing Wordlist" ),;
+
+        private final String label;
+
+        Activity( final String label )
+        {
+            this.label = label;
+        }
+
+        public String getLabel()
+        {
+            return label;
+        }
+    }
 }

server/src/main/java/password/pwm/svc/wordlist/WordlistBucket.java

@@ -0,0 +1,251 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import password.pwm.PwmApplication;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.localdb.LocalDB;
+import password.pwm.util.localdb.LocalDBException;
+import password.pwm.util.logging.PwmLogger;
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicLong;
+
+class WordlistBucket
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( WordlistBucket.class );
+    private static final String KEY_LAST_ISSUED_KEY = "_______lastKey_";
+
+    private final PwmApplication pwmApplication;
+    private final WordlistConfiguration wordlistConfiguration;
+    private final LocalDB.DB db;
+    private final WordlistType type;
+    private final AtomicLong seedlistTopKey = new AtomicLong(  );
+
+
+    WordlistBucket(
+            final PwmApplication pwmApplication,
+            final WordlistConfiguration wordlistConfiguration,
+            final WordlistType type
+    )
+            throws LocalDBException
+    {
+        this.pwmApplication = pwmApplication;
+        this.wordlistConfiguration = wordlistConfiguration;
+        this.db = wordlistConfiguration.getDb();
+        this.type = type;
+
+        final String valueOfLastKey = pwmApplication.getLocalDB().get( db, KEY_LAST_ISSUED_KEY );
+
+        seedlistTopKey.set(
+                StringUtil.isEmpty( valueOfLastKey )
+                        ? 0
+                        : Long.parseLong( valueOfLastKey )
+        );
+    }
+
+    boolean containsWord( final String word ) throws LocalDBException
+    {
+        if ( type == WordlistType.SEEDLIST )
+        {
+            throw new IllegalStateException( "unable to containWord check SEEDLIST wordlist" );
+        }
+
+        final String testWord = normalizeWord( word );
+
+        if ( testWord == null || testWord.length() < 1 )
+        {
+            return false;
+        }
+
+        final Set<String> testWords = chunkWord( testWord, this.wordlistConfiguration.getCheckSize() );
+
+        final Instant startTime = Instant.now();
+        boolean result = false;
+
+        searchLoop:
+        for ( final String t : testWords )
+        {
+            // stop checking once found
+            if ( pwmApplication.getLocalDB().contains( db, t ) )
+            {
+                result = true;
+                break searchLoop;
+            }
+        }
+
+        final TimeDuration timeDuration = TimeDuration.fromCurrent( startTime );
+        if ( timeDuration.isLongerThan( 100 ) )
+        {
+            LOGGER.debug( "wordlist search time for " + testWords.size() + " wordlist permutations was greater then 100ms: " + timeDuration.asCompactString() );
+        }
+
+        return result;
+    }
+
+    String randomSeed( ) throws PwmUnrecoverableException
+    {
+        if ( type == WordlistType.WORDLIST )
+        {
+            throw new IllegalStateException( "unable to read randomSeed from WORDLIST wordlist" );
+        }
+
+        try
+        {
+            final long seedCount = size();
+            if ( seedCount > 1000 )
+            {
+                final long randomKey = pwmApplication.getSecureService().pwmRandom().nextLong( seedCount );
+                return pwmApplication.getLocalDB().get( db, String.valueOf( randomKey ) );
+            }
+        }
+        catch ( Exception e )
+        {
+            throw PwmUnrecoverableException.newException( PwmError.ERROR_INTERNAL, "error while generating random word: " + e.getMessage() );
+        }
+
+        throw new PwmUnrecoverableException( PwmError.ERROR_INTERNAL, "seedlist word not available" );
+    }
+
+    void addWords( final Collection<String> words ) throws LocalDBException
+    {
+        pwmApplication.getLocalDB().putAll( db, getWriteTxnForValue( words ) );
+    }
+
+    long size() throws LocalDBException
+    {
+        return pwmApplication.getLocalDB().size( db );
+    }
+
+
+    void clear() throws LocalDBException
+    {
+        seedlistTopKey.set( 0 );
+        pwmApplication.getLocalDB().truncate( db );
+    }
+
+    private Map<String, String> getWriteTxnForValue( final Collection<String> words ) throws LocalDBException
+    {
+        switch ( type )
+        {
+            case SEEDLIST:
+            {
+                final Map<String, String> returnSet = new TreeMap<>();
+                for ( final String word : words )
+                {
+                    final String normalizedWord = normalizeWord( word );
+                    if ( !StringUtil.isEmpty( normalizedWord ) )
+                    {
+                        returnSet.put( String.valueOf( seedlistTopKey.incrementAndGet() ), normalizedWord );
+                        returnSet.put( KEY_LAST_ISSUED_KEY, String.valueOf( seedlistTopKey.get() ) );
+                    }
+                }
+                return Collections.unmodifiableMap( returnSet );
+            }
+
+            case WORDLIST:
+            {
+                final Map<String, String> returnSet = new TreeMap<>();
+                for ( final String word : words )
+                {
+                    final String normalizedWord = normalizeWord( word );
+                    if ( !StringUtil.isEmpty( normalizedWord ) )
+                    {
+                        returnSet.put( normalizedWord, "" );
+                    }
+                }
+                return returnSet;
+            }
+
+            default:
+                JavaHelper.unhandledSwitchStatement( type );
+        }
+
+        throw new IllegalStateException( "unreachable switch statement" );
+    }
+
+    private String normalizeWord( final String input )
+    {
+        if ( input == null )
+        {
+            return null;
+        }
+
+        String word = input.trim();
+
+        if ( word.length() < wordlistConfiguration.getMinSize() )
+        {
+            return null;
+        }
+
+        if ( word.length() > wordlistConfiguration.getMaxSize() )
+        {
+            word = word.substring( 0, wordlistConfiguration.getMaxSize() );
+        }
+
+        if ( !wordlistConfiguration.isCaseSensitive() )
+        {
+            word = word.toLowerCase();
+        }
+
+        return word.length() > 0 ? word : null;
+    }
+
+    private Set<String> chunkWord( final String input, final int size )
+    {
+        if ( StringUtil.isEmpty( input ) )
+        {
+            return Collections.emptySet();
+        }
+
+        if ( size == 0 )
+        {
+            return Collections.singleton( input );
+        }
+
+        int checkSize = size == 0 || size > input.length() ? input.length() : size;
+        final TreeSet<String> testWords = new TreeSet<>();
+        while ( checkSize <= input.length() )
+        {
+            for ( int i = 0; i + checkSize <= input.length(); i++ )
+            {
+                final String loopWord = input.substring( i, i + checkSize );
+                testWords.add( loopWord );
+            }
+            checkSize++;
+        }
+
+        return testWords;
+    }
+
+}

server/src/main/java/password/pwm/svc/wordlist/WordlistConfiguration.java

@@ -22,13 +22,21 @@
 
 package password.pwm.svc.wordlist;
 
-import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Getter;
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.config.Configuration;
+import password.pwm.config.PwmSetting;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.localdb.LocalDB;
 
 import java.io.Serializable;
 
 @Getter
-@AllArgsConstructor
+@Builder
 public class WordlistConfiguration implements Serializable
 {
     private final boolean caseSensitive;
@@ -36,4 +44,108 @@ public class WordlistConfiguration implements Serializable
     private final String autoImportUrl;
     private final int minSize;
     private final int maxSize;
+    private final PwmApplication.AppAttribute metaDataAppAttribute;
+    private final AppProperty builtInWordlistLocationProperty;
+    private final LocalDB.DB db;
+    private final PwmSetting wordlistFilenameSetting;
+
+    private final TimeDuration autoImportRecheckDuration;
+    private final TimeDuration importDurationGoal;
+    private final int importMinTransactions;
+    private final int importMaxTransactions;
+
+    private final TimeDuration inspectorFrequency;
+
+    static WordlistConfiguration fromConfiguration(
+            final Configuration configuration,
+            final WordlistType type
+    )
+    {
+        switch ( type )
+        {
+            case SEEDLIST:
+            {
+                return WordlistConfiguration.builder()
+                        .autoImportUrl( readAutoImportUrl( configuration, PwmSetting.SEEDLIST_FILENAME ) )
+                        .minSize( Integer.parseInt( configuration.readAppProperty( AppProperty.WORDLIST_CHAR_LENGTH_MIN ) ) )
+                        .maxSize( Integer.parseInt( configuration.readAppProperty( AppProperty.WORDLIST_CHAR_LENGTH_MAX ) ) )
+                        .metaDataAppAttribute( PwmApplication.AppAttribute.SEEDLIST_METADATA )
+                        .builtInWordlistLocationProperty( AppProperty.SEEDLIST_BUILTIN_PATH )
+                        .db( LocalDB.DB.SEEDLIST_WORDS )
+                        .wordlistFilenameSetting( PwmSetting.SEEDLIST_FILENAME )
+
+                        .minSize( Integer.parseInt( configuration.readAppProperty( AppProperty.WORDLIST_CHAR_LENGTH_MIN ) ) )
+                        .maxSize( Integer.parseInt( configuration.readAppProperty( AppProperty.WORDLIST_CHAR_LENGTH_MAX ) ) )
+                        .autoImportRecheckDuration( TimeDuration.of(
+                                Long.parseLong( configuration.readAppProperty( AppProperty.WORDLIST_IMPORT_AUTO_IMPORT_RECHECK_SECONDS ) ),
+                                TimeDuration.Unit.SECONDS ) )
+                        .importDurationGoal( TimeDuration.of(
+                                Long.parseLong( configuration.readAppProperty( AppProperty.WORDLIST_IMPORT_DURATION_GOAL_MS ) ),
+                                TimeDuration.Unit.MILLISECONDS ) )
+                        .importMinTransactions( Integer.parseInt( configuration.readAppProperty( AppProperty.WORDLIST_IMPORT_MIN_TRANSACTIONS ) ) )
+                        .importMaxTransactions( Integer.parseInt( configuration.readAppProperty( AppProperty.WORDLIST_IMPORT_MAX_TRANSACTIONS ) ) )
+
+                        .inspectorFrequency( TimeDuration.of(
+                                Long.parseLong( configuration.readAppProperty( AppProperty.WORDLIST_INSPECTOR_FREQUENCY_SECONDS ) ),
+                                TimeDuration.Unit.SECONDS ) )
+
+                        .build();
+            }
+
+            case WORDLIST:
+            {
+                return WordlistConfiguration.builder()
+                        .caseSensitive( configuration.readSettingAsBoolean( PwmSetting.WORDLIST_CASE_SENSITIVE )  )
+                        .checkSize( (int) configuration.readSettingAsLong( PwmSetting.PASSWORD_WORDLIST_WORDSIZE ) )
+                        .autoImportUrl( readAutoImportUrl( configuration, PwmSetting.WORDLIST_FILENAME ) )
+                        .metaDataAppAttribute( PwmApplication.AppAttribute.WORDLIST_METADATA )
+                        .builtInWordlistLocationProperty( AppProperty.WORDLIST_BUILTIN_PATH )
+                        .db( LocalDB.DB.WORDLIST_WORDS )
+                        .wordlistFilenameSetting( PwmSetting.WORDLIST_FILENAME )
+
+                        .minSize( Integer.parseInt( configuration.readAppProperty( AppProperty.WORDLIST_CHAR_LENGTH_MIN ) ) )
+                        .maxSize( Integer.parseInt( configuration.readAppProperty( AppProperty.WORDLIST_CHAR_LENGTH_MAX ) ) )
+                        .autoImportRecheckDuration( TimeDuration.of(
+                                Long.parseLong( configuration.readAppProperty( AppProperty.WORDLIST_IMPORT_AUTO_IMPORT_RECHECK_SECONDS ) ),
+                                TimeDuration.Unit.SECONDS ) )
+                        .importDurationGoal( TimeDuration.of(
+                                Long.parseLong( configuration.readAppProperty( AppProperty.WORDLIST_IMPORT_DURATION_GOAL_MS ) ),
+                                TimeDuration.Unit.MILLISECONDS ) )
+                        .importMinTransactions( Integer.parseInt( configuration.readAppProperty( AppProperty.WORDLIST_IMPORT_MIN_TRANSACTIONS ) ) )
+                        .importMaxTransactions( Integer.parseInt( configuration.readAppProperty( AppProperty.WORDLIST_IMPORT_MAX_TRANSACTIONS ) ) )
+
+                        .inspectorFrequency( TimeDuration.of(
+                                Long.parseLong( configuration.readAppProperty( AppProperty.WORDLIST_INSPECTOR_FREQUENCY_SECONDS ) ),
+                                TimeDuration.Unit.SECONDS ) )
+
+                        .build();
+            }
+
+            default:
+                JavaHelper.unhandledSwitchStatement( type );
+        }
+
+        throw new IllegalStateException( "unreachable switch statement" );
+    }
+
+
+    private static String readAutoImportUrl(
+            final Configuration configuration,
+            final PwmSetting wordlistFileSetting
+    )
+    {
+        final String inputUrl = configuration.readSettingAsString( wordlistFileSetting );
+
+        if ( StringUtil.isEmpty( inputUrl ) )
+        {
+            return null;
+        }
+
+        if ( !inputUrl.startsWith( "http:" ) && !inputUrl.startsWith( "https:" ) && !inputUrl.startsWith( "file:" ) )
+        {
+            return "file:" + inputUrl;
+        }
+
+        return inputUrl;
+    }
 }

server/src/main/java/password/pwm/svc/wordlist/WordlistImporter.java

@@ -0,0 +1,382 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import org.apache.commons.io.IOUtils;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.TransactionSizeCalculator;
+import password.pwm.util.java.ConditionalTaskExecutor;
+import password.pwm.util.java.Percent;
+import password.pwm.util.java.PwmNumberFormat;
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.localdb.LocalDBException;
+import password.pwm.util.logging.PwmLogger;
+
+import java.time.Instant;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.function.BooleanSupplier;
+
+/**
+ * @author Jason D. Rivard
+ */
+class WordlistImporter implements Runnable
+{
+    // words tarting with this prefix are ignored.
+    private static final String COMMENT_PREFIX = "!#comment:";
+
+    private final WordlistZipReader zipFileReader;
+    private final WordlistSourceType sourceType;
+    private final TransactionSizeCalculator transactionCalculator;
+    private final Set<String> bufferedWords = new TreeSet<>();
+    private final WordlistBucket wordlistBucket;
+    private final AbstractWordlist rootWordlist;
+    private final WordlistSourceInfo wordlistSourceInfo;
+    private final BooleanSupplier cancelFlag;
+
+    private ErrorInformation exitError;
+    private Instant startTime = Instant.now();
+    private long bytesSkipped;
+
+    private enum DebugKey
+    {
+        LinesRead,
+        BytesRead,
+        BytesRemaining,
+        BufferSize,
+        BytesSkipped,
+        BytesPerSecond,
+        PercentComplete,
+        ImportTime,
+        EstimatedRemainingTime,
+    }
+
+    WordlistImporter(
+            final WordlistSourceInfo wordlistSourceInfo,
+            final WordlistZipReader wordlistZipReader,
+            final WordlistSourceType sourceType,
+            final AbstractWordlist rootWordlist,
+            final BooleanSupplier cancelFlag
+    )
+    {
+        this.wordlistSourceInfo = wordlistSourceInfo;
+        this.sourceType = sourceType;
+        this.zipFileReader = wordlistZipReader;
+        this.rootWordlist = rootWordlist;
+        this.cancelFlag = cancelFlag;
+        this.wordlistBucket = rootWordlist.getWordlistBucket();
+
+        final WordlistConfiguration wordlistConfiguration = rootWordlist.getConfiguration();
+
+        transactionCalculator = new TransactionSizeCalculator(
+                TransactionSizeCalculator.Settings.builder()
+                        .durationGoal( wordlistConfiguration.getImportDurationGoal() )
+                        .minTransactions( wordlistConfiguration.getImportMinTransactions() )
+                        .maxTransactions( wordlistConfiguration.getImportMaxTransactions() )
+                        .build()
+        );
+
+    }
+
+    @Override
+    public void run()
+    {
+        String errorMsg = null;
+        try
+        {
+            doImport();
+        }
+        catch ( PwmUnrecoverableException e )
+        {
+            errorMsg = "error during import: " + e.getErrorInformation().getDetailedErrorMsg();
+        }
+        catch ( LocalDBException e )
+        {
+            errorMsg = "localDB error during import: " + e.getMessage();
+        }
+
+        if ( errorMsg != null )
+        {
+            exitError = new ErrorInformation( PwmError.ERROR_WORDLIST_IMPORT_ERROR, errorMsg, new String[]
+                    {
+                            errorMsg,
+                    }
+            );
+        }
+
+        if ( cancelFlag.getAsBoolean() )
+        {
+            getLogger().debug( "exiting import due to cancel flag" );
+        }
+    }
+
+    private void init( )
+            throws PwmUnrecoverableException,
+            LocalDBException
+    {
+        if ( cancelFlag.getAsBoolean() )
+        {
+            return;
+        }
+
+        if ( wordlistSourceInfo == null || !wordlistSourceInfo.equals( rootWordlist.readWordlistStatus().getRemoteInfo() ) )
+        {
+            rootWordlist.writeWordlistStatus( WordlistStatus.builder()
+                    .sourceType( sourceType )
+                    .build() );
+        }
+
+        final long previousBytesRead = rootWordlist.readWordlistStatus().getBytes();
+
+        if ( previousBytesRead == 0 )
+        {
+            rootWordlist.clearImpl( Wordlist.Activity.Importing );
+        }
+        else if ( previousBytesRead > 0 )
+        {
+            skipForward( previousBytesRead );
+        }
+    }
+
+    private void doImport( )
+            throws LocalDBException, PwmUnrecoverableException
+    {
+        rootWordlist.setActivity( Wordlist.Activity.Importing );
+
+        final ConditionalTaskExecutor metaUpdater = new ConditionalTaskExecutor(
+                () -> rootWordlist.writeWordlistStatus( WordlistStatus.builder()
+                        .sourceType( sourceType )
+                        .storeDate( Instant.now() )
+                        .remoteInfo( wordlistSourceInfo )
+                        .bytes( zipFileReader.getByteCount() )
+                        .build() ),
+                new ConditionalTaskExecutor.TimeDurationPredicate( TimeDuration.SECONDS_10 )
+        );
+
+        final ConditionalTaskExecutor debugOutputter = new ConditionalTaskExecutor(
+                () -> getLogger().debug( makeStatString() ),
+                new ConditionalTaskExecutor.TimeDurationPredicate( AbstractWordlist.DEBUG_OUTPUT_FREQUENCY )
+        );
+
+        try
+        {
+            debugOutputter.conditionallyExecuteTask();
+
+            init();
+
+            startTime = Instant.now();
+
+            getLogger().debug( "beginning import" );
+
+            String line;
+            do
+            {
+                line = zipFileReader.nextLine();
+                if ( line != null )
+                {
+                    addLine( line );
+
+                    debugOutputter.conditionallyExecuteTask();
+
+                    if ( bufferedWords.size() > transactionCalculator.getTransactionSize() )
+                    {
+                        flushBuffer();
+                        metaUpdater.conditionallyExecuteTask();
+                    }
+                }
+            }
+            while ( !cancelFlag.getAsBoolean() && line != null );
+
+
+            if ( cancelFlag.getAsBoolean() )
+            {
+                getLogger().warn( "pausing import" );
+            }
+            else
+            {
+                populationComplete();
+            }
+        }
+        finally
+        {
+            IOUtils.closeQuietly( zipFileReader );
+        }
+    }
+
+    private void addLine( final String word )
+    {
+
+        if ( StringUtil.isEmpty( word ) || word.startsWith( COMMENT_PREFIX ) )
+        {
+            return;
+        }
+
+        bufferedWords.add( word );
+    }
+
+    private void flushBuffer( )
+            throws LocalDBException
+    {
+        final long startTime = System.currentTimeMillis();
+
+        //add the elements
+        wordlistBucket.addWords( bufferedWords );
+
+        if ( cancelFlag.getAsBoolean() )
+        {
+            return;
+        }
+
+        //mark how long the buffer close took
+        final long commitTime = System.currentTimeMillis() - startTime;
+        transactionCalculator.recordLastTransactionDuration( commitTime );
+
+        //clear the buffers.
+        bufferedWords.clear();
+    }
+
+    private void populationComplete( )
+            throws LocalDBException
+    {
+        flushBuffer();
+        getLogger().info( makeStatString() );
+        getLogger().trace( "beginning wordlist size query" );
+        final long wordlistSize = wordlistBucket.size();
+
+        final String logMsg = "population complete, added " + wordlistSize
+                + " total words in " + TimeDuration.fromCurrent( startTime ).asCompactString();
+        getLogger().info( logMsg );
+
+        {
+            final WordlistStatus wordlistStatus = WordlistStatus.builder()
+                    .remoteInfo( wordlistSourceInfo )
+                    .storeDate( Instant.now() )
+                    .sourceType( sourceType )
+                    .completed( true )
+                    .bytes( zipFileReader.getByteCount() )
+                    .build();
+            rootWordlist.writeWordlistStatus( wordlistStatus );
+        }
+    }
+
+    private PwmLogger getLogger()
+    {
+        return this.rootWordlist.getLogger();
+    }
+
+    ErrorInformation getExitError()
+    {
+        return exitError;
+    }
+
+    private void skipForward( final long previousBytesRead )
+            throws PwmUnrecoverableException
+    {
+        final Instant startSkip = Instant.now();
+        final ConditionalTaskExecutor debugOutputter = new ConditionalTaskExecutor(
+                () -> getLogger().debug( "continuing skipping forward in wordlist"
+                        + ", " + StringUtil.formatDiskSizeforDebug( zipFileReader.getByteCount() )
+                        + " of " + StringUtil.formatDiskSizeforDebug( previousBytesRead )
+                        + " (" + TimeDuration.compactFromCurrent( startSkip ) + ")" ),
+                new ConditionalTaskExecutor.TimeDurationPredicate( AbstractWordlist.DEBUG_OUTPUT_FREQUENCY )
+        );
+
+        getLogger().debug( "will skip forward " + StringUtil.formatDiskSizeforDebug( previousBytesRead ) + " in stream that have been previously imported" );
+        while ( !cancelFlag.getAsBoolean() && bytesSkipped < ( previousBytesRead + 1024 ) )
+        {
+            zipFileReader.nextLine();
+            bytesSkipped = zipFileReader.getByteCount();
+            debugOutputter.conditionallyExecuteTask();
+        }
+        getLogger().debug( "skipped forward " + StringUtil.formatDiskSizeforDebug( previousBytesRead )
+                + " in stream (" + TimeDuration.fromCurrent( startSkip ).asCompactString() + ")" );
+    }
+
+    private String makeStatString()
+    {
+        return StringUtil.mapToString( makeStatValues(), "=", ", " );
+    }
+
+    private Map<DebugKey, String> makeStatValues()
+    {
+        final Map<DebugKey, String> stats = new TreeMap<>();
+
+        if ( wordlistSourceInfo != null )
+        {
+            final long totalBytes = wordlistSourceInfo.getBytes();
+            final long remainingBytes = totalBytes - zipFileReader.getByteCount();
+            stats.put( DebugKey.BytesRemaining, StringUtil.formatDiskSizeforDebug( remainingBytes ) );
+
+            try
+            {
+                if ( zipFileReader.getByteCount() > 1000 && TimeDuration.fromCurrent( startTime ).isLongerThan( TimeDuration.MINUTE ) )
+                {
+                    final long bytesSinceStart = zipFileReader.getByteCount() - bytesSkipped;
+                    final long elapsedSeconds = TimeDuration.fromCurrent( startTime ).as( TimeDuration.Unit.SECONDS );
+
+                    if ( elapsedSeconds > 0 )
+                    {
+                        final long bytesPerSecond = bytesSinceStart / elapsedSeconds;
+                        stats.put( DebugKey.BytesPerSecond, StringUtil.formatDiskSizeforDebug( bytesPerSecond ) );
+
+                        if ( remainingBytes > 0 )
+                        {
+                            final long remainingSeconds = remainingBytes / bytesPerSecond;
+                            stats.put( DebugKey.EstimatedRemainingTime, TimeDuration.of( remainingSeconds, TimeDuration.Unit.SECONDS ).asCompactString() );
+                        }
+                    }
+                }
+            }
+            catch ( Exception e )
+            {
+                getLogger().error( "error calculating " );
+
+                /* ignore - it's a long overflow if the estimate is off */
+            }
+
+            final Percent percent = new Percent( zipFileReader.getByteCount(), wordlistSourceInfo.getBytes() );
+            stats.put( DebugKey.PercentComplete, percent.pretty( 2 ) );
+        }
+
+        stats.put( DebugKey.LinesRead, PwmNumberFormat.forDefaultLocale().format( zipFileReader.getLineCount() ) );
+        stats.put( DebugKey.BytesRead, StringUtil.formatDiskSizeforDebug( zipFileReader.getByteCount() ) );
+
+        stats.put( DebugKey.BufferSize, PwmNumberFormat.forDefaultLocale().format( transactionCalculator.getTransactionSize() ) );
+
+        if ( bytesSkipped > 0 )
+        {
+            stats.put( DebugKey.BytesSkipped, StringUtil.formatDiskSizeforDebug( bytesSkipped ) );
+        }
+
+        stats.put( DebugKey.ImportTime, TimeDuration.fromCurrent( startTime ).asCompactString() );
+
+        return Collections.unmodifiableMap( stats );
+    }
+
+}

server/src/main/java/password/pwm/svc/wordlist/WordlistInspector.java

@@ -0,0 +1,387 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import password.pwm.PwmApplication;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.localdb.LocalDBException;
+import password.pwm.util.logging.PwmLogger;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.function.BooleanSupplier;
+
+class WordlistInspector implements Runnable
+{
+    private final AbstractWordlist rootWordlist;
+    private final PwmApplication pwmApplication;
+    private final BooleanSupplier cancelFlag;
+
+    WordlistInspector(
+            final PwmApplication pwmApplication,
+            final AbstractWordlist rootWordlist,
+            final BooleanSupplier cancelFlag
+    )
+    {
+        this.pwmApplication = pwmApplication;
+        this.rootWordlist = rootWordlist;
+        this.cancelFlag = cancelFlag;
+    }
+
+    @Override
+    public void run()
+    {
+        try
+        {
+            checkPopulation();
+        }
+        catch ( Exception e )
+        {
+            getLogger().error( "unexpected error running population worker: " + e.getMessage(), e );
+        }
+    }
+
+    private void checkPopulation( )
+            throws Exception
+    {
+        if ( cancelFlag.getAsBoolean() )
+        {
+            return;
+        }
+
+        rootWordlist.setActivity( Wordlist.Activity.ReadingWordlistFile );
+        final boolean autoImportUrlConfigured = !StringUtil.isEmpty( rootWordlist.getConfiguration().getAutoImportUrl() );
+        WordlistStatus existingStatus = rootWordlist.readWordlistStatus();
+
+        if ( checkIfClearIsNeeded( existingStatus, autoImportUrlConfigured ) )
+        {
+            rootWordlist.clearImpl( Wordlist.Activity.ReadingWordlistFile );
+        }
+
+        existingStatus = rootWordlist.readWordlistStatus();
+
+        if ( checkIfExistingOkay( existingStatus, autoImportUrlConfigured ) )
+        {
+            return;
+        }
+
+        if ( cancelFlag.getAsBoolean() )
+        {
+            return;
+        }
+
+        if ( autoImportUrlConfigured )
+        {
+        try
+        {
+            checkAutoPopulation( existingStatus );
+        }
+        catch ( PwmUnrecoverableException e )
+        {
+            getLogger().error( "error importing auto-import wordlist: " + e.getMessage() );
+            rootWordlist.setAutoImportError( e.getErrorInformation() );
+        }
+        }
+
+        existingStatus = rootWordlist.readWordlistStatus();
+
+        if ( cancelFlag.getAsBoolean() )
+        {
+            return;
+        }
+
+        boolean needsBuiltInPopulation = false;
+
+        if ( autoImportUrlConfigured
+                && rootWordlist.getAutoImportError() != null
+                && !existingStatus.isCompleted()
+        )
+        {
+            getLogger().debug( "auto-import did not complete and failed with an error, will (temporarily) import built-in wordlist." );
+            needsBuiltInPopulation = true;
+        }
+        else if ( !autoImportUrlConfigured )
+        {
+            final WordlistSource source = WordlistSource.forBuiltIn( pwmApplication, rootWordlist.getConfiguration() );
+
+            if ( existingStatus.getSourceType() != WordlistSourceType.Temporary_BuiltIn )
+            {
+                getLogger().debug( "auto-import is not configured, and existing wordlist is not of type BuiltIn, will reload." );
+                needsBuiltInPopulation = true;
+            }
+            else if ( !existingStatus.isCompleted() )
+            {
+                getLogger().debug( "existing built-in store was not completed, will re-import" );
+                needsBuiltInPopulation = true;
+            }
+            else
+            {
+                final WordlistSourceInfo builtInInfo = source.readRemoteWordlistInfo( cancelFlag );
+                if ( !builtInInfo.equals( existingStatus.getRemoteInfo() ) )
+                {
+                    getLogger().debug( "existing built-in store does not match imported wordlist, will re-import" );
+                    needsBuiltInPopulation = true;
+                }
+            }
+        }
+
+        if ( cancelFlag.getAsBoolean() )
+        {
+            return;
+        }
+
+        if ( needsBuiltInPopulation )
+        {
+            populateBuiltIn( autoImportUrlConfigured ? WordlistSourceType.Temporary_BuiltIn : WordlistSourceType.BuiltIn );
+        }
+    }
+
+    private boolean checkIfClearIsNeeded(
+            final WordlistStatus wordlistStatus,
+            final boolean autoImportUrlConfigured
+    )
+    {
+        if ( wordlistStatus == null || wordlistStatus.getSourceType() == null )
+        {
+            return true;
+        }
+
+        if ( wordlistStatus.getVersion() != WordlistStatus.CURRENT_VERSION )
+        {
+            getLogger().debug( "stored version '" + wordlistStatus.getVersion() + "' is not current version '"
+                    + WordlistStatus.CURRENT_VERSION + "', will clear" );
+            return true;
+        }
+
+        switch ( wordlistStatus.getSourceType() )
+        {
+            case AutoImport:
+            {
+                if ( !autoImportUrlConfigured )
+                {
+                    getLogger().debug( "existing stored list is AutoImport but auto-import is not configured, will clear" );
+                    return true;
+                }
+
+                final String storedImportUrl = wordlistStatus.getRemoteInfo().getImportUrl();
+                final String configuredUrl = rootWordlist.getConfiguration().getAutoImportUrl();
+                if ( !StringUtil.nullSafeEquals( storedImportUrl, configuredUrl ) )
+                {
+                    getLogger().debug( "auto import url has been modified since import, will clear" );
+                    return true;
+                }
+            }
+            break;
+
+            case BuiltIn:
+            {
+                if ( autoImportUrlConfigured )
+                {
+                    return false;
+                }
+            }
+            break;
+
+            case Temporary_BuiltIn:
+            {
+
+                if ( autoImportUrlConfigured )
+                {
+                    return false;
+                }
+            }
+            break;
+
+            case User:
+            {
+                if ( !wordlistStatus.isCompleted() )
+                {
+                    return true;
+                }
+            }
+            break;
+
+            default:
+                return false;
+        }
+        return false;
+    }
+
+    private boolean checkIfExistingOkay(
+            final WordlistStatus wordlistStatus,
+            final boolean autoImportUrlConfigured
+    )
+            throws LocalDBException
+    {
+        if ( wordlistStatus.getSourceType() == null )
+        {
+            return false;
+        }
+
+        switch ( wordlistStatus.getSourceType() )
+        {
+            case User:
+            {
+                if ( wordlistStatus.isCompleted() )
+                {
+                    return true;
+                }
+            }
+            break;
+
+
+            case BuiltIn:
+            {
+                if ( wordlistStatus.isCompleted() && wordlistStatus.getVersion() == WordlistStatus.CURRENT_VERSION && !autoImportUrlConfigured )
+                {
+                    return true;
+                }
+            }
+            break;
+
+            case Temporary_BuiltIn:
+            {
+                final WordlistSource testWordlistSource = WordlistSource.forAutoImport( pwmApplication, rootWordlist.getConfiguration() );
+                try
+                {
+                    testWordlistSource.readRemoteWordlistInfo( cancelFlag );
+                }
+                catch ( PwmUnrecoverableException e )
+                {
+                    rootWordlist.setAutoImportError( e.getErrorInformation() );
+                    getLogger().debug( "existing stored list is not type AutoImport but auto-import is configured"
+                            + ", however auto-import returns error so will keep existing built-in wordlist; error: " + e.getMessage() );
+                    return true;
+                }
+                getLogger().debug( "existing stored list is not type AutoImport but auto-import is configured, will clear" );
+                rootWordlist.clearImpl( Wordlist.Activity.ReadingWordlistFile );
+            }
+            break;
+
+            case AutoImport:
+            {
+                final Instant storageTime = wordlistStatus.getStoreDate();
+                final TimeDuration timeSinceCompletion = TimeDuration.fromCurrent( storageTime );
+                final TimeDuration recheckDuration = rootWordlist.getConfiguration().getAutoImportRecheckDuration();
+                if ( wordlistStatus.isCompleted() && timeSinceCompletion.isShorterThan( recheckDuration ) && autoImportUrlConfigured )
+                {
+                    /*
+                    getLogger().debug( "existing completed wordlist is "
+                            + timeSinceCompletion.asCompactString() + " old, which is less than recheck interval of "
+                            + recheckDuration.asCompactString() + ", skipping recheck" );
+                     */
+                    return true;
+                }
+            }
+            break;
+
+            default:
+                return false;
+        }
+
+        return false;
+    }
+
+    private void checkAutoPopulation(
+            final WordlistStatus existingStatus
+    )
+            throws IOException, PwmUnrecoverableException
+    {
+        final WordlistSource source = WordlistSource.forAutoImport( pwmApplication, rootWordlist.getConfiguration() );
+        final WordlistSourceInfo remoteInfo = source.readRemoteWordlistInfo( cancelFlag );
+
+        boolean needsAutoImport = false;
+        if ( remoteInfo == null )
+        {
+            getLogger().warn( "can't read remote wordlist data from url " + rootWordlist.getConfiguration().getAutoImportUrl() );
+        }
+        else
+        {
+            if ( !remoteInfo.equals( existingStatus.getRemoteInfo() ) )
+            {
+                getLogger().debug( "auto-import url remote hash does not equal currently stored hash, will start auto-import" );
+                needsAutoImport = true;
+            }
+            else if ( remoteInfo.getBytes() > existingStatus.getBytes() || !existingStatus.isCompleted() )
+            {
+                getLogger().debug( "auto-import did not previously complete, will continue previous import" );
+                needsAutoImport = true;
+            }
+
+            if ( needsAutoImport )
+            {
+                populateAutoImport( remoteInfo );
+            }
+        }
+    }
+
+    private void populateBuiltIn( final WordlistSourceType wordlistSourceType )
+            throws IOException, PwmUnrecoverableException
+    {
+        final WordlistSource wordlistSource = WordlistSource.forBuiltIn( pwmApplication, rootWordlist.getConfiguration() );
+        final WordlistSourceInfo wordlistSourceInfo = wordlistSource.readRemoteWordlistInfo( cancelFlag );
+        final WordlistImporter wordlistImporter = new WordlistImporter(
+                wordlistSourceInfo,
+                wordlistSource.getZipWordlistReader(),
+                wordlistSourceType,
+                rootWordlist,
+                cancelFlag );
+        wordlistImporter.run();
+    }
+
+
+    private void populateAutoImport( final WordlistSourceInfo wordlistSourceInfo )
+            throws IOException, PwmUnrecoverableException
+    {
+        rootWordlist.setAutoImportError( null );
+        final WordlistSource wordlistSource = WordlistSource.forAutoImport( pwmApplication, rootWordlist.getConfiguration() );
+        final WordlistImporter wordlistImporter = new WordlistImporter(
+                wordlistSourceInfo,
+                wordlistSource.getZipWordlistReader(),
+                WordlistSourceType.AutoImport,
+                rootWordlist,
+                cancelFlag );
+        wordlistImporter.run();
+        rootWordlist.setAutoImportError( wordlistImporter.getExitError() );
+    }
+
+    private PwmLogger getLogger()
+    {
+        return this.rootWordlist.getLogger();
+    }
+
+
+    boolean needsRunningAgain()
+    {
+        final WordlistStatus wordlistStatus = rootWordlist.readWordlistStatus();
+
+        if ( wordlistStatus.isCompleted() )
+        {
+            return false;
+        }
+
+        return true;
+    }
+}

server/src/main/java/password/pwm/svc/wordlist/WordlistManager.java

@@ -1,101 +0,0 @@
-/*
- * Password Management Servlets (PWM)
- * http://www.pwm-project.org
- *
- * Copyright (c) 2006-2009 Novell, Inc.
- * Copyright (c) 2009-2018 The PWM Project
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-package password.pwm.svc.wordlist;
-
-import password.pwm.AppProperty;
-import password.pwm.PwmApplication;
-import password.pwm.PwmConstants;
-import password.pwm.config.PwmSetting;
-import password.pwm.error.PwmException;
-import password.pwm.util.localdb.LocalDB;
-import password.pwm.util.logging.PwmLogger;
-
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-
-/**
- * @author Jason D. Rivard
- */
-public class WordlistManager extends AbstractWordlist implements Wordlist
-{
-
-    public WordlistManager( )
-    {
-        logger = PwmLogger.forClass( WordlistManager.class );
-    }
-
-
-    protected Map<String, String> getWriteTxnForValue( final String value )
-    {
-        final Map<String, String> returnSet = new TreeMap<>();
-        final Set<String> chunkedWords = chunkWord( value, this.wordlistConfiguration.getCheckSize() );
-        for ( final String word : chunkedWords )
-        {
-            returnSet.put( word, "" );
-        }
-        return returnSet;
-    }
-
-    public void init( final PwmApplication pwmApplication ) throws PwmException
-    {
-        super.init( pwmApplication );
-        final boolean caseSensitive = pwmApplication.getConfig().readSettingAsBoolean( PwmSetting.WORDLIST_CASE_SENSITIVE );
-        final int checkSize = ( int ) pwmApplication.getConfig().readSettingAsLong( PwmSetting.PASSWORD_WORDLIST_WORDSIZE );
-        final String wordlistUrl = readAutoImportUrl();
-
-        final int minSize = Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.WORDLIST_CHAR_LENGTH_MIN ) );
-        final int maxSize = Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.WORDLIST_CHAR_LENGTH_MAX ) );
-
-        this.wordlistConfiguration = new WordlistConfiguration( caseSensitive, checkSize, wordlistUrl, minSize, maxSize );
-        this.debugLabel = PwmConstants.PWM_APP_NAME + "-Wordlist";
-        backgroundStartup();
-    }
-
-    @Override
-    protected PwmApplication.AppAttribute getMetaDataAppAttribute( )
-    {
-        return PwmApplication.AppAttribute.WORDLIST_METADATA;
-    }
-
-    @Override
-    protected PwmSetting getWordlistFileSetting( )
-    {
-        return PwmSetting.WORDLIST_FILENAME;
-    }
-
-    @Override
-    protected LocalDB.DB getWordlistDB( )
-    {
-        return LocalDB.DB.WORDLIST_WORDS;
-    }
-
-    @Override
-    protected AppProperty getBuiltInWordlistLocationProperty( )
-    {
-        return AppProperty.WORDLIST_BUILTIN_PATH;
-    }
-
-
-}

server/src/main/java/password/pwm/svc/wordlist/WordlistService.java

@@ -0,0 +1,58 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import password.pwm.PwmApplication;
+import password.pwm.error.PwmException;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.logging.PwmLogger;
+
+
+/**
+ * @author Jason D. Rivard
+ */
+public class WordlistService extends AbstractWordlist implements Wordlist
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( WordlistService.class );
+
+    public WordlistService( )
+    {
+    }
+
+    public void init( final PwmApplication pwmApplication ) throws PwmException
+    {
+        super.init( pwmApplication, WordlistType.WORDLIST );
+    }
+
+    @Override
+    PwmLogger getLogger()
+    {
+        return LOGGER;
+    }
+
+    @Override
+    public boolean containsWord( final String word ) throws PwmUnrecoverableException
+    {
+        return super.containsWord( word );
+    }
+}

server/src/main/java/password/pwm/svc/wordlist/WordlistSource.java

@@ -0,0 +1,192 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.input.CountingInputStream;
+import org.apache.commons.io.output.NullOutputStream;
+import password.pwm.AppProperty;
+import password.pwm.PwmApplication;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.http.ContextManager;
+import password.pwm.http.client.PwmHttpClient;
+import password.pwm.http.client.PwmHttpClientConfiguration;
+import password.pwm.util.java.ConditionalTaskExecutor;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
+import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.secure.ChecksumInputStream;
+import password.pwm.util.secure.X509Utils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.time.Instant;
+import java.util.function.BooleanSupplier;
+
+class WordlistSource
+{
+    private static final PwmLogger LOGGER = PwmLogger.forClass( WordlistSource.class );
+
+    private final WordlistSourceType wordlistSourceType;
+    private final StreamProvider streamProvider;
+    private final String importUrl;
+
+    private WordlistSource( final WordlistSourceType wordlistSourceType, final String importUrl, final StreamProvider streamProvider )
+    {
+        this.wordlistSourceType = wordlistSourceType;
+        this.importUrl = importUrl;
+        this.streamProvider = streamProvider;
+    }
+
+    public WordlistSourceType getWordlistSourceType()
+    {
+        return wordlistSourceType;
+    }
+
+    private interface StreamProvider
+    {
+        InputStream getInputStream() throws IOException, PwmUnrecoverableException;
+    }
+
+    static WordlistSource forAutoImport( final PwmApplication pwmApplication, final WordlistConfiguration wordlistConfiguration )
+    {
+        final String importUrl = wordlistConfiguration.getAutoImportUrl();
+        return new WordlistSource( WordlistSourceType.AutoImport, importUrl, () ->
+        {
+            if ( importUrl.startsWith( "http" ) )
+            {
+                final boolean promiscuous = Boolean.parseBoolean( pwmApplication.getConfig().readAppProperty( AppProperty.HTTP_CLIENT_PROMISCUOUS_WORDLIST_ENABLE ) );
+                final PwmHttpClientConfiguration pwmHttpClientConfiguration = PwmHttpClientConfiguration.builder()
+                        .trustManager( promiscuous ? new X509Utils.PromiscuousTrustManager() : null )
+                        .build();
+                final PwmHttpClient client = new PwmHttpClient( pwmApplication, null, pwmHttpClientConfiguration );
+                return client.streamForUrl( wordlistConfiguration.getAutoImportUrl() );
+            }
+
+            try
+            {
+                final URL url = new URL( importUrl );
+                return url.openStream();
+            }
+            catch ( IOException e )
+            {
+                final String msg = "unable to open auto-import URL: " + e.getMessage();
+                throw PwmUnrecoverableException.newException( PwmError.ERROR_WORDLIST_IMPORT_ERROR, msg );
+            }
+        }
+        );
+    }
+
+    static WordlistSource forBuiltIn(
+            final PwmApplication pwmApplication,
+            final WordlistConfiguration wordlistConfiguration
+    )
+    {
+        return new WordlistSource( WordlistSourceType.BuiltIn, null, () ->
+        {
+            final ContextManager contextManager = pwmApplication.getPwmEnvironment().getContextManager();
+            if ( contextManager != null )
+            {
+                final String wordlistFilename = pwmApplication.getConfig().readAppProperty( wordlistConfiguration.getBuiltInWordlistLocationProperty() );
+                return contextManager.getResourceAsStream( wordlistFilename );
+            }
+            throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, "unable to locate builtin wordlist file" ) );
+        }
+        );
+    }
+
+    WordlistZipReader getZipWordlistReader()
+            throws IOException, PwmUnrecoverableException
+    {
+        return new WordlistZipReader( this.streamProvider.getInputStream() );
+    }
+
+    WordlistSourceInfo readRemoteWordlistInfo(
+            final BooleanSupplier cancelFlag
+    )
+            throws PwmUnrecoverableException
+    {
+        final int buffersize = 128_1024;
+        InputStream inputStream = null;
+
+        try
+        {
+            final Instant startTime = Instant.now();
+            LOGGER.debug( "reading file info for " + this.getWordlistSourceType() + " wordlist" );
+
+            inputStream = this.streamProvider.getInputStream();
+
+            final ChecksumInputStream checksumInputStream = new ChecksumInputStream( AbstractWordlist.CHECKSUM_HASH_ALG, inputStream );
+            final CountingInputStream countingInputStream = new CountingInputStream( checksumInputStream );
+
+            final ConditionalTaskExecutor debugOutputter = new ConditionalTaskExecutor(
+                    () -> LOGGER.debug( "continuing reading file info for " + getWordlistSourceType() + " wordlist"
+                            + " " + StringUtil.formatDiskSizeforDebug( countingInputStream.getByteCount() )
+                            + " (" + TimeDuration.compactFromCurrent( startTime ) + ")" ),
+                    new ConditionalTaskExecutor.TimeDurationPredicate( AbstractWordlist.DEBUG_OUTPUT_FREQUENCY )
+            );
+
+            final long bytes = JavaHelper.copyWhilePredicate(
+                    countingInputStream,
+                    new NullOutputStream(),
+                    buffersize, o -> !cancelFlag.getAsBoolean(),
+                    debugOutputter );
+
+            if ( cancelFlag.getAsBoolean() )
+            {
+                return null;
+            }
+
+            final String hash = JavaHelper.binaryArrayToHex( checksumInputStream.closeAndFinalChecksum() );
+
+            final WordlistSourceInfo wordlistSourceInfo = new WordlistSourceInfo(
+                    hash,
+                    bytes,
+                    importUrl
+            );
+
+            LOGGER.debug( "completed read of data for " + this.getWordlistSourceType() + " wordlist"
+                    + " " + StringUtil.formatDiskSizeforDebug( countingInputStream.getByteCount() )
+                    + ", " + JsonUtil.serialize( wordlistSourceInfo )
+                    + " (" + TimeDuration.compactFromCurrent( startTime ) + ")" );
+            return wordlistSourceInfo;
+        }
+        catch ( Exception e )
+        {
+            final ErrorInformation errorInformation = new ErrorInformation(
+                    PwmError.ERROR_WORDLIST_IMPORT_ERROR,
+                    "error reading from remote wordlist auto-import url: " + JavaHelper.readHostileExceptionMessage( e )
+            );
+            throw new PwmUnrecoverableException( errorInformation );
+        }
+        finally
+        {
+            IOUtils.closeQuietly( inputStream );
+        }
+    }
+}

server/src/main/java/password/pwm/svc/wordlist/WordlistSourceInfo.java

@@ -0,0 +1,35 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import lombok.Value;
+
+import java.io.Serializable;
+
+@Value
+public class WordlistSourceInfo implements Serializable
+{
+    private String checksum;
+    private long bytes;
+    private String importUrl;
+}

server/src/main/java/password/pwm/svc/wordlist/WordlistSourceType.java

@@ -0,0 +1,41 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.wordlist;
+
+import lombok.Getter;
+
+@Getter
+public enum WordlistSourceType
+{
+    BuiltIn( "Built-In" ),
+    Temporary_BuiltIn( "Built-In (auto-import failed)" ),
+    AutoImport( "Import from configured URL" ),
+    User( "Upload" ),;
+
+    private final String label;
+
+    WordlistSourceType( final String label )
+    {
+        this.label = label;
+    }
+}

server/src/main/java/password/pwm/svc/wordlist/StoredWordlistDataBean.java → server/src/main/java/password/pwm/svc/wordlist/WordlistStatus.java

@@ -23,7 +23,6 @@
 package password.pwm.svc.wordlist;
 
 import lombok.Builder;
-import lombok.Getter;
 import lombok.Value;
 
 import java.io.Serializable;
@@ -31,34 +30,15 @@ import java.time.Instant;
 
 @Value
 @Builder( toBuilder = true )
-public class StoredWordlistDataBean implements Serializable
+public class WordlistStatus implements Serializable
 {
+    public static final int CURRENT_VERSION = 3;
+
+    @Builder.Default
+    private int version = CURRENT_VERSION;
     private boolean completed;
-    private Source source;
+    private WordlistSourceType sourceType;
     private Instant storeDate;
-    private RemoteWordlistInfo remoteInfo;
+    private WordlistSourceInfo remoteInfo;
     private long bytes;
-    private int size;
-
-    @Getter
-    public enum Source
-    {
-        BuiltIn( "Built-In" ),
-        AutoImport( "Import from configured URL" ),
-        User( "Uploaded" ),;
-
-        private final String label;
-
-        Source( final String label )
-        {
-            this.label = label;
-        }
-    }
-
-    @Value
-    public static class RemoteWordlistInfo implements Serializable
-    {
-        private String checksum;
-        private long bytes;
-    }
 }

server/src/main/java/password/pwm/svc/wordlist/WordlistZipReader.java

@@ -24,6 +24,8 @@ package password.pwm.svc.wordlist;
 
 import org.apache.commons.io.input.CountingInputStream;
 import password.pwm.PwmConstants;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.secure.ChecksumInputStream;
@@ -53,8 +55,7 @@ class WordlistZipReader implements AutoCloseable, Closeable
     private BufferedReader reader;
     private ZipEntry zipEntry;
 
-    WordlistZipReader( final InputStream inputStream )
-            throws Exception
+    WordlistZipReader( final InputStream inputStream ) throws PwmUnrecoverableException
     {
         checksumInputStream = new ChecksumInputStream( AbstractWordlist.CHECKSUM_HASH_ALG, inputStream );
         countingInputStream = new CountingInputStream( checksumInputStream );
@@ -68,18 +69,25 @@ class WordlistZipReader implements AutoCloseable, Closeable
     }
 
     private void nextZipEntry( )
-            throws IOException
+            throws PwmUnrecoverableException
     {
-        zipEntry = zipStream.getNextEntry();
-
-        while ( zipEntry != null && zipEntry.isDirectory() )
+        try
         {
             zipEntry = zipStream.getNextEntry();
-        }
 
-        if ( zipEntry != null )
+            while ( zipEntry != null && zipEntry.isDirectory() )
+            {
+                zipEntry = zipStream.getNextEntry();
+            }
+
+            if ( zipEntry != null )
+            {
+                reader = new BufferedReader( new InputStreamReader( zipStream, PwmConstants.DEFAULT_CHARSET ) );
+            }
+        }
+        catch ( IOException e )
         {
-            reader = new BufferedReader( new InputStreamReader( zipStream, PwmConstants.DEFAULT_CHARSET ) );
+            throw PwmUnrecoverableException.newException( PwmError.ERROR_WORDLIST_IMPORT_ERROR, "error reading wordlist zip: " + e.getMessage() );
         }
     }
 
@@ -110,13 +118,20 @@ class WordlistZipReader implements AutoCloseable, Closeable
     }
 
     String nextLine( )
-            throws IOException
+            throws PwmUnrecoverableException
     {
         String line;
 
         do
         {
-            line = reader.readLine();
+            try
+            {
+                line = reader.readLine();
+            }
+            catch ( IOException e )
+            {
+                throw PwmUnrecoverableException.newException( PwmError.ERROR_WORDLIST_IMPORT_ERROR, "error reading zip wordlist file: " + e.getMessage() );
+            }
 
             if ( line == null )
             {

server/src/main/java/password/pwm/util/DataStore.java

@@ -56,6 +56,6 @@ public interface DataStore
     void remove( String key )
             throws PwmDataStoreException, PwmUnrecoverableException;
 
-    int size( )
+    long size( )
             throws PwmDataStoreException, PwmUnrecoverableException;
 }

server/src/main/java/password/pwm/util/RandomPasswordGenerator.java

@@ -39,7 +39,7 @@ import password.pwm.http.PwmSession;
 import password.pwm.svc.PwmService;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsManager;
-import password.pwm.svc.wordlist.SeedlistManager;
+import password.pwm.svc.wordlist.SeedlistService;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.operations.PasswordUtility;
@@ -140,7 +140,7 @@ public class RandomPasswordGenerator
             {
                 Set<String> seeds = DEFAULT_SEED_PHRASES;
 
-                final SeedlistManager seedlistManager = pwmApplication.getSeedlistManager();
+                final SeedlistService seedlistManager = pwmApplication.getSeedlistManager();
                 if ( seedlistManager != null && seedlistManager.status() == PwmService.STATUS.OPEN && seedlistManager.size() > 0 )
                 {
                     seeds = new HashSet<>();

server/src/main/java/password/pwm/util/TransactionSizeCalculator.java

@@ -22,19 +22,22 @@
 
 package password.pwm.util;
 
+import lombok.Builder;
+import lombok.Value;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.TimeDuration;
 
+import java.util.Objects;
+
 public class TransactionSizeCalculator
 {
-
     private final Settings settings;
-
     private volatile int transactionSize;
     private volatile long lastDuration = 1;
 
     public TransactionSizeCalculator( final Settings settings )
     {
+        settings.validateSettings();
         this.settings = settings;
         reset();
     }
@@ -50,6 +53,7 @@ public class TransactionSizeCalculator
         recordLastTransactionDuration( TimeDuration.of( duration, TimeDuration.Unit.MILLISECONDS ) );
     }
 
+    @SuppressWarnings( "ResultOfMethodCallIgnored" )
     public void pause( )
     {
         JavaHelper.pause( Math.min( lastDuration, settings.getDurationGoal().asMillis() * 2 ) );
@@ -57,6 +61,8 @@ public class TransactionSizeCalculator
 
     public void recordLastTransactionDuration( final TimeDuration duration )
     {
+        Objects.requireNonNull( duration );
+
         lastDuration = duration.asMillis();
         final long durationGoalMs = settings.getDurationGoal().asMillis();
         final long difference = Math.abs( duration.asMillis() - durationGoalMs );
@@ -108,18 +114,21 @@ public class TransactionSizeCalculator
         return transactionSize;
     }
 
+    @Value
+    @Builder
     public static class Settings
     {
-        private final TimeDuration durationGoal;
-        private final int maxTransactions;
-        private final int minTransactions;
+        @Builder.Default
+        private TimeDuration durationGoal = TimeDuration.of( 100, TimeDuration.Unit.MILLISECONDS );
 
-        private Settings( final TimeDuration durationGoal, final int maxTransactions, final int minTransactions )
-        {
-            this.durationGoal = durationGoal;
-            this.maxTransactions = maxTransactions;
-            this.minTransactions = minTransactions;
+        @Builder.Default
+        private int maxTransactions = 5003;
 
+        @Builder.Default
+        private int minTransactions = 3;
+
+        private void validateSettings( )
+        {
             if ( minTransactions < 1 )
             {
                 throw new IllegalArgumentException( "minTransactions must be a positive integer" );
@@ -145,51 +154,5 @@ public class TransactionSizeCalculator
                 throw new IllegalArgumentException( "durationGoal must be greater than 0ms" );
             }
         }
-
-
-        public TimeDuration getDurationGoal( )
-        {
-            return durationGoal;
-        }
-
-        public int getMaxTransactions( )
-        {
-            return maxTransactions;
-        }
-
-        public int getMinTransactions( )
-        {
-            return minTransactions;
-        }
-    }
-
-    public static class SettingsBuilder
-    {
-        private TimeDuration durationGoal = TimeDuration.of( 100, TimeDuration.Unit.MILLISECONDS );
-        private int maxTransactions = 5003;
-        private int minTransactions = 3;
-
-        public SettingsBuilder setDurationGoal( final TimeDuration durationGoal )
-        {
-            this.durationGoal = durationGoal;
-            return this;
-        }
-
-        public SettingsBuilder setMaxTransactions( final int maxTransactions )
-        {
-            this.maxTransactions = maxTransactions;
-            return this;
-        }
-
-        public SettingsBuilder setMinTransactions( final int minTransactions )
-        {
-            this.minTransactions = minTransactions;
-            return this;
-        }
-
-        public Settings createSettings( )
-        {
-            return new Settings( durationGoal, maxTransactions, minTransactions );
-        }
     }
 }

server/src/main/java/password/pwm/util/cli/commands/ExportHttpsTomcatConfigCommand.java

@@ -27,7 +27,10 @@ import password.pwm.PwmConstants;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.option.TLSVersion;
+import password.pwm.config.stored.ConfigurationReader;
+import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.cli.CliParameters;
+import password.pwm.util.java.StringUtil;
 
 import java.io.File;
 import java.io.FileInputStream;
@@ -38,6 +41,7 @@ import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Properties;
 import java.util.Set;
 
 public class ExportHttpsTomcatConfigCommand extends AbstractCliCommand
@@ -105,6 +109,29 @@ public class ExportHttpsTomcatConfigCommand extends AbstractCliCommand
         return cliParameters;
     }
 
+    /**
+     * Invoked (via reflection) by tomcatOneJar class in Onejar module.
+     * @param applicationPath application path containing configuration file.
+     * @return Properties with tomcat connector parameters.
+     * @throws PwmUnrecoverableException if problem loading config
+     */
+    public static Properties readAsProperties( final String applicationPath )
+            throws PwmUnrecoverableException
+    {
+        final File configFile = new File( applicationPath + File.separator + PwmConstants.DEFAULT_CONFIG_FILE_FILENAME );
+        final ConfigurationReader reader = new ConfigurationReader( configFile );
+        final Configuration configuration = reader.getConfiguration();
+        final String sslProtocolSettingValue = TomcatConfigWriter.getTlsProtocolsValue( configuration );
+        final Properties newProps = new Properties();
+        newProps.setProperty( "sslEnabledProtocols",  sslProtocolSettingValue );
+        final String ciphers = configuration.readSettingAsString( PwmSetting.HTTPS_CIPHERS );
+        if ( !StringUtil.isEmpty( ciphers ) )
+        {
+            newProps.setProperty( "ciphers", ciphers );
+        }
+        return newProps;
+    }
+
 
     public static class TomcatConfigWriter
     {
@@ -126,7 +153,6 @@ public class ExportHttpsTomcatConfigCommand extends AbstractCliCommand
             outputFile.write( fileContents.getBytes( PwmConstants.DEFAULT_CHARSET ) );
         }
 
-
         private static String getTlsProtocolsValue( final Configuration configuration )
         {
             final Set<TLSVersion> tlsVersions = configuration.readSettingAsOptionList( PwmSetting.HTTPS_PROTOCOLS, TLSVersion.class );
@@ -142,6 +168,5 @@ public class ExportHttpsTomcatConfigCommand extends AbstractCliCommand
             }
             return output.toString();
         }
-
     }
 }

server/src/main/java/password/pwm/util/db/DatabaseDataStore.java

@@ -82,7 +82,7 @@ public class DatabaseDataStore implements DataStore
         databaseService.getAccessor().remove( table, key );
     }
 
-    public int size( ) throws PwmDataStoreException, PwmUnrecoverableException
+    public long size( ) throws PwmDataStoreException, PwmUnrecoverableException
     {
         return databaseService.getAccessor().size( table );
     }

server/src/main/java/password/pwm/util/db/DatabaseService.java

@@ -58,9 +58,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutorService;
+
 
 public class DatabaseService implements PwmService
 {
@@ -84,7 +83,7 @@ public class DatabaseService implements PwmService
     private AtomicLoopIntIncrementer slotIncrementer;
     private final Map<Integer, DatabaseAccessorImpl> accessors = new ConcurrentHashMap<>();
 
-    private ScheduledExecutorService executorService;
+    private ExecutorService executorService;
 
     private final Map<DatabaseAboutProperty, String> debugInfo = new LinkedHashMap<>();
 
@@ -111,14 +110,12 @@ public class DatabaseService implements PwmService
         this.pwmApplication = pwmApplication;
         init();
 
-        executorService = Executors.newSingleThreadScheduledExecutor(
-                JavaHelper.makePwmThreadFactory(
-                        JavaHelper.makeThreadName( pwmApplication, this.getClass() ) + "-",
-                        true
-                ) );
+        executorService = JavaHelper.makeBackgroundExecutor( pwmApplication, this.getClass() );
 
-        final int watchdogFrequencySeconds = Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.DB_CONNECTIONS_WATCHDOG_FREQUENCY_SECONDS ) );
-        executorService.scheduleWithFixedDelay( new ConnectionMonitor(), watchdogFrequencySeconds, watchdogFrequencySeconds, TimeUnit.SECONDS );
+        final TimeDuration watchdogFrequency = TimeDuration.of(
+                Integer.parseInt( pwmApplication.getConfig().readAppProperty( AppProperty.DB_CONNECTIONS_WATCHDOG_FREQUENCY_SECONDS ) ),
+                TimeDuration.Unit.SECONDS );
+        pwmApplication.scheduleFixedRateJob( new ConnectionMonitor(), executorService, watchdogFrequency, watchdogFrequency );
     }
 
     private synchronized void init( )

server/src/main/java/password/pwm/util/java/JavaHelper.java

@@ -65,8 +65,10 @@ import java.util.TimeZone;
 import java.util.TreeSet;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Predicate;
 
@@ -322,10 +324,25 @@ public class JavaHelper
         return IOUtils.copyLarge( input, output, 0, -1, buffer );
     }
 
-    public static long copyWhilePredicate( final InputStream input, final OutputStream output, final Predicate<Long> predicate )
+    public static long copyWhilePredicate(
+            final InputStream input,
+            final OutputStream output,
+            final Predicate<Long> predicate
+    )
+            throws IOException
+    {
+        return copyWhilePredicate( input, output, 4 * 1024, predicate, null );
+    }
+
+    public static long copyWhilePredicate(
+            final InputStream input,
+            final OutputStream output,
+            final int bufferSize,
+            final Predicate<Long> predicate,
+            final ConditionalTaskExecutor condtionalTaskExecutor
+    )
             throws IOException
     {
-        final int bufferSize = 4 * 1024;
         final byte[] buffer = new byte[ bufferSize ];
         long bytesCopied;
         long totalCopied = 0;
@@ -336,6 +353,10 @@ public class JavaHelper
             {
                 totalCopied += bytesCopied;
             }
+            if ( condtionalTaskExecutor != null )
+            {
+                condtionalTaskExecutor.conditionallyExecuteTask();
+            }
             if ( !predicate.test( bytesCopied ) )
             {
                 return totalCopied;
@@ -469,7 +490,24 @@ public class JavaHelper
                 ) );
     }
 
-
+    public static ExecutorService makeBackgroundExecutor(
+            final PwmApplication pwmApplication,
+            final Class clazz
+    )
+    {
+        final ThreadPoolExecutor executor = new ThreadPoolExecutor(
+                1,
+                1,
+                10, TimeUnit.SECONDS,
+                new LinkedBlockingQueue<>(),
+                JavaHelper.makePwmThreadFactory(
+                        JavaHelper.makeThreadName( pwmApplication, clazz ) + "-",
+                        true
+                ) );
+        executor.allowCoreThreadTimeOut( true );
+        return executor;
+    }
+    
     /**
      * Copy of {@link ThreadInfo#toString()} but with the MAX_FRAMES changed from 8 to 256.
      * @param threadInfo thread information

server/src/main/java/password/pwm/util/java/StringUtil.java

@@ -183,6 +183,13 @@ public abstract class StringUtil
         return "";
     }
 
+    public static String formatDiskSizeforDebug( final long diskSize )
+    {
+        return diskSize == 0
+                ? "0"
+                : PwmNumberFormat.forDefaultLocale().format( diskSize ) + " (" + formatDiskSize( diskSize ) + ")";
+    }
+
     public static String formatDiskSize( final long diskSize )
     {
         final float count = 1000;

server/src/main/java/password/pwm/util/localdb/AbstractJDBCLocalDB.java

@@ -470,7 +470,7 @@ public abstract class AbstractJDBCLocalDB implements LocalDBProvider
         return true;
     }
 
-    public int size( final LocalDB.DB db )
+    public long size( final LocalDB.DB db )
             throws LocalDBException
     {
         preCheck( false );

server/src/main/java/password/pwm/util/localdb/LocalDB.java

@@ -102,7 +102,7 @@ public interface LocalDB
             throws LocalDBException;
 
     @ReadOperation
-    int size( DB db )
+    long size( DB db )
             throws LocalDBException;
 
     @WriteOperation

server/src/main/java/password/pwm/util/localdb/LocalDBAdaptor.java

@@ -26,23 +26,16 @@ import password.pwm.PwmApplication;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.svc.stats.EpsStatistic;
-import password.pwm.util.java.TimeDuration;
-import password.pwm.util.logging.PwmLogger;
 
 import java.io.File;
 import java.io.Serializable;
 import java.util.Collection;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 
 public class LocalDBAdaptor implements LocalDB
 {
-    private static final PwmLogger LOGGER = PwmLogger.forClass( LocalDBAdaptor.class );
-
     private final LocalDBProvider innerDB;
 
-    private final SizeCacheManager sizeCacheManager;
     private final PwmApplication pwmApplication;
 
     LocalDBAdaptor( final LocalDBProvider innerDB, final PwmApplication pwmApplication )
@@ -53,15 +46,6 @@ public class LocalDBAdaptor implements LocalDB
             throw new IllegalArgumentException( "innerDB can not be null" );
         }
 
-        if ( innerDB.flags().contains( LocalDBProvider.Flag.SlowSizeOperations ) )
-        {
-            sizeCacheManager = new SizeCacheManager();
-        }
-        else
-        {
-            sizeCacheManager = null;
-        }
-
         this.innerDB = innerDB;
 
     }
@@ -83,7 +67,7 @@ public class LocalDBAdaptor implements LocalDB
         ParameterValidator.validateKeyValue( key );
 
         final boolean value = innerDB.contains( db, key );
-        markRead( 1 );
+        markRead();
         return value;
     }
 
@@ -94,7 +78,7 @@ public class LocalDBAdaptor implements LocalDB
         ParameterValidator.validateKeyValue( key );
 
         final String value = innerDB.get( db, key );
-        markRead( 1 );
+        markRead();
         return value;
     }
 
@@ -107,54 +91,7 @@ public class LocalDBAdaptor implements LocalDB
     public LocalDBIterator<String> iterator( final DB db ) throws LocalDBException
     {
         ParameterValidator.validateDBValue( db );
-        final LocalDBIterator<String> innerIterator = innerDB.iterator( db );
-        return new SizeIterator<String>( db, innerIterator );
-    }
-
-    private class SizeIterator<K> implements LocalDBIterator<String>
-    {
-        private final LocalDBIterator<String> innerIterator;
-        private final DB db;
-        private String key;
-
-        SizeIterator( final DB db, final LocalDBIterator<String> innerIterator )
-        {
-            this.innerIterator = innerIterator;
-            this.db = db;
-        }
-
-        public boolean hasNext( )
-        {
-            return innerIterator.hasNext();
-        }
-
-        public String next( )
-        {
-            key = innerIterator.next();
-            return key;
-        }
-
-        public void remove( )
-        {
-            innerIterator.remove();
-            if ( sizeCacheManager != null )
-            {
-                try
-                {
-                    sizeCacheManager.decrementSize( db );
-                }
-                catch ( Exception e )
-                {
-                    throw new RuntimeException( e );
-                }
-            }
-        }
-
-        @Override
-        public void close( )
-        {
-            innerIterator.close();
-        }
+        return innerDB.iterator( db );
     }
 
     public Map<String, Serializable> debugInfo( )
@@ -185,17 +122,7 @@ public class LocalDBAdaptor implements LocalDB
             }
         }
 
-        try
-        {
-            innerDB.putAll( db, keyValueMap );
-        }
-        finally
-        {
-            if ( sizeCacheManager != null )
-            {
-                sizeCacheManager.clearSize( db );
-            }
-        }
+        innerDB.putAll( db, keyValueMap );
 
         markWrite( keyValueMap.size() );
     }
@@ -208,13 +135,6 @@ public class LocalDBAdaptor implements LocalDB
         ParameterValidator.validateValueValue( value );
 
         final boolean preExisting = innerDB.put( db, key, value );
-        if ( !preExisting )
-        {
-            if ( sizeCacheManager != null )
-            {
-                sizeCacheManager.incrementSize( db );
-            }
-        }
 
         markWrite( 1 );
         return preExisting;
@@ -228,14 +148,6 @@ public class LocalDBAdaptor implements LocalDB
         ParameterValidator.validateValueValue( value );
 
         final boolean success = innerDB.putIfAbsent( db, key, value );
-        if ( success )
-        {
-            if ( sizeCacheManager != null )
-            {
-                sizeCacheManager.incrementSize( db );
-            }
-        }
-
         markWrite( 1 );
         return success;
     }
@@ -247,14 +159,6 @@ public class LocalDBAdaptor implements LocalDB
         ParameterValidator.validateKeyValue( key );
 
         final boolean result = innerDB.remove( db, key );
-        if ( result )
-        {
-            if ( sizeCacheManager != null )
-            {
-                sizeCacheManager.decrementSize( db );
-            }
-        }
-
         markWrite( 1 );
         return result;
     }
@@ -281,17 +185,7 @@ public class LocalDBAdaptor implements LocalDB
 
         if ( keys.size() > 1 )
         {
-            try
-            {
-                innerDB.removeAll( db, keys );
-            }
-            finally
-            {
-                if ( sizeCacheManager != null )
-                {
-                    sizeCacheManager.clearSize( db );
-                }
-            }
+            innerDB.removeAll( db, keys );
         }
         else
         {
@@ -304,135 +198,24 @@ public class LocalDBAdaptor implements LocalDB
         markWrite( keys.size() );
     }
 
-    public int size( final DB db ) throws LocalDBException
+    public long size( final DB db ) throws LocalDBException
     {
         ParameterValidator.validateDBValue( db );
-        if ( sizeCacheManager != null )
-        {
-            return sizeCacheManager.getSizeForDB( db, innerDB );
-        }
-        else
-        {
-            return innerDB.size( db );
-        }
+        return innerDB.size( db );
     }
 
     @WriteOperation
     public void truncate( final DB db ) throws LocalDBException
     {
         ParameterValidator.validateDBValue( db );
-        try
-        {
             innerDB.truncate( db );
-        }
-        finally
-        {
-            if ( sizeCacheManager != null )
-            {
-                sizeCacheManager.clearSize( db );
-            }
-        }
     }
 
     public Status status( )
     {
-        if ( innerDB == null )
-        {
-            return Status.CLOSED;
-        }
-
         return innerDB.getStatus();
     }
 
-    private static class SizeCacheManager
-    {
-        private static final Integer CACHE_DIRTY = -1;
-        private static final Integer CACHE_WORKING = -2;
-
-        private final ConcurrentMap<DB, Integer> sizeCache = new ConcurrentHashMap<>();
-
-        private SizeCacheManager( )
-        {
-            for ( final DB db : DB.values() )
-            {
-                sizeCache.put( db, CACHE_DIRTY );
-            }
-        }
-
-        private void incrementSize( final DB db )
-        {
-            modifySize( db, +1 );
-        }
-
-        private void decrementSize( final DB db )
-        {
-            modifySize( db, -1 );
-        }
-
-        private void modifySize( final DB db, final int amount )
-        {
-            // retrieve the current cache size.
-            final Integer cachedSize = sizeCache.get( db );
-
-            //update the cached value only if there is a meaningful value cached.
-            if ( cachedSize >= 0 )
-            {
-
-                // calculate the new value
-                final int newSize = cachedSize + amount;
-
-                // replace the cached value with the new value, only if it hasn't been touched by another thread since it was
-                // retrieved from the cache a few lines ago.
-                if ( !sizeCache.replace( db, cachedSize, newSize ) )
-                {
-
-                    // the cache was modified by some other thread, and so is no longer accurate.  Mark it dirty.
-                    clearSize( db );
-                }
-            }
-
-        }
-
-        private void clearSize( final DB db )
-        {
-            sizeCache.put( db, CACHE_DIRTY );
-        }
-
-        private int getSizeForDB( final DB db, final LocalDBProvider localDBProvider ) throws LocalDBException
-        {
-            // read the cached size out of the cache store
-            final Integer cachedSize = sizeCache.get( db );
-
-            if ( cachedSize != null && cachedSize >= 0 )
-            {
-                // if there is a good cache value and its not dirty (-1) or being populated by another thread (-2)
-                return cachedSize;
-            }
-
-            final long beginTime = System.currentTimeMillis();
-
-            // mark the cache as population in progress
-            sizeCache.put( db, CACHE_WORKING );
-
-            // read the "real" value.  this is the line that might take a long time
-            final int theSize = localDBProvider.size( db );
-            final TimeDuration timeDuration = TimeDuration.fromCurrent( beginTime );
-
-            // so long as nothing else has touched the cache (perhaps another thread populated it, or someone else marked it dirty, then
-            // go ahead and update it.
-            final boolean savedInCache = sizeCache.replace( db, CACHE_WORKING, theSize );
-
-            final StringBuilder debugMsg = new StringBuilder();
-            debugMsg.append( "performed real size lookup of " ).append( theSize ).append( " for " ).append( db );
-            debugMsg.append( ": " ).append( timeDuration.asCompactString() );
-            debugMsg.append( savedInCache ? ", cached" : ", not cached" );
-            LOGGER.debug( debugMsg );
-
-            return theSize;
-        }
-
-    }
-
     private static class ParameterValidator
     {
         private static void validateDBValue( final LocalDB.DB db )
@@ -476,13 +259,13 @@ public class LocalDBAdaptor implements LocalDB
         }
     }
 
-    private void markRead( final int events )
+    private void markRead()
     {
         if ( pwmApplication != null )
         {
             if ( pwmApplication.getStatisticsManager() != null )
             {
-                pwmApplication.getStatisticsManager().updateEps( EpsStatistic.PWMDB_READS, events );
+                pwmApplication.getStatisticsManager().updateEps( EpsStatistic.PWMDB_READS, 1 );
             }
         }
     }

server/src/main/java/password/pwm/util/localdb/LocalDBDataStore.java

@@ -102,7 +102,7 @@ public class LocalDBDataStore implements DataStore
         localDB.remove( db, key );
     }
 
-    public int size( ) throws PwmDataStoreException
+    public long size( ) throws PwmDataStoreException
     {
         return localDB.size( db );
     }

server/src/main/java/password/pwm/util/localdb/LocalDBProvider.java

@@ -82,7 +82,7 @@ public interface LocalDBProvider
             throws LocalDBException;
 
     @LocalDB.ReadOperation
-    int size( LocalDB.DB db )
+    long size( LocalDB.DB db )
             throws LocalDBException;
 
     @LocalDB.WriteOperation

server/src/main/java/password/pwm/util/localdb/LocalDBUtility.java

@@ -233,11 +233,11 @@ public class LocalDBUtility
 
         final Instant startTime = Instant.now();
         final TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(
-                new TransactionSizeCalculator.SettingsBuilder()
-                        .setDurationGoal( TimeDuration.of( 100, TimeDuration.Unit.MILLISECONDS ) )
-                        .setMinTransactions( 50 )
-                        .setMaxTransactions( 5 * 1000 )
-                        .createSettings()
+                TransactionSizeCalculator.Settings.builder()
+                        .durationGoal( TimeDuration.of( 100, TimeDuration.Unit.MILLISECONDS ) )
+                        .minTransactions( 50 )
+                        .maxTransactions( 5 * 1000 )
+                        .build()
         );
 
         final Map<LocalDB.DB, Map<String, String>> transactionMap = new HashMap<>();

server/src/main/java/password/pwm/util/localdb/MemoryLocalDB.java

@@ -158,7 +158,7 @@ public class MemoryLocalDB implements LocalDBProvider
         return null != map.remove( key );
     }
 
-    public int size( final LocalDB.DB db )
+    public long size( final LocalDB.DB db )
             throws LocalDBException
     {
         opertationPreCheck();

server/src/main/java/password/pwm/util/localdb/WorkQueueProcessor.java

@@ -127,6 +127,7 @@ public final class WorkQueueProcessor<W extends Serializable>
                     new ArrayBlockingQueue<>( settings.getPreThreads() ),
                     threadFactory
             );
+            executorService.allowCoreThreadTimeOut( true );
         }
     }
 

server/src/main/java/password/pwm/util/localdb/XodusLocalDB.java

@@ -175,6 +175,7 @@ public class XodusLocalDB implements LocalDBProvider
         environmentConfig.setEnvCloseForcedly( true );
         environmentConfig.setMemoryUsage( 50 * 1024 * 1024 );
         environmentConfig.setEnvGatherStatistics( true );
+        environmentConfig.setGcUtilizationFromScratch( true );
 
         for ( final Map.Entry<String, String> entry : initParameters.entrySet() )
         {
@@ -196,13 +197,13 @@ public class XodusLocalDB implements LocalDBProvider
     }
 
     @Override
-    public int size( final LocalDB.DB db ) throws LocalDBException
+    public long size( final LocalDB.DB db ) throws LocalDBException
     {
         checkStatus( false );
         return environment.computeInReadonlyTransaction( transaction ->
         {
             final Store store = getStore( db );
-            return ( int ) store.count( transaction );
+            return store.count( transaction );
         } );
     }
 

server/src/main/java/password/pwm/util/operations/CrService.java

@@ -54,7 +54,7 @@ import password.pwm.health.HealthRecord;
 import password.pwm.ldap.LdapOperationsHelper;
 import password.pwm.ldap.LdapPermissionTester;
 import password.pwm.svc.PwmService;
-import password.pwm.svc.wordlist.WordlistManager;
+import password.pwm.svc.wordlist.WordlistService;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.TimeDuration;
@@ -302,7 +302,7 @@ public class CrService implements PwmService
 
         {
             // check responses against wordlist
-            final WordlistManager wordlistManager = pwmApplication.getWordlistManager();
+            final WordlistService wordlistManager = pwmApplication.getWordlistManager();
             if ( wordlistManager.status() == PwmService.STATUS.OPEN )
             {
                 for ( final Map.Entry<Challenge, String> entry : responseMap.entrySet() )

server/src/main/java/password/pwm/util/secure/PwmRandom.java

@@ -52,6 +52,18 @@ public class PwmRandom extends SecureRandom
         return internalRand.nextLong();
     }
 
+    public long nextLong( final long n )
+    {
+        long randomLong;
+        do
+        {
+            randomLong = internalRand.nextLong();
+        }
+        while ( randomLong < 0 );
+
+        return randomLong % n;
+    }
+
     public int nextInt( )
     {
         return internalRand.nextInt();

server/src/main/java/password/pwm/util/secure/X509Utils.java

@@ -313,7 +313,8 @@ public abstract class X509Utils
                 }
                 if ( !certTrusted )
                 {
-                    final String errorMsg = "server certificate {subject=" + loopCert.getSubjectDN().getName() + "} does not match a certificate in the configuration trust store.";
+                    final String errorMsg = "server certificate {subject=" + loopCert.getSubjectDN().getName() + "} does not match a certificate in the"
+                            + PwmConstants.PWM_APP_NAME + " configuration trust store.";
                     throw new CertificateException( errorMsg );
                 }
                 //LOGGER.trace("trusting configured certificate: " + makeDebugText(loopCert));

server/src/main/resources/password/pwm/AppProperty.properties

@@ -320,6 +320,11 @@ urlshortener.url.regex=(https?://([^:@]+(:[^@]+)?@)?([a-zA-Z0-9.]+|d{1,3}.d{1,3}
 wordlist.builtin.path=/WEB-INF/wordlist.zip
 wordlist.maxCharLength=64
 wordlist.minCharLength=2
+wordlist.import.autoImportRecheckSeconds=432000
+wordlist.import.durationGoalMS=1000
+wordlist.import.minTransactions=10
+wordlist.import.maxTransactions=200000
+wordlist.inspector.frequencySeconds=300
 ws.restClient.pwRule.haltOnError=true
 ws.restServer.signing.form.timeoutSeconds=120
 ws.restServer.statistics.defaultHistoryDays=7

server/src/main/resources/password/pwm/config/PwmSetting.xml

@@ -4003,6 +4003,7 @@
     <setting hidden="false" key="https.server.tls.protocols" level="1">
         <default>
             <value>TLS_1_2</value>
+            <value>TLS_1_3</value>
         </default>
         <options>
             <option value="SSL_2_0">SSL v2.0</option>
@@ -4010,11 +4011,12 @@
             <option value="TLS_1_0">TLS v1.0</option>
             <option value="TLS_1_1">TLS v1.1</option>
             <option value="TLS_1_2">TLS v1.2</option>
+            <option value="TLS_1_3">TLS v1.3</option>
         </options>
     </setting>
     <setting hidden="false" key="https.server.tls.ciphers" level="1">
         <default>
-            <value>TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_DHE_RSA_WITH_AES_128_GCM_SHA256,TLS_DHE_DSS_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_SHA256,TLS_ECDHE_RSA_WITH_AES_128_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_SHA,TLS_DHE_RSA_WITH_AES_128_SHA256,TLS_DHE_RSA_WITH_AES_128_SHA,TLS_DHE_DSS_WITH_AES_128_SHA256</value>
+            <value/>
         </default>
     </setting>
     <setting hidden="false" key="pwm.wordlist.location" level="1">

server/src/main/resources/password/pwm/i18n/Error.properties

@@ -164,6 +164,7 @@ Error_FileTypeIncorrect=The file type is not correct.
 Error_FileTooLarge=The file is too large.
 Error_ClusterServiceError=An error occurred with the cluster service: %1%.   Check the log files for more information.
 Error_RemoteErrorValue=Remote Error: %1%
+Error_WordlistImportError=An error occured importing the wordlist: %1%
 
 Error_ConfigUploadSuccess=File uploaded successfully
 Error_ConfigUploadFailure=File failed to upload.

webapp/src/main/webapp/WEB-INF/jsp/configmanager-wordlists.jsp

@@ -75,7 +75,7 @@
                     </button>
                 </td>
                 <td class="buttoncell">
-                    <button class="menubutton" id="MenuItem_ClearSeedlist" style="visibility: hidden;">
+                    <button class="menubutton" id="MenuItem_ClearSeedlist" disabled="disabled">
                         <pwm:if test="<%=PwmIfTest.showIcons%>"><span class="btn-icon pwm-icon pwm-icon-trash"></span></pwm:if>
                         Clear Seed List
                     </button>

webapp/src/main/webapp/public/resources/js/configmanager.js

@@ -376,6 +376,7 @@ PWM_CONFIG.initConfigManagerWordlistPage = function() {
                         var url = 'wordlists?processAction=clearWordlist&wordlist=' + type;
                         var loadFunction = function (data) {
                             PWM_MAIN.showDialog({
+                                title: PWM_MAIN.showString('Title_Success'),
                                 text: data['successMessage'], okAction: function () {
                                     PWM_MAIN.showWaitDialog({
                                         loadFunction: function(){

webapp/src/main/webapp/public/resources/js/uilibrary.js

@@ -795,7 +795,8 @@ UILibrary.displayElementsToTableContents = function(fields) {
     var htmlTable = '';
     for (var field in fields) {(function(field){
         var fieldData = fields[field];
-        htmlTable += '<tr><td>' + fieldData['label'] + '</td><td><span id="report_status_' + fieldData['key']  + '"</tr>';
+        var classValue = fieldData['type'] === 'timestamp' ? 'timestamp' : '';
+        htmlTable += '<tr><td>' + fieldData['label'] + '</td><td><span class="' + classValue + '" id="report_status_' + fieldData['key']  + '"</tr>';
     }(field)); }
     return htmlTable;
 };