app-properties for wordlist import pausing

pom.xml

@@ -82,12 +82,13 @@
             <properties>
                 <jarsigner.skip>false</jarsigner.skip>
             </properties>
-        </profile><profile>
-        <id>skip-owasp</id>
-        <properties>
-            <owasp.skip>true</owasp.skip>
-        </properties>
-    </profile>
+        </profile>
+        <profile>
+            <id>skip-owasp</id>
+            <properties>
+                <owasp.skip>true</owasp.skip>
+            </properties>
+        </profile>
     </profiles>
 
     <build>

server/src/main/java/password/pwm/AppProperty.java

@@ -385,6 +385,8 @@ public enum AppProperty
     WORDLIST_IMPORT_MAX_TRANSACTIONS                ( "wordlist.import.maxTransactions" ),
     WORDLIST_IMPORT_MAX_CHARS_TRANSACTIONS          ( "wordlist.import.maxCharsTransactions" ),
     WORDLIST_IMPORT_LINE_COMMENTS                   ( "wordlist.import.lineComments" ),
+    WORDLIST_IMPORT_PAUSE_DURATION_MS               ( "wordlist.import.pauseDurationMs" ),
+    WORDLIST_IMPORT_PAUSE_FREQUENCY_MS              ( "wordlist.import.pauseFrequencyMs" ),
     WORDLIST_INSPECTOR_FREQUENCY_SECONDS            ( "wordlist.inspector.frequencySeconds" ),
     WORDLIST_TEST_MODE                              ( "wordlist.testMode" ),
     WORDLIST_BUCKET_CHECK_TIME_WARNING_MS           ( "wordlist.bucket.checkTimeWarningMs" ),

server/src/main/java/password/pwm/config/AppConfig.java

@@ -39,6 +39,7 @@ import password.pwm.util.PasswordData;
 import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.LazySupplier;
 import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogLevel;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.secure.PwmSecurityKey;
@@ -142,6 +143,26 @@ public class AppConfig implements SettingReader
         return Boolean.parseBoolean( readAppProperty( appProperty ) );
     }
 
+    public TimeDuration readDurationAppProperty( final AppProperty appProperty )
+    {
+        final TimeDuration.Unit unit;
+        final String lcasePropName = appProperty.getKey().toLowerCase( PwmConstants.DEFAULT_LOCALE );
+        if ( lcasePropName.endsWith( "ms" ) )
+        {
+            unit = TimeDuration.Unit.MILLISECONDS;
+        }
+        else if ( lcasePropName.endsWith( "seconds" ) )
+        {
+            unit = TimeDuration.Unit.SECONDS;
+        }
+        else
+        {
+            throw new IllegalStateException( "can't read appProperty '" + appProperty.getKey() + "' as duration, unknown time unit" );
+        }
+
+        return TimeDuration.of( Long.parseLong( readAppProperty( appProperty ) ), unit );
+    }
+
     public Map<AppProperty, String> readAllNonDefaultAppProperties( )
     {
         final LinkedHashMap<AppProperty, String> nonDefaultProperties = new LinkedHashMap<>();

server/src/main/java/password/pwm/svc/wordlist/WordlistConfiguration.java

@@ -71,6 +71,9 @@ public class WordlistConfiguration implements Serializable
     private final TimeDuration importDurationGoal;
     private final TimeDuration bucketCheckLogWarningTimeout;
 
+    private final TimeDuration importPauseDuration;
+    private final TimeDuration importPauseFrequency;
+
     private final int importMinTransactions;
     private final int importMaxTransactions;
     private final long importMaxChars;
@@ -126,22 +129,16 @@ public class WordlistConfiguration implements Serializable
                 .minWordSize( Integer.parseInt( appConfig.readAppProperty( AppProperty.WORDLIST_CHAR_LENGTH_MIN ) ) )
                 .maxWordSize( Integer.parseInt( appConfig.readAppProperty( AppProperty.WORDLIST_CHAR_LENGTH_MAX ) ) )
                 .warmupLookups( Integer.parseInt( appConfig.readAppProperty( AppProperty.WORDLIST_WARMUP_COUNT ) ) )
-                .bucketCheckLogWarningTimeout( TimeDuration.of(
-                        Long.parseLong( appConfig.readAppProperty( AppProperty.WORDLIST_BUCKET_CHECK_WARNING_TIMEOUT_MS ) ),
-                        TimeDuration.Unit.MILLISECONDS ) )
-                .autoImportRecheckDuration( TimeDuration.of(
-                        Long.parseLong( appConfig.readAppProperty( AppProperty.WORDLIST_IMPORT_AUTO_IMPORT_RECHECK_SECONDS ) ),
-                        TimeDuration.Unit.SECONDS ) )
-                .importDurationGoal( TimeDuration.of(
-                        Long.parseLong( appConfig.readAppProperty( AppProperty.WORDLIST_IMPORT_DURATION_GOAL_MS ) ),
-                        TimeDuration.Unit.MILLISECONDS ) )
+                .bucketCheckLogWarningTimeout( appConfig.readDurationAppProperty( AppProperty.WORDLIST_BUCKET_CHECK_WARNING_TIMEOUT_MS ) )
+                .autoImportRecheckDuration( appConfig.readDurationAppProperty( AppProperty.WORDLIST_IMPORT_AUTO_IMPORT_RECHECK_SECONDS ) )
+                .importDurationGoal( appConfig.readDurationAppProperty( AppProperty.WORDLIST_IMPORT_DURATION_GOAL_MS ) )
                 .importMinTransactions( Integer.parseInt( appConfig.readAppProperty( AppProperty.WORDLIST_IMPORT_MIN_TRANSACTIONS ) ) )
                 .importMaxTransactions( Integer.parseInt( appConfig.readAppProperty( AppProperty.WORDLIST_IMPORT_MAX_TRANSACTIONS ) ) )
                 .importMaxChars( JavaHelper.silentParseLong( appConfig.readAppProperty( AppProperty.WORDLIST_IMPORT_MAX_CHARS_TRANSACTIONS ), 10_1024_1024 ) )
-                .inspectorFrequency( TimeDuration.of(
-                        Long.parseLong( appConfig.readAppProperty( AppProperty.WORDLIST_INSPECTOR_FREQUENCY_SECONDS ) ),
-                        TimeDuration.Unit.SECONDS ) )
+                .inspectorFrequency( appConfig.readDurationAppProperty( AppProperty.WORDLIST_INSPECTOR_FREQUENCY_SECONDS ) )
                 .importMinFreeSpace( JavaHelper.silentParseLong( appConfig.readAppProperty( AppProperty.WORDLIST_IMPORT_MIN_FREE_SPACE ), 100_000_000 ) )
+                .importPauseDuration( appConfig.readDurationAppProperty( AppProperty.WORDLIST_IMPORT_PAUSE_DURATION_MS ) )
+                .importPauseFrequency( appConfig.readDurationAppProperty( AppProperty.WORDLIST_IMPORT_PAUSE_FREQUENCY_MS ) )
                 .build();
     }
 

server/src/main/java/password/pwm/svc/wordlist/WordlistImporter.java

@@ -61,6 +61,7 @@ class WordlistImporter implements Runnable
     private final WordlistSourceInfo wordlistSourceInfo;
     private final BooleanSupplier cancelFlag;
     private final StatisticAverageBundle<StatKey> importStatistics = new StatisticAverageBundle<>( StatKey.class );
+    private final ConditionalTaskExecutor pauseTimer;
 
     private long charsInBuffer;
     private ErrorInformation exitError;
@@ -69,7 +70,7 @@ class WordlistImporter implements Runnable
     private final Map<WordType, LongAdder> seenWordTypes = new EnumMap<>( WordType.class );
     private boolean completed;
 
-    enum StatKey
+    private enum StatKey
     {
         charsPerTransaction( DebugKey.CharsPerTxn ),
         wordsPerTransaction( DebugKey.WordsPerTxn ),
@@ -129,13 +130,20 @@ class WordlistImporter implements Runnable
 
         final WordlistConfiguration wordlistConfiguration = rootWordlist.getConfiguration();
 
-        transactionCalculator = new TransactionSizeCalculator(
+        this.transactionCalculator = new TransactionSizeCalculator(
                 TransactionSizeCalculator.Settings.builder()
                         .durationGoal( wordlistConfiguration.getImportDurationGoal() )
                         .minTransactions( wordlistConfiguration.getImportMinTransactions() )
                         .maxTransactions( wordlistConfiguration.getImportMaxTransactions() )
                         .build()
         );
+
+        {
+            final TimeDuration pauseDuration = wordlistConfiguration.getImportPauseDuration();
+            this.pauseTimer = ConditionalTaskExecutor.forPeriodicTask(
+                    pauseDuration::pause,
+                    wordlistConfiguration.getImportPauseFrequency() );
+        }
     }
 
     @Override
@@ -218,10 +226,6 @@ class WordlistImporter implements Runnable
                 () -> getLogger().debug( rootWordlist.getSessionLabel(), this::makeStatString ),
                 AbstractWordlist.DEBUG_OUTPUT_FREQUENCY );
 
-        final ConditionalTaskExecutor pauseTimer = ConditionalTaskExecutor.forPeriodicTask(
-                () -> TimeDuration.of( 100, TimeDuration.Unit.MILLISECONDS ).pause(),
-                TimeDuration.SECOND );
-
         try
         {
             debugOutputter.conditionallyExecuteTask();

server/src/main/resources/password/pwm/AppProperty.properties

@@ -361,6 +361,8 @@ wordlist.import.minFreeSpace=500000000
 wordlist.import.maxTransactions=100000
 wordlist.import.maxCharsTransactions=10485760
 wordlist.import.lineComments=!#comment:
+wordlist.import.pauseDurationMs=100
+wordlist.import.pauseFrequencyMs=1000
 wordlist.inspector.frequencySeconds=300
 wordlist.testMode=false
 wordlist.bucket.checkTimeWarningMs=1000