seedlist refactoring

server/src/main/java/password/pwm/svc/wordlist/WordlistBucket.java

@@ -39,18 +39,15 @@ import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicLong;
 
 class WordlistBucket
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( WordlistBucket.class );
-    private static final String KEY_LAST_ISSUED_KEY = "_______lastKey_";
 
     private final PwmApplication pwmApplication;
     private final WordlistConfiguration wordlistConfiguration;
     private final LocalDB.DB db;
     private final WordlistType type;
-    private final AtomicLong seedlistTopKey = new AtomicLong(  );
 
 
     WordlistBucket(
@@ -64,14 +61,6 @@ class WordlistBucket
         this.wordlistConfiguration = wordlistConfiguration;
         this.db = wordlistConfiguration.getDb();
         this.type = type;
-
-        final String valueOfLastKey = pwmApplication.getLocalDB().get( db, KEY_LAST_ISSUED_KEY );
-
-        seedlistTopKey.set(
-                StringUtil.isEmpty( valueOfLastKey )
-                        ? 0
-                        : seedlistKeyToLong( valueOfLastKey )
-        );
     }
 
     boolean containsWord( final String word ) throws LocalDBException
@@ -137,9 +126,17 @@ class WordlistBucket
         throw new PwmUnrecoverableException( PwmError.ERROR_INTERNAL, "seedlist word not available" );
     }
 
-    void addWords( final Collection<String> words ) throws LocalDBException
+    void addWords( final Collection<String> words, final AbstractWordlist abstractWordlist )
+            throws LocalDBException
     {
-        pwmApplication.getLocalDB().putAll( db, getWriteTxnForValue( words ) );
+        final WordlistStatus initialStatus = abstractWordlist.readWordlistStatus();
+        final MutableLongIncrementer valueIncrementer = new MutableLongIncrementer( initialStatus.getValueCount() );
+        pwmApplication.getLocalDB().putAll( db, getWriteTxnForValue( words, valueIncrementer ) );
+        if ( initialStatus.getValueCount() != valueIncrementer.get() )
+        {
+            final WordlistStatus incrementedStatus = initialStatus.toBuilder().valueCount( valueIncrementer.get() ).build();
+            abstractWordlist.writeWordlistStatus( incrementedStatus );
+        }
     }
 
     long size() throws LocalDBException
@@ -150,11 +147,10 @@ class WordlistBucket
 
     void clear() throws LocalDBException
     {
-        seedlistTopKey.set( 0 );
         pwmApplication.getLocalDB().truncate( db );
     }
 
-    private Map<String, String> getWriteTxnForValue( final Collection<String> words ) throws LocalDBException
+    private Map<String, String> getWriteTxnForValue( final Collection<String> words, final MutableLongIncrementer valueIncrementer ) throws LocalDBException
     {
         switch ( type )
         {
@@ -166,10 +162,9 @@ class WordlistBucket
                     final String normalizedWord = normalizeWord( word );
                     if ( !StringUtil.isEmpty( normalizedWord ) )
                     {
-                        final long nextLong = seedlistTopKey.incrementAndGet();
+                        final long nextLong = valueIncrementer.getAndIncrement();
                         final String nextKey = seedlistLongToKey( nextLong );
                         returnSet.put( nextKey, normalizedWord );
-                        returnSet.put( KEY_LAST_ISSUED_KEY, nextKey );
                     }
                 }
                 return Collections.unmodifiableMap( returnSet );
@@ -183,6 +178,7 @@ class WordlistBucket
                     final String normalizedWord = normalizeWord( word );
                     if ( !StringUtil.isEmpty( normalizedWord ) )
                     {
+                        valueIncrementer.getAndIncrement();
                         returnSet.put( normalizedWord, "" );
                     }
                 }
@@ -259,4 +255,25 @@ class WordlistBucket
     {
         return Long.toString( longValue, 36 );
     }
+
+    public static class MutableLongIncrementer
+    {
+        private long value;
+
+        MutableLongIncrementer( final long value )
+        {
+            this.value = value;
+        }
+
+        public long getAndIncrement()
+        {
+            value++;
+            return value;
+        }
+
+        public long get()
+        {
+            return value;
+        }
+    }
 }

server/src/main/java/password/pwm/svc/wordlist/WordlistImporter.java

@@ -28,6 +28,7 @@ import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.TransactionSizeCalculator;
 import password.pwm.util.java.ConditionalTaskExecutor;
+import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.Percent;
 import password.pwm.util.java.PwmNumberFormat;
 import password.pwm.util.java.StringUtil;
@@ -170,7 +171,7 @@ class WordlistImporter implements Runnable
         rootWordlist.setActivity( Wordlist.Activity.Importing );
 
         final ConditionalTaskExecutor metaUpdater = new ConditionalTaskExecutor(
-                () -> rootWordlist.writeWordlistStatus( WordlistStatus.builder()
+                () -> rootWordlist.writeWordlistStatus( rootWordlist.readWordlistStatus().toBuilder()
                         .sourceType( sourceType )
                         .storeDate( Instant.now() )
                         .remoteInfo( wordlistSourceInfo )
@@ -180,7 +181,7 @@ class WordlistImporter implements Runnable
         );
 
         final ConditionalTaskExecutor debugOutputter = new ConditionalTaskExecutor(
-                () -> getLogger().debug( () -> makeStatString() ),
+                () -> getLogger().debug( this::makeStatString ),
                 new ConditionalTaskExecutor.TimeDurationPredicate( AbstractWordlist.DEBUG_OUTPUT_FREQUENCY )
         );
 
@@ -246,7 +247,7 @@ class WordlistImporter implements Runnable
         final long startTime = System.currentTimeMillis();
 
         //add the elements
-        wordlistBucket.addWords( bufferedWords );
+        wordlistBucket.addWords( bufferedWords, rootWordlist );
 
         if ( cancelFlag.getAsBoolean() )
         {
@@ -272,16 +273,16 @@ class WordlistImporter implements Runnable
         getLogger().info( () -> "population complete, added " + wordlistSize
                 + " total words in " + TimeDuration.compactFromCurrent( startTime ) );
 
-        {
-            final WordlistStatus wordlistStatus = WordlistStatus.builder()
-                    .remoteInfo( wordlistSourceInfo )
-                    .storeDate( Instant.now() )
-                    .sourceType( sourceType )
-                    .completed( true )
-                    .bytes( zipFileReader.getByteCount() )
-                    .build();
-            rootWordlist.writeWordlistStatus( wordlistStatus );
-        }
+        rootWordlist.writeWordlistStatus( rootWordlist.readWordlistStatus().toBuilder()
+                .remoteInfo( wordlistSourceInfo )
+                .storeDate( Instant.now() )
+                .sourceType( sourceType )
+                .completed( true )
+                .bytes( zipFileReader.getByteCount() )
+                .build()
+        );
+
+        getLogger().debug( () -> "final post-population status: " + JsonUtil.serialize( rootWordlist.readWordlistStatus() ) );
     }
 
     private PwmLogger getLogger()

server/src/main/java/password/pwm/svc/wordlist/WordlistStatus.java

@@ -32,13 +32,15 @@ import java.time.Instant;
 @Builder( toBuilder = true )
 public class WordlistStatus implements Serializable
 {
-    public static final int CURRENT_VERSION = 4;
+    public static final int CURRENT_VERSION = 5;
 
     @Builder.Default
     private int version = CURRENT_VERSION;
+
     private boolean completed;
     private WordlistSourceType sourceType;
     private Instant storeDate;
     private WordlistSourceInfo remoteInfo;
     private long bytes;
+    private long valueCount;
 }