wordlist populator enhancements

server/src/main/java/password/pwm/http/servlet/configmanager/ConfigManagerWordlistServlet.java

@@ -253,7 +253,7 @@ public class ConfigManagerWordlistServlet extends AbstractPwmServlet
                             wordlistType.name() + "_sha1Hash",
                             DisplayElement.Type.string,
                             "SHA1 Checksum Hash",
-                            storedWordlistDataBean.getSha1hash() ) );
+                            storedWordlistDataBean.getRemoteInfo().getChecksum() ) );
                 }
                 if ( wordlist.getAutoImportError() != null )
                 {

server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java

@@ -49,7 +49,6 @@ import password.pwm.util.localdb.LocalDBException;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.secure.ChecksumInputStream;
 import password.pwm.util.secure.PwmHashAlgorithm;
-import password.pwm.util.secure.SecureEngine;
 import password.pwm.util.secure.X509Utils;
 
 import java.io.FileNotFoundException;
@@ -252,9 +251,16 @@ abstract class AbstractWordlist implements Wordlist, PwmService
 
     public int size( )
     {
-        if ( populator != null )
+        try
+        {
+            if ( populator != null && wlStatus != STATUS.CLOSED )
+            {
+                return localDB.size( this.getWordlistDB() );
+            }
+        }
+        catch ( LocalDBException e )
         {
-            return 0;
+            logger.debug( "error reading wordlist size: " + e.getMessage() );
         }
 
         return readMetadata().getSize();
@@ -262,6 +268,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
 
     public synchronized void close( )
     {
+        wlStatus = STATUS.CLOSED;
         if ( populator != null )
         {
             try
@@ -276,7 +283,6 @@ abstract class AbstractWordlist implements Wordlist, PwmService
         }
 
         executorService.shutdown();
-        wlStatus = STATUS.CLOSED;
         localDB = null;
     }
 
@@ -343,7 +349,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
         }
         else
         {
-            return new ServiceInfoBean( Collections.<DataStorageMethod>emptyList() );
+            return new ServiceInfoBean( Collections.emptyList() );
         }
     }
 
@@ -404,7 +410,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
     {
         try
         {
-            populationManager.populateImpl( inputStream, StoredWordlistDataBean.Source.User );
+            populationManager.populateImpl( null, inputStream, StoredWordlistDataBean.Source.User );
         }
         finally
         {
@@ -458,13 +464,19 @@ abstract class AbstractWordlist implements Wordlist, PwmService
 
             if ( autoImportUrlConfigured )
             {
-                final String remoteHash = readImportUrlHash();
-                if ( remoteHash != null )
+                final StoredWordlistDataBean.RemoteWordlistInfo remoteInfo = readRemoteWordlistInfo( autoImportInputStream() );
+                final StoredWordlistDataBean.RemoteWordlistInfo localInfo = readMetadata().getRemoteInfo();
+                if ( remoteInfo != null )
                 {
-                    if ( !remoteHash.equals( readMetadata().getSha1hash() ) )
+                    if ( !remoteInfo.equals( localInfo ) )
                     {
                         logger.debug( "auto-import url remote hash does not equal currently stored hash, will start auto-import" );
-                        populateAutoImport();
+                        populateAutoImport( remoteInfo );
+                    }
+                    else if ( remoteInfo.getBytes() > readMetadata().getBytes() )
+                    {
+                        logger.debug( "auto-import did not previously complete, will continue previous import" );
+                        populateAutoImport( remoteInfo );
                     }
                 }
 
@@ -498,37 +510,45 @@ abstract class AbstractWordlist implements Wordlist, PwmService
                 }
             }
 
-            boolean needsBuiltinPopulating = false;
-            if ( !readMetadata().isCompleted() )
-            {
-                needsBuiltinPopulating = true;
-                logger.debug( "wordlist stored in database does not have a completed load status, will load built-in wordlist" );
-            }
-            else if ( StoredWordlistDataBean.Source.BuiltIn == readMetadata().getSource() )
+            if ( wlStatus != STATUS.CLOSED )
             {
-                final String builtInWordlistHash = getBuiltInWordlistHash();
-                if ( !builtInWordlistHash.equals( readMetadata().getSha1hash() ) )
+                boolean needsBuiltinPopulating = false;
+                if ( !readMetadata().isCompleted() )
                 {
-                    logger.debug( "wordlist stored in database does not have match checksum with built-in wordlist file, will load built-in wordlist" );
                     needsBuiltinPopulating = true;
+                    logger.debug( "wordlist stored in database does not have a completed load status, will load built-in wordlist" );
+                }
+                else if ( StoredWordlistDataBean.Source.BuiltIn == readMetadata().getSource() )
+                {
+                    final StoredWordlistDataBean.RemoteWordlistInfo builtInWordlistHash = getBuiltInWordlistHash();
+                    if ( !builtInWordlistHash.equals( readMetadata().getRemoteInfo() ) )
+                    {
+                        logger.debug( "wordlist stored in database does not have match checksum with built-in wordlist file, will load built-in wordlist" );
+                        needsBuiltinPopulating = true;
+                    }
                 }
-            }
 
-            if ( !needsBuiltinPopulating )
-            {
-                return;
-            }
+                if ( !needsBuiltinPopulating )
+                {
+                    return;
+                }
 
-            this.populateBuiltIn();
+                this.populateBuiltIn();
+            }
         }
 
         protected void populateBuiltIn( )
                 throws IOException, PwmUnrecoverableException
         {
-            populateImpl( getBuiltInWordlist(), StoredWordlistDataBean.Source.BuiltIn );
+            final StoredWordlistDataBean.RemoteWordlistInfo remoteWordlistInfo = readRemoteWordlistInfo( getBuiltInWordlist() );
+            populateImpl( remoteWordlistInfo, getBuiltInWordlist(), StoredWordlistDataBean.Source.BuiltIn );
         }
 
-        private void populateImpl( final InputStream inputStream, final StoredWordlistDataBean.Source source )
+        private void populateImpl(
+                final StoredWordlistDataBean.RemoteWordlistInfo remoteWordlistInfo,
+                final InputStream inputStream,
+                final StoredWordlistDataBean.Source source
+        )
                 throws IOException, PwmUnrecoverableException
         {
             if ( inputStream == null )
@@ -561,15 +581,17 @@ abstract class AbstractWordlist implements Wordlist, PwmService
                     }
                 }
 
+                if ( remoteWordlistInfo == null || !remoteWordlistInfo.equals( readMetadata().getRemoteInfo() ) )
                 {
                     // reset the wordlist metadata
                     final StoredWordlistDataBean storedWordlistDataBean = StoredWordlistDataBean.builder()
                             .source( source )
+                            .remoteInfo( remoteWordlistInfo )
                             .build();
                     writeMetadata( storedWordlistDataBean );
                 }
 
-                populator = new Populator( inputStream, source, AbstractWordlist.this, pwmApplication );
+                populator = new Populator( remoteWordlistInfo, inputStream, source, AbstractWordlist.this, pwmApplication );
                 populator.populate();
             }
             catch ( Exception e )
@@ -601,16 +623,16 @@ abstract class AbstractWordlist implements Wordlist, PwmService
             throw new PwmUnrecoverableException( new ErrorInformation( PwmError.ERROR_SERVICE_NOT_AVAILABLE, "unable to locate builtin wordlist file" ) );
         }
 
-        protected String getBuiltInWordlistHash( ) throws IOException, PwmUnrecoverableException
+        protected StoredWordlistDataBean.RemoteWordlistInfo getBuiltInWordlistHash( ) throws IOException, PwmUnrecoverableException
         {
 
             try ( InputStream inputStream = getBuiltInWordlist() )
             {
-                return SecureEngine.hash( inputStream, CHECKSUM_HASH_ALG );
+                return readRemoteWordlistInfo( inputStream );
             }
         }
 
-        public boolean populateAutoImport( )
+        public boolean populateAutoImport( final StoredWordlistDataBean.RemoteWordlistInfo remoteWordlistInfo )
                 throws IOException, PwmUnrecoverableException
         {
             autoImportError = null;
@@ -618,7 +640,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
             try
             {
                 inputStream = autoImportInputStream();
-                populateImpl( inputStream, StoredWordlistDataBean.Source.AutoImport );
+                populateImpl( remoteWordlistInfo, inputStream, StoredWordlistDataBean.Source.AutoImport );
                 return true;
             }
             catch ( Exception e )
@@ -634,21 +656,26 @@ abstract class AbstractWordlist implements Wordlist, PwmService
             return false;
         }
 
-        String readImportUrlHash( )
+        StoredWordlistDataBean.RemoteWordlistInfo readRemoteWordlistInfo( final InputStream inputStream )
         {
-            InputStream inputStream = null;
             try
             {
                 final Instant startTime = Instant.now();
-                logger.debug( "beginning read of auto-import wordlist url hash checksum from '" + wordlistConfiguration.getAutoImportUrl() + "'" );
-                inputStream = autoImportInputStream();
+                logger.debug( "beginning read of auto-import remote url data" );
+
                 final ChecksumInputStream checksumInputStream = new ChecksumInputStream( CHECKSUM_HASH_ALG, inputStream );
-                JavaHelper.copyWhilePredicate( checksumInputStream, new NullOutputStream(), o -> wlStatus != STATUS.CLOSED );
-                IOUtils.copy( checksumInputStream, new NullOutputStream() );
+                final long bytes = JavaHelper.copyWhilePredicate( checksumInputStream, new NullOutputStream(), o -> wlStatus != STATUS.CLOSED );
                 final String hash = JavaHelper.binaryArrayToHex( checksumInputStream.closeAndFinalChecksum() );
-                logger.debug( "completed read of auto-import wordlist url hash, value=" + hash + " (" + TimeDuration.fromCurrent( startTime ).asCompactString() + ")" );
+
+                final StoredWordlistDataBean.RemoteWordlistInfo remoteWordlistInfo = new StoredWordlistDataBean.RemoteWordlistInfo(
+                        hash,
+                        bytes );
+
+                logger.debug( "completed read of wordlist data: "
+                        + JsonUtil.serialize( remoteWordlistInfo )
+                        + " (" + TimeDuration.fromCurrent( startTime ).asCompactString() + ")" );
                 autoImportError = null;
-                return hash;
+                return remoteWordlistInfo;
             }
             catch ( Exception e )
             {
@@ -675,8 +702,5 @@ abstract class AbstractWordlist implements Wordlist, PwmService
             final PwmHttpClient client = new PwmHttpClient( pwmApplication, null, pwmHttpClientConfiguration );
             return client.streamForUrl( wordlistConfiguration.getAutoImportUrl() );
         }
-
-
     }
-
 }

server/src/main/java/password/pwm/svc/wordlist/Populator.java

@@ -27,21 +27,28 @@ import password.pwm.PwmApplication;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.svc.stats.EventRateMeter;
 import password.pwm.util.TransactionSizeCalculator;
+import password.pwm.util.java.ConditionalTaskExecutor;
 import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.Percent;
+import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBException;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.secure.ChecksumInputStream;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.text.DecimalFormat;
 import java.text.NumberFormat;
 import java.time.Instant;
+import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * @author Jason D. Rivard
@@ -53,21 +60,19 @@ class Populator
     // words truncated to this length, prevents massive words if the input
     private static final int MAX_LINE_LENGTH = 64;
 
-    private static final TimeDuration DEBUG_OUTPUT_FREQUENCY = TimeDuration.of( 3, TimeDuration.Unit.MINUTES );
+    private static final TimeDuration DEBUG_OUTPUT_FREQUENCY = TimeDuration.SECONDS_30;
 
     // words tarting with this prefix are ignored.
     private static final String COMMENT_PREFIX = "!#comment:";
 
     private static final NumberFormat PERCENT_FORMAT = DecimalFormat.getPercentInstance();
 
-    private final ZipReader zipFileReader;
+    private final WordlistZipReader zipFileReader;
     private final StoredWordlistDataBean.Source source;
 
-    private volatile boolean running;
-    private volatile boolean abortFlag;
+    private final AtomicBoolean running = new AtomicBoolean( false );
+    private final AtomicBoolean abortFlag = new AtomicBoolean( false );
 
-    private final PopulationStats overallStats = new PopulationStats();
-    private PopulationStats perReportStats = new PopulationStats();
     private TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(
             new TransactionSizeCalculator.SettingsBuilder()
                     .setDurationGoal( TimeDuration.of( 600, TimeDuration.Unit.MILLISECONDS ) )
@@ -76,16 +81,21 @@ class Populator
                     .createSettings()
     );
 
-    private int loopLines;
-
     private final Map<String, String> bufferedWords = new TreeMap<>();
 
     private final LocalDB localDB;
 
-    private final ChecksumInputStream checksumInputStream;
-
     private final AbstractWordlist rootWordlist;
 
+    private final StoredWordlistDataBean.RemoteWordlistInfo remoteWordlistInfo;
+
+    private final Instant startTime = Instant.now();
+
+    private final AtomicLong bytesSkipped = new AtomicLong( 0 );
+
+    private final EventRateMeter.MovingAverage byteRateMeter
+            = new EventRateMeter.MovingAverage( TimeDuration.of( 5, TimeDuration.Unit.MINUTES ) );
+
 
     static
     {
@@ -93,6 +103,7 @@ class Populator
     }
 
     Populator(
+            final StoredWordlistDataBean.RemoteWordlistInfo remoteWordlistInfo,
             final InputStream inputStream,
             final StoredWordlistDataBean.Source source,
             final AbstractWordlist rootWordlist,
@@ -100,81 +111,137 @@ class Populator
     )
             throws Exception
     {
+        this.remoteWordlistInfo = remoteWordlistInfo;
         this.source = source;
-        this.checksumInputStream = new ChecksumInputStream( AbstractWordlist.CHECKSUM_HASH_ALG, inputStream );
-        this.zipFileReader = new ZipReader( checksumInputStream );
+        this.zipFileReader = new WordlistZipReader( inputStream );
         this.localDB = pwmApplication.getLocalDB();
         this.rootWordlist = rootWordlist;
     }
 
-    private void init( ) throws LocalDBException, IOException
+    private void init( ) throws IOException, LocalDBException
     {
-        if ( abortFlag )
+        if ( abortFlag.get() )
         {
             return;
         }
 
-        localDB.truncate( rootWordlist.getWordlistDB() );
+        final long previousBytesRead = rootWordlist.readMetadata().getBytes();
 
-        if ( overallStats.getLines() > 0 )
+        if ( previousBytesRead == 0 )
         {
-            for ( int i = 0; i < overallStats.getLines(); i++ )
+            LOGGER.debug( "clearing stored wordlist" );
+            localDB.truncate( rootWordlist.getWordlistDB() );
+        }
+
+        if ( previousBytesRead > 0 )
+        {
+            final Instant startSkip = Instant.now();
+            LOGGER.debug( "skipping forward " + previousBytesRead + " bytes in stream that have been previously imported" );
+            while ( !abortFlag.get() && bytesSkipped.get() < previousBytesRead )
             {
                 zipFileReader.nextLine();
+                bytesSkipped.set( zipFileReader.getByteCount() );
             }
+            LOGGER.debug( "skipped forward " + previousBytesRead + " bytes in stream (" + TimeDuration.fromCurrent( startSkip ).asCompactString() + ")" );
         }
     }
 
-    public String makeStatString( )
+    String makeStatString( )
     {
-        if ( !running )
+        if ( !running.get() )
         {
             return "not running";
         }
 
-        final int lps = perReportStats.getElapsedSeconds() <= 0 ? 0 : perReportStats.getLines() / perReportStats.getElapsedSeconds();
+        return rootWordlist.debugLabel + " " + StringUtil.mapToString( makeStatValues() );
+    }
+
+    private Map<String, String> makeStatValues()
+    {
+        final Map<String, String> stats = new LinkedHashMap<>();
+        stats.put( "LinesRead", Long.toString( zipFileReader.getLineCount() ) );
+        stats.put( "BytesRead", Long.toString( zipFileReader.getByteCount() ) );
+        stats.put( "BufferSize", Integer.toString( transactionCalculator.getTransactionSize() ) );
+
+        final long elapsedSeconds = TimeDuration.fromCurrent( startTime ).as( TimeDuration.Unit.SECONDS );
+
+        if ( bytesSkipped.get() > 0 )
+        {
+            stats.put( "BytesSkipped", Long.toString( bytesSkipped.get() ) );
+        }
+
+        if ( elapsedSeconds > 10 )
+        {
+            stats.put( "BytesPerSecond", Double.toString( byteRateMeter.getAverage() * 1000 ) );
+        }
 
-        perReportStats = new PopulationStats();
-        return rootWordlist.debugLabel + ", lines/second="
-                + lps + ", line=" + overallStats.getLines() + ""
-                + " current zipEntry=" + zipFileReader.currentZipName();
+        if ( remoteWordlistInfo != null )
+        {
+            final Percent percent = new Percent( zipFileReader.getByteCount(), remoteWordlistInfo.getBytes() );
+            stats.put( "PercentComplete", percent.pretty( 2 ) );
+        }
+
+        stats.put( "ImportTime", TimeDuration.fromCurrent( startTime ).asCompactString() );
+
+        if ( remoteWordlistInfo != null && zipFileReader.getByteCount() > 1000 )
+        {
+            final long totalBytes = remoteWordlistInfo.getBytes();
+            final long remaingingBytes = totalBytes - zipFileReader.getByteCount();
+            final long remainingSeconds = (long) ( remaingingBytes * byteRateMeter.getAverage() * 1000 );
+
+            stats.put( "EstimatedRemainingTime", TimeDuration.of( remainingSeconds, TimeDuration.Unit.SECONDS ).asCompactString() );
+        }
+
+        return Collections.unmodifiableMap( stats );
     }
 
     @SuppressWarnings( "checkstyle:InnerAssignment" )
     void populate( ) throws IOException, LocalDBException, PwmUnrecoverableException
     {
+        final ConditionalTaskExecutor metaUpdater = new ConditionalTaskExecutor(
+                () -> rootWordlist.writeMetadata( StoredWordlistDataBean.builder()
+                        .source( source )
+                        .size( rootWordlist.size() )
+                        .storeDate( Instant.now() )
+                        .remoteInfo( remoteWordlistInfo )
+                        .bytes( zipFileReader.getByteCount() )
+                        .build() ),
+                new ConditionalTaskExecutor.TimeDurationPredicate( TimeDuration.SECONDS_10 )
+        );
+
+        final ConditionalTaskExecutor debugOutputter = new ConditionalTaskExecutor(
+                () -> LOGGER.debug( makeStatString() ),
+                new ConditionalTaskExecutor.TimeDurationPredicate( DEBUG_OUTPUT_FREQUENCY )
+        );
+
         try
         {
-            rootWordlist.writeMetadata( StoredWordlistDataBean.builder().source( source ).build() );
-            running = true;
+            debugOutputter.conditionallyExecuteTask();
+            running.set( true );
             init();
 
-            long lastReportTime = System.currentTimeMillis() - ( long ) ( DEBUG_OUTPUT_FREQUENCY.asMillis() * 0.33 );
-
             String line;
 
-            while ( !abortFlag && ( line = zipFileReader.nextLine() ) != null )
-            {
-
-                overallStats.incrementLines();
-                perReportStats.incrementLines();
+            long lastBytes = zipFileReader.getByteCount();
 
+            while ( !abortFlag.get() && ( line = zipFileReader.nextLine() ) != null )
+            {
                 addLine( line );
-                loopLines++;
 
-                if ( TimeDuration.fromCurrent( lastReportTime ).isLongerThan( DEBUG_OUTPUT_FREQUENCY.asMillis() ) )
-                {
-                    LOGGER.info( makeStatString() );
-                    lastReportTime = System.currentTimeMillis();
-                }
+                debugOutputter.conditionallyExecuteTask();
+                final long cycleBytes = zipFileReader.getByteCount() - lastBytes;
+                lastBytes = zipFileReader.getByteCount();
+                byteRateMeter.update( cycleBytes );
 
                 if ( bufferedWords.size() > transactionCalculator.getTransactionSize() )
                 {
                     flushBuffer();
+                    metaUpdater.conditionallyExecuteTask();
                 }
+
             }
 
-            if ( abortFlag )
+            if ( abortFlag.get() )
             {
                 LOGGER.warn( "pausing " + rootWordlist.debugLabel + " population" );
             }
@@ -185,8 +252,8 @@ class Populator
         }
         finally
         {
-            running = false;
-            IOUtils.closeQuietly( checksumInputStream );
+            running.set( false );
+            IOUtils.closeQuietly( zipFileReader );
         }
     }
 
@@ -218,7 +285,7 @@ class Populator
         //add the elements
         localDB.putAll( rootWordlist.getWordlistDB(), bufferedWords );
 
-        if ( abortFlag )
+        if ( abortFlag.get() )
         {
             return;
         }
@@ -227,21 +294,8 @@ class Populator
         final long commitTime = System.currentTimeMillis() - startTime;
         transactionCalculator.recordLastTransactionDuration( commitTime );
 
-        if ( bufferedWords.size() > 0 )
-        {
-            final StringBuilder sb = new StringBuilder();
-            sb.append( rootWordlist.debugLabel ).append( " " );
-            sb.append( "read " ).append( loopLines ).append( ", " );
-            sb.append( "saved " );
-            sb.append( bufferedWords.size() ).append( " words" );
-            sb.append( " (" ).append( TimeDuration.of( commitTime, TimeDuration.Unit.MILLISECONDS ).asCompactString() ).append( ")" );
-
-            LOGGER.trace( sb.toString() );
-        }
-
         //clear the buffers.
         bufferedWords.clear();
-        loopLines = 0;
     }
 
     private void populationComplete( )
@@ -259,14 +313,15 @@ class Populator
         final StringBuilder sb = new StringBuilder();
         sb.append( rootWordlist.debugLabel );
         sb.append( " population complete, added " ).append( wordlistSize );
-        sb.append( " total words in " ).append( TimeDuration.of( overallStats.getElapsedSeconds() * 1000, TimeDuration.Unit.MILLISECONDS ).asCompactString() );
+        sb.append( " total words in " ).append( TimeDuration.fromCurrent( startTime ).asCompactString() );
         {
             final StoredWordlistDataBean storedWordlistDataBean = StoredWordlistDataBean.builder()
-                    .sha1hash( JavaHelper.binaryArrayToHex( checksumInputStream.closeAndFinalChecksum() ) )
+                    .remoteInfo( remoteWordlistInfo )
                     .size( wordlistSize )
                     .storeDate( Instant.now() )
                     .source( source )
-                    .completed( !abortFlag )
+                    .completed( !abortFlag.get() )
+                    .bytes( zipFileReader.getByteCount() )
                     .build();
             rootWordlist.writeMetadata( storedWordlistDataBean );
         }
@@ -276,7 +331,7 @@ class Populator
     public void cancel( ) throws PwmUnrecoverableException
     {
         LOGGER.debug( "cancelling in-progress population" );
-        abortFlag = true;
+        abortFlag.set( true );
 
         final int maxWaitMs = 1000 * 30;
         final Instant startWaitTime = Instant.now();
@@ -293,28 +348,6 @@ class Populator
 
     public boolean isRunning( )
     {
-        return running;
-    }
-
-    private static class PopulationStats
-    {
-
-        private long startTime = System.currentTimeMillis();
-        private int lines;
-
-        public int getLines( )
-        {
-            return lines;
-        }
-
-        public void incrementLines( )
-        {
-            lines++;
-        }
-
-        public int getElapsedSeconds( )
-        {
-            return ( int ) ( System.currentTimeMillis() - startTime ) / 1000;
-        }
+        return running.get();
     }
 }

server/src/main/java/password/pwm/svc/wordlist/StoredWordlistDataBean.java

@@ -24,20 +24,23 @@ package password.pwm.svc.wordlist;
 
 import lombok.Builder;
 import lombok.Getter;
+import lombok.Value;
 
 import java.io.Serializable;
 import java.time.Instant;
 
-@Getter
-@Builder
+@Value
+@Builder( toBuilder = true )
 public class StoredWordlistDataBean implements Serializable
 {
     private boolean completed;
     private Source source;
     private Instant storeDate;
-    private String sha1hash;
+    private RemoteWordlistInfo remoteInfo;
+    private long bytes;
     private int size;
 
+    @Getter
     public enum Source
     {
         BuiltIn( "Built-In" ),
@@ -50,10 +53,12 @@ public class StoredWordlistDataBean implements Serializable
         {
             this.label = label;
         }
+    }
 
-        public String getLabel( )
-        {
-            return label;
-        }
+    @Value
+    public static class RemoteWordlistInfo implements Serializable
+    {
+        private String checksum;
+        private long bytes;
     }
 }

server/src/main/java/password/pwm/svc/wordlist/ZipReader.java → server/src/main/java/password/pwm/svc/wordlist/WordlistZipReader.java

@@ -22,35 +22,44 @@
 
 package password.pwm.svc.wordlist;
 
+import org.apache.commons.io.input.CountingInputStream;
 import password.pwm.PwmConstants;
+import password.pwm.util.java.JavaHelper;
 import password.pwm.util.logging.PwmLogger;
+import password.pwm.util.secure.ChecksumInputStream;
 
 import java.io.BufferedReader;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
 /**
  * @author Jason D. Rivard
  */
-class ZipReader implements AutoCloseable, Closeable
+class WordlistZipReader implements AutoCloseable, Closeable
 {
 
-    private static final PwmLogger LOGGER = PwmLogger.forClass( ZipReader.class );
+    private static final PwmLogger LOGGER = PwmLogger.forClass( WordlistZipReader.class );
 
     private final ZipInputStream zipStream;
+    private final ChecksumInputStream checksumInputStream;
+    private final CountingInputStream countingInputStream;
+    private final AtomicLong lineCounter = new AtomicLong( 0 );
 
     private BufferedReader reader;
     private ZipEntry zipEntry;
-    private int lineCounter = 0;
 
-    ZipReader( final InputStream inputStream )
+    WordlistZipReader( final InputStream inputStream )
             throws Exception
     {
-        zipStream = new ZipInputStream( inputStream );
+        checksumInputStream = new ChecksumInputStream( AbstractWordlist.CHECKSUM_HASH_ALG, inputStream );
+        countingInputStream = new CountingInputStream( checksumInputStream );
+
+        zipStream = new ZipInputStream( countingInputStream );
         nextZipEntry();
         if ( zipEntry == null )
         {
@@ -61,11 +70,6 @@ class ZipReader implements AutoCloseable, Closeable
     private void nextZipEntry( )
             throws IOException
     {
-        if ( zipEntry != null )
-        {
-            LOGGER.trace( "finished reading " + zipEntry.getName() + ", lines=" + lineCounter );
-        }
-
         zipEntry = zipStream.getNextEntry();
 
         while ( zipEntry != null && zipEntry.isDirectory() )
@@ -75,7 +79,6 @@ class ZipReader implements AutoCloseable, Closeable
 
         if ( zipEntry != null )
         {
-            lineCounter = 0;
             reader = new BufferedReader( new InputStreamReader( zipStream, PwmConstants.DEFAULT_CHARSET ) );
         }
     }
@@ -125,9 +128,24 @@ class ZipReader implements AutoCloseable, Closeable
 
         if ( line != null )
         {
-            lineCounter++;
+            lineCounter.incrementAndGet();
         }
 
         return line;
     }
+
+    long getLineCount()
+    {
+        return lineCounter.get();
+    }
+
+    long getByteCount()
+    {
+        return countingInputStream.getByteCount();
+    }
+
+    String getChecksum()
+    {
+        return JavaHelper.binaryArrayToHex( checksumInputStream.getInProgressChecksum() );
+    }
 }