wordlist stats and import refactoring

server/src/main/java/password/pwm/svc/wordlist/WordlistImporter.java

@@ -40,9 +40,9 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumMap;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.concurrent.CancellationException;
 import java.util.concurrent.atomic.LongAdder;
 import java.util.function.BooleanSupplier;
 
@@ -106,6 +106,7 @@ class WordlistImporter implements Runnable
         WordTypes,
         MsPerTxn,
         WordsPerTxn,
+        ChunksSaved,
         CharsPerTxn,
         ChunksPerWord,
         AvgWordLength,
@@ -145,6 +146,10 @@ class WordlistImporter implements Runnable
         {
             doImport();
         }
+        catch ( final CancellationException e )
+        {
+            getLogger().debug( rootWordlist.getSessionLabel(), () -> "stopped import due to cancel flag" );
+        }
         catch ( final PwmUnrecoverableException e )
         {
             errorMsg = "error during import: " + e.getErrorInformation().getDetailedErrorMsg();
@@ -158,20 +163,20 @@ class WordlistImporter implements Runnable
                     }
             );
         }
+    }
 
+    private void cancelCheck()
+    {
         if ( cancelFlag.getAsBoolean() )
         {
-            getLogger().debug( rootWordlist.getSessionLabel(), () -> "exiting import due to cancel flag" );
+            throw new CancellationException();
         }
     }
 
     private void initImportProcess( )
             throws PwmUnrecoverableException
     {
-        if ( cancelFlag.getAsBoolean() )
-        {
-            return;
-        }
+        cancelCheck();
 
         if ( wordlistSourceInfo == null || !wordlistSourceInfo.equals( rootWordlist.readWordlistStatus().getRemoteInfo() ) )
         {
@@ -228,6 +233,8 @@ class WordlistImporter implements Runnable
             getLogger().debug( rootWordlist.getSessionLabel(), () -> "beginning import: " + JsonUtil.serialize( rootWordlist.readWordlistStatus() ) );
             Instant lastTxnInstant = Instant.now();
 
+            final long importMaxChars = rootWordlist.getConfiguration().getImportMaxChars();
+
             String line;
             do
             {
@@ -240,7 +247,7 @@ class WordlistImporter implements Runnable
 
                     if (
                             bufferedWords.size() > transactionCalculator.getTransactionSize()
-                                    || charsInBuffer > rootWordlist.getConfiguration().getImportMaxChars()
+                                    || charsInBuffer > importMaxChars
                     )
                     {
                         flushBuffer();
@@ -251,19 +258,14 @@ class WordlistImporter implements Runnable
                         pauseTimer.conditionallyExecuteTask();
                         lastTxnInstant = Instant.now();
                     }
+
+                    cancelCheck();
                 }
             }
-            while ( !cancelFlag.getAsBoolean() && line != null );
-
+            while ( line != null );
 
-            if ( cancelFlag.getAsBoolean() )
-            {
-                getLogger().debug( rootWordlist.getSessionLabel(), () -> "pausing import" );
-            }
-            else
-            {
-                populationComplete();
-            }
+            cancelCheck();
+            populationComplete();
         }
         finally
         {
@@ -278,12 +280,9 @@ class WordlistImporter implements Runnable
             return;
         }
 
-        for ( final String commentPrefix : rootWordlist.getConfiguration().getCommentPrefixes() )
+        if ( checkIfCommentLine( input ) )
         {
-            if ( input.startsWith( commentPrefix ) )
-            {
-                return;
-            }
+            return;
         }
 
         final WordType wordType = WordType.determineWordType( input );
@@ -291,16 +290,15 @@ class WordlistImporter implements Runnable
 
         if ( wordType == WordType.RAW )
         {
-            final Optional<String> word = WordlistUtil.normalizeWordLength( input, rootWordlist.getConfiguration() );
-            if ( word.isPresent() )
+            WordlistUtil.normalizeWordLength( input, rootWordlist.getConfiguration() ).ifPresent( word ->
             {
-                final String normalizedWord = wordType.convertInputFromWordlist( this.rootWordlist.getConfiguration(), word.get() );
+                final String normalizedWord = wordType.convertInputFromWordlist( this.rootWordlist.getConfiguration(), word );
                 final Set<String> words = WordlistUtil.chunkWord( normalizedWord, rootWordlist.getConfiguration().getCheckSize() );
                 importStatistics.update( StatKey.averageWordLength, normalizedWord.length() );
                 importStatistics.update( StatKey.chunksPerWord, words.size() );
                 incrementCharBufferCounter( words );
                 bufferedWords.addAll( words );
-            }
+            } );
         }
         else
         {
@@ -318,6 +316,19 @@ class WordlistImporter implements Runnable
         }
     }
 
+    private boolean checkIfCommentLine( final String input )
+    {
+        for ( final String commentPrefix : rootWordlist.getConfiguration().getCommentPrefixes() )
+        {
+            if ( input.startsWith( commentPrefix ) )
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     private void flushBuffer( )
             throws PwmUnrecoverableException
     {
@@ -326,10 +337,7 @@ class WordlistImporter implements Runnable
         //add the elements
         wordlistBucket.addWords( bufferedWords, rootWordlist );
 
-        if ( cancelFlag.getAsBoolean() )
-        {
-            return;
-        }
+        cancelCheck();
 
         //mark how long the buffer close took
         final TimeDuration commitTime = TimeDuration.fromCurrent( startTime );
@@ -386,11 +394,13 @@ class WordlistImporter implements Runnable
 
             getLogger().debug( rootWordlist.getSessionLabel(), () -> "will skip forward " + StringUtil.formatDiskSizeforDebug( previousBytesRead )
                     + " in wordlist that has been previously imported" );
-            while ( !cancelFlag.getAsBoolean() && bytesSkipped < previousBytesRead )
+
+            while ( bytesSkipped < previousBytesRead )
             {
                 zipFileReader.nextLine();
                 bytesSkipped = zipFileReader.getByteCount();
                 debugOutputter.conditionallyExecuteTask();
+                cancelCheck();
             }
             getLogger().debug( rootWordlist.getSessionLabel(), () -> "skipped forward " + StringUtil.formatDiskSizeforDebug( previousBytesRead )
                     + " in stream (" + TimeDuration.fromCurrent( startSkipTime ).asCompactString() + ")" );
@@ -410,7 +420,6 @@ class WordlistImporter implements Runnable
         {
             final long totalBytes = wordlistSourceInfo.getBytes();
             final long remainingBytes = totalBytes - zipFileReader.getByteCount();
-            stats.put( DebugKey.BytesRemaining, StringUtil.formatDiskSizeforDebug( remainingBytes ) );
 
             try
             {
@@ -433,28 +442,27 @@ class WordlistImporter implements Runnable
             }
             catch ( final Exception e )
             {
-                getLogger().error( rootWordlist.getSessionLabel(), () -> "error calculating import statistics: " + e.getMessage() );
-
-                /* ignore - it's a long overflow if the estimate is off */
+                /* ignore - it's a long overflow or div by zero if the estimate is off */
+                getLogger().debug( rootWordlist.getSessionLabel(), () -> "error calculating import statistics: " + e.getMessage() );
             }
 
-            final Percent percent = Percent.of( zipFileReader.getByteCount(), wordlistSourceInfo.getBytes() );
-            stats.put( DebugKey.PercentComplete, percent.pretty( 2 ) );
+            stats.put( DebugKey.PercentComplete, Percent.of( zipFileReader.getByteCount(), wordlistSourceInfo.getBytes() ).pretty() );
+            stats.put( DebugKey.BytesRemaining, StringUtil.formatDiskSizeforDebug( remainingBytes ) );
         }
 
         stats.put( DebugKey.LinesRead, PwmNumberFormat.forDefaultLocale().format( zipFileReader.getLineCount() ) );
+        stats.put( DebugKey.ChunksSaved, PwmNumberFormat.forDefaultLocale().format( rootWordlist.size() ) );
         stats.put( DebugKey.BytesRead, StringUtil.formatDiskSizeforDebug( zipFileReader.getByteCount() ) );
-
         stats.put( DebugKey.DiskFreeSpace, StringUtil.formatDiskSize( wordlistBucket.spaceRemaining() ) );
+        stats.put( DebugKey.ImportTime, TimeDuration.fromCurrent( startTime ).asCompactString() );
+        stats.put( DebugKey.ZipFile, zipFileReader.currentZipName() );
+        stats.put( DebugKey.WordTypes, JsonUtil.serializeMap( seenWordTypes ) );
 
         if ( bytesSkipped > 0 )
         {
             stats.put( DebugKey.BytesSkipped, StringUtil.formatDiskSizeforDebug( bytesSkipped ) );
         }
 
-        stats.put( DebugKey.ImportTime, TimeDuration.fromCurrent( startTime ).asCompactString() );
-        stats.put( DebugKey.ZipFile, zipFileReader.currentZipName() );
-        stats.put( DebugKey.WordTypes, JsonUtil.serializeMap( seenWordTypes ) );
 
         try
         {

server/src/main/java/password/pwm/svc/wordlist/WordlistInspector.java

@@ -22,6 +22,7 @@ package password.pwm.svc.wordlist;
 
 import password.pwm.PwmApplication;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
@@ -144,7 +145,7 @@ class WordlistInspector implements Runnable
             }
             else
             {
-                final WordlistSourceInfo builtInInfo = source.readRemoteWordlistInfo( pwmApplication, cancelFlag, getLogger() );
+                final WordlistSourceInfo builtInInfo = source.readRemoteWordlistInfo( pwmApplication, rootWordlist.getSessionLabel(), cancelFlag, getLogger() );
                 if ( !builtInInfo.equals( existingStatus.getRemoteInfo() ) )
                 {
                     getLogger().debug( rootWordlist.getSessionLabel(), () -> "existing built-in store does not match imported wordlist, will re-import" );
@@ -270,7 +271,7 @@ class WordlistInspector implements Runnable
                 final WordlistSource testWordlistSource = WordlistSource.forAutoImport( pwmApplication, rootWordlist.getConfiguration() );
                 try
                 {
-                    testWordlistSource.readRemoteWordlistInfo( pwmApplication, cancelFlag, getLogger() );
+                    testWordlistSource.readRemoteWordlistInfo( pwmApplication, rootWordlist.getSessionLabel(), cancelFlag, getLogger() );
                 }
                 catch ( final PwmUnrecoverableException e )
                 {
@@ -314,7 +315,7 @@ class WordlistInspector implements Runnable
             throws IOException, PwmUnrecoverableException
     {
         final WordlistSource source = WordlistSource.forAutoImport( pwmApplication, rootWordlist.getConfiguration() );
-        final WordlistSourceInfo remoteInfo = source.readRemoteWordlistInfo( pwmApplication, cancelFlag, getLogger() );
+        final WordlistSourceInfo remoteInfo = source.readRemoteWordlistInfo( pwmApplication, rootWordlist.getSessionLabel(), cancelFlag, getLogger() );
 
         cancelCheck();
 
@@ -327,7 +328,11 @@ class WordlistInspector implements Runnable
         {
             if ( !remoteInfo.equals( existingStatus.getRemoteInfo() ) )
             {
-                getLogger().debug( rootWordlist.getSessionLabel(), () -> "auto-import url remote hash does not equal currently stored hash, will start auto-import" );
+                getLogger().debug( rootWordlist.getSessionLabel(), () -> "auto-import url remote info "
+                        + JsonUtil.serialize( remoteInfo )
+                        + " does not equal currently stored info "
+                        + JsonUtil.serialize( existingStatus.getRemoteInfo() )
+                        + ", will start auto-import" );
                 needsAutoImport = true;
             }
             else if ( !existingStatus.isCompleted() )
@@ -351,7 +356,7 @@ class WordlistInspector implements Runnable
             throws IOException, PwmUnrecoverableException
     {
         final WordlistSource wordlistSource = WordlistSource.forBuiltIn( pwmApplication, rootWordlist.getConfiguration() );
-        final WordlistSourceInfo wordlistSourceInfo = wordlistSource.readRemoteWordlistInfo( pwmApplication, cancelFlag, getLogger() );
+        final WordlistSourceInfo wordlistSourceInfo = wordlistSource.readRemoteWordlistInfo( pwmApplication, rootWordlist.getSessionLabel(), cancelFlag, getLogger() );
         final WordlistImporter wordlistImporter = new WordlistImporter(
                 wordlistSourceInfo,
                 wordlistSource.getZipWordlistReader(),

server/src/main/java/password/pwm/svc/wordlist/WordlistService.java

@@ -20,13 +20,8 @@
 
 package password.pwm.svc.wordlist;
 
-import password.pwm.error.PwmException;
 import password.pwm.error.PwmUnrecoverableException;
-import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.secure.PwmRandom;
-
-import java.time.Instant;
 
 
 /**
@@ -56,49 +51,4 @@ public class WordlistService extends AbstractWordlist implements Wordlist
     {
         return super.containsWord( this.getWordTypesCache(), word );
     }
-
-    protected void warmup()
-    {
-        getPwmApplication().getPwmScheduler().immediateExecuteRunnableInNewThread( new WarmupJob(), "wordlist-warmup" );
-    }
-
-    private class WarmupJob implements Runnable
-    {
-        @Override
-        public void run()
-        {
-
-            final Instant startTime = Instant.now();
-            final PwmRandom pwmRandom = getPwmApplication().getSecureService().pwmRandom();
-            final int warmupCount = getConfiguration().getWarmupLookups();
-
-            getLogger().trace( getSessionLabel(),
-                    () -> "beginning warmup using " + warmupCount + " random words" );
-
-            for ( int i = 0; i < warmupCount; i++ )
-            {
-                final String testWord = pwmRandom.alphaNumericString( pwmRandom.nextInt( 10 ) + 5 );
-                try
-                {
-                    containsWord( getWordTypesCache(), testWord );
-
-                    if ( status() != STATUS.OPEN )
-                    {
-                        LOGGER.trace( getSessionLabel(), () -> "exiting cancelled warmup..." );
-                        return;
-                    }
-                }
-                catch ( final PwmException e )
-                {
-                    getLogger().trace( getSessionLabel(), () -> "error during warmup word check: " + e.getMessage() );
-                }
-            }
-
-            getLogger().trace( getSessionLabel(),
-                    () -> "warmup using " + warmupCount + " random words complete",
-                    () -> TimeDuration.fromCurrent( startTime ) );
-
-            outputStats();
-        }
-    }
 }

server/src/main/java/password/pwm/svc/wordlist/WordlistSource.java

@@ -23,6 +23,7 @@ package password.pwm.svc.wordlist;
 import org.apache.commons.io.IOUtils;
 import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
+import password.pwm.bean.SessionLabel;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
@@ -43,7 +44,6 @@ import password.pwm.util.logging.PwmLogger;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
-import java.security.DigestInputStream;
 import java.time.Instant;
 import java.util.Collections;
 import java.util.EnumMap;
@@ -88,7 +88,10 @@ class WordlistSource
         InputStream getInputStream() throws IOException, PwmUnrecoverableException;
     }
 
-    static WordlistSource forAutoImport( final PwmApplication pwmApplication, final WordlistConfiguration wordlistConfiguration )
+    static WordlistSource forAutoImport(
+            final PwmApplication pwmApplication,
+            final WordlistConfiguration wordlistConfiguration
+    )
     {
         final String importUrl = wordlistConfiguration.getAutoImportUrl();
         return new WordlistSource( WordlistSourceType.AutoImport, importUrl, () ->
@@ -153,6 +156,7 @@ class WordlistSource
 
     Map<HttpHeader, String> readRemoteHeaders(
             final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
             final PwmLogger pwmLogger
     )
             throws PwmUnrecoverableException
@@ -179,50 +183,48 @@ class WordlistSource
         }
 
         final Map<HttpHeader, String> finalReturnResponses =  Collections.unmodifiableMap( returnResponses );
-        pwmLogger.debug( () -> "read remote header info for " + this.getWordlistSourceType() + " wordlist: "
+        pwmLogger.debug( sessionLabel, () -> "read remote header info for " + this.getWordlistSourceType() + " wordlist: "
                 + JsonUtil.serializeMap( finalReturnResponses ), () -> TimeDuration.fromCurrent( startTime ) );
         return finalReturnResponses;
     }
 
     WordlistSourceInfo readRemoteWordlistInfo(
             final PwmApplication pwmApplication,
+            final SessionLabel sessionLabel,
             final BooleanSupplier cancelFlag,
             final PwmLogger pwmLogger
     )
             throws PwmUnrecoverableException
     {
         final Instant startTime = Instant.now();
+        final int processId = CLOSE_COUNTER.getAndIncrement();
 
         if ( cancelFlag.getAsBoolean() )
         {
             throw new CancellationException();
         }
 
-        pwmLogger.debug( () -> "begin reading file info for " + this.getWordlistSourceType() + " wordlist" );
+        pwmLogger.debug( sessionLabel, () -> processIdLabel( processId ) + "begin reading file info for " + this.getWordlistSourceType() + " wordlist" );
 
         final long bytes;
         final String hash;
         final long lines;
 
         InputStream inputStream = null;
-        DigestInputStream checksumInputStream = null;
         WordlistZipReader zipInputStream = null;
 
         try
         {
             inputStream = this.streamProvider.getInputStream();
-            checksumInputStream = pwmApplication.getSecureService().digestInputStream( WordlistConfiguration.HASH_ALGORITHM, inputStream );
-            zipInputStream = new WordlistZipReader( checksumInputStream );
-            final ConditionalTaskExecutor debugOutputter = makeDebugLoggerExecutor( pwmLogger, startTime, zipInputStream );
+            zipInputStream = new WordlistZipReader( inputStream );
+            final ConditionalTaskExecutor debugOutputter = makeDebugLoggerExecutor( pwmLogger, processId, sessionLabel, startTime, zipInputStream );
 
-            int counter = 0;
             String nextLine;
             do
             {
-                counter++;
                 nextLine = zipInputStream.nextLine();
 
-                if ( counter % 10000 == 0 )
+                if ( zipInputStream.getLineCount() % 10_000 == 0 )
                 {
                     debugOutputter.conditionallyExecuteTask();
                 }
@@ -244,12 +246,12 @@ class WordlistSource
         }
         finally
         {
-            closeStreams( pwmLogger, checksumInputStream, inputStream );
+            closeStreams( pwmLogger, processId, sessionLabel, inputStream );
             IOUtils.closeQuietly( zipInputStream );
         }
 
         bytes = zipInputStream.getByteCount();
-        hash = JavaHelper.byteArrayToHexString( checksumInputStream.getMessageDigest().digest() );
+        hash = JavaHelper.byteArrayToHexString( zipInputStream.getHash() );
         lines = zipInputStream.getLineCount();
 
         if ( cancelFlag.getAsBoolean() )
@@ -259,8 +261,8 @@ class WordlistSource
 
         final WordlistSourceInfo wordlistSourceInfo = new WordlistSourceInfo( hash, bytes, importUrl, lines );
 
-        pwmLogger.debug( () -> "completed read of data for " + this.getWordlistSourceType() + " wordlist"
-                + " " + StringUtil.formatDiskSizeforDebug( bytes )
+        pwmLogger.debug( sessionLabel, () -> processIdLabel( processId ) + "completed read of data for " + this.getWordlistSourceType()
+                + " wordlist " + StringUtil.formatDiskSizeforDebug( bytes )
                 + ", " + JsonUtil.serialize( wordlistSourceInfo )
                 + " (" + TimeDuration.compactFromCurrent( startTime ) + ")" );
 
@@ -269,6 +271,8 @@ class WordlistSource
 
     private ConditionalTaskExecutor makeDebugLoggerExecutor(
             final PwmLogger pwmLogger,
+            final int processId,
+            final SessionLabel sessionLabel,
             final Instant startTime,
             final WordlistZipReader wordlistZipReader
     )
@@ -278,8 +282,9 @@ class WordlistSource
             @Override
             public void run()
             {
-                pwmLogger.debug( () -> "continuing reading file info for " + getWordlistSourceType() + " wordlist"
-                                + " " + StringUtil.formatDiskSize( wordlistZipReader.getByteCount() ) + " bytes read"
+                pwmLogger.debug( sessionLabel, () -> processIdLabel( processId ) + "continuing reading file info for "
+                                + getWordlistSourceType() + " wordlist"
+                                + " " + StringUtil.formatDiskSize( wordlistZipReader.getByteCount() ) + " read"
                                 + bytesPerSecondStr().orElse( "" ),
                         () -> TimeDuration.fromCurrent( startTime ) );
             }
@@ -300,25 +305,24 @@ class WordlistSource
                 AbstractWordlist.DEBUG_OUTPUT_FREQUENCY );
     }
 
-    private void closeStreams( final PwmLogger pwmLogger, final InputStream... inputStreams )
+    private void closeStreams(
+            final PwmLogger pwmLogger,
+            final int processId,
+            final SessionLabel sessionLabel,
+            final InputStream... inputStreams )
     {
-        // use a thread to close the io tasks as the close will block until the tcp socket is closed
-        final Thread closerThread = new Thread( () ->
+        final Instant startClose = Instant.now();
+        pwmLogger.trace( sessionLabel, () -> processIdLabel( processId ) + "beginning close of remote wordlist read process" );
+        for ( final InputStream inputStream : inputStreams )
         {
-            final int counter = CLOSE_COUNTER.incrementAndGet();
-
-            final Instant startClose = Instant.now();
-            pwmLogger.trace( () -> "beginning close of remote wordlist read process [" + counter + "]" );
-            for ( final InputStream inputStream : inputStreams )
-            {
-                IOUtils.closeQuietly( inputStream );
-            }
-            pwmLogger.trace( () -> "completed close of remote wordlist read process [" + counter + "]",
-                    () -> TimeDuration.fromCurrent( startClose ) );
-        } );
+            IOUtils.closeQuietly( inputStream );
+        }
+        pwmLogger.trace( sessionLabel, () -> processIdLabel( processId ) + "completed close of remote wordlist read process",
+                () -> TimeDuration.fromCurrent( startClose ) );
+    }
 
-        closerThread.setDaemon( true );
-        closerThread.setName( Thread.currentThread().getName() + "-import-close-io" );
-        closerThread.start();
+    private String processIdLabel( final int processId )
+    {
+        return "<" + processId + "> ";
     }
 }

server/src/main/java/password/pwm/svc/wordlist/WordlistZipReader.java

@@ -34,8 +34,9 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.math.BigDecimal;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.Objects;
-import java.util.concurrent.atomic.LongAdder;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
@@ -47,9 +48,11 @@ class WordlistZipReader implements AutoCloseable, Closeable
     private static final PwmLogger LOGGER = PwmLogger.forClass( WordlistZipReader.class );
 
     private final ZipInputStream zipStream;
-    private final LongAdder byteCounter = new LongAdder();
     private final EventRateMeter eventRateMeter;
-    private final LongAdder lineCounter = new LongAdder();
+    private final MessageDigest messageDigest;
+
+    private long byteCounter = 0;
+    private long lineCounter = 0;
 
     private BufferedReader reader;
     private ZipEntry zipEntry;
@@ -58,6 +61,15 @@ class WordlistZipReader implements AutoCloseable, Closeable
     {
         Objects.requireNonNull( inputStream );
 
+        try
+        {
+            messageDigest = MessageDigest.getInstance( WordlistConfiguration.HASH_ALGORITHM.getAlgName() );
+        }
+        catch ( final NoSuchAlgorithmException e )
+        {
+            throw new IllegalStateException();
+        }
+
         eventRateMeter = new EventRateMeter( TimeDuration.MINUTE );
         final CopyingInputStream copyingInputStream = new CopyingInputStream( inputStream, this::updateReadBytes );
         zipStream = new ZipInputStream( copyingInputStream );
@@ -77,7 +89,8 @@ class WordlistZipReader implements AutoCloseable, Closeable
 
         final int count = b.length;
         eventRateMeter.markEvents( count );
-        byteCounter.add( count );
+        byteCounter += count;
+        messageDigest.digest( b );
     }
 
     private void nextZipEntry( )
@@ -156,7 +169,7 @@ class WordlistZipReader implements AutoCloseable, Closeable
 
         if ( line != null )
         {
-            lineCounter.increment();
+            lineCounter++;
         }
 
         return line;
@@ -169,11 +182,16 @@ class WordlistZipReader implements AutoCloseable, Closeable
 
     long getLineCount()
     {
-        return lineCounter.sum();
+        return lineCounter;
     }
 
     long getByteCount()
     {
-        return byteCounter.sum();
+        return byteCounter;
+    }
+
+    byte[] getHash()
+    {
+        return messageDigest.digest();
     }
 }