Merge branch 'master' into enh-ngconfig

server/src/main/java/password/pwm/http/servlet/configmanager/DebugItemGenerator.java

@@ -50,6 +50,7 @@ import password.pwm.svc.stats.EpsStatistic;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.LDAPPermissionCalculator;
+import password.pwm.util.java.ClosableIterator;
 import password.pwm.util.java.DebugOutputBuilder;
 import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.java.JavaHelper;
@@ -468,9 +469,9 @@ public class DebugItemGenerator
         @Override
         public void outputItem( final DebugItemInput debugItemInput, final OutputStream outputStream ) throws Exception
         {
-            final List<FileSystemUtility.FileSummaryInformation> fileSummaryInformations = new ArrayList<>();
             final PwmApplication pwmApplication = debugItemInput.getPwmApplication();
             final File applicationPath = pwmApplication.getPwmEnvironment().getApplicationPath();
+            final List<File> interestedFiles = new ArrayList<>(  );
 
             if ( pwmApplication.getPwmEnvironment().getContextManager() != null )
             {
@@ -483,7 +484,7 @@ public class DebugItemGenerator
 
                         if ( servletRootPath != null )
                         {
-                            fileSummaryInformations.addAll( FileSystemUtility.readFileInformation( webInfPath ) );
+                            interestedFiles.add( webInfPath );
                         }
                     }
                 }
@@ -497,7 +498,7 @@ public class DebugItemGenerator
             {
                 try
                 {
-                    fileSummaryInformations.addAll( FileSystemUtility.readFileInformation( applicationPath ) );
+                    interestedFiles.add( applicationPath );
                 }
                 catch ( Exception e )
                 {
@@ -505,6 +506,8 @@ public class DebugItemGenerator
                 }
             }
 
+
+            try ( ClosableIterator<FileSystemUtility.FileSummaryInformation> iter = FileSystemUtility.readFileInformation( interestedFiles ); )
             {
                 final CSVPrinter csvPrinter = JavaHelper.makeCsvPrinter( outputStream );
                 {
@@ -516,8 +519,10 @@ public class DebugItemGenerator
                     headerRow.add( "Checksum" );
                     csvPrinter.printComment( StringUtil.join( headerRow, "," ) );
                 }
-                for ( final FileSystemUtility.FileSummaryInformation fileSummaryInformation : fileSummaryInformations )
+
+                while ( iter.hasNext() )
                 {
+                    final FileSystemUtility.FileSummaryInformation fileSummaryInformation = iter.next();
                     try
                     {
                         final List<String> dataRow = new ArrayList<>();

server/src/main/java/password/pwm/http/servlet/resource/ResourceServletService.java

@@ -31,6 +31,7 @@ import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.health.HealthRecord;
 import password.pwm.http.PwmRequest;
 import password.pwm.svc.PwmService;
+import password.pwm.util.java.ClosableIterator;
 import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.MovingAverage;
@@ -277,9 +278,15 @@ public class ResourceServletService implements PwmService
                         final File resourcePath = new File( basePath.getAbsolutePath() + File.separator + "public" + File.separator + "resources" );
                         if ( resourcePath.exists() )
                         {
-                            for ( final FileSystemUtility.FileSummaryInformation fileSummaryInformation : FileSystemUtility.readFileInformation( resourcePath ) )
+                            try ( ClosableIterator<FileSystemUtility.FileSummaryInformation> iter =
+                                          FileSystemUtility.readFileInformation( Collections.singletonList( resourcePath ) ) )
                             {
-                                checksumStream.write( JavaHelper.longToBytes( fileSummaryInformation.getChecksum() ) );
+                                while ( iter.hasNext()  )
+                                {
+                                    final FileSystemUtility.FileSummaryInformation fileSummaryInformation = iter.next();
+                                    checksumStream.write( JavaHelper.longToBytes( fileSummaryInformation.getChecksum() ) );
+                                }
+
                             }
                         }
                     }

server/src/main/java/password/pwm/util/java/ConcurrentClosableIteratorWrapper.java

@@ -0,0 +1,103 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2019 The PWM Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package password.pwm.util.java;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+
+public class ConcurrentClosableIteratorWrapper<T> implements Iterator<T>, ClosableIterator<T>
+{
+    private final AtomicReference<T> nextReference = new AtomicReference<>();
+    private final AtomicBoolean completed = new AtomicBoolean( false );
+
+    private final Supplier<Optional<T>> nextSupplier;
+    private final Runnable closeFunction;
+
+    public ConcurrentClosableIteratorWrapper( final Supplier<Optional<T>> nextFunction, final Runnable closeFunction )
+    {
+        this.nextSupplier = nextFunction;
+        this.closeFunction = closeFunction;
+    }
+
+    @Override
+    public synchronized boolean hasNext()
+    {
+        if ( completed.get() )
+        {
+            return false;
+        }
+        work();
+        return nextReference.get() != null;
+    }
+
+    @Override
+    public synchronized T next()
+    {
+        if ( completed.get() )
+        {
+            throw new NoSuchElementException(  );
+        }
+        work();
+        final T fileSummaryInformation = nextReference.get();
+        if ( fileSummaryInformation == null )
+        {
+            throw new NoSuchElementException(  );
+        }
+        nextReference.set( null );
+        return fileSummaryInformation;
+    }
+
+    @Override
+    public synchronized void close()
+    {
+        closeImpl();
+    }
+
+    private void closeImpl()
+    {
+        if ( !completed.get() )
+        {
+            completed.set( true );
+            nextReference.set( null );
+            closeFunction.run();
+        }
+    }
+
+    private void work()
+    {
+        if ( nextReference.get() == null && !completed.get() )
+        {
+            final Optional<T> next = nextSupplier.get();
+            if ( next.isPresent() )
+            {
+                nextReference.set( next.get() );
+            }
+            else
+            {
+                closeImpl();
+            }
+        }
+    }
+}

server/src/main/java/password/pwm/util/java/FileSystemUtility.java

@@ -26,7 +26,6 @@ import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.logging.PwmLogger;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
@@ -35,14 +34,19 @@ import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.ForkJoinPool;
-import java.util.concurrent.RecursiveTask;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.zip.CRC32;
@@ -51,90 +55,133 @@ public class FileSystemUtility
 {
     private static final PwmLogger LOGGER = PwmLogger.forClass( FileSystemUtility.class );
 
+    private static final int CRC_BUFFER_SIZE = 60 * 1024;
+
     private static final AtomicLoopIntIncrementer OP_COUNTER = new AtomicLoopIntIncrementer();
 
-    public static List<FileSummaryInformation> readFileInformation( final File rootFile )
+
+    public static ClosableIterator<FileSummaryInformation> readFileInformation( final List<File> rootFiles )
     {
         final Instant startTime = Instant.now();
         final int operation = OP_COUNTER.next();
-        LOGGER.trace( () -> "begin file summary load for file '" + rootFile.getAbsolutePath() + ", operation=" + operation );
-        final ForkJoinPool pool = new ForkJoinPool();
-        final RecursiveFileReaderTask task = new RecursiveFileReaderTask( rootFile );
-        final List<FileSummaryInformation> fileSummaryInformations = pool.invoke( task );
-        final AtomicLong byteCount = new AtomicLong( 0 );
-        final AtomicInteger fileCount = new AtomicInteger( 0 );
-        fileSummaryInformations.forEach( fileSummaryInformation -> byteCount.addAndGet( fileSummaryInformation.getSize() ) );
-        fileSummaryInformations.forEach( fileSummaryInformation -> fileCount.incrementAndGet() );
-        final Map<String, String> debugInfo = new LinkedHashMap<>();
-        debugInfo.put( "operation", Integer.toString( operation ) );
-        debugInfo.put( "bytes", StringUtil.formatDiskSizeforDebug( byteCount.get() ) );
-        debugInfo.put( "files", Integer.toString( fileCount.get() ) );
-        debugInfo.put( "duration", TimeDuration.compactFromCurrent( startTime ) );
-        LOGGER.trace( () -> "completed file summary load for file '" + rootFile.getAbsolutePath() + ", " + StringUtil.mapToString( debugInfo ) );
-        return fileSummaryInformations;
+
+        final int cpus = Runtime.getRuntime().availableProcessors();
+        final BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(  );
+        final ExecutorService executor = new ThreadPoolExecutor( cpus, cpus, Long.MAX_VALUE, TimeUnit.MILLISECONDS, workQueue );
+        final TaskData taskData = new TaskData( executor );
+
+        for ( final File rootFile : rootFiles )
+        {
+            LOGGER.trace( () -> "begin file summary load for file '" + rootFile.getAbsolutePath() + ", operation=" + operation );
+            executor.execute( new RecursiveFileReaderTask( rootFile, taskData ) );
+        }
+
+        return new ConcurrentClosableIteratorWrapper<>( () ->
+        {
+            while ( taskData.getWorkInProgress().get() > 0 )
+            {
+                final FileSummaryInformation next = taskData.getOutputQueue().poll();
+
+                if ( next == null )
+                {
+                    TimeDuration.of( 20, TimeDuration.Unit.MILLISECONDS ).pause();
+                }
+                else
+                {
+                    return Optional.of( next );
+                }
+            }
+            return Optional.empty();
+        },
+                () ->
+        {
+            executor.shutdown();
+            final Map<String, String> debugInfo = new LinkedHashMap<>();
+            debugInfo.put( "bytes", StringUtil.formatDiskSizeforDebug( taskData.getByteCount().get() ) );
+            debugInfo.put( "files", Integer.toString( taskData.getFileCount().get() ) );
+            debugInfo.put( "duration", TimeDuration.compactFromCurrent( startTime ) );
+            LOGGER.trace( () -> "completed file summary load for operation '" + operation + ", " + StringUtil.mapToString( debugInfo ) );
+        } );
     }
 
-    private static class RecursiveFileReaderTask extends RecursiveTask<List<FileSummaryInformation>>
+    @Value
+    private static class TaskData
+    {
+        private final AtomicLong byteCount = new AtomicLong( 0 );
+        private final AtomicInteger fileCount = new AtomicInteger( 0 );
+        private final AtomicInteger workInProgress = new AtomicInteger( 0 );
+        private final Queue<FileSummaryInformation> outputQueue = new ConcurrentLinkedQueue<>();
+
+        private Executor executor;
+
+        TaskData( final Executor executor )
+        {
+            this.executor = executor;
+        }
+    }
+
+    private static class RecursiveFileReaderTask implements Runnable
     {
         private final File theFile;
+        private final TaskData taskData;
 
-        RecursiveFileReaderTask( final File theFile )
+        RecursiveFileReaderTask( final File theFile, final TaskData taskData )
         {
             Objects.requireNonNull( theFile );
+            Objects.requireNonNull( taskData );
             this.theFile = theFile;
+            this.taskData = taskData;
+            this.taskData.getWorkInProgress().incrementAndGet();
         }
 
         @Override
-        protected List<FileSummaryInformation> compute()
+        public void run()
         {
-            final List<FileSummaryInformation> results = new ArrayList<>();
-
-            if ( theFile.isDirectory() )
+            try
             {
-                final File[] subFiles = theFile.listFiles();
-                if ( subFiles != null )
+                if ( theFile.isDirectory() )
                 {
-                    final List<RecursiveFileReaderTask> tasks = new ArrayList<>();
-                    for ( final File file : subFiles )
+                    final File[] subFiles = theFile.listFiles();
+                    if ( subFiles != null )
                     {
-                        final RecursiveFileReaderTask newTask = new RecursiveFileReaderTask( file );
-                        newTask.fork();
-                        tasks.add( newTask );
+                        for ( final File file : subFiles )
+                        {
+                            final RecursiveFileReaderTask newTask = new RecursiveFileReaderTask( file, taskData );
+                            taskData.getExecutor().execute( newTask );
+                        }
                     }
-                    tasks.forEach( recursiveFileReaderTask -> results.addAll( recursiveFileReaderTask.join() ) );
                 }
-            }
-            else
-            {
-                try
+                else
                 {
-                    results.add( fileInformationForFile( theFile ) );
-                }
-                catch ( Exception e )
-                {
-                    LOGGER.debug( () -> "error executing file summary reader: " + e.getMessage() );
+                    try
+                    {
+                        if ( theFile.exists() )
+                        {
+                            final FileSummaryInformation fileSummaryInformation = new FileSummaryInformation(
+                                    theFile.getName(),
+                                    theFile.getParentFile().getAbsolutePath(),
+                                    Instant.ofEpochMilli( theFile.lastModified() ),
+                                    theFile.length(),
+                                    crc32( theFile )
+                            );
+                            taskData.getByteCount().addAndGet( fileSummaryInformation.getSize() );
+                            taskData.getFileCount().incrementAndGet();
+                            taskData.getOutputQueue().offer( fileSummaryInformation );
+                        }
+                    }
+                    catch ( Exception e )
+                    {
+                        LOGGER.debug( () -> "error executing file summary reader: " + e.getMessage() );
+                    }
                 }
             }
-
-            return Collections.unmodifiableList( results );
+            finally
+            {
+                this.taskData.getWorkInProgress().decrementAndGet();
+            }
         }
     }
 
-    private static FileSummaryInformation fileInformationForFile( final File file )
-            throws IOException
-    {
-        if ( file == null || !file.exists() )
-        {
-            return null;
-        }
-        return new FileSummaryInformation(
-                file.getName(),
-                file.getParentFile().getAbsolutePath(),
-                Instant.ofEpochMilli( file.lastModified() ),
-                file.length(),
-                crc32( file )
-        );
-    }
 
     public static long getFileDirectorySize( final File dir )
     {
@@ -260,9 +307,9 @@ public class FileSystemUtility
             throws IOException
     {
         final CRC32 crc32 = new CRC32();
-        final FileInputStream fileInputStream = new FileInputStream( file );
-        final FileChannel fileChannel = fileInputStream.getChannel();
-        final ByteBuffer byteBuffer = ByteBuffer.allocateDirect( 1024 );
+        final FileChannel fileChannel = FileChannel.open( file.toPath() );
+        final int bufferSize = (int) Math.min( file.length(), CRC_BUFFER_SIZE );
+        final ByteBuffer byteBuffer = ByteBuffer.allocateDirect( bufferSize );
 
         while ( fileChannel.read( byteBuffer ) > 0 )
         {

server/src/main/java/password/pwm/util/secure/SecureEngine.java

@@ -20,7 +20,6 @@
 
 package password.pwm.util.secure;
 
-import org.apache.commons.io.IOUtils;
 import password.pwm.PwmConstants;
 import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
@@ -37,7 +36,6 @@ import javax.crypto.spec.GCMParameterSpec;
 import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Writer;
@@ -279,13 +277,12 @@ public class SecureEngine
     )
             throws PwmUnrecoverableException
     {
-        FileInputStream fileInputStream = null;
         try
         {
             final MessageDigest messageDigest = MessageDigest.getInstance( hashAlgorithm.getAlgName() );
-            fileInputStream = new FileInputStream( file );
-            final FileChannel fileChannel = fileInputStream.getChannel();
-            final ByteBuffer byteBuffer = ByteBuffer.allocateDirect( HASH_FILE_BUFFER_SIZE );
+            final int bufferSize = (int) Math.min( file.length(), HASH_FILE_BUFFER_SIZE );
+            final FileChannel fileChannel = FileChannel.open( file.toPath() );
+            final ByteBuffer byteBuffer = ByteBuffer.allocateDirect( bufferSize );
 
             while ( fileChannel.read( byteBuffer ) > 0 )
             {
@@ -307,10 +304,6 @@ public class SecureEngine
             final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_CRYPT_ERROR, errorMsg );
             throw new PwmUnrecoverableException( errorInformation );
         }
-        finally
-        {
-            IOUtils.closeQuietly( fileInputStream );
-        }
     }
 
     public static String hash(