wordlist disk space checking

server/src/main/java/password/pwm/AppProperty.java

@@ -196,6 +196,7 @@ public enum AppProperty
     HEALTHCHECK_MAX_FORCE_WAIT                      ( "healthCheck.maximumForceCheckWaitSeconds" ),
     HEALTH_SUPPORT_BUNDLE_WRITE_INTERVAL_SECONDS    ( "health.supportBundle.file.writeIntervalSeconds" ),
     HEALTH_SUPPORT_BUNDLE_FILE_WRITE_COUNT          ( "health.supportBundle.file.writeRetentionCount" ),
+    HEALTH_DISK_MIN_FREE_WARNING                    ( "health.disk.minFreeWarning" ),
     HEALTH_CERTIFICATE_WARN_SECONDS                 ( "health.certificate.warnSeconds" ),
     HEALTH_LDAP_CAUTION_DURATION_MS                 ( "health.ldap.cautionDurationMS" ),
     HEALTH_LDAP_PROXY_WARN_PW_EXPIRE_SECONDS        ( "health.ldap.proxy.pwExpireWarnSeconds" ),
@@ -363,6 +364,7 @@ public enum AppProperty
     WORDLIST_CHAR_LENGTH_MIN                        ( "wordlist.minCharLength" ),
     WORDLIST_IMPORT_AUTO_IMPORT_RECHECK_SECONDS     ( "wordlist.import.autoImportRecheckSeconds" ),
     WORDLIST_IMPORT_DURATION_GOAL_MS                ( "wordlist.import.durationGoalMS" ),
+    WORDLIST_IMPORT_MIN_FREE_SPACE                  ( "wordlist.import.minFreeSpace" ),
     WORDLIST_IMPORT_MIN_TRANSACTIONS                ( "wordlist.import.minTransactions" ),
     WORDLIST_IMPORT_MAX_TRANSACTIONS                ( "wordlist.import.maxTransactions" ),
     WORDLIST_IMPORT_MAX_CHARS_TRANSACTIONS          ( "wordlist.import.maxCharsTransactions" ),

server/src/main/java/password/pwm/config/profile/NewUserProfile.java

@@ -35,7 +35,7 @@ import password.pwm.error.ErrorInformation;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.util.java.TimeDuration;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 
 import java.time.Instant;
 import java.util.HashMap;

server/src/main/java/password/pwm/health/ConfigurationChecker.java

@@ -45,7 +45,7 @@ import password.pwm.util.PasswordData;
 import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -194,30 +194,31 @@ public class ConfigurationChecker implements HealthChecker
 
         for ( final PwmSetting setting : PwmSetting.values() )
         {
-            if ( setting.getSyntax() == PwmSettingSyntax.PASSWORD )
+            if (
+                    setting.getSyntax() == PwmSettingSyntax.PASSWORD
+                            && !setting.getCategory().hasProfiles()
+                            && !config.isDefaultValue( setting )
+            )
             {
-                if ( !setting.getCategory().hasProfiles() )
+                try
                 {
-                    if ( !config.isDefaultValue( setting ) )
+                    final PasswordData passwordValue = config.readSettingAsPassword( setting );
+                    final String stringValue = passwordValue.getStringValue();
+                    if ( !StringUtil.isEmpty( stringValue ) )
                     {
-                        try
+                        final int strength = PasswordUtility.judgePasswordStrength( config, stringValue );
+                        if ( strength < 50 )
                         {
-                            final PasswordData passwordValue = config.readSettingAsPassword( setting );
-                            final int strength = PasswordUtility.judgePasswordStrength( config,
-                                    passwordValue.getStringValue() );
-                            if ( strength < 50 )
-                            {
-                                records.add( HealthRecord.forMessage( HealthMessage.Config_WeakPassword,
-                                        setting.toMenuLocationDebug( null, locale ), String.valueOf( strength ) ) );
-                            }
-                        }
-                        catch ( Exception e )
-                        {
-                            LOGGER.error( SessionLabel.HEALTH_SESSION_LABEL, "error while inspecting setting "
-                                    + setting.toMenuLocationDebug( null, locale ) + ", error: " + e.getMessage() );
+                            records.add( HealthRecord.forMessage( HealthMessage.Config_WeakPassword,
+                                    setting.toMenuLocationDebug( null, locale ), String.valueOf( strength ) ) );
                         }
                     }
                 }
+                catch ( Exception e )
+                {
+                    LOGGER.error( SessionLabel.HEALTH_SESSION_LABEL, "error while inspecting setting "
+                            + setting.toMenuLocationDebug( null, locale ) + ", error: " + e.getMessage() );
+                }
             }
         }
         for ( final LdapProfile profile : config.getLdapProfiles().values() )

server/src/main/java/password/pwm/health/HealthMessage.java

@@ -83,6 +83,7 @@ public enum HealthMessage
     LocalDB_BAD( HealthStatus.WARN, HealthTopic.LocalDB ),
     LocalDB_NEW( HealthStatus.WARN, HealthTopic.LocalDB ),
     LocalDB_CLOSED( HealthStatus.WARN, HealthTopic.LocalDB ),
+    LocalDB_LowDiskSpace( HealthStatus.WARN, HealthTopic.LocalDB ),
     LocalDBLogger_NOTOPEN( HealthStatus.CAUTION, HealthTopic.LocalDB ),
     LocalDBLogger_HighRecordCount( HealthStatus.CAUTION, HealthTopic.LocalDB ),
     LocalDBLogger_OldRecordPresent( HealthStatus.CAUTION, HealthTopic.LocalDB ),
@@ -91,7 +92,8 @@ public enum HealthMessage
     ServiceClosed_LocalDBUnavail( HealthStatus.CAUTION, HealthTopic.Application ),
     ServiceClosed_AppReadOnly( HealthStatus.CAUTION, HealthTopic.Application ),
     SMS_SendFailure( HealthStatus.WARN, HealthTopic.SMS ),
-    Wordlist_AutoImportFailure( HealthStatus.WARN, HealthTopic.Configuration ),;
+    Wordlist_AutoImportFailure( HealthStatus.WARN, HealthTopic.Configuration ),
+    Wordlist_ImportInProgress( HealthStatus.CAUTION, HealthTopic.Application ),;
 
     private final HealthStatus status;
     private final HealthTopic topic;

server/src/main/java/password/pwm/health/LDAPHealthChecker.java

@@ -54,13 +54,13 @@ import password.pwm.ldap.LdapOperationsHelper;
 import password.pwm.ldap.UserInfo;
 import password.pwm.ldap.UserInfoFactory;
 import password.pwm.util.PasswordData;
-import password.pwm.util.password.RandomPasswordGenerator;
 import password.pwm.util.i18n.LocaleHelper;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
+import password.pwm.util.password.RandomPasswordGenerator;
 import password.pwm.ws.server.rest.bean.HealthData;
 
 import java.io.Serializable;
@@ -78,6 +78,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 public class LDAPHealthChecker implements HealthChecker
@@ -857,14 +858,11 @@ public class LDAPHealthChecker implements HealthChecker
                             final String value = config.getLdapProfiles().get( profile ).readSettingAsString( pwmSetting );
                             if ( value != null && !value.isEmpty() )
                             {
-                                final String errorMsg = validateDN( pwmApplication, value, profile );
-                                if ( errorMsg != null )
-                                {
-                                    returnList.add( HealthRecord.forMessage(
-                                            HealthMessage.Config_DNValueValidity,
-                                            pwmSetting.toMenuLocationDebug( profile, PwmConstants.DEFAULT_LOCALE ), errorMsg )
-                                    );
-                                }
+                                final Optional<String> errorMsg = validateDN( pwmApplication, value, profile );
+                                errorMsg.ifPresent( s -> returnList.add( HealthRecord.forMessage(
+                                        HealthMessage.Config_DNValueValidity,
+                                        pwmSetting.toMenuLocationDebug( profile, PwmConstants.DEFAULT_LOCALE ), s )
+                                ) );
                             }
                         }
                         else if ( pwmSetting.getSyntax() == PwmSettingSyntax.STRING_ARRAY )
@@ -874,14 +872,11 @@ public class LDAPHealthChecker implements HealthChecker
                             {
                                 for ( final String value : values )
                                 {
-                                    final String errorMsg = validateDN( pwmApplication, value, profile );
-                                    if ( errorMsg != null )
-                                    {
-                                        returnList.add( HealthRecord.forMessage(
-                                                HealthMessage.Config_DNValueValidity,
-                                                pwmSetting.toMenuLocationDebug( profile, PwmConstants.DEFAULT_LOCALE ), errorMsg )
-                                        );
-                                    }
+                                    final Optional<String> errorMsg = validateDN( pwmApplication, value, profile );
+                                    errorMsg.ifPresent( s -> returnList.add( HealthRecord.forMessage(
+                                            HealthMessage.Config_DNValueValidity,
+                                            pwmSetting.toMenuLocationDebug( profile, PwmConstants.DEFAULT_LOCALE ), s )
+                                    ) );
                                 }
                             }
                         }
@@ -985,7 +980,7 @@ public class LDAPHealthChecker implements HealthChecker
             }
             else
             {
-                if ( config.getLdapProfiles().keySet().contains( configuredLdapProfileID ) )
+                if ( config.getLdapProfiles().containsKey( configuredLdapProfileID ) )
                 {
                     ldapProfilesToCheck.add( configuredLdapProfileID );
                 }
@@ -1009,11 +1004,10 @@ public class LDAPHealthChecker implements HealthChecker
                     final String groupDN = userPermission.getLdapBase();
                     if ( groupDN != null && !isExampleDN( groupDN ) )
                     {
-                        final String errorMsg = validateDN( pwmApplication, groupDN, ldapProfileID );
-                        if ( errorMsg != null )
-                        {
-                            returnList.add( HealthRecord.forMessage( HealthMessage.Config_UserPermissionValidity, settingDebugName, "groupDN: " + errorMsg ) );
-                        }
+                        final Optional<String> errorMsg = validateDN( pwmApplication, groupDN, ldapProfileID );
+                        errorMsg.ifPresent( s -> returnList.add( HealthRecord.forMessage(
+                                HealthMessage.Config_UserPermissionValidity,
+                                settingDebugName, "groupDN: " + s ) ) );
                     }
                 }
                 break;
@@ -1023,11 +1017,10 @@ public class LDAPHealthChecker implements HealthChecker
                     final String baseDN = userPermission.getLdapBase();
                     if ( baseDN != null && !isExampleDN( baseDN ) )
                     {
-                        final String errorMsg = validateDN( pwmApplication, baseDN, ldapProfileID );
-                        if ( errorMsg != null )
-                        {
-                            returnList.add( HealthRecord.forMessage( HealthMessage.Config_UserPermissionValidity, settingDebugName, "baseDN: " + errorMsg ) );
-                        }
+                        final Optional<String> errorMsg = validateDN( pwmApplication, baseDN, ldapProfileID );
+                        errorMsg.ifPresent( s -> returnList.add( HealthRecord.forMessage(
+                                HealthMessage.Config_UserPermissionValidity,
+                                settingDebugName, "baseDN: " + s ) ) );
                     }
                 }
                 break;
@@ -1039,9 +1032,18 @@ public class LDAPHealthChecker implements HealthChecker
         return returnList;
     }
 
-    private static String validateDN( final PwmApplication pwmApplication, final String dnValue, final String ldapProfileID )
+    private static Optional<String> validateDN(
+            final PwmApplication pwmApplication,
+            final String dnValue,
+            final String ldapProfileID
+    )
             throws PwmUnrecoverableException
     {
+        if ( StringUtil.isEmpty( dnValue ) )
+        {
+            return Optional.empty();
+        }
+
         final ChaiProvider chaiProvider = pwmApplication.getProxyChaiProvider( ldapProfileID );
         try
         {
@@ -1050,15 +1052,15 @@ public class LDAPHealthChecker implements HealthChecker
                 final ChaiEntry baseDNEntry = chaiProvider.getEntryFactory().newChaiEntry( dnValue );
                 if ( !baseDNEntry.exists() )
                 {
-                    return "DN '" + dnValue + "' is invalid";
+                    return Optional.of( "DN '" + dnValue + "' is invalid" );
                 }
                 else
                 {
                     final String canonicalDN = baseDNEntry.readCanonicalDN();
                     if ( !dnValue.equals( canonicalDN ) )
                     {
-                        return "DN '" + dnValue + "' is not the correct canonical value, the server reports the canonical value as '"
-                                + canonicalDN + "'";
+                        return Optional.of( "DN '" + dnValue + "' is not the correct canonical value, the server reports the canonical value as '"
+                                + canonicalDN + "'" );
                     }
                 }
             }
@@ -1071,19 +1073,21 @@ public class LDAPHealthChecker implements HealthChecker
         {
             LOGGER.error( "error while evaluating ldap DN '" + dnValue + "', error: " + e.getMessage() );
         }
-        return null;
+        return Optional.empty();
     }
 
     private static boolean isExampleDN( final String dnValue )
     {
-        if ( dnValue == null )
+        if ( StringUtil.isEmpty( dnValue ) )
         {
             return false;
         }
+
         final String[] exampleSuffixes = new String[] {
                 "DC=site,DC=example,DC=net",
                 "ou=groups,o=example",
         };
+
         for ( final String suffix : exampleSuffixes )
         {
             if ( dnValue.endsWith( suffix ) )

server/src/main/java/password/pwm/health/LocalDBHealthChecker.java

@@ -20,10 +20,16 @@
 
 package password.pwm.health;
 
+import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
+import password.pwm.config.Configuration;
+import password.pwm.util.java.FileSystemUtility;
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.StringUtil;
 import password.pwm.util.localdb.LocalDB;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 public class LocalDBHealthChecker implements HealthChecker
@@ -58,6 +64,8 @@ public class LocalDBHealthChecker implements HealthChecker
             return healthRecords;
         }
 
+        healthRecords.addAll( checkSpaceRemaining( pwmApplication ) );
+
         if ( healthRecords.isEmpty() )
         {
             healthRecords.add( HealthRecord.forMessage( HealthMessage.LocalDB_OK ) );
@@ -65,4 +73,19 @@ public class LocalDBHealthChecker implements HealthChecker
 
         return healthRecords;
     }
+
+    private List<HealthRecord> checkSpaceRemaining( final PwmApplication pwmApplication )
+    {
+        final Configuration configuration = pwmApplication.getConfig();
+        final long minFreeSpace = JavaHelper.silentParseLong( configuration.readAppProperty( AppProperty.HEALTH_DISK_MIN_FREE_WARNING ), 500_000_000 );
+        final long freeSpace = FileSystemUtility.diskSpaceRemaining( pwmApplication.getLocalDB().getFileLocation() );
+
+        if ( freeSpace < minFreeSpace )
+        {
+            final String spaceValue = StringUtil.formatDiskSizeforDebug( freeSpace );
+            return Collections.singletonList( HealthRecord.forMessage( HealthMessage.LocalDB_LowDiskSpace, spaceValue ) );
+        }
+
+        return Collections.emptyList();
+    }
 }

server/src/main/java/password/pwm/http/servlet/GuestRegistrationServlet.java

@@ -60,7 +60,7 @@ import password.pwm.util.java.JavaHelper;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
 import password.pwm.util.operations.ActionExecutor;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 
 import javax.servlet.ServletException;
 import javax.servlet.annotation.WebServlet;

server/src/main/java/password/pwm/http/servlet/admin/UserDebugDataReader.java

@@ -40,7 +40,7 @@ import password.pwm.svc.PwmService;
 import password.pwm.svc.pwnotify.PwNotifyUserStatus;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 
 import java.util.Collections;
 import java.util.List;

server/src/main/java/password/pwm/http/servlet/changepw/ChangePasswordServlet.java

@@ -59,7 +59,7 @@ import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 import password.pwm.util.password.PwmPasswordRuleValidator;
 import password.pwm.util.password.RandomPasswordGenerator;
 import password.pwm.ws.server.RestResultBean;

server/src/main/java/password/pwm/http/servlet/changepw/ChangePasswordServletUtil.java

@@ -46,7 +46,7 @@ import password.pwm.ldap.auth.AuthenticationType;
 import password.pwm.svc.event.AuditEvent;
 import password.pwm.util.PasswordData;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 
 import java.util.Locale;
 import java.util.Map;

server/src/main/java/password/pwm/http/servlet/configmanager/ConfigManagerWordlistServlet.java

@@ -274,9 +274,27 @@ public class ConfigManagerWordlistServlet extends AbstractPwmServlet
                             "Last Import Attempt",
                             JavaHelper.toIsoDate( wordlist.getAutoImportError().getDate() ) ) );
                 }
+
+                if ( activity == Wordlist.Activity.Importing )
+                {
+                    final String percentComplete = wordlist.getImportPercentComplete();
+                    if ( !StringUtil.isEmpty( percentComplete ) )
+                    {
+                        presentableValues.add( new DisplayElement(
+                                "percentComplete",
+                                DisplayElement.Type.string,
+                                "Percent Complete",
+                                percentComplete ) );
+
+                    }
+                }
+
                 builder.presentableData( Collections.unmodifiableList( presentableValues ) );
             }
 
+
+
+
             if ( wordlistStatus.isCompleted() )
             {
                 if ( wordlistConfiguration.getAutoImportUrl() == null )

server/src/main/java/password/pwm/http/servlet/forgottenpw/ForgottenPasswordServlet.java

@@ -82,7 +82,7 @@ import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 import password.pwm.util.operations.cr.NMASCrOperator;
 import password.pwm.util.operations.otp.OTPUserRecord;
 import password.pwm.ws.server.RestResultBean;

server/src/main/java/password/pwm/http/servlet/forgottenpw/ForgottenPasswordStageProcessor.java

@@ -40,7 +40,7 @@ import password.pwm.ldap.UserInfo;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.util.java.JsonUtil;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 
 import java.util.ArrayList;
 import java.util.Collections;

server/src/main/java/password/pwm/http/servlet/forgottenpw/ForgottenPasswordStateMachine.java

@@ -75,7 +75,7 @@ import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 import password.pwm.util.operations.otp.OTPUserRecord;
 import password.pwm.ws.server.PresentableForm;
 import password.pwm.ws.server.PresentableFormRow;

server/src/main/java/password/pwm/http/servlet/forgottenpw/ForgottenPasswordUtil.java

@@ -70,7 +70,7 @@ import password.pwm.util.PasswordData;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 import password.pwm.util.password.RandomPasswordGenerator;
 
 import javax.servlet.ServletException;

server/src/main/java/password/pwm/http/servlet/helpdesk/HelpdeskServlet.java

@@ -85,7 +85,7 @@ import password.pwm.util.macro.MacroMachine;
 import password.pwm.util.operations.ActionExecutor;
 import password.pwm.util.operations.CrService;
 import password.pwm.util.operations.OtpService;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 import password.pwm.util.operations.otp.OTPUserRecord;
 import password.pwm.util.password.RandomPasswordGenerator;
 import password.pwm.util.secure.SecureService;

server/src/main/java/password/pwm/http/servlet/newuser/NewUserServlet.java

@@ -61,7 +61,7 @@ import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 import password.pwm.ws.server.RestResultBean;
 import password.pwm.ws.server.rest.RestCheckPasswordServer;
 import password.pwm.ws.server.rest.RestFormSigningServer;

server/src/main/java/password/pwm/http/servlet/newuser/NewUserUtils.java

@@ -71,7 +71,7 @@ import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
 import password.pwm.util.operations.ActionExecutor;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 import password.pwm.ws.client.rest.form.FormDataRequestBean;
 import password.pwm.ws.client.rest.form.FormDataResponseBean;
 import password.pwm.ws.client.rest.form.RestFormDataClient;

server/src/main/java/password/pwm/ldap/PasswordChangeProgressChecker.java

@@ -35,7 +35,7 @@ import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.Percent;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 
 import java.io.Serializable;
 import java.math.BigDecimal;

server/src/main/java/password/pwm/ldap/UserInfoReader.java

@@ -60,7 +60,7 @@ import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.operations.CrService;
 import password.pwm.util.operations.OtpService;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 import password.pwm.util.operations.otp.OTPUserRecord;
 import password.pwm.util.password.PwmPasswordRuleValidator;
 

server/src/main/java/password/pwm/ldap/auth/LDAPAuthenticationRequest.java

@@ -62,7 +62,7 @@ import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogLevel;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 
 import java.time.Instant;
 import java.util.Collections;

server/src/main/java/password/pwm/svc/token/TokenService.java

@@ -66,7 +66,7 @@ import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBDataStore;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 import password.pwm.util.secure.PwmRandom;
 
 import java.time.Instant;

server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java

@@ -36,6 +36,7 @@ import password.pwm.svc.PwmService;
 import password.pwm.util.PwmScheduler;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.Percent;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 
@@ -289,6 +290,16 @@ abstract class AbstractWordlist implements Wordlist, PwmService
             returnList.add( healthRecord );
         }
 
+        if ( backgroundImportRunning.get() )
+        {
+            final Activity activity = getActivity();
+            final String suffix = "("
+                    + ( ( activity == Activity.Importing ) ? getImportPercentComplete() : activity.getLabel() )
+                    + ")";
+            final HealthRecord healthRecord = HealthRecord.forMessage( HealthMessage.Wordlist_ImportInProgress, suffix );
+            returnList.add( healthRecord );
+        }
+
         if ( lastError != null )
         {
             final HealthRecord healthRecord = new HealthRecord( HealthStatus.WARN, HealthTopic.Application, this.getClass().getName() + " error: " + lastError.toDebugStr() );
@@ -474,4 +485,29 @@ abstract class AbstractWordlist implements Wordlist, PwmService
         return statistics;
     }
 
+    @Override
+    public String getImportPercentComplete()
+    {
+        if ( backgroundImportRunning.get() )
+        {
+            final WordlistStatus wordlistStatus = readWordlistStatus();
+            if ( wordlistStatus != null )
+            {
+                final WordlistSourceInfo wordlistSourceInfo = wordlistStatus.getRemoteInfo();
+                if ( wordlistSourceInfo != null )
+                {
+                    final long totalBytes = wordlistSourceInfo.getBytes();
+                    final long importBytes = wordlistStatus.getBytes();
+                    if ( importBytes > 0 && totalBytes > 0 )
+                    {
+                        final Percent percent = new Percent( importBytes, totalBytes );
+                        return percent.pretty( 3 );
+                    }
+                }
+
+            }
+        }
+
+        return "";
+    }
 }

server/src/main/java/password/pwm/svc/wordlist/LocalDBWordlistBucket.java

@@ -23,6 +23,7 @@ package password.pwm.svc.wordlist;
 import password.pwm.PwmApplication;
 import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.java.FileSystemUtility;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBException;
 
@@ -131,4 +132,10 @@ class LocalDBWordlistBucket extends AbstractWordlistBucket implements WordlistBu
         final PwmApplication.AppAttribute appAttribute = wordlistConfiguration.getMetaDataAppAttribute();
         pwmApplication.writeAppAttribute( appAttribute, wordlistStatus );
     }
+
+    @Override
+    public long spaceRemaining()
+    {
+        return FileSystemUtility.diskSpaceRemaining( localDB.getFileLocation() );
+    }
 }

server/src/main/java/password/pwm/svc/wordlist/MemoryWordlistBucket.java

@@ -82,4 +82,10 @@ public class MemoryWordlistBucket extends AbstractWordlistBucket
     {
         this.wordlistStatus = wordlistStatus;
     }
+
+    @Override
+    public long spaceRemaining()
+    {
+        return Long.MAX_VALUE;
+    }
 }

server/src/main/java/password/pwm/svc/wordlist/Wordlist.java

@@ -47,6 +47,8 @@ public interface Wordlist extends PwmService
 
     AbstractWordlist.Activity getActivity();
 
+    String getImportPercentComplete();
+
     enum Activity
     {
         Idle( "Idle" ),

server/src/main/java/password/pwm/svc/wordlist/WordlistBucket.java

@@ -41,4 +41,6 @@ public interface WordlistBucket
     WordlistStatus readWordlistStatus();
 
     void writeWordlistStatus( WordlistStatus wordlistStatus );
+
+    long spaceRemaining();
 }

server/src/main/java/password/pwm/svc/wordlist/WordlistConfiguration.java

@@ -47,7 +47,7 @@ import java.util.function.Supplier;
 @Builder( toBuilder = true )
 public class WordlistConfiguration implements Serializable
 {
-    static final int STREAM_BUFFER_SIZE = 1024_1024;
+    static final int STREAM_BUFFER_SIZE = 1_1024_1024;
     static final PwmHashAlgorithm HASH_ALGORITHM = PwmHashAlgorithm.SHA1;
 
     private final boolean caseSensitive;
@@ -69,6 +69,7 @@ public class WordlistConfiguration implements Serializable
     private final int importMinTransactions;
     private final int importMaxTransactions;
     private final long importMaxChars;
+    private final long importMinFreeSpace;
 
     private final TimeDuration inspectorFrequency;
 
@@ -132,6 +133,7 @@ public class WordlistConfiguration implements Serializable
                 .inspectorFrequency( TimeDuration.of(
                         Long.parseLong( configuration.readAppProperty( AppProperty.WORDLIST_INSPECTOR_FREQUENCY_SECONDS ) ),
                         TimeDuration.Unit.SECONDS ) )
+                .importMinFreeSpace( JavaHelper.silentParseLong( configuration.readAppProperty( AppProperty.WORDLIST_IMPORT_MIN_FREE_SPACE ), 100_000_000 ) )
                 .build();
     }
 
@@ -160,7 +162,7 @@ public class WordlistConfiguration implements Serializable
     {
         try
         {
-            return SecureEngine.hash( JsonUtil.serialize( WordlistConfiguration.this ), PwmHashAlgorithm.SHA1 );
+            return SecureEngine.hash( JsonUtil.serialize( WordlistConfiguration.this ), HASH_ALGORITHM );
         }
         catch ( PwmUnrecoverableException e )
         {

server/src/main/java/password/pwm/svc/wordlist/WordlistImporter.java

@@ -158,6 +158,8 @@ class WordlistImporter implements Runnable
                     .build() );
         }
 
+        checkWordlistSpaceRemaining();
+
         final long previousBytesRead = rootWordlist.readWordlistStatus().getBytes();
         seenWordTypes.putAll( rootWordlist.readWordlistStatus().getWordTypes() );
 
@@ -213,6 +215,7 @@ class WordlistImporter implements Runnable
                     {
                         flushBuffer();
                         metaUpdater.conditionallyExecuteTask();
+                        checkWordlistSpaceRemaining();
                     }
                 }
             }
@@ -282,10 +285,6 @@ class WordlistImporter implements Runnable
         }
     }
 
-    private void updateStatistics( final String word )
-    {
-    }
-
     private void flushBuffer( )
             throws PwmUnrecoverableException
     {
@@ -473,4 +472,21 @@ class WordlistImporter implements Runnable
         private final MovingAverage chunksPerWord = new MovingAverage( TimeDuration.MINUTE );
         private final MovingAverage averageWordLength = new MovingAverage( TimeDuration.MINUTE );
     }
+
+    private void checkWordlistSpaceRemaining()
+            throws PwmUnrecoverableException
+    {
+        final long freeSpace = wordlistBucket.spaceRemaining();
+        final long minFreeSpace = rootWordlist.getConfiguration().getImportMinFreeSpace();
+        if ( freeSpace < minFreeSpace )
+        {
+            final String msg = "free space remaining for wordlist storage is " + StringUtil.formatDiskSizeforDebug( freeSpace )
+                    + " which is less than the minimum of "
+                    + StringUtil.formatDiskSizeforDebug( minFreeSpace )
+                    + ", aborting import";
+
+            final ErrorInformation errorInformation = new ErrorInformation( PwmError.ERROR_WORDLIST_IMPORT_ERROR, msg );
+            throw new PwmUnrecoverableException( errorInformation );
+        }
+    }
 }

server/src/main/java/password/pwm/util/java/FileSystemUtility.java

@@ -30,7 +30,6 @@ import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
-import java.lang.reflect.Method;
 import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
@@ -139,39 +138,19 @@ public class FileSystemUtility
 
     public static long getFileDirectorySize( final File dir )
     {
-        long size = 0;
         try
         {
-            if ( dir.isFile() )
-            {
-                size = dir.length();
-            }
-            else
-            {
-                final File[] subFiles = dir.listFiles();
-                if ( subFiles != null )
-                {
-                    for ( final File file : subFiles )
-                    {
-                        if ( file.isFile() )
-                        {
-                            size += file.length();
-                        }
-                        else
-                        {
-                            size += getFileDirectorySize( file );
-                        }
-
-                    }
-                }
-            }
+            return Files.walk( dir.toPath() )
+                    .filter( path -> path.toFile().isFile() )
+                    .mapToLong( path -> path.toFile().length() )
+                    .sum();
         }
-        catch ( NullPointerException e )
+        catch ( IOException e )
         {
-            // file was deleted before file size could be read
+            LOGGER.error( "error calculating disk size of '" + dir.getAbsolutePath() + "', error: " + e.getMessage(), e );
         }
 
-        return size;
+        return -1;
     }
 
     public static File figureFilepath( final String filename, final File suggestedPath )
@@ -191,21 +170,7 @@ public class FileSystemUtility
 
     public static long diskSpaceRemaining( final File file )
     {
-        try
-        {
-            final Method getFreeSpaceMethod = File.class.getMethod( "getFreeSpace" );
-            final Object rawResult = getFreeSpaceMethod.invoke( file );
-            return ( Long ) rawResult;
-        }
-        catch ( NoSuchMethodException e )
-        {
-            /* no error, pre java 1.6 doesn't have this method */
-        }
-        catch ( Exception e )
-        {
-            LOGGER.debug( () -> "error reading file space remaining for " + file.toString() + ",: " + e.getMessage() );
-        }
-        return -1;
+        return file.getFreeSpace();
     }
 
     public static void rotateBackups( final File inputFile, final int maxRotate )

server/src/main/java/password/pwm/util/password/PasswordRuleChecks.java

@@ -41,7 +41,6 @@ import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.operations.PasswordUtility;
 
 import java.util.ArrayList;
 import java.util.Arrays;

server/src/main/java/password/pwm/util/operations/PasswordUtility.java → server/src/main/java/password/pwm/util/password/PasswordUtility.java

@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-package password.pwm.util.operations;
+package password.pwm.util.password;
 
 import com.novell.ldapchai.ChaiPasswordPolicy;
 import com.novell.ldapchai.ChaiUser;
@@ -83,11 +83,11 @@ import password.pwm.svc.stats.Statistic;
 import password.pwm.util.PasswordData;
 import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
-import password.pwm.util.password.PasswordCharCounter;
-import password.pwm.util.password.PwmPasswordRuleValidator;
+import password.pwm.util.operations.ActionExecutor;
 
 import java.io.Serializable;
 import java.time.Instant;
@@ -812,6 +812,11 @@ public class PasswordUtility
             final String password
     )
     {
+        if ( StringUtil.isEmpty( password ) )
+        {
+            return Integer.parseInt( configuration.readAppProperty( AppProperty.PASSWORD_STRENGTH_THRESHOLD_VERY_WEAK ) );
+        }
+
         final Zxcvbn zxcvbn = new Zxcvbn();
         final Strength strength = zxcvbn.measure( password );
 
@@ -836,9 +841,8 @@ public class PasswordUtility
     public static int judgePasswordStrengthUsingTraditionalAlgorithm(
             final String password
     )
-            throws PwmUnrecoverableException
     {
-        if ( password == null || password.length() < 1 )
+        if ( StringUtil.isEmpty( password ) )
         {
             return 0;
         }

server/src/main/java/password/pwm/util/password/RandomPasswordGenerator.java

@@ -40,7 +40,6 @@ import password.pwm.svc.wordlist.SeedlistService;
 import password.pwm.util.PasswordData;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.operations.PasswordUtility;
 import password.pwm.util.secure.PwmRandom;
 
 import java.time.Instant;

server/src/main/java/password/pwm/ws/server/rest/RestChallengesServer.java

@@ -47,7 +47,7 @@ import password.pwm.ldap.LdapOperationsHelper;
 import password.pwm.svc.stats.Statistic;
 import password.pwm.svc.stats.StatisticsManager;
 import password.pwm.util.operations.CrService;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 import password.pwm.ws.server.RestMethodHandler;
 import password.pwm.ws.server.RestRequest;
 import password.pwm.ws.server.RestResultBean;

server/src/main/java/password/pwm/ws/server/rest/RestCheckPasswordServer.java

@@ -42,7 +42,7 @@ import password.pwm.util.java.JsonUtil;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.java.TimeDuration;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 import password.pwm.ws.server.RestMethodHandler;
 import password.pwm.ws.server.RestRequest;
 import password.pwm.ws.server.RestResultBean;

server/src/main/java/password/pwm/ws/server/rest/RestRandomPasswordServer.java

@@ -37,7 +37,7 @@ import password.pwm.util.PasswordData;
 import password.pwm.util.password.RandomPasswordGenerator;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 import password.pwm.ws.server.RestMethodHandler;
 import password.pwm.ws.server.RestRequest;
 import password.pwm.ws.server.RestResultBean;

server/src/main/java/password/pwm/ws/server/rest/RestSetPasswordServer.java

@@ -40,7 +40,7 @@ import password.pwm.util.BasicAuthInfo;
 import password.pwm.util.PasswordData;
 import password.pwm.util.password.RandomPasswordGenerator;
 import password.pwm.util.logging.PwmLogger;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 import password.pwm.ws.server.RestMethodHandler;
 import password.pwm.ws.server.RestRequest;
 import password.pwm.ws.server.RestResultBean;

server/src/main/resources/password/pwm/AppProperty.properties

@@ -101,6 +101,7 @@ healthCheck.maximumForceCheckWaitSeconds=30
 health.supportBundle.file.writeIntervalSeconds=0
 health.supportBundle.file.writeRetentionCount=10
 health.certificate.warnSeconds=2592000
+health.disk.minFreeWarning=500000000
 health.ldap.cautionDurationMS=10800000
 health.ldap.proxy.pwExpireWarnSeconds=2592000
 health.java.maxThreads=1000
@@ -340,6 +341,7 @@ wordlist.minCharLength=2
 wordlist.import.autoImportRecheckSeconds=432000
 wordlist.import.durationGoalMS=1000
 wordlist.import.minTransactions=10
+wordlist.import.minFreeSpace=500000000
 wordlist.import.maxTransactions=100000
 wordlist.import.maxCharsTransactions=10485760
 wordlist.import.lineComments=!#comment:

server/src/main/resources/password/pwm/i18n/Health.properties

@@ -70,6 +70,7 @@ HealthMessage_TokenServiceError=An error occurred during the TokenService startu
 HealthMessage_Java_HighThreads=Java thread count is unusually large (%1% threads)
 HealthMessage_Java_SmallHeap=Java maximum memory heap size is set to default of 64MB.  Please increase the memory heap size.
 HealthMessage_Java_OK=Java platform is operating normally
+HealthMessage_LocalDB_LowDiskSpace=Free storage space remaining for LocalDB storage is %1%.  Free additional storage space to prevent a possible service interruption.
 HealthMessage_LocalDB_OK=LocalDB and related services are operating correctly
 HealthMessage_LocalDB_BAD=LocalDB is not online. Startup error: %1%
 HealthMessage_LocalDB_NEW=LocalDB status is NEW (loading) state, until LocalDB loads, statistics, online logging, word lists and other features are disabled
@@ -82,6 +83,7 @@ HealthMessage_ServiceClosed=unable to start %1% service   %2%
 HealthMessage_ServiceClosed_LocalDBUnavail=unable to start %1% service, LocalDB is not available
 HealthMessage_ServiceClosed_AppReadOnly=unable to start %1% service, application is in read-only mode
 HealthMessage_Wordlist_AutoImportFailure=Configured word list (%1%) failed to import due to error: %2% at timestamp %3%
+HealthMessage_Wordlist_ImportInProgress=Wordlist import is in progress.  Application performance may be degraded during import.  %1%
 HealthStatus_WARN=WARN
 HealthStatus_CAUTION=CAUTION
 HealthStatus_CONFIG=CONFIGURATION

server/src/test/java/password/pwm/tests/PwmPasswordJudgeTest.java

@@ -26,7 +26,7 @@ import org.mockito.Mockito;
 import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.option.StrengthMeterType;
-import password.pwm.util.operations.PasswordUtility;
+import password.pwm.util.password.PasswordUtility;
 
 import java.util.ArrayList;
 import java.util.List;