fix #2442 add adaptive.load.control

src/main/java/org/codelibs/fess/crawler/interval/FessIntervalController.java

@@ -55,6 +55,7 @@ public class FessIntervalController extends DefaultIntervalController {
 
     @Override
     protected void delayForWaitingNewUrl() {
+        ComponentUtil.getSystemHelper().calibrateCpuLoad();
         try {
             final IntervalControlHelper intervalControlHelper = ComponentUtil.getIntervalControlHelper();
             intervalControlHelper.checkCrawlerStatus();

src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java

@@ -64,6 +64,9 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
      */
     @Override
     public void store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
+        final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
+        systemHelper.calibrateCpuLoad();
+
         final long startTime = System.currentTimeMillis();
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
@@ -102,7 +105,6 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
         }
 
         if (!dataMap.containsKey(fessConfig.getIndexFieldDocId())) {
-            final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
             dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap));
         }
 

src/main/java/org/codelibs/fess/helper/SystemHelper.java

@@ -57,6 +57,7 @@ import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.config.Configurator;
 import org.codelibs.core.exception.IORuntimeException;
 import org.codelibs.core.lang.StringUtil;
+import org.codelibs.core.lang.ThreadUtil;
 import org.codelibs.core.misc.Pair;
 import org.codelibs.fess.Constants;
 import org.codelibs.fess.crawler.util.CharUtil;
@@ -69,6 +70,7 @@ import org.codelibs.fess.util.GsaConfigParser;
 import org.codelibs.fess.util.ParameterUtil;
 import org.codelibs.fess.util.ResourceUtil;
 import org.codelibs.fess.validation.FessActionValidator;
+import org.elasticsearch.monitor.os.OsProbe;
 import org.lastaflute.core.message.supplier.UserMessagesCreator;
 import org.lastaflute.web.TypicalAction;
 import org.lastaflute.web.response.HtmlResponse;
@@ -110,6 +112,12 @@ public class SystemHelper {
 
     protected long eolTime;
 
+    private short systemCpuPercent;
+
+    private long systemCpuCheckTime;
+
+    private long systemCpuCheckInterval = 1000L;
+
     @PostConstruct
     public void init() {
         if (logger.isDebugEnabled()) {
@@ -539,6 +547,38 @@ public class SystemHelper {
         }
     }
 
+    public void calibrateCpuLoad() {
+        final int percent = ComponentUtil.getFessConfig().getAdaptiveLoadControlAsInteger();
+        if (percent <= 0) {
+            return;
+        }
+        while (getSystemCpuPercent() > percent) {
+            if (logger.isInfoEnabled()) {
+                logger.info("Cpu Load {}% is greater than {}%.", getSystemCpuPercent(), percent);
+            }
+            ThreadUtil.sleep(systemCpuCheckInterval);
+        }
+    }
+
+    protected short getSystemCpuPercent() {
+        final long now = System.currentTimeMillis();
+        if (now - systemCpuCheckTime > systemCpuCheckInterval) {
+            synchronized (this) {
+                if (now - systemCpuCheckTime > systemCpuCheckInterval) {
+                    try {
+                        final OsProbe osProbe = OsProbe.getInstance();
+                        systemCpuPercent = osProbe.getSystemCpuPercent();
+                    } catch (Exception e) {
+                        logger.warn("Failed to get SystemCpuPercent.", e);
+                        return 0;
+                    }
+                    systemCpuCheckTime = now;
+                }
+            }
+        }
+        return systemCpuPercent;
+    }
+
     public String getVersion() {
         return version;
     }
@@ -554,4 +594,8 @@ public class SystemHelper {
     public String getProductVersion() {
         return productVersion;
     }
+
+    public void setSystemCpuCheckInterval(long systemCpuCheckInterval) {
+        this.systemCpuCheckInterval = systemCpuCheckInterval;
+    }
 }

src/main/java/org/codelibs/fess/indexer/IndexUpdater.java

@@ -194,6 +194,8 @@ public class IndexUpdater extends Thread {
                         ThreadUtil.sleep(interval); // 10 sec (default)
                     }
 
+                    systemHelper.calibrateCpuLoad();
+
                     docList.clear();
                     accessResultList.clear();
 

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -190,6 +190,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. 4000 */
     String MAX_LOG_OUTPUT_LENGTH = "max.log.output.length";
 
+    /** The key of the configuration. e.g. 0 */
+    String ADAPTIVE_LOAD_CONTROL = "adaptive.load.control";
+
     /** The key of the configuration. e.g. js */
     String SUPPORTED_UPLOADED_JS_EXTENTIONS = "supported.uploaded.js.extentions";
 
@@ -1895,6 +1898,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     Integer getMaxLogOutputLengthAsInteger();
 
+    /**
+     * Get the value for the key 'adaptive.load.control'. <br>
+     * The value is, e.g. 0 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getAdaptiveLoadControl();
+
+    /**
+     * Get the value for the key 'adaptive.load.control' as {@link Integer}. <br>
+     * The value is, e.g. 0 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     * @throws NumberFormatException When the property is not integer.
+     */
+    Integer getAdaptiveLoadControlAsInteger();
+
     /**
      * Get the value for the key 'supported.uploaded.js.extentions'. <br>
      * The value is, e.g. js <br>
@@ -6608,6 +6626,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return getAsInteger(FessConfig.MAX_LOG_OUTPUT_LENGTH);
         }
 
+        public String getAdaptiveLoadControl() {
+            return get(FessConfig.ADAPTIVE_LOAD_CONTROL);
+        }
+
+        public Integer getAdaptiveLoadControlAsInteger() {
+            return getAsInteger(FessConfig.ADAPTIVE_LOAD_CONTROL);
+        }
+
         public String getSupportedUploadedJsExtentions() {
             return get(FessConfig.SUPPORTED_UPLOADED_JS_EXTENTIONS);
         }
@@ -9088,6 +9114,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap.put(FessConfig.PATH_ENCODING, "UTF-8");
             defaultMap.put(FessConfig.USE_OWN_TMP_DIR, "true");
             defaultMap.put(FessConfig.MAX_LOG_OUTPUT_LENGTH, "4000");
+            defaultMap.put(FessConfig.ADAPTIVE_LOAD_CONTROL, "0");
             defaultMap.put(FessConfig.SUPPORTED_UPLOADED_JS_EXTENTIONS, "js");
             defaultMap.put(FessConfig.SUPPORTED_UPLOADED_CSS_EXTENTIONS, "css");
             defaultMap.put(FessConfig.SUPPORTED_UPLOADED_MEDIA_EXTENTIONS, "jpg,jpeg,gif,png,swf");

src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java

@@ -235,6 +235,8 @@ public class ThumbnailManager {
     }
 
     protected void process(final FessConfig fessConfig, final ThumbnailQueue entity) {
+        ComponentUtil.getSystemHelper().calibrateCpuLoad();
+
         if (logger.isDebugEnabled()) {
             logger.debug("Processing thumbnail: {}", entity);
         }

src/main/resources/fess_config.properties

@@ -150,6 +150,7 @@ java.command.path=java
 path.encoding=UTF-8
 use.own.tmp.dir=true
 max.log.output.length=4000
+adaptive.load.control=0
 supported.uploaded.js.extentions=js
 supported.uploaded.css.extentions=css
 supported.uploaded.media.extentions=jpg,jpeg,gif,png,swf