fix #2442 add adaptive.load.control
This commit is contained in:
Shinsuke Sugaya
parent
24cb1db9df
commit
29ce6dd071
7 changed files with 80 additions and 1 deletions
|
|
@ -55,6 +55,7 @@ public class FessIntervalController extends DefaultIntervalController {
|
|||
|
||||
@Override
|
||||
protected void delayForWaitingNewUrl() {
|
||||
ComponentUtil.getSystemHelper().calibrateCpuLoad();
|
||||
try {
|
||||
final IntervalControlHelper intervalControlHelper = ComponentUtil.getIntervalControlHelper();
|
||||
intervalControlHelper.checkCrawlerStatus();
|
||||
|
|
|
|||
|
|
@ -64,6 +64,9 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
|
|||
*/
|
||||
@Override
|
||||
public void store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
|
||||
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
|
||||
systemHelper.calibrateCpuLoad();
|
||||
|
||||
final long startTime = System.currentTimeMillis();
|
||||
final FessConfig fessConfig = ComponentUtil.getFessConfig();
|
||||
final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
|
||||
|
|
@ -102,7 +105,6 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
|
|||
}
|
||||
|
||||
if (!dataMap.containsKey(fessConfig.getIndexFieldDocId())) {
|
||||
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
|
||||
dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ import org.apache.logging.log4j.Logger;
|
|||
import org.apache.logging.log4j.core.config.Configurator;
|
||||
import org.codelibs.core.exception.IORuntimeException;
|
||||
import org.codelibs.core.lang.StringUtil;
|
||||
import org.codelibs.core.lang.ThreadUtil;
|
||||
import org.codelibs.core.misc.Pair;
|
||||
import org.codelibs.fess.Constants;
|
||||
import org.codelibs.fess.crawler.util.CharUtil;
|
||||
|
|
@ -69,6 +70,7 @@ import org.codelibs.fess.util.GsaConfigParser;
|
|||
import org.codelibs.fess.util.ParameterUtil;
|
||||
import org.codelibs.fess.util.ResourceUtil;
|
||||
import org.codelibs.fess.validation.FessActionValidator;
|
||||
import org.elasticsearch.monitor.os.OsProbe;
|
||||
import org.lastaflute.core.message.supplier.UserMessagesCreator;
|
||||
import org.lastaflute.web.TypicalAction;
|
||||
import org.lastaflute.web.response.HtmlResponse;
|
||||
|
|
@ -110,6 +112,12 @@ public class SystemHelper {
|
|||
|
||||
protected long eolTime;
|
||||
|
||||
private short systemCpuPercent;
|
||||
|
||||
private long systemCpuCheckTime;
|
||||
|
||||
private long systemCpuCheckInterval = 1000L;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
if (logger.isDebugEnabled()) {
|
||||
|
|
@ -539,6 +547,38 @@ public class SystemHelper {
|
|||
}
|
||||
}
|
||||
|
||||
public void calibrateCpuLoad() {
|
||||
final int percent = ComponentUtil.getFessConfig().getAdaptiveLoadControlAsInteger();
|
||||
if (percent <= 0) {
|
||||
return;
|
||||
}
|
||||
while (getSystemCpuPercent() > percent) {
|
||||
if (logger.isInfoEnabled()) {
|
||||
logger.info("Cpu Load {}% is greater than {}%.", getSystemCpuPercent(), percent);
|
||||
}
|
||||
ThreadUtil.sleep(systemCpuCheckInterval);
|
||||
}
|
||||
}
|
||||
|
||||
protected short getSystemCpuPercent() {
|
||||
final long now = System.currentTimeMillis();
|
||||
if (now - systemCpuCheckTime > systemCpuCheckInterval) {
|
||||
synchronized (this) {
|
||||
if (now - systemCpuCheckTime > systemCpuCheckInterval) {
|
||||
try {
|
||||
final OsProbe osProbe = OsProbe.getInstance();
|
||||
systemCpuPercent = osProbe.getSystemCpuPercent();
|
||||
} catch (Exception e) {
|
||||
logger.warn("Failed to get SystemCpuPercent.", e);
|
||||
return 0;
|
||||
}
|
||||
systemCpuCheckTime = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
return systemCpuPercent;
|
||||
}
|
||||
|
||||
public String getVersion() {
|
||||
return version;
|
||||
}
|
||||
|
|
@ -554,4 +594,8 @@ public class SystemHelper {
|
|||
public String getProductVersion() {
|
||||
return productVersion;
|
||||
}
|
||||
|
||||
public void setSystemCpuCheckInterval(long systemCpuCheckInterval) {
|
||||
this.systemCpuCheckInterval = systemCpuCheckInterval;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -194,6 +194,8 @@ public class IndexUpdater extends Thread {
|
|||
ThreadUtil.sleep(interval); // 10 sec (default)
|
||||
}
|
||||
|
||||
systemHelper.calibrateCpuLoad();
|
||||
|
||||
docList.clear();
|
||||
accessResultList.clear();
|
||||
|
||||
|
|
|
|||
|
|
@ -190,6 +190,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
|
|||
/** The key of the configuration. e.g. 4000 */
|
||||
String MAX_LOG_OUTPUT_LENGTH = "max.log.output.length";
|
||||
|
||||
/** The key of the configuration. e.g. 0 */
|
||||
String ADAPTIVE_LOAD_CONTROL = "adaptive.load.control";
|
||||
|
||||
/** The key of the configuration. e.g. js */
|
||||
String SUPPORTED_UPLOADED_JS_EXTENTIONS = "supported.uploaded.js.extentions";
|
||||
|
||||
|
|
@ -1895,6 +1898,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
|
|||
*/
|
||||
Integer getMaxLogOutputLengthAsInteger();
|
||||
|
||||
/**
|
||||
* Get the value for the key 'adaptive.load.control'. <br>
|
||||
* The value is, e.g. 0 <br>
|
||||
* @return The value of found property. (NotNull: if not found, exception but basically no way)
|
||||
*/
|
||||
String getAdaptiveLoadControl();
|
||||
|
||||
/**
|
||||
* Get the value for the key 'adaptive.load.control' as {@link Integer}. <br>
|
||||
* The value is, e.g. 0 <br>
|
||||
* @return The value of found property. (NotNull: if not found, exception but basically no way)
|
||||
* @throws NumberFormatException When the property is not integer.
|
||||
*/
|
||||
Integer getAdaptiveLoadControlAsInteger();
|
||||
|
||||
/**
|
||||
* Get the value for the key 'supported.uploaded.js.extentions'. <br>
|
||||
* The value is, e.g. js <br>
|
||||
|
|
@ -6608,6 +6626,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
|
|||
return getAsInteger(FessConfig.MAX_LOG_OUTPUT_LENGTH);
|
||||
}
|
||||
|
||||
public String getAdaptiveLoadControl() {
|
||||
return get(FessConfig.ADAPTIVE_LOAD_CONTROL);
|
||||
}
|
||||
|
||||
public Integer getAdaptiveLoadControlAsInteger() {
|
||||
return getAsInteger(FessConfig.ADAPTIVE_LOAD_CONTROL);
|
||||
}
|
||||
|
||||
public String getSupportedUploadedJsExtentions() {
|
||||
return get(FessConfig.SUPPORTED_UPLOADED_JS_EXTENTIONS);
|
||||
}
|
||||
|
|
@ -9088,6 +9114,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
|
|||
defaultMap.put(FessConfig.PATH_ENCODING, "UTF-8");
|
||||
defaultMap.put(FessConfig.USE_OWN_TMP_DIR, "true");
|
||||
defaultMap.put(FessConfig.MAX_LOG_OUTPUT_LENGTH, "4000");
|
||||
defaultMap.put(FessConfig.ADAPTIVE_LOAD_CONTROL, "0");
|
||||
defaultMap.put(FessConfig.SUPPORTED_UPLOADED_JS_EXTENTIONS, "js");
|
||||
defaultMap.put(FessConfig.SUPPORTED_UPLOADED_CSS_EXTENTIONS, "css");
|
||||
defaultMap.put(FessConfig.SUPPORTED_UPLOADED_MEDIA_EXTENTIONS, "jpg,jpeg,gif,png,swf");
|
||||
|
|
|
|||
|
|
@ -235,6 +235,8 @@ public class ThumbnailManager {
|
|||
}
|
||||
|
||||
protected void process(final FessConfig fessConfig, final ThumbnailQueue entity) {
|
||||
ComponentUtil.getSystemHelper().calibrateCpuLoad();
|
||||
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Processing thumbnail: {}", entity);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -150,6 +150,7 @@ java.command.path=java
|
|||
path.encoding=UTF-8
|
||||
use.own.tmp.dir=true
|
||||
max.log.output.length=4000
|
||||
adaptive.load.control=0
|
||||
supported.uploaded.js.extentions=js
|
||||
supported.uploaded.css.extentions=css
|
||||
supported.uploaded.media.extentions=jpg,jpeg,gif,png,swf
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue