0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #441 : refactoring for StringBuilder

Browse source
This commit is contained in:
Shinsuke Sugaya 2016-03-20 16:42:55 +09:00
parent 92dc3d378f
commit 487866658b
17 changed files with 119 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pom.xml
View file

@ -769,7 +769,7 @@
<dependency>
<groupId>org.codelibs</groupId>
<artifactId>corelib</artifactId>
<version>0.3.3</version>
<version>0.3.4-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>

9
src/main/java/org/codelibs/fess/api/json/JsonApiManager.java
View file

@ -32,7 +32,6 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.output.StringBuilderWriter;
import org.codelibs.core.CoreLibConstants;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
@ -141,7 +140,7 @@ public class JsonApiManager extends BaseApiManager {
int status = 0;
String errMsg = StringUtil.EMPTY;
String query = null;
final StringBuilder buf = new StringBuilder(1000);
final StringBuilder buf = new StringBuilder(1000); // TODO replace response stream
request.setAttribute(Constants.SEARCH_LOG_ACCESS_TYPE, Constants.SEARCH_LOG_ACCESS_TYPE_JSON);
try {
final SearchRenderData data = new SearchRenderData();
@ -278,7 +277,7 @@ public class JsonApiManager extends BaseApiManager {
int status = 0;
String errMsg = StringUtil.EMPTY;
final StringBuilder buf = new StringBuilder(255);
final StringBuilder buf = new StringBuilder(255); // TODO replace response stream
try {
final List<Map<String, String>> labelTypeItems = labelTypeHelper.getLabelTypeItemList();
buf.append("\"record_count\":");
@ -328,7 +327,7 @@ public class JsonApiManager extends BaseApiManager {
int status = 0;
String errMsg = StringUtil.EMPTY;
final StringBuilder buf = new StringBuilder(255);
final StringBuilder buf = new StringBuilder(255); // TODO replace response stream
try {
final List<String> popularWordList = popularWordHelper.getWordList(seed, tags, null, fields, excludes);
@ -493,7 +492,7 @@ public class JsonApiManager extends BaseApiManager {
}
}
final StringBuilder buf = new StringBuilder();
final StringBuilder buf = new StringBuilder(255); // TODO replace response stream
buf.append("\"num\":").append(docIdList.size());
if (!docIdList.isEmpty()) {
buf.append(", \"doc_ids\":[");

2
src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java
View file

@ -60,7 +60,7 @@ public class SuggestApiManager extends BaseApiManager {
ServletException {
int status = 0;
String errMsg = StringUtil.EMPTY;
final StringBuilder buf = new StringBuilder(255);
final StringBuilder buf = new StringBuilder(255); // TODO replace response stream
final RoleQueryHelper roleQueryHelper = ComponentUtil.getRoleQueryHelper();
try {

14
src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java
View file

@ -46,6 +46,7 @@ import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.extractor.Extractor;
import org.codelibs.fess.crawler.transformer.impl.AbstractTransformer;
import org.codelibs.fess.crawler.util.CrawlingParameterUtil;
import org.codelibs.fess.crawler.util.UnsafeStringBuilder;
import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigName;
import org.codelibs.fess.helper.CrawlingConfigHelper;
@ -101,7 +102,7 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
final String mimeType = responseData.getMimeType();
params.put(HttpHeaders.CONTENT_TYPE, mimeType);
params.put(HttpHeaders.CONTENT_ENCODING, responseData.getCharSet());
final StringBuilder contentMetaBuf = new StringBuilder(1000);
final UnsafeStringBuilder contentMetaBuf = new UnsafeStringBuilder(1000);
final Map<String, Object> dataMap = new HashMap<String, Object>();
final Map<String, Object> metaDataMap = new HashMap<>();
String content;
@ -163,7 +164,7 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
if (content == null) {
content = StringUtil.EMPTY;
}
final String contentMeta = contentMetaBuf.toString();
final String contentMeta = contentMetaBuf.toUnsafeString().trim();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
@ -201,7 +202,7 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
// segment
putResultDataBody(dataMap, fessConfig.getIndexFieldSegment(), sessionId);
// content
final StringBuilder buf = new StringBuilder(content.length() + 1000);
final UnsafeStringBuilder buf = new UnsafeStringBuilder(content.length() + 1000);
if (fessConfig.isCrawlerDocumentFileAppendBodyContent()) {
buf.append(content);
}
@ -211,7 +212,8 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
}
buf.append(contentMeta);
}
final String body = documentHelper.getContent(responseData, buf.toString(), dataMap);
final String bodyBase = buf.toUnsafeString().trim();
final String body = documentHelper.getContent(responseData, bodyBase, dataMap);
putResultDataBody(dataMap, fessConfig.getIndexFieldContent(), body);
if ((Constants.TRUE.equalsIgnoreCase(fieldConfigMap.get(fessConfig.getIndexFieldCache())) || fessConfig
.isCrawlerDocumentCacheEnabled()) && fessConfig.isSupportedDocumentCacheMimetypes(mimeType)) {
@ -225,8 +227,8 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
}
}
// digest
putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(), documentHelper.getDigest(responseData, buf.toString(), dataMap,
fessConfig.getCrawlerDocumentFileMaxDigestLengthAsInteger()));
putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(),
documentHelper.getDigest(responseData, bodyBase, dataMap, fessConfig.getCrawlerDocumentFileMaxDigestLengthAsInteger()));
// title
if (!dataMap.containsKey(fessConfig.getIndexFieldTitle())) {
if (url.endsWith("/")) {

11
src/main/java/org/codelibs/fess/crawler/transformer/FessXpathTransformer.java
View file

@ -46,6 +46,7 @@ import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.transformer.impl.XpathTransformer;
import org.codelibs.fess.crawler.util.CrawlingParameterUtil;
import org.codelibs.fess.crawler.util.UnsafeStringBuilder;
import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigName;
import org.codelibs.fess.helper.CrawlingConfigHelper;
@ -345,13 +346,13 @@ public class FessXpathTransformer extends XpathTransformer implements FessTransf
}
protected String getSingleNodeValue(final Document document, final String xpath, final boolean pruned) {
StringBuilder buf = null;
UnsafeStringBuilder buf = null;
NodeList list = null;
try {
list = getXPathAPI().selectNodeList(document, xpath);
for (int i = 0; i < list.getLength(); i++) {
if (buf == null) {
buf = new StringBuilder(1000);
buf = new UnsafeStringBuilder(1000);
} else {
buf.append(' ');
}
@ -369,7 +370,7 @@ public class FessXpathTransformer extends XpathTransformer implements FessTransf
if (buf == null) {
return null;
}
return buf.toString();
return buf.toUnsafeString().trim();
}
protected Node pruneNode(final Node node) {
@ -407,7 +408,7 @@ public class FessXpathTransformer extends XpathTransformer implements FessTransf
protected String getMultipleNodeValue(final Document document, final String xpath) {
NodeList nodeList = null;
final StringBuilder buf = new StringBuilder(100);
final UnsafeStringBuilder buf = new UnsafeStringBuilder(100);
try {
nodeList = getXPathAPI().selectNodeList(document, xpath);
for (int i = 0; i < nodeList.getLength(); i++) {
@ -418,7 +419,7 @@ public class FessXpathTransformer extends XpathTransformer implements FessTransf
} catch (final Exception e) {
logger.warn("Could not parse a value of " + xpath);
}
return buf.toString();
return buf.toUnsafeString().trim();
}
protected String replaceDuplicateHost(final String url) {

5
src/main/java/org/codelibs/fess/ds/impl/IndexUpdateCallbackImpl.java
View file

@ -15,8 +15,6 @@
*/
package org.codelibs.fess.ds.impl;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
@ -29,6 +27,7 @@ import org.codelibs.fess.helper.SearchLogHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -39,7 +38,7 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
protected volatile long executeTime = 0;
final List<Map<String, Object>> docList = new ArrayList<>();
final DocList docList = new DocList();
/* (non-Javadoc)
* @see org.codelibs.fess.ds.impl.IndexUpdateCallback#store(java.util.Map)

5
src/main/java/org/codelibs/fess/helper/CrawlingInfoHelper.java
View file

@ -27,6 +27,7 @@ import java.util.Map;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.CrawlingInfoService;
import org.codelibs.fess.crawler.util.UnsafeStringBuilder;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.es.config.exentity.CrawlingInfo;
@ -187,7 +188,7 @@ public class CrawlingInfoHelper implements Serializable {
}
private String generateId(final String url, final List<String> roleTypeList) {
final StringBuilder buf = new StringBuilder(1000);
final UnsafeStringBuilder buf = new UnsafeStringBuilder(1000);
buf.append(url);
if (roleTypeList != null && !roleTypeList.isEmpty()) {
Collections.sort(roleTypeList);
@ -200,7 +201,7 @@ public class CrawlingInfoHelper implements Serializable {
}
}
return normalize(buf.toString());
return normalize(buf.toUnsafeString().trim());
}
private String normalize(final String value) {

5
src/main/java/org/codelibs/fess/helper/DocumentHelper.java
View file

@ -21,6 +21,7 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.util.UnsafeStringBuilder;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
@ -38,7 +39,7 @@ public class DocumentHelper implements Serializable {
}
final int maxAlphanumTermSize = getMaxAlphanumSize();
final StringBuilder buf = new StringBuilder(content.length());
final UnsafeStringBuilder buf = new UnsafeStringBuilder(content.length());
boolean isSpace = false;
int alphanumSize = 0;
for (int i = 0; i < content.length(); i++) {
@ -68,7 +69,7 @@ public class DocumentHelper implements Serializable {
}
}
return buf.toString().trim();
return buf.toUnsafeString().trim();
}
protected int getMaxAlphanumSize() {

23
src/main/java/org/codelibs/fess/helper/IndexingHelper.java
View file

@ -19,9 +19,11 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocList;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
@ -37,7 +39,7 @@ public class IndexingHelper {
public long requestInterval = 500;
public void sendDocuments(final FessEsClient fessEsClient, final List<Map<String, Object>> docList) {
public void sendDocuments(final FessEsClient fessEsClient, final DocList docList) {
if (docList.isEmpty()) {
return;
}
@ -52,14 +54,29 @@ public class IndexingHelper {
fessEsClient.addAll(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), docList);
}
if (logger.isInfoEnabled()) {
logger.info("Sent " + docList.size() + " docs (" + (System.currentTimeMillis() - execTime) + "ms)");
final Runtime runtime = Runtime.getRuntime();
final long freeBytes = runtime.freeMemory();
final long maxBytes = runtime.maxMemory();
final long totalBytes = runtime.totalMemory();
final long usedBytes = totalBytes - freeBytes;
if (docList.getContentSize() > 0) {
logger.info("Sent " + docList.size() + " docs (Doc:{process " + docList.getProcessingTime() + "ms, send "
+ (System.currentTimeMillis() - execTime) + "ms, size "
+ FileUtils.byteCountToDisplaySize(docList.getContentSize()) + "}, Mem:{used "
+ FileUtils.byteCountToDisplaySize(usedBytes) + ", heap " + FileUtils.byteCountToDisplaySize(totalBytes)
+ ", max " + FileUtils.byteCountToDisplaySize(maxBytes) + "})");
} else {
logger.info("Sent " + docList.size() + " docs (Doc:{send " + (System.currentTimeMillis() - execTime)
+ "ms}, Mem:{used " + FileUtils.byteCountToDisplaySize(usedBytes) + ", heap "
+ FileUtils.byteCountToDisplaySize(totalBytes) + ", max " + FileUtils.byteCountToDisplaySize(maxBytes) + "})");
}
}
} finally {
docList.clear();
}
}
private void deleteOldDocuments(final FessEsClient fessEsClient, final List<Map<String, Object>> docList) {
private void deleteOldDocuments(final FessEsClient fessEsClient, final DocList docList) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final List<String> docIdList = new ArrayList<>();

4
src/main/java/org/codelibs/fess/helper/LabelTypeHelper.java
View file

@ -185,7 +185,7 @@ public class LabelTypeHelper implements Serializable {
this.value = value;
if (StringUtil.isNotBlank(includedPaths)) {
final StringBuilder buf = new StringBuilder();
final StringBuilder buf = new StringBuilder(100);
char split = 0;
for (final String path : includedPaths.split("\n")) {
if (split == 0) {
@ -199,7 +199,7 @@ public class LabelTypeHelper implements Serializable {
}
if (StringUtil.isNotBlank(excludedPaths)) {
final StringBuilder buf = new StringBuilder();
final StringBuilder buf = new StringBuilder(100);
char split = 0;
for (final String path : excludedPaths.split("\n")) {
if (split == 0) {

2
src/main/java/org/codelibs/fess/helper/SuggestHelper.java
View file

@ -110,7 +110,7 @@ public class SuggestHelper {
return;
}
final StringBuilder sb = new StringBuilder();
final StringBuilder sb = new StringBuilder(100);
final List<String> fields = new ArrayList<>();
final List<String> tags = new ArrayList<>();
final List<String> roles = new ArrayList<>();

2
src/main/java/org/codelibs/fess/helper/SystemHelper.java
View file

@ -152,7 +152,7 @@ public class SystemHelper implements Serializable {
}
try {
final StringBuilder buf = new StringBuilder();
final StringBuilder buf = new StringBuilder(path.length() + 100);
for (int i = 0; i < path.length(); i++) {
final char c = path.charAt(i);
if (CharUtil.isUrlChar(c) || c == '^' || c == '{' || c == '}' || c == '|' || c == '\\') {

31
src/main/java/org/codelibs/fess/indexer/IndexUpdater.java
View file

@ -24,6 +24,7 @@ import java.util.function.Consumer;
import javax.annotation.PreDestroy;
import javax.annotation.Resource;
import org.apache.commons.io.FileUtils;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
@ -47,6 +48,7 @@ import org.codelibs.fess.helper.SearchLogHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocList;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
@ -172,7 +174,7 @@ public class IndexUpdater extends Thread {
builder.addSort(EsAccessResult.CREATE_TIME, SortOrder.ASC);
};
final List<Map<String, Object>> docList = new ArrayList<>();
final DocList docList = new DocList();
final List<EsAccessResult> accessResultList = new ArrayList<>();
long updateTime = System.currentTimeMillis();
@ -299,11 +301,9 @@ public class IndexUpdater extends Thread {
}
}
private void processAccessResults(final List<Map<String, Object>> docList, final List<EsAccessResult> accessResultList,
final List<EsAccessResult> arList) {
private void processAccessResults(final DocList docList, final List<EsAccessResult> accessResultList, final List<EsAccessResult> arList) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final long maxDocumentRequestSize = fessConfig.getIndexerWebfsMaxDocumentRequestSizeAsInteger().longValue();
long contentSize = 0;
for (final EsAccessResult accessResult : arList) {
if (logger.isDebugEnabled()) {
logger.debug("Indexing " + accessResult.getUrl());
@ -319,6 +319,7 @@ public class IndexUpdater extends Thread {
continue;
}
final long startTime = System.currentTimeMillis();
final AccessResultData<?> accessResultData = accessResult.getAccessResultData();
if (accessResultData != null) {
accessResult.setAccessResultData(null);
@ -349,19 +350,19 @@ public class IndexUpdater extends Thread {
updateDocument(map);
docList.add(map);
final long processingTime = System.currentTimeMillis() - startTime;
docList.addProcessingTime(processingTime);
if (logger.isDebugEnabled()) {
logger.debug("Added the document(" + contentSize + " bytes). " + "The number of a document cache is "
+ docList.size() + ".");
logger.debug("Added the document(" + FileUtils.byteCountToDisplaySize(docList.getContentSize()) + ", "
+ processingTime + "ms). " + "The number of a document cache is " + docList.size() + ".");
}
if (accessResult.getContentLength() == null) {
indexingHelper.sendDocuments(fessEsClient, docList);
contentSize = 0;
} else {
contentSize += accessResult.getContentLength().longValue();
if (contentSize >= maxDocumentRequestSize) {
docList.addContentSize(accessResult.getContentLength().longValue());
if (docList.getContentSize() >= maxDocumentRequestSize) {
indexingHelper.sendDocuments(fessEsClient, docList);
contentSize = 0;
}
}
documentSize++;
@ -482,15 +483,19 @@ public class IndexUpdater extends Thread {
}
final long totalHits = ((EsResultList<EsAccessResult>) arList).getTotalHits();
if (logger.isInfoEnabled()) {
logger.info("Processing " + arList.size() + "/" + totalHits + " docs (" + (System.currentTimeMillis() - execTime) + "ms)");
if (totalHits > 0) {
logger.info("Processing " + arList.size() + "/" + totalHits + " docs (" + (System.currentTimeMillis() - execTime) + "ms)");
} else {
logger.info("Processing no docs (" + (System.currentTimeMillis() - execTime) + "ms)");
}
}
final long unprocessedDocumentSize = fessConfig.getIndexerUnprocessedDocumentSizeAsInteger().longValue();
if (totalHits > unprocessedDocumentSize) {
final IntervalControlHelper intervalControlHelper = ComponentUtil.getIntervalControlHelper();
if (totalHits > unprocessedDocumentSize && intervalControlHelper.isCrawlerRunning()) {
if (logger.isInfoEnabled()) {
logger.info("Stopped all crawler threads. " + " You have " + totalHits + " (>" + unprocessedDocumentSize + ") "
+ " unprocessed docs.");
}
final IntervalControlHelper intervalControlHelper = ComponentUtil.getIntervalControlHelper();
intervalControlHelper.setCrawlerRunning(false);
}
return arList;

4
src/main/java/org/codelibs/fess/job/CrawlJob.java
View file

@ -178,7 +178,7 @@ public class CrawlJob {
}
public String execute() {
final StringBuilder resultBuf = new StringBuilder();
final StringBuilder resultBuf = new StringBuilder(100);
final boolean runAll = webConfigIds == null && fileConfigIds == null && dataConfigIds == null;
if (sessionId == null) { // create session id
@ -255,7 +255,7 @@ public class CrawlJob {
// -cp
cmdList.add("-cp");
final StringBuilder buf = new StringBuilder();
final StringBuilder buf = new StringBuilder(100);
final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
if (StringUtil.isNotBlank(confPath)) {
buf.append(confPath);

7
src/main/java/org/codelibs/fess/job/SuggestJob.java
View file

@ -107,10 +107,9 @@ public class SuggestJob {
try {
executeSuggestCreator();
} catch (final FessSystemException e) {
throw e;
} catch (final Exception e) {
throw new FessSystemException("Failed to execute a crawl job.", e);
logger.error("Failed to purge user info.", e);
resultBuf.append(e.getMessage()).append("\n");
}
return resultBuf.toString();
@ -128,7 +127,7 @@ public class SuggestJob {
// -cp
cmdList.add("-cp");
final StringBuilder buf = new StringBuilder();
final StringBuilder buf = new StringBuilder(100);
final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
if (StringUtil.isNotBlank(confPath)) {
buf.append(confPath);

43
src/main/java/org/codelibs/fess/util/DocList.java Normal file
View file

@ -0,0 +1,43 @@
package org.codelibs.fess.util;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
public class DocList extends ArrayList<Map<String, Object>> {
private static final long serialVersionUID = 1L;
private long contentSize = 0;
private long processingTime = 0;
public void clear() {
super.clear();
contentSize = 0;
processingTime = 0;
}
public long getContentSize() {
return contentSize;
}
public void addContentSize(long contentSize) {
this.contentSize += contentSize;
}
public long getProcessingTime() {
return processingTime;
}
public void addProcessingTime(long processingTime) {
this.processingTime += processingTime;
}
@Override
public String toString() {
return "DocList [contentSize=" + contentSize + ", processingTime=" + processingTime + ", elementData="
+ Arrays.toString(toArray(new Map[size()])) + "]";
}
}

2
src/main/java/org/codelibs/fess/util/InputStreamThread.java
View file

@ -70,7 +70,7 @@ public class InputStreamThread extends Thread {
}
public String getOutput() {
final StringBuilder buf = new StringBuilder();
final StringBuilder buf = new StringBuilder(100);
for (final String value : list) {
buf.append(value).append("\n");
}