0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1711 elasticsearch 6.3

Browse source
This commit is contained in:
Shinsuke Sugaya 2018-06-16 08:21:04 +09:00
parent 71a13d8b03
commit de11bbdba9
19 changed files with 74 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
dbflute.xml
View file

@ -2,7 +2,7 @@
<project name="dbflute" basedir=".">
<property name="mydbflute.dir" value="${basedir}/mydbflute" />
<property name="target.dir" value="${basedir}/target" />
<property name="branch.name" value="master" />
<property name="branch.name" value="es-6.3" />
<property name="mydbflute.url" value="https://github.com/lastaflute/lastaflute-example-waterfront/archive/${branch.name}.zip" />
<target name="mydbflute.check">

42
plugin.xml
View file

@ -13,66 +13,66 @@
<mkdir dir="${plugins.dir}" />
<!-- analysis-fess -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="analysis-fess" />
<param name="plugin.version" value="6.2.1" />
<param name="plugin.zip.version" value="6.2.1" />
<param name="plugin.version" value="6.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="6.3.0-20180614.205104-1" />
</antcall>
<!-- analysis-ja -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="analysis-ja" />
<param name="plugin.version" value="6.2.1" />
<param name="plugin.zip.version" value="6.2.1" />
<param name="plugin.version" value="6.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="6.3.0-20180614.213049-1" />
</antcall>
<!-- analysis-synonym -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="analysis-synonym" />
<param name="plugin.version" value="6.2.1" />
<param name="plugin.zip.version" value="6.2.1" />
<param name="plugin.version" value="6.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="6.3.0-20180614.214213-1" />
</antcall>
<!-- configsync -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="configsync" />
<param name="plugin.version" value="6.2.2" />
<param name="plugin.zip.version" value="6.2.2" />
<param name="plugin.version" value="6.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="6.3.0-20180614.214445-1" />
</antcall>
<!-- dataformat -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="dataformat" />
<param name="plugin.version" value="6.2.3" />
<param name="plugin.zip.version" value="6.2.3" />
<param name="plugin.version" value="6.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="6.3.0-20180615.130428-1" />
</antcall>
<!-- langfield -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="langfield" />
<param name="plugin.version" value="6.2.1" />
<param name="plugin.zip.version" value="6.2.1" />
<param name="plugin.version" value="6.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="6.3.0-20180615.131150-1" />
</antcall>
<!-- minhash -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="minhash" />
<param name="plugin.version" value="6.2.1" />
<param name="plugin.zip.version" value="6.2.1" />
<param name="plugin.version" value="6.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="6.3.0-20180615.132619-1" />
</antcall>
<antcall target="remove.jars" />

9
pom.xml
View file

@ -63,10 +63,10 @@
<crawler.version>2.2.0-SNAPSHOT</crawler.version>
<!-- Suggest -->
<suggest.version>6.2.1</suggest.version>
<suggest.version>6.3.0-SNAPSHOT</suggest.version>
<!-- Elasticsearch -->
<elasticsearch.version>6.2.4</elasticsearch.version>
<elasticsearch.version>6.3.0</elasticsearch.version>
<elasticsearch.min.version>6.0.0</elasticsearch.min.version>
<cluster.runner.version>${elasticsearch.version}.0</cluster.runner.version>
@ -1265,6 +1265,11 @@
<artifactId>corelib</artifactId>
<version>0.4.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.codelibs</groupId>
<artifactId>curl4j</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.codelibs</groupId>
<artifactId>spnego</artifactId>

4
src/main/java/org/codelibs/fess/api/es/EsApiManager.java
View file

@ -33,8 +33,8 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.catalina.connector.ClientAbortException;
import org.codelibs.core.io.CopyUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.elasticsearch.runner.net.Curl.Method;
import org.codelibs.elasticsearch.runner.net.CurlRequest;
import org.codelibs.curl.Curl.Method;
import org.codelibs.curl.CurlRequest;
import org.codelibs.fess.Constants;
import org.codelibs.fess.api.BaseApiManager;
import org.codelibs.fess.exception.FessSystemException;

9
src/main/java/org/codelibs/fess/api/json/JsonApiManager.java
View file

@ -15,7 +15,9 @@
*/
package org.codelibs.fess.api.json;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Arrays;
@ -56,7 +58,6 @@ import org.codelibs.fess.util.FacetResponse;
import org.codelibs.fess.util.FacetResponse.Field;
import org.dbflute.optional.OptionalThing;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.script.Script;
import org.slf4j.Logger;
@ -371,9 +372,9 @@ public class JsonApiManager extends BaseJsonApiManager {
}
protected String toGeoRequestString(final GeoInfo geoInfo) {
try {
final XContentBuilder builder = XContentFactory.jsonBuilder();
return geoInfo.toQueryBuilder().toXContent(builder, ToXContent.EMPTY_PARAMS).string();
try (OutputStream out =
geoInfo.toQueryBuilder().toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).getOutputStream()) {
return ((ByteArrayOutputStream) out).toString(Constants.UTF_8);
} catch (final Exception e) {
return "{\"error\":\"" + detailedMessage(e) + "\"}";
}

2
src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java
View file

@ -41,7 +41,7 @@ import org.apache.commons.text.StringEscapeUtils;
import org.codelibs.core.exception.IORuntimeException;
import org.codelibs.core.io.CopyUtil;
import org.codelibs.core.misc.Pair;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.curl.CurlResponse;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.base.FessAdminAction;
import org.codelibs.fess.es.config.exbhv.FileConfigBhv;

4
src/main/java/org/codelibs/fess/app/web/admin/esreq/AdminEsreqAction.java
View file

@ -25,8 +25,8 @@ import java.util.Locale;
import org.codelibs.core.io.CopyUtil;
import org.codelibs.core.io.ReaderUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.elasticsearch.runner.net.CurlRequest;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.curl.CurlRequest;
import org.codelibs.curl.CurlResponse;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.base.FessAdminAction;
import org.codelibs.fess.helper.CurlHelper;

2
src/main/java/org/codelibs/fess/app/web/api/admin/backup/ApiAdminBackupAction.java
View file

@ -33,7 +33,7 @@ import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.curl.CurlResponse;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.api.ApiResult;
import org.codelibs.fess.app.web.api.ApiResult.ApiBackupFilesResponse;

7
src/main/java/org/codelibs/fess/dict/DictionaryManager.java
View file

@ -27,7 +27,8 @@ import java.util.Map;
import javax.annotation.PostConstruct;
import org.codelibs.core.io.FileUtil;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.curl.CurlResponse;
import org.codelibs.elasticsearch.runner.net.EcrCurl;
import org.codelibs.fess.Constants;
import org.codelibs.fess.util.ComponentUtil;
import org.dbflute.optional.OptionalEntity;
@ -50,7 +51,7 @@ public class DictionaryManager {
try (CurlResponse response =
ComponentUtil.getCurlHelper().get("/_configsync/file").param("fields", "path,@timestamp")
.param("size", ComponentUtil.getFessConfig().getPageDictionaryMaxFetchSize()).execute()) {
final Map<String, Object> contentMap = response.getContentAsMap();
final Map<String, Object> contentMap = response.getContent(EcrCurl.jsonParser);
@SuppressWarnings("unchecked")
final List<Map<String, Object>> fileList = (List<Map<String, Object>>) contentMap.get("file");
return fileList
@ -96,7 +97,7 @@ public class DictionaryManager {
try (CurlResponse response =
ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", dictFile.getPath())
.body(FileUtil.readUTF8(file)).execute()) {
final Map<String, Object> contentMap = response.getContentAsMap();
final Map<String, Object> contentMap = response.getContent(EcrCurl.jsonParser);
if (!Constants.TRUE.equalsIgnoreCase(contentMap.get("acknowledged").toString())) {
throw new DictionaryException("Failed to update " + dictFile.getPath());
}

10
src/main/java/org/codelibs/fess/entity/PingResponse.java
View file

@ -15,13 +15,15 @@
*/
package org.codelibs.fess.entity;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
public class PingResponse {
@ -37,10 +39,8 @@ public class PingResponse {
status = response.getStatus() == ClusterHealthStatus.RED ? 1 : 0;
clusterName = response.getClusterName();
clusterStatus = response.getStatus().toString();
try {
final XContentBuilder builder = XContentFactory.jsonBuilder();
response.toXContent(builder, ToXContent.EMPTY_PARAMS);
message = builder.string();
try (OutputStream out = response.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).getOutputStream()) {
message = ((ByteArrayOutputStream) out).toString(Constants.UTF_8);
} catch (final IOException e) {
message = "{ \"error\" : \"" + e.getMessage() + "\"}";
}

2
src/main/java/org/codelibs/fess/es/client/FessEsClient.java
View file

@ -43,9 +43,9 @@ import org.codelibs.core.exception.ResourceNotFoundRuntimeException;
import org.codelibs.core.io.FileUtil;
import org.codelibs.core.io.ResourceUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.curl.CurlResponse;
import org.codelibs.elasticsearch.runner.ElasticsearchClusterRunner;
import org.codelibs.elasticsearch.runner.ElasticsearchClusterRunner.Configs;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.Constants;
import org.codelibs.fess.entity.FacetInfo;
import org.codelibs.fess.entity.GeoInfo;

4
src/main/java/org/codelibs/fess/helper/CurlHelper.java
View file

@ -15,8 +15,8 @@
*/
package org.codelibs.fess.helper;
import org.codelibs.elasticsearch.runner.net.Curl.Method;
import org.codelibs.elasticsearch.runner.net.CurlRequest;
import org.codelibs.curl.Curl.Method;
import org.codelibs.curl.CurlRequest;
import org.codelibs.fess.util.ResourceUtil;
public class CurlHelper {

11
src/main/java/org/codelibs/fess/timer/SystemMonitorTarget.java
View file

@ -15,6 +15,8 @@
*/
package org.codelibs.fess.timer;
import java.io.ByteArrayOutputStream;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.List;
import java.util.function.Supplier;
@ -22,6 +24,7 @@ import java.util.stream.Collectors;
import org.apache.commons.text.StringEscapeUtils;
import org.codelibs.core.timer.TimeoutTarget;
import org.codelibs.fess.Constants;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
@ -179,14 +182,16 @@ public class SystemMonitorTarget implements TimeoutTarget {
try {
final FessEsClient esClient = ComponentUtil.getFessEsClient();
final NodesStatsResponse response =
esClient.admin().cluster().prepareNodesStats().ingest(false).setBreaker(false).setDiscovery(false).setFs(true)
esClient.admin().cluster().prepareNodesStats().setIngest(false).setBreaker(false).setDiscovery(false).setFs(true)
.setHttp(false).setIndices(true).setJvm(true).setOs(true).setProcess(true).setScript(false).setThreadPool(true)
.setTransport(true).execute().actionGet(10000L);
final XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
response.toXContent(builder, ToXContent.EMPTY_PARAMS);
response.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS);
builder.endObject();
stats = builder.string();
try (OutputStream out = builder.getOutputStream()) {
stats = ((ByteArrayOutputStream) out).toString(Constants.UTF_8);
}
} catch (final Exception e) {
logger.debug("Failed to access Elasticsearch stats.", e);
}

2
src/main/java/org/codelibs/fess/util/UpgradeUtil.java
View file

@ -20,7 +20,7 @@ import java.util.function.Consumer;
import org.codelibs.core.exception.ResourceNotFoundRuntimeException;
import org.codelibs.core.io.FileUtil;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.curl.CurlResponse;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;

7
src/test/java/org/codelibs/fess/helper/AccessTokenHelperTest.java
View file

@ -15,14 +15,13 @@
*/
package org.codelibs.fess.helper;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertThrows;
import org.codelibs.fess.unit.UnitFessTestCase;
import org.codelibs.fess.exception.InvalidAccessTokenException;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.unit.UnitFessTestCase;
import org.dbflute.utflute.mocklet.MockletHttpServletRequest;
public class AccessTokenHelperTest extends UnitFessTestCase {

6
src/test/java/org/codelibs/fess/helper/QueryHelperTest.java
View file

@ -115,13 +115,13 @@ public class QueryHelperTest extends UnitFessTestCase {
"{\"function_score\":{\"query\":{\"wildcard\":{\"title\":{\"wildcard\":\"*\",\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
buildQuery("allintitle:").toString().replaceAll("\\s", ""));
assertEquals(
"{\"function_score\":{\"query\":{\"match_phrase\":{\"title\":{\"query\":\"test\",\"slop\":0,\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
"{\"function_score\":{\"query\":{\"match_phrase\":{\"title\":{\"query\":\"test\",\"slop\":0,\"zero_terms_query\":\"NONE\",\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
buildQuery("allintitle:test").toString().replaceAll("\\s", ""));
assertEquals(
"{\"function_score\":{\"query\":{\"match_phrase\":{\"title\":{\"query\":\"test\",\"slop\":0,\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
"{\"function_score\":{\"query\":{\"match_phrase\":{\"title\":{\"query\":\"test\",\"slop\":0,\"zero_terms_query\":\"NONE\",\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
buildQuery("allintitle: test").toString().replaceAll("\\s", ""));
assertEquals(
"{\"function_score\":{\"query\":{\"bool\":{\"must\":[{\"match_phrase\":{\"title\":{\"query\":\"aaa\",\"slop\":0,\"boost\":1.0}}},{\"match_phrase\":{\"title\":{\"query\":\"bbb\",\"slop\":0,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
"{\"function_score\":{\"query\":{\"bool\":{\"must\":[{\"match_phrase\":{\"title\":{\"query\":\"aaa\",\"slop\":0,\"zero_terms_query\":\"NONE\",\"boost\":1.0}}},{\"match_phrase\":{\"title\":{\"query\":\"bbb\",\"slop\":0,\"zero_terms_query\":\"NONE\",\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
buildQuery("allintitle: aaa bbb").toString().replaceAll("\\s", ""));
assertEquals(

4
src/test/java/org/codelibs/fess/it/admin/SearchListTests.java
View file

@ -15,13 +15,11 @@
*/
package org.codelibs.fess.it.admin;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.hamcrest.Matchers.equalTo;
import java.util.HashMap;
import java.util.Map;
import java.util.List;
import java.util.Set;
import java.util.Map;
import org.codelibs.fess.it.CrudTestBase;
import org.junit.jupiter.api.Tag;

7
src/test/java/org/codelibs/fess/util/DocMapTest.java
View file

@ -15,12 +15,11 @@
*/
package org.codelibs.fess.util;
import java.util.Map;
import java.util.Set;
import java.util.List;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.codelibs.fess.unit.UnitFessTestCase;

5
src/test/java/org/codelibs/fess/util/KuromojiCSVUtilTest.java
View file

@ -15,11 +15,12 @@
*/
package org.codelibs.fess.util;
import java.util.List;
import java.util.Arrays;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
import java.util.Arrays;
import java.util.List;
import org.codelibs.fess.unit.UnitFessTestCase;
public class KuromojiCSVUtilTest extends UnitFessTestCase {