0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #953 elasticsearch 5.3 support

Browse source
This commit is contained in:
Shinsuke Sugaya 2017-03-30 12:44:52 +09:00
parent 33be98a9dd
commit b2029c99b3
5 changed files with 44 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
dbflute.xml
View file

@ -2,7 +2,7 @@
<project name="dbflute" basedir=".">
<property name="mydbflute.dir" value="${basedir}/mydbflute" />
<property name="target.dir" value="${basedir}/target" />
<property name="branch.name" value="es-5.2" />
<property name="branch.name" value="es-5.3" />
<property name="mydbflute.url" value="https://github.com/lastaflute/lastaflute-example-waterfront/archive/${branch.name}.zip" />
<target name="mydbflute.check">

36
plugin.xml
View file

@ -15,57 +15,57 @@
<mkdir dir="${plugins.dir}" />
<!-- analysis-fess -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="analysis-fess" />
<param name="plugin.version" value="5.2.1" />
<param name="plugin.zip.version" value="5.2.1" />
<param name="plugin.version" value="5.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="5.3.0-20170329.213757-1" />
</antcall>
<!-- analysis-ja -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="analysis-ja" />
<param name="plugin.version" value="5.2.1" />
<param name="plugin.zip.version" value="5.2.1" />
<param name="plugin.version" value="5.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="5.3.0-20170329.214734-1" />
</antcall>
<!-- analysis-synonym -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="analysis-synonym" />
<param name="plugin.version" value="5.2.1" />
<param name="plugin.zip.version" value="5.2.1" />
<param name="plugin.version" value="5.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="5.3.0-20170329.215604-1" />
</antcall>
<!-- configsync -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="configsync" />
<param name="plugin.version" value="5.2.0" />
<param name="plugin.zip.version" value="5.2.0" />
<param name="plugin.version" value="5.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="5.3.0-20170330.013642-1" />
</antcall>
<!-- dataformat -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="dataformat" />
<param name="plugin.version" value="5.2.1" />
<param name="plugin.zip.version" value="5.2.1" />
<param name="plugin.version" value="5.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="5.3.0-20170330.020712-1" />
</antcall>
<!-- langfield -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="langfield" />
<param name="plugin.version" value="5.2.1" />
<param name="plugin.zip.version" value="5.2.1" />
<param name="plugin.version" value="5.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="5.3.0-20170330.022144-1" />
</antcall>
<!-- kopf -->
<get dest="${target.dir}/kopf.zip">

8
pom.xml
View file

@ -60,12 +60,12 @@
<crawler.version>1.2.0-SNAPSHOT</crawler.version>
<!-- Suggest -->
<suggest.version>5.2.1-SNAPSHOT</suggest.version>
<suggest.version>5.3.0-SNAPSHOT</suggest.version>
<!-- Elasticsearch -->
<elasticsearch.version>5.2.2</elasticsearch.version>
<elasticsearch.min.version>5.1.0</elasticsearch.min.version>
<cluster.runner.version>5.2.2.0</cluster.runner.version>
<elasticsearch.version>5.3.0</elasticsearch.version>
<elasticsearch.min.version>5.3.0</elasticsearch.min.version>
<cluster.runner.version>5.3.0.0</cluster.runner.version>
<!-- Tomcat -->
<tomcat.version>8.5.11</tomcat.version>

28
src/main/java/org/codelibs/fess/es/client/FessEsClient.java
View file

@ -69,6 +69,8 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.DocWriteRequest.OpType;
import org.elasticsearch.action.DocWriteResponse.Result;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
@ -100,7 +102,6 @@ import org.elasticsearch.action.get.MultiGetRequest;
import org.elasticsearch.action.get.MultiGetRequestBuilder;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexRequest.OpType;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.ClearScrollRequest;
@ -135,6 +136,7 @@ import org.elasticsearch.common.settings.Settings.Builder;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
@ -318,7 +320,7 @@ public class FessEsClient implements Client {
source = FileUtil.readUTF8(filePath);
try (CurlResponse response =
Curl.post(org.codelibs.fess.util.ResourceUtil.getElasticsearchHttpUrl() + "/_configsync/file")
.param("path", path).body(source).execute()) {
.header("Content-Type", "application/json").param("path", path).body(source).execute()) {
if (response.getHttpStatusCode() == 200) {
logger.info("Register " + path + " to " + configIndex);
} else {
@ -335,7 +337,8 @@ public class FessEsClient implements Client {
}
});
try (CurlResponse response =
Curl.post(org.codelibs.fess.util.ResourceUtil.getElasticsearchHttpUrl() + "/_configsync/flush").execute()) {
Curl.post(org.codelibs.fess.util.ResourceUtil.getElasticsearchHttpUrl() + "/_configsync/flush")
.header("Content-Type", "application/json").execute()) {
if (response.getHttpStatusCode() == 200) {
logger.info("Flushed config files.");
} else {
@ -357,7 +360,8 @@ public class FessEsClient implements Client {
final String dictionaryPath = System.getProperty("fess.dictionary.path", StringUtil.EMPTY);
source = source.replaceAll(Pattern.quote("${fess.dictionary.path}"), dictionaryPath);
final CreateIndexResponse indexResponse =
client.admin().indices().prepareCreate(createdIndexName).setSource(source).execute()
client.admin().indices().prepareCreate(createdIndexName)
.setSource(source, XContentFactory.xContentType(source)).execute()
.actionGet(fessConfig.getIndexIndicesTimeout());
if (indexResponse.isAcknowledged()) {
logger.info("Created " + createdIndexName + " index.");
@ -426,7 +430,8 @@ public class FessEsClient implements Client {
}
try {
final PutMappingResponse putMappingResponse =
client.admin().indices().preparePutMapping(updatedIndexName).setType(configType).setSource(source).execute()
client.admin().indices().preparePutMapping(updatedIndexName).setType(configType)
.setSource(source, XContentFactory.xContentType(source)).execute()
.actionGet(fessConfig.getIndexIndicesTimeout());
if (putMappingResponse.isAcknowledged()) {
logger.info("Created " + updatedIndexName + "/" + configType + " mapping.");
@ -478,7 +483,8 @@ public class FessEsClient implements Client {
});
if (result.keySet().contains("index")) {
final IndexRequestBuilder requestBuilder =
client.prepareIndex(configIndex, configType, result.get("index").get("_id")).setSource(line);
client.prepareIndex(configIndex, configType, result.get("index").get("_id")).setSource(line,
XContentFactory.xContentType(line));
builder.add(requestBuilder);
}
}
@ -507,8 +513,8 @@ public class FessEsClient implements Client {
private void waitForConfigSyncStatus() {
try (CurlResponse response =
Curl.get(org.codelibs.fess.util.ResourceUtil.getElasticsearchHttpUrl() + "/_configsync/wait").param("status", "green")
.execute()) {
Curl.get(org.codelibs.fess.util.ResourceUtil.getElasticsearchHttpUrl() + "/_configsync/wait")
.header("Content-Type", "application/json").param("status", "green").execute()) {
if (response.getHttpStatusCode() == 200) {
logger.info("ConfigSync is ready.");
} else {
@ -768,13 +774,14 @@ public class FessEsClient implements Client {
final BulkResponse response = bulkRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexBulkTimeout());
if (response.hasFailures()) {
if (logger.isDebugEnabled()) {
final List<ActionRequest> requests = bulkRequestBuilder.request().requests();
@SuppressWarnings("rawtypes")
final List<DocWriteRequest> requests = bulkRequestBuilder.request().requests();
final BulkItemResponse[] items = response.getItems();
if (requests.size() == items.length) {
for (int i = 0; i < requests.size(); i++) {
final BulkItemResponse resp = items[i];
if (resp.isFailed() && resp.getFailure() != null) {
final ActionRequest req = requests.get(i);
final DocWriteRequest<?> req = requests.get(i);
final Failure failure = resp.getFailure();
logger.debug("Failed Request: " + req + "\n=>" + failure.getMessage());
}
@ -924,6 +931,7 @@ public class FessEsClient implements Client {
public boolean store(final String index, final String type, final Object obj) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
@SuppressWarnings("unchecked")
final Map<String, Object> source = obj instanceof Map ? (Map<String, Object>) obj : BeanUtil.copyBeanToNewMap(obj);
final String id = (String) source.remove(fessConfig.getIndexFieldId());
final Long version = (Long) source.remove(fessConfig.getIndexFieldVersion());

6
src/test/java/org/codelibs/fess/entity/GeoInfoTest.java
View file

@ -27,7 +27,7 @@ public class GeoInfoTest extends UnitFessTestCase {
final GeoInfo geoInfo = new GeoInfo(request);
String result =
"{\"geo_distance\":{\"location\":[150.0,34.0],\"distance\":10000.0,\"distance_type\":\"sloppy_arc\",\"validation_method\":\"STRICT\",\"ignore_unmapped\":false,\"boost\":1.0}}";
"{\"geo_distance\":{\"location\":[150.0,34.0],\"distance\":10000.0,\"distance_type\":\"arc\",\"validation_method\":\"STRICT\",\"ignore_unmapped\":false,\"boost\":1.0}}";
assertEquals(result, geoInfo.toQueryBuilder().toString().replaceAll("[ \n]", ""));
}
@ -38,7 +38,7 @@ public class GeoInfoTest extends UnitFessTestCase {
final GeoInfo geoInfo = new GeoInfo(request);
String result =
"{\"geo_distance\":{\"location\":[150.0,34.0],\"distance\":10000.0,\"distance_type\":\"sloppy_arc\",\"validation_method\":\"STRICT\",\"ignore_unmapped\":false,\"boost\":1.0}}";
"{\"geo_distance\":{\"location\":[150.0,34.0],\"distance\":10000.0,\"distance_type\":\"arc\",\"validation_method\":\"STRICT\",\"ignore_unmapped\":false,\"boost\":1.0}}";
assertEquals(result, geoInfo.toQueryBuilder().toString().replaceAll("[ \n]", ""));
}
@ -51,7 +51,7 @@ public class GeoInfoTest extends UnitFessTestCase {
final GeoInfo geoInfo = new GeoInfo(request);
String result =
"{\"bool\":{\"should\":[{\"geo_distance\":{\"location\":[151.0,35.0],\"distance\":1000.0,\"distance_type\":\"sloppy_arc\",\"validation_method\":\"STRICT\",\"ignore_unmapped\":false,\"boost\":1.0}},{\"geo_distance\":{\"location\":[150.0,34.0],\"distance\":10000.0,\"distance_type\":\"sloppy_arc\",\"validation_method\":\"STRICT\",\"ignore_unmapped\":false,\"boost\":1.0}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}}";
"{\"bool\":{\"should\":[{\"geo_distance\":{\"location\":[151.0,35.0],\"distance\":1000.0,\"distance_type\":\"arc\",\"validation_method\":\"STRICT\",\"ignore_unmapped\":false,\"boost\":1.0}},{\"geo_distance\":{\"location\":[150.0,34.0],\"distance\":10000.0,\"distance_type\":\"arc\",\"validation_method\":\"STRICT\",\"ignore_unmapped\":false,\"boost\":1.0}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}}";
assertEquals(result, geoInfo.toQueryBuilder().toString().replaceAll("[ \n]", ""));
}