0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

#1810 replace with HTTP

Browse source
This commit is contained in:
Shinsuke Sugaya 2018-08-20 21:20:53 +09:00
parent 7b99623f74
commit 50abebb82b
13 changed files with 53 additions and 146 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
pom.xml
View file

@ -60,7 +60,7 @@
<utflute.version>0.8.6</utflute.version>
<!-- Crawler -->
<crawler.version>2.2.0</crawler.version>
<crawler.version>2.3.0-SNAPSHOT</crawler.version>
<!-- Suggest -->
<suggest.version>6.3.0</suggest.version>
@ -1200,6 +1200,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.codelibs</groupId>
<artifactId>elasticsearch-httpclient</artifactId>
<version>0.1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
@ -1272,7 +1277,7 @@
<dependency>
<groupId>org.codelibs</groupId>
<artifactId>curl4j</artifactId>
<version>1.0.0</version>
<version>1.2.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.codelibs</groupId>

1
src/main/assemblies/files/fess.in.bat
View file

@ -117,5 +117,4 @@ set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Dtomcat.config.path=tomcat_config.propertie
REM External elasticsearch cluster
REM set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Dfess.es.http_address=http://localhost:9200
REM set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Dfess.es.transport_addresses=localhost:9300
REM set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Dfess.dictionary.path=%ES_HOME%/config/

3
src/main/assemblies/files/fess.in.sh
View file

@ -116,9 +116,6 @@ fi
if [ "x$ES_HTTP_URL" != "x" ]; then
FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dfess.es.http_address=$ES_HTTP_URL"
fi
if [ "x$ES_TRANSPORT_URL" != "x" ]; then
FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dfess.es.transport_addresses=$ES_TRANSPORT_URL"
fi
if [ "x$FESS_DICTIONARY_PATH" != "x" ]; then
FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dfess.dictionary.path=$FESS_DICTIONARY_PATH"
fi

4
src/main/java/org/codelibs/fess/Constants.java
View file

@ -317,9 +317,7 @@ public class Constants extends CoreLibConstants {
public static final String MATCHES_ALL_QUERY = "*:*";
public static final String FESS_ES_TRANSPORT_ADDRESSES = "fess.es.transport_addresses";
public static final String FESS_ES_CLUSTER_NAME = "fess.es.cluster_name";
public static final String FESS_ES_HTTP_ADDRESS = "fess.es.http_address";
public static final int DEFAULT_PAGE_SIZE = 20;

34
src/main/java/org/codelibs/fess/api/es/EsApiManager.java
View file

@ -36,6 +36,7 @@ import org.codelibs.core.io.CopyUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.curl.Curl.Method;
import org.codelibs.curl.CurlRequest;
import org.codelibs.curl.CurlResponse;
import org.codelibs.fess.Constants;
import org.codelibs.fess.api.BaseApiManager;
import org.codelibs.fess.exception.FessSystemException;
@ -136,7 +137,7 @@ public class EsApiManager extends BaseApiManager {
curlRequest.param(entry.getKey(), entry.getValue()[0]);
}
});
curlRequest.onConnect((req, con) -> {
final CurlResponse curlResponse = curlRequest.onConnect((req, con) -> {
con.setDoOutput(true);
if (httpMethod != Method.GET) {
try (ServletInputStream in = request.getInputStream(); OutputStream out = con.getOutputStream()) {
@ -145,27 +146,20 @@ public class EsApiManager extends BaseApiManager {
throw new WebApiException(HttpServletResponse.SC_BAD_REQUEST, e);
}
}
}).execute(con -> {
try (ServletOutputStream out = response.getOutputStream()) {
try (InputStream in = con.getInputStream()) {
response.setStatus(con.getResponseCode());
CopyUtil.copy(in, out);
} catch (final Exception e) {
response.setStatus(con.getResponseCode());
try (InputStream err = con.getErrorStream()) {
CopyUtil.copy(err, out);
}
}
} catch (final ClientAbortException e) {
}).execute();
try (ServletOutputStream out = response.getOutputStream(); InputStream in = curlResponse.getContentAsStream()) {
response.setStatus(curlResponse.getHttpStatusCode());
CopyUtil.copy(in, out);
} catch (final ClientAbortException e) {
logger.debug("Client aborts this request.", e);
} catch (final Exception e) {
if (e.getCause() instanceof ClientAbortException) {
logger.debug("Client aborts this request.", e);
} catch (final Exception e) {
if (e.getCause() instanceof ClientAbortException) {
logger.debug("Client aborts this request.", e);
} else {
throw new WebApiException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
}
} else {
throw new WebApiException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
}
});
}
}
protected void processPluginRequest(final HttpServletRequest request, final HttpServletResponse response, final String path) {

72
src/main/java/org/codelibs/fess/es/client/FessEsClient.java
View file

@ -44,6 +44,7 @@ import org.codelibs.core.io.FileUtil;
import org.codelibs.core.io.ResourceUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.curl.CurlResponse;
import org.codelibs.elasticsearch.client.HttpClient;
import org.codelibs.elasticsearch.runner.ElasticsearchClusterRunner;
import org.codelibs.elasticsearch.runner.ElasticsearchClusterRunner.Configs;
import org.codelibs.fess.Constants;
@ -130,13 +131,10 @@ import org.elasticsearch.action.update.UpdateRequestBuilder;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.Settings.Builder;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.InnerHitBuilder;
@ -150,7 +148,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilde
import org.elasticsearch.search.collapse.CollapseBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.lastaflute.core.message.UserMessages;
import org.lastaflute.di.exception.ContainerInitFailureException;
import org.slf4j.Logger;
@ -158,7 +155,6 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import com.google.common.io.BaseEncoding;
public class FessEsClient implements Client {
@ -166,7 +162,7 @@ public class FessEsClient implements Client {
protected ElasticsearchClusterRunner runner;
protected List<TransportAddress> transportAddressList = new ArrayList<>();
protected String httpAddress;
protected Client client;
@ -218,10 +214,6 @@ public class FessEsClient implements Client {
return this.runner != null;
}
public void addTransportAddress(final String host, final int port) {
transportAddressList.add(new TransportAddress(getInetAddressByName(host), port));
}
protected InetAddress getInetAddressByName(final String host) {
try {
return InetAddress.getByName(host);
@ -234,24 +226,8 @@ public class FessEsClient implements Client {
public void open() {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String transportAddressesValue = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
if (StringUtil.isNotBlank(transportAddressesValue)) {
for (final String transportAddressValue : transportAddressesValue.split(",")) {
final String[] addressPair = transportAddressValue.trim().split(":");
if (addressPair.length < 3) {
final String host = addressPair[0];
int port = 9300;
if (addressPair.length == 2) {
port = Integer.parseInt(addressPair[1]);
}
addTransportAddress(host, port);
} else {
logger.warn("Invalid address format: " + transportAddressValue);
}
}
}
if (transportAddressList.isEmpty()) {
String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
if (StringUtil.isBlank(httpAddress)) {
if (runner == null) {
runner = new ElasticsearchClusterRunner();
final Configs config = newConfigs().clusterName(fessConfig.getElasticsearchClusterName()).numOfNode(1).useLogger();
@ -273,27 +249,14 @@ public class FessEsClient implements Client {
});
runner.build(config);
}
final int port = runner.node().settings().getAsInt("transport.tcp.port", 9300);
client = createTransportClient(fessConfig, Lists.newArrayList(new TransportAddress(getInetAddressByName("localhost"), port)));
addTransportAddress("localhost", port);
final int port = runner.node().settings().getAsInt("http.port", 9200);
httpAddress = "http://localhost:" + port;
logger.warn("Embedded Elasticsearch is running. This configuration is not recommended for production use.");
} else {
client = createTransportClient(fessConfig, transportAddressList);
}
client = createHttpClient(fessConfig, httpAddress);
if (StringUtil.isBlank(transportAddressesValue)) {
final StringBuilder buf = new StringBuilder();
for (final TransportAddress transportAddress : transportAddressList) {
if (buf.length() > 0) {
buf.append(',');
}
buf.append(transportAddress.address().getHostName());
buf.append(':');
buf.append(transportAddress.address().getPort());
}
if (buf.length() > 0) {
System.setProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES, buf.toString());
}
if (StringUtil.isNotBlank(httpAddress)) {
System.setProperty(Constants.FESS_ES_HTTP_ADDRESS, httpAddress);
}
waitForYellowStatus(fessConfig);
@ -352,18 +315,9 @@ public class FessEsClient implements Client {
});
}
protected Client createTransportClient(final FessConfig fessConfig, final List<TransportAddress> transportAddressList) {
final Builder settingsBuilder = Settings.builder();
settingsBuilder.put("cluster.name", fessConfig.getElasticsearchClusterName());
settingsBuilder.put("client.transport.sniff", fessConfig.isElasticsearchTransportSniff());
settingsBuilder.put("client.transport.ping_timeout", fessConfig.getElasticsearchTransportPingTimeout());
settingsBuilder.put("client.transport.nodes_sampler_interval", fessConfig.getElasticsearchTransportNodesSamplerInterval());
final Settings settings = settingsBuilder.build();
final TransportClient transportClient = new PreBuiltTransportClient(settings);
for (final TransportAddress address : transportAddressList) {
transportClient.addTransportAddress(address);
}
return transportClient;
protected Client createHttpClient(final FessConfig fessConfig, final String host) {
final Settings settings = Settings.builder().putList("http.hosts", host).build();
return new HttpClient(settings, null);
}
public boolean existsIndex(final String indexName) {
@ -631,7 +585,7 @@ public class FessEsClient implements Client {
}
}
final String message =
"Elasticsearch (" + System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES)
"Elasticsearch (" + System.getProperty(Constants.FESS_ES_HTTP_ADDRESS)
+ ") is not available. Check the state of your Elasticsearch cluster (" + fessConfig.getElasticsearchClusterName()
+ ").";
throw new ContainerInitFailureException(message, cause);

10
src/main/java/org/codelibs/fess/exec/Crawler.java
View file

@ -203,13 +203,9 @@ public class Crawler {
initializeProbes();
final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
if (StringUtil.isNotBlank(transportAddresses)) {
System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses);
}
final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
if (StringUtil.isNotBlank(clusterName)) {
System.setProperty(EsClient.CLUSTER_NAME, clusterName);
final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
if (StringUtil.isNotBlank(httpAddress)) {
System.setProperty(EsClient.HTTP_ADDRESS, httpAddress);
}
TimeoutTask systemMonitorTask = null;

10
src/main/java/org/codelibs/fess/exec/SuggestCreator.java
View file

@ -105,13 +105,9 @@ public class SuggestCreator {
}
}
final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
if (StringUtil.isNotBlank(transportAddresses)) {
System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses);
}
final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
if (StringUtil.isNotBlank(clusterName)) {
System.setProperty(EsClient.CLUSTER_NAME, clusterName);
final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
if (StringUtil.isNotBlank(httpAddress)) {
System.setProperty(EsClient.HTTP_ADDRESS, httpAddress);
}
TimeoutTask systemMonitorTask = null;

10
src/main/java/org/codelibs/fess/exec/ThumbnailGenerator.java
View file

@ -110,13 +110,9 @@ public class ThumbnailGenerator {
}
}
final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
if (StringUtil.isNotBlank(transportAddresses)) {
System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses);
}
final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
if (StringUtil.isNotBlank(clusterName)) {
System.setProperty(EsClient.CLUSTER_NAME, clusterName);
final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
if (StringUtil.isNotBlank(httpAddress)) {
System.setProperty(EsClient.HTTP_ADDRESS, httpAddress);
}
TimeoutTask systemMonitorTask = null;

13
src/main/java/org/codelibs/fess/job/CrawlJob.java
View file

@ -266,19 +266,12 @@ public class CrawlJob {
cmdList.add(buf.toString());
if (useLocalElasticsearch) {
final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
if (StringUtil.isNotBlank(transportAddresses)) {
cmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
if (StringUtil.isNotBlank(httpAddress)) {
cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
}
}
final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
if (StringUtil.isNotBlank(clusterName)) {
cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + clusterName);
} else {
cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + fessConfig.getElasticsearchClusterName());
}
final String systemLastaEnv = System.getProperty("lasta.env");
if (StringUtil.isNotBlank(systemLastaEnv)) {
if (systemLastaEnv.equals("web")) {

13
src/main/java/org/codelibs/fess/job/GenerateThumbnailJob.java
View file

@ -197,19 +197,12 @@ public class GenerateThumbnailJob {
cmdList.add(buf.toString());
if (useLocaleElasticsearch) {
final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
if (StringUtil.isNotBlank(transportAddresses)) {
cmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
if (StringUtil.isNotBlank(httpAddress)) {
cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
}
}
final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
if (StringUtil.isNotBlank(clusterName)) {
cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + clusterName);
} else {
cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + fessConfig.getElasticsearchClusterName());
}
final String systemLastaEnv = System.getProperty("lasta.env");
if (StringUtil.isNotBlank(systemLastaEnv)) {
if (systemLastaEnv.equals("web")) {

13
src/main/java/org/codelibs/fess/job/SuggestJob.java
View file

@ -183,19 +183,12 @@ public class SuggestJob {
cmdList.add(buf.toString());
if (useLocaleElasticsearch) {
final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
if (StringUtil.isNotBlank(transportAddresses)) {
cmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
if (StringUtil.isNotBlank(httpAddress)) {
cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
}
}
final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
if (StringUtil.isNotBlank(clusterName)) {
cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + clusterName);
} else {
cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + fessConfig.getElasticsearchClusterName());
}
final String systemLastaEnv = System.getProperty("lasta.env");
if (StringUtil.isNotBlank(systemLastaEnv)) {
if (systemLastaEnv.equals("web")) {

7
src/main/resources/fess_config.properties
View file

@ -8,13 +8,6 @@
# The title of domain the application for logging
domain.title = Fess
# Elasticsearch
elasticsearch.cluster.name=elasticsearch
elasticsearch.http.url=http://localhost:9201
elasticsearch.transport.sniff=false
elasticsearch.transport.ping_timeout=1m
elasticsearch.transport.nodes_sampler_interval=5s
# Cryptographer
app.cipher.algorism=aes
app.cipher.key=___change__me___