0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tech(SSL): SSL FESS <=> ELK

Browse source
This commit is contained in:
Robin Coma Delperier 2018-04-17 17:10:48 +02:00 • committed by Shinsuke Sugaya
parent 0cc67e78dd
commit 7689191077
9 changed files with 154 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
src/main/assemblies/files/fess.in.sh
View file

@ -18,6 +18,17 @@ fi
#ES_TRANSPORT_URL=localhost:9300
#FESS_DICTIONARY_PATH=/var/lib/elasticsearch/config/
# In case the ES cluster is SSL secured (Searchguard)
#SSL_ENABLED=false
#SG_SSL_TRANSPORT_PEMKEY_FILEPATH=/path/to/the/key
#SG_SSL_TRANSPORT_PEMCERT_FILEPATH=/path/to/the/CERTIFICATE
#SG_SSL_TRANSPORT_PEMCA_FILEPATH=/path/to/the/CA
#SG_SSL_HTTP_USER_PWD=user:pwd
# SSL truststore for certificate validation over https
#JAVA_OPTS="$JAVA_OPTS -Djavax.net.ssl.trustStore=/tech/elastic/config/truststore.jks"
#JAVA_OPTS="$JAVA_OPTS -Djavax.net.ssl.trustStorePassword=changeit"
# min and max heap sizes should be set to the same value to avoid
# stop-the-world GC pauses during resize, and so that we can lock the
# heap in memory on startup to prevent any of it from being swapped
@ -112,9 +123,24 @@ fi
if [ "x$ES_HTTP_URL" != "x" ]; then
FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dfess.es.http_address=$ES_HTTP_URL"
fi
if [ "x$SSL_ENABLED" != "x" ]; then
FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dfess.es.ssl.enabled=$SSL_ENABLED"
fi
if [ "x$ES_TRANSPORT_URL" != "x" ]; then
FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dfess.es.transport_addresses=$ES_TRANSPORT_URL"
fi
if [ "x$SG_SSL_TRANSPORT_PEMKEY_FILEPATH" != "x" ]; then
FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dsg.ssl.transport.pemkeyfilepath=$SG_SSL_TRANSPORT_PEMKEY_FILEPATH"
fi
if [ "x$SG_SSL_TRANSPORT_PEMCERT_FILEPATH" != "x" ]; then
FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dsg.ssl.transport.pemcertfilepath=$SG_SSL_TRANSPORT_PEMCERT_FILEPATH"
fi
if [ "x$SG_SSL_TRANSPORT_PEMCA_FILEPATH" != "x" ]; then
FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dsg.ssl.transport.pemcafilepath=$SG_SSL_TRANSPORT_PEMCA_FILEPATH"
fi
if [ "x$SG_SSL_HTTP_USER_PWD" != "x" ]; then
FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dsg.ssl.http.user_pwd=$SG_SSL_HTTP_USER_PWD"
fi
if [ "x$FESS_DICTIONARY_PATH" != "x" ]; then
FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dfess.dictionary.path=$FESS_DICTIONARY_PATH"
fi

4
src/main/java/org/codelibs/fess/api/es/EsApiManager.java
View file

@ -42,6 +42,7 @@ import org.codelibs.fess.exception.WebApiException;
import org.codelibs.fess.mylasta.action.FessUserBean;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.ResourceUtil;
import org.codelibs.fess.util.SslUtil;
import org.lastaflute.web.servlet.request.RequestManager;
import org.lastaflute.web.servlet.session.SessionManager;
import org.slf4j.Logger;
@ -114,6 +115,9 @@ public class EsApiManager extends BaseApiManager {
final Method httpMethod = Method.valueOf(request.getMethod().toUpperCase(Locale.ROOT));
final CurlRequest curlRequest = ComponentUtil.getCurlHelper().request(httpMethod, path);
if(SslUtil.isSslSecure()) {
curlRequest.header("Authorization", SslUtil.getBasicAuthEncodedCredentials());
}
final String contentType = request.getHeader("Content-Type");
if (StringUtil.isNotEmpty(contentType)) {

15
src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java
View file

@ -41,6 +41,7 @@ import org.apache.commons.text.StringEscapeUtils;
import org.codelibs.core.exception.IORuntimeException;
import org.codelibs.core.io.CopyUtil;
import org.codelibs.core.misc.Pair;
import org.codelibs.elasticsearch.runner.net.CurlRequest;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.base.FessAdminAction;
@ -51,6 +52,7 @@ import org.codelibs.fess.es.log.exbhv.UserInfoBhv;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.RenderDataUtil;
import org.codelibs.fess.util.SslUtil;
import org.lastaflute.core.magic.async.AsyncManager;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.ActionResponse;
@ -99,7 +101,11 @@ public class AdminBackupAction extends FessAdminAction {
logger.warn("Failed to process system.properties file: " + form.bulkFile.getFileName(), e);
}
} else {
try (CurlResponse response = ComponentUtil.getCurlHelper().post("/_bulk").onConnect((req, con) -> {
CurlRequest curlRequest = ComponentUtil.getCurlHelper().post("/_bulk");
if(SslUtil.isSslSecure()) {
curlRequest.header("Authorization", SslUtil.getBasicAuthEncodedCredentials());
}
try (CurlResponse response = curlRequest.onConnect((req, con) -> {
con.setDoOutput(true);
try (InputStream in = form.bulkFile.getInputStream(); OutputStream out = con.getOutputStream()) {
CopyUtil.copy(in, out);
@ -155,8 +161,11 @@ public class AdminBackupAction extends FessAdminAction {
}
return asStream(filename).contentTypeOctetStream().stream(
out -> {
try (CurlResponse response =
ComponentUtil.getCurlHelper().get("/" + index + "/_data").param("format", "json").execute()) {
CurlRequest curlRequest = ComponentUtil.getCurlHelper().get("/" + index + "/_data").param("format", "json");
if(SslUtil.isSslSecure()) {
curlRequest.header("Authorization", SslUtil.getBasicAuthEncodedCredentials());
}
try (CurlResponse response = curlRequest.execute()) {
out.write(response.getContentAsStream());
}
});

4
src/main/java/org/codelibs/fess/app/web/admin/esreq/AdminEsreqAction.java
View file

@ -31,6 +31,7 @@ import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.base.FessAdminAction;
import org.codelibs.fess.helper.CurlHelper;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.SslUtil;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.ActionResponse;
import org.lastaflute.web.response.HtmlResponse;
@ -82,6 +83,9 @@ public class AdminEsreqAction extends FessAdminAction {
return asListHtml(() -> saveToken());
});
} else {
if(SslUtil.isSslSecure()) {
curlRequest.header("Authorization", SslUtil.getBasicAuthEncodedCredentials());
}
try (final CurlResponse response = curlRequest.body(buf.toString()).execute()) {
final File tempFile = File.createTempFile("esreq_", ".json");
try (final InputStream in = response.getContentAsStream()) {

9
src/main/java/org/codelibs/fess/app/web/api/admin/backup/ApiAdminBackupAction.java
View file

@ -33,6 +33,7 @@ import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import org.codelibs.elasticsearch.runner.net.CurlRequest;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.api.ApiResult;
@ -40,6 +41,7 @@ import org.codelibs.fess.app.web.api.ApiResult.ApiBackupFilesResponse;
import org.codelibs.fess.app.web.api.admin.FessApiAdminAction;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.SslUtil;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.JsonResponse;
import org.lastaflute.web.response.StreamResponse;
@ -93,8 +95,11 @@ public class ApiAdminBackupAction extends FessApiAdminAction {
}
return asStream(filename).contentTypeOctetStream().stream(
out -> {
try (CurlResponse response =
ComponentUtil.getCurlHelper().get("/" + index + "/_data").param("format", "json").execute()) {
CurlRequest curlRequest = ComponentUtil.getCurlHelper().get("/" + index + "/_data").param("format", "json");
if(SslUtil.isSslSecure()) {
curlRequest.header("Authorization", SslUtil.getBasicAuthEncodedCredentials());
}
try (CurlResponse response = curlRequest.execute()) {
out.write(response.getContentAsStream());
}
});

28
src/main/java/org/codelibs/fess/dict/DictionaryManager.java
View file

@ -27,9 +27,12 @@ import java.util.Map;
import javax.annotation.PostConstruct;
import org.codelibs.core.io.FileUtil;
import org.codelibs.elasticsearch.runner.net.CurlRequest;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.Constants;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.ResourceUtil;
import org.codelibs.fess.util.SslUtil;
import org.dbflute.optional.OptionalEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -47,9 +50,13 @@ public class DictionaryManager {
}
public DictionaryFile<? extends DictionaryItem>[] getDictionaryFiles() {
try (CurlResponse response =
ComponentUtil.getCurlHelper().get("/_configsync/file").param("fields", "path,@timestamp")
.param("size", ComponentUtil.getFessConfig().getPageDictionaryMaxFetchSize()).execute()) {
CurlRequest curlRequest = ComponentUtil.getCurlHelper().post("/_configsync/file").param("fields", "path,@timestamp")
.param("size", ComponentUtil.getFessConfig().getPageDictionaryMaxFetchSize());
if(SslUtil.isSslSecure()) {
curlRequest.header("Authorization", SslUtil.getBasicAuthEncodedCredentials());
}
try (CurlResponse response = curlRequest.execute()) {
final Map<String, Object> contentMap = response.getContentAsMap();
@SuppressWarnings("unchecked")
final List<Map<String, Object>> fileList = (List<Map<String, Object>>) contentMap.get("file");
@ -93,9 +100,12 @@ public class DictionaryManager {
}
// TODO use stream
try (CurlResponse response =
ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", dictFile.getPath())
.body(FileUtil.readUTF8(file)).execute()) {
CurlRequest curlRequest = ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", dictFile.getPath())
.body(FileUtil.readUTF8(file));
if(SslUtil.isSslSecure()) {
curlRequest.header("Authorization", SslUtil.getBasicAuthEncodedCredentials());
}
try (CurlResponse response = curlRequest.execute()) {
final Map<String, Object> contentMap = response.getContentAsMap();
if (!Constants.TRUE.equalsIgnoreCase(contentMap.get("acknowledged").toString())) {
throw new DictionaryException("Failed to update " + dictFile.getPath());
@ -111,7 +121,11 @@ public class DictionaryManager {
public InputStream getContentInputStream(final DictionaryFile<? extends DictionaryItem> dictFile) {
try {
return ComponentUtil.getCurlHelper().get("/_configsync/file").param("path", dictFile.getPath()).execute().getContentAsStream();
CurlRequest curlRequest = ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", dictFile.getPath());
if(SslUtil.isSslSecure()) {
curlRequest.header("Authorization", SslUtil.getBasicAuthEncodedCredentials());
}
return curlRequest.execute().getContentAsStream();
} catch (final IOException e) {
throw new DictionaryException("Failed to access " + dictFile.getPath(), e);
}

46
src/main/java/org/codelibs/fess/es/client/FessEsClient.java
View file

@ -45,6 +45,7 @@ import org.codelibs.core.io.ResourceUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.elasticsearch.runner.ElasticsearchClusterRunner;
import org.codelibs.elasticsearch.runner.ElasticsearchClusterRunner.Configs;
import org.codelibs.elasticsearch.runner.net.CurlRequest;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.Constants;
import org.codelibs.fess.entity.FacetInfo;
@ -61,6 +62,7 @@ import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocMap;
import org.codelibs.fess.util.SslUtil;
import org.dbflute.exception.IllegalBehaviorStateException;
import org.dbflute.optional.OptionalEntity;
import org.elasticsearch.ElasticsearchException;
@ -352,6 +354,23 @@ public class FessEsClient implements Client {
settingsBuilder.put("client.transport.sniff", fessConfig.isElasticsearchTransportSniff());
settingsBuilder.put("client.transport.ping_timeout", fessConfig.getElasticsearchTransportPingTimeout());
settingsBuilder.put("client.transport.nodes_sampler_interval", fessConfig.getElasticsearchTransportNodesSamplerInterval());
if(SslUtil.isSslSecure()) {
if(SslUtil.SSL_KEY_PATH != "") {
settingsBuilder.put("searchguard.ssl.transport.pemkey_filepath", SslUtil.SSL_KEY_PATH);
} else {
logger.warn("Missing SSL_KEY_PATH as sg.ssl.transport.pemkeyfilepath");
}
if(SslUtil.SSL_CERT_PATH != "") {
settingsBuilder.put("searchguard.ssl.transport.pemcert_filepath", SslUtil.SSL_CERT_PATH);
} else {
logger.warn("Missing SSL_CERT_PATH as sg.ssl.transport.pemcertfilepath");
}
if(SslUtil.SSL_CA_PATH != "") {
settingsBuilder.put("searchguard.ssl.transport.pemtrustedcas_filepath", SslUtil.SSL_CA_PATH);
} else {
logger.warn("Missing SSL_CA_PATH as sg.ssl.transport.pemcafilepath");
}
}
final Settings settings = settingsBuilder.build();
final TransportClient transportClient = new PreBuiltTransportClient(settings);
for (final TransportAddress address : transportAddressList) {
@ -375,9 +394,13 @@ public class FessEsClient implements Client {
public boolean reindex(final String fromIndex, final String toIndex, final boolean waitForCompletion) {
final String source = "{\"source\":{\"index\":\"" + fromIndex + "\"},\"dest\":{\"index\":\"" + toIndex + "\"}}";
CurlRequest curlRequest = ComponentUtil.getCurlHelper().post("/_reindex").param("wait_for_completion", Boolean.toString(waitForCompletion))
.body(source);
if(SslUtil.isSslSecure()) {
curlRequest.header("Authorization", SslUtil.getBasicAuthEncodedCredentials());
}
try (CurlResponse response =
ComponentUtil.getCurlHelper().post("/_reindex").param("wait_for_completion", Boolean.toString(waitForCompletion))
.body(source).execute()) {
curlRequest.execute()) {
if (response.getHttpStatusCode() == 200) {
return true;
} else {
@ -517,8 +540,11 @@ public class FessEsClient implements Client {
final String filePath = indexConfigPath + "/" + index + "/" + path;
try {
source = FileUtil.readUTF8(filePath);
try (CurlResponse response =
ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", path).body(source).execute()) {
CurlRequest curlRequest = ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", path).body(source);
if(SslUtil.isSslSecure()) {
curlRequest.header("Authorization", SslUtil.getBasicAuthEncodedCredentials());
}
try (CurlResponse response = curlRequest.execute()) {
if (response.getHttpStatusCode() == 200) {
logger.info("Register " + path + " to " + index);
} else {
@ -533,7 +559,11 @@ public class FessEsClient implements Client {
logger.warn("Failed to register " + filePath, e);
}
});
try (CurlResponse response = ComponentUtil.getCurlHelper().post("/_configsync/flush").execute()) {
CurlRequest curlRequest = ComponentUtil.getCurlHelper().post("/_configsync/flush");
if(SslUtil.isSslSecure()) {
curlRequest.header("Authorization", SslUtil.getBasicAuthEncodedCredentials());
}
try (CurlResponse response = curlRequest.execute()) {
if (response.getHttpStatusCode() == 200) {
logger.info("Flushed config files.");
} else {
@ -624,7 +654,11 @@ public class FessEsClient implements Client {
protected void waitForConfigSyncStatus() {
FessSystemException cause = null;
for (int i = 0; i < maxConfigSyncStatusRetry; i++) {
try (CurlResponse response = ComponentUtil.getCurlHelper().get("/_configsync/wait").param("status", "green").execute()) {
CurlRequest curlRequest = ComponentUtil.getCurlHelper().get("/_configsync/wait").param("status", "green");
if(SslUtil.isSslSecure()) {
curlRequest.header("Authorization", SslUtil.getBasicAuthEncodedCredentials());
}
try (CurlResponse response = curlRequest.execute()) {
final int httpStatusCode = response.getHttpStatusCode();
if (httpStatusCode == 200) {
logger.info("ConfigSync is ready.");

33
src/main/java/org/codelibs/fess/util/SslUtil.java Normal file
View file

@ -0,0 +1,33 @@
package org.codelibs.fess.util;
import java.io.UnsupportedEncodingException;
import java.util.Base64;
import org.codelibs.core.lang.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SslUtil {
private static final Logger logger = LoggerFactory.getLogger(SslUtil.class);
private final static String SSL_ENABLED = System.getProperty("fess.es.ssl.enabled");
private final static String SSL_CREDENTIALS = System.getProperty("sg.ssl.http.user_pwd");
public final static String SSL_KEY_PATH = System.getProperty("sg.ssl.transport.pemkeyfilepath");
public final static String SSL_CERT_PATH = System.getProperty("sg.ssl.transport.pemcertfilepath");
public final static String SSL_CA_PATH = System.getProperty("sg.ssl.transport.pemcafilepath");
public static boolean isSslSecure() {
return !StringUtil.isEmpty(SSL_ENABLED) && SSL_ENABLED.equals("true");
}
public static String getBasicAuthEncodedCredentials() {
try {
return "Basic " + Base64.getEncoder().encodeToString(SSL_CREDENTIALS.getBytes("UTF-8"));
} catch (UnsupportedEncodingException e) {
logger.error("Failed to encode https user/pwd.");
return "";
}
}
}

8
src/main/java/org/codelibs/fess/util/UpgradeUtil.java
View file

@ -16,10 +16,12 @@
package org.codelibs.fess.util;
import java.io.File;
import java.io.UnsupportedEncodingException;
import java.util.function.Consumer;
import org.codelibs.core.exception.ResourceNotFoundRuntimeException;
import org.codelibs.core.io.FileUtil;
import org.codelibs.elasticsearch.runner.net.CurlRequest;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.elasticsearch.action.ActionListener;
@ -50,7 +52,11 @@ public final class UpgradeUtil {
final String filePath = indexConfigPath + "/" + indexName + "/" + path;
try {
final String source = FileUtil.readUTF8(filePath);
try (CurlResponse response = ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", path).body(source).execute()) {
CurlRequest curlRequest = ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", path).body(source);
if(SslUtil.isSslSecure()) {
curlRequest.header("Authorization", SslUtil.getBasicAuthEncodedCredentials());
}
try (CurlResponse response = curlRequest.execute()) {
if (response.getHttpStatusCode() == 200) {
logger.info("Register " + path + " to " + indexName);
return true;