0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

merge from 10.0.x

Browse source
This commit is contained in:
Shinsuke Sugaya 2016-02-22 06:38:20 +09:00
commit 47ed63b116
36 changed files with 274 additions and 137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
pom.xml
View file

@ -59,7 +59,7 @@
<utflute.version>0.6.0F</utflute.version>
<!-- Crawler -->
<crawler.version>1.0.5-SNAPSHOT</crawler.version>
<crawler.version>1.0.5</crawler.version>
<!-- Suggest -->
<suggest.version>2.1.1</suggest.version>
@ -87,6 +87,7 @@
<packaging.fess.systemd.dir>/usr/lib/systemd/system</packaging.fess.systemd.dir>
<packaging.fess.systemd.sysctl.dir>/usr/lib/sysctl.d</packaging.fess.systemd.sysctl.dir>
<packaging.fess.tmpfilesd.dir>/usr/lib/tmpfiles.d</packaging.fess.tmpfilesd.dir>
<packaging.fess.dictionary.dir>/var/lib/elasticsearch/config</packaging.fess.dictionary.dir>
</properties>
<build>
@ -461,6 +462,13 @@
<mapping>
<directory>${packaging.fess.temp.dir}</directory>
</mapping>
<!-- dictionary (empty) -->
<mapping>
<directory>${packaging.fess.dictionary.dir}</directory>
<filemode>755</filemode>
<username>elasticsearch</username>
<groupname>elasticsearch</groupname>
</mapping>
<!-- es/plugins -->
<mapping>
<directory>${packaging.fess.home.dir}/es/plugins</directory>
@ -616,7 +624,7 @@
</goals>
</pluginExecutionFilter>
<action>
<ignore></ignore>
<ignore />
</action>
</pluginExecution>
<pluginExecution>
@ -637,7 +645,7 @@
</goals>
</pluginExecutionFilter>
<action>
<ignore></ignore>
<ignore />
</action>
</pluginExecution>
</pluginExecutions>

68
src/main/java/org/codelibs/fess/app/service/SearchService.java
View file

@ -16,7 +16,6 @@
package org.codelibs.fess.app.service;
import java.text.NumberFormat;
import java.time.LocalDateTime;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.List;
@ -29,7 +28,6 @@ import java.util.function.Consumer;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.lang3.StringUtils;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
@ -39,13 +37,8 @@ import org.codelibs.fess.entity.SearchRequestParams;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.client.FessEsClient.SearchConditionBuilder;
import org.codelibs.fess.es.client.FessEsClientException;
import org.codelibs.fess.es.log.exentity.SearchLog;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.helper.RoleQueryHelper;
import org.codelibs.fess.helper.SearchLogHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.helper.UserInfoHelper;
import org.codelibs.fess.mylasta.action.FessUserBean;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.QueryResponseList;
@ -86,9 +79,6 @@ public class SearchService {
@Resource
protected QueryHelper queryHelper;
@Resource
protected UserInfoHelper userInfoHelper;
// ===================================================================================
// Method
// ==============
@ -176,7 +166,8 @@ public class SearchService {
// search log
if (searchLogSupport) {
storeSearchLog(request, DfTypeUtil.toLocalDateTime(requestedTime), queryId, query, pageStart, pageSize, queryResponseList);
ComponentUtil.getSearchLogHelper().addSearchLog(DfTypeUtil.toLocalDateTime(requestedTime), queryId, query, pageStart, pageSize,
queryResponseList);
}
}
@ -192,61 +183,6 @@ public class SearchService {
queryContext.getQueryBuilder());
}
protected void storeSearchLog(final HttpServletRequest request, final LocalDateTime requestedTime, final String queryId,
final String query, final int pageStart, final int pageSize, final QueryResponseList queryResponseList) {
final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper();
final RoleQueryHelper roleQueryHelper = ComponentUtil.getRoleQueryHelper();
final SearchLog searchLog = new SearchLog();
if (Constants.TRUE.equals(systemProperties.getProperty(Constants.USER_INFO_PROPERTY, Constants.TRUE))) {
final String userCode = userInfoHelper.getUserCode();
if (userCode != null) {
searchLog.setUserSessionId(userCode);
}
}
searchLog.setRoles(roleQueryHelper.build().stream().toArray(n -> new String[n]));
searchLog.setQueryId(queryId);
searchLog.setHitCount(queryResponseList.getAllRecordCount());
searchLog.setResponseTime(queryResponseList.getExecTime());
searchLog.setQueryTime(queryResponseList.getQueryTime());
searchLog.setSearchWord(StringUtils.abbreviate(query, 1000));
searchLog.setSearchQuery(StringUtils.abbreviate(queryResponseList.getSearchQuery(), 1000));
searchLog.setRequestedAt(requestedTime);
searchLog.setQueryOffset(pageStart);
searchLog.setQueryPageSize(pageSize);
ComponentUtil.getRequestManager().findUserBean(FessUserBean.class).ifPresent(user -> {
searchLog.setUser(user.getUserId());
});
searchLog.setClientIp(StringUtils.abbreviate(request.getRemoteAddr(), 50));
searchLog.setReferer(StringUtils.abbreviate(request.getHeader("referer"), 1000));
searchLog.setUserAgent(StringUtils.abbreviate(request.getHeader("user-agent"), 255));
final Object accessType = request.getAttribute(Constants.SEARCH_LOG_ACCESS_TYPE);
if (Constants.SEARCH_LOG_ACCESS_TYPE_JSON.equals(accessType)) {
searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_JSON);
} else if (Constants.SEARCH_LOG_ACCESS_TYPE_XML.equals(accessType)) {
searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_XML);
} else if (Constants.SEARCH_LOG_ACCESS_TYPE_OTHER.equals(accessType)) {
searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_OTHER);
} else {
searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_WEB);
}
@SuppressWarnings("unchecked")
final Map<String, List<String>> fieldLogMap = (Map<String, List<String>>) request.getAttribute(Constants.FIELD_LOGS);
if (fieldLogMap != null) {
for (final Map.Entry<String, List<String>> logEntry : fieldLogMap.entrySet()) {
for (final String value : logEntry.getValue()) {
searchLog.addSearchFieldLogValue(logEntry.getKey(), StringUtils.abbreviate(value, 1000));
}
}
}
searchLogHelper.addSearchLog(searchLog);
}
public String[] getLanguages(final HttpServletRequest request, final SearchRequestParams params) {
if (params.getLanguages() != null) {
final Set<String> langSet = new HashSet<>();

3
src/main/java/org/codelibs/fess/app/web/admin/fileconfig/CreateForm.java
View file

@ -27,6 +27,7 @@ import org.codelibs.fess.app.web.CrudMode;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.StreamUtil;
import org.codelibs.fess.validation.UriType;
import org.codelibs.fess.validation.UriTypeValidator.ProtocolType;
import org.lastaflute.web.validation.Required;
import org.lastaflute.web.validation.theme.conversion.ValidateTypeFailure;
@ -50,7 +51,7 @@ public class CreateForm implements Serializable {
public String name;
@Required
@UriType(protocols = { "file:", "smb:" })
@UriType(protocolType = ProtocolType.FILE)
@Size(max = 4000)
public String paths;

3
src/main/java/org/codelibs/fess/app/web/admin/webconfig/CreateForm.java
View file

@ -27,6 +27,7 @@ import org.codelibs.fess.app.web.CrudMode;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.StreamUtil;
import org.codelibs.fess.validation.UriType;
import org.codelibs.fess.validation.UriTypeValidator.ProtocolType;
import org.lastaflute.web.validation.Required;
import org.lastaflute.web.validation.theme.conversion.ValidateTypeFailure;
@ -51,7 +52,7 @@ public class CreateForm implements Serializable {
public String name;
@Required
@UriType(protocols = { "http:", "https:" })
@UriType(protocolType = ProtocolType.WEB)
@Size(max = 4000)
public String urls;

3
src/main/java/org/codelibs/fess/app/web/base/FessAdminAction.java
View file

@ -24,7 +24,6 @@ import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.core.beans.util.CopyOptions;
import org.codelibs.fess.exception.UserRoleLoginException;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.util.ActivityUtil;
import org.dbflute.optional.OptionalThing;
import org.lastaflute.di.util.LdiFileUtil;
import org.lastaflute.web.login.LoginManager;
@ -111,7 +110,7 @@ public abstract class FessAdminAction extends FessBaseAction {
final String username = getUserBean().map(u -> u.getUserId()).orElse("-");
final String requestPath = runtime.getRequestPath();
final String executeName = runtime.getExecuteMethod().getName();
ActivityUtil.access(username, requestPath, executeName);
activityHelper.access(getUserBean(), requestPath, executeName);
return super.hookBefore(runtime);
}

4
src/main/java/org/codelibs/fess/app/web/base/FessBaseAction.java
View file

@ -18,6 +18,7 @@ package org.codelibs.fess.app.web.base;
import javax.annotation.Resource;
import org.codelibs.fess.app.web.base.login.FessLoginAssist;
import org.codelibs.fess.helper.ActivityHelper;
import org.codelibs.fess.mylasta.action.FessHtmlPath;
import org.codelibs.fess.mylasta.action.FessMessages;
import org.codelibs.fess.mylasta.action.FessUserBean;
@ -60,6 +61,9 @@ public abstract class FessBaseAction extends TypicalAction // has several interf
@Resource
protected FessConfig fessConfig;
@Resource
protected ActivityHelper activityHelper;
// ===================================================================================
// Hook
// ======

7
src/main/java/org/codelibs/fess/app/web/login/LoginAction.java
View file

@ -20,7 +20,6 @@ import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.admin.dashboard.AdminDashboardAction;
import org.codelibs.fess.app.web.base.FessSearchAction;
import org.codelibs.fess.mylasta.action.FessUserBean;
import org.codelibs.fess.util.ActivityUtil;
import org.codelibs.fess.util.RenderDataUtil;
import org.lastaflute.web.Execute;
import org.lastaflute.web.login.exception.LoginFailureException;
@ -60,9 +59,11 @@ public class LoginAction extends FessSearchAction {
final String username = form.username;
final String password = form.password;
form.clearSecurityInfo();
ActivityUtil.login(username);
try {
return fessLoginAssist.loginRedirect(username, password, op -> {}, () -> getHtmlResponse());
return fessLoginAssist.loginRedirect(username, password, op -> {}, () -> {
activityHelper.login(getUserBean());
return getHtmlResponse();
});
} catch (final LoginFailureException lfe) {
throwValidationError(messages -> messages.addErrorsLoginError(GLOBAL), () -> {
return asHtml(path_Login_IndexJsp);

3
src/main/java/org/codelibs/fess/app/web/logout/LogoutAction.java
View file

@ -17,7 +17,6 @@ package org.codelibs.fess.app.web.logout;
import org.codelibs.fess.app.web.base.FessSearchAction;
import org.codelibs.fess.app.web.login.LoginAction;
import org.codelibs.fess.util.ActivityUtil;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.HtmlResponse;
@ -42,7 +41,7 @@ public class LogoutAction extends FessSearchAction {
@Execute
public HtmlResponse index() {
final String username = getUserBean().map(u -> u.getUserId()).orElse("-");
ActivityUtil.logout(username);
activityHelper.logout(getUserBean());
fessLoginAssist.logout();
return redirect(LoginAction.class);
}

23
src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java
View file

@ -437,6 +437,29 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
return getHost(url);
}
protected List<String> getRoleTypes(final ResponseData responseData) {
final List<String> roleTypeList = new ArrayList<>();
if (fessConfig.isSmbRoleFromFile() && responseData.getUrl().startsWith("smb://")) {
final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
final ACE[] aces = (ACE[]) responseData.getMetaDataMap().get(SmbClient.SMB_ACCESS_CONTROL_ENTRIES);
if (aces != null) {
for (final ACE item : aces) {
final SID sid = item.getSID();
final String accountId = sambaHelper.getAccountId(sid);
if (accountId != null) {
roleTypeList.add(accountId);
}
}
if (getLogger().isDebugEnabled()) {
getLogger().debug("smbUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
}
}
}
return roleTypeList;
}
protected String getSiteOnFile(final String url, final String encoding) {
if (StringUtil.isBlank(url)) {
return StringUtil.EMPTY; // empty

2
src/main/java/org/codelibs/fess/ds/IndexUpdateCallback.java
View file

@ -19,7 +19,7 @@ import java.util.Map;
public interface IndexUpdateCallback {
boolean store(Map<String, Object> dataMap);
boolean store(Map<String, String> paramMap, Map<String, Object> dataMap);
long getDocumentSize();

2
src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java
View file

@ -234,7 +234,7 @@ public class CsvDataStoreImpl extends AbstractDataStoreImpl {
}
try {
loop = callback.store(dataMap);
loop = callback.store(paramMap, dataMap);
} catch (final CrawlingAccessException e) {
logger.warn("Crawling Access Exception at : " + dataMap, e);

2
src/main/java/org/codelibs/fess/ds/impl/DatabaseDataStoreImpl.java
View file

@ -110,7 +110,7 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
}
try {
loop = callback.store(dataMap);
loop = callback.store(paramMap, dataMap);
} catch (final Exception e) {
logger.warn("Failed to store data: " + dataMap, e);
}

18
src/main/java/org/codelibs/fess/ds/impl/FileListDataStoreImpl.java
View file

@ -143,22 +143,22 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
}
@Override
public boolean store(final Map<String, Object> dataMap) {
public boolean store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
final Object eventType = dataMap.remove(eventTypeField);
if (createEventName.equals(eventType) || modifyEventName.equals(eventType)) {
// updated file
return addDocument(dataMap);
return addDocument(paramMap, dataMap);
} else if (deleteEventName.equals(eventType)) {
// deleted file
return deleteDocument(dataMap);
return deleteDocument(paramMap, dataMap);
}
logger.warn("unknown event: " + eventType + ", data: " + dataMap);
return false;
}
protected boolean addDocument(final Map<String, Object> dataMap) {
protected boolean addDocument(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
synchronized (indexUpdateCallback) {
// required check
@ -178,7 +178,11 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
final long startTime = System.currentTimeMillis();
final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build());
responseData.setExecutionTime(System.currentTimeMillis() - startTime);
responseData.setSessionId((String) dataMap.get(Constants.SESSION_ID));
if (dataMap.containsKey(Constants.SESSION_ID)) {
responseData.setSessionId((String) dataMap.get(Constants.SESSION_ID));
} else {
responseData.setSessionId((String) paramMap.get(Constants.CRAWLING_INFO_ID));
}
final RuleManager ruleManager = SingletonLaContainer.getComponent(RuleManager.class);
final Rule rule = ruleManager.getRule(responseData);
@ -208,7 +212,7 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
dataMap.remove(fieldName);
}
return indexUpdateCallback.store(dataMap);
return indexUpdateCallback.store(paramMap, dataMap);
} else {
logger.warn("The response processor is not DefaultResponseProcessor. responseProcessor: " + responseProcessor
+ ", Data: " + dataMap);
@ -221,7 +225,7 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
}
}
protected boolean deleteDocument(final Map<String, Object> dataMap) {
protected boolean deleteDocument(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
if (logger.isDebugEnabled()) {
logger.debug("Deleting " + dataMap);

2
src/main/java/org/codelibs/fess/ds/impl/IndexUpdateCallbackImpl.java
View file

@ -45,7 +45,7 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
* @see org.codelibs.fess.ds.impl.IndexUpdateCallback#store(java.util.Map)
*/
@Override
public synchronized boolean store(final Map<String, Object> dataMap) {
public synchronized boolean store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
final long startTime = System.currentTimeMillis();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();

54
src/main/java/org/codelibs/fess/es/config/exentity/DataConfig.java
View file

@ -60,15 +60,15 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
private static final Logger logger = LoggerFactory.getLogger(DataConfig.class);
private static final String S2ROBOT_WEB_HEADER_PREFIX = "crawler.web.header.";
private static final String CRAWLER_WEB_HEADER_PREFIX = "crawler.web.header.";
private static final String S2ROBOT_WEB_AUTH = "crawler.web.auth";
private static final String CRAWLER_WEB_AUTH = "crawler.web.auth";
private static final String S2ROBOT_USERAGENT = "crawler.useragent";
private static final String CRAWLER_USERAGENT = "crawler.useragent";
private static final String S2ROBOT_PARAM_PREFIX = "crawler.param.";
private static final String CRAWLER_PARAM_PREFIX = "crawler.param.";
private static final Object S2ROBOT_FILE_AUTH = "crawler.file.auth";
private static final Object CRAWLER_FILE_AUTH = "crawler.file.auth";
private String[] labelTypeIds;
@ -240,29 +240,29 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
// parameters
for (final Map.Entry<String, String> entry : paramMap.entrySet()) {
final String key = entry.getKey();
if (key.startsWith(S2ROBOT_PARAM_PREFIX)) {
factoryParamMap.put(key.substring(S2ROBOT_PARAM_PREFIX.length()), entry.getValue());
if (key.startsWith(CRAWLER_PARAM_PREFIX)) {
factoryParamMap.put(key.substring(CRAWLER_PARAM_PREFIX.length()), entry.getValue());
}
}
// user agent
final String userAgent = paramMap.get(S2ROBOT_USERAGENT);
final String userAgent = paramMap.get(CRAWLER_USERAGENT);
if (StringUtil.isNotBlank(userAgent)) {
factoryParamMap.put(HcHttpClient.USER_AGENT_PROPERTY, userAgent);
}
// web auth
final String webAuthStr = paramMap.get(S2ROBOT_WEB_AUTH);
final String webAuthStr = paramMap.get(CRAWLER_WEB_AUTH);
if (StringUtil.isNotBlank(webAuthStr)) {
final String[] webAuthNames = webAuthStr.split(",");
final List<Authentication> basicAuthList = new ArrayList<Authentication>();
for (final String webAuthName : webAuthNames) {
final String scheme = paramMap.get(S2ROBOT_WEB_AUTH + "." + webAuthName + ".scheme");
final String hostname = paramMap.get(S2ROBOT_WEB_AUTH + "." + webAuthName + ".host");
final String port = paramMap.get(S2ROBOT_WEB_AUTH + "." + webAuthName + ".port");
final String realm = paramMap.get(S2ROBOT_WEB_AUTH + "." + webAuthName + ".realm");
final String username = paramMap.get(S2ROBOT_WEB_AUTH + "." + webAuthName + ".username");
final String password = paramMap.get(S2ROBOT_WEB_AUTH + "." + webAuthName + ".password");
final String scheme = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".scheme");
final String hostname = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".host");
final String port = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".port");
final String realm = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".realm");
final String username = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".username");
final String password = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".password");
if (StringUtil.isEmpty(username)) {
logger.warn("username is empty. webAuth:" + webAuthName);
@ -305,8 +305,8 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
Credentials credentials;
if (Constants.NTLM.equals(scheme)) {
final String workstation = paramMap.get(S2ROBOT_WEB_AUTH + "." + webAuthName + ".workstation");
final String domain = paramMap.get(S2ROBOT_WEB_AUTH + "." + webAuthName + ".domain");
final String workstation = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".workstation");
final String domain = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".domain");
credentials =
new NTCredentials(username, password == null ? StringUtil.EMPTY : password,
workstation == null ? StringUtil.EMPTY : workstation, domain == null ? StringUtil.EMPTY : domain);
@ -324,12 +324,12 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList =
new ArrayList<org.codelibs.fess.crawler.client.http.RequestHeader>();
int count = 1;
String headerName = paramMap.get(S2ROBOT_WEB_HEADER_PREFIX + count + ".name");
String headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
while (StringUtil.isNotBlank(headerName)) {
final String headerValue = paramMap.get(S2ROBOT_WEB_HEADER_PREFIX + count + ".value");
final String headerValue = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".value");
rhList.add(new org.codelibs.fess.crawler.client.http.RequestHeader(headerName, headerValue));
count++;
headerName = paramMap.get(S2ROBOT_WEB_HEADER_PREFIX + count + ".name");
headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
}
if (!rhList.isEmpty()) {
factoryParamMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY,
@ -337,18 +337,18 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
}
// file auth
final String fileAuthStr = paramMap.get(S2ROBOT_FILE_AUTH);
final String fileAuthStr = paramMap.get(CRAWLER_FILE_AUTH);
if (StringUtil.isNotBlank(fileAuthStr)) {
final String[] fileAuthNames = fileAuthStr.split(",");
final List<SmbAuthentication> smbAuthList = new ArrayList<SmbAuthentication>();
for (final String fileAuthName : fileAuthNames) {
final String scheme = paramMap.get(S2ROBOT_FILE_AUTH + "." + fileAuthName + ".scheme");
final String scheme = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".scheme");
if (Constants.SAMBA.equals(scheme)) {
final String domain = paramMap.get(S2ROBOT_FILE_AUTH + "." + fileAuthName + ".domain");
final String hostname = paramMap.get(S2ROBOT_FILE_AUTH + "." + fileAuthName + ".host");
final String port = paramMap.get(S2ROBOT_FILE_AUTH + "." + fileAuthName + ".port");
final String username = paramMap.get(S2ROBOT_FILE_AUTH + "." + fileAuthName + ".username");
final String password = paramMap.get(S2ROBOT_FILE_AUTH + "." + fileAuthName + ".password");
final String domain = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".domain");
final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
if (StringUtil.isEmpty(username)) {
logger.warn("username is empty. fileAuth:" + fileAuthName);

35
src/main/java/org/codelibs/fess/util/ActivityUtil.java → src/main/java/org/codelibs/fess/helper/ActivityHelper.java
View file

@ -13,12 +13,16 @@
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.util;
package org.codelibs.fess.helper;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import javax.annotation.PostConstruct;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.mylasta.action.FessUserBean;
import org.dbflute.optional.OptionalThing;
import org.lastaflute.web.util.LaRequestUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -27,36 +31,43 @@ import org.slf4j.LoggerFactory;
* @author shinsuke
*
*/
public class ActivityUtil {
private static Logger logger = LoggerFactory.getLogger("fess.log.audit");
public class ActivityHelper {
private Logger logger = null;
public static void login(final String username) {
protected String loggerName = "fess.log.audit";
@PostConstruct
public void init() {
logger = LoggerFactory.getLogger(loggerName);
}
public void login(final OptionalThing<FessUserBean> user) {
final StringBuilder buf = new StringBuilder(100);
buf.append("action:");
buf.append(Action.LOGIN);
buf.append('\t');
buf.append("user:");
buf.append(username);
buf.append(user.map(u -> u.getUserId()).orElse("-"));
log(buf);
}
public static void logout(final String username) {
public void logout(final OptionalThing<FessUserBean> user) {
final StringBuilder buf = new StringBuilder(100);
buf.append("action:");
buf.append(Action.LOGOUT);
buf.append('\t');
buf.append("user:");
buf.append(username);
buf.append(user.map(u -> u.getUserId()).orElse("-"));
log(buf);
}
public static void access(final String username, final String path, final String execute) {
public void access(final OptionalThing<FessUserBean> user, final String path, final String execute) {
final StringBuilder buf = new StringBuilder(100);
buf.append("action:");
buf.append(Action.ACCESS);
buf.append('\t');
buf.append("user:");
buf.append(username);
buf.append(user.map(u -> u.getUserId()).orElse("-"));
buf.append('\t');
buf.append("path:");
buf.append(path);
@ -66,7 +77,7 @@ public class ActivityUtil {
log(buf);
}
private static void log(final StringBuilder buf) {
private void log(final StringBuilder buf) {
buf.append('\t');
buf.append("ip:");
buf.append(getClientIp());
@ -90,4 +101,8 @@ public class ActivityUtil {
protected enum Action {
LOGIN, LOGOUT, ACCESS;
}
public void setLoggerName(String loggerName) {
this.loggerName = loggerName;
}
}

59
src/main/java/org/codelibs/fess/helper/SearchLogHelper.java
View file

@ -25,7 +25,9 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.lang3.StringUtils;
import org.codelibs.core.collection.LruHashMap;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
@ -40,13 +42,16 @@ import org.codelibs.fess.es.log.exentity.ClickLog;
import org.codelibs.fess.es.log.exentity.SearchFieldLog;
import org.codelibs.fess.es.log.exentity.SearchLog;
import org.codelibs.fess.es.log.exentity.UserInfo;
import org.codelibs.fess.mylasta.action.FessUserBean;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocumentUtil;
import org.codelibs.fess.util.QueryResponseList;
import org.codelibs.fess.util.StreamUtil;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.script.Script;
import org.lastaflute.di.core.SingletonLaContainer;
import org.lastaflute.web.util.LaRequestUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -71,7 +76,59 @@ public class SearchLogHelper {
userInfoCache = new LruHashMap<String, Long>(userInfoCacheSize);
}
public void addSearchLog(final SearchLog searchLog) {
public void addSearchLog(final LocalDateTime requestedTime, final String queryId, final String query, final int pageStart,
final int pageSize, final QueryResponseList queryResponseList) {
final RoleQueryHelper roleQueryHelper = ComponentUtil.getRoleQueryHelper();
final UserInfoHelper userInfoHelper = ComponentUtil.getUserInfoHelper();
final SearchLog searchLog = new SearchLog();
if (Constants.TRUE.equals(systemProperties.getProperty(Constants.USER_INFO_PROPERTY, Constants.TRUE))) {
final String userCode = userInfoHelper.getUserCode();
if (userCode != null) {
searchLog.setUserSessionId(userCode);
}
}
searchLog.setRoles(roleQueryHelper.build().stream().toArray(n -> new String[n]));
searchLog.setQueryId(queryId);
searchLog.setHitCount(queryResponseList.getAllRecordCount());
searchLog.setResponseTime(queryResponseList.getExecTime());
searchLog.setQueryTime(queryResponseList.getQueryTime());
searchLog.setSearchWord(StringUtils.abbreviate(query, 1000));
searchLog.setSearchQuery(StringUtils.abbreviate(queryResponseList.getSearchQuery(), 1000));
searchLog.setRequestedAt(requestedTime);
searchLog.setQueryOffset(pageStart);
searchLog.setQueryPageSize(pageSize);
ComponentUtil.getRequestManager().findUserBean(FessUserBean.class).ifPresent(user -> {
searchLog.setUser(user.getUserId());
});
final HttpServletRequest request = LaRequestUtil.getRequest();
searchLog.setClientIp(StringUtils.abbreviate(request.getRemoteAddr(), 50));
searchLog.setReferer(StringUtils.abbreviate(request.getHeader("referer"), 1000));
searchLog.setUserAgent(StringUtils.abbreviate(request.getHeader("user-agent"), 255));
final Object accessType = request.getAttribute(Constants.SEARCH_LOG_ACCESS_TYPE);
if (Constants.SEARCH_LOG_ACCESS_TYPE_JSON.equals(accessType)) {
searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_JSON);
} else if (Constants.SEARCH_LOG_ACCESS_TYPE_XML.equals(accessType)) {
searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_XML);
} else if (Constants.SEARCH_LOG_ACCESS_TYPE_OTHER.equals(accessType)) {
searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_OTHER);
} else {
searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_WEB);
}
@SuppressWarnings("unchecked")
final Map<String, List<String>> fieldLogMap = (Map<String, List<String>>) request.getAttribute(Constants.FIELD_LOGS);
if (fieldLogMap != null) {
for (final Map.Entry<String, List<String>> logEntry : fieldLogMap.entrySet()) {
for (final String value : logEntry.getValue()) {
searchLog.addSearchFieldLogValue(logEntry.getKey(), StringUtils.abbreviate(value, 1000));
}
}
}
searchLogQueue.add(searchLog);
}

6
src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java
View file

@ -39,6 +39,7 @@ import org.codelibs.fess.crawler.service.UrlQueueService;
import org.codelibs.fess.es.config.exentity.FileConfig;
import org.codelibs.fess.es.config.exentity.WebConfig;
import org.codelibs.fess.indexer.IndexUpdater;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.lastaflute.di.core.SingletonLaContainer;
import org.slf4j.Logger;
@ -134,6 +135,7 @@ public class WebFsIndexHelper implements Serializable {
}
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final long startTime = System.currentTimeMillis();
@ -184,7 +186,7 @@ public class WebFsIndexHelper implements Serializable {
for (final String u : urls) {
if (StringUtil.isNotBlank(u)) {
final String urlValue = u.trim();
if (!urlValue.startsWith("#")) {
if (!urlValue.startsWith("#") && fessConfig.isValidCrawlerWebProtocol(u)) {
crawler.addUrl(urlValue);
if (logger.isInfoEnabled()) {
logger.info("Target URL: " + urlValue);
@ -288,7 +290,7 @@ public class WebFsIndexHelper implements Serializable {
if (StringUtil.isNotBlank(u)) {
u = u.trim();
if (!u.startsWith("#")) {
if (!u.startsWith("file:") && !u.startsWith("smb:")) {
if (!fessConfig.isValidCrawlerFileProtocol(u)) {
if (u.startsWith("/")) {
u = "file:" + u;
} else {

3
src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java
View file

@ -2058,6 +2058,9 @@ public class FessLabels extends ActionMessages {
/** The key of the message: Name */
public static final String LABELS_backup_name = "{labels.backup_name}";
/** The key of the message: The limit of a search time was exceeded. The partial result might be displayed. */
public static final String LABELS_process_time_is_exceeded = "{labels.process_time_is_exceeded}";
/**
* Assert the property is not null.
* @param property The value of the property. (NotNull)

28
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -126,6 +126,12 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. UTF-8 */
String CRAWLER_CRAWLING_DATA_ENCODING = "crawler.crawling.data.encoding";
/** The key of the configuration. e.g. http,https */
String CRAWLER_WEB_PROTOCOLS = "crawler.web.protocols";
/** The key of the configuration. e.g. file,smb */
String CRAWLER_FILE_PROTOCOLS = "crawler.file.protocols";
/** The key of the configuration. e.g. resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.* */
String CRAWLER_METADATA_CONTENT_EXCLUDES = "crawler.metadata.content.excludes";
@ -956,6 +962,20 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
String getCrawlerCrawlingDataEncoding();
/**
* Get the value for the key 'crawler.web.protocols'. <br>
* The value is, e.g. http,https <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getCrawlerWebProtocols();
/**
* Get the value for the key 'crawler.file.protocols'. <br>
* The value is, e.g. file,smb <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getCrawlerFileProtocols();
/**
* Get the value for the key 'crawler.metadata.content.excludes'. <br>
* The value is, e.g. resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.* <br>
@ -2854,6 +2874,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return get(FessConfig.CRAWLER_CRAWLING_DATA_ENCODING);
}
public String getCrawlerWebProtocols() {
return get(FessConfig.CRAWLER_WEB_PROTOCOLS);
}
public String getCrawlerFileProtocols() {
return get(FessConfig.CRAWLER_FILE_PROTOCOLS);
}
public String getCrawlerMetadataContentExcludes() {
return get(FessConfig.CRAWLER_METADATA_CONTENT_EXCLUDES);
}

21
src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
View file

@ -503,4 +503,25 @@ public interface FessProp {
return StreamUtil.of(getAuthenticationAdminUsers().split(",")).anyMatch(s -> s.equals(username));
}
String getCrawlerWebProtocols();
public default String[] getCrawlerWebProtocolsAsArray() {
return StreamUtil.of(getCrawlerWebProtocols().split(",")).filter(s -> StringUtil.isNotBlank(s)).map(s -> s.trim() + ":")
.toArray(n -> new String[n]);
}
public default boolean isValidCrawlerWebProtocol(final String url) {
return StreamUtil.of(getCrawlerWebProtocolsAsArray()).anyMatch(s -> url.startsWith(s));
}
String getCrawlerFileProtocols();
public default String[] getCrawlerFileProtocolsAsArray() {
return StreamUtil.of(getCrawlerFileProtocols().split(",")).filter(s -> StringUtil.isNotBlank(s)).map(s -> s.trim() + ":")
.toArray(n -> new String[n]);
}
public default boolean isValidCrawlerFileProtocol(final String url) {
return StreamUtil.of(getCrawlerFileProtocolsAsArray()).anyMatch(s -> url.startsWith(s));
}
}

7
src/main/java/org/codelibs/fess/util/ComponentUtil.java
View file

@ -24,6 +24,7 @@ import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.dict.DictionaryManager;
import org.codelibs.fess.ds.DataStoreFactory;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.helper.ActivityHelper;
import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.CrawlingInfoHelper;
import org.codelibs.fess.helper.DuplicateHostHelper;
@ -55,6 +56,8 @@ import org.lastaflute.job.JobManager;
import org.lastaflute.web.servlet.request.RequestManager;
public final class ComponentUtil {
private static final String ACTIVITY_HELPER = "activityHelper";
private static final String LDAP_MANAGER = "ldapManager";
private static final String ROLE_QUERY_HELPER = "roleQueryHelper";
@ -276,6 +279,10 @@ public final class ComponentUtil {
return SingletonLaContainer.getComponent(LDAP_MANAGER);
}
public static ActivityHelper getActivityHelper() {
return SingletonLaContainer.getComponent(ACTIVITY_HELPER);
}
public static RequestManager getRequestManager() {
return SingletonLaContainer.getComponent(RequestManager.class);
}

4
src/main/java/org/codelibs/fess/validation/UriType.java
View file

@ -29,13 +29,15 @@ import java.lang.annotation.Target;
import javax.validation.Constraint;
import javax.validation.Payload;
import org.codelibs.fess.validation.UriTypeValidator.ProtocolType;
@Target({ METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER })
@Retention(RUNTIME)
@Documented
@Constraint(validatedBy = UriTypeValidator.class)
public @interface UriType {
String[] protocols();
ProtocolType protocolType();
String message() default "{org.lastaflute.validator.constraints.UriType.message}";

17
src/main/java/org/codelibs/fess/validation/UriTypeValidator.java
View file

@ -20,15 +20,22 @@ import javax.validation.ConstraintValidator;
import javax.validation.ConstraintValidatorContext;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.util.ComponentUtil;
public class UriTypeValidator implements ConstraintValidator<UriType, String> {
private String[] protocols;
@Override
public void initialize(final UriType uriType) {
protocols = uriType.protocols();
if (protocols == null || protocols.length == 0) {
throw new ConstraintDefinitionException("protocols is emtpy.");
switch (uriType.protocolType()) {
case WEB:
protocols = ComponentUtil.getFessConfig().getCrawlerWebProtocolsAsArray();
break;
case FILE:
protocols = ComponentUtil.getFessConfig().getCrawlerFileProtocolsAsArray();
break;
default:
throw new ConstraintDefinitionException("protocolType is emtpy.");
}
}
@ -58,4 +65,8 @@ public class UriTypeValidator implements ConstraintValidator<UriType, String> {
}
return true;
}
public enum ProtocolType {
WEB, FILE;
}
}

2
src/main/resources/app.xml
View file

@ -16,6 +16,8 @@
<include path="crawler/client.xml" />
<include path="crawler/mimetype.xml" />
<component name="activityHelper" class="org.codelibs.fess.helper.ActivityHelper">
</component>
<component name="labelTypeHelper" class="org.codelibs.fess.helper.LabelTypeHelper">
</component>
<component name="keyMatchHelper" class="org.codelibs.fess.helper.KeyMatchHelper">

4
src/main/resources/fess.xml
View file

@ -16,8 +16,8 @@
</component>
<component name="crawlingConfigHelper" class="org.codelibs.fess.helper.CrawlingConfigHelper">
</component>
<component name="pathMappingHelper" class="org.codelibs.fess.helper.PathMappingHelper">
</component>
<component name="pathMappingHelper" class="org.codelibs.fess.helper.PathMappingHelper">
</component>
<component name="processHelper" class="org.codelibs.fess.helper.ProcessHelper">
</component>
<component name="sambaHelper" class="org.codelibs.fess.helper.SambaHelper">

2
src/main/resources/fess_config.properties
View file

@ -74,6 +74,8 @@ crawler.document.unknown.hostname=unknown
crawler.document.use.site.encoding.on.english=false
crawler.document.append.data=true
crawler.crawling.data.encoding=UTF-8
crawler.web.protocols=http,https
crawler.file.protocols=file,smb
crawler.metadata.content.excludes=resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*
crawler.metadata.name.mapping=\
title=title:string\n\

1
src/main/resources/fess_label.properties
View file

@ -685,3 +685,4 @@ labels.general_menu_notification=Notification
labels.send_testmail=Send TestMail
labels.backup_configuration=Back Up
labels.backup_name=Name
labels.process_time_is_exceeded=The limit of a search time was exceeded. The partial result might be displayed.

1
src/main/resources/fess_label_en.properties
View file

@ -683,3 +683,4 @@ labels.notification_search_top=Search top page
labels.send_testmail=Send TestMail
labels.backup_configuration=Back Up
labels.backup_name=Name
labels.process_time_is_exceeded=The limit of a search time was exceeded. The partial result might be displayed.

1
src/main/resources/fess_label_ja.properties
View file

@ -676,3 +676,4 @@ labels.notification_search_top=\u691c\u7d22\u30c8\u30c3\u30d7\u30da\u30fc\u30b8
labels.send_testmail=\u30c6\u30b9\u30c8\u30e1\u30fc\u30eb\u306e\u9001\u4fe1
labels.backup_configuration=\u30d0\u30c3\u30af\u30a2\u30c3\u30d7
labels.backup_name=\u540d\u524d
labels.process_time_is_exceeded=\u691c\u7d22\u5f85\u3061\u6642\u9593\u306e\u4e0a\u9650\u3092\u8d85\u3048\u307e\u3057\u305f\u3002\u8868\u793a\u3055\u308c\u305f\u7d50\u679c\u306f\u691c\u7d22\u7d50\u679c\u306e\u4e00\u90e8\u3067\u3042\u308b\u53ef\u80fd\u6027\u304c\u3042\u308a\u307e\u3059\u3002

2
src/main/webapp/WEB-INF/orig/view/searchResults.jsp
View file

@ -16,7 +16,7 @@
<c:if test="${partialResults}">
<div class="alert">
<p>
<la:message key="errors.process_time_is_exceeded" />
<la:message key="labels.process_time_is_exceeded" />
</p>
</div>
</c:if>

8
src/main/webapp/WEB-INF/view/admin/design/admin_design.jsp
View file

@ -155,6 +155,14 @@
<la:option value="errorBadRequest">
<la:message key="labels.design_file_errorBadRequest" />
</la:option>
<%-- Login --%>
<la:option value="login">
<la:message key="labels.design_file_login" />
</la:option>
<%-- Profile --%>
<la:option value="profile">
<la:message key="labels.design_file_profile" />
</la:option>
</la:select>
</div>
</div>

2
src/main/webapp/WEB-INF/view/searchResults.jsp
View file

@ -16,7 +16,7 @@
<c:if test="${partialResults}">
<div class="alert">
<p>
<la:message key="errors.process_time_is_exceeded" />
<la:message key="labels.process_time_is_exceeded" />
</p>
</div>
</c:if>

BIN
src/main/webapp/images/logo-mono-nega.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 2.3 KiB

BIN
src/main/webapp/images/logo-mono.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 2.2 KiB

BIN
src/main/webapp/images/logo-small.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 724 B