0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1933 improve last modified check

Browse source
This commit is contained in:
Shinsuke Sugaya 2018-12-15 07:26:36 +09:00
parent 1bf3d8d4f2
commit e94b8a335a
3 changed files with 86 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java
View file

@ -32,7 +32,6 @@ import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.app.service.FailureUrlService;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.client.smb.SmbClient;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.UrlQueue;
@ -45,15 +44,13 @@ import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.CrawlingInfoHelper;
import org.codelibs.fess.helper.DuplicateHostHelper;
import org.codelibs.fess.helper.IndexingHelper;
import org.codelibs.fess.helper.SambaHelper;
import org.codelibs.fess.helper.PermissionHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocumentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import jcifs.SID;
public class FessCrawlerThread extends CrawlerThread {
private static final Logger logger = LoggerFactory.getLogger(FessCrawlerThread.class);
@ -66,7 +63,6 @@ public class FessCrawlerThread extends CrawlerThread {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
@ -78,30 +74,22 @@ public class FessCrawlerThread extends CrawlerThread {
dataMap.put(fessConfig.getIndexFieldUrl(), url);
final List<String> roleTypeList = new ArrayList<>();
stream(crawlingConfig.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
if (url.startsWith("smb://")) {
if (url.startsWith("smb:") || url.startsWith("file:") || url.startsWith("ftp:")) {
if (url.endsWith("/")) {
// directory
return true;
}
if (fessConfig.isSmbRoleFromFile()) {
final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
if (fessConfig.isSmbRoleFromFile() || fessConfig.isFileRoleFromFile() || fessConfig.isFtpRoleFromFile()) {
// head method
responseData = client.execute(RequestDataBuilder.newRequestData().head().url(url).build());
if (responseData == null) {
return true;
}
final SID[] sids = (SID[]) responseData.getMetaDataMap().get(SmbClient.SMB_ALLOWED_SID_ENTRIES);
if (sids != null) {
for (final SID sid : sids) {
final String accountId = sambaHelper.getAccountId(sid);
if (accountId != null) {
roleTypeList.add(accountId);
}
}
if (logger.isDebugEnabled()) {
logger.debug("smbUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
}
}
roleTypeList.addAll(permissionHelper.getSmbRoleTypeList(responseData));
roleTypeList.addAll(permissionHelper.getFileRoleTypeList(responseData));
roleTypeList.addAll(permissionHelper.getFtpRoleTypeList(responseData));
}
}
dataMap.put(fessConfig.getIndexFieldRole(), roleTypeList);

71
src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java
View file

@ -24,7 +24,6 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.tika.metadata.HttpHeaders;
@ -33,9 +32,6 @@ import org.codelibs.core.io.SerializeUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.Pair;
import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.client.fs.FileSystemClient;
import org.codelibs.fess.crawler.client.ftp.FtpClient;
import org.codelibs.fess.crawler.client.smb.SmbClient;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.ExtractData;
import org.codelibs.fess.crawler.entity.ResponseData;
@ -54,15 +50,13 @@ import org.codelibs.fess.helper.DocumentHelper;
import org.codelibs.fess.helper.FileTypeHelper;
import org.codelibs.fess.helper.LabelTypeHelper;
import org.codelibs.fess.helper.PathMappingHelper;
import org.codelibs.fess.helper.SambaHelper;
import org.codelibs.fess.helper.PermissionHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import jcifs.SID;
public abstract class AbstractFessFileTransformer extends AbstractTransformer implements FessTransformer {
private static final Logger logger = LoggerFactory.getLogger(AbstractFessFileTransformer.class);
@ -392,70 +386,15 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
protected List<String> getRoleTypes(final ResponseData responseData) {
final List<String> roleTypeList = new ArrayList<>();
final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
roleTypeList.addAll(getSmbRoleTypes(responseData));
roleTypeList.addAll(getFileRoleTypes(responseData));
roleTypeList.addAll(getFtpRoleTypes(responseData));
roleTypeList.addAll(permissionHelper.getSmbRoleTypeList(responseData));
roleTypeList.addAll(permissionHelper.getFileRoleTypeList(responseData));
roleTypeList.addAll(permissionHelper.getFtpRoleTypeList(responseData));
return roleTypeList;
}
protected List<String> getFileRoleTypes(final ResponseData responseData) {
final List<String> roleTypeList = new ArrayList<>();
if (fessConfig.isFileRoleFromFile() && responseData.getUrl().startsWith("file:")) {
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final String owner = (String) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_USER);
if (owner != null) {
roleTypeList.add(systemHelper.getSearchRoleByUser(owner));
}
final String[] groups = (String[]) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_GROUPS);
roleTypeList.addAll(stream(groups).get(stream -> stream.map(systemHelper::getSearchRoleByGroup).collect(Collectors.toList())));
if (getLogger().isDebugEnabled()) {
getLogger().debug("fileUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
}
}
return roleTypeList;
}
protected List<String> getFtpRoleTypes(final ResponseData responseData) {
final List<String> roleTypeList = new ArrayList<>();
if (fessConfig.isFtpRoleFromFile() && responseData.getUrl().startsWith("ftp:")) {
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final String owner = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_USER);
if (owner != null) {
roleTypeList.add(systemHelper.getSearchRoleByUser(owner));
}
final String group = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_GROUP);
if (group != null) {
roleTypeList.add(systemHelper.getSearchRoleByGroup(group));
}
if (getLogger().isDebugEnabled()) {
getLogger().debug("ftpUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
}
}
return roleTypeList;
}
protected List<String> getSmbRoleTypes(final ResponseData responseData) {
final List<String> roleTypeList = new ArrayList<>();
if (fessConfig.isSmbRoleFromFile() && responseData.getUrl().startsWith("smb:")) {
final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
final SID[] sids = (SID[]) responseData.getMetaDataMap().get(SmbClient.SMB_ALLOWED_SID_ENTRIES);
if (sids != null) {
for (final SID sid : sids) {
final String accountId = sambaHelper.getAccountId(sid);
if (accountId != null) {
roleTypeList.add(accountId);
}
}
if (getLogger().isDebugEnabled()) {
getLogger().debug("smbUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
}
}
}
return roleTypeList;
}
protected String getSiteOnFile(final String url, final String encoding) {
if (StringUtil.isBlank(url)) {
return StringUtil.EMPTY; // empty

74
src/main/java/org/codelibs/fess/helper/PermissionHelper.java
View file

@ -15,15 +15,30 @@
*/
package org.codelibs.fess.helper;
import static org.codelibs.core.stream.StreamUtil.stream;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.stream.Collectors;
import javax.annotation.Resource;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.client.fs.FileSystemClient;
import org.codelibs.fess.crawler.client.ftp.FtpClient;
import org.codelibs.fess.crawler.client.smb.SmbClient;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import jcifs.SID;
public class PermissionHelper {
private static final Logger logger = LoggerFactory.getLogger(PermissionHelper.class);
protected String rolePrefix = "{role}";
protected String groupPrefix = "{group}";
@ -90,4 +105,63 @@ public class PermissionHelper {
public void setUserPrefix(final String userPrefix) {
this.userPrefix = userPrefix;
}
public List<String> getSmbRoleTypeList(final ResponseData responseData) {
final List<String> roleTypeList = new ArrayList<>();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (fessConfig.isSmbRoleFromFile() && responseData.getUrl().startsWith("smb:")) {
final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
final SID[] sids = (SID[]) responseData.getMetaDataMap().get(SmbClient.SMB_ALLOWED_SID_ENTRIES);
if (sids != null) {
for (final SID sid : sids) {
final String accountId = sambaHelper.getAccountId(sid);
if (accountId != null) {
roleTypeList.add(accountId);
}
}
if (logger.isDebugEnabled()) {
logger.debug("smbUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
}
}
}
return roleTypeList;
}
public List<String> getFileRoleTypeList(final ResponseData responseData) {
final List<String> roleTypeList = new ArrayList<>();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (fessConfig.isFileRoleFromFile() && responseData.getUrl().startsWith("file:")) {
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final String owner = (String) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_USER);
if (owner != null) {
roleTypeList.add(systemHelper.getSearchRoleByUser(owner));
}
final String[] groups = (String[]) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_GROUPS);
roleTypeList.addAll(stream(groups).get(stream -> stream.map(systemHelper::getSearchRoleByGroup).collect(Collectors.toList())));
if (logger.isDebugEnabled()) {
logger.debug("fileUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
}
}
return roleTypeList;
}
public List<String> getFtpRoleTypeList(final ResponseData responseData) {
final List<String> roleTypeList = new ArrayList<>();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (fessConfig.isFtpRoleFromFile() && responseData.getUrl().startsWith("ftp:")) {
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final String owner = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_USER);
if (owner != null) {
roleTypeList.add(systemHelper.getSearchRoleByUser(owner));
}
final String group = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_GROUP);
if (group != null) {
roleTypeList.add(systemHelper.getSearchRoleByGroup(group));
}
if (logger.isDebugEnabled()) {
logger.debug("ftpUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
}
}
return roleTypeList;
}
}