diff --git a/src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java b/src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java index 1c5e61832..f8fa56cf1 100644 --- a/src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java +++ b/src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java @@ -32,7 +32,6 @@ import org.codelibs.core.lang.StringUtil; import org.codelibs.fess.app.service.FailureUrlService; import org.codelibs.fess.crawler.builder.RequestDataBuilder; import org.codelibs.fess.crawler.client.CrawlerClient; -import org.codelibs.fess.crawler.client.smb.SmbClient; import org.codelibs.fess.crawler.entity.RequestData; import org.codelibs.fess.crawler.entity.ResponseData; import org.codelibs.fess.crawler.entity.UrlQueue; @@ -45,15 +44,13 @@ import org.codelibs.fess.helper.CrawlingConfigHelper; import org.codelibs.fess.helper.CrawlingInfoHelper; import org.codelibs.fess.helper.DuplicateHostHelper; import org.codelibs.fess.helper.IndexingHelper; -import org.codelibs.fess.helper.SambaHelper; +import org.codelibs.fess.helper.PermissionHelper; import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.util.ComponentUtil; import org.codelibs.fess.util.DocumentUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import jcifs.SID; - public class FessCrawlerThread extends CrawlerThread { private static final Logger logger = LoggerFactory.getLogger(FessCrawlerThread.class); @@ -66,7 +63,6 @@ public class FessCrawlerThread extends CrawlerThread { final FessConfig fessConfig = ComponentUtil.getFessConfig(); final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper(); final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper(); - final SambaHelper sambaHelper = ComponentUtil.getSambaHelper(); final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper(); final FessEsClient fessEsClient = ComponentUtil.getFessEsClient(); @@ -78,30 +74,22 @@ public class FessCrawlerThread extends CrawlerThread { dataMap.put(fessConfig.getIndexFieldUrl(), url); final List roleTypeList = new ArrayList<>(); stream(crawlingConfig.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p))); - if (url.startsWith("smb://")) { + if (url.startsWith("smb:") || url.startsWith("file:") || url.startsWith("ftp:")) { if (url.endsWith("/")) { // directory return true; } - if (fessConfig.isSmbRoleFromFile()) { + final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper(); + if (fessConfig.isSmbRoleFromFile() || fessConfig.isFileRoleFromFile() || fessConfig.isFtpRoleFromFile()) { // head method responseData = client.execute(RequestDataBuilder.newRequestData().head().url(url).build()); if (responseData == null) { return true; } - final SID[] sids = (SID[]) responseData.getMetaDataMap().get(SmbClient.SMB_ALLOWED_SID_ENTRIES); - if (sids != null) { - for (final SID sid : sids) { - final String accountId = sambaHelper.getAccountId(sid); - if (accountId != null) { - roleTypeList.add(accountId); - } - } - if (logger.isDebugEnabled()) { - logger.debug("smbUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString()); - } - } + roleTypeList.addAll(permissionHelper.getSmbRoleTypeList(responseData)); + roleTypeList.addAll(permissionHelper.getFileRoleTypeList(responseData)); + roleTypeList.addAll(permissionHelper.getFtpRoleTypeList(responseData)); } } dataMap.put(fessConfig.getIndexFieldRole(), roleTypeList); diff --git a/src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java b/src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java index 6c7cd5bce..4861ba30f 100644 --- a/src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java +++ b/src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java @@ -24,7 +24,6 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.tika.metadata.HttpHeaders; @@ -33,9 +32,6 @@ import org.codelibs.core.io.SerializeUtil; import org.codelibs.core.lang.StringUtil; import org.codelibs.core.misc.Pair; import org.codelibs.fess.Constants; -import org.codelibs.fess.crawler.client.fs.FileSystemClient; -import org.codelibs.fess.crawler.client.ftp.FtpClient; -import org.codelibs.fess.crawler.client.smb.SmbClient; import org.codelibs.fess.crawler.entity.AccessResultData; import org.codelibs.fess.crawler.entity.ExtractData; import org.codelibs.fess.crawler.entity.ResponseData; @@ -54,15 +50,13 @@ import org.codelibs.fess.helper.DocumentHelper; import org.codelibs.fess.helper.FileTypeHelper; import org.codelibs.fess.helper.LabelTypeHelper; import org.codelibs.fess.helper.PathMappingHelper; -import org.codelibs.fess.helper.SambaHelper; +import org.codelibs.fess.helper.PermissionHelper; import org.codelibs.fess.helper.SystemHelper; import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.util.ComponentUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import jcifs.SID; - public abstract class AbstractFessFileTransformer extends AbstractTransformer implements FessTransformer { private static final Logger logger = LoggerFactory.getLogger(AbstractFessFileTransformer.class); @@ -392,70 +386,15 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im protected List getRoleTypes(final ResponseData responseData) { final List roleTypeList = new ArrayList<>(); + final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper(); - roleTypeList.addAll(getSmbRoleTypes(responseData)); - roleTypeList.addAll(getFileRoleTypes(responseData)); - roleTypeList.addAll(getFtpRoleTypes(responseData)); + roleTypeList.addAll(permissionHelper.getSmbRoleTypeList(responseData)); + roleTypeList.addAll(permissionHelper.getFileRoleTypeList(responseData)); + roleTypeList.addAll(permissionHelper.getFtpRoleTypeList(responseData)); return roleTypeList; } - protected List getFileRoleTypes(final ResponseData responseData) { - final List roleTypeList = new ArrayList<>(); - if (fessConfig.isFileRoleFromFile() && responseData.getUrl().startsWith("file:")) { - final SystemHelper systemHelper = ComponentUtil.getSystemHelper(); - final String owner = (String) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_USER); - if (owner != null) { - roleTypeList.add(systemHelper.getSearchRoleByUser(owner)); - } - final String[] groups = (String[]) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_GROUPS); - roleTypeList.addAll(stream(groups).get(stream -> stream.map(systemHelper::getSearchRoleByGroup).collect(Collectors.toList()))); - if (getLogger().isDebugEnabled()) { - getLogger().debug("fileUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString()); - } - } - return roleTypeList; - } - - protected List getFtpRoleTypes(final ResponseData responseData) { - final List roleTypeList = new ArrayList<>(); - if (fessConfig.isFtpRoleFromFile() && responseData.getUrl().startsWith("ftp:")) { - final SystemHelper systemHelper = ComponentUtil.getSystemHelper(); - final String owner = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_USER); - if (owner != null) { - roleTypeList.add(systemHelper.getSearchRoleByUser(owner)); - } - final String group = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_GROUP); - if (group != null) { - roleTypeList.add(systemHelper.getSearchRoleByGroup(group)); - } - if (getLogger().isDebugEnabled()) { - getLogger().debug("ftpUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString()); - } - } - return roleTypeList; - } - - protected List getSmbRoleTypes(final ResponseData responseData) { - final List roleTypeList = new ArrayList<>(); - if (fessConfig.isSmbRoleFromFile() && responseData.getUrl().startsWith("smb:")) { - final SambaHelper sambaHelper = ComponentUtil.getSambaHelper(); - final SID[] sids = (SID[]) responseData.getMetaDataMap().get(SmbClient.SMB_ALLOWED_SID_ENTRIES); - if (sids != null) { - for (final SID sid : sids) { - final String accountId = sambaHelper.getAccountId(sid); - if (accountId != null) { - roleTypeList.add(accountId); - } - } - if (getLogger().isDebugEnabled()) { - getLogger().debug("smbUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString()); - } - } - } - return roleTypeList; - } - protected String getSiteOnFile(final String url, final String encoding) { if (StringUtil.isBlank(url)) { return StringUtil.EMPTY; // empty diff --git a/src/main/java/org/codelibs/fess/helper/PermissionHelper.java b/src/main/java/org/codelibs/fess/helper/PermissionHelper.java index 4c9296f92..43a2f6b02 100644 --- a/src/main/java/org/codelibs/fess/helper/PermissionHelper.java +++ b/src/main/java/org/codelibs/fess/helper/PermissionHelper.java @@ -15,15 +15,30 @@ */ package org.codelibs.fess.helper; +import static org.codelibs.core.stream.StreamUtil.stream; + +import java.util.ArrayList; +import java.util.List; import java.util.Locale; +import java.util.stream.Collectors; import javax.annotation.Resource; import org.codelibs.core.lang.StringUtil; +import org.codelibs.fess.crawler.client.fs.FileSystemClient; +import org.codelibs.fess.crawler.client.ftp.FtpClient; +import org.codelibs.fess.crawler.client.smb.SmbClient; +import org.codelibs.fess.crawler.entity.ResponseData; import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.util.ComponentUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jcifs.SID; public class PermissionHelper { + private static final Logger logger = LoggerFactory.getLogger(PermissionHelper.class); + protected String rolePrefix = "{role}"; protected String groupPrefix = "{group}"; @@ -90,4 +105,63 @@ public class PermissionHelper { public void setUserPrefix(final String userPrefix) { this.userPrefix = userPrefix; } + + public List getSmbRoleTypeList(final ResponseData responseData) { + final List roleTypeList = new ArrayList<>(); + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + if (fessConfig.isSmbRoleFromFile() && responseData.getUrl().startsWith("smb:")) { + final SambaHelper sambaHelper = ComponentUtil.getSambaHelper(); + final SID[] sids = (SID[]) responseData.getMetaDataMap().get(SmbClient.SMB_ALLOWED_SID_ENTRIES); + if (sids != null) { + for (final SID sid : sids) { + final String accountId = sambaHelper.getAccountId(sid); + if (accountId != null) { + roleTypeList.add(accountId); + } + } + if (logger.isDebugEnabled()) { + logger.debug("smbUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString()); + } + } + } + return roleTypeList; + } + + public List getFileRoleTypeList(final ResponseData responseData) { + final List roleTypeList = new ArrayList<>(); + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + if (fessConfig.isFileRoleFromFile() && responseData.getUrl().startsWith("file:")) { + final SystemHelper systemHelper = ComponentUtil.getSystemHelper(); + final String owner = (String) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_USER); + if (owner != null) { + roleTypeList.add(systemHelper.getSearchRoleByUser(owner)); + } + final String[] groups = (String[]) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_GROUPS); + roleTypeList.addAll(stream(groups).get(stream -> stream.map(systemHelper::getSearchRoleByGroup).collect(Collectors.toList()))); + if (logger.isDebugEnabled()) { + logger.debug("fileUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString()); + } + } + return roleTypeList; + } + + public List getFtpRoleTypeList(final ResponseData responseData) { + final List roleTypeList = new ArrayList<>(); + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + if (fessConfig.isFtpRoleFromFile() && responseData.getUrl().startsWith("ftp:")) { + final SystemHelper systemHelper = ComponentUtil.getSystemHelper(); + final String owner = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_USER); + if (owner != null) { + roleTypeList.add(systemHelper.getSearchRoleByUser(owner)); + } + final String group = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_GROUP); + if (group != null) { + roleTypeList.add(systemHelper.getSearchRoleByGroup(group)); + } + if (logger.isDebugEnabled()) { + logger.debug("ftpUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString()); + } + } + return roleTypeList; + } }