fix #1933 improve last modified check

src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java

@@ -32,7 +32,6 @@ import org.codelibs.core.lang.StringUtil;
 import org.codelibs.fess.app.service.FailureUrlService;
 import org.codelibs.fess.crawler.builder.RequestDataBuilder;
 import org.codelibs.fess.crawler.client.CrawlerClient;
-import org.codelibs.fess.crawler.client.smb.SmbClient;
 import org.codelibs.fess.crawler.entity.RequestData;
 import org.codelibs.fess.crawler.entity.ResponseData;
 import org.codelibs.fess.crawler.entity.UrlQueue;
@@ -45,15 +44,13 @@ import org.codelibs.fess.helper.CrawlingConfigHelper;
 import org.codelibs.fess.helper.CrawlingInfoHelper;
 import org.codelibs.fess.helper.DuplicateHostHelper;
 import org.codelibs.fess.helper.IndexingHelper;
-import org.codelibs.fess.helper.SambaHelper;
+import org.codelibs.fess.helper.PermissionHelper;
 import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
 import org.codelibs.fess.util.DocumentUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import jcifs.SID;
-
 public class FessCrawlerThread extends CrawlerThread {
     private static final Logger logger = LoggerFactory.getLogger(FessCrawlerThread.class);
 
@@ -66,7 +63,6 @@ public class FessCrawlerThread extends CrawlerThread {
             final FessConfig fessConfig = ComponentUtil.getFessConfig();
             final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
             final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
-            final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
             final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
             final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
 
@@ -78,30 +74,22 @@ public class FessCrawlerThread extends CrawlerThread {
                 dataMap.put(fessConfig.getIndexFieldUrl(), url);
                 final List<String> roleTypeList = new ArrayList<>();
                 stream(crawlingConfig.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
-                if (url.startsWith("smb://")) {
+                if (url.startsWith("smb:") || url.startsWith("file:") || url.startsWith("ftp:")) {
                     if (url.endsWith("/")) {
                         // directory
                         return true;
                     }
-                    if (fessConfig.isSmbRoleFromFile()) {
+                    final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
+                    if (fessConfig.isSmbRoleFromFile() || fessConfig.isFileRoleFromFile() || fessConfig.isFtpRoleFromFile()) {
                         // head method
                         responseData = client.execute(RequestDataBuilder.newRequestData().head().url(url).build());
                         if (responseData == null) {
                             return true;
                         }
 
-                        final SID[] sids = (SID[]) responseData.getMetaDataMap().get(SmbClient.SMB_ALLOWED_SID_ENTRIES);
-                        if (sids != null) {
-                            for (final SID sid : sids) {
-                                final String accountId = sambaHelper.getAccountId(sid);
-                                if (accountId != null) {
-                                    roleTypeList.add(accountId);
-                                }
-                            }
-                            if (logger.isDebugEnabled()) {
-                                logger.debug("smbUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
-                            }
-                        }
+                        roleTypeList.addAll(permissionHelper.getSmbRoleTypeList(responseData));
+                        roleTypeList.addAll(permissionHelper.getFileRoleTypeList(responseData));
+                        roleTypeList.addAll(permissionHelper.getFtpRoleTypeList(responseData));
                     }
                 }
                 dataMap.put(fessConfig.getIndexFieldRole(), roleTypeList);

src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java

@@ -24,7 +24,6 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.tika.metadata.HttpHeaders;
@@ -33,9 +32,6 @@ import org.codelibs.core.io.SerializeUtil;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.core.misc.Pair;
 import org.codelibs.fess.Constants;
-import org.codelibs.fess.crawler.client.fs.FileSystemClient;
-import org.codelibs.fess.crawler.client.ftp.FtpClient;
-import org.codelibs.fess.crawler.client.smb.SmbClient;
 import org.codelibs.fess.crawler.entity.AccessResultData;
 import org.codelibs.fess.crawler.entity.ExtractData;
 import org.codelibs.fess.crawler.entity.ResponseData;
@@ -54,15 +50,13 @@ import org.codelibs.fess.helper.DocumentHelper;
 import org.codelibs.fess.helper.FileTypeHelper;
 import org.codelibs.fess.helper.LabelTypeHelper;
 import org.codelibs.fess.helper.PathMappingHelper;
-import org.codelibs.fess.helper.SambaHelper;
+import org.codelibs.fess.helper.PermissionHelper;
 import org.codelibs.fess.helper.SystemHelper;
 import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import jcifs.SID;
-
 public abstract class AbstractFessFileTransformer extends AbstractTransformer implements FessTransformer {
 
     private static final Logger logger = LoggerFactory.getLogger(AbstractFessFileTransformer.class);
@@ -392,67 +386,12 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
 
     protected List<String> getRoleTypes(final ResponseData responseData) {
         final List<String> roleTypeList = new ArrayList<>();
+        final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
 
-        roleTypeList.addAll(getSmbRoleTypes(responseData));
-        roleTypeList.addAll(getFileRoleTypes(responseData));
-        roleTypeList.addAll(getFtpRoleTypes(responseData));
-
-        return roleTypeList;
-    }
-
-    protected List<String> getFileRoleTypes(final ResponseData responseData) {
-        final List<String> roleTypeList = new ArrayList<>();
-        if (fessConfig.isFileRoleFromFile() && responseData.getUrl().startsWith("file:")) {
-            final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
-            final String owner = (String) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_USER);
-            if (owner != null) {
-                roleTypeList.add(systemHelper.getSearchRoleByUser(owner));
-            }
-            final String[] groups = (String[]) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_GROUPS);
-            roleTypeList.addAll(stream(groups).get(stream -> stream.map(systemHelper::getSearchRoleByGroup).collect(Collectors.toList())));
-            if (getLogger().isDebugEnabled()) {
-                getLogger().debug("fileUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
-            }
-        }
-        return roleTypeList;
-    }
-
-    protected List<String> getFtpRoleTypes(final ResponseData responseData) {
-        final List<String> roleTypeList = new ArrayList<>();
-        if (fessConfig.isFtpRoleFromFile() && responseData.getUrl().startsWith("ftp:")) {
-            final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
-            final String owner = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_USER);
-            if (owner != null) {
-                roleTypeList.add(systemHelper.getSearchRoleByUser(owner));
-            }
-            final String group = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_GROUP);
-            if (group != null) {
-                roleTypeList.add(systemHelper.getSearchRoleByGroup(group));
-            }
-            if (getLogger().isDebugEnabled()) {
-                getLogger().debug("ftpUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
-            }
-        }
-        return roleTypeList;
-    }
+        roleTypeList.addAll(permissionHelper.getSmbRoleTypeList(responseData));
+        roleTypeList.addAll(permissionHelper.getFileRoleTypeList(responseData));
+        roleTypeList.addAll(permissionHelper.getFtpRoleTypeList(responseData));
 
-    protected List<String> getSmbRoleTypes(final ResponseData responseData) {
-        final List<String> roleTypeList = new ArrayList<>();
-        if (fessConfig.isSmbRoleFromFile() && responseData.getUrl().startsWith("smb:")) {
-            final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
-            final SID[] sids = (SID[]) responseData.getMetaDataMap().get(SmbClient.SMB_ALLOWED_SID_ENTRIES);
-            if (sids != null) {
-                for (final SID sid : sids) {
-                    final String accountId = sambaHelper.getAccountId(sid);
-                    if (accountId != null) {
-                        roleTypeList.add(accountId);
-                    }
-                }
-                if (getLogger().isDebugEnabled()) {
-                    getLogger().debug("smbUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
-                }
-            }
-        }
         return roleTypeList;
     }
 

src/main/java/org/codelibs/fess/helper/PermissionHelper.java

@@ -15,15 +15,30 @@
  */
 package org.codelibs.fess.helper;
 
+import static org.codelibs.core.stream.StreamUtil.stream;
+
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Locale;
+import java.util.stream.Collectors;
 
 import javax.annotation.Resource;
 
 import org.codelibs.core.lang.StringUtil;
+import org.codelibs.fess.crawler.client.fs.FileSystemClient;
+import org.codelibs.fess.crawler.client.ftp.FtpClient;
+import org.codelibs.fess.crawler.client.smb.SmbClient;
+import org.codelibs.fess.crawler.entity.ResponseData;
 import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import jcifs.SID;
 
 public class PermissionHelper {
+    private static final Logger logger = LoggerFactory.getLogger(PermissionHelper.class);
+
     protected String rolePrefix = "{role}";
 
     protected String groupPrefix = "{group}";
@@ -90,4 +105,63 @@ public class PermissionHelper {
     public void setUserPrefix(final String userPrefix) {
         this.userPrefix = userPrefix;
     }
+
+    public List<String> getSmbRoleTypeList(final ResponseData responseData) {
+        final List<String> roleTypeList = new ArrayList<>();
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
+        if (fessConfig.isSmbRoleFromFile() && responseData.getUrl().startsWith("smb:")) {
+            final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
+            final SID[] sids = (SID[]) responseData.getMetaDataMap().get(SmbClient.SMB_ALLOWED_SID_ENTRIES);
+            if (sids != null) {
+                for (final SID sid : sids) {
+                    final String accountId = sambaHelper.getAccountId(sid);
+                    if (accountId != null) {
+                        roleTypeList.add(accountId);
+                    }
+                }
+                if (logger.isDebugEnabled()) {
+                    logger.debug("smbUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
+                }
+            }
+        }
+        return roleTypeList;
+    }
+
+    public List<String> getFileRoleTypeList(final ResponseData responseData) {
+        final List<String> roleTypeList = new ArrayList<>();
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
+        if (fessConfig.isFileRoleFromFile() && responseData.getUrl().startsWith("file:")) {
+            final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
+            final String owner = (String) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_USER);
+            if (owner != null) {
+                roleTypeList.add(systemHelper.getSearchRoleByUser(owner));
+            }
+            final String[] groups = (String[]) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_GROUPS);
+            roleTypeList.addAll(stream(groups).get(stream -> stream.map(systemHelper::getSearchRoleByGroup).collect(Collectors.toList())));
+            if (logger.isDebugEnabled()) {
+                logger.debug("fileUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
+            }
+        }
+        return roleTypeList;
+    }
+
+    public List<String> getFtpRoleTypeList(final ResponseData responseData) {
+        final List<String> roleTypeList = new ArrayList<>();
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
+        if (fessConfig.isFtpRoleFromFile() && responseData.getUrl().startsWith("ftp:")) {
+            final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
+            final String owner = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_USER);
+            if (owner != null) {
+                roleTypeList.add(systemHelper.getSearchRoleByUser(owner));
+            }
+            final String group = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_GROUP);
+            if (group != null) {
+                roleTypeList.add(systemHelper.getSearchRoleByGroup(group));
+            }
+            if (logger.isDebugEnabled()) {
+                logger.debug("ftpUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
+            }
+        }
+        return roleTypeList;
+    }
 }