fix #1267 add file system/ftp permissions

src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java

@@ -26,6 +26,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.tika.metadata.HttpHeaders;
@@ -34,6 +35,8 @@ import org.codelibs.core.io.SerializeUtil;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.core.misc.Pair;
 import org.codelibs.fess.Constants;
+import org.codelibs.fess.crawler.client.fs.FileSystemClient;
+import org.codelibs.fess.crawler.client.ftp.FtpClient;
 import org.codelibs.fess.crawler.client.smb.SmbClient;
 import org.codelibs.fess.crawler.entity.AccessResultData;
 import org.codelibs.fess.crawler.entity.ExtractData;
@@ -398,7 +401,52 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
     protected List<String> getRoleTypes(final ResponseData responseData) {
         final List<String> roleTypeList = new ArrayList<>();
 
-        if (fessConfig.isSmbRoleFromFile() && responseData.getUrl().startsWith("smb://")) {
+        roleTypeList.addAll(getSmbRoleTypes(responseData));
+        roleTypeList.addAll(getFileRoleTypes(responseData));
+        roleTypeList.addAll(getFtpRoleTypes(responseData));
+
+        return roleTypeList;
+    }
+
+    protected List<String> getFileRoleTypes(final ResponseData responseData) {
+        final List<String> roleTypeList = new ArrayList<>();
+        if (fessConfig.isFileRoleFromFile() && responseData.getUrl().startsWith("file:")) {
+            final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
+            final String owner = (String) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_USER);
+            if (owner != null) {
+                roleTypeList.add(systemHelper.getSearchRoleByUser(owner));
+            }
+            final String[] groups = (String[]) responseData.getMetaDataMap().get(FileSystemClient.FS_FILE_GROUPS);
+            roleTypeList.addAll(stream(groups).get(stream -> stream.map(systemHelper::getSearchRoleByGroup).collect(Collectors.toList())));
+            if (getLogger().isDebugEnabled()) {
+                getLogger().debug("fileUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
+            }
+        }
+        return roleTypeList;
+    }
+
+    protected List<String> getFtpRoleTypes(final ResponseData responseData) {
+        final List<String> roleTypeList = new ArrayList<>();
+        if (fessConfig.isFtpRoleFromFile() && responseData.getUrl().startsWith("ftp:")) {
+            final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
+            final String owner = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_USER);
+            if (owner != null) {
+                roleTypeList.add(systemHelper.getSearchRoleByUser(owner));
+            }
+            final String group = (String) responseData.getMetaDataMap().get(FtpClient.FTP_FILE_GROUP);
+            if (group != null) {
+                roleTypeList.add(systemHelper.getSearchRoleByGroup(group));
+            }
+            if (getLogger().isDebugEnabled()) {
+                getLogger().debug("ftpUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
+            }
+        }
+        return roleTypeList;
+    }
+
+    protected List<String> getSmbRoleTypes(final ResponseData responseData) {
+        final List<String> roleTypeList = new ArrayList<>();
+        if (fessConfig.isSmbRoleFromFile() && responseData.getUrl().startsWith("smb:")) {
             final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
             final ACE[] aces = (ACE[]) responseData.getMetaDataMap().get(SmbClient.SMB_ACCESS_CONTROL_ENTRIES);
             if (aces != null) {
@@ -414,7 +462,6 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
                 }
             }
         }
-
         return roleTypeList;
     }
 

src/main/java/org/codelibs/fess/es/config/exentity/FileConfig.java

@@ -228,7 +228,7 @@ public class FileConfig extends BsFileConfig implements CrawlingConfig {
             } else if (Constants.FTP.equals(fileAuth.getProtocolScheme())) {
                 final FtpAuthentication ftpAuth = new FtpAuthentication();
                 ftpAuth.setServer(fileAuth.getHostname());
-                ftpAuth.setPort(fileAuth.getPort());
+                ftpAuth.setPort(fileAuth.getPort() == null ? -1 : fileAuth.getPort());
                 ftpAuth.setUsername(fileAuth.getUsername());
                 ftpAuth.setPassword(fileAuth.getPassword());
                 ftpAuthList.add(ftpAuth);

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -623,6 +623,12 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. 1,2 */
     String SMB_AVAILABLE_SID_TYPES = "smb.available.sid.types";
 
+    /** The key of the configuration. e.g. true */
+    String FILE_ROLE_FROM_FILE = "file.role.from.file";
+
+    /** The key of the configuration. e.g. true */
+    String FTP_ROLE_FROM_FILE = "ftp.role.from.file";
+
     /** The key of the configuration. e.g. .fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties */
     String INDEX_BACKUP_TARGETS = "index.backup.targets";
 
@@ -3102,6 +3108,34 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     Integer getSmbAvailableSidTypesAsInteger();
 
+    /**
+     * Get the value for the key 'file.role.from.file'. <br>
+     * The value is, e.g. true <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getFileRoleFromFile();
+
+    /**
+     * Is the property for the key 'file.role.from.file' true? <br>
+     * The value is, e.g. true <br>
+     * @return The determination, true or false. (if not found, exception but basically no way)
+     */
+    boolean isFileRoleFromFile();
+
+    /**
+     * Get the value for the key 'ftp.role.from.file'. <br>
+     * The value is, e.g. true <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getFtpRoleFromFile();
+
+    /**
+     * Is the property for the key 'ftp.role.from.file' true? <br>
+     * The value is, e.g. true <br>
+     * @return The determination, true or false. (if not found, exception but basically no way)
+     */
+    boolean isFtpRoleFromFile();
+
     /**
      * Get the value for the key 'index.backup.targets'. <br>
      * The value is, e.g. .fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties <br>
@@ -6245,6 +6279,22 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return getAsInteger(FessConfig.SMB_AVAILABLE_SID_TYPES);
         }
 
+        public String getFileRoleFromFile() {
+            return get(FessConfig.FILE_ROLE_FROM_FILE);
+        }
+
+        public boolean isFileRoleFromFile() {
+            return is(FessConfig.FILE_ROLE_FROM_FILE);
+        }
+
+        public String getFtpRoleFromFile() {
+            return get(FessConfig.FTP_ROLE_FROM_FILE);
+        }
+
+        public boolean isFtpRoleFromFile() {
+            return is(FessConfig.FTP_ROLE_FROM_FILE);
+        }
+
         public String getIndexBackupTargets() {
             return get(FessConfig.INDEX_BACKUP_TARGETS);
         }
@@ -7622,6 +7672,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap.put(FessConfig.QUERY_BOOST_CONTENT_LANG, "0.5");
             defaultMap.put(FessConfig.SMB_ROLE_FROM_FILE, "true");
             defaultMap.put(FessConfig.SMB_AVAILABLE_SID_TYPES, "1,2");
+            defaultMap.put(FessConfig.FILE_ROLE_FROM_FILE, "true");
+            defaultMap.put(FessConfig.FTP_ROLE_FROM_FILE, "true");
             defaultMap.put(FessConfig.INDEX_BACKUP_TARGETS, ".fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties");
             defaultMap.put(FessConfig.INDEX_BACKUP_LOG_TARGETS, "click_log.ndjson,favorite_log.ndjson,search_log.ndjson,user_info.ndjson");
             defaultMap.put(FessConfig.FORM_ADMIN_MAX_INPUT_SIZE, "4000");

src/main/resources/fess_config.properties

@@ -314,6 +314,8 @@ query.boost.content.lang=0.5
 # acl
 smb.role.from.file=true
 smb.available.sid.types=1,2
+file.role.from.file=true
+ftp.role.from.file=true
 
 # backup
 index.backup.targets=.fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties