#493 FTP support (#500)

src/main/java/org/codelibs/fess/Constants.java

@@ -230,6 +230,8 @@ public class Constants extends CoreLibConstants {
 
     public static final String SAMBA = "SAMBA";
 
+    public static final String FTP = "FTP";
+
     public static final String[] RESERVED = { "+", "-", "&&", "||", "!", "(", ")", "{", "}", "[", "]", "^", "~", "*", "?", "\\", ";", ":",
             "/" };
 

src/main/java/org/codelibs/fess/app/web/admin/fileauth/AdminFileauthAction.java

@@ -276,6 +276,7 @@ public class AdminFileauthAction extends FessAdminAction {
         final List<Map<String, String>> itemList = new ArrayList<Map<String, String>>();
         final Locale locale = LaRequestUtil.getRequest().getLocale();
         itemList.add(createItem(ComponentUtil.getMessageManager().getMessage(locale, "labels.file_auth_scheme_samba"), Constants.SAMBA));
+        itemList.add(createItem(ComponentUtil.getMessageManager().getMessage(locale, "labels.file_auth_scheme_ftp"), Constants.FTP));
         RenderDataUtil.register(data, "protocolSchemeItems", itemList);
     }
 

src/main/java/org/codelibs/fess/app/web/admin/wizard/AdminWizardAction.java

@@ -252,7 +252,7 @@ public class AdminWizardAction extends FessAdminAction {
     }
 
     protected String convertCrawlingPath(final String path) {
-        if (path.startsWith("http:") || path.startsWith("https:") || path.startsWith("smb:")) {
+        if (path.startsWith("http:") || path.startsWith("https:") || path.startsWith("smb:") || path.startsWith("ftp:")) {
             return path;
         }
 
@@ -293,4 +293,4 @@ public class AdminWizardAction extends FessAdminAction {
         }
         return redirect(AdminWizardAction.class);
     }
-}
+}

src/main/java/org/codelibs/fess/app/web/go/GoAction.java

@@ -153,6 +153,6 @@ public class GoAction extends FessSearchAction {
     }
 
     protected boolean isFileSystemPath(final String url) {
-        return url.startsWith("file:") || url.startsWith("smb:");
+        return url.startsWith("file:") || url.startsWith("smb:") || url.startsWith("ftp:");
     }
-}
+}

src/main/java/org/codelibs/fess/es/config/exentity/DataConfig.java

@@ -34,6 +34,8 @@ import org.apache.http.impl.auth.NTLMScheme;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.fess.Constants;
 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
+import org.codelibs.fess.crawler.client.ftp.FtpAuthentication;
+import org.codelibs.fess.crawler.client.ftp.FtpClient;
 import org.codelibs.fess.crawler.client.http.Authentication;
 import org.codelibs.fess.crawler.client.http.HcHttpClient;
 import org.codelibs.fess.crawler.client.http.impl.AuthenticationImpl;
@@ -289,7 +291,8 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
         final String fileAuthStr = paramMap.get(CRAWLER_FILE_AUTH);
         if (StringUtil.isNotBlank(fileAuthStr)) {
             final String[] fileAuthNames = fileAuthStr.split(",");
-            final List<SmbAuthentication> smbAuthList = new ArrayList<SmbAuthentication>();
+            final List<SmbAuthentication> smbAuthList = new ArrayList<>();
+            final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
             for (final String fileAuthName : fileAuthNames) {
                 final String scheme = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".scheme");
                 if (Constants.SAMBA.equals(scheme)) {
@@ -317,11 +320,37 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
                     smbAuth.setUsername(username);
                     smbAuth.setPassword(password == null ? StringUtil.EMPTY : password);
                     smbAuthList.add(smbAuth);
+                } else if (Constants.FTP.equals(scheme)) {
+                    final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
+                    final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
+                    final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
+                    final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
+
+                    if (StringUtil.isEmpty(username)) {
+                        logger.warn("username is empty. fileAuth:" + fileAuthName);
+                        continue;
+                    }
+
+                    final FtpAuthentication ftpAuth = new FtpAuthentication();
+                    ftpAuth.setServer(hostname);
+                    if (StringUtil.isNotBlank(port)) {
+                        try {
+                            ftpAuth.setPort(Integer.parseInt(port));
+                        } catch (final NumberFormatException e) {
+                            logger.warn("Failed to parse " + port, e);
+                        }
+                    }
+                    ftpAuth.setUsername(username);
+                    ftpAuth.setPassword(password == null ? StringUtil.EMPTY : password);
+                    ftpAuthList.add(ftpAuth);
                 }
             }
             if (!smbAuthList.isEmpty()) {
                 factoryParamMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
             }
+            if (!ftpAuthList.isEmpty()) {
+                factoryParamMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
+            }
         }
 
     }

src/main/java/org/codelibs/fess/es/config/exentity/FileConfig.java

@@ -27,6 +27,8 @@ import org.codelibs.core.lang.StringUtil;
 import org.codelibs.fess.Constants;
 import org.codelibs.fess.app.service.FileAuthenticationService;
 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
+import org.codelibs.fess.crawler.client.ftp.FtpAuthentication;
+import org.codelibs.fess.crawler.client.ftp.FtpClient;
 import org.codelibs.fess.crawler.client.smb.SmbAuthentication;
 import org.codelibs.fess.crawler.client.smb.SmbClient;
 import org.codelibs.fess.es.config.bsentity.BsFileConfig;
@@ -210,7 +212,8 @@ public class FileConfig extends BsFileConfig implements CrawlingConfig {
 
         // auth params
         final List<FileAuthentication> fileAuthList = fileAuthenticationService.getFileAuthenticationList(getId());
-        final List<SmbAuthentication> smbAuthList = new ArrayList<SmbAuthentication>();
+        final List<SmbAuthentication> smbAuthList = new ArrayList<>();
+        final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
         for (final FileAuthentication fileAuth : fileAuthList) {
             if (Constants.SAMBA.equals(fileAuth.getProtocolScheme())) {
                 final SmbAuthentication smbAuth = new SmbAuthentication();
@@ -222,9 +225,17 @@ public class FileConfig extends BsFileConfig implements CrawlingConfig {
                 smbAuth.setUsername(fileAuth.getUsername());
                 smbAuth.setPassword(fileAuth.getPassword());
                 smbAuthList.add(smbAuth);
+            } else if (Constants.FTP.equals(fileAuth.getProtocolScheme())) {
+                final FtpAuthentication ftpAuth = new FtpAuthentication();
+                ftpAuth.setServer(fileAuth.getHostname());
+                ftpAuth.setPort(fileAuth.getPort());
+                ftpAuth.setUsername(fileAuth.getUsername());
+                ftpAuth.setPassword(fileAuth.getPassword());
+                ftpAuthList.add(ftpAuth);
             }
         }
         paramMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
+        paramMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
 
     }
 

src/main/java/org/codelibs/fess/helper/ViewHelper.java

@@ -216,16 +216,18 @@ public class ViewHelper implements Serializable {
             return "#";
         }
 
-        final boolean isSmbUrl = url.startsWith("smb:");
+        final boolean isFileUrl = url.startsWith("smb:") || url.startsWith("ftp:");
 
         // replacing url with mapping data
         url = pathMappingHelper.replaceUrl(url);
 
         if (url.startsWith("smb:")) {
             url = url.replace("smb:", "file:");
+        } else if (url.startsWith("ftp:")) {
+            url = url.replace("ftp:", "file:");
         }
 
-        if (url.startsWith("http:") && isSmbUrl) {
+        if (url.startsWith("http:") && isFileUrl) {
             final StringBuilder buf = new StringBuilder(url.length() + 100);
             for (final char c : url.toCharArray()) {
                 if (CharUtil.isUrlChar(c)) {

src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java

@@ -1584,6 +1584,9 @@ public class FessLabels extends ActionMessages {
     /** The key of the message: Samba */
     public static final String LABELS_file_auth_scheme_samba = "{labels.file_auth_scheme_samba}";
 
+    /** The key of the message: FTP */
+    public static final String LABELS_file_auth_scheme_ftp = "{labels.file_auth_scheme_ftp}";
+
     /** The key of the message: {0}/{1} ({2} items) */
     public static final String LABELS_pagination_page_guide_msg = "{labels.pagination_page_guide_msg}";
 

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -143,7 +143,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. http,https */
     String CRAWLER_WEB_PROTOCOLS = "crawler.web.protocols";
 
-    /** The key of the configuration. e.g. file,smb */
+    /** The key of the configuration. e.g. file,smb,ftp */
     String CRAWLER_FILE_PROTOCOLS = "crawler.file.protocols";
 
     /** The key of the configuration. e.g. resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.* */
@@ -1224,7 +1224,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
 
     /**
      * Get the value for the key 'crawler.file.protocols'. <br>
-     * The value is, e.g. file,smb <br>
+     * The value is, e.g. file,smb,ftp <br>
      * @return The value of found property. (NotNull: if not found, exception but basically no way)
      */
     String getCrawlerFileProtocols();

src/main/java/org/codelibs/fess/util/ComponentUtil.java

@@ -126,6 +126,8 @@ public final class ComponentUtil {
 
     private static final String SAMBA_HELPER = "sambaHelper";
 
+    private static final String FTP_HELPER = "ftpHelper";
+
     private static final String VIEW_HELPER = "viewHelper";
 
     private static final String SYSTEM_HELPER = "systemHelper";

src/main/resources/fess_config.properties

@@ -86,7 +86,7 @@ crawler.document.max.alphanum.term.size=20
 crawler.document.max.symbol.term.size=10
 crawler.crawling.data.encoding=UTF-8
 crawler.web.protocols=http,https
-crawler.file.protocols=file,smb
+crawler.file.protocols=file,smb,ftp
 crawler.metadata.content.excludes=resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*
 crawler.metadata.name.mapping=\
 title=title:string\n\

src/main/resources/fess_label.properties

@@ -518,6 +518,7 @@ labels.file_auth_password=Password
 labels.file_auth_parameters=Parameters
 labels.file_auth_file_crawling_config=File System Config
 labels.file_auth_scheme_samba=Samba
+labels.file_auth_scheme_ftp=FTP
 labels.pagination_page_guide_msg={0}/{1} ({2} items)
 labels.list_could_not_find_crud_table=No data.
 labels.scheduledjob_configuration=Job Scheduler

src/main/resources/fess_label_en.properties

@@ -518,6 +518,7 @@ labels.file_auth_password=Password
 labels.file_auth_parameters=Parameters
 labels.file_auth_file_crawling_config=File System Config
 labels.file_auth_scheme_samba=Samba
+labels.file_auth_scheme_ftp=FTP
 labels.pagination_page_guide_msg={0}/{1} ({2} items)
 labels.list_could_not_find_crud_table=No data.
 labels.scheduledjob_configuration=Job Scheduler

src/main/resources/fess_label_ja.properties

@@ -514,6 +514,7 @@ labels.file_auth_password=\u30d1\u30b9\u30ef\u30fc\u30c9
 labels.file_auth_parameters=\u30d1\u30e9\u30e1\u30fc\u30bf
 labels.file_auth_file_crawling_config=\u30d5\u30a1\u30a4\u30eb\u30af\u30ed\u30fc\u30eb\u8a2d\u5b9a
 labels.file_auth_scheme_samba=Samba
+labels.file_auth_scheme_ftp=FTP
 labels.pagination_page_guide_msg={0}/{1} ({2} \u4ef6)
 labels.list_could_not_find_crud_table=\u767b\u9332\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002
 labels.scheduledjob_configuration=\u30b8\u30e7\u30d6\u30b9\u30b1\u30b8\u30e5\u30fc\u30e9