0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

#493 FTP support (#500)

Browse source
This commit is contained in:
Keiichi Watanabe 2016-05-09 22:40:27 +09:00 • committed by Shinsuke Sugaya
parent 2069b7d540
commit b87aa56dcc
14 changed files with 64 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/org/codelibs/fess/Constants.java
View file

@ -230,6 +230,8 @@ public class Constants extends CoreLibConstants {
public static final String SAMBA = "SAMBA";
public static final String FTP = "FTP";
public static final String[] RESERVED = { "+", "-", "&&", "||", "!", "(", ")", "{", "}", "[", "]", "^", "~", "*", "?", "\\", ";", ":",
"/" };

1
src/main/java/org/codelibs/fess/app/web/admin/fileauth/AdminFileauthAction.java
View file

@ -276,6 +276,7 @@ public class AdminFileauthAction extends FessAdminAction {
final List<Map<String, String>> itemList = new ArrayList<Map<String, String>>();
final Locale locale = LaRequestUtil.getRequest().getLocale();
itemList.add(createItem(ComponentUtil.getMessageManager().getMessage(locale, "labels.file_auth_scheme_samba"), Constants.SAMBA));
itemList.add(createItem(ComponentUtil.getMessageManager().getMessage(locale, "labels.file_auth_scheme_ftp"), Constants.FTP));
RenderDataUtil.register(data, "protocolSchemeItems", itemList);
}

4
src/main/java/org/codelibs/fess/app/web/admin/wizard/AdminWizardAction.java
View file

@ -252,7 +252,7 @@ public class AdminWizardAction extends FessAdminAction {
}
protected String convertCrawlingPath(final String path) {
if (path.startsWith("http:") || path.startsWith("https:") || path.startsWith("smb:")) {
if (path.startsWith("http:") || path.startsWith("https:") || path.startsWith("smb:") || path.startsWith("ftp:")) {
return path;
}
@ -293,4 +293,4 @@ public class AdminWizardAction extends FessAdminAction {
}
return redirect(AdminWizardAction.class);
}
}
}

4
src/main/java/org/codelibs/fess/app/web/go/GoAction.java
View file

@ -153,6 +153,6 @@ public class GoAction extends FessSearchAction {
}
protected boolean isFileSystemPath(final String url) {
return url.startsWith("file:") || url.startsWith("smb:");
return url.startsWith("file:") || url.startsWith("smb:") || url.startsWith("ftp:");
}
}
}

31
src/main/java/org/codelibs/fess/es/config/exentity/DataConfig.java
View file

@ -34,6 +34,8 @@ import org.apache.http.impl.auth.NTLMScheme;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.client.ftp.FtpAuthentication;
import org.codelibs.fess.crawler.client.ftp.FtpClient;
import org.codelibs.fess.crawler.client.http.Authentication;
import org.codelibs.fess.crawler.client.http.HcHttpClient;
import org.codelibs.fess.crawler.client.http.impl.AuthenticationImpl;
@ -289,7 +291,8 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
final String fileAuthStr = paramMap.get(CRAWLER_FILE_AUTH);
if (StringUtil.isNotBlank(fileAuthStr)) {
final String[] fileAuthNames = fileAuthStr.split(",");
final List<SmbAuthentication> smbAuthList = new ArrayList<SmbAuthentication>();
final List<SmbAuthentication> smbAuthList = new ArrayList<>();
final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
for (final String fileAuthName : fileAuthNames) {
final String scheme = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".scheme");
if (Constants.SAMBA.equals(scheme)) {
@ -317,11 +320,37 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
smbAuth.setUsername(username);
smbAuth.setPassword(password == null ? StringUtil.EMPTY : password);
smbAuthList.add(smbAuth);
} else if (Constants.FTP.equals(scheme)) {
final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
if (StringUtil.isEmpty(username)) {
logger.warn("username is empty. fileAuth:" + fileAuthName);
continue;
}
final FtpAuthentication ftpAuth = new FtpAuthentication();
ftpAuth.setServer(hostname);
if (StringUtil.isNotBlank(port)) {
try {
ftpAuth.setPort(Integer.parseInt(port));
} catch (final NumberFormatException e) {
logger.warn("Failed to parse " + port, e);
}
}
ftpAuth.setUsername(username);
ftpAuth.setPassword(password == null ? StringUtil.EMPTY : password);
ftpAuthList.add(ftpAuth);
}
}
if (!smbAuthList.isEmpty()) {
factoryParamMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
}
if (!ftpAuthList.isEmpty()) {
factoryParamMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
}
}
}

13
src/main/java/org/codelibs/fess/es/config/exentity/FileConfig.java
View file

@ -27,6 +27,8 @@ import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.FileAuthenticationService;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.client.ftp.FtpAuthentication;
import org.codelibs.fess.crawler.client.ftp.FtpClient;
import org.codelibs.fess.crawler.client.smb.SmbAuthentication;
import org.codelibs.fess.crawler.client.smb.SmbClient;
import org.codelibs.fess.es.config.bsentity.BsFileConfig;
@ -210,7 +212,8 @@ public class FileConfig extends BsFileConfig implements CrawlingConfig {
// auth params
final List<FileAuthentication> fileAuthList = fileAuthenticationService.getFileAuthenticationList(getId());
final List<SmbAuthentication> smbAuthList = new ArrayList<SmbAuthentication>();
final List<SmbAuthentication> smbAuthList = new ArrayList<>();
final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
for (final FileAuthentication fileAuth : fileAuthList) {
if (Constants.SAMBA.equals(fileAuth.getProtocolScheme())) {
final SmbAuthentication smbAuth = new SmbAuthentication();
@ -222,9 +225,17 @@ public class FileConfig extends BsFileConfig implements CrawlingConfig {
smbAuth.setUsername(fileAuth.getUsername());
smbAuth.setPassword(fileAuth.getPassword());
smbAuthList.add(smbAuth);
} else if (Constants.FTP.equals(fileAuth.getProtocolScheme())) {
final FtpAuthentication ftpAuth = new FtpAuthentication();
ftpAuth.setServer(fileAuth.getHostname());
ftpAuth.setPort(fileAuth.getPort());
ftpAuth.setUsername(fileAuth.getUsername());
ftpAuth.setPassword(fileAuth.getPassword());
ftpAuthList.add(ftpAuth);
}
}
paramMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
paramMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
}

6
src/main/java/org/codelibs/fess/helper/ViewHelper.java
View file

@ -216,16 +216,18 @@ public class ViewHelper implements Serializable {
return "#";
}
final boolean isSmbUrl = url.startsWith("smb:");
final boolean isFileUrl = url.startsWith("smb:") || url.startsWith("ftp:");
// replacing url with mapping data
url = pathMappingHelper.replaceUrl(url);
if (url.startsWith("smb:")) {
url = url.replace("smb:", "file:");
} else if (url.startsWith("ftp:")) {
url = url.replace("ftp:", "file:");
}
if (url.startsWith("http:") && isSmbUrl) {
if (url.startsWith("http:") && isFileUrl) {
final StringBuilder buf = new StringBuilder(url.length() + 100);
for (final char c : url.toCharArray()) {
if (CharUtil.isUrlChar(c)) {

3
src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java
View file

@ -1584,6 +1584,9 @@ public class FessLabels extends ActionMessages {
/** The key of the message: Samba */
public static final String LABELS_file_auth_scheme_samba = "{labels.file_auth_scheme_samba}";
/** The key of the message: FTP */
public static final String LABELS_file_auth_scheme_ftp = "{labels.file_auth_scheme_ftp}";
/** The key of the message: {0}/{1} ({2} items) */
public static final String LABELS_pagination_page_guide_msg = "{labels.pagination_page_guide_msg}";

4
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -143,7 +143,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. http,https */
String CRAWLER_WEB_PROTOCOLS = "crawler.web.protocols";
/** The key of the configuration. e.g. file,smb */
/** The key of the configuration. e.g. file,smb,ftp */
String CRAWLER_FILE_PROTOCOLS = "crawler.file.protocols";
/** The key of the configuration. e.g. resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.* */
@ -1224,7 +1224,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/**
* Get the value for the key 'crawler.file.protocols'. <br>
* The value is, e.g. file,smb <br>
* The value is, e.g. file,smb,ftp <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getCrawlerFileProtocols();

2
src/main/java/org/codelibs/fess/util/ComponentUtil.java
View file

@ -126,6 +126,8 @@ public final class ComponentUtil {
private static final String SAMBA_HELPER = "sambaHelper";
private static final String FTP_HELPER = "ftpHelper";
private static final String VIEW_HELPER = "viewHelper";
private static final String SYSTEM_HELPER = "systemHelper";

2
src/main/resources/fess_config.properties
View file

@ -86,7 +86,7 @@ crawler.document.max.alphanum.term.size=20
crawler.document.max.symbol.term.size=10
crawler.crawling.data.encoding=UTF-8
crawler.web.protocols=http,https
crawler.file.protocols=file,smb
crawler.file.protocols=file,smb,ftp
crawler.metadata.content.excludes=resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*
crawler.metadata.name.mapping=\
title=title:string\n\

1
src/main/resources/fess_label.properties
View file

@ -518,6 +518,7 @@ labels.file_auth_password=Password
labels.file_auth_parameters=Parameters
labels.file_auth_file_crawling_config=File System Config
labels.file_auth_scheme_samba=Samba
labels.file_auth_scheme_ftp=FTP
labels.pagination_page_guide_msg={0}/{1} ({2} items)
labels.list_could_not_find_crud_table=No data.
labels.scheduledjob_configuration=Job Scheduler

1
src/main/resources/fess_label_en.properties
View file

@ -518,6 +518,7 @@ labels.file_auth_password=Password
labels.file_auth_parameters=Parameters
labels.file_auth_file_crawling_config=File System Config
labels.file_auth_scheme_samba=Samba
labels.file_auth_scheme_ftp=FTP
labels.pagination_page_guide_msg={0}/{1} ({2} items)
labels.list_could_not_find_crud_table=No data.
labels.scheduledjob_configuration=Job Scheduler

1
src/main/resources/fess_label_ja.properties
View file

@ -514,6 +514,7 @@ labels.file_auth_password=\u30d1\u30b9\u30ef\u30fc\u30c9
labels.file_auth_parameters=\u30d1\u30e9\u30e1\u30fc\u30bf
labels.file_auth_file_crawling_config=\u30d5\u30a1\u30a4\u30eb\u30af\u30ed\u30fc\u30eb\u8a2d\u5b9a
labels.file_auth_scheme_samba=Samba
labels.file_auth_scheme_ftp=FTP
labels.pagination_page_guide_msg={0}/{1} ({2} \u4ef6)
labels.list_could_not_find_crud_table=\u767b\u9332\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002
labels.scheduledjob_configuration=\u30b8\u30e7\u30d6\u30b9\u30b1\u30b8\u30e5\u30fc\u30e9