0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1677 remove datastore impl

Browse source
This commit is contained in:
Shinsuke Sugaya 2018-05-31 18:46:09 +09:00
parent e9650b004c
commit 845e8392a2
17 changed files with 29 additions and 1933 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
src/main/java/org/codelibs/fess/ds/impl/AbstractDataStoreImpl.java → src/main/java/org/codelibs/fess/ds/AbstractDataStore.java
View file

@ -13,7 +13,7 @@
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.ds.impl;
package org.codelibs.fess.ds;
import static org.codelibs.core.stream.StreamUtil.stream;
@ -25,8 +25,7 @@ import java.util.Map;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.ds.DataStore;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.ds.callback.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.helper.CrawlingInfoHelper;
import org.codelibs.fess.helper.SystemHelper;
@ -36,14 +35,20 @@ import org.codelibs.fess.util.GroovyUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public abstract class AbstractDataStoreImpl implements DataStore {
public abstract class AbstractDataStore implements DataStore {
private static final Logger logger = LoggerFactory.getLogger(AbstractDataStoreImpl.class);
private static final Logger logger = LoggerFactory.getLogger(AbstractDataStore.class);
public String mimeType = "application/datastore";
protected boolean alive = true;
public void register() {
ComponentUtil.getDataStoreFactory().add(getName(), this);
}
protected abstract String getName();
@Override
public void stop() {
alive = false;

1
src/main/java/org/codelibs/fess/ds/DataStore.java
View file

@ -17,6 +17,7 @@ package org.codelibs.fess.ds;
import java.util.Map;
import org.codelibs.fess.ds.callback.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.DataConfig;
public interface DataStore {

10
src/main/java/org/codelibs/fess/ds/DataStoreFactory.java
View file

@ -16,18 +16,27 @@
package org.codelibs.fess.ds;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DataStoreFactory {
private static final Logger logger = LoggerFactory.getLogger(DataStoreFactory.class);
protected Map<String, DataStore> dataStoreMap = new LinkedHashMap<>();
public void add(final String name, final DataStore dataStore) {
if (name == null || dataStore == null) {
throw new IllegalArgumentException("name or dataStore is null.");
}
if (logger.isInfoEnabled()) {
logger.info("Loaded " + name);
}
dataStoreMap.put(name, dataStore);
}
@ -39,6 +48,7 @@ public class DataStoreFactory {
final Set<String> nameSet = dataStoreMap.keySet();
final List<String> nameList = new ArrayList<>();
nameList.addAll(nameSet);
Collections.sort(nameList);
return nameList;
}

7
src/main/java/org/codelibs/fess/ds/impl/FileListIndexUpdateCallbackImpl.java → src/main/java/org/codelibs/fess/ds/callback/FileListIndexUpdateCallbackImpl.java
View file

@ -13,7 +13,7 @@
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.ds.impl;
package org.codelibs.fess.ds.callback;
import static org.codelibs.core.stream.StreamUtil.stream;
@ -40,7 +40,6 @@ import org.codelibs.fess.crawler.processor.impl.DefaultResponseProcessor;
import org.codelibs.fess.crawler.rule.Rule;
import org.codelibs.fess.crawler.rule.RuleManager;
import org.codelibs.fess.crawler.transformer.Transformer;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.exception.DataStoreCrawlingException;
import org.codelibs.fess.helper.IndexingHelper;
@ -67,8 +66,8 @@ public class FileListIndexUpdateCallbackImpl implements IndexUpdateCallback {
private int executorTerminationTimeout = 300;
protected FileListIndexUpdateCallbackImpl(final IndexUpdateCallback indexUpdateCallback,
final CrawlerClientFactory crawlerClientFactory, final int nThreads) {
public FileListIndexUpdateCallbackImpl(final IndexUpdateCallback indexUpdateCallback, final CrawlerClientFactory crawlerClientFactory,
final int nThreads) {
this.indexUpdateCallback = indexUpdateCallback;
this.crawlerClientFactory = crawlerClientFactory;
executor = newFixedThreadPool(nThreads < 1 ? 1 : nThreads);

2
src/main/java/org/codelibs/fess/ds/IndexUpdateCallback.java → src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallback.java
View file

@ -13,7 +13,7 @@
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.ds;
package org.codelibs.fess.ds.callback;
import java.util.Map;

5
src/main/java/org/codelibs/fess/ds/impl/IndexUpdateCallbackImpl.java → src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java
View file

@ -13,14 +13,13 @@
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.ds.impl;
package org.codelibs.fess.ds.callback;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.PostConstruct;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.exception.DataStoreException;
import org.codelibs.fess.helper.CrawlingInfoHelper;
@ -51,7 +50,7 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
}
/* (non-Javadoc)
* @see org.codelibs.fess.ds.impl.IndexUpdateCallback#store(java.util.Map)
* @see org.codelibs.fess.ds.callback.IndexUpdateCallback#store(java.util.Map)
*/
@Override
public void store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {

429
src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java
View file

@ -1,429 +0,0 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.ds.impl;
import static org.codelibs.core.stream.StreamUtil.stream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Pattern;
import org.apache.commons.text.StringEscapeUtils;
import org.codelibs.core.io.CloseableUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.FailureUrlService;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.exception.MultipleCrawlingAccessException;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.exception.DataStoreCrawlingException;
import org.codelibs.fess.exception.DataStoreException;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.orangesignal.csv.CsvConfig;
import com.orangesignal.csv.CsvReader;
public class CsvDataStoreImpl extends AbstractDataStoreImpl {
private static final Logger logger = LoggerFactory.getLogger(CsvDataStoreImpl.class);
protected static final String ESCAPE_CHARACTER_PARAM = "escapeCharacter";
protected static final String QUOTE_CHARACTER_PARAM = "quoteCharacter";
protected static final String SEPARATOR_CHARACTER_PARAM = "separatorCharacter";
protected static final String SKIP_LINES_PARAM = "skipLines";
protected static final String IGNORE_LINE_PATTERNS_PARAM = "ignoreLinePatterns";
protected static final String IGNORE_EMPTY_LINES_PARAM = "ignoreEmptyLines";
protected static final String IGNORE_TRAILING_WHITESPACES_PARAM = "ignoreTrailingWhitespaces";
protected static final String IGNORE_LEADING_WHITESPACES_PARAM = "ignoreLeadingWhitespaces";
protected static final String NULL_STRING_PARAM = "nullString";
protected static final String BREAK_STRING_PARAM = "breakString";
protected static final String ESCAPE_DISABLED_PARAM = "escapeDisabled";
protected static final String QUOTE_DISABLED_PARAM = "quoteDisabled";
protected static final String CSV_FILE_ENCODING_PARAM = "fileEncoding";
protected static final String CSV_FILES_PARAM = "files";
protected static final String CSV_DIRS_PARAM = "directories";
protected static final String CELL_PREFIX = "cell";
public String[] csvFileSuffixs = new String[] { ".csv", ".tsv" };
protected List<File> getCsvFileList(final Map<String, String> paramMap) {
String value = paramMap.get(CSV_FILES_PARAM);
final List<File> fileList = new ArrayList<>();
if (StringUtil.isBlank(value)) {
value = paramMap.get(CSV_DIRS_PARAM);
if (StringUtil.isBlank(value)) {
throw new DataStoreException(CSV_FILES_PARAM + " and " + CSV_DIRS_PARAM + " are blank.");
}
logger.info(CSV_DIRS_PARAM + "=" + value);
final String[] values = value.split(",");
for (final String path : values) {
final File dir = new File(path);
if (dir.isDirectory()) {
stream(dir.listFiles()).of(
stream -> stream.filter(f -> isCsvFile(f.getParentFile(), f.getName()))
.sorted((f1, f2) -> (int) (f1.lastModified() - f2.lastModified())).forEach(f -> fileList.add(f)));
} else {
logger.warn(path + " is not a directory.");
}
}
} else {
logger.info(CSV_FILES_PARAM + "=" + value);
final String[] values = value.split(",");
for (final String path : values) {
final File file = new File(path);
if (file.isFile() && isCsvFile(file.getParentFile(), file.getName())) {
fileList.add(file);
} else {
logger.warn(path + " is not found.");
}
}
}
if (fileList.isEmpty() && logger.isDebugEnabled()) {
logger.debug("No csv files in " + value);
}
return fileList;
}
protected boolean isCsvFile(final File parentFile, final String filename) {
final String name = filename.toLowerCase(Locale.ROOT);
for (final String suffix : csvFileSuffixs) {
if (name.endsWith(suffix)) {
return true;
}
}
return false;
}
protected String getCsvFileEncoding(final Map<String, String> paramMap) {
final String value = paramMap.get(CSV_FILE_ENCODING_PARAM);
if (StringUtil.isBlank(value)) {
return Constants.UTF_8;
}
return value;
}
protected boolean hasHeaderLine(final Map<String, String> paramMap) {
final String value = paramMap.get("hasHeaderLine");
if (StringUtil.isBlank(value)) {
return false;
}
try {
return Boolean.parseBoolean(value);
} catch (final Exception e) {
return false;
}
}
@Override
protected void storeData(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
final long readInterval = getReadInterval(paramMap);
final List<File> csvFileList = getCsvFileList(paramMap);
if (csvFileList.isEmpty()) {
logger.warn("No CSV file.");
return;
}
final String csvFileEncoding = getCsvFileEncoding(paramMap);
final boolean hasHeaderLine = hasHeaderLine(paramMap);
final CsvConfig csvConfig = buildCsvConfig(paramMap);
for (final File csvFile : csvFileList) {
processCsv(dataConfig, callback, paramMap, scriptMap, defaultDataMap, csvConfig, csvFile, readInterval, csvFileEncoding,
hasHeaderLine);
}
}
protected void processCsv(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap, final CsvConfig csvConfig, final File csvFile,
final long readInterval, final String csvFileEncoding, final boolean hasHeaderLine) {
logger.info("Loading " + csvFile.getAbsolutePath());
CsvReader csvReader = null;
try {
csvReader = new CsvReader(new BufferedReader(new InputStreamReader(new FileInputStream(csvFile), csvFileEncoding)), csvConfig);
List<String> headerList = null;
if (hasHeaderLine) {
headerList = csvReader.readValues();
}
List<String> list;
boolean loop = true;
while ((list = csvReader.readValues()) != null && loop && alive) {
final Map<String, Object> dataMap = new HashMap<>();
dataMap.putAll(defaultDataMap);
final Map<String, Object> resultMap = new LinkedHashMap<>();
resultMap.putAll(paramMap);
resultMap.put("csvfile", csvFile.getAbsolutePath());
resultMap.put("csvfilename", csvFile.getName());
resultMap.put("crawlingConfig", dataConfig);
boolean foundValues = false;
for (int i = 0; i < list.size(); i++) {
String key = null;
String value = list.get(i);
if (value == null) {
value = StringUtil.EMPTY;
}
if (StringUtil.isNotBlank(value)) {
foundValues = true;
}
if (headerList != null && headerList.size() > i) {
key = headerList.get(i);
if (StringUtil.isNotBlank(key)) {
resultMap.put(key, value);
}
}
key = CELL_PREFIX + Integer.toString(i + 1);
resultMap.put(key, value);
}
if (!foundValues) {
logger.debug("No data in line: {}", resultMap);
continue;
}
if (logger.isDebugEnabled()) {
for (final Map.Entry<String, Object> entry : resultMap.entrySet()) {
logger.debug(entry.getKey() + "=" + entry.getValue());
}
}
final Map<String, Object> crawlingContext = new HashMap<>();
crawlingContext.put("doc", dataMap);
resultMap.put("crawlingContext", crawlingContext);
for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
final Object convertValue = convertValue(entry.getValue(), resultMap);
if (convertValue != null) {
dataMap.put(entry.getKey(), convertValue);
}
}
if (logger.isDebugEnabled()) {
for (final Map.Entry<String, Object> entry : dataMap.entrySet()) {
logger.debug(entry.getKey() + "=" + entry.getValue());
}
}
try {
callback.store(paramMap, dataMap);
} catch (final CrawlingAccessException e) {
logger.warn("Crawling Access Exception at : " + dataMap, e);
Throwable target = e;
if (target instanceof MultipleCrawlingAccessException) {
final Throwable[] causes = ((MultipleCrawlingAccessException) target).getCauses();
if (causes.length > 0) {
target = causes[causes.length - 1];
}
}
String errorName;
final Throwable cause = target.getCause();
if (cause != null) {
errorName = cause.getClass().getCanonicalName();
} else {
errorName = target.getClass().getCanonicalName();
}
String url;
if (target instanceof DataStoreCrawlingException) {
final DataStoreCrawlingException dce = (DataStoreCrawlingException) target;
url = dce.getUrl();
if (dce.aborted()) {
loop = false;
}
} else {
url = csvFile.getAbsolutePath() + ":" + csvReader.getLineNumber();
}
final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
failureUrlService.store(dataConfig, errorName, url, target);
} catch (final Throwable t) {
logger.warn("Crawling Access Exception at : " + dataMap, t);
final String url = csvFile.getAbsolutePath() + ":" + csvReader.getLineNumber();
final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
failureUrlService.store(dataConfig, t.getClass().getCanonicalName(), url, t);
}
if (readInterval > 0) {
sleep(readInterval);
}
}
} catch (final Exception e) {
throw new DataStoreException("Failed to crawl data when reading csv file.", e);
} finally {
CloseableUtil.closeQuietly(csvReader);
}
}
protected CsvConfig buildCsvConfig(final Map<String, String> paramMap) {
final CsvConfig csvConfig = new CsvConfig();
if (paramMap.containsKey(SEPARATOR_CHARACTER_PARAM)) {
final String value = paramMap.get(SEPARATOR_CHARACTER_PARAM);
if (StringUtil.isNotBlank(value)) {
try {
csvConfig.setSeparator(StringEscapeUtils.unescapeJava(value).charAt(0));
} catch (final Exception e) {
logger.warn("Failed to load " + SEPARATOR_CHARACTER_PARAM, e);
}
}
}
if (paramMap.containsKey(QUOTE_CHARACTER_PARAM)) {
final String value = paramMap.get(QUOTE_CHARACTER_PARAM);
if (StringUtil.isNotBlank(value)) {
try {
csvConfig.setQuote(value.charAt(0));
} catch (final Exception e) {
logger.warn("Failed to load " + QUOTE_CHARACTER_PARAM, e);
}
}
}
if (paramMap.containsKey(ESCAPE_CHARACTER_PARAM)) {
final String value = paramMap.get(ESCAPE_CHARACTER_PARAM);
if (StringUtil.isNotBlank(value)) {
try {
csvConfig.setEscape(value.charAt(0));
} catch (final Exception e) {
logger.warn("Failed to load " + ESCAPE_CHARACTER_PARAM, e);
}
}
}
if (paramMap.containsKey(QUOTE_DISABLED_PARAM)) {
final String value = paramMap.get(QUOTE_DISABLED_PARAM);
if (StringUtil.isNotBlank(value)) {
try {
// デフォルトでは無効となっている囲み文字を有効にします。
csvConfig.setQuoteDisabled(Boolean.parseBoolean(value));
} catch (final Exception e) {
logger.warn("Failed to load " + QUOTE_DISABLED_PARAM, e);
}
}
}
if (paramMap.containsKey(ESCAPE_DISABLED_PARAM)) {
final String value = paramMap.get(ESCAPE_DISABLED_PARAM);
if (StringUtil.isNotBlank(value)) {
try {
// デフォルトでは無効となっているエスケープ文字を有効にします。
csvConfig.setEscapeDisabled(Boolean.parseBoolean(value));
} catch (final Exception e) {
logger.warn("Failed to load " + ESCAPE_DISABLED_PARAM, e);
}
}
}
if (paramMap.containsKey(BREAK_STRING_PARAM)) {
final String value = paramMap.get(BREAK_STRING_PARAM);
if (StringUtil.isNotBlank(value)) {
// 項目値中の改行を \n で置換えます。
csvConfig.setBreakString(value);
}
}
if (paramMap.containsKey(NULL_STRING_PARAM)) {
final String value = paramMap.get(NULL_STRING_PARAM);
if (StringUtil.isNotBlank(value)) {
// null 値扱いする文字列を指定します。
csvConfig.setNullString(value);
}
}
if (paramMap.containsKey(IGNORE_LEADING_WHITESPACES_PARAM)) {
final String value = paramMap.get(IGNORE_LEADING_WHITESPACES_PARAM);
if (StringUtil.isNotBlank(value)) {
try {
// 項目値前のホワイトスペースを除去します。
csvConfig.setIgnoreLeadingWhitespaces(Boolean.parseBoolean(value));
} catch (final Exception e) {
logger.warn("Failed to load " + IGNORE_LEADING_WHITESPACES_PARAM, e);
}
}
}
if (paramMap.containsKey(IGNORE_TRAILING_WHITESPACES_PARAM)) {
final String value = paramMap.get(IGNORE_TRAILING_WHITESPACES_PARAM);
if (StringUtil.isNotBlank(value)) {
try {
// 項目値後のホワイトスペースを除去します。
csvConfig.setIgnoreTrailingWhitespaces(Boolean.parseBoolean(value));
} catch (final Exception e) {
logger.warn("Failed to load " + IGNORE_TRAILING_WHITESPACES_PARAM, e);
}
}
}
if (paramMap.containsKey(IGNORE_EMPTY_LINES_PARAM)) {
final String value = paramMap.get(IGNORE_EMPTY_LINES_PARAM);
if (StringUtil.isNotBlank(value)) {
try {
// 空行を無視するようにします。
csvConfig.setIgnoreEmptyLines(Boolean.parseBoolean(value));
} catch (final Exception e) {
logger.warn("Failed to load " + IGNORE_EMPTY_LINES_PARAM, e);
}
}
}
if (paramMap.containsKey(IGNORE_LINE_PATTERNS_PARAM)) {
final String value = paramMap.get(IGNORE_LINE_PATTERNS_PARAM);
if (StringUtil.isNotBlank(value)) {
// 正規表現による無視する行パターンを設定します。(この例では # で始まる行)
csvConfig.setIgnoreLinePatterns(Pattern.compile(value));
}
}
if (paramMap.containsKey(SKIP_LINES_PARAM)) {
final String value = paramMap.get(SKIP_LINES_PARAM);
if (StringUtil.isNotBlank(value)) {
try {
// 最初の1行目をスキップして読込みます。
csvConfig.setSkipLines(Integer.parseInt(value));
} catch (final Exception e) {
logger.warn("Failed to load " + SKIP_LINES_PARAM, e);
}
}
}
return csvConfig;
}
}

100
src/main/java/org/codelibs/fess/ds/impl/CsvListDataStoreImpl.java
View file

@ -1,100 +0,0 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.ds.impl;
import java.io.File;
import java.util.Map;
import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.exception.DataStoreException;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.orangesignal.csv.CsvConfig;
public class CsvListDataStoreImpl extends CsvDataStoreImpl {
private static final Logger logger = LoggerFactory.getLogger(CsvListDataStoreImpl.class);
public boolean deleteProcessedFile = true;
public long csvFileTimestampMargin = 60 * 1000L;// 1min
public boolean ignoreDataStoreException = true;
@Override
protected boolean isCsvFile(final File parentFile, final String filename) {
if (super.isCsvFile(parentFile, filename)) {
final File file = new File(parentFile, filename);
final long now = System.currentTimeMillis();
return now - file.lastModified() > csvFileTimestampMargin;
}
return false;
}
@Override
protected void storeData(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
int nThreads = 1;
if (paramMap.containsKey(Constants.NUM_OF_THREADS)) {
try {
nThreads = Integer.parseInt(paramMap.get(Constants.NUM_OF_THREADS));
} catch (final NumberFormatException e) {
logger.warn(Constants.NUM_OF_THREADS + " is not int value.", e);
}
}
final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getCrawlerClientFactory();
dataConfig.initializeClientFactory(crawlerClientFactory);
try {
final FileListIndexUpdateCallbackImpl fileListIndexUpdateCallback =
new FileListIndexUpdateCallbackImpl(callback, crawlerClientFactory, nThreads);
super.storeData(dataConfig, fileListIndexUpdateCallback, paramMap, scriptMap, defaultDataMap);
fileListIndexUpdateCallback.commit();
} catch (final Exception e) {
throw new DataStoreException(e);
}
}
@Override
protected void processCsv(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap, final CsvConfig csvConfig, final File csvFile,
final long readInterval, final String csvFileEncoding, final boolean hasHeaderLine) {
try {
super.processCsv(dataConfig, callback, paramMap, scriptMap, defaultDataMap, csvConfig, csvFile, readInterval, csvFileEncoding,
hasHeaderLine);
// delete csv file
if (deleteProcessedFile && !csvFile.delete()) {
logger.warn("Failed to delete {}", csvFile.getAbsolutePath());
}
} catch (final DataStoreException e) {
if (ignoreDataStoreException) {
logger.error("Failed to process " + csvFile.getAbsolutePath(), e);
// rename csv file, or delete it if failed
if (!csvFile.renameTo(new File(csvFile.getParent(), csvFile.getName() + ".txt")) && !csvFile.delete()) {
logger.warn("Failed to delete {}", csvFile.getAbsolutePath());
}
} else {
throw e;
}
}
}
}

284
src/main/java/org/codelibs/fess/ds/impl/DatabaseDataStoreImpl.java
View file

@ -1,284 +0,0 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.ds.impl;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.app.service.FailureUrlService;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.exception.MultipleCrawlingAccessException;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.exception.DataStoreCrawlingException;
import org.codelibs.fess.exception.DataStoreException;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
private static final Logger logger = LoggerFactory.getLogger(DatabaseDataStoreImpl.class);
private static final String SQL_PARAM = "sql";
private static final String URL_PARAM = "url";
private static final String PASSWORD_PARAM = "password";
private static final String USERNAME_PARAM = "username";
private static final String DRIVER_PARAM = "driver";
protected String getDriverClass(final Map<String, String> paramMap) {
final String driverName = paramMap.get(DRIVER_PARAM);
if (StringUtil.isBlank(driverName)) {
throw new DataStoreException("JDBC driver is null");
}
return driverName;
}
protected String getUsername(final Map<String, String> paramMap) {
return paramMap.get(USERNAME_PARAM);
}
protected String getPassword(final Map<String, String> paramMap) {
return paramMap.get(PASSWORD_PARAM);
}
protected String getUrl(final Map<String, String> paramMap) {
return paramMap.get(URL_PARAM);
}
protected String getSql(final Map<String, String> paramMap) {
final String sql = paramMap.get(SQL_PARAM);
if (StringUtil.isBlank(sql)) {
throw new DataStoreException("sql is null");
}
return sql;
}
@Override
protected void storeData(final DataConfig config, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
final long readInterval = getReadInterval(paramMap);
Connection con = null;
Statement stmt = null;
ResultSet rs = null;
try {
Class.forName(getDriverClass(paramMap));
final String jdbcUrl = getUrl(paramMap);
final String username = getUsername(paramMap);
final String password = getPassword(paramMap);
if (StringUtil.isNotEmpty(username)) {
con = DriverManager.getConnection(jdbcUrl, username, password);
} else {
con = DriverManager.getConnection(jdbcUrl);
}
final String sql = getSql(paramMap);
stmt = con.createStatement();
rs = stmt.executeQuery(sql); // SQL generated by an administrator
boolean loop = true;
while (rs.next() && loop && alive) {
final Map<String, Object> dataMap = new HashMap<>();
dataMap.putAll(defaultDataMap);
final Map<String, Object> crawlingContext = new HashMap<>();
crawlingContext.put("doc", dataMap);
for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
final Object convertValue =
convertValue(entry.getValue(), new ResultSetParamMap(config, crawlingContext, rs, paramMap));
if (convertValue != null) {
dataMap.put(entry.getKey(), convertValue);
}
}
try {
callback.store(paramMap, dataMap);
} catch (final CrawlingAccessException e) {
logger.warn("Crawling Access Exception at : " + dataMap, e);
Throwable target = e;
if (target instanceof MultipleCrawlingAccessException) {
final Throwable[] causes = ((MultipleCrawlingAccessException) target).getCauses();
if (causes.length > 0) {
target = causes[causes.length - 1];
}
}
String errorName;
final Throwable cause = target.getCause();
if (cause != null) {
errorName = cause.getClass().getCanonicalName();
} else {
errorName = target.getClass().getCanonicalName();
}
String url;
if (target instanceof DataStoreCrawlingException) {
final DataStoreCrawlingException dce = (DataStoreCrawlingException) target;
url = dce.getUrl();
if (dce.aborted()) {
loop = false;
}
} else {
url = sql + ":" + rs.getRow();
}
final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
failureUrlService.store(config, errorName, url, target);
} catch (final Throwable t) {
logger.warn("Crawling Access Exception at : " + dataMap, t);
final String url = sql + ":" + rs.getRow();
final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
failureUrlService.store(config, t.getClass().getCanonicalName(), url, t);
}
if (readInterval > 0) {
sleep(readInterval);
}
}
} catch (final Exception e) {
throw new DataStoreException("Failed to crawl data in DB.", e);
} finally {
try {
if (rs != null) {
rs.close();
}
} catch (final SQLException e) {
logger.warn("Failed to close a result set.", e);
} finally {
try {
if (stmt != null) {
stmt.close();
}
} catch (final SQLException e) {
logger.warn("Failed to close a statement.", e);
} finally {
try {
if (con != null) {
con.close();
}
} catch (final SQLException e) {
logger.warn("Failed to close a db connection.", e);
}
}
}
}
}
protected static class ResultSetParamMap implements Map<String, Object> {
private final Map<String, Object> paramMap = new HashMap<>();
public ResultSetParamMap(final DataConfig config, final Map<String, Object> crawlingContext, final ResultSet resultSet,
final Map<String, String> paramMap) {
this.paramMap.putAll(paramMap);
this.paramMap.put("crawlingConfig", config);
this.paramMap.put("crawlingContext", crawlingContext);
try {
final ResultSetMetaData metaData = resultSet.getMetaData();
final int columnCount = metaData.getColumnCount();
for (int i = 0; i < columnCount; i++) {
try {
final String label = metaData.getColumnLabel(i + 1);
final String value = resultSet.getString(i + 1);
this.paramMap.put(label, value);
} catch (final SQLException e) {
logger.warn("Failed to parse data in a result set. The column is " + (i + 1) + ".", e);
}
}
} catch (final Exception e) {
throw new FessSystemException("Failed to access meta data.", e);
}
}
@Override
public void clear() {
paramMap.clear();
}
@Override
public boolean containsKey(final Object key) {
return paramMap.containsKey(key);
}
@Override
public boolean containsValue(final Object value) {
return paramMap.containsValue(value);
}
@Override
public Set<java.util.Map.Entry<String, Object>> entrySet() {
return paramMap.entrySet();
}
@Override
public Object get(final Object key) {
return paramMap.get(key);
}
@Override
public boolean isEmpty() {
return paramMap.isEmpty();
}
@Override
public Set<String> keySet() {
return paramMap.keySet();
}
@Override
public Object put(final String key, final Object value) {
return paramMap.put(key, value);
}
@Override
public void putAll(final Map<? extends String, ? extends Object> m) {
paramMap.putAll(m);
}
@Override
public Object remove(final Object key) {
return paramMap.remove(key);
}
@Override
public int size() {
return paramMap.size();
}
@Override
public Collection<Object> values() {
return paramMap.values();
}
}
}

258
src/main/java/org/codelibs/fess/ds/impl/EsDataStoreImpl.java
View file

@ -1,258 +0,0 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.ds.impl;
import static org.codelibs.core.stream.StreamUtil.split;
import java.net.InetAddress;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.stream.Collectors;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.FailureUrlService;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.exception.MultipleCrawlingAccessException;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.exception.DataStoreCrawlingException;
import org.codelibs.fess.exception.DataStoreException;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class EsDataStoreImpl extends AbstractDataStoreImpl {
private static final String PREFERENCE = "preference";
private static final String QUERY = "query";
private static final String FIELDS = "fields";
private static final String SIZE = "size";
private static final String TYPE = "type";
private static final String TIMEOUT = "timeout";
private static final String SCROLL = "scroll";
private static final String INDEX = "index";
private static final String HOSTS = "hosts";
private static final String SETTINGS_PREFIX = "settings.";
private static final Logger logger = LoggerFactory.getLogger(EsDataStoreImpl.class);
@Override
protected void storeData(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
final String hostsStr = paramMap.get(HOSTS);
if (StringUtil.isBlank(hostsStr)) {
logger.info("hosts is empty.");
return;
}
final long readInterval = getReadInterval(paramMap);
final Settings settings =
Settings.builder()
.putProperties(
paramMap.entrySet()
.stream()
.filter(e -> e.getKey().startsWith(SETTINGS_PREFIX))
.collect(
Collectors.toMap(e -> e.getKey().replaceFirst("^settings\\.", StringUtil.EMPTY),
e -> e.getValue())), s -> s).build();
logger.info("Connecting to " + hostsStr + " with [" + settings.toDelimitedString(',') + "]");
final TransportAddress[] addresses = split(hostsStr, ",").get(stream -> stream.map(h -> {
final String[] values = h.trim().split(":");
try {
if (values.length == 1) {
return new TransportAddress(InetAddress.getByName(values[0]), 9300);
} else if (values.length == 2) {
return new TransportAddress(InetAddress.getByName(values[0]), Integer.parseInt(values[1]));
}
} catch (final Exception e) {
logger.warn("Failed to parse address: " + h, e);
}
return null;
}).filter(v -> v != null).toArray(n -> new TransportAddress[n]));
try (PreBuiltTransportClient client = new PreBuiltTransportClient(settings)) {
client.addTransportAddresses(addresses);
processData(dataConfig, callback, paramMap, scriptMap, defaultDataMap, readInterval, client);
}
}
protected void processData(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap, final long readInterval, final Client client) {
final boolean deleteProcessedDoc = paramMap.getOrDefault("delete.processed.doc", Constants.FALSE).equalsIgnoreCase(Constants.TRUE);
final String[] indices;
if (paramMap.containsKey(INDEX)) {
indices = paramMap.get(INDEX).trim().split(",");
} else {
indices = new String[] { "_all" };
}
final String scroll = paramMap.containsKey(SCROLL) ? paramMap.get(SCROLL).trim() : "1m";
final String timeout = paramMap.containsKey(TIMEOUT) ? paramMap.get(TIMEOUT).trim() : "1m";
final SearchRequestBuilder builder = client.prepareSearch(indices);
if (paramMap.containsKey(TYPE)) {
builder.setTypes(paramMap.get(TYPE).trim().split(","));
}
if (paramMap.containsKey(SIZE)) {
builder.setSize(Integer.parseInt(paramMap.get(SIZE)));
}
if (paramMap.containsKey(FIELDS)) {
builder.setFetchSource(paramMap.get(FIELDS).trim().split(","), null);
}
builder.setQuery(QueryBuilders.wrapperQuery(paramMap.containsKey(QUERY) ? paramMap.get(QUERY).trim() : "{\"match_all\":{}}"));
builder.setScroll(scroll);
builder.setPreference(paramMap.containsKey(PREFERENCE) ? paramMap.get(PREFERENCE).trim() : Constants.SEARCH_PREFERENCE_LOCAL);
try {
SearchResponse response = builder.execute().actionGet(timeout);
String scrollId = response.getScrollId();
while (scrollId != null) {
final SearchHits searchHits = response.getHits();
final SearchHit[] hits = searchHits.getHits();
if (hits.length == 0) {
scrollId = null;
break;
}
boolean loop = true;
final BulkRequestBuilder bulkRequest = deleteProcessedDoc ? client.prepareBulk() : null;
for (final SearchHit hit : hits) {
if (!alive || !loop) {
break;
}
final Map<String, Object> dataMap = new HashMap<>();
dataMap.putAll(defaultDataMap);
final Map<String, Object> resultMap = new LinkedHashMap<>();
resultMap.putAll(paramMap);
resultMap.put("index", hit.getIndex());
resultMap.put("type", hit.getType());
resultMap.put("id", hit.getId());
resultMap.put("version", Long.valueOf(hit.getVersion()));
resultMap.put("hit", hit);
resultMap.put("source", hit.getSourceAsMap());
resultMap.put("crawlingConfig", dataConfig);
if (logger.isDebugEnabled()) {
for (final Map.Entry<String, Object> entry : resultMap.entrySet()) {
logger.debug(entry.getKey() + "=" + entry.getValue());
}
}
final Map<String, Object> crawlingContext = new HashMap<>();
crawlingContext.put("doc", dataMap);
resultMap.put("crawlingContext", crawlingContext);
for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
final Object convertValue = convertValue(entry.getValue(), resultMap);
if (convertValue != null) {
dataMap.put(entry.getKey(), convertValue);
}
}
if (logger.isDebugEnabled()) {
for (final Map.Entry<String, Object> entry : dataMap.entrySet()) {
logger.debug(entry.getKey() + "=" + entry.getValue());
}
}
try {
callback.store(paramMap, dataMap);
} catch (final CrawlingAccessException e) {
logger.warn("Crawling Access Exception at : " + dataMap, e);
Throwable target = e;
if (target instanceof MultipleCrawlingAccessException) {
final Throwable[] causes = ((MultipleCrawlingAccessException) target).getCauses();
if (causes.length > 0) {
target = causes[causes.length - 1];
}
}
String errorName;
final Throwable cause = target.getCause();
if (cause != null) {
errorName = cause.getClass().getCanonicalName();
} else {
errorName = target.getClass().getCanonicalName();
}
String url;
if (target instanceof DataStoreCrawlingException) {
final DataStoreCrawlingException dce = (DataStoreCrawlingException) target;
url = dce.getUrl();
if (dce.aborted()) {
loop = false;
}
} else {
url = hit.getIndex() + "/" + hit.getType() + "/" + hit.getId();
}
final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
failureUrlService.store(dataConfig, errorName, url, target);
} catch (final Throwable t) {
logger.warn("Crawling Access Exception at : " + dataMap, t);
final String url = hit.getIndex() + "/" + hit.getType() + "/" + hit.getId();
final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
failureUrlService.store(dataConfig, t.getClass().getCanonicalName(), url, t);
}
if (bulkRequest != null) {
bulkRequest.add(client.prepareDelete(hit.getIndex(), hit.getType(), hit.getId()));
}
if (readInterval > 0) {
sleep(readInterval);
}
}
if (bulkRequest != null && bulkRequest.numberOfActions() > 0) {
final BulkResponse bulkResponse = bulkRequest.execute().actionGet(timeout);
if (bulkResponse.hasFailures()) {
logger.warn(bulkResponse.buildFailureMessage());
}
}
if (!alive) {
break;
}
response = client.prepareSearchScroll(scrollId).setScroll(scroll).execute().actionGet(timeout);
scrollId = response.getScrollId();
}
} catch (final Exception e) {
throw new DataStoreException("Failed to crawl data when acessing elasticsearch.", e);
}
}
}

55
src/main/java/org/codelibs/fess/ds/impl/EsListDataStoreImpl.java
View file

@ -1,55 +0,0 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.ds.impl;
import java.util.Map;
import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.exception.DataStoreException;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class EsListDataStoreImpl extends EsDataStoreImpl {
private static final Logger logger = LoggerFactory.getLogger(EsListDataStoreImpl.class);
@Override
protected void storeData(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
int nThreads = 1;
if (paramMap.containsKey(Constants.NUM_OF_THREADS)) {
try {
nThreads = Integer.parseInt(paramMap.get(Constants.NUM_OF_THREADS));
} catch (final NumberFormatException e) {
logger.warn(Constants.NUM_OF_THREADS + " is not int value.", e);
}
}
final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getCrawlerClientFactory();
dataConfig.initializeClientFactory(crawlerClientFactory);
try {
final FileListIndexUpdateCallbackImpl fileListIndexUpdateCallback =
new FileListIndexUpdateCallbackImpl(callback, crawlerClientFactory, nThreads);
super.storeData(dataConfig, fileListIndexUpdateCallback, paramMap, scriptMap, defaultDataMap);
fileListIndexUpdateCallback.commit();
} catch (final Exception e) {
throw new DataStoreException(e);
}
}
}

481
src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java
View file

@ -1,481 +0,0 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.ds.impl;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.elasticsearch.runner.net.Curl;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.client.http.HcHttpClient;
import org.codelibs.fess.crawler.client.http.RequestHeader;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.es.config.exentity.CrawlingConfigWrapper;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
/**
* @author Keiichi Watanabe
*/
public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
private static final Logger logger = LoggerFactory.getLogger(GitBucketDataStoreImpl.class);
private static final int MAX_DEPTH = 20;
protected static final String TOKEN_PARAM = "token";
protected static final String GITBUCKET_URL_PARAM = "url";
protected static final String PRIVATE_REPOSITORY_PARAM = "is_private";
protected static final String COLLABORATORS_PARAM = "collaborators";
@Override
protected void storeData(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
final String rootURL = getRootURL(paramMap);
final String authToken = getAuthToken(paramMap);
final long readInterval = getReadInterval(paramMap);
// Non-emptiness Check for URL and Token
if (rootURL.isEmpty() || authToken.isEmpty()) {
logger.warn("parameter \"" + TOKEN_PARAM + "\" and \"" + GITBUCKET_URL_PARAM + "\" are required");
return;
}
// Get List of Repositories
final List<Map<String, Object>> repositoryList = getRepositoryList(rootURL, authToken);
if (repositoryList.isEmpty()) {
logger.warn("Token is invalid or no Repository");
return;
}
// Get Labels
final Map<String, String> pluginInfo = getFessPluginInfo(rootURL, authToken);
final String sourceLabel = pluginInfo.get("source_label");
final String issueLabel = pluginInfo.get("issue_label");
final String wikiLabel = pluginInfo.get("wiki_label");
final CrawlingConfig crawlingConfig = new CrawlingConfigWrapper(dataConfig) {
@Override
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory crawlerClientFactory) {
final Map<String, Object> paramMap = super.initializeClientFactory(crawlerClientFactory);
final List<RequestHeader> headerList = new ArrayList<>();
final RequestHeader[] headers = (RequestHeader[]) paramMap.get(HcHttpClient.REQUERT_HEADERS_PROPERTY);
if (headers != null) {
for (final RequestHeader header : headers) {
headerList.add(header);
}
}
headerList.add(new RequestHeader("Authorization", "token " + authToken));
headerList.add(new RequestHeader("Accept", "application/vnd.github.v3.raw"));
paramMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, headerList.toArray(new RequestHeader[headerList.size()]));
return paramMap;
}
};
// Crawl each repository
for (final Map<String, Object> repository : repositoryList) {
try {
final String owner = (String) repository.get("owner");
final String name = (String) repository.get("name");
// Since old gitbucket-fess-plugin does not return "branch", it refers instead of "master".
final String branch = (String) repository.getOrDefault("branch", "master");
final int issueCount = (int) repository.get("issue_count");
final int pullCount = (int) repository.get("pull_count");
final List<String> roleList = createRoleList(owner, repository);
// branch is empty when git repository is empty.
if (StringUtil.isNotEmpty(branch)) {
final String refStr = getGitRef(rootURL, authToken, owner, name, branch);
if (logger.isInfoEnabled()) {
logger.info("Crawl " + owner + "/" + name);
}
// crawl and store file contents recursively
crawlFileContents(
rootURL,
authToken,
owner,
name,
refStr,
StringUtil.EMPTY,
0,
readInterval,
path -> {
storeFileContent(rootURL, authToken, sourceLabel, owner, name, refStr, roleList, path, crawlingConfig,
callback, paramMap, scriptMap, defaultDataMap);
if (readInterval > 0) {
sleep(readInterval);
}
});
}
if (logger.isInfoEnabled()) {
logger.info("Crawl issues in " + owner + "/" + name);
}
// store issues
for (int issueId = 1; issueId <= issueCount + pullCount; issueId++) {
storeIssueById(rootURL, authToken, issueLabel, owner, name, new Integer(issueId), roleList, crawlingConfig, callback,
paramMap, scriptMap, defaultDataMap);
if (readInterval > 0) {
sleep(readInterval);
}
}
if (logger.isInfoEnabled()) {
logger.info("Crawl Wiki in " + owner + "/" + name);
}
// crawl Wiki
storeWikiContents(rootURL, authToken, wikiLabel, owner, name, roleList, crawlingConfig, callback, paramMap, scriptMap,
defaultDataMap, readInterval);
} catch (final Exception e) {
logger.warn("Failed to access to " + repository, e);
}
}
}
protected String getRootURL(final Map<String, String> paramMap) {
if (paramMap.containsKey(GITBUCKET_URL_PARAM)) {
final String url = paramMap.get(GITBUCKET_URL_PARAM);
if (!url.endsWith("/")) {
return url + "/";
}
return url;
}
return StringUtil.EMPTY;
}
protected String getAuthToken(final Map<String, String> paramMap) {
if (paramMap.containsKey(TOKEN_PARAM)) {
return paramMap.get(TOKEN_PARAM);
}
return StringUtil.EMPTY;
}
protected Map<String, String> getFessPluginInfo(final String rootURL, final String authToken) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String url = rootURL + "api/v3/fess/info";
try (CurlResponse curlResponse =
Curl.get(url).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
@SuppressWarnings({ "rawtypes", "unchecked" })
final Map<String, String> map = (Map) curlResponse.getContentAsMap();
assert (map.containsKey("version"));
assert (map.containsKey("source_label") && map.containsKey("wiki_label") && map.containsKey("issue_label"));
return map;
} catch (final Exception e) {
logger.warn("Failed to access to " + url, e);
return Collections.emptyMap();
}
}
protected List<Map<String, Object>> getRepositoryList(final String rootURL, final String authToken) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String url = rootURL + "api/v3/fess/repos";
int totalCount = -1; // initialize with dummy value
final List<Map<String, Object>> repoList = new ArrayList<>();
do {
final String urlWithOffset = url + "?offset=" + repoList.size();
try (CurlResponse curlResponse =
Curl.get(urlWithOffset).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
final Map<String, Object> map = curlResponse.getContentAsMap();
assert (map.containsKey("total_count"));
assert (map.containsKey("response_count"));
assert (map.containsKey("repositories"));
totalCount = (int) map.get("total_count");
final int responseCount = (int) map.get("response_count");
if (responseCount == 0) {
break;
}
@SuppressWarnings("unchecked")
final List<Map<String, Object>> repos = (ArrayList<Map<String, Object>>) map.get("repositories");
repoList.addAll(repos);
} catch (final Exception e) {
logger.warn("Failed to access to " + urlWithOffset, e);
break;
}
} while (repoList.size() < totalCount);
if (logger.isInfoEnabled()) {
logger.info("There exist " + repoList.size() + " repositories");
}
return repoList;
}
protected String getGitRef(final String rootURL, final String authToken, final String owner, final String name, final String branch) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String url = encode(rootURL, "api/v3/repos/" + owner + "/" + name + "/git/refs/heads/" + branch, null);
try (CurlResponse curlResponse =
Curl.get(url).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
final Map<String, Object> map = curlResponse.getContentAsMap();
assert (map.containsKey("object"));
@SuppressWarnings("unchecked")
final Map<String, String> objmap = (Map<String, String>) map.get("object");
assert (objmap.containsKey("sha"));
return objmap.get("sha");
} catch (final Exception e) {
logger.warn("Failed to access to " + url, e);
return branch;
}
}
private List<String> createRoleList(final String owner, final Map<String, Object> repository) {
Boolean isPrivate = true;
if (repository.containsKey(PRIVATE_REPOSITORY_PARAM)) {
isPrivate = (Boolean) repository.get(PRIVATE_REPOSITORY_PARAM);
}
if (!isPrivate) {
return Collections.singletonList("Rguest");
}
@SuppressWarnings("unchecked")
final List<String> collaboratorList = (List<String>) repository.get(COLLABORATORS_PARAM);
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
collaboratorList.add(owner);
return collaboratorList.stream().map(user -> systemHelper.getSearchRoleByUser(user)).collect(Collectors.toList());
}
private List<Object> parseList(final InputStream is) { // TODO This function should be moved to CurlResponse
try {
return JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, is).list();
} catch (final Exception e) {
logger.warn("Failed to parse a list.", e);
return Collections.emptyList();
}
}
private void storeFileContent(final String rootURL, final String authToken, final String sourceLabel, final String owner,
final String name, final String refStr, final List<String> roleList, final String path, final CrawlingConfig crawlingConfig,
final IndexUpdateCallback callback, final Map<String, String> paramMap, final Map<String, String> scriptMap,
final Map<String, Object> defaultDataMap) {
final String apiUrl = encode(rootURL, "api/v3/repos/" + owner + "/" + name + "/contents/" + path, null);
final String viewUrl = encode(rootURL, owner + "/" + name + "/blob/" + refStr + "/" + path, null);
if (logger.isInfoEnabled()) {
logger.info("Get a content from " + apiUrl);
}
final Map<String, Object> dataMap = new HashMap<>();
dataMap.putAll(defaultDataMap);
dataMap.putAll(ComponentUtil.getDocumentHelper().processRequest(crawlingConfig, paramMap.get("crawlingInfoId"),
apiUrl + "?ref=" + refStr + "&large_file=true"));
dataMap.put("url", viewUrl);
dataMap.put("role", roleList);
dataMap.put("label", Collections.singletonList(sourceLabel));
// TODO scriptMap
callback.store(paramMap, dataMap);
return;
}
private void storeIssueById(final String rootURL, final String authToken, final String issueLabel, final String owner,
final String name, final Integer issueId, final List<String> roleList, final CrawlingConfig crawlingConfig,
final IndexUpdateCallback callback, final Map<String, String> paramMap, final Map<String, String> scriptMap,
final Map<String, Object> defaultDataMap) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String issueUrl = rootURL + "api/v3/repos/" + owner + "/" + name + "/issues/" + issueId.toString();
final String viewUrl = rootURL + owner + "/" + name + "/issues/" + issueId.toString();
if (logger.isInfoEnabled()) {
logger.info("Get a content from " + issueUrl);
}
final Map<String, Object> dataMap = new HashMap<>();
String contentStr = "";
dataMap.putAll(defaultDataMap);
// Get issue description
// FIXME: Use `ComponentUtil.getDocumentHelper().processRequest` instead of `Curl.get`
try (CurlResponse curlResponse =
Curl.get(issueUrl).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
final Map<String, Object> map = curlResponse.getContentAsMap();
dataMap.put("title", map.getOrDefault("title", ""));
contentStr = (String) map.getOrDefault("body", "");
} catch (final Exception e) {
logger.warn("Failed to access to " + issueUrl, e);
}
final String commentsStr = String.join("\n", getIssueComments(issueUrl, authToken));
contentStr += "\n" + commentsStr;
dataMap.put("content", contentStr);
dataMap.put("url", viewUrl);
dataMap.put("role", roleList);
dataMap.put("label", Collections.singletonList(issueLabel));
// TODO scriptMap
callback.store(paramMap, dataMap);
return;
}
private List<String> getIssueComments(final String issueUrl, final String authToken) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String commentsUrl = issueUrl + "/comments";
final List<String> commentList = new ArrayList<>();
try (CurlResponse curlResponse =
Curl.get(commentsUrl).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
final String commentsJson = curlResponse.getContentAsString();
final List<Map<String, Object>> comments =
new ObjectMapper().readValue(commentsJson, new TypeReference<List<Map<String, Object>>>() {
});
for (final Map<String, Object> comment : comments) {
if (comment.containsKey("body")) {
commentList.add((String) comment.get("body"));
}
}
} catch (final Exception e) {
logger.warn("Failed to access to " + issueUrl, e);
}
return commentList;
}
@SuppressWarnings("unchecked")
private void storeWikiContents(final String rootURL, final String authToken, final String wikiLabel, final String owner,
final String name, final List<String> roleList, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback,
final Map<String, String> paramMap, final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap,
final long readInterval) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String wikiUrl = rootURL + "api/v3/fess/" + owner + "/" + name + "/wiki";
List<String> pageList = Collections.emptyList();
// Get list of pages
try (CurlResponse curlResponse =
Curl.get(wikiUrl).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
final Map<String, Object> map = curlResponse.getContentAsMap();
pageList = (List<String>) map.get("pages");
} catch (final Exception e) {
logger.warn("Failed to access to " + wikiUrl, e);
}
for (final String page : pageList) {
final String pageUrl = wikiUrl + "/contents/" + page + ".md";
final String viewUrl = rootURL + owner + "/" + name + "/wiki/" + page;
if (logger.isInfoEnabled()) {
logger.info("Get a content from " + pageUrl);
}
final Map<String, Object> dataMap = new HashMap<>();
dataMap.putAll(defaultDataMap);
dataMap.putAll(ComponentUtil.getDocumentHelper().processRequest(crawlingConfig, paramMap.get("crawlingInfoId"),
pageUrl.replace("+", "%20")));
dataMap.put("url", viewUrl);
dataMap.put("role", roleList);
dataMap.put("label", Collections.singletonList(wikiLabel));
// TODO scriptMap
callback.store(paramMap, dataMap);
if (logger.isDebugEnabled()) {
logger.debug("Stored " + pageUrl);
}
if (readInterval > 0) {
sleep(readInterval);
}
}
}
protected void crawlFileContents(final String rootURL, final String authToken, final String owner, final String name,
final String refStr, final String path, final int depth, final long readInterval, final Consumer<String> consumer) {
if (MAX_DEPTH <= depth) {
return;
}
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String url = encode(rootURL, "api/v3/repos/" + owner + "/" + name + "/contents/" + path, "ref=" + refStr);
try (CurlResponse curlResponse =
Curl.get(url).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
final InputStream iStream = curlResponse.getContentAsStream();
final List<Object> fileList = parseList(iStream);
for (int i = 0; i < fileList.size(); ++i) {
@SuppressWarnings("unchecked")
final Map<String, String> file = (Map<String, String>) fileList.get(i);
final String newPath = path.isEmpty() ? file.get("name") : path + "/" + file.get("name");
switch (file.get("type")) {
case "file":
consumer.accept(newPath);
break;
case "dir":
if (readInterval > 0) {
sleep(readInterval);
}
crawlFileContents(rootURL, authToken, owner, name, refStr, newPath, depth + 1, readInterval, consumer);
break;
}
}
} catch (final Exception e) {
logger.warn("Failed to access to " + url, e);
}
}
protected String encode(final String rootURL, final String path, final String query) {
try {
final URI rootURI = new URI(rootURL);
final URI uri =
new URI(rootURI.getScheme(), rootURI.getUserInfo(), rootURI.getHost(), rootURI.getPort(), rootURI.getPath() + path,
query, null);
return uri.toASCIIString();
} catch (final URISyntaxException e) {
logger.warn("Failed to parse " + rootURL + path + "?" + query, e);
if (StringUtil.isEmpty(query)) {
return rootURL + path;
}
return rootURL + path + "?" + query;
}
}
}

161
src/main/java/org/codelibs/fess/ds/impl/JsonDataStoreImpl.java
View file

@ -1,161 +0,0 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.ds.impl;
import static org.codelibs.core.stream.StreamUtil.stream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.exception.DataStoreException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
public class JsonDataStoreImpl extends AbstractDataStoreImpl {
private static final Logger logger = LoggerFactory.getLogger(JsonDataStoreImpl.class);
private static final String FILE_ENCODING_PARAM = "fileEncoding";
private static final String FILES_PARAM = "files";
private static final String DIRS_PARAM = "directories";
private String[] fileSuffixes = new String[] { ".json", ".jsonl" };
@Override
protected void storeData(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
final String fileEncoding = getFileEncoding(paramMap);
final List<File> fileList = getFileList(paramMap);
if (fileList.isEmpty()) {
logger.warn("No files to process");
return;
}
for (final File file : fileList) {
processFile(dataConfig, callback, paramMap, scriptMap, defaultDataMap, file, fileEncoding);
}
}
private List<File> getFileList(final Map<String, String> paramMap) {
String value = paramMap.get(FILES_PARAM);
final List<File> fileList = new ArrayList<>();
if (StringUtil.isBlank(value)) {
value = paramMap.get(DIRS_PARAM);
if (StringUtil.isBlank(value)) {
throw new DataStoreException(FILES_PARAM + " and " + DIRS_PARAM + " are blank.");
}
logger.info(DIRS_PARAM + "=" + value);
final String[] values = value.split(",");
for (final String path : values) {
final File dir = new File(path);
if (dir.isDirectory()) {
stream(dir.listFiles()).of(
stream -> stream.filter(f -> isDesiredFile(f.getParentFile(), f.getName()))
.sorted((f1, f2) -> (int) (f1.lastModified() - f2.lastModified())).forEach(fileList::add));
} else {
logger.warn(path + " is not a directory.");
}
}
} else {
logger.info(FILES_PARAM + "=" + value);
final String[] values = value.split(",");
for (final String path : values) {
final File file = new File(path);
if (file.isFile() && isDesiredFile(file.getParentFile(), file.getName())) {
fileList.add(file);
} else {
logger.warn(path + " is not found.");
}
}
}
if (fileList.isEmpty() && logger.isDebugEnabled()) {
logger.debug("No files in " + value);
}
return fileList;
}
private boolean isDesiredFile(final File parentFile, final String filename) {
final String name = filename.toLowerCase(Locale.ROOT);
for (final String suffix : fileSuffixes) {
if (name.endsWith(suffix)) {
return true;
}
}
return false;
}
private String getFileEncoding(final Map<String, String> paramMap) {
final String value = paramMap.get(FILE_ENCODING_PARAM);
if (StringUtil.isBlank(value)) {
return Constants.UTF_8;
}
return value;
}
private void processFile(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap, final File file, final String fileEncoding) {
final ObjectMapper objectMapper = new ObjectMapper();
logger.info("Loading " + file.getAbsolutePath());
try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), fileEncoding))) {
for (String line; (line = br.readLine()) != null;) {
final Map<String, Object> source = objectMapper.readValue(line, new TypeReference<Map<String, Object>>() {
});
final Map<String, Object> resultMap = new LinkedHashMap<>();
final Map<String, Object> dataMap = new HashMap<>(defaultDataMap);
resultMap.putAll(paramMap);
resultMap.putAll(source);
for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
final Object convertValue = convertValue(entry.getValue(), resultMap);
if (convertValue != null) {
dataMap.put(entry.getKey(), convertValue);
}
}
callback.store(paramMap, dataMap);
}
} catch (final FileNotFoundException e) {
logger.warn("Source file " + file + " does not exist.", e);
} catch (final IOException e) {
logger.warn("IO Error occurred while reading source file.", e);
}
}
public void setFileSuffixes(final String[] fileSuffixes) {
this.fileSuffixes = fileSuffixes;
}
}

2
src/main/java/org/codelibs/fess/helper/DataIndexHelper.java
View file

@ -26,7 +26,7 @@ import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.FailureUrlService;
import org.codelibs.fess.ds.DataStore;
import org.codelibs.fess.ds.DataStoreFactory;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.ds.callback.IndexUpdateCallback;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.mylasta.direction.FessConfig;

50
src/main/resources/fess_ds.xml
View file

@ -3,55 +3,7 @@
"http://dbflute.org/meta/lastadi10.dtd">
<components>
<component name="dataStoreFactory" class="org.codelibs.fess.ds.DataStoreFactory">
<postConstruct name="add">
<arg>"DatabaseDataStore"</arg>
<arg>databaseDataStore</arg>
</postConstruct>
<postConstruct name="add">
<arg>"CsvDataStore"</arg>
<arg>csvDataStore</arg>
</postConstruct>
<postConstruct name="add">
<arg>"CsvListDataStore"</arg>
<arg>csvListDataStore</arg>
</postConstruct>
<postConstruct name="add">
<arg>"EsDataStore"</arg>
<arg>esDataStore</arg>
</postConstruct>
<postConstruct name="add">
<arg>"EsListDataStore"</arg>
<arg>esListDataStore</arg>
</postConstruct>
<postConstruct name="add">
<arg>"GitBucketDataStore"</arg>
<arg>gitBucketDataStore</arg>
</postConstruct>
<postConstruct name="add">
<arg>"JsonDataStore"</arg>
<arg>jsonDataStore</arg>
</postConstruct>
</component>
<component name="databaseDataStore" class="org.codelibs.fess.ds.impl.DatabaseDataStoreImpl">
<component name="indexUpdateCallback" class="org.codelibs.fess.ds.callback.IndexUpdateCallbackImpl" instance="prototype">
</component>
<component name="csvDataStore" class="org.codelibs.fess.ds.impl.CsvDataStoreImpl">
<!--
<property name="csvFileSuffixs">new String[] { ".csv", ".tsv" }</property>
-->
</component>
<component name="csvListDataStore" class="org.codelibs.fess.ds.impl.CsvListDataStoreImpl">
</component>
<component name="esDataStore" class="org.codelibs.fess.ds.impl.EsDataStoreImpl">
</component>
<component name="esListDataStore" class="org.codelibs.fess.ds.impl.EsListDataStoreImpl">
</component>
<component name="gitBucketDataStore" class="org.codelibs.fess.ds.impl.GitBucketDataStoreImpl">
</component>
<component name="jsonDataStore" class="org.codelibs.fess.ds.impl.JsonDataStoreImpl">
</component>
<component name="indexUpdateCallback" class="org.codelibs.fess.ds.impl.IndexUpdateCallbackImpl" instance="prototype">
</component>
</components>

60
src/test/java/org/codelibs/fess/ds/impl/DatabaseDataStoreImplTest.java
View file

@ -1,60 +0,0 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.ds.impl;
import java.util.HashMap;
import java.util.Map;
import org.codelibs.fess.unit.UnitFessTestCase;
public class DatabaseDataStoreImplTest extends UnitFessTestCase {
public DatabaseDataStoreImpl databaseDataStore;
@Override
public void setUp() throws Exception {
super.setUp();
databaseDataStore = new DatabaseDataStoreImpl();
}
public void test_convertValue() {
String value;
final Map<String, Object> paramMap = new HashMap<>();
paramMap.put("param1", "PARAM1");
paramMap.put("param2", "PARAM2+");
paramMap.put("param3", "PARAM3*");
value = "\"abc\"";
assertEquals("abc", databaseDataStore.convertValue(value, paramMap));
value = "param1";
assertEquals("PARAM1", databaseDataStore.convertValue(value, paramMap));
value = "param2";
assertEquals("PARAM2+", databaseDataStore.convertValue(value, paramMap));
value = "\"123\"+param2+\",\"+param3+\"abc\"";
assertEquals("123PARAM2+,PARAM3*abc", databaseDataStore.convertValue(value, paramMap));
value = null;
assertEquals("", databaseDataStore.convertValue(value, paramMap));
value = "";
assertEquals("", databaseDataStore.convertValue(value, paramMap));
value = " ";
assertNull(databaseDataStore.convertValue(value, paramMap));
}
}

42
src/test/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImplTest.java
View file

@ -1,42 +0,0 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.ds.impl;
import org.codelibs.fess.unit.UnitFessTestCase;
public class GitBucketDataStoreImplTest extends UnitFessTestCase {
public GitBucketDataStoreImpl dataStore;
@Override
public void setUp() throws Exception {
super.setUp();
dataStore = new GitBucketDataStoreImpl();
}
public void test_encode() {
String rootURL = "https://gitbucket.com/";
String path;
String query;
path = "api/v3/repos/";
query = "aaa=111";
assertEquals("https://gitbucket.com/api/v3/repos/?aaa=111", dataStore.encode(rootURL, path, query));
path = "api/v3/repos/AA BB";
query = "aaa=11 11";
assertEquals("https://gitbucket.com/api/v3/repos/AA%20BB?aaa=11%2011", dataStore.encode(rootURL, path, query));
}
}