0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

working on #72

Browse source
This commit is contained in:
Shinsuke Sugaya 2013-12-22 20:42:57 +09:00
parent 223787003e
commit bb28d55594
18 changed files with 1161 additions and 339 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/main/java/jp/sf/fess/dic/DictionaryException.java Normal file
View file

@ -0,0 +1,17 @@
package jp.sf.fess.dic;
import jp.sf.fess.FessSystemException;
public class DictionaryException extends FessSystemException {
private static final long serialVersionUID = 1L;
public DictionaryException(final String message, final Throwable cause) {
super(message, cause);
}
public DictionaryException(final String message) {
super(message);
}
}

164
src/main/java/jp/sf/fess/dic/DictionaryFile.java Normal file
View file

@ -0,0 +1,164 @@
package jp.sf.fess.dic;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
public abstract class DictionaryFile {
public abstract String getName();
public abstract PagingList<DictionaryItem> selectList(int offset, int size);
public abstract void insert(DictionaryItem item);
public abstract void update(DictionaryItem item);
public abstract void delete(DictionaryItem item);
public static class PagingList<E> implements List<E> {
private final List<E> parent;
protected int allRecordCount;
protected int pageSize;
protected int currentPageNumber;
public PagingList(final List<E> list, final int offset, final int size,
final int allRecordCount) {
this.parent = list;
this.allRecordCount = allRecordCount;
pageSize = size;
currentPageNumber = offset / size + 1;
}
@Override
public int size() {
return parent.size();
}
@Override
public boolean isEmpty() {
return parent.isEmpty();
}
@Override
public boolean contains(final Object o) {
return parent.contains(o);
}
@Override
public Iterator<E> iterator() {
return parent.iterator();
}
@Override
public Object[] toArray() {
return parent.toArray();
}
@Override
public <T> T[] toArray(final T[] a) {
return parent.toArray(a);
}
@Override
public boolean add(final E e) {
return parent.add(e);
}
@Override
public boolean remove(final Object o) {
return parent.remove(o);
}
@Override
public boolean containsAll(final Collection<?> c) {
return parent.containsAll(c);
}
@Override
public boolean addAll(final Collection<? extends E> c) {
return parent.addAll(c);
}
@Override
public boolean addAll(final int index, final Collection<? extends E> c) {
return parent.addAll(index, c);
}
@Override
public boolean removeAll(final Collection<?> c) {
return parent.retainAll(c);
}
@Override
public boolean retainAll(final Collection<?> c) {
return parent.retainAll(c);
}
@Override
public void clear() {
parent.clear();
}
@Override
public E get(final int index) {
return parent.get(index);
}
@Override
public E set(final int index, final E element) {
return parent.set(index, element);
}
@Override
public void add(final int index, final E element) {
parent.add(index, element);
}
@Override
public E remove(final int index) {
return parent.remove(index);
}
@Override
public int indexOf(final Object o) {
return parent.indexOf(o);
}
@Override
public int lastIndexOf(final Object o) {
return parent.lastIndexOf(o);
}
@Override
public ListIterator<E> listIterator() {
return parent.listIterator();
}
@Override
public ListIterator<E> listIterator(final int index) {
return parent.listIterator(index);
}
@Override
public List<E> subList(final int fromIndex, final int toIndex) {
return parent.subList(fromIndex, toIndex);
}
public int getAllRecordCount() {
return allRecordCount;
}
public int getPageSize() {
return pageSize;
}
public int getCurrentPageNumber() {
return currentPageNumber;
}
}
}

5
src/main/java/jp/sf/fess/dic/DictionaryItem.java Normal file
View file

@ -0,0 +1,5 @@
package jp.sf.fess.dic;
public abstract class DictionaryItem {
}

41
src/main/java/jp/sf/fess/dic/DictionaryLocator.java Normal file
View file

@ -0,0 +1,41 @@
package jp.sf.fess.dic;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import jp.sf.fess.util.ResourceUtil;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.AbstractFileFilter;
public abstract class DictionaryLocator {
protected List<String> searchPathList = new ArrayList<String>();
public abstract Map<String, DictionaryFile> find();
protected File[] findFiles(final String path, final String filenamePrefix,
final List<String> excludedSet) {
final Collection<File> files = FileUtils.listFiles(new File(path),
new AbstractFileFilter() {
@Override
public boolean accept(final File dir, final String name) {
return name.startsWith(filenamePrefix);
}
}, new AbstractFileFilter() {
@Override
public boolean accept(final File dir, final String name) {
return excludedSet == null
|| !excludedSet.contains(name);
}
});
return files.toArray(new File[files.size()]);
}
public void addSearchPath(final String path) {
searchPathList.add(ResourceUtil.resolve(path));
}
}

98
src/main/java/jp/sf/fess/dic/DictionaryManager.java Normal file
View file

@ -0,0 +1,98 @@
package jp.sf.fess.dic;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import jp.sf.fess.dic.synonym.SynonymFile;
import org.seasar.extension.timer.TimeoutManager;
import org.seasar.extension.timer.TimeoutTarget;
import org.seasar.extension.timer.TimeoutTask;
import org.seasar.framework.container.annotation.tiger.DestroyMethod;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DictionaryManager {
private static final Logger logger = LoggerFactory
.getLogger(DictionaryManager.class);
protected Map<String, DictionaryFile> dicFileMap;
public long keepAlive = 5 * 60 * 1000; // 5min
public int watcherTimeout = 60; // 1min
protected volatile long lifetime = 0;
protected TimeoutTask watcherTargetTask;
protected List<DictionaryLocator> locatorList = new ArrayList<DictionaryLocator>();
@InitMethod
public void init() {
// start
final WatcherTarget watcherTarget = new WatcherTarget();
watcherTargetTask = TimeoutManager.getInstance().addTimeoutTarget(
watcherTarget, watcherTimeout, true);
}
@DestroyMethod
public void destroy() {
if (watcherTargetTask != null && !watcherTargetTask.isStopped()) {
watcherTargetTask.stop();
}
}
public DictionaryFile[] getDictionaryFiles() {
final Map<String, DictionaryFile> fileMap = getDictionaryFileMap();
final Collection<DictionaryFile> values = fileMap.values();
return values.toArray(new SynonymFile[values.size()]);
}
public DictionaryFile getDictionaryFile(final String uri) {
final Map<String, DictionaryFile> fileMap = getDictionaryFileMap();
return fileMap.get(uri);
}
protected Map<String, DictionaryFile> getDictionaryFileMap() {
synchronized (this) {
if (lifetime > System.currentTimeMillis() && dicFileMap != null) {
lifetime = System.currentTimeMillis() + keepAlive;
return dicFileMap;
}
final Map<String, DictionaryFile> newFileMap = new TreeMap<String, DictionaryFile>();
for (final DictionaryLocator locator : locatorList) {
newFileMap.putAll(locator.find());
}
dicFileMap = newFileMap;
lifetime = System.currentTimeMillis() + keepAlive;
return dicFileMap;
}
}
public void addLocator(final DictionaryLocator locator) {
locatorList.add(locator);
}
protected class WatcherTarget implements TimeoutTarget {
@Override
public void expired() {
synchronized (DictionaryManager.this) {
if (lifetime <= System.currentTimeMillis()
&& dicFileMap != null) {
if (logger.isDebugEnabled()) {
logger.debug("Cleaning synonym files: " + dicFileMap);
}
dicFileMap = null;
}
}
}
}
}

331
src/main/java/jp/sf/fess/dic/synonym/SynonymFile.java Normal file
View file

@ -0,0 +1,331 @@
package jp.sf.fess.dic.synonym;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import jp.sf.fess.Constants;
import jp.sf.fess.dic.DictionaryException;
import jp.sf.fess.dic.DictionaryFile;
import jp.sf.fess.dic.DictionaryItem;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
public class SynonymFile extends DictionaryFile {
private final File file;
List<DictionaryItem> synonymItemList;
public SynonymFile(final File file) {
this.file = file;
}
@Override
public String getName() {
return file.getAbsolutePath();
}
@Override
public synchronized PagingList<DictionaryItem> selectList(final int offset,
final int size) {
if (synonymItemList == null) {
reload(null);
}
if (offset >= synonymItemList.size() || offset < 0) {
return new PagingList<DictionaryItem>(
Collections.<DictionaryItem> emptyList(), offset, size,
synonymItemList.size());
}
int toIndex = offset + size;
if (toIndex > synonymItemList.size()) {
toIndex = synonymItemList.size();
}
return new PagingList<DictionaryItem>(synonymItemList.subList(offset,
toIndex), offset, size, synonymItemList.size());
}
@Override
public synchronized void insert(final DictionaryItem item) {
final SynonymItem synonymItem = (SynonymItem) item;
BufferedWriter bw = null;
try {
bw = new BufferedWriter(new OutputStreamWriter(
new FileOutputStream(file, true), Constants.UTF_8));
bw.newLine();
bw.write(synonymItem.toLineString());
bw.flush();
synonymItemList.add(new SynonymItem(synonymItemList.size() + 1,
synonymItem.getInputs(), synonymItem.getOutputs()));
} catch (final IOException e) {
throw new DictionaryException("Failed to write: " + item, e);
} finally {
IOUtils.closeQuietly(bw);
}
}
@Override
public synchronized void update(final DictionaryItem item) {
reload(new SynonymUpdater(file, (SynonymItem) item));
}
@Override
public synchronized void delete(final DictionaryItem item) {
final SynonymItem synonymItem = (SynonymItem) item;
synonymItem.setNewInputs(new String[0]);
synonymItem.setNewOutputs(new String[0]);
reload(new SynonymUpdater(file, synonymItem));
}
protected void reload(final SynonymUpdater updater) {
final List<DictionaryItem> itemList = new ArrayList<DictionaryItem>();
BufferedReader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(
new FileInputStream(file), Constants.UTF_8));
int id = 0;
String line = null;
while ((line = reader.readLine()) != null) {
if (line.length() == 0 || line.charAt(0) == '#') {
if (updater != null) {
updater.write(line);
}
continue; // ignore empty lines and comments
}
String inputs[];
String outputs[];
final List<String> sides = split(line, "=>");
if (sides.size() > 1) { // explicit mapping
if (sides.size() != 2) {
throw new DictionaryException(
"more than one explicit mapping specified on the same line");
}
final List<String> inputStrings = split(sides.get(0), ",");
inputs = new String[inputStrings.size()];
for (int i = 0; i < inputs.length; i++) {
inputs[i] = unescape(inputStrings.get(i)).trim();
}
final List<String> outputStrings = split(sides.get(1), ",");
outputs = new String[outputStrings.size()];
for (int i = 0; i < outputs.length; i++) {
outputs[i] = unescape(outputStrings.get(i)).trim();
}
if (inputs.length > 0 && outputs.length > 0) {
id++;
final SynonymItem item = new SynonymItem(id, inputs,
outputs);
if (updater != null) {
final SynonymItem newItem = updater.write(item);
if (newItem != null) {
itemList.add(newItem);
} else {
id--;
}
} else {
itemList.add(item);
}
}
} else {
final List<String> inputStrings = split(line, ",");
inputs = new String[inputStrings.size()];
for (int i = 0; i < inputs.length; i++) {
inputs[i] = unescape(inputStrings.get(i)).trim();
}
if (inputs.length > 0) {
id++;
final SynonymItem item = new SynonymItem(id, inputs,
inputs);
if (updater != null) {
final SynonymItem newItem = updater.write(item);
if (newItem != null) {
itemList.add(newItem);
} else {
id--;
}
} else {
itemList.add(item);
}
}
}
}
if (updater != null) {
updater.commit();
}
} catch (final IOException e) {
throw new DictionaryException("Failed to parse "
+ file.getAbsolutePath(), e);
} finally {
IOUtils.closeQuietly(reader);
synonymItemList = itemList;
if (updater != null) {
updater.close();
}
}
}
private static List<String> split(final String s, final String separator) {
final List<String> list = new ArrayList<String>(2);
StringBuilder sb = new StringBuilder();
int pos = 0;
final int end = s.length();
while (pos < end) {
if (s.startsWith(separator, pos)) {
if (sb.length() > 0) {
list.add(sb.toString());
sb = new StringBuilder();
}
pos += separator.length();
continue;
}
char ch = s.charAt(pos++);
if (ch == '\\') {
sb.append(ch);
if (pos >= end) {
break; // ERROR, or let it go?
}
ch = s.charAt(pos++);
}
sb.append(ch);
}
if (sb.length() > 0) {
list.add(sb.toString());
}
return list;
}
private String unescape(final String s) {
if (s.indexOf("\\") >= 0) {
final StringBuilder sb = new StringBuilder();
for (int i = 0; i < s.length(); i++) {
final char ch = s.charAt(i);
if (ch == '\\' && i < s.length() - 1) {
sb.append(s.charAt(++i));
} else {
sb.append(ch);
}
}
return sb.toString();
}
return s;
}
protected static class SynonymUpdater {
protected boolean isCommit = false;
protected File oldFile;
protected File newFile;
protected Writer writer;
protected SynonymItem item;
protected SynonymUpdater(final File file, final SynonymItem newItem) {
try {
newFile = File.createTempFile("synonym", ".txt");
writer = new BufferedWriter(new OutputStreamWriter(
new FileOutputStream(newFile), Constants.UTF_8));
} catch (final IOException e) {
if (newFile != null) {
newFile.delete();
}
throw new DictionaryException(
"Failed to write a synonym file.", e);
}
oldFile = file;
item = newItem;
}
public SynonymItem write(final SynonymItem oldItem) {
try {
if (item.getId() == oldItem.getId() && item.isUpdated()) {
if (item.equals(oldItem)) {
try {
if (!item.isDeleted()) {
// update
writer.write(item.toLineString());
writer.write(System.lineSeparator());
return new SynonymItem(item.getId(),
item.getNewInputs(),
item.getNewOutputs());
} else {
return null;
}
} finally {
item.setNewInputs(null);
item.setNewOutputs(null);
}
} else {
throw new DictionaryException(
"Synonym file was updated: old=" + oldItem
+ " : new=" + item);
}
} else {
writer.write(oldItem.toLineString());
writer.write(System.lineSeparator());
return oldItem;
}
} catch (final IOException e) {
throw new DictionaryException("Failed to write: " + oldItem
+ " -> " + item, e);
}
}
public void write(final String line) {
try {
writer.write(line);
writer.write(System.lineSeparator());
} catch (final IOException e) {
throw new DictionaryException("Failed to write: " + line, e);
}
}
public void commit() {
isCommit = true;
}
public void close() {
try {
writer.flush();
} catch (final IOException e) {
// ignore
}
IOUtils.closeQuietly(writer);
if (isCommit) {
try {
FileUtils.copyFile(newFile, oldFile);
newFile.delete();
} catch (final IOException e) {
throw new DictionaryException("Failed to replace "
+ oldFile.getAbsolutePath() + " with "
+ newFile.getAbsolutePath(), e);
}
} else {
newFile.delete();
}
}
}
}

38
src/main/java/jp/sf/fess/synonym/SynonymItem.java → src/main/java/jp/sf/fess/dic/synonym/SynonymItem.java
View file

@ -1,8 +1,12 @@
package jp.sf.fess.synonym;
package jp.sf.fess.dic.synonym;
import java.util.Arrays;
public class SynonymItem {
import jp.sf.fess.dic.DictionaryItem;
import org.apache.commons.lang3.StringUtils;
public class SynonymItem extends DictionaryItem {
private final String[] inputs;
private final String[] outputs;
@ -56,6 +60,10 @@ public class SynonymItem {
return newInputs != null && newOutputs != null;
}
public boolean isDeleted() {
return isUpdated() && newInputs.length == 0;
}
public void sort() {
if (inputs != null) {
Arrays.sort(inputs);
@ -103,4 +111,30 @@ public class SynonymItem {
return true;
}
@Override
public String toString() {
return "SynonymItem [id=" + id + ", inputs=" + Arrays.toString(inputs)
+ ", outputs=" + Arrays.toString(outputs) + ", newInputs="
+ Arrays.toString(newInputs) + ", newOutputs="
+ Arrays.toString(newOutputs) + "]";
}
public String toLineString() {
if (isUpdated()) {
if (Arrays.equals(newInputs, newOutputs)) {
return StringUtils.join(newInputs, ",");
} else {
return StringUtils.join(newInputs, ",") + "=>"
+ StringUtils.join(newOutputs, ",");
}
} else {
if (Arrays.equals(inputs, outputs)) {
return StringUtils.join(inputs, ",");
} else {
return StringUtils.join(inputs, ",") + "=>"
+ StringUtils.join(outputs, ",");
}
}
}
}

38
src/main/java/jp/sf/fess/dic/synonym/SynonymLocator.java Normal file
View file

@ -0,0 +1,38 @@
package jp.sf.fess.dic.synonym;
import java.io.File;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import jp.sf.fess.dic.DictionaryFile;
import jp.sf.fess.dic.DictionaryLocator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SynonymLocator extends DictionaryLocator {
private static final Logger logger = LoggerFactory
.getLogger(SynonymLocator.class);
public String synonymFilePrefix = "synonym";
public List<String> excludedSynonymList;
@Override
public Map<String, DictionaryFile> find() {
final Map<String, DictionaryFile> fileMap = new HashMap<String, DictionaryFile>();
for (final String path : searchPathList) {
final File[] files = findFiles(path, synonymFilePrefix,
excludedSynonymList);
for (final File file : files) {
if (logger.isInfoEnabled()) {
logger.info("Synonym File: " + file.getAbsolutePath());
}
fileMap.put(file.getAbsolutePath(), new SynonymFile(file));
}
}
return fileMap;
}
}

13
src/main/java/jp/sf/fess/synonym/SynonymException.java
View file

@ -1,13 +0,0 @@
package jp.sf.fess.synonym;
import jp.sf.fess.FessSystemException;
public class SynonymException extends FessSystemException {
private static final long serialVersionUID = 1L;
public SynonymException(final String message, final Throwable cause) {
super(message, cause);
}
}

144
src/main/java/jp/sf/fess/synonym/SynonymFile.java
View file

@ -1,144 +0,0 @@
package jp.sf.fess.synonym;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import jp.sf.fess.Constants;
import org.apache.commons.io.IOUtils;
public class SynonymFile {
private final File file;
final List<SynonymItem> synonymItemList = new ArrayList<SynonymItem>();
public SynonymFile(final File file) {
this.file = file;
}
public void clear() {
synchronized (synonymItemList) {
synonymItemList.clear();
}
}
public void load() {
BufferedReader reader = null;
synchronized (synonymItemList) {
try {
reader = new BufferedReader(new InputStreamReader(
new FileInputStream(file), Constants.UTF_8));
int id = 0;
String line = null;
while ((line = reader.readLine()) != null) {
if (line.length() == 0 || line.charAt(0) == '#') {
continue; // ignore empty lines and comments
}
String inputs[];
String outputs[];
final String sides[] = split(line, "=>");
if (sides.length > 1) { // explicit mapping
if (sides.length != 2) {
throw new IllegalArgumentException(
"more than one explicit mapping specified on the same line");
}
final String inputStrings[] = split(sides[0], ",");
inputs = new String[inputStrings.length];
for (int i = 0; i < inputs.length; i++) {
inputs[i] = unescape(inputStrings[i]).trim();
}
final String outputStrings[] = split(sides[1], ",");
outputs = new String[outputStrings.length];
for (int i = 0; i < outputs.length; i++) {
outputs[i] = unescape(outputStrings[i]).trim();
}
if (inputs.length > 0 && outputs.length > 0) {
final SynonymItem item = new SynonymItem(id,
inputs, inputs);
id++;
synonymItemList.add(item);
}
} else {
final String inputStrings[] = split(line, ",");
inputs = new String[inputStrings.length];
for (int i = 0; i < inputs.length; i++) {
inputs[i] = unescape(inputStrings[i]).trim();
}
if (inputs.length > 0) {
final SynonymItem item = new SynonymItem(id,
inputs, inputs);
id++;
synonymItemList.add(item);
}
}
}
} catch (final IOException e) {
throw new SynonymException("Failed to parse "
+ file.getAbsolutePath(), e);
} finally {
IOUtils.closeQuietly(reader);
}
}
}
private static String[] split(final String s, final String separator) {
final List<String> list = new ArrayList<String>(2);
StringBuilder sb = new StringBuilder();
int pos = 0;
final int end = s.length();
while (pos < end) {
if (s.startsWith(separator, pos)) {
if (sb.length() > 0) {
list.add(sb.toString());
sb = new StringBuilder();
}
pos += separator.length();
continue;
}
char ch = s.charAt(pos++);
if (ch == '\\') {
sb.append(ch);
if (pos >= end) {
break; // ERROR, or let it go?
}
ch = s.charAt(pos++);
}
sb.append(ch);
}
if (sb.length() > 0) {
list.add(sb.toString());
}
return list.toArray(new String[list.size()]);
}
private String unescape(final String s) {
if (s.indexOf("\\") >= 0) {
final StringBuilder sb = new StringBuilder();
for (int i = 0; i < s.length(); i++) {
final char ch = s.charAt(i);
if (ch == '\\' && i < s.length() - 1) {
sb.append(s.charAt(++i));
} else {
sb.append(ch);
}
}
return sb.toString();
}
return s;
}
}

136
src/main/java/jp/sf/fess/synonym/SynonymManager.java
View file

@ -1,136 +0,0 @@
package jp.sf.fess.synonym;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import jp.sf.fess.util.ResourceUtil;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.AbstractFileFilter;
import org.seasar.extension.timer.TimeoutManager;
import org.seasar.extension.timer.TimeoutTarget;
import org.seasar.extension.timer.TimeoutTask;
import org.seasar.framework.container.annotation.tiger.DestroyMethod;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SynonymManager {
private static final Logger logger = LoggerFactory
.getLogger(SynonymManager.class);
protected List<String> searchPathList = new ArrayList<String>();
protected Map<String, SynonymFile> synonymFileMap;
public String synonymFilePrefix = "synonym";
// public String[] excludedSynonymDirs = new String[] { "data", "txlog",
// "lib", "bin", "contrib" };
public Set<String> excludedSynonymSet;
public long keepAlive = 5 * 60 * 1000; // 5min
public int watcherTimeout = 60; // 1min
protected volatile long lifetime = 0;
protected TimeoutTask watcherTargetTask;
@InitMethod
public void init() {
// start
final WatcherTarget watcherTarget = new WatcherTarget();
watcherTargetTask = TimeoutManager.getInstance().addTimeoutTarget(
watcherTarget, watcherTimeout, true);
}
@DestroyMethod
public void destroy() {
if (watcherTargetTask != null && !watcherTargetTask.isStopped()) {
watcherTargetTask.stop();
}
}
public SynonymFile[] getSynonymFiles() {
final Map<String, SynonymFile> fileMap = getSynonymFileMap();
final Collection<SynonymFile> values = fileMap.values();
return values.toArray(new SynonymFile[values.size()]);
}
public SynonymFile getSynonymFile(final String uri) {
final Map<String, SynonymFile> fileMap = getSynonymFileMap();
return fileMap.get(uri);
}
protected Map<String, SynonymFile> getSynonymFileMap() {
synchronized (this) {
if (lifetime > System.currentTimeMillis() && synonymFileMap != null) {
lifetime = System.currentTimeMillis() + keepAlive;
return synonymFileMap;
}
final Map<String, SynonymFile> newFileMap = new LinkedHashMap<String, SynonymFile>();
for (final String path : searchPathList) {
final File[] files = findFiles(path);
for (final File file : files) {
if (logger.isInfoEnabled()) {
logger.info("Synonym File: " + file.getAbsolutePath());
}
newFileMap.put(file.getAbsolutePath(),
new SynonymFile(file));
}
}
synonymFileMap = newFileMap;
lifetime = System.currentTimeMillis() + keepAlive;
return synonymFileMap;
}
}
protected File[] findFiles(final String path) {
final Collection<File> files = FileUtils.listFiles(new File(path),
new AbstractFileFilter() {
@Override
public boolean accept(final File dir, final String name) {
return name.startsWith(synonymFilePrefix);
}
}, new AbstractFileFilter() {
@Override
public boolean accept(final File dir, final String name) {
return excludedSynonymSet == null
|| !excludedSynonymSet.contains(name);
}
});
return files.toArray(new File[files.size()]);
}
public void addSearchPath(final String path) {
searchPathList.add(ResourceUtil.resolve(path));
}
protected class WatcherTarget implements TimeoutTarget {
@Override
public void expired() {
synchronized (SynonymManager.this) {
if (lifetime <= System.currentTimeMillis()
&& synonymFileMap != null) {
if (logger.isDebugEnabled()) {
logger.debug("Cleaning synonym files: "
+ synonymFileMap);
}
synonymFileMap = null;
}
}
}
}
}

1
src/main/resources/app.dicon
View file

@ -10,6 +10,7 @@
<include path="fess_suggest.dicon"/>
<include path="fess_job.dicon"/>
<include path="fess_api.dicon"/>
<include path="fess_dic.dicon"/>
<include path="mobylet.dicon"/>

19
src/main/resources/fess_dic.dicon Normal file
View file

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//SEASAR//DTD S2Container 2.4//EN"
"http://www.seasar.org/dtd/components24.dtd">
<components>
<component name="synonymManager" class="jp.sf.fess.dic.DictionaryManager">
<initMethod name="addLocator">
<arg>synonymLocator</arg>
</initMethod>
</component>
<component name="synonymLocator" class="jp.sf.fess.dic.synonym.SynonymLocator">
<property name="excludedSynonymList">{"data", "txlog",
"lib", "bin", "contrib"}</property>
<initMethod name="addSearchPath">
<arg>"${solr.solr.home}"</arg>
</initMethod>
</component>
</components>

11
src/main/resources/fess_synonym.dicon
View file

@ -1,11 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//SEASAR//DTD S2Container 2.4//EN"
"http://www.seasar.org/dtd/components24.dtd">
<components>
<component name="synonymManager" class="jp.sf.fess.synonym.SynonymManager">
<initMethod name="addSearchPath">
<arg>"${solr.solr.home}"</arg>
</initMethod>
</component>
</components>

75
src/test/java/jp/sf/fess/dic/DictionaryManagerTest.java Normal file
View file

@ -0,0 +1,75 @@
/*
* Copyright 2009-2013 the Fess Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package jp.sf.fess.dic;
import java.io.File;
import java.util.HashSet;
import jp.sf.fess.Constants;
import jp.sf.fess.dic.synonym.SynonymLocator;
import org.apache.commons.io.FileUtils;
import org.seasar.extension.unit.S2TestCase;
import org.seasar.framework.util.FileUtil;
public class DictionaryManagerTest extends S2TestCase {
private File testDir;
private File synonymFile1;
@Override
protected void setUp() throws Exception {
testDir = File.createTempFile("synonymtest", "_dir");
testDir.delete();
testDir.mkdirs();
synonymFile1 = new File(testDir, "synonym.txt");
FileUtil.write(synonymFile1.getAbsolutePath(),
"abc=>123\nxyz,890".getBytes(Constants.UTF_8));
}
@Override
protected void tearDown() throws Exception {
FileUtils.deleteDirectory(testDir);
}
public void test_getSynonymFiles() throws Exception {
final DictionaryManager dictionaryManager = new DictionaryManager();
dictionaryManager.keepAlive = 1000;
dictionaryManager.watcherTimeout = 1;
final SynonymLocator synonymLocator = new SynonymLocator();
synonymLocator.excludedSynonymSet = new HashSet<String>();
synonymLocator.excludedSynonymSet.add("data");
synonymLocator.addSearchPath(testDir.getAbsolutePath());
dictionaryManager.init();
final DictionaryFile[] synonymFiles = dictionaryManager
.getDictionaryFiles();
assertEquals(1, synonymFiles.length);
assertNotNull(dictionaryManager.dicFileMap);
Thread.sleep(2000);
assertNull(dictionaryManager.dicFileMap);
final DictionaryFile[] synonymFiles2 = dictionaryManager
.getDictionaryFiles();
assertEquals(1, synonymFiles2.length);
assertNotNull(dictionaryManager.dicFileMap);
Thread.sleep(2000);
assertNull(dictionaryManager.dicFileMap);
}
}

199
src/test/java/jp/sf/fess/dic/synonym/SynonymFileTest.java Normal file
View file

@ -0,0 +1,199 @@
/*
* Copyright 2009-2013 the Fess Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package jp.sf.fess.dic.synonym;
import java.io.File;
import jp.sf.fess.Constants;
import jp.sf.fess.dic.DictionaryFile.PagingList;
import jp.sf.fess.dic.DictionaryItem;
import org.seasar.extension.unit.S2TestCase;
import org.seasar.framework.util.FileUtil;
public class SynonymFileTest extends S2TestCase {
private File file1;
@Override
protected void setUp() throws Exception {
file1 = File.createTempFile("synonym", ".txt");
FileUtil.write(file1.getAbsolutePath(),
"a1=>A1\nb1,b2 => B1\nc1 => C1, C2\nx1,X1\ny1, Y1, y2"
.getBytes(Constants.UTF_8));
}
@Override
protected void tearDown() throws Exception {
file1.delete();
}
public void test_selectList() {
final SynonymFile synonymFile = new SynonymFile(file1);
final PagingList<DictionaryItem> itemList1 = synonymFile.selectList(0,
20);
assertEquals(5, itemList1.size());
assertEquals(5, itemList1.getAllRecordCount());
assertEquals(1, itemList1.getCurrentPageNumber());
assertEquals(20, itemList1.getPageSize());
final PagingList<DictionaryItem> itemList2 = synonymFile.selectList(4,
2);
assertEquals(1, itemList2.size());
assertEquals(5, itemList2.getAllRecordCount());
assertEquals(3, itemList2.getCurrentPageNumber());
assertEquals(2, itemList2.getPageSize());
assertEquals(0, synonymFile.selectList(5, 5).size());
assertEquals(0, synonymFile.selectList(-1, 5).size());
}
public void test_selectList2() {
final SynonymFile synonymFile = new SynonymFile(file1);
final PagingList<DictionaryItem> itemList = synonymFile
.selectList(0, 5);
assertEquals(1, ((SynonymItem) itemList.get(0)).getInputs().length);
assertEquals(1, ((SynonymItem) itemList.get(0)).getOutputs().length);
assertEquals("a1", ((SynonymItem) itemList.get(0)).getInputs()[0]);
assertEquals("A1", ((SynonymItem) itemList.get(0)).getOutputs()[0]);
assertFalse(((SynonymItem) itemList.get(0)).isUpdated());
assertEquals(2, ((SynonymItem) itemList.get(1)).getInputs().length);
assertEquals(1, ((SynonymItem) itemList.get(1)).getOutputs().length);
assertEquals("b1", ((SynonymItem) itemList.get(1)).getInputs()[0]);
assertEquals("b2", ((SynonymItem) itemList.get(1)).getInputs()[1]);
assertEquals("B1", ((SynonymItem) itemList.get(1)).getOutputs()[0]);
assertFalse(((SynonymItem) itemList.get(1)).isUpdated());
assertEquals(1, ((SynonymItem) itemList.get(2)).getInputs().length);
assertEquals(2, ((SynonymItem) itemList.get(2)).getOutputs().length);
assertEquals("c1", ((SynonymItem) itemList.get(2)).getInputs()[0]);
assertEquals("C1", ((SynonymItem) itemList.get(2)).getOutputs()[0]);
assertEquals("C2", ((SynonymItem) itemList.get(2)).getOutputs()[1]);
assertFalse(((SynonymItem) itemList.get(2)).isUpdated());
assertEquals(2, ((SynonymItem) itemList.get(3)).getInputs().length);
assertEquals(2, ((SynonymItem) itemList.get(3)).getOutputs().length);
assertEquals("X1", ((SynonymItem) itemList.get(3)).getInputs()[0]);
assertEquals("x1", ((SynonymItem) itemList.get(3)).getInputs()[1]);
assertEquals("X1", ((SynonymItem) itemList.get(3)).getOutputs()[0]);
assertEquals("x1", ((SynonymItem) itemList.get(3)).getOutputs()[1]);
assertFalse(((SynonymItem) itemList.get(3)).isUpdated());
assertEquals(3, ((SynonymItem) itemList.get(4)).getInputs().length);
assertEquals(3, ((SynonymItem) itemList.get(4)).getOutputs().length);
assertEquals("Y1", ((SynonymItem) itemList.get(4)).getInputs()[0]);
assertEquals("y1", ((SynonymItem) itemList.get(4)).getInputs()[1]);
assertEquals("y2", ((SynonymItem) itemList.get(4)).getInputs()[2]);
assertEquals("Y1", ((SynonymItem) itemList.get(4)).getOutputs()[0]);
assertEquals("y1", ((SynonymItem) itemList.get(4)).getOutputs()[1]);
assertEquals("y2", ((SynonymItem) itemList.get(4)).getOutputs()[2]);
assertFalse(((SynonymItem) itemList.get(4)).isUpdated());
}
public void test_insert() {
final SynonymFile synonymFile = new SynonymFile(file1);
final PagingList<DictionaryItem> itemList1 = synonymFile.selectList(0,
20);
assertEquals(5, itemList1.size());
final SynonymItem synonymItem1 = new SynonymItem(0, new String[] {
"z1", "z2" }, new String[] { "Z1", "Z2" });
synonymFile.insert(synonymItem1);
final PagingList<DictionaryItem> itemList2 = synonymFile.selectList(0,
20);
assertEquals(6, itemList2.size());
assertEquals("z1", ((SynonymItem) itemList2.get(5)).getInputs()[0]);
assertEquals("z2", ((SynonymItem) itemList2.get(5)).getInputs()[1]);
assertEquals("Z1", ((SynonymItem) itemList2.get(5)).getOutputs()[0]);
assertEquals("Z2", ((SynonymItem) itemList2.get(5)).getOutputs()[1]);
final SynonymItem synonymItem2 = new SynonymItem(0, new String[] {
"z1", "z2" }, new String[] { "z1", "z2" });
synonymFile.insert(synonymItem2);
final PagingList<DictionaryItem> itemList3 = synonymFile.selectList(0,
20);
assertEquals(7, itemList3.size());
assertEquals("z1", ((SynonymItem) itemList3.get(6)).getInputs()[0]);
assertEquals("z2", ((SynonymItem) itemList3.get(6)).getInputs()[1]);
assertEquals("z1", ((SynonymItem) itemList3.get(6)).getOutputs()[0]);
assertEquals("z2", ((SynonymItem) itemList3.get(6)).getOutputs()[1]);
}
public void test_update() {
final SynonymFile synonymFile = new SynonymFile(file1);
final PagingList<DictionaryItem> itemList1 = synonymFile.selectList(0,
20);
assertEquals(5, itemList1.size());
final SynonymItem synonymItem1 = (SynonymItem) itemList1.get(0);
synonymItem1.setNewInputs(new String[] { "a1", "a2" });
synonymItem1.setNewOutputs(new String[] { "A1", "A2" });
synonymFile.update(synonymItem1);
final PagingList<DictionaryItem> itemList2 = synonymFile.selectList(0,
20);
assertEquals(5, itemList2.size());
final SynonymItem synonymItem2 = (SynonymItem) itemList2.get(0);
assertEquals(2, synonymItem2.getInputs().length);
assertEquals(2, synonymItem2.getOutputs().length);
assertEquals("a1", synonymItem2.getInputs()[0]);
assertEquals("a2", synonymItem2.getInputs()[1]);
assertEquals("A1", synonymItem2.getOutputs()[0]);
assertEquals("A2", synonymItem2.getOutputs()[1]);
assertFalse(synonymItem2.isUpdated());
final SynonymItem synonymItem3 = (SynonymItem) itemList2.get(2);
synonymItem3.setNewInputs(new String[] { "c1", "c2" });
synonymItem3.setNewOutputs(new String[] { "c1", "c2" });
synonymFile.update(synonymItem3);
final PagingList<DictionaryItem> itemList3 = synonymFile.selectList(0,
20);
assertEquals(5, itemList3.size());
final SynonymItem synonymItem4 = (SynonymItem) itemList3.get(2);
assertEquals(2, synonymItem4.getInputs().length);
assertEquals(2, synonymItem4.getOutputs().length);
assertEquals("c1", synonymItem4.getInputs()[0]);
assertEquals("c2", synonymItem4.getInputs()[1]);
assertEquals("c1", synonymItem4.getOutputs()[0]);
assertEquals("c2", synonymItem4.getOutputs()[1]);
assertFalse(synonymItem2.isUpdated());
}
public void test_delete() throws Exception {
final SynonymFile synonymFile = new SynonymFile(file1);
final PagingList<DictionaryItem> itemList1 = synonymFile.selectList(0,
20);
assertEquals(5, itemList1.size());
final SynonymItem synonymItem1 = (SynonymItem) itemList1.get(0);
synonymFile.delete(synonymItem1);
final PagingList<DictionaryItem> itemList2 = synonymFile.selectList(0,
20);
assertEquals(4, itemList2.size());
final SynonymItem synonymItem2 = (SynonymItem) itemList2.get(3);
synonymFile.delete(synonymItem2);
final PagingList<DictionaryItem> itemList3 = synonymFile.selectList(0,
20);
assertEquals(3, itemList3.size());
assertEquals("b1,b2=>B1" + System.lineSeparator() + "c1=>C1,C2"
+ System.lineSeparator() + "X1,x1" + System.lineSeparator(),
new String(FileUtil.getBytes(file1), Constants.UTF_8));
}
}

120
src/test/java/jp/sf/fess/dic/synonym/SynonymItemTest.java Normal file
View file

@ -0,0 +1,120 @@
/*
* Copyright 2009-2013 the Fess Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package jp.sf.fess.dic.synonym;
import org.seasar.extension.unit.S2TestCase;
public class SynonymItemTest extends S2TestCase {
public void test_new1() {
final SynonymItem synonymItem = new SynonymItem(1, new String[] { "a",
"A" }, new String[] { "b", "B" });
assertEquals(1, synonymItem.getId());
assertEquals(2, synonymItem.getInputs().length);
assertEquals("A", synonymItem.getInputs()[0]);
assertEquals("a", synonymItem.getInputs()[1]);
assertEquals(2, synonymItem.getOutputs().length);
assertEquals("B", synonymItem.getOutputs()[0]);
assertEquals("b", synonymItem.getOutputs()[1]);
assertNull(synonymItem.getNewInputs());
assertNull(synonymItem.getNewOutputs());
assertFalse(synonymItem.isUpdated());
assertFalse(synonymItem.isDeleted());
synonymItem.setNewInputs(new String[] { "1", "2" });
synonymItem.setNewOutputs(new String[] { "3", "4" });
assertTrue(synonymItem.isUpdated());
assertFalse(synonymItem.isDeleted());
synonymItem.setNewInputs(new String[0]);
synonymItem.setNewOutputs(new String[0]);
assertTrue(synonymItem.isUpdated());
assertTrue(synonymItem.isDeleted());
}
public void test_new2() {
final SynonymItem synonymItem = new SynonymItem(1, new String[] { "A",
"a" }, new String[] { "B", "b" });
assertEquals(1, synonymItem.getId());
assertEquals(2, synonymItem.getInputs().length);
assertEquals("A", synonymItem.getInputs()[0]);
assertEquals("a", synonymItem.getInputs()[1]);
assertEquals(2, synonymItem.getOutputs().length);
assertEquals("B", synonymItem.getOutputs()[0]);
assertEquals("b", synonymItem.getOutputs()[1]);
assertNull(synonymItem.getNewInputs());
assertNull(synonymItem.getNewOutputs());
assertFalse(synonymItem.isUpdated());
assertFalse(synonymItem.isDeleted());
synonymItem.setNewInputs(new String[] { "2", "1" });
synonymItem.setNewOutputs(new String[] { "4", "3" });
assertTrue(synonymItem.isUpdated());
assertFalse(synonymItem.isDeleted());
synonymItem.setNewInputs(new String[0]);
synonymItem.setNewOutputs(new String[0]);
assertTrue(synonymItem.isUpdated());
assertTrue(synonymItem.isDeleted());
}
public void test_equals1() {
final SynonymItem synonymItem1 = new SynonymItem(1, new String[] { "a",
"A" }, new String[] { "b", "B" });
assertTrue(synonymItem1.equals(synonymItem1));
assertTrue(synonymItem1.equals(new SynonymItem(1, new String[] { "A",
"a" }, new String[] { "B", "b" })));
assertTrue(synonymItem1.equals(new SynonymItem(2, new String[] { "A",
"a" }, new String[] { "B", "b" })));
assertFalse(synonymItem1.equals(new SynonymItem(2, new String[] { "A",
"a" }, new String[] { "B", })));
assertFalse(synonymItem1.equals(new SynonymItem(2,
new String[] { "A" }, new String[] { "B", "b" })));
assertFalse(synonymItem1.equals(new SynonymItem(1, new String[] { "A",
"a" }, new String[] { "B", "c" })));
assertFalse(synonymItem1.equals(new SynonymItem(1, new String[] { "A",
"c" }, new String[] { "B", "b" })));
}
public void test_equals2() {
final SynonymItem synonymItem1 = new SynonymItem(1,
new String[] { "a" }, new String[] { "b" });
assertTrue(synonymItem1.equals(synonymItem1));
assertTrue(synonymItem1.equals(new SynonymItem(1, new String[] { "a" },
new String[] { "b" })));
assertFalse(synonymItem1.equals(new SynonymItem(2,
new String[] { "a" }, new String[] { "B", })));
assertFalse(synonymItem1.equals(new SynonymItem(2,
new String[] { "A" }, new String[] { "b" })));
}
public void test_toLineString() {
assertEquals("a1,a2,a3=>b1,b2,b3",
new SynonymItem(1, new String[] { "a1", "a2", "a3" },
new String[] { "b1", "b2", "b3" }).toLineString());
assertEquals("a=>b", new SynonymItem(1, new String[] { "a" },
new String[] { "b" }).toLineString());
assertEquals("A,a=>B,b", new SynonymItem(1, new String[] { "a", "A" },
new String[] { "b", "B" }).toLineString());
assertEquals("A,a", new SynonymItem(1, new String[] { "a", "A" },
new String[] { "a", "A" }).toLineString());
assertEquals("a", new SynonymItem(1, new String[] { "a" },
new String[] { "a" }).toLineString());
}
}

50
src/test/java/jp/sf/fess/synonym/SynonymManagerTest.java → src/test/java/jp/sf/fess/dic/synonym/SynonymLocatorTest.java
View file

@ -14,18 +14,20 @@
* governing permissions and limitations under the License.
*/
package jp.sf.fess.synonym;
package jp.sf.fess.dic.synonym;
import java.io.File;
import java.util.HashSet;
import java.util.Map;
import jp.sf.fess.Constants;
import jp.sf.fess.dic.DictionaryFile;
import org.apache.commons.io.FileUtils;
import org.seasar.extension.unit.S2TestCase;
import org.seasar.framework.util.FileUtil;
public class SynonymManagerTest extends S2TestCase {
public class SynonymLocatorTest extends S2TestCase {
private File testDir;
@ -75,37 +77,19 @@ public class SynonymManagerTest extends S2TestCase {
FileUtils.deleteDirectory(testDir);
}
public void test_findFiles() {
final SynonymManager synonymManager = new SynonymManager();
synonymManager.excludedSynonymSet = new HashSet<String>();
synonymManager.excludedSynonymSet.add("data");
final File[] files = synonymManager
.findFiles(testDir.getAbsolutePath());
assertEquals(2, files.length);
assertEquals(synonymFile1.getAbsolutePath(), files[0].getAbsolutePath());
assertEquals(synonymFile3.getAbsolutePath(), files[1].getAbsolutePath());
public void test_find() {
final SynonymLocator synonymLocator = new SynonymLocator();
synonymLocator.excludedSynonymSet = new HashSet<String>();
synonymLocator.excludedSynonymSet.add("data");
synonymLocator.addSearchPath(testDir.getAbsolutePath());
final Map<String, DictionaryFile> fileMap = synonymLocator.find();
assertEquals(2, fileMap.size());
final DictionaryFile dicFile1 = fileMap.get(synonymFile1
.getAbsolutePath());
final DictionaryFile dicFile2 = fileMap.get(synonymFile3
.getAbsolutePath());
assertEquals(synonymFile1.getAbsolutePath(), dicFile1.getName());
assertEquals(synonymFile3.getAbsolutePath(), dicFile2.getName());
}
public void test_getSynonymFiles() throws Exception {
final SynonymManager synonymManager = new SynonymManager();
synonymManager.keepAlive = 1000;
synonymManager.watcherTimeout = 1;
synonymManager.excludedSynonymSet = new HashSet<String>();
synonymManager.excludedSynonymSet.add("data");
synonymManager.addSearchPath(testDir.getAbsolutePath());
synonymManager.init();
final SynonymFile[] synonymFiles = synonymManager.getSynonymFiles();
assertEquals(2, synonymFiles.length);
assertNotNull(synonymManager.synonymFileMap);
Thread.sleep(2000);
assertNull(synonymManager.synonymFileMap);
final SynonymFile[] synonymFiles2 = synonymManager.getSynonymFiles();
assertEquals(2, synonymFiles2.length);
assertNotNull(synonymManager.synonymFileMap);
Thread.sleep(2000);
assertNull(synonymManager.synonymFileMap);
}
}