0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #2205 move fess-ds to plugin

Browse source
This commit is contained in:
Shinsuke Sugaya 2019-08-15 09:49:24 +09:00
parent 5f621eaa78
commit dd40684477
10 changed files with 175 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -6,6 +6,7 @@
/src/main/webapp/WEB-INF/classes/
/src/main/webapp/WEB-INF/lib/
/src/main/webapp/WEB-INF/site/
/src/main/webapp/WEB-INF/plugin/
/src/main/webapp/WEB-INF/env/crawler/lib/
/src/main/webapp/WEB-INF/env/suggest/lib/
/src/main/webapp/WEB-INF/env/thumbnail/lib/

28
deps.xml
View file

@ -16,6 +16,8 @@
<mkdir dir="${target.dir}" />
<delete dir="${webinf.dir}/lib" />
<mkdir dir="${webinf.dir}/lib" />
<delete dir="${webinf.dir}/plugin" />
<mkdir dir="${webinf.dir}/plugin" />
<delete dir="${crawler.dir}/lib" />
<mkdir dir="${crawler.dir}/lib" />
<delete dir="${suggest.dir}/lib" />
@ -42,7 +44,7 @@
<cutdirsmapper dirs="2" />
</unzip>
<!-- fess-ds-atlassian -->
<antcall target="install.lib.jar">
<antcall target="install.plugin.jar">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="jar.groupId" value="org/codelibs/fess" />
<param name="jar.artifactId" value="fess-ds-atlassian" />
@ -50,7 +52,7 @@
<param name="file.version" value="13.2.0" />
</antcall>
<!-- fess-ds-box -->
<antcall target="install.lib.jar">
<antcall target="install.plugin.jar">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="jar.groupId" value="org/codelibs/fess" />
<param name="jar.artifactId" value="fess-ds-box" />
@ -58,7 +60,7 @@
<param name="file.version" value="13.2.0" />
</antcall>
<!-- fess-ds-csv -->
<antcall target="install.lib.jar">
<antcall target="install.plugin.jar">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="jar.groupId" value="org/codelibs/fess" />
<param name="jar.artifactId" value="fess-ds-csv" />
@ -66,7 +68,7 @@
<param name="file.version" value="13.2.0" />
</antcall>
<!-- fess-ds-db -->
<antcall target="install.lib.jar">
<antcall target="install.plugin.jar">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="jar.groupId" value="org/codelibs/fess" />
<param name="jar.artifactId" value="fess-ds-db" />
@ -74,7 +76,7 @@
<param name="file.version" value="13.2.0" />
</antcall>
<!-- fess-ds-dropbox -->
<antcall target="install.lib.jar">
<antcall target="install.plugin.jar">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="jar.groupId" value="org/codelibs/fess" />
<param name="jar.artifactId" value="fess-ds-dropbox" />
@ -82,7 +84,7 @@
<param name="file.version" value="13.2.0" />
</antcall>
<!-- fess-ds-elasticsearch -->
<antcall target="install.lib.jar">
<antcall target="install.plugin.jar">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="jar.groupId" value="org/codelibs/fess" />
<param name="jar.artifactId" value="fess-ds-elasticsearch" />
@ -90,7 +92,7 @@
<param name="file.version" value="13.2.0" />
</antcall>
<!-- fess-ds-gitbucket -->
<antcall target="install.lib.jar">
<antcall target="install.plugin.jar">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="jar.groupId" value="org/codelibs/fess" />
<param name="jar.artifactId" value="fess-ds-gitbucket" />
@ -98,7 +100,7 @@
<param name="file.version" value="13.2.0" />
</antcall>
<!-- fess-ds-gsuite -->
<antcall target="install.lib.jar">
<antcall target="install.plugin.jar">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="jar.groupId" value="org/codelibs/fess" />
<param name="jar.artifactId" value="fess-ds-gsuite" />
@ -106,7 +108,7 @@
<param name="file.version" value="13.2.0" />
</antcall>
<!-- fess-ds-json -->
<antcall target="install.lib.jar">
<antcall target="install.plugin.jar">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="jar.groupId" value="org/codelibs/fess" />
<param name="jar.artifactId" value="fess-ds-json" />
@ -114,7 +116,7 @@
<param name="file.version" value="13.2.0" />
</antcall>
<!-- fess-ds-office365 -->
<antcall target="install.lib.jar">
<antcall target="install.plugin.jar">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="jar.groupId" value="org/codelibs/fess" />
<param name="jar.artifactId" value="fess-ds-office365" />
@ -122,7 +124,7 @@
<param name="file.version" value="13.2.0" />
</antcall>
<!-- fess-ds-slack -->
<antcall target="install.lib.jar">
<antcall target="install.plugin.jar">
<param name="repo.url" value="${maven.release.repo.url}" />
<param name="jar.groupId" value="org/codelibs/fess" />
<param name="jar.artifactId" value="fess-ds-slack" />
@ -143,11 +145,11 @@
todir="${thumbnail.dir}/lib"/>
</target>
<target name="install.lib.jar">
<target name="install.plugin.jar">
<get dest="${target.dir}">
<url url="${repo.url}/${jar.groupId}/${jar.artifactId}/${jar.version}/${jar.artifactId}-${file.version}.jar" />
</get>
<copy file="${target.dir}/${jar.artifactId}-${file.version}.jar"
todir="${webinf.dir}/lib"/>
todir="${webinf.dir}/plugin"/>
</target>
</project>

4
src/main/java/org/codelibs/fess/app/web/admin/dataconfig/AdminDataconfigAction.java
View file

@ -323,9 +323,9 @@ public class AdminDataconfigAction extends FessAdminAction {
}
protected void registerHandlerNames(final RenderData data) {
final List<String> dataStoreNameList = dataStoreFactory.getDataStoreNameList();
final String[] dataStoreNames = dataStoreFactory.getDataStoreNames();
final List<Map<String, String>> itemList = new ArrayList<>();
for (final String name : dataStoreNameList) {
for (final String name : dataStoreNames) {
final Map<String, String> map = new HashMap<>();
map.put(Constants.ITEM_LABEL, name);
map.put(Constants.ITEM_VALUE, name);

81
src/main/java/org/codelibs/fess/ds/DataStoreFactory.java
View file

@ -15,21 +15,43 @@
*/
package org.codelibs.fess.ds;
import java.util.ArrayList;
import java.util.Collections;
import java.io.File;
import java.io.InputStream;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.helper.PluginHelper;
import org.codelibs.fess.util.ResourceUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
public class DataStoreFactory {
private static final Logger logger = LoggerFactory.getLogger(DataStoreFactory.class);
protected Map<String, DataStore> dataStoreMap = new LinkedHashMap<>();
protected String[] dataStoreNames = StringUtil.EMPTY_STRINGS;
protected long lastLoadedTime = 0;
public void add(final String name, final DataStore dataStore) {
if (name == null || dataStore == null) {
throw new IllegalArgumentException("name or dataStore is null.");
@ -37,19 +59,58 @@ public class DataStoreFactory {
if (logger.isDebugEnabled()) {
logger.debug("Loaded " + name);
}
dataStoreMap.put(name, dataStore);
dataStoreMap.put(name.toLowerCase(Locale.ROOT), dataStore);
dataStoreMap.put(dataStore.getClass().getSimpleName().toLowerCase(Locale.ROOT), dataStore);
}
public DataStore getDataStore(final String name) {
return dataStoreMap.get(name);
if (name == null) {
return null;
}
return dataStoreMap.get(name.toLowerCase(Locale.ROOT));
}
public List<String> getDataStoreNameList() {
final Set<String> nameSet = dataStoreMap.keySet();
final List<String> nameList = new ArrayList<>();
nameList.addAll(nameSet);
Collections.sort(nameList);
return nameList;
public String[] getDataStoreNames() {
if (System.currentTimeMillis() - lastLoadedTime > 60000L) {
final List<String> nameList = loadDataStoreNameList();
dataStoreNames = nameList.toArray(n -> new String[nameList.size()]);
}
return dataStoreNames;
}
protected List<String> loadDataStoreNameList() {
final Set<String> nameSet = new HashSet<>();
final File[] jarFiles = ResourceUtil.getPluginJarFiles(PluginHelper.ArtifactType.DATA_STORE.getId());
for (final File jarFile : jarFiles) {
try (FileSystem fs = FileSystems.newFileSystem(jarFile.toPath(), ClassLoader.getSystemClassLoader())) {
final Path xmlPath = fs.getPath("fess_ds++.xml");
try (InputStream is = Files.newInputStream(xmlPath)) {
final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
final DocumentBuilder builder = factory.newDocumentBuilder();
final Document doc = builder.parse(is);
final NodeList nodeList = doc.getElementsByTagName("component");
for (int i = 0; i < nodeList.getLength(); i++) {
final Node node = nodeList.item(i);
final NamedNodeMap attributes = node.getAttributes();
if (attributes != null) {
final Node classAttr = attributes.getNamedItem("class");
if (classAttr != null) {
final String value = classAttr.getNodeValue();
if (StringUtil.isNotBlank(value)) {
final String[] values = value.split("\\.");
nameSet.add(values[values.length - 1]);
}
}
}
}
}
} catch (final Exception e) {
logger.warn("Failed to load " + jarFile.getAbsolutePath(), e);
}
}
return nameSet.stream().sorted().collect(Collectors.toList());
}
}

30
src/main/java/org/codelibs/fess/exception/PluginException.java Normal file
View file

@ -0,0 +1,30 @@
/*
* Copyright 2012-2019 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.exception;
public class PluginException extends FessSystemException {
private static final long serialVersionUID = 1L;
public PluginException(String message, Throwable cause) {
super(message, cause);
}
public PluginException(String message) {
super(message);
}
}

69
src/main/java/org/codelibs/fess/helper/PluginHelper.java
View file

@ -21,6 +21,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
@ -28,9 +29,12 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.servlet.ServletContext;
import org.codelibs.core.io.CopyUtil;
import org.codelibs.curl.Curl;
import org.codelibs.curl.CurlResponse;
import org.codelibs.fess.exception.PluginException;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.ResourceUtil;
import org.codelibs.nekohtml.parsers.DOMParser;
@ -41,8 +45,6 @@ import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import javax.servlet.ServletContext;
// TODO: refactoring, exception handling, improving codes
public class PluginHelper {
private static final Logger logger = LoggerFactory.getLogger(PluginHelper.class);
@ -64,12 +66,12 @@ public class PluginHelper {
protected List<Artifact> processRepository(final ArtifactType artifactType, final String url) {
final List<Artifact> list = new ArrayList<>();
final String repoContent = getRepositoryContentAsString(url);
final String repoContent = getRepositoryContent(url);
final Matcher matcher = Pattern.compile("href=\"[^\"]*(" + artifactType.getId() + "[a-zA-Z0-9\\-]+)/?\"").matcher(repoContent);
while (matcher.find()) {
final String name = matcher.group(1);
final String pluginUrl = url + (url.endsWith("/") ? name + "/" : "/" + name + "/");
final String mavenMetadata = getRepositoryContentAsString(pluginUrl + "maven-metadata.xml");
final String mavenMetadata = getRepositoryContent(pluginUrl + "maven-metadata.xml");
try (final StringReader reader = new StringReader(mavenMetadata)) {
final DOMParser parser = new DOMParser();
parser.parse(new InputSource(reader));
@ -86,7 +88,7 @@ public class PluginHelper {
return list;
}
protected String getRepositoryContentAsString(final String url) {
protected String getRepositoryContent(final String url) {
try (final CurlResponse response = Curl.get(url).execute()) {
return response.getContentAsString();
} catch (final IOException e) {
@ -94,16 +96,8 @@ public class PluginHelper {
}
}
protected InputStream getRepositoryContentAsStream(final String url) {
try (final CurlResponse response = Curl.get(url).execute()) {
return response.getContentAsStream();
} catch (final IOException e) {
throw new IORuntimeException(e);
}
}
public Artifact[] getInstalledArtifacts(final ArtifactType artifactType) {
final File[] jarFiles = ResourceUtil.getJarFiles(artifactType.getId());
final File[] jarFiles = ResourceUtil.getPluginJarFiles(artifactType.getId());
final List<Artifact> list = new ArrayList<>(jarFiles.length);
for (final File file : jarFiles) {
list.add(getArtifactFromFileName(artifactType, file.getName()));
@ -119,39 +113,32 @@ public class PluginHelper {
return new Artifact(artifactName, artifactVersion, null);
}
public boolean installArtifact(Artifact artifact) {
try (final InputStream in = getRepositoryContentAsStream(artifact.getUrl())) {
final String fileName = artifact.getName() + "-" + artifact.getVersion() + ".jar";
final File file = Paths.get(getLibPath().toString(), fileName).toFile();
if (!file.exists()) {
file.createNewFile();
CopyUtil.copy(in, file);
} else {
final File tempFile = File.createTempFile(fileName, "bk");
CopyUtil.copy(file, tempFile);
CopyUtil.copy(in, file);
tempFile.delete();
public void installArtifact(Artifact artifact) {
final String fileName = artifact.getFileName();
try (final CurlResponse response = Curl.get(artifact.getUrl()).execute()) {
try (final InputStream in = response.getContentAsStream()) {
CopyUtil.copy(in, ResourceUtil.getPluginPath(fileName).toFile());
}
return file.exists();
} catch (final Exception e) {
logger.warn("Failed to install the artifact " + artifact.getName(), e);
return false;
throw new PluginException("Failed to install the artifact " + artifact.getName(), e);
}
}
public boolean deleteInstalledArtifact(Artifact artifact) {
final String fileName = artifact.getName() + "-" + artifact.getVersion() + ".jar";
final File file = Paths.get(getLibPath().toString(), fileName).toFile();
if (file != null && file.exists() && file.delete()) {
return true;
} else {
logger.warn("Failed to delete the artifact " + file.getAbsolutePath());
return false;
public void deleteInstalledArtifact(Artifact artifact) {
final String fileName = artifact.getFileName();
final Path jarPath = Paths.get(getPluginPath().toString(), fileName);
if (!Files.exists(jarPath)) {
throw new PluginException(fileName + " does not exist.");
}
try {
Files.delete(jarPath);
} catch (IOException e) {
throw new PluginException("Failed to delete the artifact " + fileName, e);
}
}
protected Path getLibPath() {
return Paths.get(ComponentUtil.getComponent(ServletContext.class).getRealPath("/WEB-INF/lib/"));
protected Path getPluginPath() {
return Paths.get(ComponentUtil.getComponent(ServletContext.class).getRealPath("/WEB-INF/plugin"));
}
public static class Artifact {
@ -173,6 +160,10 @@ public class PluginHelper {
return version;
}
public String getFileName() {
return name + "-" + version + ".jar";
}
public String getUrl() {
return url;
}

3
src/main/java/org/codelibs/fess/job/CrawlJob.java
View file

@ -237,6 +237,9 @@ public class CrawlJob extends ExecJob {
// WEB-INF/env/crawler/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF"
+ File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator);
// WEB-INF/plugin
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin"
+ File.separator);
final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
if (targetLibDir.isDirectory()) {
appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);

3
src/main/java/org/codelibs/fess/job/GenerateThumbnailJob.java
View file

@ -134,6 +134,9 @@ public class GenerateThumbnailJob extends ExecJob {
// WEB-INF/env/thumbnail/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF"
+ File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator);
// WEB-INF/plugin
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin"
+ File.separator);
final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
if (targetLibDir.isDirectory()) {
appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);

3
src/main/java/org/codelibs/fess/job/SuggestJob.java
View file

@ -121,6 +121,9 @@ public class SuggestJob extends ExecJob {
// WEB-INF/env/suggest/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF"
+ File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator);
// WEB-INF/plugin
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin"
+ File.separator);
final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
if (targetLibDir.isDirectory()) {
appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);

17
src/main/java/org/codelibs/fess/util/ResourceUtil.java
View file

@ -111,6 +111,10 @@ public class ResourceUtil {
return getPath("site", names);
}
public static Path getPluginPath(final String... names) {
return getPath("plutin", names);
}
public static Path getProjectPropertiesFile() {
return getPath("", "project.properties");
}
@ -156,6 +160,19 @@ public class ResourceUtil {
return libDir.listFiles((FilenameFilter) (file, name) -> name.startsWith(namePrefix));
}
public static File[] getPluginJarFiles(final String namePrefix) {
final ServletContext context = LaServletContextUtil.getServletContext();
if (context == null) {
return new File[0];
}
final String libPath = context.getRealPath("/WEB-INF/plugin");
if (StringUtil.isBlank(libPath)) {
return new File[0];
}
final File libDir = new File(libPath);
return libDir.listFiles((FilenameFilter) (file, name) -> name.startsWith(namePrefix));
}
public static String resolve(final String value) {
if (value == null) {
return null;