diff --git a/.gitignore b/.gitignore index 96b1dbb06..e7d3759ae 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /src/main/webapp/WEB-INF/classes/ /src/main/webapp/WEB-INF/lib/ /src/main/webapp/WEB-INF/site/ +/src/main/webapp/WEB-INF/plugin/ /src/main/webapp/WEB-INF/env/crawler/lib/ /src/main/webapp/WEB-INF/env/suggest/lib/ /src/main/webapp/WEB-INF/env/thumbnail/lib/ diff --git a/deps.xml b/deps.xml index 6721beb9b..f7be6bea6 100644 --- a/deps.xml +++ b/deps.xml @@ -16,6 +16,8 @@ + + @@ -42,7 +44,7 @@ - + @@ -50,7 +52,7 @@ - + @@ -58,7 +60,7 @@ - + @@ -66,7 +68,7 @@ - + @@ -74,7 +76,7 @@ - + @@ -82,7 +84,7 @@ - + @@ -90,7 +92,7 @@ - + @@ -98,7 +100,7 @@ - + @@ -106,7 +108,7 @@ - + @@ -114,7 +116,7 @@ - + @@ -122,7 +124,7 @@ - + @@ -143,11 +145,11 @@ todir="${thumbnail.dir}/lib"/> - + + todir="${webinf.dir}/plugin"/> diff --git a/src/main/java/org/codelibs/fess/app/web/admin/dataconfig/AdminDataconfigAction.java b/src/main/java/org/codelibs/fess/app/web/admin/dataconfig/AdminDataconfigAction.java index 032e772de..f0fe61f9e 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/dataconfig/AdminDataconfigAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/dataconfig/AdminDataconfigAction.java @@ -323,9 +323,9 @@ public class AdminDataconfigAction extends FessAdminAction { } protected void registerHandlerNames(final RenderData data) { - final List dataStoreNameList = dataStoreFactory.getDataStoreNameList(); + final String[] dataStoreNames = dataStoreFactory.getDataStoreNames(); final List> itemList = new ArrayList<>(); - for (final String name : dataStoreNameList) { + for (final String name : dataStoreNames) { final Map map = new HashMap<>(); map.put(Constants.ITEM_LABEL, name); map.put(Constants.ITEM_VALUE, name); diff --git a/src/main/java/org/codelibs/fess/ds/DataStoreFactory.java b/src/main/java/org/codelibs/fess/ds/DataStoreFactory.java index e84555765..bb862908b 100644 --- a/src/main/java/org/codelibs/fess/ds/DataStoreFactory.java +++ b/src/main/java/org/codelibs/fess/ds/DataStoreFactory.java @@ -15,21 +15,43 @@ */ package org.codelibs.fess.ds; -import java.util.ArrayList; -import java.util.Collections; +import java.io.File; +import java.io.InputStream; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.codelibs.core.lang.StringUtil; +import org.codelibs.fess.helper.PluginHelper; +import org.codelibs.fess.util.ResourceUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; public class DataStoreFactory { private static final Logger logger = LoggerFactory.getLogger(DataStoreFactory.class); protected Map dataStoreMap = new LinkedHashMap<>(); + protected String[] dataStoreNames = StringUtil.EMPTY_STRINGS; + + protected long lastLoadedTime = 0; + public void add(final String name, final DataStore dataStore) { if (name == null || dataStore == null) { throw new IllegalArgumentException("name or dataStore is null."); @@ -37,19 +59,58 @@ public class DataStoreFactory { if (logger.isDebugEnabled()) { logger.debug("Loaded " + name); } - dataStoreMap.put(name, dataStore); + dataStoreMap.put(name.toLowerCase(Locale.ROOT), dataStore); + dataStoreMap.put(dataStore.getClass().getSimpleName().toLowerCase(Locale.ROOT), dataStore); } public DataStore getDataStore(final String name) { - return dataStoreMap.get(name); + if (name == null) { + return null; + } + return dataStoreMap.get(name.toLowerCase(Locale.ROOT)); } - public List getDataStoreNameList() { - final Set nameSet = dataStoreMap.keySet(); - final List nameList = new ArrayList<>(); - nameList.addAll(nameSet); - Collections.sort(nameList); - return nameList; + public String[] getDataStoreNames() { + if (System.currentTimeMillis() - lastLoadedTime > 60000L) { + final List nameList = loadDataStoreNameList(); + dataStoreNames = nameList.toArray(n -> new String[nameList.size()]); + } + return dataStoreNames; + } + + protected List loadDataStoreNameList() { + final Set nameSet = new HashSet<>(); + final File[] jarFiles = ResourceUtil.getPluginJarFiles(PluginHelper.ArtifactType.DATA_STORE.getId()); + for (final File jarFile : jarFiles) { + try (FileSystem fs = FileSystems.newFileSystem(jarFile.toPath(), ClassLoader.getSystemClassLoader())) { + final Path xmlPath = fs.getPath("fess_ds++.xml"); + try (InputStream is = Files.newInputStream(xmlPath)) { + final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + final DocumentBuilder builder = factory.newDocumentBuilder(); + + final Document doc = builder.parse(is); + final NodeList nodeList = doc.getElementsByTagName("component"); + for (int i = 0; i < nodeList.getLength(); i++) { + final Node node = nodeList.item(i); + final NamedNodeMap attributes = node.getAttributes(); + if (attributes != null) { + final Node classAttr = attributes.getNamedItem("class"); + if (classAttr != null) { + final String value = classAttr.getNodeValue(); + if (StringUtil.isNotBlank(value)) { + final String[] values = value.split("\\."); + nameSet.add(values[values.length - 1]); + } + } + } + } + } + } catch (final Exception e) { + logger.warn("Failed to load " + jarFile.getAbsolutePath(), e); + } + } + return nameSet.stream().sorted().collect(Collectors.toList()); } } diff --git a/src/main/java/org/codelibs/fess/exception/PluginException.java b/src/main/java/org/codelibs/fess/exception/PluginException.java new file mode 100644 index 000000000..41f796191 --- /dev/null +++ b/src/main/java/org/codelibs/fess/exception/PluginException.java @@ -0,0 +1,30 @@ +/* + * Copyright 2012-2019 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.exception; + +public class PluginException extends FessSystemException { + + private static final long serialVersionUID = 1L; + + public PluginException(String message, Throwable cause) { + super(message, cause); + } + + public PluginException(String message) { + super(message); + } + +} diff --git a/src/main/java/org/codelibs/fess/helper/PluginHelper.java b/src/main/java/org/codelibs/fess/helper/PluginHelper.java index e5c299063..c9935eda6 100644 --- a/src/main/java/org/codelibs/fess/helper/PluginHelper.java +++ b/src/main/java/org/codelibs/fess/helper/PluginHelper.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.StringReader; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -28,9 +29,12 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.servlet.ServletContext; + import org.codelibs.core.io.CopyUtil; import org.codelibs.curl.Curl; import org.codelibs.curl.CurlResponse; +import org.codelibs.fess.exception.PluginException; import org.codelibs.fess.util.ComponentUtil; import org.codelibs.fess.util.ResourceUtil; import org.codelibs.nekohtml.parsers.DOMParser; @@ -41,8 +45,6 @@ import org.w3c.dom.Document; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; -import javax.servlet.ServletContext; - // TODO: refactoring, exception handling, improving codes public class PluginHelper { private static final Logger logger = LoggerFactory.getLogger(PluginHelper.class); @@ -64,12 +66,12 @@ public class PluginHelper { protected List processRepository(final ArtifactType artifactType, final String url) { final List list = new ArrayList<>(); - final String repoContent = getRepositoryContentAsString(url); + final String repoContent = getRepositoryContent(url); final Matcher matcher = Pattern.compile("href=\"[^\"]*(" + artifactType.getId() + "[a-zA-Z0-9\\-]+)/?\"").matcher(repoContent); while (matcher.find()) { final String name = matcher.group(1); final String pluginUrl = url + (url.endsWith("/") ? name + "/" : "/" + name + "/"); - final String mavenMetadata = getRepositoryContentAsString(pluginUrl + "maven-metadata.xml"); + final String mavenMetadata = getRepositoryContent(pluginUrl + "maven-metadata.xml"); try (final StringReader reader = new StringReader(mavenMetadata)) { final DOMParser parser = new DOMParser(); parser.parse(new InputSource(reader)); @@ -86,7 +88,7 @@ public class PluginHelper { return list; } - protected String getRepositoryContentAsString(final String url) { + protected String getRepositoryContent(final String url) { try (final CurlResponse response = Curl.get(url).execute()) { return response.getContentAsString(); } catch (final IOException e) { @@ -94,16 +96,8 @@ public class PluginHelper { } } - protected InputStream getRepositoryContentAsStream(final String url) { - try (final CurlResponse response = Curl.get(url).execute()) { - return response.getContentAsStream(); - } catch (final IOException e) { - throw new IORuntimeException(e); - } - } - public Artifact[] getInstalledArtifacts(final ArtifactType artifactType) { - final File[] jarFiles = ResourceUtil.getJarFiles(artifactType.getId()); + final File[] jarFiles = ResourceUtil.getPluginJarFiles(artifactType.getId()); final List list = new ArrayList<>(jarFiles.length); for (final File file : jarFiles) { list.add(getArtifactFromFileName(artifactType, file.getName())); @@ -119,39 +113,32 @@ public class PluginHelper { return new Artifact(artifactName, artifactVersion, null); } - public boolean installArtifact(Artifact artifact) { - try (final InputStream in = getRepositoryContentAsStream(artifact.getUrl())) { - final String fileName = artifact.getName() + "-" + artifact.getVersion() + ".jar"; - final File file = Paths.get(getLibPath().toString(), fileName).toFile(); - if (!file.exists()) { - file.createNewFile(); - CopyUtil.copy(in, file); - } else { - final File tempFile = File.createTempFile(fileName, "bk"); - CopyUtil.copy(file, tempFile); - CopyUtil.copy(in, file); - tempFile.delete(); + public void installArtifact(Artifact artifact) { + final String fileName = artifact.getFileName(); + try (final CurlResponse response = Curl.get(artifact.getUrl()).execute()) { + try (final InputStream in = response.getContentAsStream()) { + CopyUtil.copy(in, ResourceUtil.getPluginPath(fileName).toFile()); } - return file.exists(); } catch (final Exception e) { - logger.warn("Failed to install the artifact " + artifact.getName(), e); - return false; + throw new PluginException("Failed to install the artifact " + artifact.getName(), e); } } - public boolean deleteInstalledArtifact(Artifact artifact) { - final String fileName = artifact.getName() + "-" + artifact.getVersion() + ".jar"; - final File file = Paths.get(getLibPath().toString(), fileName).toFile(); - if (file != null && file.exists() && file.delete()) { - return true; - } else { - logger.warn("Failed to delete the artifact " + file.getAbsolutePath()); - return false; + public void deleteInstalledArtifact(Artifact artifact) { + final String fileName = artifact.getFileName(); + final Path jarPath = Paths.get(getPluginPath().toString(), fileName); + if (!Files.exists(jarPath)) { + throw new PluginException(fileName + " does not exist."); + } + try { + Files.delete(jarPath); + } catch (IOException e) { + throw new PluginException("Failed to delete the artifact " + fileName, e); } } - protected Path getLibPath() { - return Paths.get(ComponentUtil.getComponent(ServletContext.class).getRealPath("/WEB-INF/lib/")); + protected Path getPluginPath() { + return Paths.get(ComponentUtil.getComponent(ServletContext.class).getRealPath("/WEB-INF/plugin")); } public static class Artifact { @@ -173,6 +160,10 @@ public class PluginHelper { return version; } + public String getFileName() { + return name + "-" + version + ".jar"; + } + public String getUrl() { return url; } diff --git a/src/main/java/org/codelibs/fess/job/CrawlJob.java b/src/main/java/org/codelibs/fess/job/CrawlJob.java index b0489cdac..db47f48d4 100644 --- a/src/main/java/org/codelibs/fess/job/CrawlJob.java +++ b/src/main/java/org/codelibs/fess/job/CrawlJob.java @@ -237,6 +237,9 @@ public class CrawlJob extends ExecJob { // WEB-INF/env/crawler/lib appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF" + File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator); + // WEB-INF/plugin + appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin" + + File.separator); final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib"); if (targetLibDir.isDirectory()) { appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator); diff --git a/src/main/java/org/codelibs/fess/job/GenerateThumbnailJob.java b/src/main/java/org/codelibs/fess/job/GenerateThumbnailJob.java index 280ae7b31..fc6c931ca 100644 --- a/src/main/java/org/codelibs/fess/job/GenerateThumbnailJob.java +++ b/src/main/java/org/codelibs/fess/job/GenerateThumbnailJob.java @@ -134,6 +134,9 @@ public class GenerateThumbnailJob extends ExecJob { // WEB-INF/env/thumbnail/lib appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF" + File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator); + // WEB-INF/plugin + appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin" + + File.separator); final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib"); if (targetLibDir.isDirectory()) { appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator); diff --git a/src/main/java/org/codelibs/fess/job/SuggestJob.java b/src/main/java/org/codelibs/fess/job/SuggestJob.java index 4bb3ab4f2..7af5101ca 100644 --- a/src/main/java/org/codelibs/fess/job/SuggestJob.java +++ b/src/main/java/org/codelibs/fess/job/SuggestJob.java @@ -121,6 +121,9 @@ public class SuggestJob extends ExecJob { // WEB-INF/env/suggest/lib appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF" + File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator); + // WEB-INF/plugin + appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin" + + File.separator); final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib"); if (targetLibDir.isDirectory()) { appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator); diff --git a/src/main/java/org/codelibs/fess/util/ResourceUtil.java b/src/main/java/org/codelibs/fess/util/ResourceUtil.java index 2658eb0a8..5a089f0a6 100644 --- a/src/main/java/org/codelibs/fess/util/ResourceUtil.java +++ b/src/main/java/org/codelibs/fess/util/ResourceUtil.java @@ -111,6 +111,10 @@ public class ResourceUtil { return getPath("site", names); } + public static Path getPluginPath(final String... names) { + return getPath("plutin", names); + } + public static Path getProjectPropertiesFile() { return getPath("", "project.properties"); } @@ -156,6 +160,19 @@ public class ResourceUtil { return libDir.listFiles((FilenameFilter) (file, name) -> name.startsWith(namePrefix)); } + public static File[] getPluginJarFiles(final String namePrefix) { + final ServletContext context = LaServletContextUtil.getServletContext(); + if (context == null) { + return new File[0]; + } + final String libPath = context.getRealPath("/WEB-INF/plugin"); + if (StringUtil.isBlank(libPath)) { + return new File[0]; + } + final File libDir = new File(libPath); + return libDir.listFiles((FilenameFilter) (file, name) -> name.startsWith(namePrefix)); + } + public static String resolve(final String value) { if (value == null) { return null;