fix #2222 install jar by artifacts.yaml

src/main/java/org/codelibs/fess/es/config/exentity/FileConfig.java

@@ -28,9 +28,7 @@ import org.codelibs.fess.Constants;
 import org.codelibs.fess.app.service.FileAuthenticationService;
 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
 import org.codelibs.fess.crawler.client.ftp.FtpAuthentication;
-import org.codelibs.fess.crawler.client.ftp.FtpClient;
 import org.codelibs.fess.crawler.client.smb.SmbAuthentication;
-import org.codelibs.fess.crawler.client.smb.SmbClient;
 import org.codelibs.fess.es.config.bsentity.BsFileConfig;
 import org.codelibs.fess.helper.SystemHelper;
 import org.codelibs.fess.util.ComponentUtil;

src/main/java/org/codelibs/fess/helper/PluginHelper.java

@@ -25,15 +25,19 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import javax.xml.XMLConstants;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 
+import org.apache.commons.lang3.StringUtils;
 import org.codelibs.core.io.CopyUtil;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.curl.Curl;
@@ -50,6 +54,8 @@ import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
@@ -62,7 +68,13 @@ public class PluginHelper {
                 public Artifact[] load(ArtifactType key) {
                     final List<Artifact> list = new ArrayList<>();
                     for (final String url : getRepositories()) {
-                        list.addAll(processRepository(key, url));
+                        if (url.endsWith(".yaml")) {
+                            if (key == ArtifactType.UNKNOWN) {
+                                list.addAll(loadArtifactsFromRepository(url));
+                            }
+                        } else {
+                            list.addAll(processRepository(key, url));
+                        }
                     }
                     return list.toArray(new Artifact[list.size()]);
                 }
@@ -81,6 +93,22 @@ public class PluginHelper {
                 stream -> stream.map(String::trim).toArray(n -> new String[n]));
     }
 
+    protected List<Artifact> loadArtifactsFromRepository(final String url) {
+        final String content = getRepositoryContent(url);
+        ObjectMapper objectMapper = new YAMLMapper();
+        try {
+            @SuppressWarnings("unchecked")
+            List<Map<?, ?>> result = objectMapper.readValue(content, List.class);
+            if (result != null) {
+                return result.stream().map(o -> new Artifact((String) o.get("name"), (String) o.get("version"), (String) o.get("url")))
+                        .collect(Collectors.toList());
+            }
+            return Collections.emptyList();
+        } catch (Exception e) {
+            throw new PluginException("Failed to access " + url, e);
+        }
+    }
+
     protected List<Artifact> processRepository(final ArtifactType artifactType, final String url) {
         final List<Artifact> list = new ArrayList<>();
         final String repoContent = getRepositoryContent(url);
@@ -162,6 +190,23 @@ public class PluginHelper {
     }
 
     public Artifact[] getInstalledArtifacts(final ArtifactType artifactType) {
+        if (artifactType == ArtifactType.UNKNOWN) {
+            final File[] jarFiles = ResourceUtil.getPluginJarFiles((d, n) -> {
+                for (ArtifactType type : ArtifactType.values()) {
+                    if (n.startsWith(type.getId())) {
+                        return false;
+                    }
+                }
+                return true;
+            });
+            final List<Artifact> list = new ArrayList<>(jarFiles.length);
+            for (final File file : jarFiles) {
+                list.add(getArtifactFromFileName(artifactType, file.getName()));
+            }
+            list.sort((a, b) -> a.getName().compareTo(b.getName()));
+            return list.toArray(new Artifact[list.size()]);
+        }
+
         final File[] jarFiles = ResourceUtil.getPluginJarFiles(artifactType.getId());
         final List<Artifact> list = new ArrayList<>(jarFiles.length);
         for (final File file : jarFiles) {
@@ -171,12 +216,22 @@ public class PluginHelper {
         return list.toArray(new Artifact[list.size()]);
     }
 
-    protected Artifact getArtifactFromFileName(final ArtifactType artifactType, final String fileName) {
-        final String convertedFileName = fileName.substring(artifactType.getId().length() + 1, fileName.lastIndexOf('.'));
-        final int firstIndexOfDash = convertedFileName.indexOf('-');
-        final String artifactName = artifactType.getId() + "-" + convertedFileName.substring(0, firstIndexOfDash);
-        final String artifactVersion = convertedFileName.substring(firstIndexOfDash + 1);
-        return new Artifact(artifactName, artifactVersion);
+    protected Artifact getArtifactFromFileName(final ArtifactType artifactType, final String name) {
+        String baseName = StringUtils.removeEndIgnoreCase(name, ".jar");
+        List<String> nameList = new ArrayList<>();
+        List<String> versionList = new ArrayList<>();
+        boolean isName = true;
+        for (String value : baseName.split("-")) {
+            if (isName && value.length() > 0 && value.charAt(0) >= '0' && value.charAt(0) <= '9') {
+                isName = false;
+            }
+            if (isName) {
+                nameList.add(value);
+            } else {
+                versionList.add(value);
+            }
+        }
+        return new Artifact(nameList.stream().collect(Collectors.joining("-")), versionList.stream().collect(Collectors.joining("-")));
     }
 
     public void installArtifact(final Artifact artifact) {
@@ -282,7 +337,7 @@ public class PluginHelper {
     }
 
     public enum ArtifactType {
-        DATA_STORE("fess-ds"), THEME("fess-theme"), UNKNOWN("unknown");
+        DATA_STORE("fess-ds"), THEME("fess-theme"), UNKNOWN("jar");
 
         private final String id;
 

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -1430,7 +1430,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. homeDirectory */
     String LDAP_ATTR_HOME_DIRECTORY = "ldap.attr.homeDirectory";
 
-    /** The key of the configuration. e.g. https://repo.maven.apache.org/maven2/org/codelibs/fess/ */
+    /** The key of the configuration. e.g. https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://fess.codelibs.org/plugin/artifacts.yaml */
     String PLUGIN_REPOSITORIES = "plugin.repositories";
 
     /** The key of the configuration. e.g.  */
@@ -5939,7 +5939,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
 
     /**
      * Get the value for the key 'plugin.repositories'. <br>
-     * The value is, e.g. https://repo.maven.apache.org/maven2/org/codelibs/fess/ <br>
+     * The value is, e.g. https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://fess.codelibs.org/plugin/artifacts.yaml <br>
      * @return The value of found property. (NotNull: if not found, exception but basically no way)
      */
     String getPluginRepositories();
@@ -8774,7 +8774,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap.put(FessConfig.LDAP_ATTR_UID_NUMBER, "uidNumber");
             defaultMap.put(FessConfig.LDAP_ATTR_GID_NUMBER, "gidNumber");
             defaultMap.put(FessConfig.LDAP_ATTR_HOME_DIRECTORY, "homeDirectory");
-            defaultMap.put(FessConfig.PLUGIN_REPOSITORIES, "https://repo.maven.apache.org/maven2/org/codelibs/fess/");
+            defaultMap.put(FessConfig.PLUGIN_REPOSITORIES,
+                    "https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://fess.codelibs.org/plugin/artifacts.yaml");
             defaultMap.put(FessConfig.PLUGIN_VERSION_FILTER, "");
             defaultMap.put(FessConfig.lasta_di_SMART_DEPLOY_MODE, "hot");
             defaultMap.put(FessConfig.DEVELOPMENT_HERE, "true");

src/main/java/org/codelibs/fess/util/ResourceUtil.java

@@ -172,10 +172,14 @@ public class ResourceUtil {
         if (!libDir.exists()) {
             return new File[0];
         }
-        return libDir.listFiles((FilenameFilter) (file, name) -> name.startsWith(namePrefix));
+        return libDir.listFiles((file, name) -> name.startsWith(namePrefix));
     }
 
     public static File[] getPluginJarFiles(final String namePrefix) {
+        return getPluginJarFiles((file, name) -> name.startsWith(namePrefix));
+    }
+
+    public static File[] getPluginJarFiles(final FilenameFilter filter) {
         final ServletContext context = LaServletContextUtil.getServletContext();
         if (context == null) {
             return new File[0];
@@ -188,7 +192,7 @@ public class ResourceUtil {
         if (!libDir.exists()) {
             return new File[0];
         }
-        return libDir.listFiles((FilenameFilter) (file, name) -> name.startsWith(namePrefix));
+        return libDir.listFiles(filter);
     }
 
     public static String resolve(final String value) {

src/main/resources/fess_config.properties

@@ -705,6 +705,6 @@ ldap.attr.homeDirectory=homeDirectory
 # ----------------------------------------------------------
 #                                                  Scheduler
 #                                                     ------
-#plugin.repositories=https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/
-plugin.repositories=https://repo.maven.apache.org/maven2/org/codelibs/fess/
+#plugin.repositories=https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/,https://fess.codelibs.org/plugin/artifacts.yaml
+plugin.repositories=https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://fess.codelibs.org/plugin/artifacts.yaml
 plugin.version.filter=

src/test/java/org/codelibs/fess/helper/PluginHelperTest.java

@@ -53,6 +53,12 @@ public class PluginHelperTest extends UnitFessTestCase {
                     } catch (Exception e) {
                         return "";
                     }
+                } else if (url.contains("plugin/repo.yaml")) {
+                    try (InputStream is = ResourceUtil.getResourceAsStream(url)) {
+                        return new String(InputStreamUtil.getBytes(is), Constants.UTF_8);
+                    } catch (Exception e) {
+                        return "";
+                    }
                 }
                 throw new FessSystemException("unknown");
             }
@@ -90,4 +96,15 @@ public class PluginHelperTest extends UnitFessTestCase {
         assertEquals("fess-ds-atlassian", artifact.getName());
         assertEquals("13.2.1-20190708.212247-1", artifact.getVersion());
     }
+
+    public void test_getArtifactFromFileName3() {
+        Artifact artifact = pluginHelper.getArtifactFromFileName(ArtifactType.UNKNOWN, "mysql-connector-java-8.0.17.jar");
+        assertEquals("mysql-connector-java", artifact.getName());
+        assertEquals("8.0.17", artifact.getVersion());
+    }
+
+    public void test_loadYaml() {
+        List<Artifact> artifacts = pluginHelper.loadArtifactsFromRepository("plugin/repo.yaml");
+        assertEquals(2, artifacts.size());
+    }
 }

src/test/resources/plugin/repo.yaml

@@ -0,0 +1,6 @@
+- name: mysql-connector-java
+  version: 8.0.17
+  url: https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.17/mysql-connector-java-8.0.17.jar
+- name: mysql-connector-java
+  version: 6.0.6
+  url: https://repo1.maven.org/maven2/mysql/mysql-connector-java/6.0.6/mysql-connector-java-6.0.6.jar