#2203 add pluginhelper

src/main/java/org/codelibs/fess/helper/PluginHelper.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright 2012-2019 CodeLibs Project and the Others.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package org.codelibs.fess.helper;
+
+import static org.codelibs.core.stream.StreamUtil.split;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.codelibs.curl.Curl;
+import org.codelibs.curl.CurlResponse;
+import org.codelibs.fess.util.ComponentUtil;
+import org.lastaflute.di.exception.IORuntimeException;
+
+public class PluginHelper {
+
+    public Artifact[] getArtifacts(PluginType pluginType) {
+        List<Artifact> list = new ArrayList<>();
+        for (String url : getRepositories()) {
+            list.addAll(processRepository(pluginType, url));
+        }
+        return list.toArray(new Artifact[list.size()]);
+    }
+
+    protected String[] getRepositories() {
+        return split(ComponentUtil.getFessConfig().getPluginRepositories(), ",").get(
+                stream -> stream.map(s -> s.trim()).toArray(n -> new String[n]));
+    }
+
+    protected List<Artifact> processRepository(PluginType pluginType, String url) {
+        List<Artifact> list = new ArrayList<>();
+        String repoContent = getRepositoryContent(url);
+        Matcher matcher = Pattern.compile("href=\"[^\"]*(" + pluginType.getId() + "[a-zA-Z0-9\\-]+)/?\"").matcher(repoContent);
+        while (matcher.find()) {
+            String name = matcher.group(1);
+            // TODO parse maven-metadata.xml from url/name to set version and url
+            list.add(new Artifact(name, "TODO", "TODO"));
+        }
+        return list;
+    }
+
+    protected String getRepositoryContent(String url) {
+        try (CurlResponse response = Curl.get(url).execute()) {
+            return response.getContentAsString();
+        } catch (IOException e) {
+            throw new IORuntimeException(e);
+        }
+    }
+
+    public static class Artifact {
+        protected final String name;
+        protected final String version;
+        protected final String url;
+
+        public Artifact(String name, String version, String url) {
+            this.name = name;
+            this.version = version;
+            this.url = url;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public String getVersion() {
+            return version;
+        }
+
+        public String getUrl() {
+            return url;
+        }
+    }
+
+    public enum PluginType {
+        DATA_STORE("fess-ds");
+
+        private String id;
+
+        private PluginType(String id) {
+            this.id = id;
+        }
+
+        public String getId() {
+            return id;
+        }
+    }
+}

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -1427,6 +1427,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. homeDirectory */
     String LDAP_ATTR_HOME_DIRECTORY = "ldap.attr.homeDirectory";
 
+    /** The key of the configuration. e.g. https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/ */
+    String PLUGIN_REPOSITORIES = "plugin.repositories";
+
     /**
      * Get the value of property as {@link String}.
      * @param propertyKey The key of the property. (NotNull)
@@ -5921,6 +5924,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     String getLdapAttrHomeDirectory();
 
+    /**
+     * Get the value for the key 'plugin.repositories'. <br>
+     * The value is, e.g. https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/ <br>
+     * comment: ------
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getPluginRepositories();
+
     /**
      * The simple implementation for configuration.
      * @author FreeGen
@@ -8286,6 +8297,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return get(FessConfig.LDAP_ATTR_HOME_DIRECTORY);
         }
 
+        public String getPluginRepositories() {
+            return get(FessConfig.PLUGIN_REPOSITORIES);
+        }
+
         @Override
         protected java.util.Map<String, String> prepareGeneratedDefaultMap() {
             java.util.Map<String, String> defaultMap = super.prepareGeneratedDefaultMap();
@@ -8719,6 +8734,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap.put(FessConfig.LDAP_ATTR_UID_NUMBER, "uidNumber");
             defaultMap.put(FessConfig.LDAP_ATTR_GID_NUMBER, "gidNumber");
             defaultMap.put(FessConfig.LDAP_ATTR_HOME_DIRECTORY, "homeDirectory");
+            defaultMap
+                    .put(FessConfig.PLUGIN_REPOSITORIES,
+                            "https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/");
             defaultMap.put(FessConfig.lasta_di_SMART_DEPLOY_MODE, "hot");
             defaultMap.put(FessConfig.DEVELOPMENT_HERE, "true");
             defaultMap.put(FessConfig.ENVIRONMENT_TITLE, "Local Development");

src/main/resources/fess_config.properties

@@ -701,3 +701,7 @@ ldap.attr.uidNumber=uidNumber
 ldap.attr.gidNumber=gidNumber
 ldap.attr.homeDirectory=homeDirectory
 
+# ----------------------------------------------------------
+#                                                  Scheduler
+#                                                     ------
+plugin.repositories=https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/

src/test/java/org/codelibs/fess/helper/PluginHelperTest.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2012-2019 CodeLibs Project and the Others.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package org.codelibs.fess.helper;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+import org.codelibs.core.io.InputStreamUtil;
+import org.codelibs.core.io.ResourceUtil;
+import org.codelibs.fess.Constants;
+import org.codelibs.fess.exception.FessSystemException;
+import org.codelibs.fess.helper.PluginHelper.Artifact;
+import org.codelibs.fess.helper.PluginHelper.PluginType;
+import org.codelibs.fess.unit.UnitFessTestCase;
+import org.lastaflute.di.exception.IORuntimeException;
+
+public class PluginHelperTest extends UnitFessTestCase {
+    private PluginHelper pluginHelper;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        pluginHelper = new PluginHelper() {
+            protected String[] getRepositories() {
+                return new String[] { "plugin/repo1.html", "plugin/repo2.html" };
+            }
+
+            protected String getRepositoryContent(String url) {
+                if (url.contains("repo1")) {
+                    try (InputStream is = ResourceUtil.getResourceAsStream(url)) {
+                        return new String(InputStreamUtil.getBytes(is), Constants.UTF_8);
+                    } catch (IOException e) {
+                        throw new IORuntimeException(e);
+                    }
+                } else if (url.contains("repo2")) {
+                    try (InputStream is = ResourceUtil.getResourceAsStream(url)) {
+                        return new String(InputStreamUtil.getBytes(is), Constants.UTF_8);
+                    } catch (IOException e) {
+                        throw new IORuntimeException(e);
+                    }
+                }
+                throw new FessSystemException("unknown");
+            }
+        };
+    }
+
+    public void test_processRepository1() {
+        List<Artifact> list = pluginHelper.processRepository(PluginType.DATA_STORE, "plugin/repo1.html");
+        assertEquals(11, list.size());
+        assertEquals("fess-ds-atlassian", list.get(0).getName());
+    }
+
+    public void test_processRepository2() {
+        List<Artifact> list = pluginHelper.processRepository(PluginType.DATA_STORE, "plugin/repo2.html");
+        assertEquals(14, list.size());
+        assertEquals("fess-ds-atlassian", list.get(0).getName());
+    }
+}

src/test/resources/plugin/repo1.html

@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+	<title>Central Repository: org/codelibs/fess</title>
+	<meta name="viewport" content="width=device-width, initial-scale=1.0">
+	<style>
+body {
+	background: #fff;
+}
+	</style>
+</head>
+
+<body>
+	<header>
+		<h1>org/codelibs/fess</h1>
+	</header>
+	<hr/>
+	<main>
+		<pre id="contents">
+<a href="../">../</a>
+<a href="fess/" title="fess/">fess/</a>                                                            -         -      
+<a href="fess-crawler/" title="fess-crawler/">fess-crawler/</a>                                                    -         -      
+<a href="fess-crawler-db/" title="fess-crawler-db/">fess-crawler-db/</a>                                                 -         -      
+<a href="fess-crawler-db-h2/" title="fess-crawler-db-h2/">fess-crawler-db-h2/</a>                                              -         -      
+<a href="fess-crawler-db-mysql/" title="fess-crawler-db-mysql/">fess-crawler-db-mysql/</a>                                           -         -      
+<a href="fess-crawler-es/" title="fess-crawler-es/">fess-crawler-es/</a>                                                 -         -      
+<a href="fess-crawler-lasta/" title="fess-crawler-lasta/">fess-crawler-lasta/</a>                                              -         -      
+<a href="fess-crawler-parent/" title="fess-crawler-parent/">fess-crawler-parent/</a>                                             -         -      
+<a href="fess-crawler-webdriver/" title="fess-crawler-webdriver/">fess-crawler-webdriver/</a>                                          -         -      
+<a href="fess-ds-atlassian/" title="fess-ds-atlassian/">fess-ds-atlassian/</a>                                               -         -      
+<a href="fess-ds-box/" title="fess-ds-box/">fess-ds-box/</a>                                                     -         -      
+<a href="fess-ds-csv/" title="fess-ds-csv/">fess-ds-csv/</a>                                                     -         -      
+<a href="fess-ds-db/" title="fess-ds-db/">fess-ds-db/</a>                                                      -         -      
+<a href="fess-ds-dropbox/" title="fess-ds-dropbox/">fess-ds-dropbox/</a>                                                 -         -      
+<a href="fess-ds-elasticsearch/" title="fess-ds-elasticsearch/">fess-ds-elasticsearch/</a>                                           -         -      
+<a href="fess-ds-gitbucket/" title="fess-ds-gitbucket/">fess-ds-gitbucket/</a>                                               -         -      
+<a href="fess-ds-gsuite/" title="fess-ds-gsuite/">fess-ds-gsuite/</a>                                                  -         -      
+<a href="fess-ds-json/" title="fess-ds-json/">fess-ds-json/</a>                                                    -         -      
+<a href="fess-ds-office365/" title="fess-ds-office365/">fess-ds-office365/</a>                                               -         -      
+<a href="fess-ds-slack/" title="fess-ds-slack/">fess-ds-slack/</a>                                                   -         -      
+<a href="fess-suggest/" title="fess-suggest/">fess-suggest/</a>                                                    -         -      
+<a href="fess-xpack/" title="fess-xpack/">fess-xpack/</a>                                                      -         -      
+		</pre>
+	</main>
+	<hr/>
+</body>
+
+</html>

src/test/resources/plugin/repo2.html

@@ -0,0 +1,251 @@
+<html>
+  <head>
+    <title>Index of /repositories/snapshots/org/codelibs/fess</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+
+    <link rel="icon" type="image/png" href="https://oss.sonatype.org/favicon.png">
+    <!--[if IE]>
+    <link rel="SHORTCUT ICON" href="https://oss.sonatype.org/favicon.ico"/>
+    <![endif]-->
+
+    <link rel="stylesheet" href="https://oss.sonatype.org/static/css/Sonatype-content.css?2.14.13-01" type="text/css" media="screen" title="no title" charset="utf-8">
+  </head>
+  <body>
+    <h1>Index of /repositories/snapshots/org/codelibs/fess</h1>
+    <table cellspacing="10">
+      <tr>
+        <th align="left">Name</th>
+        <th>Last Modified</th>
+        <th>Size</th>
+        <th>Description</th>
+      </tr>
+      <tr>
+        <td><a href="../">Parent Directory</a></td>
+      </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-crawler-db-h2/">fess-crawler-db-h2/</a></td>
+            <td>Fri Jan 13 13:53:34 UTC 2017</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-crawler-db-mysql/">fess-crawler-db-mysql/</a></td>
+            <td>Fri Jan 13 13:53:56 UTC 2017</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-crawler-db/">fess-crawler-db/</a></td>
+            <td>Fri Jan 13 13:54:37 UTC 2017</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-crawler-es/">fess-crawler-es/</a></td>
+            <td>Mon Aug 12 06:14:22 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-crawler-lasta/">fess-crawler-lasta/</a></td>
+            <td>Mon Aug 12 06:13:10 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-crawler-parent/">fess-crawler-parent/</a></td>
+            <td>Mon Aug 12 06:08:50 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-crawler-webdriver/">fess-crawler-webdriver/</a></td>
+            <td>Mon Aug 12 06:14:36 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-crawler/">fess-crawler/</a></td>
+            <td>Mon Aug 12 06:11:31 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-db-h2/">fess-db-h2/</a></td>
+            <td>Sun Jul 05 13:18:45 UTC 2015</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-db-mysql/">fess-db-mysql/</a></td>
+            <td>Sun Jul 05 13:21:09 UTC 2015</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-db/">fess-db/</a></td>
+            <td>Sun Jul 05 13:16:39 UTC 2015</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-atlassian/">fess-ds-atlassian/</a></td>
+            <td>Mon Jul 15 03:55:47 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-box/">fess-ds-box/</a></td>
+            <td>Sat Jul 20 22:22:01 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-csv/">fess-ds-csv/</a></td>
+            <td>Mon Jul 15 03:55:46 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-db/">fess-ds-db/</a></td>
+            <td>Mon Jul 15 03:55:54 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-dropbox/">fess-ds-dropbox/</a></td>
+            <td>Wed Jul 10 13:08:03 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-elasticsearch/">fess-ds-elasticsearch/</a></td>
+            <td>Mon Jul 15 03:55:56 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-gitbucket/">fess-ds-gitbucket/</a></td>
+            <td>Mon Jul 15 03:55:45 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-gsuite/">fess-ds-gsuite/</a></td>
+            <td>Mon Jul 15 03:55:55 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-json/">fess-ds-json/</a></td>
+            <td>Mon Jul 15 03:56:05 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-office365/">fess-ds-office365/</a></td>
+            <td>Mon Jul 15 03:56:12 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-s3/">fess-ds-s3/</a></td>
+            <td>Thu Aug 08 02:27:37 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-salesforce/">fess-ds-salesforce/</a></td>
+            <td>Sat Jul 20 22:36:24 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-sample/">fess-ds-sample/</a></td>
+            <td>Mon Jun 10 04:43:07 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-ds-slack/">fess-ds-slack/</a></td>
+            <td>Mon Jul 15 03:55:47 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-suggest/">fess-suggest/</a></td>
+            <td>Mon Aug 12 04:18:25 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-xpack/">fess-xpack/</a></td>
+            <td>Sun Jun 02 23:11:23 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+                  <tr>
+            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess/">fess/</a></td>
+            <td>Mon Aug 12 06:44:55 UTC 2019</td>
+            <td align="right">
+                              &nbsp;
+                          </td>
+            <td></td>
+          </tr>
+            </table>
+  </body>
+</html>