diff --git a/src/main/java/org/codelibs/fess/helper/PluginHelper.java b/src/main/java/org/codelibs/fess/helper/PluginHelper.java new file mode 100644 index 000000000..169fcfc28 --- /dev/null +++ b/src/main/java/org/codelibs/fess/helper/PluginHelper.java @@ -0,0 +1,103 @@ +/* + * Copyright 2012-2019 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.helper; + +import static org.codelibs.core.stream.StreamUtil.split; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.codelibs.curl.Curl; +import org.codelibs.curl.CurlResponse; +import org.codelibs.fess.util.ComponentUtil; +import org.lastaflute.di.exception.IORuntimeException; + +public class PluginHelper { + + public Artifact[] getArtifacts(PluginType pluginType) { + List list = new ArrayList<>(); + for (String url : getRepositories()) { + list.addAll(processRepository(pluginType, url)); + } + return list.toArray(new Artifact[list.size()]); + } + + protected String[] getRepositories() { + return split(ComponentUtil.getFessConfig().getPluginRepositories(), ",").get( + stream -> stream.map(s -> s.trim()).toArray(n -> new String[n])); + } + + protected List processRepository(PluginType pluginType, String url) { + List list = new ArrayList<>(); + String repoContent = getRepositoryContent(url); + Matcher matcher = Pattern.compile("href=\"[^\"]*(" + pluginType.getId() + "[a-zA-Z0-9\\-]+)/?\"").matcher(repoContent); + while (matcher.find()) { + String name = matcher.group(1); + // TODO parse maven-metadata.xml from url/name to set version and url + list.add(new Artifact(name, "TODO", "TODO")); + } + return list; + } + + protected String getRepositoryContent(String url) { + try (CurlResponse response = Curl.get(url).execute()) { + return response.getContentAsString(); + } catch (IOException e) { + throw new IORuntimeException(e); + } + } + + public static class Artifact { + protected final String name; + protected final String version; + protected final String url; + + public Artifact(String name, String version, String url) { + this.name = name; + this.version = version; + this.url = url; + } + + public String getName() { + return name; + } + + public String getVersion() { + return version; + } + + public String getUrl() { + return url; + } + } + + public enum PluginType { + DATA_STORE("fess-ds"); + + private String id; + + private PluginType(String id) { + this.id = id; + } + + public String getId() { + return id; + } + } +} diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index ecbab6391..6a3250889 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -1427,6 +1427,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. homeDirectory */ String LDAP_ATTR_HOME_DIRECTORY = "ldap.attr.homeDirectory"; + /** The key of the configuration. e.g. https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/ */ + String PLUGIN_REPOSITORIES = "plugin.repositories"; + /** * Get the value of property as {@link String}. * @param propertyKey The key of the property. (NotNull) @@ -5921,6 +5924,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ String getLdapAttrHomeDirectory(); + /** + * Get the value for the key 'plugin.repositories'.
+ * The value is, e.g. https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/
+ * comment: ------ + * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getPluginRepositories(); + /** * The simple implementation for configuration. * @author FreeGen @@ -8286,6 +8297,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return get(FessConfig.LDAP_ATTR_HOME_DIRECTORY); } + public String getPluginRepositories() { + return get(FessConfig.PLUGIN_REPOSITORIES); + } + @Override protected java.util.Map prepareGeneratedDefaultMap() { java.util.Map defaultMap = super.prepareGeneratedDefaultMap(); @@ -8719,6 +8734,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction defaultMap.put(FessConfig.LDAP_ATTR_UID_NUMBER, "uidNumber"); defaultMap.put(FessConfig.LDAP_ATTR_GID_NUMBER, "gidNumber"); defaultMap.put(FessConfig.LDAP_ATTR_HOME_DIRECTORY, "homeDirectory"); + defaultMap + .put(FessConfig.PLUGIN_REPOSITORIES, + "https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/"); defaultMap.put(FessConfig.lasta_di_SMART_DEPLOY_MODE, "hot"); defaultMap.put(FessConfig.DEVELOPMENT_HERE, "true"); defaultMap.put(FessConfig.ENVIRONMENT_TITLE, "Local Development"); diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index 7e0f67668..58880b8e0 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -701,3 +701,7 @@ ldap.attr.uidNumber=uidNumber ldap.attr.gidNumber=gidNumber ldap.attr.homeDirectory=homeDirectory +# ---------------------------------------------------------- +# Scheduler +# ------ +plugin.repositories=https://repo.maven.apache.org/maven2/org/codelibs/fess/,https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/ diff --git a/src/test/java/org/codelibs/fess/helper/PluginHelperTest.java b/src/test/java/org/codelibs/fess/helper/PluginHelperTest.java new file mode 100644 index 000000000..7de6a15ac --- /dev/null +++ b/src/test/java/org/codelibs/fess/helper/PluginHelperTest.java @@ -0,0 +1,72 @@ +/* + * Copyright 2012-2019 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.helper; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import org.codelibs.core.io.InputStreamUtil; +import org.codelibs.core.io.ResourceUtil; +import org.codelibs.fess.Constants; +import org.codelibs.fess.exception.FessSystemException; +import org.codelibs.fess.helper.PluginHelper.Artifact; +import org.codelibs.fess.helper.PluginHelper.PluginType; +import org.codelibs.fess.unit.UnitFessTestCase; +import org.lastaflute.di.exception.IORuntimeException; + +public class PluginHelperTest extends UnitFessTestCase { + private PluginHelper pluginHelper; + + @Override + public void setUp() throws Exception { + super.setUp(); + pluginHelper = new PluginHelper() { + protected String[] getRepositories() { + return new String[] { "plugin/repo1.html", "plugin/repo2.html" }; + } + + protected String getRepositoryContent(String url) { + if (url.contains("repo1")) { + try (InputStream is = ResourceUtil.getResourceAsStream(url)) { + return new String(InputStreamUtil.getBytes(is), Constants.UTF_8); + } catch (IOException e) { + throw new IORuntimeException(e); + } + } else if (url.contains("repo2")) { + try (InputStream is = ResourceUtil.getResourceAsStream(url)) { + return new String(InputStreamUtil.getBytes(is), Constants.UTF_8); + } catch (IOException e) { + throw new IORuntimeException(e); + } + } + throw new FessSystemException("unknown"); + } + }; + } + + public void test_processRepository1() { + List list = pluginHelper.processRepository(PluginType.DATA_STORE, "plugin/repo1.html"); + assertEquals(11, list.size()); + assertEquals("fess-ds-atlassian", list.get(0).getName()); + } + + public void test_processRepository2() { + List list = pluginHelper.processRepository(PluginType.DATA_STORE, "plugin/repo2.html"); + assertEquals(14, list.size()); + assertEquals("fess-ds-atlassian", list.get(0).getName()); + } +} diff --git a/src/test/resources/plugin/repo1.html b/src/test/resources/plugin/repo1.html new file mode 100644 index 000000000..df3a47cb5 --- /dev/null +++ b/src/test/resources/plugin/repo1.html @@ -0,0 +1,49 @@ + + + + + Central Repository: org/codelibs/fess + + + + + +
+

org/codelibs/fess

+
+
+
+ 
+../
+fess/                                                            -         -      
+fess-crawler/                                                    -         -      
+fess-crawler-db/                                                 -         -      
+fess-crawler-db-h2/                                              -         -      
+fess-crawler-db-mysql/                                           -         -      
+fess-crawler-es/                                                 -         -      
+fess-crawler-lasta/                                              -         -      
+fess-crawler-parent/                                             -         -      
+fess-crawler-webdriver/                                          -         -      
+fess-ds-atlassian/                                               -         -      
+fess-ds-box/                                                     -         -      
+fess-ds-csv/                                                     -         -      
+fess-ds-db/                                                      -         -      
+fess-ds-dropbox/                                                 -         -      
+fess-ds-elasticsearch/                                           -         -      
+fess-ds-gitbucket/                                               -         -      
+fess-ds-gsuite/                                                  -         -      
+fess-ds-json/                                                    -         -      
+fess-ds-office365/                                               -         -      
+fess-ds-slack/                                                   -         -      
+fess-suggest/                                                    -         -      
+fess-xpack/                                                      -         -      
+
+
+
+ + + \ No newline at end of file diff --git a/src/test/resources/plugin/repo2.html b/src/test/resources/plugin/repo2.html new file mode 100644 index 000000000..4450d6390 --- /dev/null +++ b/src/test/resources/plugin/repo2.html @@ -0,0 +1,251 @@ + + + Index of /repositories/snapshots/org/codelibs/fess + + + + + + + + +

Index of /repositories/snapshots/org/codelibs/fess

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameLast ModifiedSizeDescription
Parent Directory
fess-crawler-db-h2/Fri Jan 13 13:53:34 UTC 2017 +   +
fess-crawler-db-mysql/Fri Jan 13 13:53:56 UTC 2017 +   +
fess-crawler-db/Fri Jan 13 13:54:37 UTC 2017 +   +
fess-crawler-es/Mon Aug 12 06:14:22 UTC 2019 +   +
fess-crawler-lasta/Mon Aug 12 06:13:10 UTC 2019 +   +
fess-crawler-parent/Mon Aug 12 06:08:50 UTC 2019 +   +
fess-crawler-webdriver/Mon Aug 12 06:14:36 UTC 2019 +   +
fess-crawler/Mon Aug 12 06:11:31 UTC 2019 +   +
fess-db-h2/Sun Jul 05 13:18:45 UTC 2015 +   +
fess-db-mysql/Sun Jul 05 13:21:09 UTC 2015 +   +
fess-db/Sun Jul 05 13:16:39 UTC 2015 +   +
fess-ds-atlassian/Mon Jul 15 03:55:47 UTC 2019 +   +
fess-ds-box/Sat Jul 20 22:22:01 UTC 2019 +   +
fess-ds-csv/Mon Jul 15 03:55:46 UTC 2019 +   +
fess-ds-db/Mon Jul 15 03:55:54 UTC 2019 +   +
fess-ds-dropbox/Wed Jul 10 13:08:03 UTC 2019 +   +
fess-ds-elasticsearch/Mon Jul 15 03:55:56 UTC 2019 +   +
fess-ds-gitbucket/Mon Jul 15 03:55:45 UTC 2019 +   +
fess-ds-gsuite/Mon Jul 15 03:55:55 UTC 2019 +   +
fess-ds-json/Mon Jul 15 03:56:05 UTC 2019 +   +
fess-ds-office365/Mon Jul 15 03:56:12 UTC 2019 +   +
fess-ds-s3/Thu Aug 08 02:27:37 UTC 2019 +   +
fess-ds-salesforce/Sat Jul 20 22:36:24 UTC 2019 +   +
fess-ds-sample/Mon Jun 10 04:43:07 UTC 2019 +   +
fess-ds-slack/Mon Jul 15 03:55:47 UTC 2019 +   +
fess-suggest/Mon Aug 12 04:18:25 UTC 2019 +   +
fess-xpack/Sun Jun 02 23:11:23 UTC 2019 +   +
fess/Mon Aug 12 06:44:55 UTC 2019 +   +
+ +