Merge pull request #1062 from kw-udon/fix-gitbucketdatastore

Improve GitBucketDataStoreImpl

src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java

@@ -41,6 +41,9 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 /**
  * @author Keiichi Watanabe
  */
@@ -184,32 +187,37 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
         }
     }
 
-    protected List<String> getSourceLabelList(final String rootURL, final String authToken) {
-        final String url = rootURL + "api/v3/fess/label";
-        try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) {
-            final Map<String, Object> map = curlResponse.getContentAsMap();
-            assert (map.containsKey("source_label"));
-            @SuppressWarnings("unchecked")
-            final List<String> sourceLabels = (List<String>) map.get("source_label");
-            return sourceLabels;
-        } catch (final Exception e) {
-            logger.warn("Failed to access to " + rootURL, e);
-            return Collections.emptyList();
-        }
-    }
-
     protected List<Map<String, Object>> getRepositoryList(final String rootURL, final String authToken) {
         final String url = rootURL + "api/v3/fess/repos";
-        try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) {
-            final Map<String, Object> map = curlResponse.getContentAsMap();
-            assert (map.containsKey("repositories"));
-            @SuppressWarnings("unchecked")
-            final List<Map<String, Object>> repoList = (List<Map<String, Object>>) map.get("repositories");
-            return repoList;
-        } catch (final Exception e) {
-            logger.warn("Failed to access to " + rootURL, e);
-            return Collections.emptyList();
-        }
+        int totalCount = -1; // initialize with dummy value
+        final List<Map<String, Object>> repoList = new ArrayList<>();
+
+        do {
+            final String urlWithOffset = url + "?offset=" + repoList.size();
+
+            try (CurlResponse curlResponse = Curl.get(urlWithOffset).header("Authorization", "token " + authToken).execute()) {
+                final Map<String, Object> map = curlResponse.getContentAsMap();
+
+                assert (map.containsKey("total_count"));
+                assert (map.containsKey("response_count"));
+                assert (map.containsKey("repositories"));
+
+                totalCount = (int) map.get("total_count");
+                int responseCount = (int) map.get("response_count");
+                if (responseCount == 0)
+                    break;
+
+                @SuppressWarnings("unchecked")
+                List<Map<String, Object>> repos = (ArrayList<Map<String, Object>>) map.get("repositories");
+                repoList.addAll(repos);
+            } catch (final Exception e) {
+                logger.warn("Failed to access to " + rootURL, e);
+                break;
+            }
+        } while (repoList.size() < totalCount);
+
+        logger.info("There exist " + repoList.size() + " repositories");
+        return repoList;
     }
 
     protected String getGitRef(final String rootURL, final String authToken, final String owner, final String name, final String branch) {
@@ -305,8 +313,8 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
             logger.warn("Failed to access to " + issueUrl, e);
         }
 
-        // FIXME: Get issue comments from `commentsUrl`
-        // How to parse JSON-style list?
+        final String commentsStr = String.join("\n", getIssueComments(issueUrl, authToken));
+        contentStr += "\n" + commentsStr;
 
         dataMap.put("content", contentStr);
         dataMap.put("url", viewUrl);
@@ -320,6 +328,27 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
         return;
     }
 
+    private List<String> getIssueComments(final String issueUrl, final String authToken) {
+        final String commentsUrl = issueUrl + "/comments";
+        final List<String> commentList = new ArrayList<String>();
+
+        try (CurlResponse curlResponse = Curl.get(commentsUrl).header("Authorization", "token " + authToken).execute()) {
+            final String commentsJson = curlResponse.getContentAsString();
+            List<Map<String, Object>> comments = new ObjectMapper().readValue(commentsJson, new TypeReference<List<Map<String, Object>>>() {
+            });
+
+            for (Map<String, Object> comment : comments) {
+                if (comment.containsKey("body")) {
+                    commentList.add((String) comment.get("body"));
+                }
+            }
+        } catch (final Exception e) {
+            logger.warn("Failed to access to " + issueUrl, e);
+        }
+
+        return commentList;
+    }
+
     @SuppressWarnings("unchecked")
     private void storeWikiContents(final String rootURL, final String authToken, final String wikiLabel, final String owner,
             final String name, final List<String> roleList, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback,