diff --git a/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java b/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java index aef42a5c3..3735be4b9 100644 --- a/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java +++ b/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java @@ -41,6 +41,9 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + /** * @author Keiichi Watanabe */ @@ -184,32 +187,37 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { } } - protected List getSourceLabelList(final String rootURL, final String authToken) { - final String url = rootURL + "api/v3/fess/label"; - try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) { - final Map map = curlResponse.getContentAsMap(); - assert (map.containsKey("source_label")); - @SuppressWarnings("unchecked") - final List sourceLabels = (List) map.get("source_label"); - return sourceLabels; - } catch (final Exception e) { - logger.warn("Failed to access to " + rootURL, e); - return Collections.emptyList(); - } - } - protected List> getRepositoryList(final String rootURL, final String authToken) { final String url = rootURL + "api/v3/fess/repos"; - try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) { - final Map map = curlResponse.getContentAsMap(); - assert (map.containsKey("repositories")); - @SuppressWarnings("unchecked") - final List> repoList = (List>) map.get("repositories"); - return repoList; - } catch (final Exception e) { - logger.warn("Failed to access to " + rootURL, e); - return Collections.emptyList(); - } + int totalCount = -1; // initialize with dummy value + final List> repoList = new ArrayList<>(); + + do { + final String urlWithOffset = url + "?offset=" + repoList.size(); + + try (CurlResponse curlResponse = Curl.get(urlWithOffset).header("Authorization", "token " + authToken).execute()) { + final Map map = curlResponse.getContentAsMap(); + + assert (map.containsKey("total_count")); + assert (map.containsKey("response_count")); + assert (map.containsKey("repositories")); + + totalCount = (int) map.get("total_count"); + int responseCount = (int) map.get("response_count"); + if (responseCount == 0) + break; + + @SuppressWarnings("unchecked") + List> repos = (ArrayList>) map.get("repositories"); + repoList.addAll(repos); + } catch (final Exception e) { + logger.warn("Failed to access to " + rootURL, e); + break; + } + } while (repoList.size() < totalCount); + + logger.info("There exist " + repoList.size() + " repositories"); + return repoList; } protected String getGitRef(final String rootURL, final String authToken, final String owner, final String name, final String branch) { @@ -305,8 +313,8 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { logger.warn("Failed to access to " + issueUrl, e); } - // FIXME: Get issue comments from `commentsUrl` - // How to parse JSON-style list? + final String commentsStr = String.join("\n", getIssueComments(issueUrl, authToken)); + contentStr += "\n" + commentsStr; dataMap.put("content", contentStr); dataMap.put("url", viewUrl); @@ -320,6 +328,27 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { return; } + private List getIssueComments(final String issueUrl, final String authToken) { + final String commentsUrl = issueUrl + "/comments"; + final List commentList = new ArrayList(); + + try (CurlResponse curlResponse = Curl.get(commentsUrl).header("Authorization", "token " + authToken).execute()) { + final String commentsJson = curlResponse.getContentAsString(); + List> comments = new ObjectMapper().readValue(commentsJson, new TypeReference>>() { + }); + + for (Map comment : comments) { + if (comment.containsKey("body")) { + commentList.add((String) comment.get("body")); + } + } + } catch (final Exception e) { + logger.warn("Failed to access to " + issueUrl, e); + } + + return commentList; + } + @SuppressWarnings("unchecked") private void storeWikiContents(final String rootURL, final String authToken, final String wikiLabel, final String owner, final String name, final List roleList, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback,