diff --git a/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java b/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java index edd5ac16a..b571a2864 100644 --- a/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java +++ b/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java @@ -92,8 +92,9 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { }; for (final Map repository : repositoryList) { try { - final String name = (String) repository.get("name"); final String owner = (String) repository.get("owner"); + final String name = (String) repository.get("name"); + final String refStr = getGitRef(rootURL, authToken, owner, name, "master"); final List roleList = createRoleList(owner, repository); collectFileNames( @@ -101,12 +102,13 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { authToken, owner, name, + refStr, StringUtil.EMPTY, 0, readInterval, path -> { - storeFileContent(rootURL, authToken, sourceLabels, owner, name, roleList, path, crawlingConfig, callback, - paramMap, scriptMap, defaultDataMap); + storeFileContent(rootURL, authToken, sourceLabels, owner, name, refStr, roleList, path, crawlingConfig, + callback, paramMap, scriptMap, defaultDataMap); if (readInterval > 0) { sleep(readInterval); } @@ -164,6 +166,21 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { } } + protected String getGitRef(final String rootURL, final String authToken, final String owner, final String name, final String branch) { + final String url = rootURL + "api/v3/repos/" + owner + "/" + name + "/git/refs/heads/" + branch; + try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) { + final Map map = curlResponse.getContentAsMap(); + assert (map.containsKey("object")); + @SuppressWarnings("unchecked") + final Map objmap = (Map) map.get("object"); + assert (objmap.containsKey("sha")); + return objmap.get("sha"); + } catch (final Exception e) { + logger.warn("Failed to access to " + rootURL, e); + return branch; + } + } + private List createRoleList(final String owner, final Map repository) { Boolean isPrivate = true; if (repository.containsKey(PRIVATE_REPOSITORY_PARAM)) { @@ -190,11 +207,11 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { } private void storeFileContent(final String rootURL, final String authToken, final List sourceLabels, final String owner, - final String name, final List roleList, final String path, final CrawlingConfig crawlingConfig, + final String name, final String refStr, final List roleList, final String path, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback, final Map paramMap, final Map scriptMap, final Map defaultDataMap) { final String apiUrl = rootURL + "api/v3/repos/" + owner + "/" + name + "/contents/" + path; - final String viewUrl = rootURL + owner + "/" + name + "/blob/master/" + path; + final String viewUrl = rootURL + owner + "/" + name + "/blob/" + refStr + "/" + path; if (logger.isInfoEnabled()) { logger.info("Get a content from " + apiUrl); @@ -202,7 +219,7 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { final Map dataMap = new HashMap<>(); dataMap.putAll(defaultDataMap); dataMap.putAll(ComponentUtil.getDocumentHelper().processRequest(crawlingConfig, paramMap.get("crawlingInfoId"), - apiUrl + "?large_file=true")); + apiUrl + "?ref=" + refStr + "&large_file=true")); dataMap.put("url", viewUrl); dataMap.put("role", roleList); @@ -215,14 +232,14 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { return; } - protected void collectFileNames(final String rootURL, final String authToken, final String owner, final String name, final String path, - final int depth, final long readInterval, final Consumer consumer) { + protected void collectFileNames(final String rootURL, final String authToken, final String owner, final String name, + final String refStr, final String path, final int depth, final long readInterval, final Consumer consumer) { if (MAX_DEPTH <= depth) { return; } - final String url = rootURL + "api/v3/repos/" + owner + "/" + name + "/contents/" + path; + final String url = rootURL + "api/v3/repos/" + owner + "/" + name + "/contents/" + path + "?ref=" + refStr; try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) { final InputStream iStream = curlResponse.getContentAsStream(); @@ -240,7 +257,7 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { if (readInterval > 0) { sleep(readInterval); } - collectFileNames(rootURL, authToken, owner, name, newPath, depth + 1, readInterval, consumer); + collectFileNames(rootURL, authToken, owner, name, refStr, newPath, depth + 1, readInterval, consumer); break; } }