|
@@ -59,6 +59,7 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
|
|
|
|
|
|
final String rootURL = getRootURL(paramMap);
|
|
|
final String authToken = getAuthToken(paramMap);
|
|
|
+ final List<String> sourceLabels = getSourceLabelList(rootURL, authToken);
|
|
|
final long readInterval = getReadInterval(paramMap);
|
|
|
|
|
|
if (rootURL.isEmpty() || authToken.isEmpty()) {
|
|
@@ -91,8 +92,9 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
|
|
|
};
|
|
|
for (final Map<String, Object> repository : repositoryList) {
|
|
|
try {
|
|
|
- final String name = (String) repository.get("name");
|
|
|
final String owner = (String) repository.get("owner");
|
|
|
+ final String name = (String) repository.get("name");
|
|
|
+ final String refStr = getGitRef(rootURL, authToken, owner, name, "master");
|
|
|
final List<String> roleList = createRoleList(owner, repository);
|
|
|
|
|
|
collectFileNames(
|
|
@@ -100,12 +102,13 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
|
|
|
authToken,
|
|
|
owner,
|
|
|
name,
|
|
|
+ refStr,
|
|
|
StringUtil.EMPTY,
|
|
|
0,
|
|
|
readInterval,
|
|
|
path -> {
|
|
|
- storeFileContent(rootURL, authToken, owner, name, roleList, path, crawlingConfig, callback, paramMap,
|
|
|
- scriptMap, defaultDataMap);
|
|
|
+ storeFileContent(rootURL, authToken, sourceLabels, owner, name, refStr, roleList, path, crawlingConfig,
|
|
|
+ callback, paramMap, scriptMap, defaultDataMap);
|
|
|
if (readInterval > 0) {
|
|
|
sleep(readInterval);
|
|
|
}
|
|
@@ -135,6 +138,20 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
|
|
|
return StringUtil.EMPTY;
|
|
|
}
|
|
|
|
|
|
+ protected List<String> getSourceLabelList(final String rootURL, final String authToken) {
|
|
|
+ final String url = rootURL + "api/v3/fess/label";
|
|
|
+ try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) {
|
|
|
+ final Map<String, Object> map = curlResponse.getContentAsMap();
|
|
|
+ assert (map.containsKey("source_label"));
|
|
|
+ @SuppressWarnings("unchecked")
|
|
|
+ final List<String> sourceLabels = (List<String>) map.get("source_label");
|
|
|
+ return sourceLabels;
|
|
|
+ } catch (final Exception e) {
|
|
|
+ logger.warn("Failed to access to " + rootURL, e);
|
|
|
+ return Collections.emptyList();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
protected List<Map<String, Object>> getRepositoryList(final String rootURL, final String authToken) {
|
|
|
final String url = rootURL + "api/v3/fess/repos";
|
|
|
try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) {
|
|
@@ -149,6 +166,21 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ protected String getGitRef(final String rootURL, final String authToken, final String owner, final String name, final String branch) {
|
|
|
+ final String url = rootURL + "api/v3/repos/" + owner + "/" + name + "/git/refs/heads/" + branch;
|
|
|
+ try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) {
|
|
|
+ final Map<String, Object> map = curlResponse.getContentAsMap();
|
|
|
+ assert (map.containsKey("object"));
|
|
|
+ @SuppressWarnings("unchecked")
|
|
|
+ final Map<String, String> objmap = (Map<String, String>) map.get("object");
|
|
|
+ assert (objmap.containsKey("sha"));
|
|
|
+ return objmap.get("sha");
|
|
|
+ } catch (final Exception e) {
|
|
|
+ logger.warn("Failed to access to " + rootURL, e);
|
|
|
+ return branch;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
private List<String> createRoleList(final String owner, final Map<String, Object> repository) {
|
|
|
Boolean isPrivate = true;
|
|
|
if (repository.containsKey(PRIVATE_REPOSITORY_PARAM)) {
|
|
@@ -174,11 +206,12 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- private void storeFileContent(final String rootURL, final String authToken, final String owner, final String name,
|
|
|
- final List<String> roleList, final String path, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback,
|
|
|
- final Map<String, String> paramMap, final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
|
|
|
+ private void storeFileContent(final String rootURL, final String authToken, final List<String> sourceLabels, final String owner,
|
|
|
+ final String name, final String refStr, final List<String> roleList, final String path, final CrawlingConfig crawlingConfig,
|
|
|
+ final IndexUpdateCallback callback, final Map<String, String> paramMap, final Map<String, String> scriptMap,
|
|
|
+ final Map<String, Object> defaultDataMap) {
|
|
|
final String apiUrl = rootURL + "api/v3/repos/" + owner + "/" + name + "/contents/" + path;
|
|
|
- final String viewUrl = rootURL + owner + "/" + name + "/blob/master/" + path;
|
|
|
+ final String viewUrl = rootURL + owner + "/" + name + "/blob/" + refStr + "/" + path;
|
|
|
|
|
|
if (logger.isInfoEnabled()) {
|
|
|
logger.info("Get a content from " + apiUrl);
|
|
@@ -186,10 +219,11 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
|
|
|
final Map<String, Object> dataMap = new HashMap<>();
|
|
|
dataMap.putAll(defaultDataMap);
|
|
|
dataMap.putAll(ComponentUtil.getDocumentHelper().processRequest(crawlingConfig, paramMap.get("crawlingInfoId"),
|
|
|
- apiUrl + "?large_file=true"));
|
|
|
+ apiUrl + "?ref=" + refStr + "&large_file=true"));
|
|
|
|
|
|
dataMap.put("url", viewUrl);
|
|
|
dataMap.put("role", roleList);
|
|
|
+ dataMap.put("label", sourceLabels);
|
|
|
|
|
|
// TODO scriptMap
|
|
|
|
|
@@ -198,14 +232,14 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
- protected void collectFileNames(final String rootURL, final String authToken, final String owner, final String name, final String path,
|
|
|
- final int depth, final long readInterval, final Consumer<String> consumer) {
|
|
|
+ protected void collectFileNames(final String rootURL, final String authToken, final String owner, final String name,
|
|
|
+ final String refStr, final String path, final int depth, final long readInterval, final Consumer<String> consumer) {
|
|
|
|
|
|
if (MAX_DEPTH <= depth) {
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
- final String url = rootURL + "api/v3/repos/" + owner + "/" + name + "/contents/" + path;
|
|
|
+ final String url = rootURL + "api/v3/repos/" + owner + "/" + name + "/contents/" + path + "?ref=" + refStr;
|
|
|
|
|
|
try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) {
|
|
|
final InputStream iStream = curlResponse.getContentAsStream();
|
|
@@ -223,7 +257,7 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
|
|
|
if (readInterval > 0) {
|
|
|
sleep(readInterval);
|
|
|
}
|
|
|
- collectFileNames(rootURL, authToken, owner, name, newPath, depth + 1, readInterval, consumer);
|
|
|
+ collectFileNames(rootURL, authToken, owner, name, refStr, newPath, depth + 1, readInterval, consumer);
|
|
|
break;
|
|
|
}
|
|
|
}
|