diff --git a/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java b/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java index c58ccc951..6b5659a0c 100644 --- a/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java +++ b/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java @@ -17,12 +17,14 @@ package org.codelibs.fess.ds.impl; import java.io.InputStream; import java.util.ArrayList; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Consumer; +import org.apache.commons.io.FilenameUtils; import org.codelibs.core.lang.StringUtil; import org.codelibs.elasticsearch.runner.net.Curl; import org.codelibs.elasticsearch.runner.net.CurlResponse; @@ -33,7 +35,6 @@ import org.codelibs.fess.ds.IndexUpdateCallback; import org.codelibs.fess.es.config.exentity.CrawlingConfig; import org.codelibs.fess.es.config.exentity.CrawlingConfigWrapper; import org.codelibs.fess.es.config.exentity.DataConfig; -import org.codelibs.fess.util.ComponentUtil; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -125,7 +126,6 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { protected List> getRepositoryList(final String rootURL, final String authToken) { final String url = rootURL + "api/v3/fess/repos"; try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) { - curlResponse.getContentAsString(); final Map map = curlResponse.getContentAsMap(); assert (map.containsKey("repositories")); @SuppressWarnings("unchecked") @@ -149,20 +149,47 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { private void storeFileContent(final String rootURL, final String authToken, final String owner, final String name, final String path, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback, final Map paramMap, final Map scriptMap, final Map defaultDataMap) { - final String url = rootURL + owner + "/" + name + "/blob/master/" + path; + final String url = rootURL + "api/v3/repos/" + owner + "/" + name + "/contents/" + path; if (logger.isInfoEnabled()) { logger.info("Get a content from " + url); } final Map dataMap = new HashMap<>(); dataMap.putAll(defaultDataMap); - dataMap.putAll(ComponentUtil.getDocumentHelper().processRequest(crawlingConfig, paramMap.get("crawlingInfoId"), url + "?raw=true")); + // FIXME Use DocumentHelper + // dataMap.putAll(ComponentUtil.getDocumentHelper().processRequest(crawlingConfig, paramMap.get("crawlingInfoId"), url)); + dataMap.putAll(processContentRequest(authToken, url)); + // TODO scriptMap callback.store(paramMap, dataMap); return; } + + private Map processContentRequest(final String authToken, final String url) { // FIXME should be replaced by DocumentHelper + final Map dataMap = new HashMap<>(); + try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) { + final Map map = curlResponse.getContentAsMap(); + String content = StringUtil.EMPTY;; + if (map.containsKey("content")) { + content = (String) map.get("content"); + } + + if (map.containsKey("encoding") && map.get("encoding").equals("base64")) { + content = new String(Base64.getDecoder().decode(content)); + } + + dataMap.put("title", FilenameUtils.getName(url)); + dataMap.put("url", url); + dataMap.put("content", content); + + return dataMap; + } catch (final Exception e) { + logger.warn("Failed to get " + url, e); + return Collections.emptyMap(); + } + } protected void collectFileNames(final String rootURL, final String authToken, final String owner, final String name, final String path, final int depth, final long readInterval, Consumer consumer) {