0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch '10.3.x'

Browse source
This commit is contained in:
Shinsuke Sugaya 2016-10-18 06:32:28 +09:00
commit 6f3b72b6d0
6 changed files with 69 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
pom.xml
View file

@ -40,9 +40,9 @@
<!-- Main Framework -->
<dbflute.version>1.1.1</dbflute.version>
<lastaflute.version>0.8.3</lastaflute.version>
<lastaflute.version>0.8.4</lastaflute.version>
<lasta.taglib.version>0.7.1</lasta.taglib.version>
<lasta.job.version>0.2.3-RC2</lasta.job.version>
<lasta.job.version>0.2.3-RC1</lasta.job.version>
<mailflute.version>0.5.1</mailflute.version>
<!-- Partner Library -->
@ -54,7 +54,7 @@
<!-- Testing -->
<junit.version>4.12</junit.version>
<utflute.version>0.6.0F</utflute.version>
<utflute.version>0.6.1B</utflute.version>
<!-- Crawler -->
<crawler.version>1.0.11</crawler.version>
@ -67,7 +67,7 @@
<cluster.runner.version>2.4.0.0</cluster.runner.version>
<!-- Tomcat -->
<tomcat.version>8.5.4</tomcat.version>
<tomcat.version>8.5.5</tomcat.version>
<tomcat.boot.version>0.5.0</tomcat.boot.version>
<!-- DEB & RPM build -->
@ -1098,7 +1098,7 @@
<dependency>
<groupId>org.codelibs</groupId>
<artifactId>corelib</artifactId>
<version>0.3.5</version>
<version>0.3.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.codelibs</groupId>

67
src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java
View file

@ -17,6 +17,8 @@ package org.codelibs.fess.app.web.admin.backup;
import static org.codelibs.core.stream.StreamUtil.stream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -33,6 +35,7 @@ import org.codelibs.core.lang.StringUtil;
import org.codelibs.elasticsearch.runner.net.Curl;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.app.web.base.FessAdminAction;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.RenderDataUtil;
import org.codelibs.fess.util.ResourceUtil;
import org.lastaflute.core.magic.async.AsyncManager;
@ -70,20 +73,29 @@ public class AdminBackupAction extends FessAdminAction {
validate(form, messages -> {}, () -> asListHtml());
verifyToken(() -> asListHtml());
asyncManager.async(() -> {
try (CurlResponse response = Curl.post(ResourceUtil.getElasticsearchHttpUrl() + "/_bulk").onConnect((req, con) -> {
con.setDoOutput(true);
try (InputStream in = form.bulkFile.getInputStream(); OutputStream out = con.getOutputStream()) {
CopyUtil.copy(in, out);
final String fileName = form.bulkFile.getFileName();
if (fileName.startsWith("system") && fileName.endsWith(".properties")) {
try (final InputStream in = form.bulkFile.getInputStream()) {
ComponentUtil.getSystemProperties().load(in);
} catch (IOException e) {
throw new IORuntimeException(e);
logger.warn("Failed to process system.properties file: " + form.bulkFile.getFileName(), e);
}
}).execute()) {
if (logger.isDebugEnabled()) {
logger.debug("Bulk Response:\n" + response.getContentAsString());
} else {
try (CurlResponse response = Curl.post(ResourceUtil.getElasticsearchHttpUrl() + "/_bulk").onConnect((req, con) -> {
con.setDoOutput(true);
try (InputStream in = form.bulkFile.getInputStream(); OutputStream out = con.getOutputStream()) {
CopyUtil.copy(in, out);
} catch (IOException e) {
throw new IORuntimeException(e);
}
}).execute()) {
if (logger.isDebugEnabled()) {
logger.debug("Bulk Response:\n" + response.getContentAsString());
}
systemHelper.reloadConfiguration();
} catch (final Exception e) {
logger.warn("Failed to process bulk file: " + form.bulkFile.getFileName(), e);
}
systemHelper.reloadConfiguration();
} catch (final Exception e) {
logger.warn("Failed to process bulk file: " + form.bulkFile.getFileName(), e);
}
});
saveInfo(messages -> messages.addSuccessBulkProcessStarted(GLOBAL));
@ -93,13 +105,34 @@ public class AdminBackupAction extends FessAdminAction {
@Execute
public ActionResponse download(final String id) {
if (stream(fessConfig.getIndexBackupTargetsAsArray()).get(stream -> stream.anyMatch(s -> s.equals(id)))) {
return asStream(id + ".bulk").contentTypeOctetStream().stream(
out -> {
try (CurlResponse response =
Curl.get(ResourceUtil.getElasticsearchHttpUrl() + "/" + id + "/_data").param("format", "json").execute()) {
out.write(response.getContentAsStream());
if (id.equals("system.properties")) {
return asStream(id).contentTypeOctetStream().stream(out -> {
try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
ComponentUtil.getSystemProperties().store(baos, id);
try (final InputStream in = new ByteArrayInputStream(baos.toByteArray())) {
out.write(in);
}
});
}
});
} else {
final String index;
final String filename;
if (id.endsWith(".bulk")) {
index = id.substring(0, id.length() - 5);
filename = id;
} else {
index = id;
filename = id + ".bulk";
}
return asStream(filename).contentTypeOctetStream().stream(
out -> {
try (CurlResponse response =
Curl.get(ResourceUtil.getElasticsearchHttpUrl() + "/" + index + "/_data").param("format", "json")
.execute()) {
out.write(response.getContentAsStream());
}
});
}
}
throwValidationError(messages -> messages.addErrorsCouldNotFindBackupIndex(GLOBAL), () -> {
return asListHtml();

47
src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java
View file

@ -17,7 +17,6 @@ package org.codelibs.fess.ds.impl;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -36,6 +35,8 @@ import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.es.config.exentity.CrawlingConfigWrapper;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -84,6 +85,7 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
}
}
headerList.add(new RequestHeader("Authorization", "token " + authToken));
headerList.add(new RequestHeader("Accept", "application/vnd.github.v3.raw"));
paramMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, headerList.toArray(new RequestHeader[headerList.size()]));
return paramMap;
}
@ -156,16 +158,12 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
if (!isPrivate) {
return Collections.singletonList("Rguest");
}
@SuppressWarnings("unchecked")
final List<String> collaboratorList = (List<String>) repository.get(COLLABORATORS_PARAM);
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
collaboratorList.add(owner);
return collaboratorList.stream().map(user -> "1" + user).collect(Collectors.toList());
}
private List<String> createLabelList(final String owner, final String name) {
final List<String> labelList = new ArrayList<String>();
Collections.addAll(labelList, "GitBucket", owner + "/" + name);
return labelList;
return collaboratorList.stream().map(user -> systemHelper.getSearchRoleByUser(user)).collect(Collectors.toList());
}
private List<Object> parseList(final InputStream is) { // TODO This function should be moved to CurlResponse
@ -188,12 +186,12 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
}
final Map<String, Object> dataMap = new HashMap<>();
dataMap.putAll(defaultDataMap);
// FIXME Use DocumentHelper
// dataMap.putAll(ComponentUtil.getDocumentHelper().processRequest(crawlingConfig, paramMap.get("crawlingInfoId"), url));
dataMap.putAll(processContentRequest(authToken, apiUrl, viewUrl));
dataMap.putAll(ComponentUtil.getDocumentHelper().processRequest(crawlingConfig, paramMap.get("crawlingInfoId"),
apiUrl + "?large_file=true"));
dataMap.put("title", FilenameUtils.getName(apiUrl));
dataMap.put("url", viewUrl);
dataMap.put("role", roleList);
dataMap.put("label", createLabelList(owner, name));
// TODO scriptMap
@ -202,31 +200,6 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
return;
}
private Map<String, String> processContentRequest(final String authToken, final String apiUrl, final String viewUrl) { // FIXME should be replaced by DocumentHelper
final Map<String, String> dataMap = new HashMap<>();
try (CurlResponse curlResponse = Curl.get(apiUrl).header("Authorization", "token " + authToken).execute()) {
final Map<String, Object> map = curlResponse.getContentAsMap();
String content = StringUtil.EMPTY;
;
if (map.containsKey("content")) {
content = (String) map.get("content");
}
if (map.containsKey("encoding") && map.get("encoding").equals("base64")) {
content = new String(Base64.getDecoder().decode(content));
}
dataMap.put("title", FilenameUtils.getName(apiUrl));
dataMap.put("url", viewUrl);
dataMap.put("content", content);
return dataMap;
} catch (final Exception e) {
logger.warn("Failed to get " + apiUrl, e);
return Collections.emptyMap();
}
}
protected void collectFileNames(final String rootURL, final String authToken, final String owner, final String name, final String path,
final int depth, final long readInterval, Consumer<String> consumer) {

1
src/main/java/org/codelibs/fess/job/CrawlJob.java
View file

@ -325,6 +325,7 @@ public class CrawlJob {
ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
if (ownTmpDir.mkdirs()) {
cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
cmdList.add("-Dpdfbox.fontcache=" + ownTmpDir.getAbsolutePath());
} else {
ownTmpDir = null;
}

4
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -517,7 +517,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. 1,2 */
String SMB_AVAILABLE_SID_TYPES = "smb.available.sid.types";
/** The key of the configuration. e.g. .fess_basic_config,.fess_config,.fess_user */
/** The key of the configuration. e.g. .fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties */
String INDEX_BACKUP_TARGETS = "index.backup.targets";
/** The key of the configuration. e.g. admin */
@ -2535,7 +2535,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/**
* Get the value for the key 'index.backup.targets'. <br>
* The value is, e.g. .fess_basic_config,.fess_config,.fess_user <br>
* The value is, e.g. .fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties <br>
* comment: backup
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/

2
src/main/resources/fess_config.properties
View file

@ -270,7 +270,7 @@ smb.role.from.file=true
smb.available.sid.types=1,2
# backup
index.backup.targets=.fess_basic_config,.fess_config,.fess_user
index.backup.targets=.fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties
# ========================================================================================
# Web