add SearchApiTests
This commit is contained in:
yfujita
Shinsuke Sugaya
parent
f97c8185d3
commit
94ac746805
3 changed files with 690 additions and 163 deletions
227
src/test/java/org/codelibs/fess/it/CrawlTestBase.java
Normal file
227
src/test/java/org/codelibs/fess/it/CrawlTestBase.java
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
/*
|
||||
* Copyright 2012-2017 CodeLibs Project and the Others.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
* either express or implied. See the License for the specific language
|
||||
* governing permissions and limitations under the License.
|
||||
*/
|
||||
package org.codelibs.fess.it;
|
||||
|
||||
import io.restassured.path.json.JsonPath;
|
||||
import io.restassured.response.Response;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static io.restassured.RestAssured.given;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class CrawlTestBase extends ITBase {
|
||||
private static final Logger logger = LoggerFactory.getLogger(CrawlTestBase.class);
|
||||
|
||||
private static final String DOC_INDEX_NAME = "fess.search";
|
||||
|
||||
private static final String DOC_TYPE_NAME = "doc";
|
||||
|
||||
protected static void createJob(final Map<String, Object> requestBody) {
|
||||
checkMethodBase(requestBody).put("/api/admin/scheduler/setting").then().body("response.created", equalTo(true))
|
||||
.body("response.status", equalTo(0));
|
||||
}
|
||||
|
||||
protected static void startJob(final String namePrefix) {
|
||||
for (int i = 0; i < 30; i++) {
|
||||
final Map<String, Object> requestBody = new HashMap<>();
|
||||
final String schedulerId = getSchedulerIds(namePrefix).get(0);
|
||||
final Response response = checkMethodBase(requestBody).post("/api/admin/scheduler/" + schedulerId + "/start");
|
||||
if (response.getBody().jsonPath().getInt("response.status") == 0) {
|
||||
logger.info("Start scheduler \"" + schedulerId + "\"");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Thread.sleep(1000L);
|
||||
} catch (InterruptedException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
assertTrue(false, "could not start job.");
|
||||
}
|
||||
|
||||
protected static void waitJob(final String namePrefix) throws InterruptedException {
|
||||
Boolean isRunning = false;
|
||||
int count = 0;
|
||||
|
||||
while (count < 300 && !isRunning) { // Wait until the crawler starts
|
||||
Thread.sleep(500);
|
||||
count++;
|
||||
final Map<String, Object> scheduler = getSchedulerItem(namePrefix);
|
||||
assertTrue(scheduler.containsKey("running"));
|
||||
isRunning = (Boolean) scheduler.get("running");
|
||||
}
|
||||
if (300 <= count) {
|
||||
logger.info("Time out: Failed to start crawler)");
|
||||
assertTrue(false); // Time Out
|
||||
}
|
||||
|
||||
logger.info("Crawler is running");
|
||||
count = 0;
|
||||
isRunning = true;
|
||||
while (count < 300 && isRunning) { // Wait until the crawler terminates
|
||||
Thread.sleep(1000);
|
||||
count++;
|
||||
final Map<String, Object> scheduler = getSchedulerItem(namePrefix);
|
||||
assertTrue(scheduler.containsKey("running"));
|
||||
isRunning = (Boolean) scheduler.get("running");
|
||||
|
||||
}
|
||||
if (300 <= count) {
|
||||
logger.info("Time out: Crawler takes too much time");
|
||||
//TODO assertTrue(false); // Time Out
|
||||
}
|
||||
|
||||
logger.info("Crawler terminated");
|
||||
}
|
||||
|
||||
protected static String createWebConfig(final Map<String, Object> requestBody) {
|
||||
String response = checkMethodBase(requestBody).put("/api/admin/webconfig/setting").asString();
|
||||
JsonPath jsonPath = JsonPath.from(response);
|
||||
assertTrue(jsonPath.getBoolean("response.created"));
|
||||
assertEquals(0, jsonPath.getInt("response.status"));
|
||||
return jsonPath.getString("response.id");
|
||||
}
|
||||
|
||||
protected static List<String> getWebConfigIds(final String namePrefix) {
|
||||
final String response = getJsonResponse("/api/admin/webconfig/settings");
|
||||
final List<String> idList = JsonPath.from(response).getList(getResponsePath(namePrefix) + ".id");
|
||||
return idList;
|
||||
}
|
||||
|
||||
protected static String createFileConfig(final Map<String, Object> requestBody) {
|
||||
String response = checkMethodBase(requestBody).put("/api/admin/fileconfig/setting").asString();
|
||||
JsonPath jsonPath = JsonPath.from(response);
|
||||
assertTrue(jsonPath.getBoolean("response.created"));
|
||||
assertEquals(0, jsonPath.getInt("response.status"));
|
||||
return jsonPath.getString("response.id");
|
||||
}
|
||||
|
||||
protected static List<String> getFileConfigIds(final String namePrefix) {
|
||||
final String response = getJsonResponse("/api/admin/fileconfig/settings");
|
||||
final List<String> idList = JsonPath.from(response).getList(getResponsePath(namePrefix) + ".id");
|
||||
return idList;
|
||||
}
|
||||
|
||||
protected static String getResponsePath(final String namePrefix) {
|
||||
return "response.settings.findAll {it.name.startsWith(\"" + namePrefix + "\")}";
|
||||
}
|
||||
|
||||
protected static String getJsonResponse(final String path) {
|
||||
final Map<String, Object> searchBody = new HashMap<>();
|
||||
final String response = checkMethodBase(searchBody).get(path).asString();
|
||||
return response;
|
||||
}
|
||||
|
||||
protected static String buildWebConfigJobScript(final String webCofigId) {
|
||||
return String.format("return container.getComponent(\"crawlJob\")" + ".logLevel(\"info\")" + ".sessionId(\"%s\")"
|
||||
+ ".webConfigIds([\"%s\"] as String[])" + ".jobExecutor(executor).execute();", webCofigId, webCofigId);
|
||||
}
|
||||
|
||||
protected static String buildFileConfigJobScript(final String fileConfigId) {
|
||||
return String.format("return container.getComponent(\"crawlJob\")" + ".logLevel(\"info\")" + ".sessionId(\"%s\")"
|
||||
+ ".fileConfigIds([\"%s\"] as String[])" + ".jobExecutor(executor).execute();", fileConfigId, fileConfigId);
|
||||
}
|
||||
|
||||
protected static Map<String, Object> getSchedulerItem(final String namePrefix) {
|
||||
final String response = getJsonResponse("/api/admin/scheduler/settings");
|
||||
final List<Map<String, Object>> itemList = JsonPath.from(response).getList(getResponsePath(namePrefix));
|
||||
assertEquals(1, itemList.size());
|
||||
return itemList.get(0);
|
||||
}
|
||||
|
||||
protected static List<String> getSchedulerIds(final String namePrefix) {
|
||||
final String response = getJsonResponse("/api/admin/scheduler/settings");
|
||||
final List<String> idList = JsonPath.from(response).getList(getResponsePath(namePrefix) + ".id");
|
||||
return idList;
|
||||
}
|
||||
|
||||
protected static List<Map<String, Object>> readJobLog(final String namePrefix) {
|
||||
final List<Map<String, Object>> logList = readLogItems("joblog");
|
||||
final List<Map<String, Object>> resList = new ArrayList<>();
|
||||
for (Map<String, Object> elem : logList) {
|
||||
if (elem.containsKey("job_name") && elem.get("job_name").equals(namePrefix + "Scheduler")) {
|
||||
resList.add(elem);
|
||||
}
|
||||
}
|
||||
return resList;
|
||||
}
|
||||
|
||||
protected static List<Map<String, Object>> readLogItems(final String apiName) {
|
||||
final Map<String, Object> searchBody = new HashMap<>();
|
||||
final String response = checkMethodBase(searchBody).get("/api/admin/" + apiName + "/logs").asString();
|
||||
final List<Map<String, Object>> itemList = JsonPath.from(response).getList("response.logs");
|
||||
return itemList;
|
||||
}
|
||||
|
||||
protected static Response deleteMethod(final String path) {
|
||||
return given().header("Authorization", getTestToken()).delete(path);
|
||||
}
|
||||
|
||||
protected static void deleteDocuments(final String queryString) {
|
||||
List<String> docIds = new ArrayList<>();
|
||||
Response response =
|
||||
given().param("scroll", "1m").param("q", queryString)
|
||||
.get(getEsUrl() + "/" + DOC_INDEX_NAME + "/" + DOC_TYPE_NAME + "/_search");
|
||||
JsonPath jsonPath = JsonPath.from(response.asString());
|
||||
String scrollId = jsonPath.getString("_scroll_id");
|
||||
while (true) {
|
||||
List<String> resultIds = jsonPath.getList("hits.hits._id");
|
||||
if (resultIds.size() == 0) {
|
||||
break;
|
||||
}
|
||||
docIds.addAll(resultIds);
|
||||
Map<String, Object> scrollBody = new HashMap<>();
|
||||
scrollBody.put("scroll", "1m");
|
||||
scrollBody.put("scroll_id", scrollId);
|
||||
response = given().body(scrollBody).get(getEsUrl() + "/_search/scroll");
|
||||
jsonPath = JsonPath.from(response.asString());
|
||||
}
|
||||
|
||||
for (String docId : docIds) {
|
||||
given().delete(getEsUrl() + "/" + DOC_INDEX_NAME + "/" + DOC_TYPE_NAME + "/" + docId);
|
||||
}
|
||||
}
|
||||
|
||||
protected static List<Map<String, Object>> readCrawlingInfo(final String configId) {
|
||||
final List<Map<String, Object>> logList = readLogItems("crawlinginfo");
|
||||
final List<Map<String, Object>> resList = new ArrayList<>();
|
||||
for (Map<String, Object> elem : logList) {
|
||||
if (elem.containsKey("session_id") && elem.get("session_id").equals(configId)) {
|
||||
resList.add(elem);
|
||||
}
|
||||
}
|
||||
return resList;
|
||||
}
|
||||
|
||||
protected static List<Map<String, Object>> readFailureUrl(final String configId) {
|
||||
final List<Map<String, Object>> logList = readLogItems("failureurl");
|
||||
final List<Map<String, Object>> resList = new ArrayList<>();
|
||||
for (Map<String, Object> elem : logList) {
|
||||
if (elem.containsKey("thread_name") && elem.get("thread_name").toString().startsWith("Crawler-" + configId)) {
|
||||
resList.add(elem);
|
||||
}
|
||||
}
|
||||
return resList;
|
||||
}
|
||||
}
|
||||
|
|
@ -15,18 +15,16 @@
|
|||
*/
|
||||
package org.codelibs.fess.it.admin;
|
||||
|
||||
import static io.restassured.RestAssured.given;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.codelibs.fess.Constants;
|
||||
import org.codelibs.fess.it.ITBase;
|
||||
import org.codelibs.fess.it.CrawlTestBase;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
|
|
@ -38,7 +36,6 @@ import org.slf4j.LoggerFactory;
|
|||
|
||||
import io.restassured.RestAssured;
|
||||
import io.restassured.path.json.JsonPath;
|
||||
import io.restassured.response.Response;
|
||||
|
||||
/**
|
||||
* Integration Tests which need an execution of crawler
|
||||
|
|
@ -48,7 +45,7 @@ import io.restassured.response.Response;
|
|||
* - /api/admin/searchlist
|
||||
* */
|
||||
@Tag("it")
|
||||
public class CrawlerLogTests extends ITBase {
|
||||
public class CrawlerLogTests extends CrawlTestBase {
|
||||
private static final Logger logger = LoggerFactory.getLogger(CrawlerLogTests.class);
|
||||
|
||||
private static final String NAME_PREFIX = "crawlingInfoTest_";
|
||||
|
|
@ -65,15 +62,15 @@ public class CrawlerLogTests extends ITBase {
|
|||
createWebConfig();
|
||||
logger.info("WebConfig is created");
|
||||
refresh();
|
||||
webConfigId = getWebConfigIds().get(0);
|
||||
webConfigId = getWebConfigIds(NAME_PREFIX).get(0);
|
||||
|
||||
createJob();
|
||||
logger.info("Job is created");
|
||||
refresh();
|
||||
|
||||
startJob();
|
||||
startJob(NAME_PREFIX);
|
||||
|
||||
waitJob();
|
||||
waitJob(NAME_PREFIX);
|
||||
refresh();
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
|
|
@ -92,17 +89,17 @@ public class CrawlerLogTests extends ITBase {
|
|||
|
||||
@AfterAll
|
||||
protected static void tearDownAll() {
|
||||
final List<Map<String, Object>> jobLogList = readJobLog();
|
||||
final List<Map<String, Object>> jobLogList = readJobLog(NAME_PREFIX);
|
||||
for (Map<String, Object> elem : jobLogList) {
|
||||
deleteMethod("/api/admin/joblog/log/" + elem.get("id"));
|
||||
}
|
||||
|
||||
final List<Map<String, Object>> crawlingInfoList = readCrawlingInfo();
|
||||
final List<Map<String, Object>> crawlingInfoList = readCrawlingInfo(webConfigId);
|
||||
for (Map<String, Object> elem : crawlingInfoList) {
|
||||
deleteMethod("/api/admin/crawlinginfo/log/" + elem.get("id"));
|
||||
}
|
||||
|
||||
final List<Map<String, Object>> failureUrlList = readFailureUrl();
|
||||
final List<Map<String, Object>> failureUrlList = readFailureUrl(webConfigId);
|
||||
for (Map<String, Object> elem : failureUrlList) {
|
||||
deleteMethod("/api/admin/failurelog/log/" + elem.get("id"));
|
||||
}
|
||||
|
|
@ -112,11 +109,11 @@ public class CrawlerLogTests extends ITBase {
|
|||
checkMethodBase(requestBody).delete("/api/admin/searchlist/query");
|
||||
refresh();
|
||||
|
||||
for (String sId : getSchedulerIds()) {
|
||||
for (String sId : getSchedulerIds(NAME_PREFIX)) {
|
||||
deleteMethod("/api/admin/scheduler/setting/" + sId);
|
||||
}
|
||||
|
||||
for (String wId : getWebConfigIds()) {
|
||||
for (String wId : getWebConfigIds(NAME_PREFIX)) {
|
||||
deleteMethod("/api/admin/webconfig/setting/" + wId);
|
||||
}
|
||||
|
||||
|
|
@ -162,9 +159,7 @@ public class CrawlerLogTests extends ITBase {
|
|||
requestBody.put("boost", 100);
|
||||
requestBody.put("available", true);
|
||||
requestBody.put("sort_order", 0);
|
||||
|
||||
checkMethodBase(requestBody).put("/api/admin/webconfig/setting").then().body("response.created", equalTo(true))
|
||||
.body("response.status", equalTo(0));
|
||||
createWebConfig(requestBody);
|
||||
}
|
||||
|
||||
private static void createJob() {
|
||||
|
|
@ -176,152 +171,64 @@ public class CrawlerLogTests extends ITBase {
|
|||
requestBody.put("crawler", Constants.ON);
|
||||
requestBody.put("job_logging", Constants.ON);
|
||||
requestBody.put("available", Constants.ON);
|
||||
requestBody.put("script_data", buildJobScript());
|
||||
|
||||
checkMethodBase(requestBody).put("/api/admin/scheduler/setting").then().body("response.created", equalTo(true))
|
||||
.body("response.status", equalTo(0));
|
||||
}
|
||||
|
||||
private static void startJob() {
|
||||
for (int i = 0; i < 30; i++) {
|
||||
final Map<String, Object> requestBody = new HashMap<>();
|
||||
final String schedulerId = getSchedulerIds().get(0);
|
||||
final Response response = checkMethodBase(requestBody).post("/api/admin/scheduler/" + schedulerId + "/start");
|
||||
if (response.getBody().jsonPath().getInt("response.status") == 0) {
|
||||
logger.info("Start scheduler \"" + schedulerId + "\"");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Thread.sleep(1000L);
|
||||
} catch (InterruptedException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
assertTrue(false, "could not start job.");
|
||||
}
|
||||
|
||||
private static void waitJob() throws InterruptedException {
|
||||
Boolean isRunning = false;
|
||||
int count = 0;
|
||||
|
||||
while (count < 300 && !isRunning) { // Wait until the crawler starts
|
||||
Thread.sleep(500);
|
||||
count++;
|
||||
final Map<String, Object> scheduler = getSchedulerItem();
|
||||
assertTrue(scheduler.containsKey("running"));
|
||||
isRunning = (Boolean) scheduler.get("running");
|
||||
}
|
||||
if (300 <= count) {
|
||||
logger.info("Time out: Failed to start crawler)");
|
||||
assertTrue(false); // Time Out
|
||||
}
|
||||
|
||||
logger.info("Crawler is running");
|
||||
count = 0;
|
||||
isRunning = true;
|
||||
while (count < 300 && isRunning) { // Wait until the crawler terminates
|
||||
Thread.sleep(1000);
|
||||
count++;
|
||||
final Map<String, Object> scheduler = getSchedulerItem();
|
||||
assertTrue(scheduler.containsKey("running"));
|
||||
isRunning = (Boolean) scheduler.get("running");
|
||||
|
||||
}
|
||||
if (300 <= count) {
|
||||
logger.info("Time out: Crawler takes too much time");
|
||||
assertTrue(false); // Time Out
|
||||
}
|
||||
|
||||
logger.info("Crawler terminated");
|
||||
requestBody.put("script_data", buildWebConfigJobScript(webConfigId));
|
||||
createJob(requestBody);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test for JobLog
|
||||
* */
|
||||
private void testReadJobLog() {
|
||||
final List<Map<String, Object>> logList = readJobLog();
|
||||
final List<Map<String, Object>> logList = readJobLog(NAME_PREFIX);
|
||||
assertEquals(1, logList.size());
|
||||
}
|
||||
|
||||
private void testDeleteJobLog() {
|
||||
final List<Map<String, Object>> logList = readJobLog();
|
||||
final List<Map<String, Object>> logList = readJobLog(NAME_PREFIX);
|
||||
for (Map<String, Object> elem : logList) {
|
||||
deleteMethod("/api/admin/joblog/log/" + elem.get("id")).then().body("response.status", equalTo(0));
|
||||
}
|
||||
|
||||
final List<Map<String, Object>> afterList = readJobLog();
|
||||
final List<Map<String, Object>> afterList = readJobLog(NAME_PREFIX);
|
||||
assertEquals(0, afterList.size()); // check if logs are successfully deleted
|
||||
}
|
||||
|
||||
private static List<Map<String, Object>> readJobLog() {
|
||||
final List<Map<String, Object>> logList = readLogItems("joblog");
|
||||
final List<Map<String, Object>> resList = new ArrayList<>();
|
||||
for (Map<String, Object> elem : logList) {
|
||||
if (elem.containsKey("job_name") && elem.get("job_name").equals(NAME_PREFIX + "Scheduler")) {
|
||||
resList.add(elem);
|
||||
}
|
||||
}
|
||||
return resList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test for CrawlingInfo
|
||||
* */
|
||||
private void testReadCrawlingInfo() {
|
||||
final List<Map<String, Object>> logList = readCrawlingInfo();
|
||||
final List<Map<String, Object>> logList = readCrawlingInfo(webConfigId);
|
||||
assertEquals(1, logList.size());
|
||||
}
|
||||
|
||||
private void testDeleteCrawlingInfo() {
|
||||
final List<Map<String, Object>> logList = readCrawlingInfo();
|
||||
final List<Map<String, Object>> logList = readCrawlingInfo(webConfigId);
|
||||
for (Map<String, Object> elem : logList) {
|
||||
deleteMethod("/api/admin/crawlinginfo/log/" + elem.get("id")).then().body("response.status", equalTo(0));
|
||||
}
|
||||
|
||||
final List<Map<String, Object>> afterList = readCrawlingInfo();
|
||||
final List<Map<String, Object>> afterList = readCrawlingInfo(webConfigId);
|
||||
assertEquals(0, afterList.size()); // check if logs are successfully deleted
|
||||
}
|
||||
|
||||
private static List<Map<String, Object>> readCrawlingInfo() {
|
||||
final List<Map<String, Object>> logList = readLogItems("crawlinginfo");
|
||||
final List<Map<String, Object>> resList = new ArrayList<>();
|
||||
for (Map<String, Object> elem : logList) {
|
||||
if (elem.containsKey("session_id") && elem.get("session_id").equals(webConfigId)) {
|
||||
resList.add(elem);
|
||||
}
|
||||
}
|
||||
return resList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test for FailureUrl
|
||||
* */
|
||||
private void testReadFailureUrl() {
|
||||
final List<Map<String, Object>> logList = readFailureUrl();
|
||||
final List<Map<String, Object>> logList = readFailureUrl(webConfigId);
|
||||
assertEquals(1, logList.size());
|
||||
}
|
||||
|
||||
private void testDeleteFailureUrl() {
|
||||
final List<Map<String, Object>> logList = readFailureUrl();
|
||||
final List<Map<String, Object>> logList = readFailureUrl(webConfigId);
|
||||
for (Map<String, Object> elem : logList) {
|
||||
deleteMethod("/api/admin/failureurl/log/" + elem.get("id")).then().body("response.status", equalTo(0));
|
||||
}
|
||||
|
||||
final List<Map<String, Object>> afterList = readFailureUrl();
|
||||
final List<Map<String, Object>> afterList = readFailureUrl(webConfigId);
|
||||
assertEquals(0, afterList.size()); // check if logs are successfully deleted
|
||||
}
|
||||
|
||||
private static List<Map<String, Object>> readFailureUrl() {
|
||||
final List<Map<String, Object>> logList = readLogItems("failureurl");
|
||||
final List<Map<String, Object>> resList = new ArrayList<>();
|
||||
for (Map<String, Object> elem : logList) {
|
||||
if (elem.containsKey("thread_name") && elem.get("thread_name").toString().startsWith("Crawler-" + webConfigId)) {
|
||||
resList.add(elem);
|
||||
}
|
||||
}
|
||||
return resList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test for SearchList
|
||||
* */
|
||||
|
|
@ -352,52 +259,4 @@ public class CrawlerLogTests extends ITBase {
|
|||
final List<Map<String, Object>> results = JsonPath.from(response).getList("response.docs");
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Utilities
|
||||
* */
|
||||
private static List<Map<String, Object>> readLogItems(final String apiName) {
|
||||
final Map<String, Object> searchBody = new HashMap<>();
|
||||
final String response = checkMethodBase(searchBody).get("/api/admin/" + apiName + "/logs").asString();
|
||||
final List<Map<String, Object>> itemList = JsonPath.from(response).getList("response.logs");
|
||||
return itemList;
|
||||
}
|
||||
|
||||
private static String getJsonResponse(final String path) {
|
||||
final Map<String, Object> searchBody = new HashMap<>();
|
||||
final String response = checkMethodBase(searchBody).get(path).asString();
|
||||
return response;
|
||||
}
|
||||
|
||||
private static String getResponsePath() {
|
||||
return "response.settings.findAll {it.name.startsWith(\"" + NAME_PREFIX + "\")}";
|
||||
}
|
||||
|
||||
private static List<String> getWebConfigIds() {
|
||||
final String response = getJsonResponse("/api/admin/webconfig/settings");
|
||||
final List<String> idList = JsonPath.from(response).getList(getResponsePath() + ".id");
|
||||
return idList;
|
||||
}
|
||||
|
||||
private static List<String> getSchedulerIds() {
|
||||
final String response = getJsonResponse("/api/admin/scheduler/settings");
|
||||
final List<String> idList = JsonPath.from(response).getList(getResponsePath() + ".id");
|
||||
return idList;
|
||||
}
|
||||
|
||||
private static Map<String, Object> getSchedulerItem() {
|
||||
final String response = getJsonResponse("/api/admin/scheduler/settings");
|
||||
final List<Map<String, Object>> itemList = JsonPath.from(response).getList(getResponsePath());
|
||||
assertEquals(1, itemList.size());
|
||||
return itemList.get(0);
|
||||
}
|
||||
|
||||
private static String buildJobScript() {
|
||||
return String.format("return container.getComponent(\"crawlJob\")" + ".logLevel(\"info\")" + ".sessionId(\"%s\")"
|
||||
+ ".webConfigIds([\"%s\"] as String[])" + ".jobExecutor(executor).execute();", webConfigId, webConfigId);
|
||||
}
|
||||
|
||||
private static Response deleteMethod(final String path) {
|
||||
return given().header("Authorization", getTestToken()).delete(path);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
441
src/test/java/org/codelibs/fess/it/search/SearchApiTests.java
Normal file
441
src/test/java/org/codelibs/fess/it/search/SearchApiTests.java
Normal file
|
|
@ -0,0 +1,441 @@
|
|||
package org.codelibs.fess.it.search;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
|
||||
import io.restassured.RestAssured;
|
||||
import io.restassured.path.json.JsonPath;
|
||||
import io.restassured.response.Response;
|
||||
import org.codelibs.fess.Constants;
|
||||
import org.codelibs.fess.it.CrawlTestBase;
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Integration Tests which need an execution of crawler
|
||||
* - /json
|
||||
* */
|
||||
@Tag("it")
|
||||
public class SearchApiTests extends CrawlTestBase {
|
||||
private static final Logger logger = LoggerFactory.getLogger(SearchApiTests.class);
|
||||
private static final String NAME_PREFIX = "searchApiTest_";
|
||||
private static final String DEFAULT_TESTDATA_PATH = "/tmp/fess-testdata";
|
||||
private static final String CRAWL_LABEL = NAME_PREFIX + "_label";
|
||||
private static final String TEST_LABEL = "tools";
|
||||
private static String fileConfigId;
|
||||
private static String labelId;
|
||||
private static String crawlLabelId;
|
||||
|
||||
@BeforeAll
|
||||
protected static void initAll() {
|
||||
RestAssured.baseURI = getFessUrl();
|
||||
settingTestToken();
|
||||
|
||||
// create and execute a file crawler
|
||||
try {
|
||||
labelId = createLabel();
|
||||
crawlLabelId = createCrawlLabel();
|
||||
|
||||
createFileConfig();
|
||||
logger.info("FileConfig is created");
|
||||
refresh();
|
||||
fileConfigId = getFileConfigIds(NAME_PREFIX).get(0);
|
||||
|
||||
createJob();
|
||||
logger.info("Job is created");
|
||||
refresh();
|
||||
|
||||
startJob(NAME_PREFIX);
|
||||
|
||||
waitJob(NAME_PREFIX);
|
||||
refresh();
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
assertTrue(false);
|
||||
}
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
protected void init() {
|
||||
refresh();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
protected void tearDown() {
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
protected static void tearDownAll() {
|
||||
deleteDocuments("label:" + CRAWL_LABEL);
|
||||
deleteLabel(labelId);
|
||||
deleteLabel(crawlLabelId);
|
||||
|
||||
final List<Map<String, Object>> jobLogList = readJobLog(NAME_PREFIX);
|
||||
for (Map<String, Object> elem : jobLogList) {
|
||||
deleteMethod("/api/admin/joblog/log/" + elem.get("id"));
|
||||
}
|
||||
|
||||
final List<Map<String, Object>> crawlingInfoList = readCrawlingInfo(fileConfigId);
|
||||
for (Map<String, Object> elem : crawlingInfoList) {
|
||||
deleteMethod("/api/admin/crawlinginfo/log/" + elem.get("id"));
|
||||
}
|
||||
|
||||
final List<Map<String, Object>> failureUrlList = readFailureUrl(fileConfigId);
|
||||
for (Map<String, Object> elem : failureUrlList) {
|
||||
deleteMethod("/api/admin/failurelog/log/" + elem.get("id"));
|
||||
}
|
||||
|
||||
final Map<String, Object> requestBody = new HashMap<>();
|
||||
requestBody.put("q", "Example Domain");
|
||||
checkMethodBase(requestBody).delete("/api/admin/searchlist/query");
|
||||
refresh();
|
||||
|
||||
for (String sId : getSchedulerIds(NAME_PREFIX)) {
|
||||
deleteMethod("/api/admin/scheduler/setting/" + sId);
|
||||
}
|
||||
|
||||
for (String fId : getFileConfigIds(NAME_PREFIX)) {
|
||||
deleteMethod("/api/admin/fileconfig/setting/" + fId);
|
||||
}
|
||||
|
||||
deleteTestToken();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void searchTestWith1Word() throws Exception {
|
||||
String query = "java";
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("q", query);
|
||||
params.put("num", "100");
|
||||
String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString();
|
||||
assertTrue(JsonPath.from(response).getInt("response.record_count") > 0);
|
||||
List<Map<String, Object>> docs = JsonPath.from(response).getList("response.result");
|
||||
for (Map<String, Object> doc : docs) {
|
||||
assertTrue(doc.entrySet().stream().anyMatch(entry -> entry.getValue().toString().toLowerCase().contains(query.toLowerCase())),
|
||||
doc.toString());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void searchTestWithMultipleWord() throws Exception {
|
||||
String query = "java public";
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("q", query);
|
||||
params.put("num", "100");
|
||||
String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString();
|
||||
assertTrue(JsonPath.from(response).getInt("response.record_count") > 0);
|
||||
List<Map<String, Object>> docs = JsonPath.from(response).getList("response.result");
|
||||
for (Map<String, Object> doc : docs) {
|
||||
for (String keyword : query.split(" ")) {
|
||||
assertTrue(
|
||||
doc.entrySet().stream()
|
||||
.anyMatch(entry -> entry.getValue().toString().toLowerCase().contains(keyword.toLowerCase())),
|
||||
doc.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void searchTestWithAndOperation() throws Exception {
|
||||
String query = "java AND public";
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("q", query);
|
||||
params.put("num", "100");
|
||||
String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString();
|
||||
assertTrue(JsonPath.from(response).getInt("response.record_count") > 0);
|
||||
List<Map<String, Object>> docs = JsonPath.from(response).getList("response.result");
|
||||
for (Map<String, Object> doc : docs) {
|
||||
for (String keyword : query.split(" ")) {
|
||||
if (keyword.equals("AND")) {
|
||||
continue;
|
||||
}
|
||||
assertTrue(
|
||||
doc.entrySet().stream()
|
||||
.anyMatch(entry -> entry.getValue().toString().toLowerCase().contains(keyword.toLowerCase())),
|
||||
doc.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void searchTestWithOrOperation() throws Exception {
|
||||
String query = "java OR public";
|
||||
|
||||
Map<String, String> andParams = new HashMap<>();
|
||||
andParams.put("q", query.replace("OR", "AND"));
|
||||
String andResponse = checkMethodBase(new HashMap<>()).params(andParams).get("/json").asString();
|
||||
int andRecordCount = JsonPath.from(andResponse).getInt("response.record_count");
|
||||
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("q", query);
|
||||
params.put("num", "100");
|
||||
String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString();
|
||||
assertTrue(JsonPath.from(response).getInt("response.record_count") > andRecordCount);
|
||||
List<Map<String, Object>> docs = JsonPath.from(response).getList("response.result");
|
||||
for (Map<String, Object> doc : docs) {
|
||||
boolean match = false;
|
||||
for (String keyword : query.split(" ")) {
|
||||
if (keyword.equals("OR")) {
|
||||
continue;
|
||||
}
|
||||
if (doc.entrySet().stream().anyMatch(entry -> entry.getValue().toString().toLowerCase().contains(keyword.toLowerCase()))) {
|
||||
match = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assertTrue(match, doc.toString());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void searchTestWithNotOperation() throws Exception {
|
||||
String query = "java NOT public";
|
||||
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("q", query);
|
||||
params.put("num", "100");
|
||||
String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString();
|
||||
assertTrue(JsonPath.from(response).getInt("response.record_count") > 0);
|
||||
List<Map<String, Object>> docs = JsonPath.from(response).getList("response.result");
|
||||
for (Map<String, Object> doc : docs) {
|
||||
String[] keywords = query.split(" ");
|
||||
assertTrue(
|
||||
doc.entrySet().stream()
|
||||
.anyMatch(entry -> entry.getValue().toString().toLowerCase().contains(keywords[0].toLowerCase())),
|
||||
doc.toString());
|
||||
assertFalse(
|
||||
doc.entrySet().stream()
|
||||
.anyMatch(entry -> entry.getValue().toString().toLowerCase().contains(keywords[2].toLowerCase())),
|
||||
doc.toString());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void searchTestWithLabel() throws Exception {
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("fields.label", TEST_LABEL);
|
||||
params.put("num", "100");
|
||||
String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString();
|
||||
assertTrue(JsonPath.from(response).getInt("response.record_count") > 0);
|
||||
List<Map<String, Object>> docs = JsonPath.from(response).getList("response.result");
|
||||
for (Map<String, Object> doc : docs) {
|
||||
assertTrue(doc.get("url").toString().toLowerCase().contains(TEST_LABEL), doc.toString());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void searchTestWithFieldQery() throws Exception {
|
||||
String query = "content:java";
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("q", query);
|
||||
params.put("num", "100");
|
||||
String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString();
|
||||
assertTrue(JsonPath.from(response).getInt("response.record_count") > 0);
|
||||
List<Map<String, Object>> docs = JsonPath.from(response).getList("response.result");
|
||||
for (Map<String, Object> doc : docs) {
|
||||
String[] fieldAndValue = query.split(":");
|
||||
assertTrue(doc.get("content_description").toString().toLowerCase().contains(fieldAndValue[1]), doc.toString());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void searchTestWithSort() throws Exception {
|
||||
String sortField = "content_length";
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("q", "*");
|
||||
params.put("sort", sortField + ".asc");
|
||||
params.put("num", "100");
|
||||
String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString();
|
||||
assertTrue(JsonPath.from(response).getInt("response.record_count") > 10);
|
||||
List<Map<String, Object>> docs = JsonPath.from(response).getList("response.result");
|
||||
int prevVal = 0;
|
||||
for (Map<String, Object> doc : docs) {
|
||||
int sortValue = Integer.parseInt(doc.get(sortField).toString());
|
||||
assertTrue(sortValue >= prevVal);
|
||||
prevVal = sortValue;
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void searchTestWithWildcard() throws Exception {
|
||||
String field = "filetype";
|
||||
String query = "others";
|
||||
String wcQuery1 = "othe*";
|
||||
String wcQuery2 = "oth??s";
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("q", field + ":" + query);
|
||||
params.put("num", "100");
|
||||
String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString();
|
||||
int recordCount = JsonPath.from(response).getInt("response.record_count");
|
||||
assertTrue(recordCount > 0);
|
||||
|
||||
Map<String, String> wcParams1 = new HashMap<>();
|
||||
wcParams1.put("q", field + ":" + wcQuery1);
|
||||
wcParams1.put("num", "100");
|
||||
String wcResponse1 = checkMethodBase(new HashMap<>()).params(wcParams1).get("/json").asString();
|
||||
int wcRecordCount1 = JsonPath.from(wcResponse1).getInt("response.record_count");
|
||||
assertEquals(recordCount, wcRecordCount1);
|
||||
|
||||
Map<String, String> wcParams2 = new HashMap<>();
|
||||
wcParams2.put("q", field + ":" + wcQuery2);
|
||||
wcParams2.put("num", "100");
|
||||
String wcResponse2 = checkMethodBase(new HashMap<>()).params(wcParams2).get("/json").asString();
|
||||
int wcRecordCount2 = JsonPath.from(wcResponse2).getInt("response.record_count");
|
||||
assertEquals(recordCount, wcRecordCount2);
|
||||
|
||||
List<Map<String, Object>> docs1 = JsonPath.from(wcResponse1).getList("response.result");
|
||||
for (Map<String, Object> doc : docs1) {
|
||||
assertTrue(doc.get(field).toString().toLowerCase().contains(query), doc.toString());
|
||||
}
|
||||
List<Map<String, Object>> docs2 = JsonPath.from(wcResponse2).getList("response.result");
|
||||
for (Map<String, Object> doc : docs2) {
|
||||
assertTrue(doc.get(field).toString().toLowerCase().contains(query), doc.toString());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void searchTestWithRange() throws Exception {
|
||||
String field = "content_length";
|
||||
int from = 100;
|
||||
int to = 1000;
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("q", field + ":[" + from + " TO " + to + "]");
|
||||
params.put("num", "100");
|
||||
String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString();
|
||||
assertTrue(JsonPath.from(response).getInt("response.record_count") > 0);
|
||||
List<Map<String, Object>> docs = JsonPath.from(response).getList("response.result");
|
||||
for (Map<String, Object> doc : docs) {
|
||||
int value = Integer.parseInt(doc.get(field).toString());
|
||||
assertTrue(value >= from);
|
||||
assertTrue(value <= to);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void searchTestWithBoost() throws Exception {
|
||||
String field = "content";
|
||||
String query1 = "java";
|
||||
String query2 = "sample";
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("q", field + ":" + query1 + "^1000 OR " + field + ":" + query2);
|
||||
params.put("num", "100");
|
||||
String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString();
|
||||
assertTrue(JsonPath.from(response).getInt("response.record_count") > 0);
|
||||
List<Map<String, Object>> docs = JsonPath.from(response).getList("response.result");
|
||||
boolean switchFlg = false;
|
||||
for (Map<String, Object> doc : docs) {
|
||||
if (!switchFlg) {
|
||||
boolean contains = doc.get("content_description").toString().toLowerCase().contains(query1);
|
||||
if (!contains) {
|
||||
switchFlg = true;
|
||||
}
|
||||
} else {
|
||||
assertTrue(doc.get("content_description").toString().toLowerCase().contains(query2), doc.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void searchTestWithFuzzy() throws Exception {
|
||||
String field = "content";
|
||||
String query = "java";
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("q", field + ":" + query);
|
||||
params.put("num", "100");
|
||||
String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString();
|
||||
int recordCount = JsonPath.from(response).getInt("response.record_count");
|
||||
assertTrue(recordCount > 0);
|
||||
|
||||
Map<String, String> fuzzyParams1 = new HashMap<>();
|
||||
fuzzyParams1.put("q", field + ":" + query + "~");
|
||||
fuzzyParams1.put("num", "100");
|
||||
String fuzzyResponse1 = checkMethodBase(new HashMap<>()).params(fuzzyParams1).get("/json").asString();
|
||||
int wcRecordCount1 = JsonPath.from(fuzzyResponse1).getInt("response.record_count");
|
||||
assertTrue(recordCount < wcRecordCount1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void searchTestWithInUrl() throws Exception {
|
||||
String query = "tools";
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("q", "inurl:" + query);
|
||||
params.put("num", "100");
|
||||
String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString();
|
||||
assertTrue(JsonPath.from(response).getInt("response.record_count") > 0);
|
||||
List<Map<String, Object>> docs = JsonPath.from(response).getList("response.result");
|
||||
for (Map<String, Object> doc : docs) {
|
||||
assertTrue(doc.get("url").toString().toLowerCase().contains(query), doc.toString());
|
||||
}
|
||||
}
|
||||
|
||||
private static void createFileConfig() {
|
||||
final Map<String, Object> requestBody = new HashMap<>();
|
||||
final String paths = "file:" + getTestDataPath();
|
||||
requestBody.put("name", NAME_PREFIX + "FileConfig");
|
||||
requestBody.put("paths", paths);
|
||||
requestBody.put("excluded_paths", ".*\\.git.*");
|
||||
requestBody.put("max_access_count", 100);
|
||||
requestBody.put("num_of_thread", 1);
|
||||
requestBody.put("interval_time", 100);
|
||||
requestBody.put("boost", 100);
|
||||
requestBody.put("permissions", "{role}guest");
|
||||
requestBody.put("available", true);
|
||||
requestBody.put("sort_order", 0);
|
||||
createFileConfig(requestBody);
|
||||
}
|
||||
|
||||
private static void createJob() {
|
||||
final Map<String, Object> requestBody = new HashMap<>();
|
||||
requestBody.put("name", NAME_PREFIX + "Scheduler");
|
||||
requestBody.put("target", "all");
|
||||
requestBody.put("script_type", "groovy");
|
||||
requestBody.put("sort_order", 0);
|
||||
requestBody.put("crawler", Constants.ON);
|
||||
requestBody.put("job_logging", Constants.ON);
|
||||
requestBody.put("available", Constants.ON);
|
||||
requestBody.put("script_data", buildFileConfigJobScript(fileConfigId));
|
||||
createJob(requestBody);
|
||||
}
|
||||
|
||||
private static String createLabel() {
|
||||
Map<String, Object> labelBody = new HashMap<>();
|
||||
labelBody.put("name", TEST_LABEL);
|
||||
labelBody.put("value", TEST_LABEL);
|
||||
labelBody.put("included_paths", ".*tools.*");
|
||||
Response response = checkMethodBase(labelBody).put("/api/admin/labeltype/setting");
|
||||
JsonPath jsonPath = JsonPath.from(response.asString());
|
||||
assertTrue(jsonPath.getBoolean("response.created"));
|
||||
assertEquals(0, jsonPath.getInt("response.status"));
|
||||
return jsonPath.get("response.id");
|
||||
}
|
||||
|
||||
private static String createCrawlLabel() {
|
||||
Map<String, Object> labelBody = new HashMap<>();
|
||||
labelBody.put("name", CRAWL_LABEL);
|
||||
labelBody.put("value", CRAWL_LABEL);
|
||||
labelBody.put("included_paths", ".*");
|
||||
Response response = checkMethodBase(labelBody).put("/api/admin/labeltype/setting");
|
||||
JsonPath jsonPath = JsonPath.from(response.asString());
|
||||
assertTrue(jsonPath.getBoolean("response.created"));
|
||||
assertEquals(0, jsonPath.getInt("response.status"));
|
||||
return jsonPath.get("response.id");
|
||||
}
|
||||
|
||||
protected static void deleteLabel(String id) {
|
||||
checkMethodBase(new HashMap<>()).delete("/api/admin/labeltype/setting/" + id);
|
||||
}
|
||||
|
||||
public static String getTestDataPath() {
|
||||
return System.getProperty("test.testdata.path", DEFAULT_TESTDATA_PATH);
|
||||
}
|
||||
|
||||
protected String getApiPath() {
|
||||
return "json";
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue