diff --git a/src/test/java/org/codelibs/fess/it/CrawlTestBase.java b/src/test/java/org/codelibs/fess/it/CrawlTestBase.java new file mode 100644 index 000000000..ee716adee --- /dev/null +++ b/src/test/java/org/codelibs/fess/it/CrawlTestBase.java @@ -0,0 +1,227 @@ +/* + * Copyright 2012-2017 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.it; + +import io.restassured.path.json.JsonPath; +import io.restassured.response.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CrawlTestBase extends ITBase { + private static final Logger logger = LoggerFactory.getLogger(CrawlTestBase.class); + + private static final String DOC_INDEX_NAME = "fess.search"; + + private static final String DOC_TYPE_NAME = "doc"; + + protected static void createJob(final Map requestBody) { + checkMethodBase(requestBody).put("/api/admin/scheduler/setting").then().body("response.created", equalTo(true)) + .body("response.status", equalTo(0)); + } + + protected static void startJob(final String namePrefix) { + for (int i = 0; i < 30; i++) { + final Map requestBody = new HashMap<>(); + final String schedulerId = getSchedulerIds(namePrefix).get(0); + final Response response = checkMethodBase(requestBody).post("/api/admin/scheduler/" + schedulerId + "/start"); + if (response.getBody().jsonPath().getInt("response.status") == 0) { + logger.info("Start scheduler \"" + schedulerId + "\""); + return; + } + try { + Thread.sleep(1000L); + } catch (InterruptedException e) { + // ignore + } + } + assertTrue(false, "could not start job."); + } + + protected static void waitJob(final String namePrefix) throws InterruptedException { + Boolean isRunning = false; + int count = 0; + + while (count < 300 && !isRunning) { // Wait until the crawler starts + Thread.sleep(500); + count++; + final Map scheduler = getSchedulerItem(namePrefix); + assertTrue(scheduler.containsKey("running")); + isRunning = (Boolean) scheduler.get("running"); + } + if (300 <= count) { + logger.info("Time out: Failed to start crawler)"); + assertTrue(false); // Time Out + } + + logger.info("Crawler is running"); + count = 0; + isRunning = true; + while (count < 300 && isRunning) { // Wait until the crawler terminates + Thread.sleep(1000); + count++; + final Map scheduler = getSchedulerItem(namePrefix); + assertTrue(scheduler.containsKey("running")); + isRunning = (Boolean) scheduler.get("running"); + + } + if (300 <= count) { + logger.info("Time out: Crawler takes too much time"); + //TODO assertTrue(false); // Time Out + } + + logger.info("Crawler terminated"); + } + + protected static String createWebConfig(final Map requestBody) { + String response = checkMethodBase(requestBody).put("/api/admin/webconfig/setting").asString(); + JsonPath jsonPath = JsonPath.from(response); + assertTrue(jsonPath.getBoolean("response.created")); + assertEquals(0, jsonPath.getInt("response.status")); + return jsonPath.getString("response.id"); + } + + protected static List getWebConfigIds(final String namePrefix) { + final String response = getJsonResponse("/api/admin/webconfig/settings"); + final List idList = JsonPath.from(response).getList(getResponsePath(namePrefix) + ".id"); + return idList; + } + + protected static String createFileConfig(final Map requestBody) { + String response = checkMethodBase(requestBody).put("/api/admin/fileconfig/setting").asString(); + JsonPath jsonPath = JsonPath.from(response); + assertTrue(jsonPath.getBoolean("response.created")); + assertEquals(0, jsonPath.getInt("response.status")); + return jsonPath.getString("response.id"); + } + + protected static List getFileConfigIds(final String namePrefix) { + final String response = getJsonResponse("/api/admin/fileconfig/settings"); + final List idList = JsonPath.from(response).getList(getResponsePath(namePrefix) + ".id"); + return idList; + } + + protected static String getResponsePath(final String namePrefix) { + return "response.settings.findAll {it.name.startsWith(\"" + namePrefix + "\")}"; + } + + protected static String getJsonResponse(final String path) { + final Map searchBody = new HashMap<>(); + final String response = checkMethodBase(searchBody).get(path).asString(); + return response; + } + + protected static String buildWebConfigJobScript(final String webCofigId) { + return String.format("return container.getComponent(\"crawlJob\")" + ".logLevel(\"info\")" + ".sessionId(\"%s\")" + + ".webConfigIds([\"%s\"] as String[])" + ".jobExecutor(executor).execute();", webCofigId, webCofigId); + } + + protected static String buildFileConfigJobScript(final String fileConfigId) { + return String.format("return container.getComponent(\"crawlJob\")" + ".logLevel(\"info\")" + ".sessionId(\"%s\")" + + ".fileConfigIds([\"%s\"] as String[])" + ".jobExecutor(executor).execute();", fileConfigId, fileConfigId); + } + + protected static Map getSchedulerItem(final String namePrefix) { + final String response = getJsonResponse("/api/admin/scheduler/settings"); + final List> itemList = JsonPath.from(response).getList(getResponsePath(namePrefix)); + assertEquals(1, itemList.size()); + return itemList.get(0); + } + + protected static List getSchedulerIds(final String namePrefix) { + final String response = getJsonResponse("/api/admin/scheduler/settings"); + final List idList = JsonPath.from(response).getList(getResponsePath(namePrefix) + ".id"); + return idList; + } + + protected static List> readJobLog(final String namePrefix) { + final List> logList = readLogItems("joblog"); + final List> resList = new ArrayList<>(); + for (Map elem : logList) { + if (elem.containsKey("job_name") && elem.get("job_name").equals(namePrefix + "Scheduler")) { + resList.add(elem); + } + } + return resList; + } + + protected static List> readLogItems(final String apiName) { + final Map searchBody = new HashMap<>(); + final String response = checkMethodBase(searchBody).get("/api/admin/" + apiName + "/logs").asString(); + final List> itemList = JsonPath.from(response).getList("response.logs"); + return itemList; + } + + protected static Response deleteMethod(final String path) { + return given().header("Authorization", getTestToken()).delete(path); + } + + protected static void deleteDocuments(final String queryString) { + List docIds = new ArrayList<>(); + Response response = + given().param("scroll", "1m").param("q", queryString) + .get(getEsUrl() + "/" + DOC_INDEX_NAME + "/" + DOC_TYPE_NAME + "/_search"); + JsonPath jsonPath = JsonPath.from(response.asString()); + String scrollId = jsonPath.getString("_scroll_id"); + while (true) { + List resultIds = jsonPath.getList("hits.hits._id"); + if (resultIds.size() == 0) { + break; + } + docIds.addAll(resultIds); + Map scrollBody = new HashMap<>(); + scrollBody.put("scroll", "1m"); + scrollBody.put("scroll_id", scrollId); + response = given().body(scrollBody).get(getEsUrl() + "/_search/scroll"); + jsonPath = JsonPath.from(response.asString()); + } + + for (String docId : docIds) { + given().delete(getEsUrl() + "/" + DOC_INDEX_NAME + "/" + DOC_TYPE_NAME + "/" + docId); + } + } + + protected static List> readCrawlingInfo(final String configId) { + final List> logList = readLogItems("crawlinginfo"); + final List> resList = new ArrayList<>(); + for (Map elem : logList) { + if (elem.containsKey("session_id") && elem.get("session_id").equals(configId)) { + resList.add(elem); + } + } + return resList; + } + + protected static List> readFailureUrl(final String configId) { + final List> logList = readLogItems("failureurl"); + final List> resList = new ArrayList<>(); + for (Map elem : logList) { + if (elem.containsKey("thread_name") && elem.get("thread_name").toString().startsWith("Crawler-" + configId)) { + resList.add(elem); + } + } + return resList; + } +} diff --git a/src/test/java/org/codelibs/fess/it/admin/CrawlerLogTests.java b/src/test/java/org/codelibs/fess/it/admin/CrawlerLogTests.java index 566a252a5..07d1f5798 100644 --- a/src/test/java/org/codelibs/fess/it/admin/CrawlerLogTests.java +++ b/src/test/java/org/codelibs/fess/it/admin/CrawlerLogTests.java @@ -15,18 +15,16 @@ */ package org.codelibs.fess.it.admin; -import static io.restassured.RestAssured.given; import static org.hamcrest.Matchers.equalTo; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.codelibs.fess.Constants; -import org.codelibs.fess.it.ITBase; +import org.codelibs.fess.it.CrawlTestBase; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -38,7 +36,6 @@ import org.slf4j.LoggerFactory; import io.restassured.RestAssured; import io.restassured.path.json.JsonPath; -import io.restassured.response.Response; /** * Integration Tests which need an execution of crawler @@ -48,7 +45,7 @@ import io.restassured.response.Response; * - /api/admin/searchlist * */ @Tag("it") -public class CrawlerLogTests extends ITBase { +public class CrawlerLogTests extends CrawlTestBase { private static final Logger logger = LoggerFactory.getLogger(CrawlerLogTests.class); private static final String NAME_PREFIX = "crawlingInfoTest_"; @@ -65,15 +62,15 @@ public class CrawlerLogTests extends ITBase { createWebConfig(); logger.info("WebConfig is created"); refresh(); - webConfigId = getWebConfigIds().get(0); + webConfigId = getWebConfigIds(NAME_PREFIX).get(0); createJob(); logger.info("Job is created"); refresh(); - startJob(); + startJob(NAME_PREFIX); - waitJob(); + waitJob(NAME_PREFIX); refresh(); } catch (InterruptedException e) { e.printStackTrace(); @@ -92,17 +89,17 @@ public class CrawlerLogTests extends ITBase { @AfterAll protected static void tearDownAll() { - final List> jobLogList = readJobLog(); + final List> jobLogList = readJobLog(NAME_PREFIX); for (Map elem : jobLogList) { deleteMethod("/api/admin/joblog/log/" + elem.get("id")); } - final List> crawlingInfoList = readCrawlingInfo(); + final List> crawlingInfoList = readCrawlingInfo(webConfigId); for (Map elem : crawlingInfoList) { deleteMethod("/api/admin/crawlinginfo/log/" + elem.get("id")); } - final List> failureUrlList = readFailureUrl(); + final List> failureUrlList = readFailureUrl(webConfigId); for (Map elem : failureUrlList) { deleteMethod("/api/admin/failurelog/log/" + elem.get("id")); } @@ -112,11 +109,11 @@ public class CrawlerLogTests extends ITBase { checkMethodBase(requestBody).delete("/api/admin/searchlist/query"); refresh(); - for (String sId : getSchedulerIds()) { + for (String sId : getSchedulerIds(NAME_PREFIX)) { deleteMethod("/api/admin/scheduler/setting/" + sId); } - for (String wId : getWebConfigIds()) { + for (String wId : getWebConfigIds(NAME_PREFIX)) { deleteMethod("/api/admin/webconfig/setting/" + wId); } @@ -162,9 +159,7 @@ public class CrawlerLogTests extends ITBase { requestBody.put("boost", 100); requestBody.put("available", true); requestBody.put("sort_order", 0); - - checkMethodBase(requestBody).put("/api/admin/webconfig/setting").then().body("response.created", equalTo(true)) - .body("response.status", equalTo(0)); + createWebConfig(requestBody); } private static void createJob() { @@ -176,152 +171,64 @@ public class CrawlerLogTests extends ITBase { requestBody.put("crawler", Constants.ON); requestBody.put("job_logging", Constants.ON); requestBody.put("available", Constants.ON); - requestBody.put("script_data", buildJobScript()); - - checkMethodBase(requestBody).put("/api/admin/scheduler/setting").then().body("response.created", equalTo(true)) - .body("response.status", equalTo(0)); - } - - private static void startJob() { - for (int i = 0; i < 30; i++) { - final Map requestBody = new HashMap<>(); - final String schedulerId = getSchedulerIds().get(0); - final Response response = checkMethodBase(requestBody).post("/api/admin/scheduler/" + schedulerId + "/start"); - if (response.getBody().jsonPath().getInt("response.status") == 0) { - logger.info("Start scheduler \"" + schedulerId + "\""); - return; - } - try { - Thread.sleep(1000L); - } catch (InterruptedException e) { - // ignore - } - } - assertTrue(false, "could not start job."); - } - - private static void waitJob() throws InterruptedException { - Boolean isRunning = false; - int count = 0; - - while (count < 300 && !isRunning) { // Wait until the crawler starts - Thread.sleep(500); - count++; - final Map scheduler = getSchedulerItem(); - assertTrue(scheduler.containsKey("running")); - isRunning = (Boolean) scheduler.get("running"); - } - if (300 <= count) { - logger.info("Time out: Failed to start crawler)"); - assertTrue(false); // Time Out - } - - logger.info("Crawler is running"); - count = 0; - isRunning = true; - while (count < 300 && isRunning) { // Wait until the crawler terminates - Thread.sleep(1000); - count++; - final Map scheduler = getSchedulerItem(); - assertTrue(scheduler.containsKey("running")); - isRunning = (Boolean) scheduler.get("running"); - - } - if (300 <= count) { - logger.info("Time out: Crawler takes too much time"); - assertTrue(false); // Time Out - } - - logger.info("Crawler terminated"); + requestBody.put("script_data", buildWebConfigJobScript(webConfigId)); + createJob(requestBody); } /** * Test for JobLog * */ private void testReadJobLog() { - final List> logList = readJobLog(); + final List> logList = readJobLog(NAME_PREFIX); assertEquals(1, logList.size()); } private void testDeleteJobLog() { - final List> logList = readJobLog(); + final List> logList = readJobLog(NAME_PREFIX); for (Map elem : logList) { deleteMethod("/api/admin/joblog/log/" + elem.get("id")).then().body("response.status", equalTo(0)); } - final List> afterList = readJobLog(); + final List> afterList = readJobLog(NAME_PREFIX); assertEquals(0, afterList.size()); // check if logs are successfully deleted } - private static List> readJobLog() { - final List> logList = readLogItems("joblog"); - final List> resList = new ArrayList<>(); - for (Map elem : logList) { - if (elem.containsKey("job_name") && elem.get("job_name").equals(NAME_PREFIX + "Scheduler")) { - resList.add(elem); - } - } - return resList; - } - /** * Test for CrawlingInfo * */ private void testReadCrawlingInfo() { - final List> logList = readCrawlingInfo(); + final List> logList = readCrawlingInfo(webConfigId); assertEquals(1, logList.size()); } private void testDeleteCrawlingInfo() { - final List> logList = readCrawlingInfo(); + final List> logList = readCrawlingInfo(webConfigId); for (Map elem : logList) { deleteMethod("/api/admin/crawlinginfo/log/" + elem.get("id")).then().body("response.status", equalTo(0)); } - final List> afterList = readCrawlingInfo(); + final List> afterList = readCrawlingInfo(webConfigId); assertEquals(0, afterList.size()); // check if logs are successfully deleted } - private static List> readCrawlingInfo() { - final List> logList = readLogItems("crawlinginfo"); - final List> resList = new ArrayList<>(); - for (Map elem : logList) { - if (elem.containsKey("session_id") && elem.get("session_id").equals(webConfigId)) { - resList.add(elem); - } - } - return resList; - } - /** * Test for FailureUrl * */ private void testReadFailureUrl() { - final List> logList = readFailureUrl(); + final List> logList = readFailureUrl(webConfigId); assertEquals(1, logList.size()); } private void testDeleteFailureUrl() { - final List> logList = readFailureUrl(); + final List> logList = readFailureUrl(webConfigId); for (Map elem : logList) { deleteMethod("/api/admin/failureurl/log/" + elem.get("id")).then().body("response.status", equalTo(0)); } - final List> afterList = readFailureUrl(); + final List> afterList = readFailureUrl(webConfigId); assertEquals(0, afterList.size()); // check if logs are successfully deleted } - private static List> readFailureUrl() { - final List> logList = readLogItems("failureurl"); - final List> resList = new ArrayList<>(); - for (Map elem : logList) { - if (elem.containsKey("thread_name") && elem.get("thread_name").toString().startsWith("Crawler-" + webConfigId)) { - resList.add(elem); - } - } - return resList; - } - /** * Test for SearchList * */ @@ -352,52 +259,4 @@ public class CrawlerLogTests extends ITBase { final List> results = JsonPath.from(response).getList("response.docs"); return results; } - - /** - * Utilities - * */ - private static List> readLogItems(final String apiName) { - final Map searchBody = new HashMap<>(); - final String response = checkMethodBase(searchBody).get("/api/admin/" + apiName + "/logs").asString(); - final List> itemList = JsonPath.from(response).getList("response.logs"); - return itemList; - } - - private static String getJsonResponse(final String path) { - final Map searchBody = new HashMap<>(); - final String response = checkMethodBase(searchBody).get(path).asString(); - return response; - } - - private static String getResponsePath() { - return "response.settings.findAll {it.name.startsWith(\"" + NAME_PREFIX + "\")}"; - } - - private static List getWebConfigIds() { - final String response = getJsonResponse("/api/admin/webconfig/settings"); - final List idList = JsonPath.from(response).getList(getResponsePath() + ".id"); - return idList; - } - - private static List getSchedulerIds() { - final String response = getJsonResponse("/api/admin/scheduler/settings"); - final List idList = JsonPath.from(response).getList(getResponsePath() + ".id"); - return idList; - } - - private static Map getSchedulerItem() { - final String response = getJsonResponse("/api/admin/scheduler/settings"); - final List> itemList = JsonPath.from(response).getList(getResponsePath()); - assertEquals(1, itemList.size()); - return itemList.get(0); - } - - private static String buildJobScript() { - return String.format("return container.getComponent(\"crawlJob\")" + ".logLevel(\"info\")" + ".sessionId(\"%s\")" - + ".webConfigIds([\"%s\"] as String[])" + ".jobExecutor(executor).execute();", webConfigId, webConfigId); - } - - private static Response deleteMethod(final String path) { - return given().header("Authorization", getTestToken()).delete(path); - } } diff --git a/src/test/java/org/codelibs/fess/it/search/SearchApiTests.java b/src/test/java/org/codelibs/fess/it/search/SearchApiTests.java new file mode 100644 index 000000000..54bfcca7d --- /dev/null +++ b/src/test/java/org/codelibs/fess/it/search/SearchApiTests.java @@ -0,0 +1,441 @@ +package org.codelibs.fess.it.search; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import io.restassured.RestAssured; +import io.restassured.path.json.JsonPath; +import io.restassured.response.Response; +import org.codelibs.fess.Constants; +import org.codelibs.fess.it.CrawlTestBase; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Integration Tests which need an execution of crawler + * - /json + * */ +@Tag("it") +public class SearchApiTests extends CrawlTestBase { + private static final Logger logger = LoggerFactory.getLogger(SearchApiTests.class); + private static final String NAME_PREFIX = "searchApiTest_"; + private static final String DEFAULT_TESTDATA_PATH = "/tmp/fess-testdata"; + private static final String CRAWL_LABEL = NAME_PREFIX + "_label"; + private static final String TEST_LABEL = "tools"; + private static String fileConfigId; + private static String labelId; + private static String crawlLabelId; + + @BeforeAll + protected static void initAll() { + RestAssured.baseURI = getFessUrl(); + settingTestToken(); + + // create and execute a file crawler + try { + labelId = createLabel(); + crawlLabelId = createCrawlLabel(); + + createFileConfig(); + logger.info("FileConfig is created"); + refresh(); + fileConfigId = getFileConfigIds(NAME_PREFIX).get(0); + + createJob(); + logger.info("Job is created"); + refresh(); + + startJob(NAME_PREFIX); + + waitJob(NAME_PREFIX); + refresh(); + } catch (InterruptedException e) { + e.printStackTrace(); + assertTrue(false); + } + } + + @BeforeEach + protected void init() { + refresh(); + } + + @AfterEach + protected void tearDown() { + } + + @AfterAll + protected static void tearDownAll() { + deleteDocuments("label:" + CRAWL_LABEL); + deleteLabel(labelId); + deleteLabel(crawlLabelId); + + final List> jobLogList = readJobLog(NAME_PREFIX); + for (Map elem : jobLogList) { + deleteMethod("/api/admin/joblog/log/" + elem.get("id")); + } + + final List> crawlingInfoList = readCrawlingInfo(fileConfigId); + for (Map elem : crawlingInfoList) { + deleteMethod("/api/admin/crawlinginfo/log/" + elem.get("id")); + } + + final List> failureUrlList = readFailureUrl(fileConfigId); + for (Map elem : failureUrlList) { + deleteMethod("/api/admin/failurelog/log/" + elem.get("id")); + } + + final Map requestBody = new HashMap<>(); + requestBody.put("q", "Example Domain"); + checkMethodBase(requestBody).delete("/api/admin/searchlist/query"); + refresh(); + + for (String sId : getSchedulerIds(NAME_PREFIX)) { + deleteMethod("/api/admin/scheduler/setting/" + sId); + } + + for (String fId : getFileConfigIds(NAME_PREFIX)) { + deleteMethod("/api/admin/fileconfig/setting/" + fId); + } + + deleteTestToken(); + } + + @Test + public void searchTestWith1Word() throws Exception { + String query = "java"; + Map params = new HashMap<>(); + params.put("q", query); + params.put("num", "100"); + String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString(); + assertTrue(JsonPath.from(response).getInt("response.record_count") > 0); + List> docs = JsonPath.from(response).getList("response.result"); + for (Map doc : docs) { + assertTrue(doc.entrySet().stream().anyMatch(entry -> entry.getValue().toString().toLowerCase().contains(query.toLowerCase())), + doc.toString()); + } + } + + @Test + public void searchTestWithMultipleWord() throws Exception { + String query = "java public"; + Map params = new HashMap<>(); + params.put("q", query); + params.put("num", "100"); + String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString(); + assertTrue(JsonPath.from(response).getInt("response.record_count") > 0); + List> docs = JsonPath.from(response).getList("response.result"); + for (Map doc : docs) { + for (String keyword : query.split(" ")) { + assertTrue( + doc.entrySet().stream() + .anyMatch(entry -> entry.getValue().toString().toLowerCase().contains(keyword.toLowerCase())), + doc.toString()); + } + } + } + + @Test + public void searchTestWithAndOperation() throws Exception { + String query = "java AND public"; + Map params = new HashMap<>(); + params.put("q", query); + params.put("num", "100"); + String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString(); + assertTrue(JsonPath.from(response).getInt("response.record_count") > 0); + List> docs = JsonPath.from(response).getList("response.result"); + for (Map doc : docs) { + for (String keyword : query.split(" ")) { + if (keyword.equals("AND")) { + continue; + } + assertTrue( + doc.entrySet().stream() + .anyMatch(entry -> entry.getValue().toString().toLowerCase().contains(keyword.toLowerCase())), + doc.toString()); + } + } + } + + @Test + public void searchTestWithOrOperation() throws Exception { + String query = "java OR public"; + + Map andParams = new HashMap<>(); + andParams.put("q", query.replace("OR", "AND")); + String andResponse = checkMethodBase(new HashMap<>()).params(andParams).get("/json").asString(); + int andRecordCount = JsonPath.from(andResponse).getInt("response.record_count"); + + Map params = new HashMap<>(); + params.put("q", query); + params.put("num", "100"); + String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString(); + assertTrue(JsonPath.from(response).getInt("response.record_count") > andRecordCount); + List> docs = JsonPath.from(response).getList("response.result"); + for (Map doc : docs) { + boolean match = false; + for (String keyword : query.split(" ")) { + if (keyword.equals("OR")) { + continue; + } + if (doc.entrySet().stream().anyMatch(entry -> entry.getValue().toString().toLowerCase().contains(keyword.toLowerCase()))) { + match = true; + break; + } + } + assertTrue(match, doc.toString()); + } + } + + @Test + public void searchTestWithNotOperation() throws Exception { + String query = "java NOT public"; + + Map params = new HashMap<>(); + params.put("q", query); + params.put("num", "100"); + String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString(); + assertTrue(JsonPath.from(response).getInt("response.record_count") > 0); + List> docs = JsonPath.from(response).getList("response.result"); + for (Map doc : docs) { + String[] keywords = query.split(" "); + assertTrue( + doc.entrySet().stream() + .anyMatch(entry -> entry.getValue().toString().toLowerCase().contains(keywords[0].toLowerCase())), + doc.toString()); + assertFalse( + doc.entrySet().stream() + .anyMatch(entry -> entry.getValue().toString().toLowerCase().contains(keywords[2].toLowerCase())), + doc.toString()); + } + } + + @Test + public void searchTestWithLabel() throws Exception { + Map params = new HashMap<>(); + params.put("fields.label", TEST_LABEL); + params.put("num", "100"); + String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString(); + assertTrue(JsonPath.from(response).getInt("response.record_count") > 0); + List> docs = JsonPath.from(response).getList("response.result"); + for (Map doc : docs) { + assertTrue(doc.get("url").toString().toLowerCase().contains(TEST_LABEL), doc.toString()); + } + } + + @Test + public void searchTestWithFieldQery() throws Exception { + String query = "content:java"; + Map params = new HashMap<>(); + params.put("q", query); + params.put("num", "100"); + String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString(); + assertTrue(JsonPath.from(response).getInt("response.record_count") > 0); + List> docs = JsonPath.from(response).getList("response.result"); + for (Map doc : docs) { + String[] fieldAndValue = query.split(":"); + assertTrue(doc.get("content_description").toString().toLowerCase().contains(fieldAndValue[1]), doc.toString()); + } + } + + @Test + public void searchTestWithSort() throws Exception { + String sortField = "content_length"; + Map params = new HashMap<>(); + params.put("q", "*"); + params.put("sort", sortField + ".asc"); + params.put("num", "100"); + String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString(); + assertTrue(JsonPath.from(response).getInt("response.record_count") > 10); + List> docs = JsonPath.from(response).getList("response.result"); + int prevVal = 0; + for (Map doc : docs) { + int sortValue = Integer.parseInt(doc.get(sortField).toString()); + assertTrue(sortValue >= prevVal); + prevVal = sortValue; + } + } + + @Test + public void searchTestWithWildcard() throws Exception { + String field = "filetype"; + String query = "others"; + String wcQuery1 = "othe*"; + String wcQuery2 = "oth??s"; + Map params = new HashMap<>(); + params.put("q", field + ":" + query); + params.put("num", "100"); + String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString(); + int recordCount = JsonPath.from(response).getInt("response.record_count"); + assertTrue(recordCount > 0); + + Map wcParams1 = new HashMap<>(); + wcParams1.put("q", field + ":" + wcQuery1); + wcParams1.put("num", "100"); + String wcResponse1 = checkMethodBase(new HashMap<>()).params(wcParams1).get("/json").asString(); + int wcRecordCount1 = JsonPath.from(wcResponse1).getInt("response.record_count"); + assertEquals(recordCount, wcRecordCount1); + + Map wcParams2 = new HashMap<>(); + wcParams2.put("q", field + ":" + wcQuery2); + wcParams2.put("num", "100"); + String wcResponse2 = checkMethodBase(new HashMap<>()).params(wcParams2).get("/json").asString(); + int wcRecordCount2 = JsonPath.from(wcResponse2).getInt("response.record_count"); + assertEquals(recordCount, wcRecordCount2); + + List> docs1 = JsonPath.from(wcResponse1).getList("response.result"); + for (Map doc : docs1) { + assertTrue(doc.get(field).toString().toLowerCase().contains(query), doc.toString()); + } + List> docs2 = JsonPath.from(wcResponse2).getList("response.result"); + for (Map doc : docs2) { + assertTrue(doc.get(field).toString().toLowerCase().contains(query), doc.toString()); + } + } + + @Test + public void searchTestWithRange() throws Exception { + String field = "content_length"; + int from = 100; + int to = 1000; + Map params = new HashMap<>(); + params.put("q", field + ":[" + from + " TO " + to + "]"); + params.put("num", "100"); + String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString(); + assertTrue(JsonPath.from(response).getInt("response.record_count") > 0); + List> docs = JsonPath.from(response).getList("response.result"); + for (Map doc : docs) { + int value = Integer.parseInt(doc.get(field).toString()); + assertTrue(value >= from); + assertTrue(value <= to); + } + } + + @Test + public void searchTestWithBoost() throws Exception { + String field = "content"; + String query1 = "java"; + String query2 = "sample"; + Map params = new HashMap<>(); + params.put("q", field + ":" + query1 + "^1000 OR " + field + ":" + query2); + params.put("num", "100"); + String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString(); + assertTrue(JsonPath.from(response).getInt("response.record_count") > 0); + List> docs = JsonPath.from(response).getList("response.result"); + boolean switchFlg = false; + for (Map doc : docs) { + if (!switchFlg) { + boolean contains = doc.get("content_description").toString().toLowerCase().contains(query1); + if (!contains) { + switchFlg = true; + } + } else { + assertTrue(doc.get("content_description").toString().toLowerCase().contains(query2), doc.toString()); + } + } + } + + @Test + public void searchTestWithFuzzy() throws Exception { + String field = "content"; + String query = "java"; + Map params = new HashMap<>(); + params.put("q", field + ":" + query); + params.put("num", "100"); + String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString(); + int recordCount = JsonPath.from(response).getInt("response.record_count"); + assertTrue(recordCount > 0); + + Map fuzzyParams1 = new HashMap<>(); + fuzzyParams1.put("q", field + ":" + query + "~"); + fuzzyParams1.put("num", "100"); + String fuzzyResponse1 = checkMethodBase(new HashMap<>()).params(fuzzyParams1).get("/json").asString(); + int wcRecordCount1 = JsonPath.from(fuzzyResponse1).getInt("response.record_count"); + assertTrue(recordCount < wcRecordCount1); + } + + @Test + public void searchTestWithInUrl() throws Exception { + String query = "tools"; + Map params = new HashMap<>(); + params.put("q", "inurl:" + query); + params.put("num", "100"); + String response = checkMethodBase(new HashMap<>()).params(params).get("/json").asString(); + assertTrue(JsonPath.from(response).getInt("response.record_count") > 0); + List> docs = JsonPath.from(response).getList("response.result"); + for (Map doc : docs) { + assertTrue(doc.get("url").toString().toLowerCase().contains(query), doc.toString()); + } + } + + private static void createFileConfig() { + final Map requestBody = new HashMap<>(); + final String paths = "file:" + getTestDataPath(); + requestBody.put("name", NAME_PREFIX + "FileConfig"); + requestBody.put("paths", paths); + requestBody.put("excluded_paths", ".*\\.git.*"); + requestBody.put("max_access_count", 100); + requestBody.put("num_of_thread", 1); + requestBody.put("interval_time", 100); + requestBody.put("boost", 100); + requestBody.put("permissions", "{role}guest"); + requestBody.put("available", true); + requestBody.put("sort_order", 0); + createFileConfig(requestBody); + } + + private static void createJob() { + final Map requestBody = new HashMap<>(); + requestBody.put("name", NAME_PREFIX + "Scheduler"); + requestBody.put("target", "all"); + requestBody.put("script_type", "groovy"); + requestBody.put("sort_order", 0); + requestBody.put("crawler", Constants.ON); + requestBody.put("job_logging", Constants.ON); + requestBody.put("available", Constants.ON); + requestBody.put("script_data", buildFileConfigJobScript(fileConfigId)); + createJob(requestBody); + } + + private static String createLabel() { + Map labelBody = new HashMap<>(); + labelBody.put("name", TEST_LABEL); + labelBody.put("value", TEST_LABEL); + labelBody.put("included_paths", ".*tools.*"); + Response response = checkMethodBase(labelBody).put("/api/admin/labeltype/setting"); + JsonPath jsonPath = JsonPath.from(response.asString()); + assertTrue(jsonPath.getBoolean("response.created")); + assertEquals(0, jsonPath.getInt("response.status")); + return jsonPath.get("response.id"); + } + + private static String createCrawlLabel() { + Map labelBody = new HashMap<>(); + labelBody.put("name", CRAWL_LABEL); + labelBody.put("value", CRAWL_LABEL); + labelBody.put("included_paths", ".*"); + Response response = checkMethodBase(labelBody).put("/api/admin/labeltype/setting"); + JsonPath jsonPath = JsonPath.from(response.asString()); + assertTrue(jsonPath.getBoolean("response.created")); + assertEquals(0, jsonPath.getInt("response.status")); + return jsonPath.get("response.id"); + } + + protected static void deleteLabel(String id) { + checkMethodBase(new HashMap<>()).delete("/api/admin/labeltype/setting/" + id); + } + + public static String getTestDataPath() { + return System.getProperty("test.testdata.path", DEFAULT_TESTDATA_PATH); + } + + protected String getApiPath() { + return "json"; + } +}