fix #2241 replace unicode format

src/main/java/org/codelibs/fess/helper/ViewHelper.java

@@ -132,7 +132,7 @@ public class ViewHelper {
 
     protected String escapedHighlightPost = null;
 
-    protected Set<Integer> hihglightTerminalCharSet = new HashSet<>();
+    protected Set<Integer> highlightTerminalCharSet = new HashSet<>();
 
     protected ActionHook actionHook = new ActionHook();
 
@@ -146,7 +146,9 @@ public class ViewHelper {
         highlightTagPre = fessConfig.getQueryHighlightTagPre();
         highlightTagPost = fessConfig.getQueryHighlightTagPost();
         highlightedFields = fessConfig.getQueryHighlightContentDescriptionFieldsAsArray();
-        fessConfig.getQueryHighlightTerminalChars().codePoints().forEach(hihglightTerminalCharSet::add);
+        for (int v : fessConfig.getQueryHighlightTerminalCharsAsArray()) {
+            highlightTerminalCharSet.add(v);
+        }
         try {
             final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
             servletContext.setSessionTrackingModes(fessConfig.getSessionTrackingModesAsSet().stream().map(SessionTrackingMode::valueOf)
@@ -211,7 +213,7 @@ public class ViewHelper {
         int pos = escaped.indexOf(escapedHighlightPre);
         while (pos >= 0) {
             final int c = escaped.codePointAt(pos);
-            if (Character.isISOControl(c) || hihglightTerminalCharSet.contains(c)) {
+            if (Character.isISOControl(c) || highlightTerminalCharSet.contains(c)) {
                 break;
             }
             pos--;

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -657,7 +657,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. true */
     String QUERY_REPLACE_TERM_WITH_PREFIX_QUERY = "query.replace.term.with.prefix.query";
 
-    /** The key of the configuration. e.g. !.?։؟۔܀܁܂।၊။።፧፨᙮᠃᠉‼‽⁇⁈⁉。﹒﹗！．？｡ */
+    /** The key of the configuration. e.g. u0021u002Cu002Eu003Fu0589u061Fu06D4u0700u0701u0702u0964u104Au104Bu1362u1367u1368u166Eu1803u1809u203Cu203Du2047u2048u2049u3002uFE52uFE57uFF01uFF0EuFF1FuFF61 */
     String QUERY_HIGHLIGHT_TERMINAL_CHARS = "query.highlight.terminal.chars";
 
     /** The key of the configuration. e.g. 60 */
@@ -675,13 +675,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. </strong> */
     String QUERY_HIGHLIGHT_TAG_POST = "query.highlight.tag.post";
 
-    /** The key of the configuration. e.g.  */
+    /** The key of the configuration. e.g. u0009u000Au0013u0020 */
     String QUERY_HIGHLIGHT_BOUNDARY_CHARS = "query.highlight.boundary.chars";
 
     /** The key of the configuration. e.g. 20 */
     String QUERY_HIGHLIGHT_BOUNDARY_MAX_SCAN = "query.highlight.boundary.max.scan";
 
-    /** The key of the configuration. e.g. sentence */
+    /** The key of the configuration. e.g. chars */
     String QUERY_HIGHLIGHT_BOUNDARY_SCANNER = "query.highlight.boundary.scanner";
 
     /** The key of the configuration. e.g. default */
@@ -3396,7 +3396,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
 
     /**
      * Get the value for the key 'query.highlight.terminal.chars'. <br>
-     * The value is, e.g. !.?։؟۔܀܁܂।၊။።፧፨᙮᠃᠉‼‽⁇⁈⁉。﹒﹗！．？｡ <br>
+     * The value is, e.g. u0021u002Cu002Eu003Fu0589u061Fu06D4u0700u0701u0702u0964u104Au104Bu1362u1367u1368u166Eu1803u1809u203Cu203Du2047u2048u2049u3002uFE52uFE57uFF01uFF0EuFF1FuFF61 <br>
      * @return The value of found property. (NotNull: if not found, exception but basically no way)
      */
     String getQueryHighlightTerminalChars();
@@ -3454,19 +3454,11 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
 
     /**
      * Get the value for the key 'query.highlight.boundary.chars'. <br>
-     * The value is, e.g.  <br>
+     * The value is, e.g. u0009u000Au0013u0020 <br>
      * @return The value of found property. (NotNull: if not found, exception but basically no way)
      */
     String getQueryHighlightBoundaryChars();
 
-    /**
-     * Get the value for the key 'query.highlight.boundary.chars' as {@link Integer}. <br>
-     * The value is, e.g.  <br>
-     * @return The value of found property. (NotNull: if not found, exception but basically no way)
-     * @throws NumberFormatException When the property is not integer.
-     */
-    Integer getQueryHighlightBoundaryCharsAsInteger();
-
     /**
      * Get the value for the key 'query.highlight.boundary.max.scan'. <br>
      * The value is, e.g. 20 <br>
@@ -3484,7 +3476,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
 
     /**
      * Get the value for the key 'query.highlight.boundary.scanner'. <br>
-     * The value is, e.g. sentence <br>
+     * The value is, e.g. chars <br>
      * @return The value of found property. (NotNull: if not found, exception but basically no way)
      */
     String getQueryHighlightBoundaryScanner();
@@ -7143,10 +7135,6 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return get(FessConfig.QUERY_HIGHLIGHT_BOUNDARY_CHARS);
         }
 
-        public Integer getQueryHighlightBoundaryCharsAsInteger() {
-            return getAsInteger(FessConfig.QUERY_HIGHLIGHT_BOUNDARY_CHARS);
-        }
-
         public String getQueryHighlightBoundaryMaxScan() {
             return get(FessConfig.QUERY_HIGHLIGHT_BOUNDARY_MAX_SCAN);
         }
@@ -8743,15 +8731,17 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap.put(FessConfig.QUERY_GEO_FIELDS, "location");
             defaultMap.put(FessConfig.QUERY_BROWSER_LANG_PARAMETER_NAME, "browser_lang");
             defaultMap.put(FessConfig.QUERY_REPLACE_TERM_WITH_PREFIX_QUERY, "true");
-            defaultMap.put(FessConfig.QUERY_HIGHLIGHT_TERMINAL_CHARS, "!.?։؟۔܀܁܂।၊။።፧፨᙮᠃᠉‼‽⁇⁈⁉。﹒﹗！．？｡");
+            defaultMap
+                    .put(FessConfig.QUERY_HIGHLIGHT_TERMINAL_CHARS,
+                            "u0021u002Cu002Eu003Fu0589u061Fu06D4u0700u0701u0702u0964u104Au104Bu1362u1367u1368u166Eu1803u1809u203Cu203Du2047u2048u2049u3002uFE52uFE57uFF01uFF0EuFF1FuFF61");
             defaultMap.put(FessConfig.QUERY_HIGHLIGHT_FRAGMENT_SIZE, "60");
             defaultMap.put(FessConfig.QUERY_HIGHLIGHT_NUMBER_OF_FRAGMENTS, "2");
             defaultMap.put(FessConfig.QUERY_HIGHLIGHT_TYPE, "fvh");
             defaultMap.put(FessConfig.QUERY_HIGHLIGHT_TAG_PRE, "<strong>");
             defaultMap.put(FessConfig.QUERY_HIGHLIGHT_TAG_POST, "</strong>");
-            defaultMap.put(FessConfig.QUERY_HIGHLIGHT_BOUNDARY_CHARS, "\t\n ");
+            defaultMap.put(FessConfig.QUERY_HIGHLIGHT_BOUNDARY_CHARS, "u0009u000Au0013u0020");
             defaultMap.put(FessConfig.QUERY_HIGHLIGHT_BOUNDARY_MAX_SCAN, "20");
-            defaultMap.put(FessConfig.QUERY_HIGHLIGHT_BOUNDARY_SCANNER, "sentence");
+            defaultMap.put(FessConfig.QUERY_HIGHLIGHT_BOUNDARY_SCANNER, "chars");
             defaultMap.put(FessConfig.QUERY_HIGHLIGHT_ENCODER, "default");
             defaultMap.put(FessConfig.QUERY_HIGHLIGHT_FORCE_SOURCE, "false");
             defaultMap.put(FessConfig.QUERY_HIGHLIGHT_FRAGMENTER, "span");

src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java

@@ -71,6 +71,8 @@ import org.lastaflute.web.validation.theme.typed.LongTypeValidator;
 
 public interface FessProp {
 
+    String QUERY_HIGHLIGHT_TERMINAL_CHARS = "queryHighlightTerminalChars";
+
     String QUERY_HIGHLIGHT_BOUNDARY_CHARS = "queryHighlightBoundaryChars";
 
     String QUERY_TRACK_TOTAL_HITS_VALUE = "queryTrackTotalHitsValue";
@@ -2028,16 +2030,18 @@ public interface FessProp {
     String getQueryHighlightBoundaryChars();
 
     default char[] getQueryHighlightBoundaryCharsAsArray() {
-        char[] chars = (char[]) propMap.get(QUERY_HIGHLIGHT_BOUNDARY_CHARS);
-        if (chars == null) {
-            final String value = getQueryHighlightBoundaryChars();
-            chars = new char[value.length()];
-            for (int i = 0; i < value.length(); i++) {
-                chars[i] = value.charAt(i);
-            }
-            propMap.put(QUERY_HIGHLIGHT_BOUNDARY_CHARS, chars);
+        final int[] values = getCrawlerDocumentCharsAsArray(QUERY_HIGHLIGHT_BOUNDARY_CHARS, getQueryHighlightBoundaryChars());
+        final char[] chars = new char[values.length];
+        for (int i = 0; i < values.length; i++) {
+            chars[i] = (char) values[i];
         }
         return chars;
     }
 
+    String getQueryHighlightTerminalChars();
+
+    default int[] getQueryHighlightTerminalCharsAsArray() {
+        return getCrawlerDocumentCharsAsArray(QUERY_HIGHLIGHT_TERMINAL_CHARS, getQueryHighlightTerminalChars());
+    }
+
 }

src/main/resources/fess_config.properties

@@ -334,15 +334,15 @@ query.track.total.hits=10000
 query.geo.fields=location
 query.browser.lang.parameter.name=browser_lang
 query.replace.term.with.prefix.query=true
-query.highlight.terminal.chars=\u0021\u002E\u003F\u0589\u061F\u06D4\u0700\u0701\u0702\u0964\u104A\u104B\u1362\u1367\u1368\u166E\u1803\u1809\u203C\u203D\u2047\u2048\u2049\u3002\uFE52\uFE57\uFF01\uFF0E\uFF1F\uFF61
+query.highlight.terminal.chars=u0021u002Cu002Eu003Fu0589u061Fu06D4u0700u0701u0702u0964u104Au104Bu1362u1367u1368u166Eu1803u1809u203Cu203Du2047u2048u2049u3002uFE52uFE57uFF01uFF0EuFF1FuFF61
 query.highlight.fragment.size=60
 query.highlight.number.of.fragments=2
 query.highlight.type=fvh
 query.highlight.tag.pre=<strong>
 query.highlight.tag.post=</strong>
-query.highlight.boundary.chars=\u0009\u000A\u0013\u0020
+query.highlight.boundary.chars=u0009u000Au0013u0020
 query.highlight.boundary.max.scan=20
-query.highlight.boundary.scanner=sentence
+query.highlight.boundary.scanner=chars
 query.highlight.encoder=default
 query.highlight.force.source=false
 query.highlight.fragmenter=span