fix #1350 improve _id generetion

src/main/java/org/codelibs/fess/helper/CrawlingInfoHelper.java

@@ -18,6 +18,7 @@ package org.codelibs.fess.helper;
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
@@ -231,7 +232,13 @@ public class CrawlingInfoHelper {
                 encodedBuf.append(c);
             } else {
                 try {
-                    encodedBuf.append(URLEncoder.encode(String.valueOf(c), Constants.UTF_8));
+                    final String target = String.valueOf(c);
+                    final String converted = URLEncoder.encode(target, Constants.UTF_8);
+                    if (target.equals(converted)) {
+                        encodedBuf.append(Base64.getUrlEncoder().encodeToString(target.getBytes(Constants.CHARSET_UTF_8)));
+                    } else {
+                        encodedBuf.append(converted);
+                    }
                 } catch (final UnsupportedEncodingException e) {
                     // NOP
                 }
@@ -239,10 +246,14 @@ public class CrawlingInfoHelper {
         }
 
         final String id = encodedBuf.toString();
-        if (id.length() <= urlIdPrefixLength) {
+        if (id.getBytes(Constants.CHARSET_UTF_8).length <= urlIdPrefixLength) {
             return id;
         }
-        return id.substring(0, urlIdPrefixLength) + MessageDigestUtil.digest("SHA-256", id.substring(urlIdPrefixLength));
+        final String longId = id.substring(0, urlIdPrefixLength) + MessageDigestUtil.digest("SHA-256", id.substring(urlIdPrefixLength));
+        if (longId.getBytes(Constants.CHARSET_UTF_8).length <= urlIdPrefixLength + 64) {
+            return longId;
+        }
+        return longId.substring(0, urlIdPrefixLength + 64);
     }
 
     public void setMaxSessionIdsInList(final int maxSessionIdsInList) {

src/test/java/org/codelibs/fess/helper/CrawlingInfoHelperTest.java

@@ -91,5 +91,16 @@ public class CrawlingInfoHelperTest extends UnitFessTestCase {
             assertEquals(509, crawlingInfoHelper.generateId(value.substring(0, 520), null).length());
             assertEquals(509, crawlingInfoHelper.generateId(value.toString(), null).length());
         }
+
+        StringBuilder buf = new StringBuilder();
+        for (int i = 0; i < 550; i++) {
+            buf.append('a');
+        }
+        assertEquals("aaaaaaaaaaaaaaa9f4390f8d30c2dd92ec9f095b65e2b9ae9b0a925a5258e241c9f1e910f734318",
+                crawlingInfoHelper.generateId(buf.substring(0, 500), null).substring(430));
+        assertEquals("aaaaaaaaaaaaaaa635361c48bb9eab14198e76ea8ab7f1a41685d6ad62aa9146d301d4f17eb0ae0",
+                crawlingInfoHelper.generateId(buf.substring(0, 510), null).substring(430));
+        assertEquals("aaaaaaaaaaaaaaa8af881bc88895bd9d8cea975a7d06dc0275d9db9d57f138216936b65e8b06489",
+                crawlingInfoHelper.generateId(buf.substring(0, 520), null).substring(430));
     }
 }