0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

redirect check and improve logging

Browse source
This commit is contained in:
Shinsuke Sugaya 2017-07-18 06:42:34 +09:00
parent e5c2865408
commit 9250d4f8c8
2 changed files with 46 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java
View file

@ -57,6 +57,8 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
protected String name;
protected int maxRedirectCount = 10;
public void addCondition(final String key, final String regex) {
conditionMap.put(key, regex);
}
@ -173,30 +175,43 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
}
protected boolean process(final String id, final Predicate<ResponseData> consumer) {
return process(id, (configId, url) -> {
final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
if (config == null) {
throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
}
final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getComponent(CrawlerClientFactory.class);
config.initializeClientFactory(crawlerClientFactory);
final CrawlerClient client = crawlerClientFactory.getClient(url);
if (client == null) {
throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);
}
try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build())) {
return consumer.test(responseData);
} catch (final CrawlingAccessException e) {
if (logger.isDebugEnabled()) {
throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
} else {
throw new ThumbnailGenerationException(e.getMessage());
}
} catch (final Exception e) {
throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
}
});
return process(id,
(configId, url) -> {
final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
if (config == null) {
throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
}
final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getComponent(CrawlerClientFactory.class);
config.initializeClientFactory(crawlerClientFactory);
final CrawlerClient client = crawlerClientFactory.getClient(url);
if (client == null) {
throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);
}
String u = url;
for (int i = 0; i < maxRedirectCount; i++) {
try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(u).build())) {
if (StringUtil.isNotBlank(responseData.getRedirectLocation())) {
u = responseData.getRedirectLocation();
continue;
}
if (StringUtil.isBlank(responseData.getUrl())) {
throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url
+ " (Response URL is empty)");
}
return consumer.test(responseData);
} catch (final CrawlingAccessException e) {
if (logger.isDebugEnabled()) {
throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
} else {
throw new ThumbnailGenerationException(e.getMessage());
}
} catch (final Exception e) {
throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
}
}
throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url + " (Redirect Loop)");
});
}
public void setGeneratorList(final List<String> generatorList) {
@ -212,4 +227,8 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
this.name = name;
}
public void setMaxRedirectCount(int maxRedirectCount) {
this.maxRedirectCount = maxRedirectCount;
}
}

6
src/main/java/org/codelibs/fess/thumbnail/impl/HtmlTagBasedGenerator.java
View file

@ -97,15 +97,15 @@ public class HtmlTagBasedGenerator extends BaseThumbnailGenerator {
created = true;
break;
case FAILED:
logger.warn("Failed to create thumbnail: " + thumbnailId);
logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + responseData.getUrl());
break;
case INVALID_SIZE:
if (logger.isDebugEnabled()) {
logger.debug("Invalid thumbnail size: " + thumbnailId);
logger.debug("Invalid thumbnail size: " + thumbnailId + " -> " + responseData.getUrl());
}
break;
default:
logger.error("Unknown thumbnail result: " + thumbnailId);
logger.error("Unknown thumbnail result: " + thumbnailId + " -> " + responseData.getUrl());
break;
}
} catch (final Throwable t) {