|
@@ -57,6 +57,8 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
|
|
|
|
|
|
protected String name;
|
|
protected String name;
|
|
|
|
|
|
|
|
+ protected int maxRedirectCount = 10;
|
|
|
|
+
|
|
public void addCondition(final String key, final String regex) {
|
|
public void addCondition(final String key, final String regex) {
|
|
conditionMap.put(key, regex);
|
|
conditionMap.put(key, regex);
|
|
}
|
|
}
|
|
@@ -173,30 +175,43 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
|
|
}
|
|
}
|
|
|
|
|
|
protected boolean process(final String id, final Predicate<ResponseData> consumer) {
|
|
protected boolean process(final String id, final Predicate<ResponseData> consumer) {
|
|
- return process(id, (configId, url) -> {
|
|
|
|
- final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
|
|
|
|
- CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
|
|
|
|
- if (config == null) {
|
|
|
|
- throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
|
|
|
|
- }
|
|
|
|
- final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getComponent(CrawlerClientFactory.class);
|
|
|
|
- config.initializeClientFactory(crawlerClientFactory);
|
|
|
|
- final CrawlerClient client = crawlerClientFactory.getClient(url);
|
|
|
|
- if (client == null) {
|
|
|
|
- throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);
|
|
|
|
- }
|
|
|
|
- try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build())) {
|
|
|
|
- return consumer.test(responseData);
|
|
|
|
- } catch (final CrawlingAccessException e) {
|
|
|
|
- if (logger.isDebugEnabled()) {
|
|
|
|
- throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
|
|
|
|
- } else {
|
|
|
|
- throw new ThumbnailGenerationException(e.getMessage());
|
|
|
|
- }
|
|
|
|
- } catch (final Exception e) {
|
|
|
|
- throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
|
|
|
|
- }
|
|
|
|
- });
|
|
|
|
|
|
+ return process(id,
|
|
|
|
+ (configId, url) -> {
|
|
|
|
+ final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
|
|
|
|
+ CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
|
|
|
|
+ if (config == null) {
|
|
|
|
+ throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
|
|
|
|
+ }
|
|
|
|
+ final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getComponent(CrawlerClientFactory.class);
|
|
|
|
+ config.initializeClientFactory(crawlerClientFactory);
|
|
|
|
+ final CrawlerClient client = crawlerClientFactory.getClient(url);
|
|
|
|
+ if (client == null) {
|
|
|
|
+ throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);
|
|
|
|
+ }
|
|
|
|
+ String u = url;
|
|
|
|
+ for (int i = 0; i < maxRedirectCount; i++) {
|
|
|
|
+ try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(u).build())) {
|
|
|
|
+ if (StringUtil.isNotBlank(responseData.getRedirectLocation())) {
|
|
|
|
+ u = responseData.getRedirectLocation();
|
|
|
|
+ continue;
|
|
|
|
+ }
|
|
|
|
+ if (StringUtil.isBlank(responseData.getUrl())) {
|
|
|
|
+ throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url
|
|
|
|
+ + " (Response URL is empty)");
|
|
|
|
+ }
|
|
|
|
+ return consumer.test(responseData);
|
|
|
|
+ } catch (final CrawlingAccessException e) {
|
|
|
|
+ if (logger.isDebugEnabled()) {
|
|
|
|
+ throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
|
|
|
|
+ } else {
|
|
|
|
+ throw new ThumbnailGenerationException(e.getMessage());
|
|
|
|
+ }
|
|
|
|
+ } catch (final Exception e) {
|
|
|
|
+ throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url + " (Redirect Loop)");
|
|
|
|
+ });
|
|
}
|
|
}
|
|
|
|
|
|
public void setGeneratorList(final List<String> generatorList) {
|
|
public void setGeneratorList(final List<String> generatorList) {
|
|
@@ -212,4 +227,8 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
|
|
this.name = name;
|
|
this.name = name;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+ public void setMaxRedirectCount(int maxRedirectCount) {
|
|
|
|
+ this.maxRedirectCount = maxRedirectCount;
|
|
|
|
+ }
|
|
|
|
+
|
|
}
|
|
}
|