add contentWaitDuration

codelibs · Dec 17, 2024 · cf267ca · cf267ca
1 parent ee88689
commit cf267ca
Showing 1 changed file with 12 additions and 0 deletions.
diff --git a/src/main/java/org/codelibs/fess/crawler/client/http/PlaywrightClient.java b/src/main/java/org/codelibs/fess/crawler/client/http/PlaywrightClient.java
@@ -37,6 +37,7 @@
 import org.apache.http.auth.UsernamePasswordCredentials;
 import org.codelibs.core.exception.IORuntimeException;
 import org.codelibs.core.lang.StringUtil;
+import org.codelibs.core.lang.ThreadUtil;
 import org.codelibs.core.misc.Tuple4;
 import org.codelibs.core.stream.StreamUtil;
 import org.codelibs.fess.crawler.Constants;
@@ -86,6 +87,8 @@ public class PlaywrightClient extends AbstractCrawlerClient {
 
     protected static final String RENDERED_STATE = "renderedState";
 
+    protected static final String CONTENT_WAIT_DURATION = "contentWaitDuration";
+
     protected static final String IGNORE_HTTPS_ERRORS_PROPERTY = "ignoreHttpsErrors";
 
     protected static final String PROXY_BYPASS_PROPERTY = "proxyBypass";
@@ -106,6 +109,8 @@ public class PlaywrightClient extends AbstractCrawlerClient {
 
     protected LoadState renderedState = LoadState.NETWORKIDLE;
 
+    protected long contentWaitDuration = 0;
+
     protected Tuple4<Playwright, Browser, BrowserContext, Page> worker;
 
     @Resource
@@ -128,6 +133,8 @@ public void init() {
                 renderedState = LoadState.valueOf(renderedStateParam);
             }
 
+            contentWaitDuration = getInitParameter(CONTENT_WAIT_DURATION, 0L, Long.class);
+
             final Boolean shared = getInitParameter(SHARED_CLIENT, Boolean.FALSE, Boolean.class);
             if (shared) {
                 if (SHARED_WORKER == null) {
@@ -284,6 +291,11 @@ public ResponseData execute(final RequestData request) {
 
                 page.waitForLoadState(renderedState);
 
+                if (contentWaitDuration > 0L) {
+                    logger.debug("Waiting {} ms before downloading the content.", contentWaitDuration);
+                    ThreadUtil.sleep(contentWaitDuration);
+                }
+
                 if (logger.isDebugEnabled()) {
                     logger.debug("Loaded: Base URL: {}, Response URL: {}", url, response.url());
                 }