feat(website): add chrome crawl_chrome_send thread safe

Author: j-mendez

Cargo.lock

@@ -138,6 +138,11 @@ name = "chrome_remote"
 path = "chrome_remote.rs"
 required-features = ["spider/sync", "spider/chrome"]
 
+[[example]]
+name = "chrome_sendable"
+path = "chrome_sendable.rs"
+required-features = ["spider/sync", "spider/chrome"]
+
 [[example]]
 name = "real_world"
 path = "real_world.rs"

examples/Cargo.toml

@@ -0,0 +1,67 @@
+//! cargo run --example chrome_sendable --features="chrome chrome_intercept"
+
+extern crate spider;
+use crate::spider::tokio::io::AsyncWriteExt;
+use spider::features::chrome_common::RequestInterceptConfiguration;
+use spider::tokio;
+use spider::website::Website;
+use std::io::Result;
+
+async fn crawl_website(website: &Website, url: &str) -> Result<()> {
+    let start = crate::tokio::time::Instant::now();
+
+    website.crawl_chrome_send(Some(url)).await;
+
+    println!(
+        "Time elapsed in website.crawl({}) is: {:?}",
+        url,
+        start.elapsed(),
+    );
+
+    Ok(())
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut stdout = tokio::io::stdout();
+    let mut website = Website::default();
+
+    website
+        .with_limit(5)
+        .with_chrome_intercept(RequestInterceptConfiguration::new(true))
+        .with_stealth(true)
+        .with_fingerprint(true)
+        .with_chrome_connection(Some("http://127.0.0.1:9222/json/version".into()));
+
+    website.configure_setup().await;
+
+    let mut rx2 = website.subscribe(16).unwrap();
+
+    let handle = tokio::spawn(async move {
+        while let Ok(page) = rx2.recv().await {
+            let _ = stdout
+                .write_all(
+                    format!(
+                        "- {} -- Bytes transferred {:?} -- HTML Size {:?}\n",
+                        page.get_url(),
+                        page.bytes_transferred.unwrap_or_default(),
+                        page.get_html_bytes_u8().len()
+                    )
+                    .as_bytes(),
+                )
+                .await;
+        }
+    });
+
+    let _ = tokio::join!(
+        crawl_website(&website, "https://choosealicense.com"),
+        crawl_website(&website, "https://jeffmendez.com"),
+        crawl_website(&website, "https://example.com"),
+    );
+
+    drop(website);
+
+    let _ = handle.await;
+
+    Ok(())
+}

examples/chrome_sendable.rs

@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.32.6"
+version = "2.32.8"
 authors = [
     "j-mendez <[email protected]>"
 ]

spider/Cargo.toml

@@ -1,3 +1,5 @@
+use std::time::Duration;
+
 use crate::utils::log;
 use crate::{configuration::Configuration, tokio_stream::StreamExt};
 use chromiumoxide::cdp::browser_protocol::browser::{
@@ -7,6 +9,7 @@ use chromiumoxide::cdp::browser_protocol::{
     browser::BrowserContextId, network::CookieParam, target::CreateTargetParams,
 };
 use chromiumoxide::error::CdpError;
+use chromiumoxide::handler::REQUEST_TIMEOUT;
 use chromiumoxide::page::DISABLE_DIALOGS;
 use chromiumoxide::Page;
 use chromiumoxide::{handler::HandlerConfig, Browser, BrowserConfig};
@@ -86,7 +89,7 @@ pub fn get_browser_config(
         .disable_default_args()
         .request_timeout(match request_timeout.as_ref() {
             Some(timeout) => **timeout,
-            _ => Default::default(),
+            _ => Duration::from_millis(REQUEST_TIMEOUT),
         });
 
     let builder = if cache_enabled {
@@ -156,7 +159,7 @@ pub fn get_browser_config(
         .no_sandbox()
         .request_timeout(match request_timeout.as_ref() {
             Some(timeout) => **timeout,
-            _ => Default::default(),
+            _ => Duration::from_millis(REQUEST_TIMEOUT),
         })
         .with_head();
 
@@ -221,7 +224,7 @@ fn create_handler_config(config: &Configuration) -> HandlerConfig {
     HandlerConfig {
         request_timeout: match config.request_timeout.as_ref() {
             Some(timeout) => **timeout,
-            _ => Default::default(),
+            _ => Duration::from_millis(REQUEST_TIMEOUT),
         },
         request_intercept: config.chrome_intercept.enabled,
         cache_enabled: config.cache,
@@ -716,10 +719,10 @@ impl BrowserController {
             // assume close will always happen.
             self.closed = true;
             if let Some(id) = self.browser.2.take() {
+                let _ = self.browser.0.quit_incognito_context_base(id).await;
                 if let Some(handler) = self.browser.1.take() {
                     // we have to quit the context until https://chromedevtools.github.io/devtools-protocol/tot/Target/#method-createBrowserContext
                     // disposeOnDetach comes out of Experimental.
-                    let _ = self.browser.0.quit_incognito_context_base(id).await;
                     handler.abort();
                 }
             }