Dev

.gitignore

@@ -21,3 +21,4 @@ Cargo.lock
 tags
 
 .DS_Store
+all_code.txt

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "capp"
-version = "0.4.1"
+version = "0.4.3"
 edition = "2021"
 license = "MIT"
 authors = ["oiwn"]
@@ -39,8 +39,8 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0" }
 serde_yaml = "0.9"
 thiserror = { version = "1" }
-tokio = { version = "1.40", features = ["full"] }
-uuid = { version = "1.10", features = ["v4", "serde"] }
+tokio = { version = "1.41", features = ["full"] }
+uuid = { version = "1.11", features = ["v4", "serde"] }
 rustis = { version = "0.13", features = ["tokio-runtime"], optional = true }
 tracing = "0.1"
 tracing-subscriber = "0.3"
@@ -49,21 +49,22 @@ tracing-futures = "0.2"
 indexmap = "2.6"
 url = "2.5"
 regex = "1.11"
+rand = { version = "0.8", optional = true } 
 
 [dev-dependencies]
 capp = { path = ".", features = ["http", "healthcheck", "redis"] }
-hyper = { version = "1.3", features = ["server", "http1"] }
+hyper = { version = "1.5", features = ["server", "http1"] }
 http-body-util = "0.1"
 bytes = "1.6"
 pin-project-lite = "0.2"
 dotenvy = "0.15"
-scraper = "0.19"
+scraper = "0.21"
 rand = "0.8"
 md5 = "0.7"
 url = "2.5"
 base64 = "0.22"
 
 [features]
-http = ["dep:backoff", "dep:reqwest"]
+http = ["dep:backoff", "dep:reqwest", "dep:rand"]
 healthcheck = ["dep:reqwest"]
 redis = ["dep:rustis"]

notes.org

@@ -1,6 +1,17 @@
 #+title: Notes
 
 * Tasks
+** Need some kind of statistics
+** Figure out if it's possible to implement similar api for capp-rs:
+```
+ let app_state = AppState {
+        sqlite: Arc::new(sqlite),
+};
+let router = axum::Router::new()
+        .route("/authors", post(create_author))
+        .layer(trace_layer)
+        .with_state(app_state);
+```
 ** need workflows
 - [ ] coverage
 - [X] tests/clippy/check/fmt

src/http.rs

@@ -1,11 +1,11 @@
+use crate::proxy::{ProxyProvider, RandomProxyProvider};
 use backoff::ExponentialBackoffBuilder;
-use std::time::Duration;
 
 #[derive(Debug)]
 pub struct HttpClientParams<'a> {
     pub timeout: u64,
     pub connect_timeout: u64,
-    pub proxy: Option<reqwest::Proxy>,
+    pub proxy_provider: Option<Box<dyn ProxyProvider + Send + Sync>>,
     pub user_agent: &'a str,
 }
 
@@ -31,20 +31,22 @@ impl<'a> HttpClientParams<'a> {
             .expect("No timeout field in config");
         let connect_timeout = http_config["connect_timeout"]
             .as_u64()
-            .expect("No connect_timout field in config");
-        let need_proxy: bool = http_config["proxy"]["use"]
-            .as_bool()
-            .expect("No use field for proxy in config");
-        let proxy = if need_proxy {
-            let proxy_uri = http_config["proxy"]["uri"].as_str().unwrap();
-            Some(reqwest::Proxy::all(proxy_uri).expect("Error setting up proxy"))
-        } else {
-            None
-        };
+            .expect("No connect_timeout field in config");
+
+        let proxy_provider =
+            if http_config["proxy"]["use"].as_bool().unwrap_or(false) {
+                Some(Box::new(
+                    RandomProxyProvider::from_config(&http_config["proxy"])
+                        .expect("Failed to create proxy provider"),
+                ) as Box<dyn ProxyProvider + Send + Sync>)
+            } else {
+                None
+            };
+
         Self {
             timeout,
             connect_timeout,
-            proxy,
+            proxy_provider,
             user_agent,
         }
     }
@@ -63,23 +65,26 @@ pub fn parse_proxy_uri(uri: &str) -> String {
 }
 */
 
-/// Helper to create typical crawling request with sane defaultsx
+// Modified build_http_client function
 pub fn build_http_client(
     params: HttpClientParams,
 ) -> Result<reqwest::Client, reqwest::Error> {
     let mut client_builder = reqwest::ClientBuilder::new()
         .use_rustls_tls()
         .danger_accept_invalid_certs(true)
-        .timeout(Duration::from_secs(params.timeout))
-        .connect_timeout(Duration::from_secs(params.connect_timeout))
+        .timeout(std::time::Duration::from_secs(params.timeout))
+        .connect_timeout(std::time::Duration::from_secs(params.connect_timeout))
         .user_agent(params.user_agent);
 
-    if let Some(proxy) = params.proxy {
-        client_builder = client_builder.proxy(proxy);
+    if let Some(proxy_provider) = params.proxy_provider {
+        if let Some(proxy_uri) = proxy_provider.get_proxy() {
+            client_builder = client_builder.proxy(
+                reqwest::Proxy::all(&proxy_uri).expect("Failed to create proxy"),
+            );
+        }
     }
 
-    let client = client_builder.build()?;
-    Ok(client)
+    client_builder.build()
 }
 
 /// Fetch url with retries (with sane defaults),
@@ -145,7 +150,7 @@ mod tests {
         let client = build_http_client(HttpClientParams {
             timeout: 10,
             connect_timeout: 5,
-            proxy: None,
+            proxy_provider: None,
             user_agent: "hello",
         });
 

src/lib.rs

@@ -29,7 +29,7 @@
 //!
 //! ```toml
 //! [dependencies]
-//! capp = "0.4.1"
+//! capp = "0.4"
 //! ```
 //!
 //! Check examples!
@@ -49,20 +49,20 @@ pub mod healthcheck;
 pub mod http;
 pub mod manager;
 pub mod prelude;
+#[cfg(feature = "http")]
+pub mod proxy;
 pub mod queue;
 pub mod router;
 pub mod task;
-// #[cfg(test)]
-// mod support;
-// #[cfg(test)]
-// pub mod test_utils;
 
 // re-export
 pub use async_trait;
 #[cfg(feature = "http")]
 pub use backoff;
 pub use derive_builder;
 pub use indexmap;
+#[cfg(feature = "http")]
+pub use rand;
 pub use regex;
 #[cfg(feature = "http")]
 pub use reqwest;

src/prelude.rs

@@ -1,4 +1,9 @@
+pub use crate::config::*;
+#[cfg(feature = "http")]
+pub use crate::http::*;
 pub use crate::manager::*;
+#[cfg(feature = "http")]
+pub use crate::proxy::*;
 pub use crate::queue::*;
 pub use crate::router::*;
 pub use crate::task::*;