Merge pull request #25 from epi052/add-request-to-response-trait

responses now have a `.request()` method to access the request that generated the response

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxfuzz"
-version = "1.0.0-rc.10"
+version = "1.0.0-rc.11"
 edition = "2021"
 authors = ["Ben 'epi' Risher (@epi052)"]
 license = "Apache-2.0"
@@ -47,7 +47,7 @@ tokio = { version = "1.20", optional = true, features = [
 num = { version = "0.4" }
 cfg-if = { version = "1.0" }
 dyn-clone = { version = "1.0.9" }
-libafl = { version = "0.9.0", default-features = false, features = ["std"] }
+libafl = { version = "0.10.1", default-features = false, features = ["std"] }
 url = { version = "2.2", features = ["serde"] }
 ## optional
 serde = { version = "1.0", optional = true, features = ["derive", "rc"] }
@@ -63,9 +63,9 @@ regex = { version = "1.6" }
 serde_regex = { version = "1.1.0" }
 lazy_static = { version = "1.4" }
 futures = { version = "0.3", optional = true }
-base64 = { version = "0.13.1", optional = true }
+base64 = { version = "0.21.2", optional = true }
 hex = { version = "0.4.3", optional = true }
-flume = { version = "0.10.14" }
+flume = { version = "0.11.0" }
 
 [dev-dependencies]
 http = { version = "0.2" }

Makefile.toml

@@ -7,7 +7,7 @@
 dependencies = ["upgrade-deps", "update"]
 
 [tasks.check]
-dependencies = ["clippy", "fmt", "test-lib", "test-doc", "doc"]
+dependencies = ["clippy", "fmt", "test-lib", "test-doc", "doc", "semver"]
 
 [tasks.test]
 clear = true
@@ -56,3 +56,9 @@ clear = true
 script = """
 cargo nextest run --all-features --retries 10 --lib "${@}"
 """
+
+[tasks.semver]
+clear = true
+script = """
+cargo semver-checks check-release
+"""

README.md

@@ -71,7 +71,7 @@ The easiest way to get started is to include FeroxFuzz in your project's `Cargo.
 
 ```toml
 [dependencies]
-feroxfuzz = { version = "1.0.0-rc.10" }
+feroxfuzz = { version = "1.0.0-rc.11" }
 ```
 
 ## Docs

src/client/async_client.rs

@@ -109,11 +109,7 @@ impl AsyncRequests for AsyncClient {
         let parsed_version = parse_version(request.version())?;
 
         // build out the reqwest::Request from our mutated feroxfuzz::Request
-
-        let request_id = request.id;
-        let request_method = request.method().as_str()?.to_string();
-
-        let reqwest_request = self.build_request(parsed_version, request)?;
+        let reqwest_request = self.build_request(parsed_version, request.clone())?;
 
         // start timer for the request
         let now = Instant::now();
@@ -126,13 +122,9 @@ impl AsyncRequests for AsyncClient {
             .map_err(reqwest_to_ferox_error)?;
 
         // build the AsyncResponse, the await is for reqwest's asynchronous read of the response body
-        let response = AsyncResponse::try_from_reqwest_response(
-            request_id,
-            request_method,
-            reqwest_response,
-            now.elapsed(),
-        )
-        .await?;
+        let response =
+            AsyncResponse::try_from_reqwest_response(request, reqwest_response, now.elapsed())
+                .await?;
 
         Ok(response)
     }

src/client/blocking_client.rs

@@ -78,11 +78,8 @@ impl BlockingRequests for BlockingClient {
         // enum as its input type for that part of the request
         let parsed_version = parse_version(request.version())?;
 
-        let request_id = request.id;
-        let request_method = request.method().as_str()?.to_string();
-
         // build out the reqwest Request from our mutated Request
-        let reqwest_request = self.build_request(parsed_version, request)?;
+        let reqwest_request = self.build_request(parsed_version, request.clone())?;
 
         // start timer for the request
         let now = Instant::now();
@@ -94,12 +91,8 @@ impl BlockingRequests for BlockingClient {
             .map_err(reqwest_to_ferox_error)?;
 
         // build the AsyncResponse, the await is for reqwest's asynchronous read of the response body
-        let response = BlockingResponse::try_from_reqwest_response(
-            request_id,
-            request_method,
-            reqwest_response,
-            now.elapsed(),
-        )?;
+        let response =
+            BlockingResponse::try_from_reqwest_response(request, reqwest_response, now.elapsed())?;
 
         Ok(response)
     }

src/deciders/length.rs

@@ -24,7 +24,7 @@ use serde::{Deserialize, Serialize};
 /// ```
 /// # use http::response;
 /// # use feroxfuzz::responses::{Response, AsyncResponse};
-/// # use feroxfuzz::requests::RequestId;
+/// # use feroxfuzz::requests::Request;
 /// # use feroxfuzz::error::FeroxFuzzError;
 /// # use tokio_test;
 /// # use std::time::Duration;
@@ -43,9 +43,8 @@ use serde::{Deserialize, Serialize};
 /// # tokio_test::block_on(async {
 /// // for testing; normally a Response comes as a result of a sent request
 /// let reqwest_response = http::response::Builder::new().body("0123456789").unwrap();
-/// let id = RequestId::new(0);
 /// let elapsed = Duration::from_secs(1);
-/// let response = AsyncResponse::try_from_reqwest_response(id, String::from("GET"), reqwest_response.into(), elapsed).await?;
+/// let response = AsyncResponse::try_from_reqwest_response(Request::default(), reqwest_response.into(), elapsed).await?;
 ///
 /// // also not relevant to the current example, but needed to make the call to .post_send_hook
 /// let corpus = Wordlist::with_words(["a", "b", "c"]).name("chars").build();

src/deciders/regex.rs

@@ -141,14 +141,13 @@ where
 /// # use feroxfuzz::deciders::{ResponseRegexDecider, DeciderHooks, LogicOperation, Deciders};
 /// # use feroxfuzz::observers::ResponseObserver;
 /// # use feroxfuzz::responses::BlockingResponse;
-/// # use feroxfuzz::requests::{Request, RequestId};
+/// # use feroxfuzz::requests::Request;
 /// # use regex::bytes::Regex;
 /// # fn main() -> Result<(), FeroxFuzzError> {
 /// // for testing; normally a Response comes as a result of a sent request
 /// let reqwest_response = http::response::Builder::new().status(200).body("XyZDeRpZyX").unwrap();
-/// let id = RequestId::new(0);
 /// let elapsed = Duration::from_secs(1);
-/// let response = BlockingResponse::try_from_reqwest_response(id, String::from("GET"), reqwest_response.into(), elapsed)?;
+/// let response = BlockingResponse::try_from_reqwest_response(Request::default(), reqwest_response.into(), elapsed)?;
 ///
 /// // not relevant to the current example, but needed to make the call to .post_send_hook
 /// let mut state = SharedState::with_corpus(RangeCorpus::with_stop(10).name("corpus").build()?);

src/deciders/status_code.rs

@@ -24,7 +24,7 @@ use serde::{Deserialize, Serialize};
 /// ```
 /// # use http::response;
 /// # use feroxfuzz::responses::{Response, AsyncResponse};
-/// # use feroxfuzz::requests::RequestId;
+/// # use feroxfuzz::requests::Request;
 /// # use feroxfuzz::error::FeroxFuzzError;
 /// # use tokio_test;
 /// # use std::time::Duration;
@@ -43,9 +43,8 @@ use serde::{Deserialize, Serialize};
 /// # tokio_test::block_on(async {
 /// // for testing; normally a Response comes as a result of a sent request
 /// let reqwest_response = http::response::Builder::new().status(200).body("").unwrap();
-/// let id = RequestId::new(0);
 /// let elapsed = Duration::from_secs(1);
-/// let response = AsyncResponse::try_from_reqwest_response(id, String::from("GET"), reqwest_response.into(), elapsed).await?;
+/// let response = AsyncResponse::try_from_reqwest_response(Request::default(), reqwest_response.into(), elapsed).await?;
 ///
 /// // also not relevant to the current example, but needed to make the call to .post_send_hook
 /// let corpus = Wordlist::with_words(["a", "b", "c"]).name("chars").build();

src/fuzzers/async_fuzzer.rs

@@ -22,7 +22,7 @@ use crate::events::{
     StopFuzzing,
 };
 use crate::mutators::Mutators;
-use crate::observers::Observers;
+use crate::observers::{Observers, ResponseObserver};
 use crate::processors::Processors;
 use crate::requests::Request;
 use crate::responses::{AsyncResponse, Response};
@@ -51,14 +51,6 @@ const NUM_POST_PROCESSORS: usize = 6;
 /// initial `Result` that is returned from the `tokio::spawn` call
 static mut STOP_FUZZING_FLAG: bool = false;
 
-/// internal type used to pass a single object from the `tokio::spawn`
-/// call to the `FuturesUnordered` stream
-#[derive(Debug, Clone)]
-struct RequestFuture {
-    request: Request,
-    response: AsyncResponse,
-}
-
 /// A fuzzer that sends requests asynchronously
 #[derive(Clone, Debug, Default)]
 pub struct AsyncFuzzer<A, D, M, O, P, S>
@@ -425,8 +417,6 @@ where
             // spawn a new task to send the request, and when received, send the response
             // across the mpsc channel to the second/post-send loop
             let sent = tx.send(tokio::spawn(async move {
-                let cloned_request = mutated_request.clone();
-
                 // send the request, and await the response
                 let result = cloned_client.send(mutated_request).await;
 
@@ -436,17 +426,7 @@ where
                 // for reference: the permit is acquired at the top of the loop
                 drop(permit);
 
-                match result {
-                    // if the request was successful, return the response
-                    // and the request that generated it
-                    Ok(response) => Ok(RequestFuture {
-                        response,
-                        request: cloned_request,
-                    }),
-                    // otherwise, allow the error to bubble up to the processing
-                    // loop
-                    Err(err) => Err(err),
-                }
+                result
             }));
 
             // UnboundedChannel::send can only error if the receiver has called close()
@@ -529,7 +509,7 @@ async fn process_responses<D, O, P>(
     mut observers: O,
     mut processors: P,
     post_send_logic: LogicOperation,
-    receiver: flume::Receiver<JoinHandle<Result<RequestFuture, FeroxFuzzError>>>,
+    receiver: flume::Receiver<JoinHandle<Result<AsyncResponse, FeroxFuzzError>>>,
 ) -> Result<(), FeroxFuzzError>
 where
     D: Deciders<O, AsyncResponse> + Send + Clone,
@@ -539,7 +519,7 @@ where
     // second loop handles responses
     //
     // outer loop awaits the actual response, which is a double-nested Result
-    // Result<Result<RequestFuture, FeroxFuzzError>, tokio::task::JoinError>
+    // Result<Result<AsyncResponse, FeroxFuzzError>, tokio::task::JoinError>
     tracing::debug!("entering the response processing loop...");
 
     while let Ok(handle) = receiver.recv_async().await {
@@ -563,7 +543,7 @@ where
             continue;
         };
 
-        let Ok(request_future) = task_result else {
+        let Ok(response) = task_result else {
             let error = task_result.err().unwrap();
 
             // feroxfuzz::client::Client::send returned an error, which is a client error
@@ -574,13 +554,25 @@ where
             continue;
         };
 
-        // unpack the request_future into its request and response parts for convenience
-        let (request, response) = (request_future.request, request_future.response);
-
-        let request_id = response.id(); // grab the id; only used for logging
-
         observers.call_post_send_hooks(response);
 
+        // at this point, we still need a reference to the request
+        //
+        // the response observer takes ownership of the response, so we can grab a
+        // reference to the response observer and then extract out the request
+        // from the observer's reference to the response. this is a rather
+        // convoluted way of avoiding an unnecessary clone while not having to
+        // rework a bunch of internal implementation details
+        let response_observer = observers
+            .match_name::<ResponseObserver<AsyncResponse>>("ResponseObserver")
+            // reasonable to assume one exists, if not, we can figure it out then
+            // if someone comes along with a use-case for not using one, we can
+            // figure it out then
+            .unwrap();
+
+        let request = response_observer.request();
+        let request_id = request.id();
+
         let decision = deciders.call_post_send_hooks(&state, &observers, None, post_send_logic);
 
         if state.update(&observers, decision.as_ref()).is_err() {
@@ -597,7 +589,7 @@ where
             Some(Action::AddToCorpus(name, corpus_item_type, flow_control)) => {
                 match corpus_item_type {
                     CorpusItemType::Request => {
-                        if let Err(err) = state.add_request_fields_to_corpus(&name, &request) {
+                        if let Err(err) = state.add_request_fields_to_corpus(&name, request) {
                             warn!("Could not add {:?} to corpus[{name}]: {:?}", &request, err);
                         }
                     }

src/mutators/afl.rs

@@ -19,7 +19,6 @@ use crate::{atomic_load, AsBytes};
 
 use libafl::bolts::rands::Rand;
 use libafl::inputs::HasBytesVec;
-use libafl::mutators::mutations::{buffer_copy, buffer_self_copy};
 use libafl::state::{HasMaxSize, HasRand};
 #[cfg(feature = "serde")]
 use serde::{
@@ -36,6 +35,40 @@ pub use libafl::mutators::mutations::{
     DwordInterestingMutator, QwordAddMutator, WordAddMutator, WordInterestingMutator,
 };
 
+// note: the following two functions are from libafl, they were a part of the public api
+// and then were later made private. there wasn't a public replacement, so in the interest
+// of time, they're copied here
+
+/// Mem move in the own vec
+#[inline]
+pub fn buffer_self_copy<T>(data: &mut [T], from: usize, to: usize, len: usize) {
+    debug_assert!(!data.is_empty());
+    debug_assert!(from + len <= data.len());
+    debug_assert!(to + len <= data.len());
+    if len != 0 && from != to {
+        let ptr = data.as_mut_ptr();
+        unsafe {
+            core::ptr::copy(ptr.add(from), ptr.add(to), len);
+        }
+    }
+}
+
+/// Mem move between vecs
+#[inline]
+pub fn buffer_copy<T>(dst: &mut [T], src: &[T], from: usize, to: usize, len: usize) {
+    debug_assert!(!dst.is_empty());
+    debug_assert!(!src.is_empty());
+    debug_assert!(from + len <= src.len());
+    debug_assert!(to + len <= dst.len());
+    let dst_ptr = dst.as_mut_ptr();
+    let src_ptr = src.as_ptr();
+    if len != 0 {
+        unsafe {
+            core::ptr::copy(src_ptr.add(from), dst_ptr.add(to), len);
+        }
+    }
+}
+
 /// An enum wrapper for libafl mutators that facilitates static dispatch
 #[derive(Debug)]
 #[non_exhaustive]
@@ -601,6 +634,7 @@ impl Mutator for CrossoverInsertMutator {
 
             // perform the actual mutation
             input.bytes_mut().resize(size + len, 0);
+
             buffer_self_copy(input.bytes_mut(), to, to + len, size - to);
             buffer_copy(input.bytes_mut(), other_entry.as_bytes(), from, to, len);
         }

src/observers/mod.rs

@@ -98,7 +98,7 @@ where
 mod tests {
     use super::*;
     use crate::build_observers;
-    use crate::requests::{Request, RequestId};
+    use crate::requests::Request;
     use crate::responses::AsyncResponse;
     use std::time::Duration;
 
@@ -120,12 +120,10 @@ mod tests {
 
         let request = Request::new();
 
-        let id = RequestId::new(0);
         let elapsed = Duration::from_secs(1);
         let reqwest_response = http::response::Response::new("{\"stuff\":\"things\"}");
         let response = AsyncResponse::try_from_reqwest_response(
-            id,
-            String::from("GET"),
+            request.clone(),
             reqwest_response.into(),
             elapsed,
         )