Copa support & Fix several bugs (#258)

* [~] unified interfaces of win_filter

* [+] add a new congestion controller, Copa (NSDI'18)

* [~] unified MSS definition

* [~] disable additive increase of cwnd in BBR

* [~] bug fix for rate sampling

* [+] calculate MSS according to settings

* [~] bug fix for cubic

* [~] add copa in the document of testing

Author: yangfurong

.gitignore

@@ -1,3 +1,4 @@
 build/
 bss/
 *.swp
+third_party/

CMakeLists.txt

@@ -225,6 +225,7 @@ set(
     CONGESTION_CONTROL_SOURCES
     "src/congestion_control/xqc_cubic.c"
     "src/congestion_control/xqc_bbr.c"
+    "src/congestion_control/xqc_copa.c"
     "src/congestion_control/xqc_window_filter.c"
     "src/congestion_control/xqc_sample.c"
 )

cmake/CMakeLists.txt

@@ -226,6 +226,7 @@ set(
     CONGESTION_CONTROL_SOURCES
     "src/congestion_control/xqc_cubic.c"
     "src/congestion_control/xqc_bbr.c"
+    "src/congestion_control/xqc_copa.c"
     "src/congestion_control/xqc_window_filter.c"
     "src/congestion_control/xqc_sample.c"
 )

docs/docs-zh/Testing-zh.md

@@ -35,7 +35,9 @@ openssl req -newkey rsa:2048 -x509 -nodes -keyout "$keyfile" -new -out "$certfil
 |   -p   | Server port. |
 |   -P   | Number of Parallel requests per single connection. Default 1. |
 |   -n   | Total number of requests to send. Defaults 1. |
-|   -c   | Congestion Control Algorithm. r:reno b:bbr c:cubic B:bbr2 bbr+ bbr2+ |
+|   -c   | Congestion Control Algorithm. r:reno b:bbr c:cubic B:bbr2 bbr+ bbr2+ P:Copa |
+|   -A   | Copa: the addtive increasing coefficient of 1/delta, default:1.0 |
+|   -D   | Copa: the delta paramter, default:0.05 |
 |   -C   | Pacing on. |
 |   -t   | Connection timeout. Default 3 seconds. |
 |   -T   | Transport layer. No HTTP3. |
@@ -63,7 +65,9 @@ openssl req -newkey rsa:2048 -x509 -nodes -keyout "$keyfile" -new -out "$certfil
 | :----: | ----  |
 |   -p   | Server port. |
 |   -e   | Echo. Send received body. |
-|   -c   | Congestion Control Algorithm. r:reno b:bbr c:cubic B:bbr2 bbr+ bbr2+ |
+|   -c   | Congestion Control Algorithm. r:reno b:bbr c:cubic B:bbr2 bbr+ bbr2+ P:Copa |
+|   -A   | Copa: the addtive increasing coefficient of 1/delta, default:1.0 |
+|   -D   | Copa: the delta paramter, default:0.05 |
 |   -C   | Pacing on. |
 |   -s   | Body size to send. |
 |   -w   | Write received body to file. |

include/xquic/xquic.h

@@ -627,6 +627,15 @@ typedef struct xqc_cc_params_s {
     uint32_t    expect_bw;
     uint32_t    max_expect_bw;
     uint32_t    cc_optimization_flags;
+    /* 0 < delta <= delta_max, default 0.05, ->0 = more throughput-oriented */
+    double      copa_delta_base; 
+    /* 0 < delta_max <= 1.0, default 0.5 */
+    double      copa_delta_max;
+    /* 
+     * 1.0 <= delta_ai_unit, default 1.0, greater values mean more aggressive
+     * when Copa competes with loss-based CCAs.
+     */
+    double      copa_delta_ai_unit;
 } xqc_cc_params_t;
 
 typedef struct xqc_congestion_control_callback_s {
@@ -657,8 +666,8 @@ typedef struct xqc_congestion_control_callback_s {
     /* This function is used by BBR and Cubic*/
     void (*xqc_cong_ctl_restart_from_idle)(void *cong_ctl, uint64_t arg);
 
-    /* For BBR */
-    void (*xqc_cong_ctl_bbr)(void *cong_ctl, xqc_sample_t *sampler);
+    /* For BBR & Copa */
+    void (*xqc_cong_ctl_on_ack_multiple_pkts)(void *cong_ctl, xqc_sample_t *sampler);
 
     /* initialize bbr */
     void (*xqc_cong_ctl_init_bbr)(void *cong_ctl, xqc_sample_t *sampler, xqc_cc_params_t cc_params);
@@ -680,6 +689,7 @@ XQC_EXPORT_PUBLIC_API XQC_EXTERN const xqc_cong_ctrl_callback_t xqc_bbr2_cb;
 #endif
 XQC_EXPORT_PUBLIC_API XQC_EXTERN const xqc_cong_ctrl_callback_t xqc_bbr_cb;
 XQC_EXPORT_PUBLIC_API XQC_EXTERN const xqc_cong_ctrl_callback_t xqc_cubic_cb;
+XQC_EXPORT_PUBLIC_API XQC_EXTERN const xqc_cong_ctrl_callback_t xqc_copa_cb;
 
 
 /**

scripts/case_test.sh

@@ -786,6 +786,32 @@ else
 fi
 
 
+clear_log
+echo -e "Copa with default parameters (delta=0.05, ai_unit=1.0) ...\c"
+result=`./test_client -s 10240000 -l e -t 1 -E -c P|grep ">>>>>>>> pass"`
+errlog=`grep_err_log`
+echo "$result"
+if [ -z "$errlog" ] && [ "$result" == ">>>>>>>> pass:1" ]; then
+    case_print_result "copa_with_default_parameters" "pass"
+else
+    case_print_result "copa_with_default_parameters" "fail"
+    echo "$errlog"
+fi
+
+clear_log
+echo -e "Copa with customized parameters (delta=0.5, ai_unit=5.0) ...\c"
+result=`./test_client -s 10240000 -l e -t 1 -E -c P -A 5.0 -D 0.5|grep ">>>>>>>> pass"`
+errlog=`grep_err_log`
+echo "$result"
+if [ -z "$errlog" ] && [ "$result" == ">>>>>>>> pass:1" ]; then
+    case_print_result "copa_with_customized_parameters" "pass"
+else
+    case_print_result "copa_with_customized_parameters" "fail"
+    echo "$errlog"
+fi
+
+
+
 clear_log
 result=`./test_client -s 10240000 -l e -t 1 -E -x 26|grep ">>>>>>>> pass"`
 errlog=`grep_err_log`

scripts/xquic.lds

@@ -54,6 +54,7 @@ XQUIC_VERS_1.0 {
         xqc_bbr2_cb;
         xqc_reno_cb;
         xqc_cubic_cb;
+        xqc_copa_cb;
         xqc_conn_is_ready_to_send_early_data;
         xqc_h3_conn_send_ping;
         xqc_conn_send_ping;

src/congestion_control/xqc_bbr.c

@@ -20,7 +20,7 @@
 #include "src/transport/xqc_send_ctl.h"
 #include "src/transport/xqc_packet.h"
 
-#define XQC_BBR_MAX_DATAGRAMSIZE    XQC_QUIC_MSS
+#define XQC_BBR_MAX_DATAGRAMSIZE    XQC_MSS
 #define XQC_BBR_MIN_WINDOW          (4 * XQC_BBR_MAX_DATAGRAMSIZE)
 #define XQC_BBR_MAX_WINDOW          (100 * XQC_BBR_MAX_DATAGRAMSIZE)
 /* The RECOMMENDED value is the minimum of 10 * kMaxDatagramSize and max(2* kMaxDatagramSize, 14720)) */
@@ -147,10 +147,12 @@ xqc_bbr_init(void *cong_ctl, xqc_sample_t *sampler, xqc_cc_params_t cc_params)
         bbr->rttvar_compensation_on = 1;
     }
 #endif
+#ifndef XQC_BBR_DISABLE_CWND_AI
     bbr->beyond_target_cwnd = 0;
     bbr->snd_cwnd_cnt_bytes = 0;
     bbr->ai_scale = 1;
     bbr->ai_scale_accumulated_bytes = 0;
+#endif
     bbr->min_rtt = sampler->srtt ? sampler->srtt : XQC_BBR_INF;
     bbr->min_rtt_stamp = now;
     bbr->probe_rtt_min_us = sampler->srtt ? sampler->srtt : XQC_BBR_INF;
@@ -266,7 +268,7 @@ static uint32_t
 xqc_bbr_compensate_cwnd_for_rttvar(xqc_bbr_t *bbr, xqc_sample_t *sampler)
 {
     xqc_usec_t srtt = sampler->srtt;
-    xqc_usec_t recent_max_rtt = xqc_win_filter_get_u64(&bbr->max_rtt);
+    xqc_usec_t recent_max_rtt = xqc_win_filter_get(&bbr->max_rtt);
     xqc_usec_t compensation_thresh = (1 + bbr->rtt_compensation_thresh) *
                                      bbr->min_rtt;
     uint32_t cwnd_addition = 0;
@@ -546,6 +548,8 @@ xqc_bbr_update_min_rtt(xqc_bbr_t *bbr, xqc_sample_t *sampler)
         xqc_log(sampler->send_ctl->ctl_conn->log, XQC_LOG_DEBUG, "|minrtt expire|rtt:%ui, old_rtt:%ui|",
                 bbr->probe_rtt_min_us,
                 bbr->min_rtt);
+
+#ifndef XQC_BBR_DISABLE_CWND_AI
         if (bbr->probe_rtt_min_us_stamp != bbr->min_rtt_stamp
             || min_rtt_expired)
         {
@@ -562,6 +566,8 @@ xqc_bbr_update_min_rtt(xqc_bbr_t *bbr, xqc_sample_t *sampler)
                 bbr->ai_scale_accumulated_bytes = 0;
             }
         }
+#endif
+
         bbr->min_rtt = bbr->probe_rtt_min_us;
         bbr->min_rtt_stamp = bbr->probe_rtt_min_us_stamp;
     }
@@ -712,14 +718,16 @@ xqc_bbr_reset_cwnd(void *cong_ctl)
     }
     /* reset recovery start time in any case */
     bbr->recovery_start_time = 0;
+#ifndef XQC_BBR_DISABLE_CWND_AI
     /* If losses happened, we do not increase cwnd beyond target_cwnd. */
     bbr->snd_cwnd_cnt_bytes = 0;
     bbr->beyond_target_cwnd = 0;
     bbr->ai_scale = 1;
     bbr->ai_scale_accumulated_bytes = 0;
+#endif
 }
 
-
+#ifndef XQC_BBR_DISABLE_CWND_AI
 static void
 xqc_bbr_cong_avoid_ai(xqc_bbr_t *bbr, uint32_t cwnd, uint32_t acked)
 {
@@ -744,6 +752,7 @@ xqc_bbr_cong_avoid_ai(xqc_bbr_t *bbr, uint32_t cwnd, uint32_t acked)
         bbr->ai_scale_accumulated_bytes -= delta * cwnd_thresh;
     }
 }
+#endif
 
 static void 
 xqc_bbr_set_cwnd(xqc_bbr_t *bbr, xqc_sample_t *sampler)
@@ -772,6 +781,7 @@ xqc_bbr_set_cwnd(xqc_bbr_t *bbr, xqc_sample_t *sampler)
         xqc_bbr_modulate_cwnd_for_recovery(bbr, sampler);
         if (!bbr->packet_conservation) {
             if (bbr->full_bandwidth_reached) {
+#ifndef XQC_BBR_DISABLE_CWND_AI
                 if ((bbr->congestion_window + sampler->acked 
                     >= (target_cwnd + bbr->beyond_target_cwnd))
                     && sampler->send_ctl->ctl_is_cwnd_limited)
@@ -789,6 +799,7 @@ xqc_bbr_set_cwnd(xqc_bbr_t *bbr, xqc_sample_t *sampler)
                             sampler->acked,
                             bbr->snd_cwnd_cnt_bytes,
                             bbr->beyond_target_cwnd);
+#endif
                 bbr->congestion_window = xqc_min(target_cwnd, 
                                                 bbr->congestion_window + 
                                                 sampler->acked);
@@ -820,11 +831,13 @@ xqc_bbr_on_lost(void *cong_ctl, xqc_usec_t lost_sent_time)
      */
     xqc_bbr_save_cwnd(bbr);
     bbr->recovery_start_time = xqc_monotonic_timestamp();
+#ifndef XQC_BBR_DISABLE_CWND_AI
     /* If losses happened, we do not increase cwnd beyond target_cwnd. */
     bbr->snd_cwnd_cnt_bytes = 0;
     bbr->beyond_target_cwnd = 0;
     bbr->ai_scale = 1;
     bbr->ai_scale_accumulated_bytes = 0;
+#endif
 }
 
 static void 
@@ -870,14 +883,14 @@ xqc_bbr_on_ack(void *cong_ctl, xqc_sample_t *sampler)
     /* maintain windowed max rtt here */
     if (bbr->rttvar_compensation_on) {
         if (sampler->rtt >= 0) {
-            xqc_usec_t last_max_rtt = xqc_win_filter_get_u64(&bbr->max_rtt);
-            xqc_win_filter_max_u64(&bbr->max_rtt, bbr->max_rtt_win_len,
+            xqc_usec_t last_max_rtt = xqc_win_filter_get(&bbr->max_rtt);
+            xqc_win_filter_max(&bbr->max_rtt, bbr->max_rtt_win_len,
                                    bbr->round_cnt, sampler->rtt);
             xqc_log(sampler->send_ctl->ctl_conn->log, XQC_LOG_DEBUG, 
                     "|rttvar_compensation|windowed max rtt info|"
                     "rtt %ui, last_max %ui, max %ui|",
                     sampler->rtt, last_max_rtt, 
-                    xqc_win_filter_get_u64(&bbr->max_rtt));
+                    xqc_win_filter_get(&bbr->max_rtt));
         }
     }
 #endif
@@ -1034,7 +1047,7 @@ static xqc_bbr_info_interface_t xqc_bbr_info_cb = {
 const xqc_cong_ctrl_callback_t xqc_bbr_cb = {
     .xqc_cong_ctl_size                    = xqc_bbr_size,
     .xqc_cong_ctl_init_bbr                = xqc_bbr_init,
-    .xqc_cong_ctl_bbr                     = xqc_bbr_on_ack,
+    .xqc_cong_ctl_on_ack_multiple_pkts    = xqc_bbr_on_ack,
     .xqc_cong_ctl_get_cwnd                = xqc_bbr_get_cwnd,
     .xqc_cong_ctl_get_pacing_rate         = xqc_bbr_get_pacing_rate,
     .xqc_cong_ctl_get_bandwidth_estimate  = xqc_bbr_get_bandwidth,

src/congestion_control/xqc_bbr.h

@@ -15,6 +15,7 @@ typedef char bool;
 #define TRUE 1
 #define FALSE 0
 #define MSEC2SEC 1000000
+#define XQC_BBR_DISABLE_CWND_AI
 
 typedef enum {
     /* Start phase quickly to fill pipe */
@@ -127,10 +128,12 @@ typedef struct xqc_bbr_s {
     uint64_t               probe_rtt_min_us;
     uint64_t               probe_rtt_min_us_stamp;
 
+#ifndef XQC_BBR_DISABLE_CWND_AI
     uint32_t               snd_cwnd_cnt_bytes; /* For AI */
     uint32_t               beyond_target_cwnd; /* To compete with buffer fillers */
     uint32_t               ai_scale_accumulated_bytes;
     uint32_t               ai_scale;
+#endif
 
 #if XQC_BBR_RTTVAR_COMPENSATION_ENABLED
     /* CWND compensation for RTT variation+ */

src/congestion_control/xqc_bbr2.c

@@ -12,7 +12,7 @@
 #include "src/transport/xqc_send_ctl.h"
 #include "src/transport/xqc_packet.h"
 
-#define XQC_BBR2_MAX_DATAGRAM_SIZE XQC_QUIC_MSS
+#define XQC_BBR2_MAX_DATAGRAM_SIZE XQC_MSS
 #define XQC_BBR2_MIN_WINDOW (4 * XQC_BBR2_MAX_DATAGRAM_SIZE)
 /* The RECOMMENDED value is the minimum of 10 * kMaxDatagramSize and max(2* kMaxDatagramSize, 14720)). */
 #define XQC_BBR2_INITIAL_WINDOW (32 * XQC_BBR2_MAX_DATAGRAM_SIZE)
@@ -151,7 +151,7 @@ xqc_bbr2_init(void *cong_ctl, xqc_sample_t *sampler, xqc_cc_params_t cc_params)
     memset(bbr2, 0, sizeof(*bbr2));
 
 #if XQC_BBR2_PLUS_ENABLED
-    xqc_win_filter_reset_u64(&bbr2->max_rtt, 0, 0);
+    xqc_win_filter_reset(&bbr2->max_rtt, 0, 0);
     bbr2->max_rtt_win_len = xqc_bbr2_windowed_max_rtt_win_size;
     if (cc_params.cc_optimization_flags & XQC_BBR2_FLAG_RTTVAR_COMPENSATION) {
         bbr2->rtt_compensation_on = 1;
@@ -1102,7 +1102,7 @@ static uint32_t
 xqc_bbr2_compensate_cwnd_for_rttvar(xqc_bbr2_t *bbr2, xqc_sample_t *sampler)
 {
     xqc_usec_t srtt = sampler->srtt;
-    xqc_usec_t recent_max_rtt = xqc_win_filter_get_u64(&bbr2->max_rtt);
+    xqc_usec_t recent_max_rtt = xqc_win_filter_get(&bbr2->max_rtt);
     xqc_usec_t compensation_thresh = (1 + bbr2->rtt_compensation_thresh) * 
                                      bbr2->min_rtt;
     uint32_t cwnd_addition = 0;
@@ -1422,14 +1422,14 @@ xqc_bbr2_on_ack(void *cong_ctl, xqc_sample_t *sampler)
     /* maintain windowed max rtt here */
     if (bbr2->rtt_compensation_on) {
         if (sampler->rtt >= 0) {
-            xqc_usec_t last_max_rtt = xqc_win_filter_get_u64(&bbr2->max_rtt);
-            xqc_win_filter_max_u64(&bbr2->max_rtt, bbr2->max_rtt_win_len,
+            xqc_usec_t last_max_rtt = xqc_win_filter_get(&bbr2->max_rtt);
+            xqc_win_filter_max(&bbr2->max_rtt, bbr2->max_rtt_win_len,
                                 bbr2->round_cnt, sampler->rtt);
             xqc_log(sampler->send_ctl->ctl_conn->log, XQC_LOG_DEBUG, 
                     "|BBRv2 Plus|windowed max rtt info: rtt %ui, "
                     "last_max %ui, max %ui|",
                     sampler->rtt, last_max_rtt, 
-                    xqc_win_filter_get_u64(&bbr2->max_rtt));
+                    xqc_win_filter_get(&bbr2->max_rtt));
         }
     }
 #endif
@@ -1611,7 +1611,7 @@ static xqc_bbr_info_interface_t xqc_bbr2_info_cb = {
 const xqc_cong_ctrl_callback_t xqc_bbr2_cb = {
     .xqc_cong_ctl_size                   = xqc_bbr2_size,
     .xqc_cong_ctl_init_bbr               = xqc_bbr2_init,
-    .xqc_cong_ctl_bbr                    = xqc_bbr2_on_ack,
+    .xqc_cong_ctl_on_ack_multiple_pkts   = xqc_bbr2_on_ack,
     .xqc_cong_ctl_get_cwnd               = xqc_bbr2_get_cwnd,
     .xqc_cong_ctl_get_pacing_rate        = xqc_bbr2_get_pacing_rate,
     .xqc_cong_ctl_get_bandwidth_estimate = xqc_bbr2_get_bandwidth,