Fixed test issues for dualtor topologies. (#11921)

What is the motivation for this PR?
In some test cases the packets go to unselected ToR in case of active-active links or forwarded by the unselected ToR in case of active-standby links.
Some tests verify queue counters, but this check fails for UC1 in presence of active active links as UC1 always has gRPC traffic flowing through it.
How did you do it?
First issue can be addressed by using the following fixtures in case of active-standby (wherever missing, commit 1) :-
a) toggle_all_simulator_ports_to_rand_selected_tor
b) toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m

To handle this in case of active-active links new fixtures are introduced (commit 3 & 4)
a) setup_standby_ports_on_rand_unselected_tor
b) setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m
c) setup_standby_ports_on_rand_unselected_tor_unconditionally
d) setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally

Second issue can be addressed by simply skipping the check for UC1 in presence of active-active links. (commit 2)

How did you verify/test it?
Verified on Arista 7260 device using dualtor/dualtor-aa topology.

Author: vivekverma-arista

git

@@ -43,7 +43,14 @@
 
 
 __all__ = ['tor_mux_intf', 'tor_mux_intfs', 'ptf_server_intf', 't1_upper_tor_intfs', 't1_lower_tor_intfs',
-           'upper_tor_host', 'lower_tor_host', 'force_active_tor', 'force_standby_tor']
+           'upper_tor_host', 'lower_tor_host', 'force_active_tor', 'force_standby_tor',
+           'config_active_active_dualtor_active_standby', 'validate_active_active_dualtor_setup',
+           'setup_standby_ports_on_rand_selected_tor',
+           'setup_standby_ports_on_rand_unselected_tor',
+           'setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m',
+           'setup_standby_ports_on_rand_unselected_tor_unconditionally',
+           'setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally',
+           ]
 
 logger = logging.getLogger(__name__)
 
@@ -1588,9 +1595,7 @@ def check_active_active_port_status(duthost, ports, status):
 
 
 @pytest.fixture
-def config_active_active_dualtor_active_standby(
-    duthosts, active_active_ports, tbinfo, validate_active_active_dualtor_setup                         # noqa F811
-):
+def config_active_active_dualtor_active_standby(duthosts, active_active_ports, tbinfo):                         # noqa F811
     """Config the active-active dualtor that one ToR as active and the other as standby."""
     if not ('dualtor' in tbinfo['topo']['name'] and active_active_ports):
         yield
@@ -1607,16 +1612,18 @@ def check_active_active_port_status(duthost, ports, status):
                 return False
         return True
 
-    def _config_the_active_active_dualtor(active_tor, standby_tor, ports):
+    def _config_the_active_active_dualtor(active_tor, standby_tor, ports, unconditionally=False):
         active_side_commands = []
         standby_side_commands = []
+        logging.info("Configuring {} as active".format(active_tor.hostname))
+        logging.info("Configuring {} as standby".format(standby_tor.hostname))
         for port in ports:
             if port not in active_active_ports:
                 raise ValueError("Port {} is not in the active-active ports".format(port))
             active_side_commands.append("config mux mode active {}".format(port))
             standby_side_commands.append("config mux mode standby {}".format(port))
 
-        if not check_active_active_port_status(active_tor, ports, 'active'):
+        if not check_active_active_port_status(active_tor, ports, 'active') or unconditionally:
             active_tor.shell_cmds(cmds=active_side_commands)
         standby_tor.shell_cmds(cmds=standby_side_commands)
 
@@ -1749,3 +1756,66 @@ def log_flap_counter(flap_counters):
         if unexpected_flap_mux_ports:
             logging.error(error_str)
             raise ValueError(error_str)
+
+
+@pytest.fixture
+def setup_standby_ports_on_rand_selected_tor(active_active_ports, rand_selected_dut, rand_unselected_dut,                  # noqa F811
+                                             config_active_active_dualtor_active_standby,                                  # noqa F811
+                                             validate_active_active_dualtor_setup):                                        # noqa F811
+    if active_active_ports:
+        config_active_active_dualtor_active_standby(rand_unselected_dut, rand_selected_dut, active_active_ports)
+    return
+
+
+@pytest.fixture
+def setup_standby_ports_on_rand_unselected_tor(active_active_ports, rand_selected_dut, rand_unselected_dut,                  # noqa F811
+                                               config_active_active_dualtor_active_standby,
+                                               validate_active_active_dualtor_setup):
+    if active_active_ports:
+        config_active_active_dualtor_active_standby(rand_selected_dut, rand_unselected_dut, active_active_ports)
+    return
+
+
+@pytest.fixture
+def setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m(
+    active_active_ports,                                                   # noqa F811
+    enum_rand_one_per_hwsku_frontend_hostname,
+    config_active_active_dualtor_active_standby,
+    validate_active_active_dualtor_setup,
+    upper_tor_host,
+    lower_tor_host,
+    duthosts
+):
+    if active_active_ports:
+        active_tor = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
+        standby_tor = upper_tor_host if active_tor == lower_tor_host else lower_tor_host
+        config_active_active_dualtor_active_standby(active_tor, standby_tor, active_active_ports)
+    return
+
+
+@pytest.fixture
+def setup_standby_ports_on_rand_unselected_tor_unconditionally(
+    active_active_ports,                                                   # noqa F811
+    rand_selected_dut,
+    rand_unselected_dut,
+    config_active_active_dualtor_active_standby
+):
+    if active_active_ports:
+        config_active_active_dualtor_active_standby(rand_selected_dut, rand_unselected_dut, active_active_ports, True)
+    return
+
+
+@pytest.fixture
+def setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally(
+    active_active_ports,                                                   # noqa F811
+    enum_rand_one_per_hwsku_frontend_hostname,
+    config_active_active_dualtor_active_standby,
+    upper_tor_host,
+    lower_tor_host,
+    duthosts
+):
+    if active_active_ports:
+        active_tor = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
+        standby_tor = upper_tor_host if active_tor == lower_tor_host else lower_tor_host
+        config_active_active_dualtor_active_standby(active_tor, standby_tor, active_active_ports, True)
+    return

tests/common/dualtor/dual_tor_utils.py

@@ -306,6 +306,7 @@ def start_dhcp_monitor_debug_counter(duthost):
 
 
 def test_dhcp_relay_default(ptfhost, dut_dhcp_relay_data, validate_dut_routes_exist, testing_config,
+                            setup_standby_ports_on_rand_unselected_tor,												# noqa F811
                             rand_unselected_dut, toggle_all_simulator_ports_to_rand_selected_tor_m):     # noqa F811
     """Test DHCP relay functionality on T0 topology.
        For each DHCP relay agent running on the DuT, verify DHCP packets are relayed properly
@@ -508,6 +509,7 @@ def test_dhcp_relay_start_with_uplinks_down(ptfhost, dut_dhcp_relay_data, valida
 
 
 def test_dhcp_relay_unicast_mac(ptfhost, dut_dhcp_relay_data, validate_dut_routes_exist, testing_config,
+                                setup_standby_ports_on_rand_unselected_tor,				 # noqa F811
                                 toggle_all_simulator_ports_to_rand_selected_tor_m):     # noqa F811
     """Test DHCP relay functionality on T0 topology with unicast mac
        Instead of using broadcast MAC, use unicast MAC of DUT and verify that DHCP relay functionality is entact.
@@ -546,6 +548,7 @@ def test_dhcp_relay_unicast_mac(ptfhost, dut_dhcp_relay_data, validate_dut_route
 
 
 def test_dhcp_relay_random_sport(ptfhost, dut_dhcp_relay_data, validate_dut_routes_exist, testing_config,
+                                 setup_standby_ports_on_rand_unselected_tor,				 # noqa F811
                                  toggle_all_simulator_ports_to_rand_selected_tor_m):    # noqa F811
     """Test DHCP relay functionality on T0 topology with random source port (sport)
        If the client is SNAT'd, the source port could be changed to a non-standard port (i.e., not 68).

tests/dhcp_relay/test_dhcp_relay.py

@@ -149,6 +149,7 @@ def verifyFdbArp(duthost, dst_ip, dst_mac, dst_intf):
 
 @pytest.mark.parametrize("drop_reason", ["L3_EGRESS_LINK_DOWN"])
 def test_neighbor_link_down(testbed_params, setup_counters, duthosts, rand_one_dut_hostname,
+                            setup_standby_ports_on_rand_unselected_tor,                             # noqa F811
                             toggle_all_simulator_ports_to_rand_selected_tor_m, mock_server,         # noqa F811
                             send_dropped_traffic, drop_reason, generate_dropped_packet, tbinfo):
     """

tests/drop_packets/test_configurable_drop_counters.py

@@ -278,8 +278,8 @@ def setup_active_active_ports(active_active_ports, rand_selected_dut, rand_unsel
 def test_encap_with_mirror_session(rand_selected_dut, rand_selected_interface,              # noqa F811
                                    ptfadapter, tbinfo, setup_mirror_session,
                                    toggle_all_simulator_ports_to_rand_unselected_tor,       # noqa F811
-                                   setup_active_active_ports,                               # noqa F811
-                                   tunnel_traffic_monitor):                                 # noqa F811
+                                   tunnel_traffic_monitor,                                  # noqa F811
+                                   setup_standby_ports_on_rand_selected_tor):               # noqa F811
     """
     A test case to verify the bounced back packet from Standby ToR to T1 doesn't have an unexpected vlan id (4095)
     The issue can happen if the bounced back packets egressed from the monitor port of mirror session

tests/dualtor/test_ipinip.py

@@ -335,18 +335,6 @@ def runcmd():
         pytest.skip('file {} not found'.format(file))
 
 
-@pytest.fixture
-def setup_active_active_ports(active_active_ports, rand_selected_dut, rand_unselected_dut,                  # noqa F811
-                              config_active_active_dualtor_active_standby,                                  # noqa F811
-                              validate_active_active_dualtor_setup):                                        # noqa F811
-    if active_active_ports:
-        logging.info("Configuring {} as active".format(rand_unselected_dut.hostname))
-        logging.info("Configuring {} as standby".format(rand_selected_dut.hostname))
-        config_active_active_dualtor_active_standby(rand_unselected_dut, rand_selected_dut, active_active_ports)
-
-    return
-
-
 @pytest.mark.parametrize("dscp", [3, 4, 2, 6])      # lossless queue is 3 or 4 or 2 or 6.
 def test_dscp_to_queue_during_encap_on_standby(
     dscp,
@@ -358,7 +346,7 @@ def test_dscp_to_queue_during_encap_on_standby(
     duthosts,
     rand_one_dut_hostname,
     write_standby,
-    setup_active_active_ports
+    setup_standby_ports_on_rand_selected_tor        # noqa F811
 ):
     """
     Test if DSCP to Q mapping for outer header is matching with inner header during encap on standby
@@ -423,7 +411,7 @@ def test_ecn_during_encap_on_standby(
     rand_selected_interface, ptfadapter,                    # noqa F811
     tbinfo, rand_selected_dut, tunnel_traffic_monitor,      # noqa F811
     write_standby,
-    setup_active_active_ports
+    setup_standby_ports_on_rand_selected_tor                # noqa F811
 ):
     """
     Test if the ECN stamping on outer header is matching with inner during encap on standby

tests/dualtor/test_tor_ecn.py

@@ -794,6 +794,7 @@ def set_traffic_action(self, duthost, action):
 
     def test_pfcwd_actions(self, request, fake_storm, setup_pfc_test, setup_dut_test_params, enum_fanout_graph_facts,  # noqa F811
                            ptfhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts,
+                           setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally,         # noqa F811
                            pause_icmp_responder, toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m): # noqa F811
         """
         PFCwd functional test
@@ -867,6 +868,7 @@ def test_pfcwd_actions(self, request, fake_storm, setup_pfc_test, setup_dut_test
 
     def test_pfcwd_multi_port(self, request, fake_storm, setup_pfc_test, setup_dut_test_params, enum_fanout_graph_facts,  # noqa F811
                               ptfhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts,
+                              setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally,         # noqa F811
                               pause_icmp_responder, toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m): # noqa F811
         """
         Tests pfcwd behavior when 2 ports are under pfc storm one after the other
@@ -944,6 +946,7 @@ def test_pfcwd_multi_port(self, request, fake_storm, setup_pfc_test, setup_dut_t
 
     def test_pfcwd_mmu_change(self, request, fake_storm, setup_pfc_test, setup_dut_test_params, enum_fanout_graph_facts,   # noqa F811
                               ptfhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts, dualtor_ports, # noqa F811
+                              setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally,       # noqa F811
                               pause_icmp_responder, toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m): # noqa F811
         """
         Tests if mmu changes impact Pfcwd functionality
@@ -1034,6 +1037,7 @@ def test_pfcwd_mmu_change(self, request, fake_storm, setup_pfc_test, setup_dut_t
 
     def test_pfcwd_port_toggle(self, request, fake_storm, setup_pfc_test, setup_dut_test_params, enum_fanout_graph_facts,  # noqa F811
                                tbinfo, ptfhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts,
+                               setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally,         # noqa F811
                                pause_icmp_responder, toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m): # noqa F811
         """
         Test PfCWD functionality after toggling port

tests/pfcwd/test_pfcwd_function.py

@@ -251,6 +251,7 @@ def test_tunnel_decap_dscp_to_queue_mapping(ptfhost, rand_selected_dut, rand_uns
 
 
 def test_separated_qos_map_on_tor(ptfhost, rand_selected_dut, rand_unselected_dut,
+                                  setup_standby_ports_on_rand_unselected_tor,
         toggle_all_simulator_ports_to_rand_selected_tor, tbinfo, ptfadapter, dut_qos_maps_module):  # noqa F811
     """
     The test case is to verify separated DSCP_TO_TC_MAP/TC_TO_QUEUE_MAP on uplink and downlink ports of dualtor
@@ -359,6 +360,7 @@ def pfc_pause_test(storm_handler, peer_info, prio, ptfadapter, dut, port, queue,
 
 
 def test_pfc_pause_extra_lossless_standby(ptfhost, fanouthosts, rand_selected_dut, rand_unselected_dut,
+                                          setup_standby_ports_on_rand_selected_tor,
         toggle_all_simulator_ports_to_rand_unselected_tor, tbinfo, ptfadapter, conn_graph_facts, fanout_graph_facts, dut_config): # noqa F811
     """
     The test case is to verify PFC pause frame can pause extra lossless queues in dualtor deployment.
@@ -436,6 +438,7 @@ def test_pfc_pause_extra_lossless_standby(ptfhost, fanouthosts, rand_selected_du
 
 
 def test_pfc_pause_extra_lossless_active(ptfhost, fanouthosts, rand_selected_dut, rand_unselected_dut,
+                                         setup_standby_ports_on_rand_unselected_tor,
         toggle_all_simulator_ports_to_rand_selected_tor, tbinfo, ptfadapter, conn_graph_facts, fanout_graph_facts, dut_config): # noqa F811
     """
     The test case is to verify PFC pause frame can pause extra lossless queues in dualtor deployment.

tests/qos/test_tunnel_qos_remap.py

@@ -7,6 +7,8 @@
 import pytest
 import ptf.testutils as testutils
 import ptf.packet as scapy
+from tests.common.dualtor.mux_simulator_control import \
+            toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m   # noqa F401
 
 from ptf.mask import Mask
 from natsort import natsorted
@@ -374,7 +376,9 @@ def get_dev_port_and_route(duthost, asichost, dst_prefix_set):
 
 def test_route_flap(duthosts, tbinfo, ptfhost, ptfadapter,
                     get_function_conpleteness_level, announce_default_routes,
-                    enum_rand_one_per_hwsku_frontend_hostname, enum_rand_one_frontend_asic_index, loganalyzer):
+                    enum_rand_one_per_hwsku_frontend_hostname, enum_rand_one_frontend_asic_index,
+                    setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m,                     # noqa F811
+                    toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m, loganalyzer):    # noqa F811
     ptf_ip = tbinfo['ptf_ip']
     common_config = tbinfo['topo']['properties']['configuration_properties'].get(
         'common', {})

tests/route/test_route_flap.py

@@ -75,6 +75,7 @@ def build_icmp_packet(vlan_id, src_mac="00:22:00:00:00:02", dst_mac="ff:ff:ff:ff
 @pytest.mark.po2vlan
 def test_snmp_fdb_send_tagged(ptfadapter, duthosts, rand_one_dut_hostname,          # noqa F811
                               toggle_all_simulator_ports_to_rand_selected_tor_m,    # noqa F811
+                              setup_standby_ports_on_rand_unselected_tor,           # noqa F811
                               rand_selected_dut, tbinfo, ports_list, localhost, creds_all_duts): # noqa F811
     """
     Send tagged packets from each port.