From 7dec0010fdc251bf44ccaebe8e885816e0282595 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 11:32:36 -0700 Subject: [PATCH 01/17] First pass on session_log fixes --- netmiko/session_log.py | 33 ++++++++++++++++++++++++++----- tests/test_netmiko_session_log.py | 5 +++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/netmiko/session_log.py b/netmiko/session_log.py index 9ead1316c..4a2cba10c 100644 --- a/netmiko/session_log.py +++ b/netmiko/session_log.py @@ -12,6 +12,7 @@ def __init__( file_encoding: str = "utf-8", no_log: Optional[Dict[str, Any]] = None, record_writes: bool = False, + slog_buffer: Optional[io.StringIO] = None, ) -> None: if no_log is None: self.no_log = {} @@ -30,6 +31,13 @@ def __init__( else: self.session_log = None + # In order to ensure all the no_log entries get hidden properly + # we must first store everying in memory and then write out to file. + # Otherwise, we might miss the data we are supposed to hide (they span + # multiple reads). + if slog_buffer is None: + self.slog_buffer = io.StringIO() + # Ensures last write operations prior to disconnect are recorded. self.fin = False @@ -49,15 +57,24 @@ def open(self) -> None: def close(self) -> None: """Close the session_log file (if it is a file that we opened).""" + self.flush() if self.session_log and self._session_log_close: self.session_log.close() self.session_log = None - def write(self, data: str) -> None: - if self.session_log is not None and len(data) > 0: - # Hide the password and secret in the session_log - for hidden_data in self.no_log.values(): - data = data.replace(hidden_data, "********") + def no_log_filter(self, data: str) -> str: + """Filter content from the session_log.""" + for hidden_data in self.no_log.values(): + data = data.replace(hidden_data, "********") + return data + + def flush(self) -> None: + """Force the slog_buffer to be written out to the actual file""" + + if self.session_log is not None: + self.slog_buffer.seek(0) + data = self.slog_buffer.read() + data = self.no_log_filter(data) if isinstance(self.session_log, io.BufferedIOBase): self.session_log.write(write_bytes(data, encoding=self.file_encoding)) @@ -67,4 +84,10 @@ def write(self, data: str) -> None: assert isinstance(self.session_log, io.BufferedIOBase) or isinstance( self.session_log, io.TextIOBase ) + + # Flush the underlying file self.session_log.flush() + + def write(self, data: str) -> None: + if len(data) > 0: + self.slog_buffer.write(data) diff --git a/tests/test_netmiko_session_log.py b/tests/test_netmiko_session_log.py index bc41398ab..ef8469524 100644 --- a/tests/test_netmiko_session_log.py +++ b/tests/test_netmiko_session_log.py @@ -171,6 +171,8 @@ def test_unicode(device_slog): conn.session_log.write("\nTesting unicode\n") conn.session_log.write(smiley_face) conn.session_log.write(smiley_face) + time.sleep(1) + conn.session_log.flush() file_name = device_slog["session_log"] with open(file_name, "r") as f: @@ -241,3 +243,6 @@ def test_session_log_custom_secrets(device_slog): assert sanitize_secrets["secret3"] not in session_log if sanitize_secrets.get("supersecret") is not None: assert sanitize_secrets["supersecret"] not in session_log + + +def test_session_log_no_log(device_slog): From ed2b4bd66da0b9cc756d3cc81701e139715fc171 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 11:36:50 -0700 Subject: [PATCH 02/17] Remove incomplete test --- tests/test_netmiko_session_log.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_netmiko_session_log.py b/tests/test_netmiko_session_log.py index ef8469524..955dfd056 100644 --- a/tests/test_netmiko_session_log.py +++ b/tests/test_netmiko_session_log.py @@ -243,6 +243,3 @@ def test_session_log_custom_secrets(device_slog): assert sanitize_secrets["secret3"] not in session_log if sanitize_secrets.get("supersecret") is not None: assert sanitize_secrets["supersecret"] not in session_log - - -def test_session_log_no_log(device_slog): From ea4c598f8584d059dff5acb85d16a92279f2e7e2 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 12:09:21 -0700 Subject: [PATCH 03/17] Force session_log flush on certain events --- netmiko/base_connection.py | 18 ++++++++++++++++++ netmiko/session_log.py | 16 +++++++++++----- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/netmiko/base_connection.py b/netmiko/base_connection.py index 12191c391..878b70702 100644 --- a/netmiko/base_connection.py +++ b/netmiko/base_connection.py @@ -102,6 +102,20 @@ def wrapper_decorator(self: "BaseConnection", *args: Any, **kwargs: Any) -> Any: return cast(F, wrapper_decorator) +def flush_session_log(func: F) -> F: + @functools.wraps(func) + def wrapper_decorator(self: "BaseConnection", *args: Any, **kwargs: Any) -> Any: + try: + return_val = func(self, *args, **kwargs) + finally: + # Always flush the session_log + if self.session_log: + self.session_log.flush() + return return_val + + return cast(F, wrapper_decorator) + + def log_writes(func: F) -> F: """Handle both session_log and log of writes.""" @@ -391,6 +405,7 @@ def __init__( elif isinstance(session_log, SessionLog): # SessionLog object self.session_log = session_log + self.session_log.open() else: raise ValueError( "session_log must be a path to a file, a file handle, " @@ -1467,6 +1482,7 @@ def command_echo_read(self, cmd: str, read_timeout: float) -> str: pass return new_data + @flush_session_log @select_cmd_verify def send_command_timing( self, @@ -1615,6 +1631,7 @@ def _prompt_handler(self, auto_find_prompt: bool) -> str: prompt = self.base_prompt return re.escape(prompt.strip()) + @flush_session_log @select_cmd_verify def send_command( self, @@ -2138,6 +2155,7 @@ def send_config_from_file( commands = cfg_file.readlines() return self.send_config_set(commands, **kwargs) + @flush_session_log def send_config_set( self, config_commands: Union[str, Sequence[str], Iterator[str], TextIO, None] = None, diff --git a/netmiko/session_log.py b/netmiko/session_log.py index 4a2cba10c..80adea31c 100644 --- a/netmiko/session_log.py +++ b/netmiko/session_log.py @@ -31,10 +31,10 @@ def __init__( else: self.session_log = None - # In order to ensure all the no_log entries get hidden properly + # In order to ensure all the no_log entries get hidden properly, # we must first store everying in memory and then write out to file. - # Otherwise, we might miss the data we are supposed to hide (they span - # multiple reads). + # Otherwise, we might miss the data we are supposed to hide (since + # the no_log data potentially spans multiple reads). if slog_buffer is None: self.slog_buffer = io.StringIO() @@ -68,12 +68,18 @@ def no_log_filter(self, data: str) -> str: data = data.replace(hidden_data, "********") return data + def _read_buffer(self) -> str: + self.slog_buffer.seek(0) + data = self.slog_buffer.read() + # Once read, create a new buffer + self.slog_buffer = io.StringIO() + return data + def flush(self) -> None: """Force the slog_buffer to be written out to the actual file""" if self.session_log is not None: - self.slog_buffer.seek(0) - data = self.slog_buffer.read() + data = self._read_buffer() data = self.no_log_filter(data) if isinstance(self.session_log, io.BufferedIOBase): From fbce3c7bb77ff1d1952a0da7a23353a56a03bcdd Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 13:19:40 -0700 Subject: [PATCH 04/17] Updating session log testing --- ..._session_log_append-cisco881_slog_append.log | 17 +++++++++++++++++ ..._log_append-cisco881_slog_append_compare.log | 15 +++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/SLOG/test_session_log_append-cisco881_slog_append.log create mode 100644 tests/SLOG/test_session_log_append-cisco881_slog_append_compare.log diff --git a/tests/SLOG/test_session_log_append-cisco881_slog_append.log b/tests/SLOG/test_session_log_append-cisco881_slog_append.log new file mode 100644 index 000000000..40fa67d00 --- /dev/null +++ b/tests/SLOG/test_session_log_append-cisco881_slog_append.log @@ -0,0 +1,17 @@ +Initial file contents + +cisco1> +cisco1>terminal width 511 +cisco1>terminal length 0 +cisco1> +cisco1> +cisco1>show ip interface brief +Interface IP-Address OK? Method Status Protocol +FastEthernet0 unassigned YES unset down down +FastEthernet1 unassigned YES unset down down +FastEthernet2 unassigned YES unset down down +FastEthernet3 unassigned YES unset down down +FastEthernet4 10.220.88.20 YES NVRAM up up +Vlan1 unassigned YES unset down down +cisco1> +cisco1>exit diff --git a/tests/SLOG/test_session_log_append-cisco881_slog_append_compare.log b/tests/SLOG/test_session_log_append-cisco881_slog_append_compare.log new file mode 100644 index 000000000..00bb16745 --- /dev/null +++ b/tests/SLOG/test_session_log_append-cisco881_slog_append_compare.log @@ -0,0 +1,15 @@ +cisco1> +cisco1>terminal width 511 +cisco1>terminal length 0 +cisco1> +cisco1> +cisco1>show ip interface brief +Interface IP-Address OK? Method Status Protocol +FastEthernet0 unassigned YES unset down down +FastEthernet1 unassigned YES unset down down +FastEthernet2 unassigned YES unset down down +FastEthernet3 unassigned YES unset down down +FastEthernet4 10.220.88.20 YES NVRAM up up +Vlan1 unassigned YES unset down down +cisco1> +cisco1>exit From 7848bf7653e9e267ecf5cdce6d0dd542d69dacb2 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 13:24:47 -0700 Subject: [PATCH 05/17] SLOG files --- tests/SLOG/cisco881_slog.log | 1 - tests/SLOG/cisco881_slog_append.log | 17 +------ tests/SLOG/cisco881_slog_wr.log | 19 ------- tests/SLOG/device_log.log | 0 tests/SLOG/netmiko.log | 77 ++++++++++++++++++++++++++--- 5 files changed, 72 insertions(+), 42 deletions(-) create mode 100644 tests/SLOG/device_log.log diff --git a/tests/SLOG/cisco881_slog.log b/tests/SLOG/cisco881_slog.log index 1e00df96e..00bb16745 100644 --- a/tests/SLOG/cisco881_slog.log +++ b/tests/SLOG/cisco881_slog.log @@ -1,4 +1,3 @@ - cisco1> cisco1>terminal width 511 cisco1>terminal length 0 diff --git a/tests/SLOG/cisco881_slog_append.log b/tests/SLOG/cisco881_slog_append.log index 213ca5b50..aaecd375c 100644 --- a/tests/SLOG/cisco881_slog_append.log +++ b/tests/SLOG/cisco881_slog_append.log @@ -1,5 +1,6 @@ Initial file contents + cisco1> cisco1>terminal width 511 cisco1>terminal length 0 @@ -15,19 +16,3 @@ FastEthernet4 10.220.88.20 YES NVRAM up up Vlan1 unassigned YES unset down down cisco1> cisco1>exit -cisco1> -cisco1>terminal width 511 -cisco1>terminal length 0 -cisco1> -cisco1> -Testing password and secret replacement -This is my password ******** -This is my secret ******** - -cisco1> -cisco1>terminal width 511 -cisco1>terminal length 0 -cisco1> -cisco1> -Testing unicode -😁😁 diff --git a/tests/SLOG/cisco881_slog_wr.log b/tests/SLOG/cisco881_slog_wr.log index 71a29c32b..e69de29bb 100644 --- a/tests/SLOG/cisco881_slog_wr.log +++ b/tests/SLOG/cisco881_slog_wr.log @@ -1,19 +0,0 @@ - -terminal width 511 -cisco1> -cisco1>terminal width 511 -cisco1>terminal length 0 -terminal length 0 -cisco1> - -cisco1> - -cisco1>enable -enable -Password: ******** - -cisco1# - -cisco1# - -cisco1#exit diff --git a/tests/SLOG/device_log.log b/tests/SLOG/device_log.log new file mode 100644 index 000000000..e69de29bb diff --git a/tests/SLOG/netmiko.log b/tests/SLOG/netmiko.log index 3c3c8b275..92b448c2d 100644 --- a/tests/SLOG/netmiko.log +++ b/tests/SLOG/netmiko.log @@ -38,8 +38,8 @@ write_channel: b'terminal width 511\n' read_channel: cisco1> cisco1> -read_channel: terminal widt -read_channel: h 511 +read_channel: terminal wid +read_channel: th 511 cisco1> Pattern found: (terminal width 511) cisco1> @@ -64,6 +64,16 @@ write_channel: b'\n' read_channel: read_channel: cisco1> + +Parenthesis found in pattern. + +pattern: (\#|>) + + +This can be problemtic when used in read_until_pattern(). + +You should ensure that you use either non-capture groups i.e. '(?:' or that the +parenthesis completely wrap the pattern '(pattern)' Pattern found: (\#|>) cisco1> read_channel: @@ -72,8 +82,8 @@ write_channel: b'\n' write_channel: b'terminal width 511\n' read_channel: cisco1> cisco1> -read_channel: terminal wid -read_channel: th 511 +read_channel: terminal widt +read_channel: h 511 cisco1> Pattern found: (terminal width 511) cisco1> cisco1>terminal width 511 @@ -97,6 +107,16 @@ write_channel: b'\n' read_channel: read_channel: cisco1> + +Parenthesis found in pattern. + +pattern: (\#|>) + + +This can be problemtic when used in read_until_pattern(). + +You should ensure that you use either non-capture groups i.e. '(?:' or that the +parenthesis completely wrap the pattern '(pattern)' Pattern found: (\#|>) cisco1> read_channel: @@ -110,8 +130,8 @@ read_channel: [find_prompt()]: prompt is cisco1> write_channel: b'show ip interface brief\n' read_channel: -read_channel: show ip inter -read_channel: face brief +read_channel: show ip interf +read_channel: ace brief Interface IP-Address OK? Method Status Protocol FastEthernet0 unassigned YES unset down down FastEthernet1 unassigned YES unset down down @@ -129,3 +149,48 @@ cisco1> Pattern found: ([>#]) cisco1> write_channel: b'exit\n' +write_channel: b'\n' +write_channel: b'terminal width 511\n' +read_channel: +cisco1> +cisco1> +read_channel: terminal wid +read_channel: th 511 +cisco1> +Pattern found: (terminal width 511) +cisco1> +cisco1>terminal width 511 +In disable_paging +Command: terminal length 0 + +write_channel: b'terminal length 0\n' +read_channel: +read_channel: terminal lengt +read_channel: h 0 +cisco1> +Pattern found: (terminal\ length\ 0) +cisco1>terminal length 0 + +cisco1>terminal length 0 +Exiting disable_paging +read_channel: +Clear buffer detects data in the channel +read_channel: +write_channel: b'\n' +read_channel: +read_channel: +cisco1> + +Parenthesis found in pattern. + +pattern: (\#|>) + + +This can be problemtic when used in read_until_pattern(). + +You should ensure that you use either non-capture groups i.e. '(?:' or that the +parenthesis completely wrap the pattern '(pattern)' +Pattern found: (\#|>) +cisco1> +read_channel: +[find_prompt()]: prompt is cisco1> From 496077b8d34450547ff3a3bcbbdc1cc1140ca2d4 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 13:46:21 -0700 Subject: [PATCH 06/17] Updating SLOG test files --- tests/SLOG/test_session_log_secrets-cisco881_slog.log | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tests/SLOG/test_session_log_secrets-cisco881_slog.log diff --git a/tests/SLOG/test_session_log_secrets-cisco881_slog.log b/tests/SLOG/test_session_log_secrets-cisco881_slog.log new file mode 100644 index 000000000..da0e4508f --- /dev/null +++ b/tests/SLOG/test_session_log_secrets-cisco881_slog.log @@ -0,0 +1,8 @@ +cisco1> +cisco1>terminal width 511 +cisco1>terminal length 0 +cisco1> +cisco1> +Testing password and secret replacement +This is my password ******** +This is my secret ******** From 31c4cb30ca2005459e097678ebc2c6be2f4a3a6e Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 13:46:33 -0700 Subject: [PATCH 07/17] Improving session_log testing --- tests/conftest.py | 19 +++++++++++++++++- tests/test_netmiko_session_log.py | 32 ++++++++++++++++++++++++++----- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b5931febb..c8feff91b 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -117,7 +117,7 @@ def net_connect_slog_wr(request): @pytest.fixture(scope="module") def device_slog(request): """ - Create the SSH connection to the remote device. Modify session_log init arguments. + Modify session_log init arguments. Return the netmiko device (not connected) """ @@ -131,6 +131,23 @@ def device_slog(request): return device +@pytest.fixture(scope="function") +def device_slog_test_name(request): + """ + Modify session_log init arguments. + + Return the netmiko device (not connected) + """ + device_under_test = request.config.getoption("test_device") + test_devices = parse_yaml(PWD + "/etc/test_devices.yml") + device = test_devices[device_under_test] + # Fictional secret + device["secret"] = "invalid" + device["verbose"] = False + test_name = request.node.name + return (device, test_name) + + @pytest.fixture(scope="module") def device_failed_key(request): """ diff --git a/tests/test_netmiko_session_log.py b/tests/test_netmiko_session_log.py index 955dfd056..62bbf90c7 100644 --- a/tests/test_netmiko_session_log.py +++ b/tests/test_netmiko_session_log.py @@ -7,6 +7,12 @@ from netmiko.session_log import SessionLog +def add_test_name_to_file_name(initial_fname, test_name): + dir_name, f_name = initial_fname.split("/") + new_file_name = f"{dir_name}/{test_name}-{f_name}" + return new_file_name + + def calc_md5(file_name=None, contents=None): """Compute MD5 hash of file.""" if contents is not None: @@ -98,30 +104,46 @@ def test_session_log_write(net_connect_slog_wr, commands, expected_responses): assert session_log_md5 == compare_log_md5 -def test_session_log_append(device_slog, commands, expected_responses): +def test_session_log_append(device_slog_test_name, commands, expected_responses): """Verify session_log matches expected content, but when channel writes are also logged.""" - session_file = expected_responses["session_log_append"] + + device_slog = device_slog_test_name[0] + test_name = device_slog_test_name[1] + slog_file = expected_responses['session_log_append'] + session_file = add_test_name_to_file_name(slog_file, test_name) + # Create a starting file with open(session_file, "wb") as f: f.write(b"Initial file contents\n\n") # The netmiko connection has not been established yet. device_slog["session_log"] = session_file + device_slog["session_log_file_mode"] = "append" conn = ConnectHandler(**device_slog) command = commands["basic"] session_action(conn, command) - compare_file = expected_responses["compare_log_append"] + compare_file_base = expected_responses['compare_log_append'] + dir_name, f_name = compare_file_base.split("/") + compare_file = f"{dir_name}/{test_name}-{f_name}" session_log_md5_append(session_file, compare_file) -def test_session_log_secrets(device_slog): +def test_session_log_secrets(device_slog_test_name): """Verify session_log does not contain password or secret.""" + device_slog = device_slog_test_name[0] + test_name = device_slog_test_name[1] + slog_file = device_slog["session_log"] + + new_slog_file = add_test_name_to_file_name(slog_file, test_name) + device_slog["session_log"] = new_slog_file + conn = ConnectHandler(**device_slog) conn.session_log.write("\nTesting password and secret replacement\n") conn.session_log.write("This is my password {}\n".format(conn.password)) conn.session_log.write("This is my secret {}\n".format(conn.secret)) + conn.session_log.flush() file_name = device_slog["session_log"] with open(file_name, "r") as f: @@ -130,6 +152,7 @@ def test_session_log_secrets(device_slog): assert conn.password not in session_log if conn.secret: assert conn.secret not in session_log + assert "terminal width" in session_log def test_logging_filter_secrets(net_connect_slog_wr): @@ -211,7 +234,6 @@ def test_session_log_custom_secrets(device_slog): } # Create custom session log custom_log = SessionLog(file_name="SLOG/device_log.log", no_log=sanitize_secrets) - custom_log.open() # Pass in custom SessionLog obj to session_log attribute device_slog["session_log"] = custom_log From 7a92f2b5ea4a33d1841fb8e65daa051f6c2de78f Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 13:54:13 -0700 Subject: [PATCH 08/17] Update SLOG test files --- tests/SLOG/test_session_log_secrets-cisco881_slog.log | 3 +++ tests/SLOG/test_unicode-cisco881_slog.log | 8 ++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/SLOG/test_unicode-cisco881_slog.log diff --git a/tests/SLOG/test_session_log_secrets-cisco881_slog.log b/tests/SLOG/test_session_log_secrets-cisco881_slog.log index da0e4508f..db7940a0e 100644 --- a/tests/SLOG/test_session_log_secrets-cisco881_slog.log +++ b/tests/SLOG/test_session_log_secrets-cisco881_slog.log @@ -1,3 +1,4 @@ + cisco1> cisco1>terminal width 511 cisco1>terminal length 0 @@ -6,3 +7,5 @@ cisco1> Testing password and secret replacement This is my password ******** This is my secret ******** + +cisco1>exit diff --git a/tests/SLOG/test_unicode-cisco881_slog.log b/tests/SLOG/test_unicode-cisco881_slog.log new file mode 100644 index 000000000..5fa74b4c6 --- /dev/null +++ b/tests/SLOG/test_unicode-cisco881_slog.log @@ -0,0 +1,8 @@ +cisco1> +cisco1>terminal width 511 +cisco1>terminal length 0 +cisco1> +cisco1> +Testing unicode +😁😁 +cisco1>exit From 9666a567104194db6e20564052c9aa47792bb598 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 14:08:22 -0700 Subject: [PATCH 09/17] Improving session_log testing --- ...sion_log_bytesio-cisco881_slog_compare.log | 22 +++++++++++++++++ tests/test_netmiko_session_log.py | 24 +++++++++++++++---- 2 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 tests/SLOG/test_session_log_bytesio-cisco881_slog_compare.log diff --git a/tests/SLOG/test_session_log_bytesio-cisco881_slog_compare.log b/tests/SLOG/test_session_log_bytesio-cisco881_slog_compare.log new file mode 100644 index 000000000..ad78858dd --- /dev/null +++ b/tests/SLOG/test_session_log_bytesio-cisco881_slog_compare.log @@ -0,0 +1,22 @@ + +terminal width 511 +cisco1> +cisco1>terminal width 511 +cisco1>terminal length 0 +terminal length 0 +cisco1> + +cisco1> + +cisco1>show ip interface brief +show ip interface brief +Interface IP-Address OK? Method Status Protocol +FastEthernet0 unassigned YES unset down down +FastEthernet1 unassigned YES unset down down +FastEthernet2 unassigned YES unset down down +FastEthernet3 unassigned YES unset down down +FastEthernet4 10.220.88.20 YES NVRAM up up +Vlan1 unassigned YES unset down down +cisco1> + +cisco1>exit diff --git a/tests/test_netmiko_session_log.py b/tests/test_netmiko_session_log.py index 62bbf90c7..b37a99202 100644 --- a/tests/test_netmiko_session_log.py +++ b/tests/test_netmiko_session_log.py @@ -143,7 +143,9 @@ def test_session_log_secrets(device_slog_test_name): conn.session_log.write("\nTesting password and secret replacement\n") conn.session_log.write("This is my password {}\n".format(conn.password)) conn.session_log.write("This is my secret {}\n".format(conn.secret)) + time.sleep(1) conn.session_log.flush() + conn.disconnect() file_name = device_slog["session_log"] with open(file_name, "r") as f: @@ -186,16 +188,23 @@ def test_logging_filter_secrets(net_connect_slog_wr): assert nc.secret not in netmiko_log -def test_unicode(device_slog): +def test_unicode(device_slog_test_name): """Verify that you can write unicode characters into the session_log.""" + device_slog = device_slog_test_name[0] + test_name = device_slog_test_name[1] + slog_file = device_slog["session_log"] + + new_slog_file = add_test_name_to_file_name(slog_file, test_name) + device_slog["session_log"] = new_slog_file + conn = ConnectHandler(**device_slog) smiley_face = "\N{grinning face with smiling eyes}" conn.session_log.write("\nTesting unicode\n") conn.session_log.write(smiley_face) conn.session_log.write(smiley_face) - time.sleep(1) - conn.session_log.flush() + + conn.disconnect() file_name = device_slog["session_log"] with open(file_name, "r") as f: @@ -203,19 +212,26 @@ def test_unicode(device_slog): assert smiley_face in session_log -def test_session_log_bytesio(device_slog, commands, expected_responses): +def test_session_log_bytesio(device_slog_test_name, commands, expected_responses): """Verify session_log matches expected content, but when channel writes are also logged.""" + device_slog = device_slog_test_name[0] + test_name = device_slog_test_name[1] + slog_file = device_slog["session_log"] + s_log = io.BytesIO() # The netmiko connection has not been established yet. device_slog["session_log"] = s_log device_slog["session_log_file_mode"] = "write" + device_slog["session_log_record_writes"] = True conn = ConnectHandler(**device_slog) command = commands["basic"] session_action(conn, command) + conn.disconnect() compare_file = expected_responses["compare_log"] + compare_file = add_test_name_to_file_name(compare_file, test_name) compare_log_md5 = calc_md5(file_name=compare_file) log_content = s_log.getvalue() From ad04fdf649d741f1a7c15ea13167978e5a408b84 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 14:14:32 -0700 Subject: [PATCH 10/17] Session log testing --- tests/SLOG/device_log.log | 0 .../test_session_log_custom_secrets-cisco881_slog.log | 11 +++++++++++ 2 files changed, 11 insertions(+) delete mode 100644 tests/SLOG/device_log.log create mode 100644 tests/SLOG/test_session_log_custom_secrets-cisco881_slog.log diff --git a/tests/SLOG/device_log.log b/tests/SLOG/device_log.log deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/SLOG/test_session_log_custom_secrets-cisco881_slog.log b/tests/SLOG/test_session_log_custom_secrets-cisco881_slog.log new file mode 100644 index 000000000..63b2a3dd5 --- /dev/null +++ b/tests/SLOG/test_session_log_custom_secrets-cisco881_slog.log @@ -0,0 +1,11 @@ + +cisco1> +cisco1>terminal width 511 +cisco1>terminal length 0 +cisco1> +cisco1> +Testing password and secret replacement +This is my first secret ******** +This is my second secret ******** +This is my third secret ******** +This is my super secret ******** From 90d5bac4d16e668e5f7649b6ab6abc7ea40fac08 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 14:26:19 -0700 Subject: [PATCH 11/17] Session log testing --- ...ssion_log_custom_secrets-cisco881_slog.log | 2 +- .../test_session_log_no_log-cisco881_slog.log | 20 ++++++++ tests/test_netmiko_session_log.py | 49 ++++++++++++++++++- 3 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 tests/SLOG/test_session_log_no_log-cisco881_slog.log diff --git a/tests/SLOG/test_session_log_custom_secrets-cisco881_slog.log b/tests/SLOG/test_session_log_custom_secrets-cisco881_slog.log index 63b2a3dd5..822e301b8 100644 --- a/tests/SLOG/test_session_log_custom_secrets-cisco881_slog.log +++ b/tests/SLOG/test_session_log_custom_secrets-cisco881_slog.log @@ -1,7 +1,7 @@ cisco1> cisco1>terminal width 511 -cisco1>terminal length 0 +cisco1>******** cisco1> cisco1> Testing password and secret replacement diff --git a/tests/SLOG/test_session_log_no_log-cisco881_slog.log b/tests/SLOG/test_session_log_no_log-cisco881_slog.log new file mode 100644 index 000000000..ec8b732be --- /dev/null +++ b/tests/SLOG/test_session_log_no_log-cisco881_slog.log @@ -0,0 +1,20 @@ + +terminal width 511 + +cisco1> +cisco1>terminal width 511 +cisco1>terminal length 0 +terminal length 0 +cisco1> + +cisco1> + +cisco1>******** +******** +Translating "********" + +% Bad IP address or host name +% Unknown command or computer name, or unable to find computer address +cisco1> + +cisco1>exit diff --git a/tests/test_netmiko_session_log.py b/tests/test_netmiko_session_log.py index b37a99202..ce00eae75 100644 --- a/tests/test_netmiko_session_log.py +++ b/tests/test_netmiko_session_log.py @@ -238,18 +238,53 @@ def test_session_log_bytesio(device_slog_test_name, commands, expected_responses session_log_md5 = calc_md5(contents=log_content) assert session_log_md5 == compare_log_md5 +def test_session_log_no_log(device_slog_test_name): + device_slog = device_slog_test_name[0] + test_name = device_slog_test_name[1] + slog_file = device_slog["session_log"] + new_slog_file = add_test_name_to_file_name(slog_file, test_name) + + # record_writes as well to make sure both the write and read-echo don't get logged. + device_slog["session_log"] = new_slog_file + device_slog["session_log_record_writes"] = True + + conn = ConnectHandler(**device_slog) + + # After connection change the password to "invalid" + fake_password = "invalid" + conn.password = fake_password + + # Now try to actually send the fake password as a show command + conn.send_command(fake_password) + + with open(new_slog_file, "r") as f: + session_log = f.read() + + assert fake_password not in session_log + # One for read, one for write, one for Cisco "translating" + assert session_log.count("********") == 3 -def test_session_log_custom_secrets(device_slog): + # Do disconnect after test (to make sure send_command() actually flushes session_log buffer) + conn.disconnect() + + +def test_session_log_no_log2(device_slog_test_name): """Verify session_log does not contain custom words.""" + device_slog = device_slog_test_name[0] + test_name = device_slog_test_name[1] + slog_file = device_slog["session_log"] + new_slog_file = add_test_name_to_file_name(slog_file, test_name) + # Dict of words that should be sanitized in session log sanitize_secrets = { "secret1": "admin_username", "secret2": "snmp_auth_secret", "secret3": "snmp_priv_secret", "supersecret": "supersecret", + "data1": "terminal length 0", # Hide something that Netmiko sends } # Create custom session log - custom_log = SessionLog(file_name="SLOG/device_log.log", no_log=sanitize_secrets) + custom_log = SessionLog(file_name=new_slog_file, no_log=sanitize_secrets) # Pass in custom SessionLog obj to session_log attribute device_slog["session_log"] = custom_log @@ -268,11 +303,21 @@ def test_session_log_custom_secrets(device_slog): conn.session_log.write( "This is my super secret {}\n".format(sanitize_secrets["supersecret"]) ) + time.sleep(1) + conn.session_log.flush() # Retrieve the file name. file_name = custom_log.file_name with open(file_name, "r") as f: session_log = f.read() + + # Ensure file exists and has logging content + assert "terminal width" in session_log + + # 'terminal length 0' should be hidden in the session_log + assert "terminal length" not in session_log + assert ">********" in session_log + if sanitize_secrets.get("secret1") is not None: assert sanitize_secrets["secret1"] not in session_log if sanitize_secrets.get("secret2") is not None: From b5f6541751c1b926e177245ee315a4f4276c5ebe Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 14:53:29 -0700 Subject: [PATCH 12/17] More session_log testing --- ...ssion_log_custom_secrets-cisco881_slog.log | 11 ---- .../test_session_log_no_log-cisco881_slog.log | 5 +- ...t_session_log_no_log_cfg-cisco881_slog.log | 36 +++++++++++++ tests/test_netmiko_session_log.py | 54 +++++++++++++++++-- 4 files changed, 90 insertions(+), 16 deletions(-) delete mode 100644 tests/SLOG/test_session_log_custom_secrets-cisco881_slog.log create mode 100644 tests/SLOG/test_session_log_no_log_cfg-cisco881_slog.log diff --git a/tests/SLOG/test_session_log_custom_secrets-cisco881_slog.log b/tests/SLOG/test_session_log_custom_secrets-cisco881_slog.log deleted file mode 100644 index 822e301b8..000000000 --- a/tests/SLOG/test_session_log_custom_secrets-cisco881_slog.log +++ /dev/null @@ -1,11 +0,0 @@ - -cisco1> -cisco1>terminal width 511 -cisco1>******** -cisco1> -cisco1> -Testing password and secret replacement -This is my first secret ******** -This is my second secret ******** -This is my third secret ******** -This is my super secret ******** diff --git a/tests/SLOG/test_session_log_no_log-cisco881_slog.log b/tests/SLOG/test_session_log_no_log-cisco881_slog.log index ec8b732be..fe1a573f4 100644 --- a/tests/SLOG/test_session_log_no_log-cisco881_slog.log +++ b/tests/SLOG/test_session_log_no_log-cisco881_slog.log @@ -1,6 +1,5 @@ terminal width 511 - cisco1> cisco1>terminal width 511 cisco1>terminal length 0 @@ -13,6 +12,10 @@ cisco1>******** ******** Translating "********" +% Bad IP address or host name +% Unknown command or computer name, or unable to find computer address +cisco1>******** +******** % Bad IP address or host name % Unknown command or computer name, or unable to find computer address cisco1> diff --git a/tests/SLOG/test_session_log_no_log_cfg-cisco881_slog.log b/tests/SLOG/test_session_log_no_log_cfg-cisco881_slog.log new file mode 100644 index 000000000..1cea6d55a --- /dev/null +++ b/tests/SLOG/test_session_log_no_log_cfg-cisco881_slog.log @@ -0,0 +1,36 @@ + +terminal width 511 +cisco1> +cisco1>terminal width 511 +cisco1>terminal length 0 +terminal length 0 +cisco1> + +cisco1> + +cisco1>enable +enable +Password: 88newclass + +cisco1# + +cisco1# + +cisco1#configure terminal +configure terminal +Enter configuration commands, one per line. End with CNTL/Z. +cisco1(config)# + +cisco1(config)#******** +******** +cisco1(config)#no logging console +no logging console +cisco1(config)# + +cisco1(config)#end +end +cisco1# + +cisco1# + +cisco1#exit diff --git a/tests/test_netmiko_session_log.py b/tests/test_netmiko_session_log.py index ce00eae75..1c28f0f4b 100644 --- a/tests/test_netmiko_session_log.py +++ b/tests/test_netmiko_session_log.py @@ -238,7 +238,10 @@ def test_session_log_bytesio(device_slog_test_name, commands, expected_responses session_log_md5 = calc_md5(contents=log_content) assert session_log_md5 == compare_log_md5 + def test_session_log_no_log(device_slog_test_name): + """Test no_log works properly for show commands.""" + device_slog = device_slog_test_name[0] test_name = device_slog_test_name[1] slog_file = device_slog["session_log"] @@ -256,20 +259,63 @@ def test_session_log_no_log(device_slog_test_name): # Now try to actually send the fake password as a show command conn.send_command(fake_password) + time.sleep(1) + conn.send_command_timing(fake_password) with open(new_slog_file, "r") as f: session_log = f.read() assert fake_password not in session_log - # One for read, one for write, one for Cisco "translating" - assert session_log.count("********") == 3 + # One for read, one for write, one for Cisco "translating" (send_command) + # Plus one for read, one for write (send_command_timing) + assert session_log.count("********") == 5 # Do disconnect after test (to make sure send_command() actually flushes session_log buffer) conn.disconnect() -def test_session_log_no_log2(device_slog_test_name): - """Verify session_log does not contain custom words.""" +def test_session_log_no_log_cfg(device_slog_test_name, commands): + """Test no_log works properly for config commands.""" + device_slog = device_slog_test_name[0] + test_name = device_slog_test_name[1] + slog_file = device_slog["session_log"] + new_slog_file = add_test_name_to_file_name(slog_file, test_name) + + # Likely "logging buffered 20000" (hide/obscure this command) + config_command1 = commands["config"][0] + + # Likely "no logging console" (this command should show up in the log) + config_command2 = commands["config"][1] + + # Dict of strings that should be sanitized + no_log = { + "cfg1": config_command1, + } + # Create custom session log (use 'record_writes' just to test that situation) + custom_log = SessionLog(file_name=new_slog_file, no_log=no_log, record_writes=True) + + # Pass in custom SessionLog obj to session_log attribute + device_slog["session_log"] = custom_log + device_slog["secret"] = device_slog["password"] + + conn = ConnectHandler(**device_slog) + conn.enable() + conn.send_config_set([config_command1, config_command2]) + + # Check the results + with open(new_slog_file, "r") as f: + session_log = f.read() + + assert config_command1 not in session_log + assert session_log.count("********") == 2 + assert config_command2 in session_log + + # Make sure send_config_set flushes the session_log (so disconnect after the asserts) + conn.disconnect() + + +def test_session_log_custom_session_log(device_slog_test_name): + """Verify session_log does not contain custom words (use SessionLog obj).""" device_slog = device_slog_test_name[0] test_name = device_slog_test_name[1] slog_file = device_slog["session_log"] From 099a56a47eb3bce0e0d3c11c78b3039d3c9b47df Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 14:58:18 -0700 Subject: [PATCH 13/17] Session Log testing --- ...sion_log_custom_session_log-cisco881_slog.log | 16 ++++++++++++++++ tests/test_netmiko_session_log.py | 5 +++++ 2 files changed, 21 insertions(+) create mode 100644 tests/SLOG/test_session_log_custom_session_log-cisco881_slog.log diff --git a/tests/SLOG/test_session_log_custom_session_log-cisco881_slog.log b/tests/SLOG/test_session_log_custom_session_log-cisco881_slog.log new file mode 100644 index 000000000..46efc9a5f --- /dev/null +++ b/tests/SLOG/test_session_log_custom_session_log-cisco881_slog.log @@ -0,0 +1,16 @@ + +cisco1> +cisco1>terminal width 511 +cisco1>******** +cisco1> +cisco1> +Testing password and secret replacement +This is my first secret ******** +This is my second secret ******** +This is my third secret ******** +This is my super secret ******** + +!Testing send_command() and send_command_timing() filtering +cisco1>******** +cisco1>******** +cisco1> \ No newline at end of file diff --git a/tests/test_netmiko_session_log.py b/tests/test_netmiko_session_log.py index 1c28f0f4b..058ccf45f 100644 --- a/tests/test_netmiko_session_log.py +++ b/tests/test_netmiko_session_log.py @@ -352,6 +352,11 @@ def test_session_log_custom_session_log(device_slog_test_name): time.sleep(1) conn.session_log.flush() + # Use send_command and send_command_timing to send something that should be filtered + conn.session_log.write("\n!Testing send_command() and send_command_timing() filtering") + conn.send_command(sanitize_secrets["data1"]) + conn.send_command_timing(sanitize_secrets["data1"]) + # Retrieve the file name. file_name = custom_log.file_name with open(file_name, "r") as f: From c18a47a8319105cbb7cac5ea1ffb985cf9b745db Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 15:15:12 -0700 Subject: [PATCH 14/17] Session log testing --- .gitignore | 1 + tests/SLOG/cisco881_slog.log | 15 --------------- ...ssion_log_custom_session_log-cisco881_slog.log | 1 - .../test_session_log_secrets-cisco881_slog.log | 1 - tests/test_netmiko_session_log.py | 15 ++++++++++++--- 5 files changed, 13 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 1980c4777..5399c9ef7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ tests/etc/test_devices.yml tests/etc/commands.yml tests/etc/responses.yml tests/etc/test_devices_exc.yml +tests/SLOG/test_logging_filter_secrets-netmiko.log examples/SECRET_DEVICE_CREDS.py ./build diff --git a/tests/SLOG/cisco881_slog.log b/tests/SLOG/cisco881_slog.log index 00bb16745..e69de29bb 100644 --- a/tests/SLOG/cisco881_slog.log +++ b/tests/SLOG/cisco881_slog.log @@ -1,15 +0,0 @@ -cisco1> -cisco1>terminal width 511 -cisco1>terminal length 0 -cisco1> -cisco1> -cisco1>show ip interface brief -Interface IP-Address OK? Method Status Protocol -FastEthernet0 unassigned YES unset down down -FastEthernet1 unassigned YES unset down down -FastEthernet2 unassigned YES unset down down -FastEthernet3 unassigned YES unset down down -FastEthernet4 10.220.88.20 YES NVRAM up up -Vlan1 unassigned YES unset down down -cisco1> -cisco1>exit diff --git a/tests/SLOG/test_session_log_custom_session_log-cisco881_slog.log b/tests/SLOG/test_session_log_custom_session_log-cisco881_slog.log index 46efc9a5f..29a7e2fc1 100644 --- a/tests/SLOG/test_session_log_custom_session_log-cisco881_slog.log +++ b/tests/SLOG/test_session_log_custom_session_log-cisco881_slog.log @@ -1,4 +1,3 @@ - cisco1> cisco1>terminal width 511 cisco1>******** diff --git a/tests/SLOG/test_session_log_secrets-cisco881_slog.log b/tests/SLOG/test_session_log_secrets-cisco881_slog.log index db7940a0e..1725a6e5f 100644 --- a/tests/SLOG/test_session_log_secrets-cisco881_slog.log +++ b/tests/SLOG/test_session_log_secrets-cisco881_slog.log @@ -1,4 +1,3 @@ - cisco1> cisco1>terminal width 511 cisco1>terminal length 0 diff --git a/tests/test_netmiko_session_log.py b/tests/test_netmiko_session_log.py index 058ccf45f..8b9eb6332 100644 --- a/tests/test_netmiko_session_log.py +++ b/tests/test_netmiko_session_log.py @@ -157,13 +157,22 @@ def test_session_log_secrets(device_slog_test_name): assert "terminal width" in session_log -def test_logging_filter_secrets(net_connect_slog_wr): +def test_logging_filter_secrets(device_slog_test_name): """Verify logging DEBUG output does not contain password or secret.""" - nc = net_connect_slog_wr + device_slog = device_slog_test_name[0] + test_name = device_slog_test_name[1] + + # No session_log for this test + del device_slog["session_log"] + + # Set the secret to be correct + device_slog["secret"] = device_slog["password"] + + nc = ConnectHandler(**device_slog) # setup logger to output to file - file_name = "SLOG/netmiko.log" + file_name = f"SLOG/{test_name}-netmiko.log" netmikologger = logging.getLogger("netmiko") netmikologger.setLevel(logging.DEBUG) file_handler = logging.FileHandler(file_name) From 4c50cdbe31d4e93c6b391a2b530570698279cb6d Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 15:16:47 -0700 Subject: [PATCH 15/17] Fixing SLOG file --- tests/SLOG/cisco881_slog_wr.log | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/SLOG/cisco881_slog_wr.log b/tests/SLOG/cisco881_slog_wr.log index e69de29bb..5d735e3f0 100644 --- a/tests/SLOG/cisco881_slog_wr.log +++ b/tests/SLOG/cisco881_slog_wr.log @@ -0,0 +1,30 @@ + +terminal width 511 + +cisco1> +cisco1>terminal width 511 +cisco1>terminal length 0 +terminal length 0 +cisco1> + +cisco1> + +cisco1>show foooooooo +show foooooooo + ^ +% Invalid input detected at '^' marker. + +cisco1> + +cisco1>show ip interface brief +show ip interface brief +Interface IP-Address OK? Method Status Protocol +FastEthernet0 unassigned YES unset down down +FastEthernet1 unassigned YES unset down down +FastEthernet2 unassigned YES unset down down +FastEthernet3 unassigned YES unset down down +FastEthernet4 10.220.88.20 YES NVRAM up up +Vlan1 unassigned YES unset down down +cisco1> + +cisco1>exit From 6cfa1a6db628b12831167d29bd7e7900613286b9 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 15:18:28 -0700 Subject: [PATCH 16/17] SLOG file --- tests/SLOG/cisco881_slog.log | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/SLOG/cisco881_slog.log b/tests/SLOG/cisco881_slog.log index e69de29bb..00bb16745 100644 --- a/tests/SLOG/cisco881_slog.log +++ b/tests/SLOG/cisco881_slog.log @@ -0,0 +1,15 @@ +cisco1> +cisco1>terminal width 511 +cisco1>terminal length 0 +cisco1> +cisco1> +cisco1>show ip interface brief +Interface IP-Address OK? Method Status Protocol +FastEthernet0 unassigned YES unset down down +FastEthernet1 unassigned YES unset down down +FastEthernet2 unassigned YES unset down down +FastEthernet3 unassigned YES unset down down +FastEthernet4 10.220.88.20 YES NVRAM up up +Vlan1 unassigned YES unset down down +cisco1> +cisco1>exit From 5f8b3bc8e18ffb810363daad2704e77abc46c441 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 30 Oct 2023 15:19:43 -0700 Subject: [PATCH 17/17] Minor linting --- tests/test_netmiko_session_log.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_netmiko_session_log.py b/tests/test_netmiko_session_log.py index 8b9eb6332..39b13c6bd 100644 --- a/tests/test_netmiko_session_log.py +++ b/tests/test_netmiko_session_log.py @@ -109,7 +109,7 @@ def test_session_log_append(device_slog_test_name, commands, expected_responses) device_slog = device_slog_test_name[0] test_name = device_slog_test_name[1] - slog_file = expected_responses['session_log_append'] + slog_file = expected_responses["session_log_append"] session_file = add_test_name_to_file_name(slog_file, test_name) # Create a starting file @@ -124,7 +124,7 @@ def test_session_log_append(device_slog_test_name, commands, expected_responses) command = commands["basic"] session_action(conn, command) - compare_file_base = expected_responses['compare_log_append'] + compare_file_base = expected_responses["compare_log_append"] dir_name, f_name = compare_file_base.split("/") compare_file = f"{dir_name}/{test_name}-{f_name}" session_log_md5_append(session_file, compare_file) @@ -225,7 +225,6 @@ def test_session_log_bytesio(device_slog_test_name, commands, expected_responses """Verify session_log matches expected content, but when channel writes are also logged.""" device_slog = device_slog_test_name[0] test_name = device_slog_test_name[1] - slog_file = device_slog["session_log"] s_log = io.BytesIO() @@ -336,7 +335,7 @@ def test_session_log_custom_session_log(device_slog_test_name): "secret2": "snmp_auth_secret", "secret3": "snmp_priv_secret", "supersecret": "supersecret", - "data1": "terminal length 0", # Hide something that Netmiko sends + "data1": "terminal length 0", # Hide something that Netmiko sends } # Create custom session log custom_log = SessionLog(file_name=new_slog_file, no_log=sanitize_secrets) @@ -362,7 +361,9 @@ def test_session_log_custom_session_log(device_slog_test_name): conn.session_log.flush() # Use send_command and send_command_timing to send something that should be filtered - conn.session_log.write("\n!Testing send_command() and send_command_timing() filtering") + conn.session_log.write( + "\n!Testing send_command() and send_command_timing() filtering" + ) conn.send_command(sanitize_secrets["data1"]) conn.send_command_timing(sanitize_secrets["data1"])