From 2d209e6f9328c7605d41ac718eb3d8c16694e957 Mon Sep 17 00:00:00 2001 From: MatejKastak Date: Sat, 12 Nov 2022 21:39:28 +0100 Subject: [PATCH 01/10] WIP --- editors/vscode/package.json | 32 ++++++++ tests/unit/test_snippet_string.py | 27 +++++++ tests/unit/test_snippets.py | 102 +++++++++++++++++++++++++ yls/completer.py | 21 +++++- yls/server.py | 4 +- yls/snippet_string.py | 74 +++++++++++++++++++ yls/snippets.py | 119 ++++++++++++++++++++++++++++++ 7 files changed, 374 insertions(+), 5 deletions(-) create mode 100644 tests/unit/test_snippet_string.py create mode 100644 tests/unit/test_snippets.py create mode 100644 yls/snippet_string.py create mode 100644 yls/snippets.py diff --git a/editors/vscode/package.json b/editors/vscode/package.json index c8ec3c8..31f440f 100644 --- a/editors/vscode/package.json +++ b/editors/vscode/package.json @@ -46,6 +46,38 @@ "type": "string", "default": "", "description": "Specifies an absolute path to the directory used to store samples and module data. Used only for local YARI. (example: /home/user/samples)" + }, + "yls.snippets.metaEntries": { + "type": "object", + "default": {}, + "description": "A set of metadata entries to insert into rules. Empty values will create meta keys with a tabstop, and built-in variables are accepted (though transforms are ignored).", + "maxProperties": 50, + "additionalProperties": true + }, + "yls.snippets.sortMeta": { + "type": "boolean", + "default": true, + "description": "Sort the metadata entries in alphabetical order by keys. Otherwise, insert keys as listed." + }, + "yls.snippets.condition": { + "type": "boolean", + "default": true, + "description": "Enable the condition snippet on YARA rules. Has no effect on the presence of the condition section in the rule snippet." + }, + "yls.snippets.meta": { + "type": "boolean", + "default": true, + "description": "Enable the meta snippet on YARA rules. Has no effect on the presence of the meta section in the rule snippet." + }, + "yls.snippets.rule": { + "type": "boolean", + "default": true, + "description": "Enable the rule skeleton snippet on YARA rules. Settings for other snippets do not affect the sections provided in this snippet." + }, + "yls.snippets.strings": { + "type": "boolean", + "default": true, + "description": "Enable the strings snippet on YARA rules. Has no effect on the presence of the strings section in the rule snippet." } } }, diff --git a/tests/unit/test_snippet_string.py b/tests/unit/test_snippet_string.py new file mode 100644 index 0000000..9e4b15c --- /dev/null +++ b/tests/unit/test_snippet_string.py @@ -0,0 +1,27 @@ +# type: ignore + +from yls.snippet_string import SnippetString + + +def test_simple(): + snip = SnippetString("test") + assert str(snip) == "test" + + +def test_placeholder(): + snip = SnippetString() + snip.append_placeholder("one") + snip.append_placeholder("two") + assert str(snip) == "${1:one}${2:two}" + + +def test_complex(): + snip = SnippetString() + snip.append_text("text\n") + snip.append_placeholder("one") + snip.append_tabstop() + snip.append_placeholder("two") + snip.append_variable("CLIPBOARD", "") + snip.append_variable("INVALID_VARIABLE", "default") + snip.append_choice(("c", "ch", "choice")) + assert str(snip) == "text\n${1:one}$2${3:two}${CLIPBOARD}${default}${4|c,ch,choice|}" diff --git a/tests/unit/test_snippets.py b/tests/unit/test_snippets.py new file mode 100644 index 0000000..c660d84 --- /dev/null +++ b/tests/unit/test_snippets.py @@ -0,0 +1,102 @@ +# type: ignore + +import pytest + +from yls.snippets import SnippetGenerator + + +@pytest.mark.parametrize( + "config, expected", + ( + ( + None, + """rule ${1:my_rule} { +\tmeta: +\t\t${2:KEY} = ${3:"VALUE"} +\tstrings: +\t\t${4:\\$name} = ${5|"string",/regex/,{ HEX }|} +\tcondition: +\t\t${6:any of them} +} +""", + ), + ( + {}, + """rule ${1:my_rule} { +\tmeta: +\t\t${2:KEY} = ${3:"VALUE"} +\tstrings: +\t\t${4:\\$name} = ${5|"string",/regex/,{ HEX }|} +\tcondition: +\t\t${6:any of them} +} +""", + ), + ( + {"metaEntries": {"author": "test user", "hash": ""}}, + """rule ${1:my_rule} { +\tmeta: +\t\tauthor = "test user" +\t\thash = "$2" +\tstrings: +\t\t${3:\\$name} = ${4|"string",/regex/,{ HEX }|} +\tcondition: +\t\t${5:any of them} +} +""", + ), + ( + {"metaEntries": {"filename": "${TM_FILENAME}"}}, + """rule ${1:my_rule} { +\tmeta: +\t\tfilename = "${TM_FILENAME}" +\tstrings: +\t\t${2:\\$name} = ${3|"string",/regex/,{ HEX }|} +\tcondition: +\t\t${4:any of them} +} +""", + ), + ( + { + "metaEntries": { + "author": "", + "date": "${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}", + } + }, + """rule ${1:my_rule} { +\tmeta: +\t\tauthor = "$2" +\t\tdate = "${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}" +\tstrings: +\t\t${3:\\$name} = ${4|"string",/regex/,{ HEX }|} +\tcondition: +\t\t${5:any of them} +} +""", + ), + ( + { + "metaEntries": { + "date": "${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}", + "author": "", + }, + "sortMeta": True, + }, + """rule ${1:my_rule} { +\tmeta: +\t\tauthor = "$2" +\t\tdate = "${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}" +\tstrings: +\t\t${3:\\$name} = ${4|"string",/regex/,{ HEX }|} +\tcondition: +\t\t${5:any of them} +} +""", + ), + ), +) +def test_basic(config, expected): + generator = SnippetGenerator(config) + + assert expected == generator.generate() diff --git a/yls/completer.py b/yls/completer.py index 72019f2..a618f69 100644 --- a/yls/completer.py +++ b/yls/completer.py @@ -13,6 +13,7 @@ from yls.plugin_manager_provider import PluginManagerProvider from yls.strings import estimate_string_type from yls.strings import string_modifiers_completion_items +from yls.snippets import SnippetGenerator log = logging.getLogger(__name__) @@ -26,8 +27,9 @@ def __init__(self, ls: Any): self.ls = ls self.completion_cache = completion.CompletionCache.from_yaramod(self.ls.ymod) - def complete(self, params: lsp_types.CompletionParams) -> lsp_types.CompletionList: - return lsp_types.CompletionList(is_incomplete=False, items=self._complete(params)) + async def complete(self, params: lsp_types.CompletionParams) -> lsp_types.CompletionList: + items = await self._complete(params) + return lsp_types.CompletionList(is_incomplete=False, items=items) def signature_help(self, params: lsp_types.CompletionParams) -> lsp_types.SignatureHelp | None: signatures = self._signature_help(params) @@ -71,7 +73,7 @@ def _signature_help( return info - def _complete(self, params: lsp_types.CompletionParams) -> list[lsp_types.CompletionItem]: + async def _complete(self, params: lsp_types.CompletionParams) -> list[lsp_types.CompletionItem]: document = self.ls.workspace.get_document(params.text_document.uri) res = [] @@ -100,6 +102,10 @@ def _complete(self, params: lsp_types.CompletionParams) -> list[lsp_types.Comple log.debug("[COMPLETION] Adding last valid yara file") res += self.complete_last_valid_yara_file(document, params, word) + # Dynamic snippets + log.debug("[COMPLETION] Adding dynamic snippets") + res += await self.complete_dynamic_snippets() + # Plugin completion log.debug("COMPLETION] Adding completion items from plugings") res += utils.flatten_list( @@ -238,3 +244,12 @@ def complete_condition_keywords( res.append(item) return res + + async def complete_dynamic_snippets(self) -> list[lsp_types.CompletionItem]: + config = await utils.get_config_from_editor(self.ls, "yls.snippets") + log.debug(f"[COMPLETION] lsp configuration {config=}") + if config is None: + return [] + + generator = SnippetGenerator(config) + return generator.generate_snippets() diff --git a/yls/server.py b/yls/server.py index 6351a66..936366b 100644 --- a/yls/server.py +++ b/yls/server.py @@ -227,13 +227,13 @@ def initiliazed(ls: YaraLanguageServer, _params: Any) -> None: @SERVER.feature(COMPLETION, lsp_types.CompletionOptions(trigger_characters=["."])) -def completion( +async def completion( ls: YaraLanguageServer, params: lsp_types.CompletionParams ) -> lsp_types.CompletionList: """Code completion.""" utils.log_command(COMPLETION) - return ls.completer.complete(params) + return await ls.completer.complete(params) @SERVER.feature(SIGNATURE_HELP, lsp_types.SignatureHelpOptions(trigger_characters=["("])) diff --git a/yls/snippet_string.py b/yls/snippet_string.py new file mode 100644 index 0000000..5a8594b --- /dev/null +++ b/yls/snippet_string.py @@ -0,0 +1,74 @@ +from __future__ import annotations +from typing import Iterable + +SUPPORTED_VARIABLES = { + "TM_SELECTED_TEXT", + "TM_CURRENT_LINE", + "TM_CURRENT_WORD", + "TM_LINE_INDEX", + "TM_LINE_NUMBER", + "TM_FILENAME", + "TM_FILENAME_BASE", + "TM_DIRECTORY", + "TM_FILEPATH", + "RELATIVE_FILEPATH", + "CLIPBOARD", + "WORKSPACE_NAME", + "WORKSPACE_FOLDER", + "CURSOR_INDEX", + "CURSOR_NUMBER", + "CURRENT_YEAR", + "CURRENT_YEAR_SHORT", + "CURRENT_MONTH", + "CURRENT_MONTH_NAME", + "CURRENT_MONTH_NAME_SHORT", + "CURRENT_DATE", + "CURRENT_DAY_NAME", + "CURRENT_DAY_NAME_SHORT", + "CURRENT_HOUR", + "CURRENT_MINUTE", + "CURRENT_SECOND", + "CURRENT_SECONDS_UNIX", + "RANDOM", + "RANDOM_HEX", + "UUID", + "BLOCK_COMMENT_START", + "BLOCK_COMMENT_END", + "LINE_COMMENT", +} + + +class SnippetString: + cur_idx: int + value: str + + def __init__(self, value: str = ""): + self.value = value + self.cur_idx = 1 + + def append_choice(self, values: Iterable[str]) -> None: + self.value += f"${{{self.get_and_inc()}|{','.join(values)}|}}" + + def append_placeholder(self, value: str) -> None: + self.value += f"${{{self.get_and_inc()}:{value}}}" + + def append_tabstop(self) -> None: + self.value += f"${self.get_and_inc()}" + + def append_text(self, value: str) -> None: + """WARNING: For now you are expected to escape the string if necessary.""" + self.value += value + + def append_variable(self, name: str, default_value: str) -> None: + if name in SUPPORTED_VARIABLES: + self.value += f"${{{name}}}" + else: + self.value += f"${{{default_value}}}" + + def get_and_inc(self) -> int: + i = self.cur_idx + self.cur_idx += 1 + return i + + def __str__(self) -> str: + return self.value diff --git a/yls/snippets.py b/yls/snippets.py new file mode 100644 index 0000000..e040294 --- /dev/null +++ b/yls/snippets.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +from typing import Any + +import pygls.lsp.types as lsp_types + +from yls.snippet_string import SnippetString +from yls import utils + + +class SnippetGenerator: + config: dict[str, Any] + + def __init__(self, config: dict[str, Any] | None = None): + self.config = config or {} + + def generate_snippets(self) -> list[lsp_types.CompletionItem]: + res = [] + if self.config.get("meta", False): + item = lsp_types.CompletionItem( + label="meta", + detail="Generate a meta section (YARA)", + kind=lsp_types.CompletionItemKind.Snippet, + insert_text='meta:\n\t$1 = "$2"', + insert_text_format=lsp_types.InsertTextFormat.Snippet, + documentation=utils.markdown_content('meta:\n\tKEY = "VALUE"'), + sort_text="a", + ) + res.append(item) + + if self.config.get("condition", False): + item = lsp_types.CompletionItem( + label="condition", + detail="Generate a condition section (YARA)", + kind=lsp_types.CompletionItemKind.Snippet, + insert_text="condition:\n\t${1:conditions}", + insert_text_format=lsp_types.InsertTextFormat.Snippet, + documentation=utils.markdown_content("condition:\n\tCONDITIONS"), + sort_text="a", + ) + res.append(item) + + if self.config.get("strings", False): + item = lsp_types.CompletionItem( + label="strings", + detail="Generate a strings skeleton (YARA)", + kind=lsp_types.CompletionItemKind.Snippet, + insert_text='strings:\n\t\\$${1:name} = "${2:string}"', + insert_text_format=lsp_types.InsertTextFormat.Snippet, + documentation=utils.markdown_content('strings:\n\t$NAME = "STRING"'), + sort_text="a", + ) + res.append(item) + + if self.config.get("rule", False): + item = lsp_types.CompletionItem( + label="rule", + detail="Generate a rule skeleton (YARA)", + kind=lsp_types.CompletionItemKind.Snippet, + insert_text="rule ${1:$TM_FILENAME_BASE} {\n\t", + insert_text_format=lsp_types.InsertTextFormat.Snippet, + documentation=utils.markdown_content("rule NAME {"), + sort_text="a", + ) + res.append(item) + + return res + + def generate_rule_snippet(self, snippet: SnippetString) -> None: + snippet.append_text("rule ") + snippet.append_placeholder("my_rule") + snippet.append_text(" {\n") + self.generate_meta_snippet(snippet) + self.generate_string_snippet(snippet) + snippet.append_text("\n") + self.generate_condition_snippet(snippet) + snippet.append_text("\n}\n") + + @staticmethod + def generate_condition_snippet(snippet: SnippetString) -> None: + snippet.append_text("\tcondition:\n\t\t") + snippet.append_placeholder("any of them") + + @staticmethod + def generate_string_snippet(snippet: SnippetString) -> None: + snippet.append_text("\tstrings:\n\t\t") + snippet.append_placeholder(r"\$name") + snippet.append_text(" = ") + snippet.append_choice(('"string"', "/regex/", "{ HEX }")) + + def generate_meta_snippet(self, snippet: SnippetString) -> None: + meta_config_dict = self.config.get("metaEntries", {}) + should_sort_meta = self.config.get("sortMeta", False) + meta_config: list[tuple[str, str]] = list(meta_config_dict.items()) + if should_sort_meta: + meta_config = sorted(meta_config) + + snippet.append_text("\tmeta:\n") + + if len(meta_config) == 0: + snippet.append_text("\t\t") + snippet.append_placeholder("KEY") + snippet.append_text(" = ") + snippet.append_placeholder('"VALUE"') + snippet.append_text("\n") + else: + for key, value in meta_config: + if value == "": + snippet.append_text(f'\t\t{key} = "') + snippet.append_tabstop() + snippet.append_text('"\n') + else: + snippet.append_text(f'\t\t{key} = "{value}"\n') + + def generate(self) -> str: + snippet = SnippetString() + self.generate_rule_snippet(snippet) + + return str(snippet) From edf1bcd117614b3171c213e8b342dca6afb1c0c6 Mon Sep 17 00:00:00 2001 From: MatejKastak Date: Sat, 19 Nov 2022 23:01:03 +0100 Subject: [PATCH 02/10] new --- pyproject.toml | 3 +- pytest-yls/pytest_yls/plugin.py | 142 ++++++++++++++++++++++++++++---- yls/completer.py | 1 + 3 files changed, 128 insertions(+), 18 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c1ef6c1..f4f86da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,8 @@ black = "^22.3.0" isort = "^5.10.1" poethepoet = "^0.13.1" pytest-black = "^0.3.12" -pytest-yls = "^0.1.0" +wrapt = "^1.14.1" +pytest-yls = {path = "pytest-yls"} [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/pytest-yls/pytest_yls/plugin.py b/pytest-yls/pytest_yls/plugin.py index 9c3a20c..f440d5e 100644 --- a/pytest-yls/pytest_yls/plugin.py +++ b/pytest-yls/pytest_yls/plugin.py @@ -9,16 +9,20 @@ import pathlib import subprocess import time +import wrapt from collections import defaultdict from pathlib import Path from threading import Thread from typing import Any +from unittest.mock import patch import pytest import pytest_yls.utils as _utils import yaramod +import pygls.protocol from pygls.lsp import methods from pygls.lsp import types +from pygls.lsp import LSP_METHODS_MAP from pygls.server import LanguageServer from tenacity import retry from tenacity import stop_after_delay @@ -61,6 +65,9 @@ class Context: server: YaraLanguageServer cursor_pos: types.Position | None = None + # Config from the client (editor) + config: dict[str, Any] + NOTIFICATION_TIMEOUT_SECONDS = 2.00 CALL_TIMEOUT_SECONDS = 2.00 LANGUAGE_ID = "yara" @@ -71,23 +78,26 @@ def __init__( client: LanguageServer, server: YaraLanguageServer, tmp_path: Path, - config: Any, + pytest_config: Any, files: dict[str, str] | None = None, is_valid_yara_rules_repo: bool = False, + config: dict[str, Any] | None = None, ): + log.error("BBBBBBBBBBBBBBBBBBB") self.client = client self.server = server self.cursor_pos = None self.files = files or {} + self.config = config or {} self.is_valid_yara_rules_repo = is_valid_yara_rules_repo self.notification_timeout_seconds = ( - int(config.getoption("yls_notification_timeout")) - if config.getoption("yls_notification_timeout") + int(pytest_config.getoption("yls_notification_timeout")) + if pytest_config.getoption("yls_notification_timeout") else self.NOTIFICATION_TIMEOUT_SECONDS ) self.call_timeout_seconds = ( - int(config.getoption("yls_call_timeout")) - if config.getoption("yls_call_timeout") + int(pytest_config.getoption("yls_call_timeout")) + if pytest_config.getoption("yls_call_timeout") else self.CALL_TIMEOUT_SECONDS ) @@ -101,13 +111,15 @@ def __init__( new_files = {} yara_rules_root = pathlib.PurePath("yara-rules") subprocess.run( - ["git", "init", str(tmp_path / yara_rules_root)], capture_output=True, check=True + ["git", "init", str(tmp_path / yara_rules_root)], + capture_output=True, + check=True, ) for name, contents in self.files.items(): new_name = yara_rules_root / pathlib.PurePath(name) new_files[str(new_name)] = contents - schema_json = config.stash.get(SCHEMA_JSON_KEY, "") + schema_json = pytest_config.stash.get(SCHEMA_JSON_KEY, "") new_files[str(yara_rules_root / "schema.json")] = schema_json self.files = new_files @@ -138,6 +150,9 @@ def __init__( name = next(iter(self.files)) self.open_file(name) + # Setup the config handler + client.editor_config = self.config + def open_file(self, name: str) -> None: path = self.tmp_path / name if not path.exists(): @@ -159,7 +174,9 @@ def open_file(self, name: str) -> None: self.opened_file = path def send_request(self, feature: str, params: Any) -> Any: - return self.client.lsp.send_request(feature, params).result(self.call_timeout_seconds) + return self.client.lsp.send_request(feature, params).result( + self.call_timeout_seconds + ) def notify(self, feature: str, params: Any) -> None: self.client.lsp.notify(feature, params) @@ -190,7 +207,9 @@ def get_cursor_range(self) -> types.Range | None: raise ValueError("No cursor in current workspace is set") return types.Range( start=self.cursor_pos, - end=types.Position(line=self.cursor_pos.line, character=self.cursor_pos.character + 1), + end=types.Position( + line=self.cursor_pos.line, character=self.cursor_pos.character + 1 + ), ) def get_file_path(self, name: str) -> Path: @@ -214,7 +233,10 @@ def get_diagnostics(self) -> list[types.Diagnostic]: ) res.append( types.Diagnostic( - range=_range, message=diag.message, severity=diag.severity, source=diag.source + range=_range, + message=diag.message, + severity=diag.severity, + source=diag.source, ) ) @@ -246,7 +268,9 @@ def prep(contents: str) -> Context: yar_file = tmp_path / "file.yar" yar_file.write_text(contents) - return Context(client, server, tmp_path, pytestconfig, {yar_file: contents}, False) + return Context( + client, server, tmp_path, pytestconfig, {yar_file: contents}, False + ) return prep @@ -256,9 +280,19 @@ def yls_prepare_with_settings(client_server: Any, tmp_path: Any, pytestconfig) - client, server = client_server def prep( - files: dict[str, str] | None = None, is_valid_yara_rules_repo: bool = False + files: dict[str, str] | None = None, + is_valid_yara_rules_repo: bool = False, + config: dict[str, Any] | None = None, ) -> Context: - return Context(client, server, tmp_path, pytestconfig, files, is_valid_yara_rules_repo) + return Context( + client, + server, + tmp_path, + pytestconfig, + files, + is_valid_yara_rules_repo, + config=config, + ) return prep @@ -280,6 +314,26 @@ def hook(ls: LanguageServer, params: Any) -> None: ls.yls_notifications[feature_name].append(params) +def configuration_hook(editor_config, params) -> None: + os.system("notify-send hi configuration hook") + log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {params=}") + # items = params["items"] + items = params.items + log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {items=}") + assert len(items) >= 1, "we currently only support single requests" + log.error(editor_config) + config = editor_config + item = items[0].section + log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {item=}") + try: + for part in item.split("."): + config = config[part] + except KeyError: + config = None + + return [config] + + @pytest.fixture(scope="session") def client_server() -> Any: """A fixture to setup a client/server""" @@ -295,7 +349,7 @@ def client_server() -> Any: target=server.start_io, args=(os.fdopen(c2s_r, "rb"), os.fdopen(s2c_w, "wb")) ) - server_thread.daemon = True + # server_thread.daemon = True server_thread.start() # Add thread id to the server (just for testing) @@ -306,16 +360,70 @@ def client_server() -> Any: reset_hooks(client) + client.editor_config = {} + + log.error("Adding a workspace configuration hook") + + os.system("notify-send hi") + + @client.feature(methods.WORKSPACE_CONFIGURATION) + def _hook(ls, params): + os.system("notify-send hi hook") + log.error("PLLLLLLLLLLLLLLLLLLLLLLLZ") + configuration_hook(ls, params) + client_thread = Thread( - target=client.start_io, args=(os.fdopen(s2c_r, "rb"), os.fdopen(c2s_w, "wb")) + target=start_editor, + args=(client, os.fdopen(s2c_r, "rb"), os.fdopen(c2s_w, "wb")), ) - client_thread.daemon = True + # client_thread.daemon = True client_thread.start() yield client, server client.lsp.notify(methods.EXIT) server.lsp.notify(methods.EXIT) - server_thread.join() client_thread.join() + server_thread.join() + + +def start_editor(client, stdin, stdout): + # original_methods_map = LSP_METHODS_MAP.copy() + # new_methods_map = original_methods_map.copy() + # new_methods_map[methods.WORKSPACE_CONFIGURATION] = ( + # None, + # _params, + # types.ConfigurationParams, + # ) + + original_deserialize_params = pygls.protocol.deserialize_params + + # def _deserialize_message(data, get_params_type=get_method_params_type): + def _deserialize_params(data, get_params_type): + # raise ValueError("AAAAAAAAAAAAAAAAAAAAAAAAAAAAA") + method = data.get("method") + params = data.get("params") + if method == methods.WORKSPACE_CONFIGURATION and params is not None: + data["params"] = pygls.protocol.dict_to_object(**params) + return data + + return original_deserialize_params(data, get_params_type) + + # log.error(f"{new_methods_map=}") + log.error(f"{client=}") + log.error(f"{dir(client)=}") + log.error(f"{dir(client.lsp)=}") + + # patch("pygls.protocol.pygls.lsp.LSP_METHODS_MAP", new_methods_map).start() + # patch("pygls.protocol.LSP_METHODS_MAP", new_methods_map).start() + # patch("pygls.lsp.LSP_METHODS_MAP", new_methods_map).start() + + # patch("pygls.protocol.pygls.lsp.LSP_METHODS_MAP", new_methods_map).start() + # patch("pygls.protocol.LSP_METHODS_MAP", new_methods_map).start() + # patch("pygls.lsp.get_method_return_type", _params).start() + patch("pygls.protocol.deserialize_params", _deserialize_params).start() + # patch("pygls.protocol.get_method_return_type", _params).start() + # patch("pygls.protocol.pygls.lsp.get_method_return_type", _params).start() + + client.start_io(stdin, stdout) diff --git a/yls/completer.py b/yls/completer.py index a618f69..e59881f 100644 --- a/yls/completer.py +++ b/yls/completer.py @@ -105,6 +105,7 @@ async def _complete(self, params: lsp_types.CompletionParams) -> list[lsp_types. # Dynamic snippets log.debug("[COMPLETION] Adding dynamic snippets") res += await self.complete_dynamic_snippets() + log.debug("[COMPLETION] Adding dynamic snippets done") # Plugin completion log.debug("COMPLETION] Adding completion items from plugings") From f19f4ebc9ef67349b4edd1737e87127c12d4f313 Mon Sep 17 00:00:00 2001 From: MatejKastak Date: Sat, 19 Nov 2022 23:26:56 +0100 Subject: [PATCH 03/10] refactor --- pytest-yls/pytest_yls/plugin.py | 25 ++++++++++++++++++++++--- yls/utils.py | 1 + 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/pytest-yls/pytest_yls/plugin.py b/pytest-yls/pytest_yls/plugin.py index f440d5e..dde27c2 100644 --- a/pytest-yls/pytest_yls/plugin.py +++ b/pytest-yls/pytest_yls/plugin.py @@ -133,6 +133,19 @@ def __init__( yar_file.write_text(new_text) self.cursor_pos = cursor_pos or self.cursor_pos + original_deserialize_params = pygls.protocol.deserialize_params + + def _deserialize_params(data, get_params_type): + method = data.get("method") + params = data.get("params") + if method == methods.WORKSPACE_CONFIGURATION and params is not None: + data["params"] = pygls.protocol.dict_to_object(**params) + return data + + return original_deserialize_params(data, get_params_type) + + patch("pygls.protocol.deserialize_params", _deserialize_params).start() + # Reset the captured notifications self.client.yls_notifications = defaultdict(list) @@ -151,6 +164,7 @@ def __init__( self.open_file(name) # Setup the config handler + log.error("CCCCCCC") client.editor_config = self.config def open_file(self, name: str) -> None: @@ -314,10 +328,11 @@ def hook(ls: LanguageServer, params: Any) -> None: ls.yls_notifications[feature_name].append(params) -def configuration_hook(editor_config, params) -> None: +def configuration_hook(ls, params) -> None: os.system("notify-send hi configuration hook") log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {params=}") - # items = params["items"] + editor_config = ls.editor_config + log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {editor_config=}") items = params.items log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {items=}") assert len(items) >= 1, "we currently only support single requests" @@ -330,6 +345,10 @@ def configuration_hook(editor_config, params) -> None: config = config[part] except KeyError: config = None + log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {config=}") + + if config is None: + return [] return [config] @@ -368,7 +387,7 @@ def client_server() -> Any: @client.feature(methods.WORKSPACE_CONFIGURATION) def _hook(ls, params): - os.system("notify-send hi hook") + os.system("notify-send configuration") log.error("PLLLLLLLLLLLLLLLLLLLLLLLZ") configuration_hook(ls, params) diff --git a/yls/utils.py b/yls/utils.py index ede9291..76040ea 100644 --- a/yls/utils.py +++ b/yls/utils.py @@ -82,6 +82,7 @@ def log_command(command: str) -> None: def logging_prolog(plugin_manager: PluginManager) -> None: """Emit dependency version information.""" log.info("Starting yls language server...") + log.info(f"Current file: {__file__}") log.info(f"System platform: {platform.system()}") log.info(f"YLS version: {__version__}") log.info(f"Yaramod version: {yaramod.YARAMOD_VERSION}") From 9ca8eef55adde7c366770e24186944529e1ffa70 Mon Sep 17 00:00:00 2001 From: MatejKastak Date: Sun, 20 Nov 2022 13:27:22 +0100 Subject: [PATCH 04/10] new lock --- poetry.lock | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index b2427fd..95b4cfb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -480,16 +480,21 @@ toml = ">=0.7.1" [[package]] name = "pytest-yls" -version = "0.1.0" +version = "1.2.1" description = "Pytest plugin to test the YLS as a whole." category = "dev" optional = false -python-versions = ">=3.8,<4.0" +python-versions = "^3.8" +develop = false [package.dependencies] -pytest = ">=7.1.2,<8.0.0" -tenacity = ">=8.0.1,<9.0.0" -yls = ">=1,<2" +pytest = "^7.1.2" +tenacity = "^8.0.1" +yls = "^1" + +[package.source] +type = "directory" +url = "pytest-yls" [[package]] name = "pyyaml" @@ -600,7 +605,7 @@ python-versions = ">=3.7" [metadata] lock-version = "1.1" python-versions = ">=3.8,<3.12" -content-hash = "70ec90af03dc9b94202850cd6970b1bdfaf80b33f6dd3ad017a85e5004d9581a" +content-hash = "ba3047d70fedfe29670c153c447a87c8d124b973d0f55608369fe7bc086f52de" [metadata.files] astroid = [] From 26788cd5db9559c0f5122ef5d04947e94fbdf61f Mon Sep 17 00:00:00 2001 From: MatejKastak Date: Sun, 20 Nov 2022 13:48:55 +0100 Subject: [PATCH 05/10] test --- poetry.lock | 2 +- pytest-yls/pytest_yls/plugin.py | 5 +---- yls/completer.py | 6 ++++++ yls/utils.py | 1 + 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 95b4cfb..da116d0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -525,7 +525,7 @@ pbr = ">=2.0.0,<2.1.0 || >2.1.0" [[package]] name = "tenacity" -version = "8.0.1" +version = "8.1.0" description = "Retry code until it succeeds" category = "dev" optional = false diff --git a/pytest-yls/pytest_yls/plugin.py b/pytest-yls/pytest_yls/plugin.py index dde27c2..558ec99 100644 --- a/pytest-yls/pytest_yls/plugin.py +++ b/pytest-yls/pytest_yls/plugin.py @@ -328,7 +328,7 @@ def hook(ls: LanguageServer, params: Any) -> None: ls.yls_notifications[feature_name].append(params) -def configuration_hook(ls, params) -> None: +def configuration_hook(ls, params): os.system("notify-send hi configuration hook") log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {params=}") editor_config = ls.editor_config @@ -347,9 +347,6 @@ def configuration_hook(ls, params) -> None: config = None log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {config=}") - if config is None: - return [] - return [config] diff --git a/yls/completer.py b/yls/completer.py index e59881f..9f454fe 100644 --- a/yls/completer.py +++ b/yls/completer.py @@ -247,6 +247,12 @@ def complete_condition_keywords( return res async def complete_dynamic_snippets(self) -> list[lsp_types.CompletionItem]: + config = await utils.get_config_from_editor(self.ls, "test") + log.debug(f"[COMPLETION] lsp configuration {config=}") + + config = await utils.get_config_from_editor(self.ls, "this.cannot.exist") + log.debug(f"[COMPLETION] lsp configuration {config=}") + config = await utils.get_config_from_editor(self.ls, "yls.snippets") log.debug(f"[COMPLETION] lsp configuration {config=}") if config is None: diff --git a/yls/utils.py b/yls/utils.py index 76040ea..242c6ce 100644 --- a/yls/utils.py +++ b/yls/utils.py @@ -772,6 +772,7 @@ async def get_config_from_editor(ls: Any, section: str) -> Any: items=[lsp_types.ConfigurationItem(scope_uri="", section=section)] ) ) + log.error(f"{config=}") # We should get list of configuration items here with exactly one element assert isinstance(config, list) From c1c2eee03ec398c35b5873b9d58297846ef561e6 Mon Sep 17 00:00:00 2001 From: MatejKastak Date: Sun, 20 Nov 2022 21:08:07 +0100 Subject: [PATCH 06/10] WORKS!!!! --- pytest-yls/pytest_yls/plugin.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pytest-yls/pytest_yls/plugin.py b/pytest-yls/pytest_yls/plugin.py index 558ec99..4a5d9b6 100644 --- a/pytest-yls/pytest_yls/plugin.py +++ b/pytest-yls/pytest_yls/plugin.py @@ -9,11 +9,10 @@ import pathlib import subprocess import time -import wrapt from collections import defaultdict from pathlib import Path from threading import Thread -from typing import Any +from typing import Any, List from unittest.mock import patch import pytest @@ -386,7 +385,7 @@ def client_server() -> Any: def _hook(ls, params): os.system("notify-send configuration") log.error("PLLLLLLLLLLLLLLLLLLLLLLLZ") - configuration_hook(ls, params) + return configuration_hook(ls, params) client_thread = Thread( target=start_editor, @@ -431,6 +430,18 @@ def _deserialize_params(data, get_params_type): log.error(f"{dir(client)=}") log.error(f"{dir(client.lsp)=}") + original_get_method_return_type = pygls.lsp.get_method_return_type + + # pylint: disable=dangerous-default-value + def _get_method_return_type(method_name, lsp_methods_map=LSP_METHODS_MAP): + if method_name == methods.WORKSPACE_CONFIGURATION: + log.warning( + "[TESTS] We are altering the return value for get_method_return_type" + ) + return List[Any] + + return original_get_method_return_type(method_name, lsp_methods_map) + # patch("pygls.protocol.pygls.lsp.LSP_METHODS_MAP", new_methods_map).start() # patch("pygls.protocol.LSP_METHODS_MAP", new_methods_map).start() # patch("pygls.lsp.LSP_METHODS_MAP", new_methods_map).start() @@ -439,6 +450,8 @@ def _deserialize_params(data, get_params_type): # patch("pygls.protocol.LSP_METHODS_MAP", new_methods_map).start() # patch("pygls.lsp.get_method_return_type", _params).start() patch("pygls.protocol.deserialize_params", _deserialize_params).start() + patch("pygls.protocol.get_method_return_type", _get_method_return_type).start() + # patch("pygls.protocol.get_method_params_type", _raise).start() # patch("pygls.protocol.get_method_return_type", _params).start() # patch("pygls.protocol.pygls.lsp.get_method_return_type", _params).start() From ab923664030d4c46bd21249e54156ca73c7b55df Mon Sep 17 00:00:00 2001 From: MatejKastak Date: Sun, 20 Nov 2022 21:21:06 +0100 Subject: [PATCH 07/10] cleanup --- poetry.lock | 2 +- pyproject.toml | 1 - pytest-yls/pytest_yls/plugin.py | 70 ++++++--------------------------- 3 files changed, 14 insertions(+), 59 deletions(-) diff --git a/poetry.lock b/poetry.lock index da116d0..e9a5efc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -605,7 +605,7 @@ python-versions = ">=3.7" [metadata] lock-version = "1.1" python-versions = ">=3.8,<3.12" -content-hash = "ba3047d70fedfe29670c153c447a87c8d124b973d0f55608369fe7bc086f52de" +content-hash = "594dc625822e0fc7037dbc03d83b83b79d321b6876f2b64adebe2858187be113" [metadata.files] astroid = [] diff --git a/pyproject.toml b/pyproject.toml index f4f86da..95dce3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,6 @@ black = "^22.3.0" isort = "^5.10.1" poethepoet = "^0.13.1" pytest-black = "^0.3.12" -wrapt = "^1.14.1" pytest-yls = {path = "pytest-yls"} [build-system] diff --git a/pytest-yls/pytest_yls/plugin.py b/pytest-yls/pytest_yls/plugin.py index 4a5d9b6..289722f 100644 --- a/pytest-yls/pytest_yls/plugin.py +++ b/pytest-yls/pytest_yls/plugin.py @@ -82,7 +82,6 @@ def __init__( is_valid_yara_rules_repo: bool = False, config: dict[str, Any] | None = None, ): - log.error("BBBBBBBBBBBBBBBBBBB") self.client = client self.server = server self.cursor_pos = None @@ -132,19 +131,6 @@ def __init__( yar_file.write_text(new_text) self.cursor_pos = cursor_pos or self.cursor_pos - original_deserialize_params = pygls.protocol.deserialize_params - - def _deserialize_params(data, get_params_type): - method = data.get("method") - params = data.get("params") - if method == methods.WORKSPACE_CONFIGURATION and params is not None: - data["params"] = pygls.protocol.dict_to_object(**params) - return data - - return original_deserialize_params(data, get_params_type) - - patch("pygls.protocol.deserialize_params", _deserialize_params).start() - # Reset the captured notifications self.client.yls_notifications = defaultdict(list) @@ -162,9 +148,8 @@ def _deserialize_params(data, get_params_type): name = next(iter(self.files)) self.open_file(name) - # Setup the config handler - log.error("CCCCCCC") - client.editor_config = self.config + # Setup the editor configuration + self.client.editor_config = self.config def open_file(self, name: str) -> None: path = self.tmp_path / name @@ -328,23 +313,17 @@ def hook(ls: LanguageServer, params: Any) -> None: def configuration_hook(ls, params): - os.system("notify-send hi configuration hook") - log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {params=}") + """WORKSPACE_CONFIGURATION hook""" editor_config = ls.editor_config - log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {editor_config=}") items = params.items - log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {items=}") assert len(items) >= 1, "we currently only support single requests" - log.error(editor_config) config = editor_config item = items[0].section - log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {item=}") try: for part in item.split("."): config = config[part] except KeyError: config = None - log.error(f"AAAAAAAAAAAAAAAAAAAAAAAAAAAA {config=}") return [config] @@ -377,15 +356,8 @@ def client_server() -> Any: client.editor_config = {} - log.error("Adding a workspace configuration hook") - - os.system("notify-send hi") - - @client.feature(methods.WORKSPACE_CONFIGURATION) - def _hook(ls, params): - os.system("notify-send configuration") - log.error("PLLLLLLLLLLLLLLLLLLLLLLLZ") - return configuration_hook(ls, params) + # Hook configuration requests + client.feature(methods.WORKSPACE_CONFIGURATION)(configuration_hook) client_thread = Thread( target=start_editor, @@ -404,32 +376,26 @@ def _hook(ls, params): def start_editor(client, stdin, stdout): - # original_methods_map = LSP_METHODS_MAP.copy() - # new_methods_map = original_methods_map.copy() - # new_methods_map[methods.WORKSPACE_CONFIGURATION] = ( - # None, - # _params, - # types.ConfigurationParams, - # ) + """Hook client editor) methods for configuration. + We need to do this kind of setup because it is really hard to change LSP_METHODS_MAP. + If you want to change the configuration just call Context.set_configuration(). + """ + log.info("[TESTS] Setting up the client (editor) hooks...") original_deserialize_params = pygls.protocol.deserialize_params - # def _deserialize_message(data, get_params_type=get_method_params_type): def _deserialize_params(data, get_params_type): - # raise ValueError("AAAAAAAAAAAAAAAAAAAAAAAAAAAAA") method = data.get("method") params = data.get("params") if method == methods.WORKSPACE_CONFIGURATION and params is not None: + log.warning( + "[TESTS] We are altering the return value for deserialize_params" + ) data["params"] = pygls.protocol.dict_to_object(**params) return data return original_deserialize_params(data, get_params_type) - # log.error(f"{new_methods_map=}") - log.error(f"{client=}") - log.error(f"{dir(client)=}") - log.error(f"{dir(client.lsp)=}") - original_get_method_return_type = pygls.lsp.get_method_return_type # pylint: disable=dangerous-default-value @@ -442,17 +408,7 @@ def _get_method_return_type(method_name, lsp_methods_map=LSP_METHODS_MAP): return original_get_method_return_type(method_name, lsp_methods_map) - # patch("pygls.protocol.pygls.lsp.LSP_METHODS_MAP", new_methods_map).start() - # patch("pygls.protocol.LSP_METHODS_MAP", new_methods_map).start() - # patch("pygls.lsp.LSP_METHODS_MAP", new_methods_map).start() - - # patch("pygls.protocol.pygls.lsp.LSP_METHODS_MAP", new_methods_map).start() - # patch("pygls.protocol.LSP_METHODS_MAP", new_methods_map).start() - # patch("pygls.lsp.get_method_return_type", _params).start() patch("pygls.protocol.deserialize_params", _deserialize_params).start() patch("pygls.protocol.get_method_return_type", _get_method_return_type).start() - # patch("pygls.protocol.get_method_params_type", _raise).start() - # patch("pygls.protocol.get_method_return_type", _params).start() - # patch("pygls.protocol.pygls.lsp.get_method_return_type", _params).start() client.start_io(stdin, stdout) From 6c276e0cc0cf7a15260a7d4099e031f667a58934 Mon Sep 17 00:00:00 2001 From: MatejKastak Date: Sun, 20 Nov 2022 21:22:12 +0100 Subject: [PATCH 08/10] formatting --- pytest-yls/pytest_yls/plugin.py | 47 +++++++++------------------------ yls/completer.py | 2 +- yls/server.py | 2 +- yls/snippet_string.py | 1 + yls/snippets.py | 2 +- 5 files changed, 17 insertions(+), 37 deletions(-) diff --git a/pytest-yls/pytest_yls/plugin.py b/pytest-yls/pytest_yls/plugin.py index 289722f..47c2dd1 100644 --- a/pytest-yls/pytest_yls/plugin.py +++ b/pytest-yls/pytest_yls/plugin.py @@ -12,16 +12,17 @@ from collections import defaultdict from pathlib import Path from threading import Thread -from typing import Any, List +from typing import Any +from typing import List from unittest.mock import patch +import pygls.protocol import pytest import pytest_yls.utils as _utils import yaramod -import pygls.protocol +from pygls.lsp import LSP_METHODS_MAP from pygls.lsp import methods from pygls.lsp import types -from pygls.lsp import LSP_METHODS_MAP from pygls.server import LanguageServer from tenacity import retry from tenacity import stop_after_delay @@ -109,9 +110,7 @@ def __init__( new_files = {} yara_rules_root = pathlib.PurePath("yara-rules") subprocess.run( - ["git", "init", str(tmp_path / yara_rules_root)], - capture_output=True, - check=True, + ["git", "init", str(tmp_path / yara_rules_root)], capture_output=True, check=True ) for name, contents in self.files.items(): new_name = yara_rules_root / pathlib.PurePath(name) @@ -172,9 +171,7 @@ def open_file(self, name: str) -> None: self.opened_file = path def send_request(self, feature: str, params: Any) -> Any: - return self.client.lsp.send_request(feature, params).result( - self.call_timeout_seconds - ) + return self.client.lsp.send_request(feature, params).result(self.call_timeout_seconds) def notify(self, feature: str, params: Any) -> None: self.client.lsp.notify(feature, params) @@ -205,9 +202,7 @@ def get_cursor_range(self) -> types.Range | None: raise ValueError("No cursor in current workspace is set") return types.Range( start=self.cursor_pos, - end=types.Position( - line=self.cursor_pos.line, character=self.cursor_pos.character + 1 - ), + end=types.Position(line=self.cursor_pos.line, character=self.cursor_pos.character + 1), ) def get_file_path(self, name: str) -> Path: @@ -231,10 +226,7 @@ def get_diagnostics(self) -> list[types.Diagnostic]: ) res.append( types.Diagnostic( - range=_range, - message=diag.message, - severity=diag.severity, - source=diag.source, + range=_range, message=diag.message, severity=diag.severity, source=diag.source ) ) @@ -266,9 +258,7 @@ def prep(contents: str) -> Context: yar_file = tmp_path / "file.yar" yar_file.write_text(contents) - return Context( - client, server, tmp_path, pytestconfig, {yar_file: contents}, False - ) + return Context(client, server, tmp_path, pytestconfig, {yar_file: contents}, False) return prep @@ -283,13 +273,7 @@ def prep( config: dict[str, Any] | None = None, ) -> Context: return Context( - client, - server, - tmp_path, - pytestconfig, - files, - is_valid_yara_rules_repo, - config=config, + client, server, tmp_path, pytestconfig, files, is_valid_yara_rules_repo, config=config ) return prep @@ -360,8 +344,7 @@ def client_server() -> Any: client.feature(methods.WORKSPACE_CONFIGURATION)(configuration_hook) client_thread = Thread( - target=start_editor, - args=(client, os.fdopen(s2c_r, "rb"), os.fdopen(c2s_w, "wb")), + target=start_editor, args=(client, os.fdopen(s2c_r, "rb"), os.fdopen(c2s_w, "wb")) ) # client_thread.daemon = True @@ -388,9 +371,7 @@ def _deserialize_params(data, get_params_type): method = data.get("method") params = data.get("params") if method == methods.WORKSPACE_CONFIGURATION and params is not None: - log.warning( - "[TESTS] We are altering the return value for deserialize_params" - ) + log.warning("[TESTS] We are altering the return value for deserialize_params") data["params"] = pygls.protocol.dict_to_object(**params) return data @@ -401,9 +382,7 @@ def _deserialize_params(data, get_params_type): # pylint: disable=dangerous-default-value def _get_method_return_type(method_name, lsp_methods_map=LSP_METHODS_MAP): if method_name == methods.WORKSPACE_CONFIGURATION: - log.warning( - "[TESTS] We are altering the return value for get_method_return_type" - ) + log.warning("[TESTS] We are altering the return value for get_method_return_type") return List[Any] return original_get_method_return_type(method_name, lsp_methods_map) diff --git a/yls/completer.py b/yls/completer.py index 9f454fe..25db7ce 100644 --- a/yls/completer.py +++ b/yls/completer.py @@ -11,9 +11,9 @@ from yls import utils from yls.completion import CONDITION_KEYWORDS from yls.plugin_manager_provider import PluginManagerProvider +from yls.snippets import SnippetGenerator from yls.strings import estimate_string_type from yls.strings import string_modifiers_completion_items -from yls.snippets import SnippetGenerator log = logging.getLogger(__name__) diff --git a/yls/server.py b/yls/server.py index 936366b..d0abf51 100644 --- a/yls/server.py +++ b/yls/server.py @@ -30,11 +30,11 @@ from yls import icons from yls import linting from yls import utils -from yls.version import __version__ from yls.completer import Completer from yls.hookspecs import ErrorMessage from yls.hover import Hoverer from yls.plugin_manager_provider import PluginManagerProvider +from yls.version import __version__ from yls.yaramod_provider import YaramodProvider log = logging.getLogger(__name__) diff --git a/yls/snippet_string.py b/yls/snippet_string.py index 5a8594b..62ba9cb 100644 --- a/yls/snippet_string.py +++ b/yls/snippet_string.py @@ -1,4 +1,5 @@ from __future__ import annotations + from typing import Iterable SUPPORTED_VARIABLES = { diff --git a/yls/snippets.py b/yls/snippets.py index e040294..6d89052 100644 --- a/yls/snippets.py +++ b/yls/snippets.py @@ -4,8 +4,8 @@ import pygls.lsp.types as lsp_types -from yls.snippet_string import SnippetString from yls import utils +from yls.snippet_string import SnippetString class SnippetGenerator: From 7af725651c0f73a399b21e2927cbdc1321a021b0 Mon Sep 17 00:00:00 2001 From: MatejKastak Date: Sun, 20 Nov 2022 21:31:16 +0100 Subject: [PATCH 09/10] clean --- pytest-yls/pytest_yls/plugin.py | 4 ++-- yls/utils.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pytest-yls/pytest_yls/plugin.py b/pytest-yls/pytest_yls/plugin.py index 47c2dd1..8300f5b 100644 --- a/pytest-yls/pytest_yls/plugin.py +++ b/pytest-yls/pytest_yls/plugin.py @@ -327,7 +327,7 @@ def client_server() -> Any: target=server.start_io, args=(os.fdopen(c2s_r, "rb"), os.fdopen(s2c_w, "wb")) ) - # server_thread.daemon = True + server_thread.daemon = True server_thread.start() # Add thread id to the server (just for testing) @@ -347,7 +347,7 @@ def client_server() -> Any: target=start_editor, args=(client, os.fdopen(s2c_r, "rb"), os.fdopen(c2s_w, "wb")) ) - # client_thread.daemon = True + client_thread.daemon = True client_thread.start() yield client, server diff --git a/yls/utils.py b/yls/utils.py index 242c6ce..76040ea 100644 --- a/yls/utils.py +++ b/yls/utils.py @@ -772,7 +772,6 @@ async def get_config_from_editor(ls: Any, section: str) -> Any: items=[lsp_types.ConfigurationItem(scope_uri="", section=section)] ) ) - log.error(f"{config=}") # We should get list of configuration items here with exactly one element assert isinstance(config, list) From b8837b5c31f98a0fa350021670a693cb84c7489e Mon Sep 17 00:00:00 2001 From: MatejKastak Date: Tue, 22 Nov 2022 12:18:35 +0100 Subject: [PATCH 10/10] refactor --- pyproject.toml | 2 + tests/unit/test_snippets.py | 19 +-- yls/completer.py | 16 +-- yls/snippets.py | 223 ++++++++++++++++++------------------ yls/utils.py | 4 +- 5 files changed, 125 insertions(+), 139 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 95dce3d..c3c3d1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ asyncio_mode = "auto" disallow_untyped_calls = false disallow_untyped_defs = true disallow_untyped_decorators = false +disallow_any_generics = false ignore_missing_imports = true strict = true warn_unused_ignores = true @@ -83,6 +84,7 @@ warn_unused_ignores = true good-names = "ls,logger,x,y,c,e,i,j,n,m,f" ignored-classes = "_HookRelay" extension-pkg-allow-list = "yaramod" +ignore-patterns = "^tests/.*" [tool.pylint.messages_control] disable = """, diff --git a/tests/unit/test_snippets.py b/tests/unit/test_snippets.py index c660d84..358e81b 100644 --- a/tests/unit/test_snippets.py +++ b/tests/unit/test_snippets.py @@ -2,24 +2,13 @@ import pytest -from yls.snippets import SnippetGenerator +from yls import snippets +from yls.snippet_string import SnippetString @pytest.mark.parametrize( "config, expected", ( - ( - None, - """rule ${1:my_rule} { -\tmeta: -\t\t${2:KEY} = ${3:"VALUE"} -\tstrings: -\t\t${4:\\$name} = ${5|"string",/regex/,{ HEX }|} -\tcondition: -\t\t${6:any of them} -} -""", - ), ( {}, """rule ${1:my_rule} { @@ -97,6 +86,4 @@ ), ) def test_basic(config, expected): - generator = SnippetGenerator(config) - - assert expected == generator.generate() + assert expected == str(snippets._generate_rule_snippet(SnippetString(), config)) diff --git a/yls/completer.py b/yls/completer.py index 25db7ce..bb2f1d3 100644 --- a/yls/completer.py +++ b/yls/completer.py @@ -7,11 +7,11 @@ import pygls.lsp.types as lsp_types from pygls.workspace import Document +from yls import snippets from yls import completion from yls import utils from yls.completion import CONDITION_KEYWORDS from yls.plugin_manager_provider import PluginManagerProvider -from yls.snippets import SnippetGenerator from yls.strings import estimate_string_type from yls.strings import string_modifiers_completion_items @@ -247,16 +247,10 @@ def complete_condition_keywords( return res async def complete_dynamic_snippets(self) -> list[lsp_types.CompletionItem]: - config = await utils.get_config_from_editor(self.ls, "test") - log.debug(f"[COMPLETION] lsp configuration {config=}") - - config = await utils.get_config_from_editor(self.ls, "this.cannot.exist") - log.debug(f"[COMPLETION] lsp configuration {config=}") - + # Get the configuration each time, in case it changed during the runtime config = await utils.get_config_from_editor(self.ls, "yls.snippets") - log.debug(f"[COMPLETION] lsp configuration {config=}") - if config is None: + log.debug(f"[COMPLETION] LSP configuration {config=}") + if config is None or not isinstance(config, dict): return [] - generator = SnippetGenerator(config) - return generator.generate_snippets() + return snippets.generate_snippets_from_configuration(config) diff --git a/yls/snippets.py b/yls/snippets.py index 6d89052..9e37082 100644 --- a/yls/snippets.py +++ b/yls/snippets.py @@ -1,119 +1,122 @@ from __future__ import annotations -from typing import Any - import pygls.lsp.types as lsp_types from yls import utils from yls.snippet_string import SnippetString -class SnippetGenerator: - config: dict[str, Any] - - def __init__(self, config: dict[str, Any] | None = None): - self.config = config or {} - - def generate_snippets(self) -> list[lsp_types.CompletionItem]: - res = [] - if self.config.get("meta", False): - item = lsp_types.CompletionItem( - label="meta", - detail="Generate a meta section (YARA)", - kind=lsp_types.CompletionItemKind.Snippet, - insert_text='meta:\n\t$1 = "$2"', - insert_text_format=lsp_types.InsertTextFormat.Snippet, - documentation=utils.markdown_content('meta:\n\tKEY = "VALUE"'), - sort_text="a", - ) - res.append(item) - - if self.config.get("condition", False): - item = lsp_types.CompletionItem( - label="condition", - detail="Generate a condition section (YARA)", - kind=lsp_types.CompletionItemKind.Snippet, - insert_text="condition:\n\t${1:conditions}", - insert_text_format=lsp_types.InsertTextFormat.Snippet, - documentation=utils.markdown_content("condition:\n\tCONDITIONS"), - sort_text="a", - ) - res.append(item) - - if self.config.get("strings", False): - item = lsp_types.CompletionItem( - label="strings", - detail="Generate a strings skeleton (YARA)", - kind=lsp_types.CompletionItemKind.Snippet, - insert_text='strings:\n\t\\$${1:name} = "${2:string}"', - insert_text_format=lsp_types.InsertTextFormat.Snippet, - documentation=utils.markdown_content('strings:\n\t$NAME = "STRING"'), - sort_text="a", - ) - res.append(item) - - if self.config.get("rule", False): - item = lsp_types.CompletionItem( - label="rule", - detail="Generate a rule skeleton (YARA)", - kind=lsp_types.CompletionItemKind.Snippet, - insert_text="rule ${1:$TM_FILENAME_BASE} {\n\t", - insert_text_format=lsp_types.InsertTextFormat.Snippet, - documentation=utils.markdown_content("rule NAME {"), - sort_text="a", - ) - res.append(item) - - return res - - def generate_rule_snippet(self, snippet: SnippetString) -> None: - snippet.append_text("rule ") - snippet.append_placeholder("my_rule") - snippet.append_text(" {\n") - self.generate_meta_snippet(snippet) - self.generate_string_snippet(snippet) - snippet.append_text("\n") - self.generate_condition_snippet(snippet) - snippet.append_text("\n}\n") - - @staticmethod - def generate_condition_snippet(snippet: SnippetString) -> None: - snippet.append_text("\tcondition:\n\t\t") - snippet.append_placeholder("any of them") - - @staticmethod - def generate_string_snippet(snippet: SnippetString) -> None: - snippet.append_text("\tstrings:\n\t\t") - snippet.append_placeholder(r"\$name") +def generate_snippets_from_configuration( + config: dict | None = None, +) -> list[lsp_types.CompletionItem]: + """Generate dynamic snippets from the user configuration. + + User can specify the configuration in the editor and this function will generate a custom snippet for it. + """ + + # Default is empty config + config = config or {} + + res = [] + if config.get("meta", False): + item = lsp_types.CompletionItem( + label="meta", + detail="Generate a meta section (YARA)", + kind=lsp_types.CompletionItemKind.Snippet, + insert_text=str(_generate_meta_snippet(SnippetString(), config)), + insert_text_format=lsp_types.InsertTextFormat.Snippet, + documentation=utils.markdown_content('meta:\n\tKEY = "VALUE"'), + sort_text="a", + ) + res.append(item) + + if config.get("condition", False): + item = lsp_types.CompletionItem( + label="condition", + detail="Generate a condition section (YARA)", + kind=lsp_types.CompletionItemKind.Snippet, + insert_text=str(_generate_condition_snippet(SnippetString())), + insert_text_format=lsp_types.InsertTextFormat.Snippet, + documentation=utils.markdown_content("condition:\n\tCONDITIONS"), + sort_text="a", + ) + res.append(item) + + if config.get("strings", False): + item = lsp_types.CompletionItem( + label="strings", + detail="Generate a strings skeleton (YARA)", + kind=lsp_types.CompletionItemKind.Snippet, + insert_text=str(_generate_string_snippet(SnippetString())), + insert_text_format=lsp_types.InsertTextFormat.Snippet, + documentation=utils.markdown_content('strings:\n\t$NAME = "STRING"'), + sort_text="a", + ) + res.append(item) + + if config.get("rule", False): + item = lsp_types.CompletionItem( + label="rule", + detail="Generate a rule skeleton (YARA)", + kind=lsp_types.CompletionItemKind.Snippet, + insert_text=str(_generate_rule_snippet(SnippetString(), config)), + insert_text_format=lsp_types.InsertTextFormat.Snippet, + documentation=utils.markdown_content("rule NAME {"), + sort_text="a", + ) + res.append(item) + + return res + + +def _generate_rule_snippet(snippet: SnippetString, config: dict) -> SnippetString: + snippet.append_text("rule ") + snippet.append_placeholder("my_rule") + snippet.append_text(" {\n") + snippet = _generate_meta_snippet(snippet, config) + snippet = _generate_string_snippet(snippet) + snippet.append_text("\n") + snippet = _generate_condition_snippet(snippet) + snippet.append_text("\n}\n") + return snippet + + +def _generate_condition_snippet(snippet: SnippetString) -> SnippetString: + snippet.append_text("\tcondition:\n\t\t") + snippet.append_placeholder("any of them") + return snippet + + +def _generate_string_snippet(snippet: SnippetString) -> SnippetString: + snippet.append_text("\tstrings:\n\t\t") + snippet.append_placeholder(r"\$name") + snippet.append_text(" = ") + snippet.append_choice(('"string"', "/regex/", "{ HEX }")) + return snippet + + +def _generate_meta_snippet(snippet: SnippetString, config: dict) -> SnippetString: + meta_config_dict = config.get("metaEntries", {}) + should_sort_meta = config.get("sortMeta", False) + meta_config: list[tuple[str, str]] = list(meta_config_dict.items()) + if should_sort_meta: + meta_config = sorted(meta_config) + + snippet.append_text("\tmeta:\n") + + if len(meta_config) == 0: + snippet.append_text("\t\t") + snippet.append_placeholder("KEY") snippet.append_text(" = ") - snippet.append_choice(('"string"', "/regex/", "{ HEX }")) - - def generate_meta_snippet(self, snippet: SnippetString) -> None: - meta_config_dict = self.config.get("metaEntries", {}) - should_sort_meta = self.config.get("sortMeta", False) - meta_config: list[tuple[str, str]] = list(meta_config_dict.items()) - if should_sort_meta: - meta_config = sorted(meta_config) - - snippet.append_text("\tmeta:\n") - - if len(meta_config) == 0: - snippet.append_text("\t\t") - snippet.append_placeholder("KEY") - snippet.append_text(" = ") - snippet.append_placeholder('"VALUE"') - snippet.append_text("\n") - else: - for key, value in meta_config: - if value == "": - snippet.append_text(f'\t\t{key} = "') - snippet.append_tabstop() - snippet.append_text('"\n') - else: - snippet.append_text(f'\t\t{key} = "{value}"\n') - - def generate(self) -> str: - snippet = SnippetString() - self.generate_rule_snippet(snippet) - - return str(snippet) + snippet.append_placeholder('"VALUE"') + snippet.append_text("\n") + else: + for key, value in meta_config: + if value == "": + snippet.append_text(f'\t\t{key} = "') + snippet.append_tabstop() + snippet.append_text('"\n') + else: + snippet.append_text(f'\t\t{key} = "{value}"\n') + + return snippet diff --git a/yls/utils.py b/yls/utils.py index 76040ea..1f96f68 100644 --- a/yls/utils.py +++ b/yls/utils.py @@ -774,7 +774,7 @@ async def get_config_from_editor(ls: Any, section: str) -> Any: ) # We should get list of configuration items here with exactly one element - assert isinstance(config, list) - assert len(config) == 1 + assert isinstance(config, list), "get configuration result should be a list" + assert len(config) >= 1, "did not receive correct configuration response" return config[0]