From 29b5666797e221946bb590eafd1f7ab82b682bbb Mon Sep 17 00:00:00 2001 From: Fabrice Le Fessant Date: Thu, 1 Feb 2024 23:16:39 +0100 Subject: [PATCH] Remove cobol_parser/cobol_parser.ml --- .drom | 4 +- src/lsp/cobol_lsp/document.ml | 10 +-- src/lsp/cobol_lsp/lsp_semtoks.ml | 2 +- src/lsp/cobol_parser/cobol_parser.ml | 66 ------------------- src/lsp/cobol_parser/dune | 2 +- .../{text_keywords.ml => keywords.ml} | 0 .../{text_keywords.mli => keywords.mli} | 0 .../{parser_engine.ml => main.ml} | 49 ++++++++++++-- .../{parser_engine.mli => main.mli} | 23 ++++++- .../{parser_options.ml => options.ml} | 0 .../{parser_outputs.ml => outputs.ml} | 0 src/lsp/cobol_parser/package.toml | 2 +- src/lsp/cobol_parser/text_lexer.ml | 6 +- src/lsp/superbol_free_lib/command_pp.ml | 2 +- test/cobol_data/test_memory.ml | 2 +- test/cobol_parsing/parser_testing.ml | 14 ++-- .../test_combined_relations_parsing.ml | 8 +-- test/cobol_typeck/typeck_testing.ml | 2 +- test/cobol_unit/test_qualmap.ml | 2 +- test/output-tests/gnucobol.ml | 2 +- test/output-tests/reparse.ml | 2 +- 21 files changed, 92 insertions(+), 106 deletions(-) delete mode 100644 src/lsp/cobol_parser/cobol_parser.ml rename src/lsp/cobol_parser/{text_keywords.ml => keywords.ml} (100%) rename src/lsp/cobol_parser/{text_keywords.mli => keywords.mli} (100%) rename src/lsp/cobol_parser/{parser_engine.ml => main.ml} (94%) rename src/lsp/cobol_parser/{parser_engine.mli => main.mli} (91%) rename src/lsp/cobol_parser/{parser_options.ml => options.ml} (100%) rename src/lsp/cobol_parser/{parser_outputs.ml => outputs.ml} (100%) diff --git a/.drom b/.drom index f0f0e1cb2..ca0d126ef 100644 --- a/.drom +++ b/.drom @@ -5,7 +5,7 @@ version:0.9.0 # hash of toml configuration files # used for generation of all files -93d2e460909a3ddf6a2ed971ff2e8639:. +902350756f0d7d15400b20a907430e9e:. # end context for . # begin context for .github/workflows/workflow.yml @@ -345,7 +345,7 @@ e74b6a4502967712b07002d1fb8b4292:src/lsp/cobol_data_old/dune # begin context for src/lsp/cobol_parser/dune # file src/lsp/cobol_parser/dune -09762dde0adad2bf9c7c37823db7919c:src/lsp/cobol_parser/dune +0b682680037cd6c542be12c29b35715d:src/lsp/cobol_parser/dune # end context for src/lsp/cobol_parser/dune # begin context for src/lsp/cobol_parser/version.mlt diff --git a/src/lsp/cobol_lsp/document.ml b/src/lsp/cobol_lsp/document.ml index 46ff390bf..e99e1b7d2 100644 --- a/src/lsp/cobol_lsp/document.ml +++ b/src/lsp/cobol_lsp/document.ml @@ -34,7 +34,7 @@ module TYPES = struct and rewinder = (Cobol_ptree.Types.compilation_group option, Cobol_common.Behaviors.eidetic) Cobol_parser.Outputs.output - Cobol_parser.rewinder + Cobol_parser.Main.rewinder (** Raised by {!val:Document.checked}. *) exception Unparseable of Lsp.Types.DocumentUri.t @@ -66,7 +66,7 @@ type t = document let uri { textdoc; _ } = Lsp.Text_document.documentUri textdoc let rewindable_parse ({ project; textdoc; _ } as doc) = - Cobol_parser.rewindable_parse_with_artifacts + Cobol_parser.Main.rewindable_parse_with_artifacts ~options:Cobol_parser.Options.{ default with recovery = EnableRecovery { silence_benign_recoveries = true }; @@ -95,7 +95,7 @@ let check doc ptree = Cobol_typeck.Engine.compilation_group ~config ptree |> Cobol_typeck.Engine.translate_diagnostics ~config |> DIAGS.map_result ~f:begin fun checked -> - Cobol_parser.artifacts ptree, Some rewinder, Some checked + Cobol_parser.Main.artifacts ptree, Some rewinder, Some checked end end ptree in @@ -115,7 +115,7 @@ let reparse_and_analyze ?position ({ copybook; rewinder; textdoc; _ } as doc) = { doc with artifacts = no_artifacts; rewinder = None; checked = None } | Some position, Some rewinder -> check doc @@ - Cobol_parser.rewind_and_parse rewinder ~position @@ + Cobol_parser.Main.rewind_and_parse rewinder ~position @@ Cobol_preproc.Preprocess.reset_preprocessor_for_string @@ Lsp.Text_document.text textdoc @@ -156,7 +156,7 @@ let first_change_pos changes = acc end Int.(max_int, max_int) changes (* can |changes|=0 really happen? *) in - Cobol_parser.Indexed { line; char } + Cobol_parser.Main.Indexed { line; char } let update ({ textdoc; _ } as doc) changes = let position = first_change_pos changes in diff --git a/src/lsp/cobol_lsp/lsp_semtoks.ml b/src/lsp/cobol_lsp/lsp_semtoks.ml index 12a33049a..68a8f4254 100644 --- a/src/lsp/cobol_lsp/lsp_semtoks.ml +++ b/src/lsp/cobol_lsp/lsp_semtoks.ml @@ -14,7 +14,7 @@ open Cobol_common.Srcloc.TYPES open Cobol_common (* Srcloc, Visitor *) open Cobol_common.Srcloc.INFIX -open Cobol_parser.Tokens +open Cobol_parser.Grammar_tokens module TOKTYP = struct type t = { index: int; name: string } diff --git a/src/lsp/cobol_parser/cobol_parser.ml b/src/lsp/cobol_parser/cobol_parser.ml deleted file mode 100644 index fea3b33da..000000000 --- a/src/lsp/cobol_parser/cobol_parser.ml +++ /dev/null @@ -1,66 +0,0 @@ -(**************************************************************************) -(* *) -(* SuperBOL OSS Studio *) -(* *) -(* Copyright (c) 2022-2023 OCamlPro SAS *) -(* *) -(* All rights reserved. *) -(* This source code is licensed under the GNU Affero General Public *) -(* License version 3 found in the LICENSE.md file in the root directory *) -(* of this source tree. *) -(* *) -(**************************************************************************) - -(** {1 Exported modules} *) - -(** Options to tune the parser engine *) -module Options = Parser_options - -(** Output types for the engine *) -module Outputs = Parser_outputs - -module Tokens = Grammar_tokens -module Keywords = Text_keywords - -(** {1 Exported functions} *) - -include Parser_engine - -(** {1 Modules and functions exported for testing purposes} - - Signatures of modules below may change unexpectedly. *) - -module INTERNAL = struct - - (** {2 COBOL tokens} *) - - module Tokens = Grammar_tokens - - let pp_token = Text_tokenizer.pp_token - let pp_tokens = Text_tokenizer.pp_tokens - let pp_tokens' = Text_tokenizer.pp_tokens' - - (** {2 COBOL grammar} *) - - module Grammar (* : Grammar_sig.S *) = Grammar - - (** {2 Dummy parser} *) - - (** Parser with dummy source locations, that can be fed directly with a - list of tokens *) - module Dummy = struct - module Tags = struct - let loc = Cobol_common.Srcloc.dummy - end - - let parse_as item toks = - let toks = ref toks - and dummy_lexer = Lexing.from_string ~with_positions:false "" in - item begin fun _ -> match !toks () with - | Seq.Nil -> Grammar_tokens.EOF - | Cons (x, tl) -> toks := tl; x - end dummy_lexer - - let parse_list_as parse lx = parse_as parse (List.to_seq lx) - end -end diff --git a/src/lsp/cobol_parser/dune b/src/lsp/cobol_parser/dune index 4164f945c..74e86dd7a 100644 --- a/src/lsp/cobol_parser/dune +++ b/src/lsp/cobol_parser/dune @@ -36,7 +36,7 @@ (flags --inspection --table --only-tokens)) (rule - (targets text_keywords.ml) + (targets keywords.ml) (enabled_if (<> %{profile} "release")) (deps grammar.cmly) (mode promote) diff --git a/src/lsp/cobol_parser/text_keywords.ml b/src/lsp/cobol_parser/keywords.ml similarity index 100% rename from src/lsp/cobol_parser/text_keywords.ml rename to src/lsp/cobol_parser/keywords.ml diff --git a/src/lsp/cobol_parser/text_keywords.mli b/src/lsp/cobol_parser/keywords.mli similarity index 100% rename from src/lsp/cobol_parser/text_keywords.mli rename to src/lsp/cobol_parser/keywords.mli diff --git a/src/lsp/cobol_parser/parser_engine.ml b/src/lsp/cobol_parser/main.ml similarity index 94% rename from src/lsp/cobol_parser/parser_engine.ml rename to src/lsp/cobol_parser/main.ml index bd73eaa0c..fea41c2c9 100644 --- a/src/lsp/cobol_parser/parser_engine.ml +++ b/src/lsp/cobol_parser/main.ml @@ -15,8 +15,8 @@ module DIAGS = Cobol_common.Diagnostics open Cobol_common.Diagnostics.TYPES open Cobol_common.Srcloc.INFIX -open Parser_options (* import types for options *) -open Parser_outputs (* import types for outputs *) +open Options (* import types for options *) +open Outputs (* import types for outputs *) module Tokzr = Text_tokenizer module Overlay_manager = Grammar_utils.Overlay_manager @@ -94,12 +94,12 @@ and 'm persist = (** Initializes a parser state, given a preprocessor. *) let make_parser - (type m) Parser_options.{ verbose; show; recovery; config } + (type m) Options.{ verbose; show; recovery; config } ?show_if_verbose ~(tokenizer_memory: m memory) pp = let tokzr: m Tokzr.state = let memory: m Tokzr.memory = match tokenizer_memory with - | Parser_options.Amnesic -> Tokzr.amnesic - | Parser_options.Eidetic -> Tokzr.eidetic + | Options.Amnesic -> Tokzr.amnesic + | Options.Eidetic -> Tokzr.eidetic in let module Config = (val config) in Tokzr.init ~verbose ?show_if_verbose ~memory Config.words @@ -580,7 +580,7 @@ let rewindable_parse let parse (type m) ~(memory: m memory) - ?(options = Parser_options.default) + ?(options = Options.default) : Cobol_preproc.Preprocess.t -> (Cobol_ptree.Types.compilation_group option, m) output with_diags = parse_once ~options ~memory @@ -592,7 +592,7 @@ let parse_with_artifacts = parse ~memory:Eidetic let rewindable_parse (type m) ~(memory: m memory) - ?(options = Parser_options.default) + ?(options = Options.default) : Cobol_preproc.Preprocess.t -> (((Cobol_ptree.Types.compilation_group option, m) output as 'x) * 'x rewinder) with_diags = @@ -608,3 +608,38 @@ let rewind_and_parse { rewind_n_parse } rewind_preproc ~position = let artifacts : (_, Cobol_common.Behaviors.eidetic) output -> _ = function | WithArtifacts (_, artifacts) -> artifacts + +module INTERNAL = struct + + (** {2 COBOL tokens} *) + + module Tokens = Grammar_tokens + + let pp_token = Text_tokenizer.pp_token + let pp_tokens = Text_tokenizer.pp_tokens + let pp_tokens' = Text_tokenizer.pp_tokens' + + (** {2 COBOL grammar} *) + + module Grammar (* : Grammar_sig.S *) = Grammar + + (** {2 Dummy parser} *) + + (** Parser with dummy source locations, that can be fed directly with a + list of tokens *) + module Dummy = struct + module Tags = struct + let loc = Cobol_common.Srcloc.dummy + end + + let parse_as item toks = + let toks = ref toks + and dummy_lexer = Lexing.from_string ~with_positions:false "" in + item begin fun _ -> match !toks () with + | Seq.Nil -> Grammar_tokens.EOF + | Cons (x, tl) -> toks := tl; x + end dummy_lexer + + let parse_list_as parse lx = parse_as parse (List.to_seq lx) + end +end diff --git a/src/lsp/cobol_parser/parser_engine.mli b/src/lsp/cobol_parser/main.mli similarity index 91% rename from src/lsp/cobol_parser/parser_engine.mli rename to src/lsp/cobol_parser/main.mli index 6b101dbed..015c772d7 100644 --- a/src/lsp/cobol_parser/parser_engine.mli +++ b/src/lsp/cobol_parser/main.mli @@ -11,8 +11,8 @@ (* *) (**************************************************************************) -open Parser_options -open Parser_outputs +open Options +open Outputs (** Parsing functions essentially parse a stream of tokens that is produced by a given preprocessor (typically returned by @@ -36,7 +36,7 @@ open Parser_outputs (** Simple parsing functions traverse the inputs once to produce a result. *) type 'm simple_parsing - = ?options:Parser_options.parser_options + = ?options:Options.parser_options -> Cobol_preproc.Preprocess.t -> (Cobol_ptree.Types.compilation_group option, 'm) output Cobol_common.Diagnostics.with_diags @@ -113,3 +113,20 @@ val rewind_and_parse val artifacts : (_, Cobol_common.Behaviors.eidetic) output -> artifacts + +module INTERNAL : sig + + val pp_token : Text_tokenizer.token Pretty.printer + val pp_tokens : Text_tokenizer.tokens Pretty.printer + val pp_tokens' : ?fsep:Pretty.simple -> Text_tokenizer.tokens Pretty.printer + + module Dummy : sig + module Tags : sig + val loc : Cobol_common.Srcloc.t + end + val parse_list_as : (('a -> Grammar.token) -> Lexing.lexbuf -> 'b) -> + Grammar.token list -> 'b +end + + +end diff --git a/src/lsp/cobol_parser/parser_options.ml b/src/lsp/cobol_parser/options.ml similarity index 100% rename from src/lsp/cobol_parser/parser_options.ml rename to src/lsp/cobol_parser/options.ml diff --git a/src/lsp/cobol_parser/parser_outputs.ml b/src/lsp/cobol_parser/outputs.ml similarity index 100% rename from src/lsp/cobol_parser/parser_outputs.ml rename to src/lsp/cobol_parser/outputs.ml diff --git a/src/lsp/cobol_parser/package.toml b/src/lsp/cobol_parser/package.toml index 57e2bbdb3..382e0fd1e 100644 --- a/src/lsp/cobol_parser/package.toml +++ b/src/lsp/cobol_parser/package.toml @@ -95,7 +95,7 @@ dune-trailer = """ (flags --inspection --table --only-tokens)) (rule - (targets text_keywords.ml) + (targets keywords.ml) (enabled_if (<> %{profile} "release")) (deps grammar.cmly) (mode promote) diff --git a/src/lsp/cobol_parser/text_lexer.ml b/src/lsp/cobol_parser/text_lexer.ml index b1d41c9b8..a5949ce37 100644 --- a/src/lsp/cobol_parser/text_lexer.ml +++ b/src/lsp/cobol_parser/text_lexer.ml @@ -86,7 +86,7 @@ let __init_puncts = List.iter begin fun (punct, token) -> Hashtbl.add punct_of_token token punct; Hashtbl.add token_of_punct punct token - end Text_keywords.puncts + end Keywords.puncts let __init_default_keywords = List.iter begin fun (kwd, token) -> @@ -94,10 +94,10 @@ let __init_default_keywords = (* Every default token needs to be reserved explicitly *) Hashtbl.add __token_of_keyword kwd { token; enabled = true; reserved = false } - end Text_keywords.keywords + end Keywords.keywords let silenced_keywords = - StringSet.of_list Text_keywords.silenced_keywords + StringSet.of_list Keywords.silenced_keywords (* --- *) diff --git a/src/lsp/superbol_free_lib/command_pp.ml b/src/lsp/superbol_free_lib/command_pp.ml index 6ee752266..2c4182f98 100644 --- a/src/lsp/superbol_free_lib/command_pp.ml +++ b/src/lsp/superbol_free_lib/command_pp.ml @@ -64,7 +64,7 @@ let cmd = in input |> Cobol_preproc.Preprocess.preprocessor ~options:preproc_options |> - Cobol_parser.parse_simple ~options:common.parser_options + Cobol_parser.Main.parse_simple ~options:common.parser_options in let my_text = Cobol_preproc.Src_input.from ~filename:file ~f:parse in Format.eprintf "%a@." Cobol_common.Diagnostics.Set.pp my_text.diags; diff --git a/test/cobol_data/test_memory.ml b/test/cobol_data/test_memory.ml index 5fd40e083..8cc0498c0 100644 --- a/test/cobol_data/test_memory.ml +++ b/test/cobol_data/test_memory.ml @@ -11,7 +11,7 @@ (* *) (**************************************************************************) -open Testing_helpers.Make (Cobol_parser.INTERNAL.Dummy.Tags) +open Testing_helpers.Make (Cobol_parser.Main.INTERNAL.Dummy.Tags) module NEL = Cobol_common.Basics.NEL module Memory = Cobol_data.Memory diff --git a/test/cobol_parsing/parser_testing.ml b/test/cobol_parsing/parser_testing.ml index 049e7a3b1..fbf9eb373 100644 --- a/test/cobol_parsing/parser_testing.ml +++ b/test/cobol_parsing/parser_testing.ml @@ -32,7 +32,7 @@ let show_parsed_tokens ?(verbose = false) ?(with_locations = false) ?source_format ?filename contents = let DIAGS.{ result = WithArtifacts (_, { tokens; _ }); _ } = preproc ?source_format ?filename contents |> - Cobol_parser.parse_with_artifacts + Cobol_parser.Main.parse_with_artifacts ~options:Cobol_parser.Options.{ default with verbose; @@ -40,12 +40,12 @@ let show_parsed_tokens ?(verbose = false) ?(with_locations = false) } in (if with_locations - then Cobol_parser.INTERNAL.pp_tokens' ~fsep:"@\n" - else Cobol_parser.INTERNAL.pp_tokens) Fmt.stdout (Lazy.force tokens) + then Cobol_parser.Main.INTERNAL.pp_tokens' ~fsep:"@\n" + else Cobol_parser.Main.INTERNAL.pp_tokens) Fmt.stdout (Lazy.force tokens) let show_diagnostics ?(verbose = false) ?source_format ?filename contents = preproc ?source_format ?filename contents |> - Cobol_parser.parse_simple + Cobol_parser.Main.parse_simple ~options:Cobol_parser.Options.{ default with verbose; @@ -154,7 +154,7 @@ let rewindable_parse verbose; libpath = []; source_format; config = Option.value config ~default:default.config; } |> - Cobol_parser.rewindable_parse_simple + Cobol_parser.Main.rewindable_parse_simple ~options:Cobol_parser.Options.{ default with verbose; recovery = DisableRecovery; @@ -166,8 +166,8 @@ let rewindable_parse (** Note: won't show detailed source locations as the openned file is neither actually on disk nor registered via {!Srcloc.register_file_contents}. *) let rewind_n_parse ~f rewinder { line; char; _ } preproc_rewind = - let DIAGS.{ result = Only ptree, rewinder; diags } = - Cobol_parser.rewind_and_parse rewinder preproc_rewind + let DIAGS.{ result = Cobol_parser.Outputs.Only ptree, rewinder; diags } = + Cobol_parser.Main.rewind_and_parse rewinder preproc_rewind ~position:(Indexed { line; char }) in f ptree diags; diff --git a/test/cobol_parsing/test_combined_relations_parsing.ml b/test/cobol_parsing/test_combined_relations_parsing.ml index dacf6e2f0..af1c39625 100644 --- a/test/cobol_parsing/test_combined_relations_parsing.ml +++ b/test/cobol_parsing/test_combined_relations_parsing.ml @@ -14,10 +14,10 @@ open Alcotest open Cobol_ptree.Types -open Testing_helpers.Make (Cobol_parser.INTERNAL.Dummy.Tags) -open Cobol_parser.INTERNAL.Tokens -open Cobol_parser.INTERNAL.Grammar -open Cobol_parser.INTERNAL.Dummy +open Testing_helpers.Make (Cobol_parser.Main.INTERNAL.Dummy.Tags) +open Cobol_parser.Grammar_tokens +open Cobol_parser.Grammar +open Cobol_parser.Main.INTERNAL.Dummy let condition: condition testable = testable pp_condition (=) let parse_condition = parse_list_as standalone_condition diff --git a/test/cobol_typeck/typeck_testing.ml b/test/cobol_typeck/typeck_testing.ml index b5831be16..cc68413db 100644 --- a/test/cobol_typeck/typeck_testing.ml +++ b/test/cobol_typeck/typeck_testing.ml @@ -18,7 +18,7 @@ module DIAGS = Cobol_common.Diagnostics let show_diagnostics ?(show_data = false) ?(verbose = false) ?source_format ?filename contents = preproc ?source_format ?filename contents |> - Cobol_parser.parse_simple + Cobol_parser.Main.parse_simple ~options:Cobol_parser.Options.{ default with verbose; diff --git a/test/cobol_unit/test_qualmap.ml b/test/cobol_unit/test_qualmap.ml index 01d5cf9e7..c7733bde4 100644 --- a/test/cobol_unit/test_qualmap.ml +++ b/test/cobol_unit/test_qualmap.ml @@ -12,7 +12,7 @@ (**************************************************************************) open Cobol_unit.Qualmap -open Testing_helpers.Make (Cobol_parser.INTERNAL.Dummy.Tags) +open Testing_helpers.Make (Cobol_parser.Main.INTERNAL.Dummy.Tags) module NEL = Cobol_common.Basics.NEL diff --git a/test/output-tests/gnucobol.ml b/test/output-tests/gnucobol.ml index 673588e2c..4a4251ba8 100644 --- a/test/output-tests/gnucobol.ml +++ b/test/output-tests/gnucobol.ml @@ -192,7 +192,7 @@ let do_check_parse (test_filename, contents, _, { check_loc; input |> Cobol_preproc.Preprocess.preprocessor ~options:Cobol_preproc.Options.{ default with source_format } |> - Cobol_parser.parse_simple + Cobol_parser.Main.parse_simple in try let input = setup_input ~filename contents in diff --git a/test/output-tests/reparse.ml b/test/output-tests/reparse.ml index 0e81cc3ab..02cf9bfe2 100644 --- a/test/output-tests/reparse.ml +++ b/test/output-tests/reparse.ml @@ -18,7 +18,7 @@ open Testsuite_utils let reparse_file ~source_format ~config filename = let parse ~source_format input = - Cobol_parser.parse_simple + Cobol_parser.Main.parse_simple ~options:Cobol_parser.Options.{ default with recovery = DisableRecovery