Remove cobol_parser/cobol_parser.ml

src/lsp/cobol_lsp/document.ml

@@ -34,7 +34,7 @@ module TYPES = struct
   and rewinder =
     (Cobol_ptree.Types.compilation_group option,
      Cobol_common.Behaviors.eidetic) Cobol_parser.Outputs.output
-      Cobol_parser.rewinder
+      Cobol_parser.Main.rewinder
 
   (** Raised by {!val:Document.checked}. *)
   exception Unparseable of Lsp.Types.DocumentUri.t
@@ -66,7 +66,7 @@ type t = document
 let uri { textdoc; _ } = Lsp.Text_document.documentUri textdoc
 
 let rewindable_parse ({ project; textdoc; _ } as doc) =
-  Cobol_parser.rewindable_parse_with_artifacts
+  Cobol_parser.Main.rewindable_parse_with_artifacts
     ~options:Cobol_parser.Options.{
         default with
         recovery = EnableRecovery { silence_benign_recoveries = true };
@@ -95,7 +95,7 @@ let check doc ptree =
       Cobol_typeck.Engine.compilation_group ~config ptree |>
       Cobol_typeck.Engine.translate_diagnostics ~config |>
       DIAGS.map_result ~f:begin fun checked ->
-        Cobol_parser.artifacts ptree, Some rewinder, Some checked
+        Cobol_parser.Main.artifacts ptree, Some rewinder, Some checked
       end
     end ptree
   in
@@ -115,7 +115,7 @@ let reparse_and_analyze ?position ({ copybook; rewinder; textdoc; _ } as doc) =
       { doc with artifacts = no_artifacts; rewinder = None; checked = None }
   | Some position, Some rewinder ->
       check doc @@
-      Cobol_parser.rewind_and_parse rewinder ~position @@
+      Cobol_parser.Main.rewind_and_parse rewinder ~position @@
       Cobol_preproc.Preprocess.reset_preprocessor_for_string @@
       Lsp.Text_document.text textdoc
 
@@ -156,7 +156,7 @@ let first_change_pos changes =
           acc
     end Int.(max_int, max_int) changes      (* can |changes|=0 really happen? *)
   in
-  Cobol_parser.Indexed { line; char }
+  Cobol_parser.Main.Indexed { line; char }
 
 let update ({ textdoc; _ } as doc) changes =
   let position = first_change_pos changes in

src/lsp/cobol_lsp/lsp_semtoks.ml

@@ -14,7 +14,7 @@
 open Cobol_common.Srcloc.TYPES
 open Cobol_common                                          (* Srcloc, Visitor *)
 open Cobol_common.Srcloc.INFIX
-open Cobol_parser.Tokens
+open Cobol_parser.Grammar_tokens
 
 module TOKTYP = struct
   type t = { index: int; name: string }

src/lsp/cobol_parser/parser_engine.ml → src/lsp/cobol_parser/main.ml

@@ -15,8 +15,8 @@ module DIAGS = Cobol_common.Diagnostics
 
 open Cobol_common.Diagnostics.TYPES
 open Cobol_common.Srcloc.INFIX
-open Parser_options                               (* import types for options *)
-open Parser_outputs                               (* import types for outputs *)
+open Options                               (* import types for options *)
+open Outputs                               (* import types for outputs *)
 
 module Tokzr = Text_tokenizer
 module Overlay_manager = Grammar_utils.Overlay_manager
@@ -94,12 +94,12 @@ and 'm persist =
 
 (** Initializes a parser state, given a preprocessor. *)
 let make_parser
-    (type m) Parser_options.{ verbose; show; recovery; config }
+    (type m) Options.{ verbose; show; recovery; config }
     ?show_if_verbose ~(tokenizer_memory: m memory) pp =
   let tokzr: m Tokzr.state =
     let memory: m Tokzr.memory = match tokenizer_memory with
-      | Parser_options.Amnesic -> Tokzr.amnesic
-      | Parser_options.Eidetic -> Tokzr.eidetic
+      | Options.Amnesic -> Tokzr.amnesic
+      | Options.Eidetic -> Tokzr.eidetic
     in
     let module Config = (val config) in
     Tokzr.init ~verbose ?show_if_verbose ~memory Config.words
@@ -580,7 +580,7 @@ let rewindable_parse
 let parse
     (type m)
     ~(memory: m memory)
-    ?(options = Parser_options.default)
+    ?(options = Options.default)
   : Cobol_preproc.Preprocess.t ->
     (Cobol_ptree.Types.compilation_group option, m) output with_diags =
   parse_once ~options ~memory
@@ -592,7 +592,7 @@ let parse_with_artifacts = parse ~memory:Eidetic
 let rewindable_parse
     (type m)
     ~(memory: m memory)
-    ?(options = Parser_options.default)
+    ?(options = Options.default)
   : Cobol_preproc.Preprocess.t ->
     (((Cobol_ptree.Types.compilation_group option, m) output as 'x) * 'x rewinder)
       with_diags =
@@ -608,3 +608,38 @@ let rewind_and_parse { rewind_n_parse } rewind_preproc ~position =
 let artifacts
   : (_, Cobol_common.Behaviors.eidetic) output -> _ = function
   | WithArtifacts (_, artifacts) -> artifacts
+
+module INTERNAL = struct
+
+  (** {2 COBOL tokens} *)
+
+  module Tokens = Grammar_tokens
+
+  let pp_token = Text_tokenizer.pp_token
+  let pp_tokens = Text_tokenizer.pp_tokens
+  let pp_tokens' = Text_tokenizer.pp_tokens'
+
+  (** {2 COBOL grammar} *)
+
+  module Grammar (* : Grammar_sig.S *) = Grammar
+
+  (** {2 Dummy parser} *)
+
+  (** Parser with dummy source locations, that can be fed directly with a
+      list of tokens *)
+  module Dummy = struct
+    module Tags = struct
+      let loc = Cobol_common.Srcloc.dummy
+    end
+
+    let parse_as item toks =
+      let toks = ref toks
+      and dummy_lexer = Lexing.from_string ~with_positions:false "" in
+      item begin fun _ -> match !toks () with
+        | Seq.Nil -> Grammar_tokens.EOF
+        | Cons (x, tl) -> toks := tl; x
+      end dummy_lexer
+
+    let parse_list_as parse lx = parse_as parse (List.to_seq lx)
+  end
+end

src/lsp/cobol_parser/parser_engine.mli → src/lsp/cobol_parser/main.mli

@@ -11,8 +11,8 @@
 (*                                                                        *)
 (**************************************************************************)
 
-open Parser_options
-open Parser_outputs
+open Options
+open Outputs
 
 (** Parsing functions essentially parse a stream of tokens that is produced by a
     given preprocessor (typically returned by
@@ -36,7 +36,7 @@ open Parser_outputs
 
 (** Simple parsing functions traverse the inputs once to produce a result. *)
 type 'm simple_parsing
-  = ?options:Parser_options.parser_options
+  = ?options:Options.parser_options
   -> Cobol_preproc.Preprocess.t
   -> (Cobol_ptree.Types.compilation_group option, 'm) output
     Cobol_common.Diagnostics.with_diags
@@ -113,3 +113,20 @@ val rewind_and_parse
 
 val artifacts
   : (_, Cobol_common.Behaviors.eidetic) output -> artifacts
+
+module INTERNAL : sig
+
+  val pp_token : Text_tokenizer.token Pretty.printer
+  val pp_tokens : Text_tokenizer.tokens Pretty.printer
+  val pp_tokens' : ?fsep:Pretty.simple -> Text_tokenizer.tokens Pretty.printer
+
+  module Dummy : sig
+    module Tags : sig
+      val loc : Cobol_common.Srcloc.t
+    end
+    val parse_list_as : (('a -> Grammar.token) -> Lexing.lexbuf -> 'b) ->
+      Grammar.token list -> 'b
+end
+
+
+end

src/lsp/cobol_parser/package.toml

@@ -95,7 +95,7 @@ dune-trailer = """
         (flags --inspection --table --only-tokens))
 
 (rule
-  (targets text_keywords.ml)
+  (targets keywords.ml)
   (enabled_if (<> %{profile} "release"))
   (deps    grammar.cmly)
   (mode    promote)

src/lsp/cobol_parser/text_lexer.ml

@@ -86,18 +86,18 @@ let __init_puncts =
   List.iter begin fun (punct, token) ->
     Hashtbl.add punct_of_token token punct;
     Hashtbl.add token_of_punct punct token
-  end Text_keywords.puncts
+  end Keywords.puncts
 
 let __init_default_keywords =
   List.iter begin fun (kwd, token) ->
     Hashtbl.add keyword_of_token token kwd;
     (* Every default token needs to be reserved explicitly *)
     Hashtbl.add __token_of_keyword kwd
       { token; enabled = true; reserved = false }
-  end Text_keywords.keywords
+  end Keywords.keywords
 
 let silenced_keywords =
-  StringSet.of_list Text_keywords.silenced_keywords
+  StringSet.of_list Keywords.silenced_keywords
 
 (* --- *)
 

src/lsp/superbol_free_lib/command_pp.ml

@@ -64,7 +64,7 @@ let cmd =
                    in
                    input |>
                    Cobol_preproc.Preprocess.preprocessor ~options:preproc_options |>
-                   Cobol_parser.parse_simple ~options:common.parser_options
+                   Cobol_parser.Main.parse_simple ~options:common.parser_options
                  in
                  let my_text = Cobol_preproc.Src_input.from ~filename:file ~f:parse in
                  Format.eprintf "%a@." Cobol_common.Diagnostics.Set.pp my_text.diags;

test/cobol_data/test_memory.ml

@@ -11,7 +11,7 @@
 (*                                                                        *)
 (**************************************************************************)
 
-open Testing_helpers.Make (Cobol_parser.INTERNAL.Dummy.Tags)
+open Testing_helpers.Make (Cobol_parser.Main.INTERNAL.Dummy.Tags)
 module NEL = Cobol_common.Basics.NEL
 
 module Memory = Cobol_data.Memory

test/cobol_parsing/parser_testing.ml

@@ -32,20 +32,20 @@ let show_parsed_tokens ?(verbose = false) ?(with_locations = false)
     ?source_format ?filename contents =
   let DIAGS.{ result = WithArtifacts (_, { tokens; _ }); _ } =
     preproc ?source_format ?filename contents |>
-    Cobol_parser.parse_with_artifacts
+    Cobol_parser.Main.parse_with_artifacts
       ~options:Cobol_parser.Options.{
           default with
           verbose;
           recovery = EnableRecovery { silence_benign_recoveries = true };
         }
   in
   (if with_locations
-   then Cobol_parser.INTERNAL.pp_tokens' ~fsep:"@\n"
-   else Cobol_parser.INTERNAL.pp_tokens) Fmt.stdout (Lazy.force tokens)
+   then Cobol_parser.Main.INTERNAL.pp_tokens' ~fsep:"@\n"
+   else Cobol_parser.Main.INTERNAL.pp_tokens) Fmt.stdout (Lazy.force tokens)
 
 let show_diagnostics ?(verbose = false) ?source_format ?filename contents =
   preproc ?source_format ?filename contents |>
-  Cobol_parser.parse_simple
+  Cobol_parser.Main.parse_simple
     ~options:Cobol_parser.Options.{
         default with
         verbose;
@@ -154,7 +154,7 @@ let rewindable_parse
           verbose; libpath = []; source_format;
           config = Option.value config ~default:default.config;
         } |>
-    Cobol_parser.rewindable_parse_simple
+    Cobol_parser.Main.rewindable_parse_simple
       ~options:Cobol_parser.Options.{
           default with
           verbose; recovery = DisableRecovery;
@@ -166,8 +166,8 @@ let rewindable_parse
 (** Note: won't show detailed source locations as the openned file is neither
     actually on disk nor registered via {!Srcloc.register_file_contents}. *)
 let rewind_n_parse ~f rewinder { line; char; _ } preproc_rewind =
-  let DIAGS.{ result = Only ptree, rewinder; diags } =
-    Cobol_parser.rewind_and_parse rewinder preproc_rewind
+  let DIAGS.{ result = Cobol_parser.Outputs.Only ptree, rewinder; diags } =
+    Cobol_parser.Main.rewind_and_parse rewinder preproc_rewind
       ~position:(Indexed { line; char })
   in
   f ptree diags;

test/cobol_parsing/test_combined_relations_parsing.ml

@@ -14,10 +14,10 @@
 open Alcotest
 
 open Cobol_ptree.Types
-open Testing_helpers.Make (Cobol_parser.INTERNAL.Dummy.Tags)
-open Cobol_parser.INTERNAL.Tokens
-open Cobol_parser.INTERNAL.Grammar
-open Cobol_parser.INTERNAL.Dummy
+open Testing_helpers.Make (Cobol_parser.Main.INTERNAL.Dummy.Tags)
+open Cobol_parser.Grammar_tokens
+open Cobol_parser.Grammar
+open Cobol_parser.Main.INTERNAL.Dummy
 
 let condition: condition testable = testable pp_condition (=)
 let parse_condition = parse_list_as standalone_condition

test/cobol_typeck/typeck_testing.ml

@@ -18,7 +18,7 @@ module DIAGS = Cobol_common.Diagnostics
 let show_diagnostics ?(show_data = false) ?(verbose = false)
     ?source_format ?filename contents =
   preproc ?source_format ?filename contents |>
-  Cobol_parser.parse_simple
+  Cobol_parser.Main.parse_simple
     ~options:Cobol_parser.Options.{
         default with
         verbose;

test/cobol_unit/test_qualmap.ml

@@ -12,7 +12,7 @@
 (**************************************************************************)
 
 open Cobol_unit.Qualmap
-open Testing_helpers.Make (Cobol_parser.INTERNAL.Dummy.Tags)
+open Testing_helpers.Make (Cobol_parser.Main.INTERNAL.Dummy.Tags)
 
 module NEL = Cobol_common.Basics.NEL