Skip to content

Commit 6e857da

Browse files
ratmiceratmice
committed
Initial implementation of test_files in the %grmtools section for `CTParserBuilder.
This allows quoted string values to be specified within the %grmtools section, A `test_files` value using it and implements `CTParserBuilder` for that key.
1 parent 762f259 commit 6e857da

File tree

17 files changed

+128
-46
lines changed

17 files changed

+128
-46
lines changed

‎cfgrammar/src/lib/header.rs

+46-23
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ pub enum Setting<T> {
124124
arg: Namespaced<T>,
125125
},
126126
Num(u64, T),
127+
String(String, T),
127128
}
128129

129130
/// Parser for the `%grmtools` section
@@ -178,6 +179,7 @@ impl From<Value<Span>> for Value<Location> {
178179
},
179180
},
180181
Setting::Num(num, num_loc) => Setting::Num(num, num_loc.into()),
182+
Setting::String(s, str_loc) => Setting::String(s, str_loc.into()),
181183
}),
182184
}
183185
}
@@ -190,6 +192,7 @@ lazy_static! {
190192
.build()
191193
.unwrap();
192194
static ref RE_DIGITS: Regex = Regex::new(r"^[0-9]+").unwrap();
195+
static ref RE_STRING: Regex = Regex::new(r#"^\"(\\.|[^"\\])*\""#).unwrap();
193196
}
194197

195198
const MAGIC: &str = "%grmtools";
@@ -244,38 +247,50 @@ impl<'input> GrmtoolsSectionParser<'input> {
244247
i = self.parse_ws(num_span.end());
245248
Ok((key_name, key_span, Value::Setting(val), i))
246249
}
247-
None => {
248-
let (path_val, j) = self.parse_namespaced(i)?;
249-
i = self.parse_ws(j);
250-
if let Some(j) = self.lookahead_is("(", i) {
251-
let (arg, j) = self.parse_namespaced(j)?;
250+
None => match RE_STRING.find(&self.src[i..]) {
251+
Some(m) => {
252+
let end = i + m.end();
253+
// Trim the leading and trailing quotes.
254+
let str_span = Span::new(i + m.start() + 1, end - 1);
255+
let str = &self.src[str_span.start()..str_span.end()];
256+
let setting = Setting::String(str.to_string(), str_span);
257+
// After the trailing quotes.
258+
i = self.parse_ws(end);
259+
Ok((key_name, key_span, Value::Setting(setting), i))
260+
}
261+
None => {
262+
let (path_val, j) = self.parse_namespaced(i)?;
252263
i = self.parse_ws(j);
253-
if let Some(j) = self.lookahead_is(")", i) {
264+
if let Some(j) = self.lookahead_is("(", i) {
265+
let (arg, j) = self.parse_namespaced(j)?;
254266
i = self.parse_ws(j);
267+
if let Some(j) = self.lookahead_is(")", i) {
268+
i = self.parse_ws(j);
269+
Ok((
270+
key_name,
271+
key_span,
272+
Value::Setting(Setting::Constructor {
273+
ctor: path_val,
274+
arg,
275+
}),
276+
i,
277+
))
278+
} else {
279+
Err(HeaderError {
280+
kind: HeaderErrorKind::ExpectedToken(')'),
281+
locations: vec![Span::new(i, i)],
282+
})
283+
}
284+
} else {
255285
Ok((
256286
key_name,
257287
key_span,
258-
Value::Setting(Setting::Constructor {
259-
ctor: path_val,
260-
arg,
261-
}),
288+
Value::Setting(Setting::Unitary(path_val)),
262289
i,
263290
))
264-
} else {
265-
Err(HeaderError {
266-
kind: HeaderErrorKind::ExpectedToken(')'),
267-
locations: vec![Span::new(i, i)],
268-
})
269291
}
270-
} else {
271-
Ok((
272-
key_name,
273-
key_span,
274-
Value::Setting(Setting::Unitary(path_val)),
275-
i,
276-
))
277292
}
278-
}
293+
},
279294
}
280295
} else {
281296
Ok((key_name, key_span, Value::Flag(true, key_span), i))
@@ -489,6 +504,14 @@ impl<T: Clone> TryFrom<&Value<T>> for YaccKind {
489504
),
490505
locations: vec![loc.clone()],
491506
}),
507+
Value::Setting(Setting::String(_, loc)) => Err(HeaderError {
508+
kind: HeaderErrorKind::ConversionError(
509+
"From<YaccKind>",
510+
"Cannot convert string to YaccKind",
511+
),
512+
locations: vec![loc.clone()],
513+
}),
514+
492515
Value::Setting(Setting::Unitary(Namespaced {
493516
namespace,
494517
member: (yk_value, yk_value_loc),

‎lrlex/Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,4 @@ bincode.workspace = true
3838
serde = { workspace = true, optional = true }
3939
prettyplease.workspace = true
4040
syn.workspace = true
41+
glob.workspace = true

‎lrlex/src/lib/ctbuilder.rs

+25-11
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ use cfgrammar::{
2525
span::Location,
2626
Spanned,
2727
};
28+
use glob::glob;
2829
use lazy_static::lazy_static;
2930
use lrpar::{CTParserBuilder, LexerTypes};
3031
use num_traits::{AsPrimitive, PrimInt, Unsigned};
@@ -64,6 +65,13 @@ impl<T: Clone> TryFrom<&Value<T>> for LexerKind {
6465
),
6566
locations: vec![loc.clone()],
6667
}),
68+
Value::Setting(Setting::String(_, loc)) => Err(HeaderError {
69+
kind: HeaderErrorKind::ConversionError(
70+
"LexerKind",
71+
"Expected `LexerKind` found string",
72+
),
73+
locations: vec![loc.clone()],
74+
}),
6775
Value::Setting(Setting::Constructor {
6876
ctor:
6977
Namespaced {
@@ -496,26 +504,32 @@ where
496504
if let Some(ref lrcfg) = self.lrpar_config {
497505
let mut lexerdef = lexerdef.clone();
498506
let mut ctp = CTParserBuilder::<LexerTypesT>::new().inspect_rt(Box::new(
499-
move |yacc_header, rtpb, rule_ids_map| {
507+
move |yacc_header, rtpb, rule_ids_map, grm_path| {
500508
let owned_map = rule_ids_map
501509
.iter()
502510
.map(|(x, y)| (&**x, *y))
503511
.collect::<HashMap<_, _>>();
504512
lexerdef.set_rule_ids(&owned_map);
505513
yacc_header.mark_used(&"test_files".to_string());
506-
// We need to add HeaderValue::String or glob mechanism for this.
507514
let test_glob = yacc_header.get("test_files");
508-
if let Some(_test_glob) = test_glob {
509-
// We need to do something with testglob, read input files etc
510-
// then read some input files, etc.
511-
let placeholder_input = String::new();
512-
let l: LRNonStreamingLexer<LexerTypesT> =
513-
lexerdef.lexer(&placeholder_input);
514-
for e in rtpb.parse_noaction(&l) {
515-
Err(e.to_string())?
515+
match test_glob {
516+
Some(HeaderValue(_, Value::Setting(Setting::String(test_files, _)))) => {
517+
let path_joined = grm_path.parent().unwrap().join(test_files);
518+
for path in
519+
glob(&path_joined.to_string_lossy()).map_err(|e| e.to_string())?
520+
{
521+
let path = path?;
522+
let input = fs::read_to_string(&path)?;
523+
let l: LRNonStreamingLexer<LexerTypesT> = lexerdef.lexer(&input);
524+
for e in rtpb.parse_noaction(&l) {
525+
Err(format!("parsing {}: {}", path.display(), e))?
526+
}
527+
}
528+
Ok(())
516529
}
530+
Some(_) => Err("Invalid value for setting 'test_files'".into()),
531+
None => Ok(()),
517532
}
518-
Ok(())
519533
},
520534
));
521535
ctp = lrcfg(ctp);

‎lrpar/cttests/src/grmtools_section.test

+9-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
grammar: |
22
%grmtools{yacckind: Grmtools}
3-
%token MAGIC IDENT NUM
3+
%token MAGIC IDENT NUM STRING
44
%epp MAGIC "%grmtools"
55
%%
66
start -> Result<Header<Span>, Vec<HeaderError<Span>>>
@@ -101,6 +101,13 @@ grammar: |
101101
let n = str::parse::<u64>($lexer.span_str(num_span));
102102
Setting::Num(n.expect("convertible"), num_span)
103103
}
104+
| STRING {
105+
let string_span = $1.as_ref().unwrap().span();
106+
// Trim the leading and trailing " characters.
107+
let string_span = Span::new(string_span.start() + 1, string_span.end() - 1);
108+
let s = $lexer.span_str(string_span).to_string();
109+
Setting::String(s, string_span)
110+
}
104111
| namespaced '(' namespaced ')' { Setting::Constructor{ctor: $1, arg: $3} }
105112
;
106113

@@ -140,4 +147,5 @@ lexer: |
140147
\) ')'
141148
:: '::'
142149
: ':'
150+
\"(\\.|[^"\\])*\" 'STRING'
143151
[\n\t\ ] ;

‎lrpar/examples/calc_actions/src/calc.y

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
%grmtools {yacckind: Grmtools}
1+
%grmtools {
2+
yacckind: Grmtools,
3+
test_files: "input.txt",
4+
}
25
%start Expr
36
%avoid_insert "INT"
47
%%

‎lrpar/examples/calc_actions/src/input.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
5 + 4 * 3

‎lrpar/examples/calc_ast/src/calc.y

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
%grmtools {yacckind: Grmtools}
1+
%grmtools {
2+
yacckind: Grmtools,
3+
test_files: "input.txt",
4+
}
25
%start Expr
36
%avoid_insert "INT"
47
%expect-unused Unmatched "UNMATCHED"

‎lrpar/examples/calc_ast/src/input.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
5 + 4 * 3

‎lrpar/examples/calc_parsetree/src/calc.y

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
%grmtools{yacckind: Original(GenericParseTree)}
1+
%grmtools{
2+
yacckind: Original(GenericParseTree),
3+
test_files: "input.txt",
4+
}
25
%start Expr
36
%avoid_insert "INT"
47
%%

‎lrpar/examples/calc_parsetree/src/input.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
5 + 4 * 3

‎lrpar/examples/clone_param/src/input.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
0++++---

‎lrpar/examples/clone_param/src/param.l

+1
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@
33
\- "Decr"
44
\+ "Incr"
55
[\n\t\ ] ;
6+
. 'UNMATCHED'

‎lrpar/examples/clone_param/src/param.y

+14-6
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,22 @@
1-
%grmtools {yacckind: Grmtools}
1+
%grmtools {
2+
yacckind: Grmtools,
3+
test_files: "input.txt",
4+
}
5+
%expect-unused Unmatched "UNMATCHED"
26
%token Incr Decr
37
%parse-param val: Rc<RefCell<i64>>
48
%%
59
Expr -> () : "INT" Ops {
6-
*val.borrow_mut() += parse_int($lexer.span_str($1.map_err(|_| "<evaluation aborted>").unwrap().span())).unwrap()
7-
};
10+
*val.borrow_mut() += parse_int($lexer.span_str($1.map_err(|_| "<evaluation aborted>").unwrap().span())).unwrap()
11+
};
812
Ops -> ():
9-
%empty {}
10-
| Ops Incr { *val.borrow_mut() += 1; }
11-
| Ops Decr { *val.borrow_mut() -= 1; };
13+
%empty {}
14+
| Ops Incr { *val.borrow_mut() += 1; }
15+
| Ops Decr { *val.borrow_mut() -= 1; }
16+
;
17+
Unmatched -> ():
18+
"UNMATCHED" { }
19+
;
1220
%%
1321
use std::{ rc::Rc, cell::RefCell, error::Error };
1422

‎lrpar/examples/start_states/src/comment.y

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
%grmtools{yacckind: Original(GenericParseTree)}
1+
%grmtools{
2+
yacckind: Original(GenericParseTree),
3+
test_files: "input.txt",
4+
}
25
%start Expr
36
%%
47
Expr: Expr Text | ;

‎lrpar/examples/start_states/src/input.txt

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
coment /* */
2+
nested comment /* /* */ */

‎lrpar/src/lib/ctbuilder.rs

+3-1
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,7 @@ where
248248
&'b mut Header<Location>,
249249
RTParserBuilder<LexerTypesT::StorageT, LexerTypesT>,
250250
&'b HashMap<String, LexerTypesT::StorageT>,
251+
&PathBuf,
251252
) -> Result<(), Box<dyn Error>>,
252253
>,
253254
>,
@@ -448,6 +449,7 @@ where
448449
&'b mut Header<Location>,
449450
RTParserBuilder<'y, StorageT, LexerTypesT>,
450451
&'b HashMap<String, StorageT>,
452+
&PathBuf,
451453
) -> Result<(), Box<dyn Error>>,
452454
>,
453455
) -> Self {
@@ -762,7 +764,7 @@ where
762764
} else {
763765
rt
764766
};
765-
inspector_rt(&mut header, rt, &rule_ids)?
767+
inspector_rt(&mut header, rt, &rule_ids, grmp)?
766768
}
767769

768770
let unused_keys = header.unused();

‎lrpar/src/lib/parser.rs

+7
Original file line numberDiff line numberDiff line change
@@ -670,6 +670,13 @@ impl TryFrom<&Value<Location>> for RecoveryKind {
670670
),
671671
locations: vec![loc.clone()],
672672
}),
673+
Value::Setting(Setting::String(_, loc)) => Err(HeaderError {
674+
kind: HeaderErrorKind::ConversionError(
675+
"RecoveryKind",
676+
"Cannot convert string to RecoveryKind",
677+
),
678+
locations: vec![loc.clone()],
679+
}),
673680
Value::Setting(Setting::Unitary(Namespaced {
674681
namespace,
675682
member: (kind, kind_loc),

0 commit comments

Comments
 (0)