internal/core/adt: adjustments to closedness algo

Embedding within a struct that refer to a
definition close that struct. We need to therefore
make a distinction between fields that are not
within the embedding and in the same struct
(always allowed) or outside that struct (may not
be allowed).

Before we tracked this with a single inclusion
list of IDs, which meant that an embedding would
have to include all IDs from nodes that are
outside of its outer struct scope. This was
complicated to manage.

Now we track each embedding and outer scope
separately, effectively having the main groups
and separate allow and deny lists. This means
we now track more groups, but the groups will
be smaller. In future CLs we can now more
easily filter unnecessary groups, though, for
instance if we have structs with a single
embedding.

Fixes #3920
Issue #3934

Signed-off-by: Marcel van Lohuizen <[email protected]>
Change-Id: Ia925281624280d04a10587b089884f9a83588b95
Dispatch-Trailer: {"type":"trybot","CL":1216387,"patchset":3,"ref":"refs/changes/87/1216387/3","targetBranch":"master"}

Author: mpvl

cue/testdata/benchmarks/disjunctelim.txtar

@@ -50,8 +50,8 @@ Unifications: 22
 Conjuncts:    843
 Disjuncts:    618
 
-CloseIDElems: 63
-NumCloseIDs: 4
+CloseIDElems: 54
+NumCloseIDs: 3
 -- diff/-out/evalalpha/stats<==>+out/eval/stats --
 diff old new
 --- old
@@ -74,8 +74,8 @@ diff old new
 +Conjuncts:    843
 +Disjuncts:    618
 +
-+CloseIDElems: 63
-+NumCloseIDs: 4
++CloseIDElems: 54
++NumCloseIDs: 3
 -- out/eval/stats --
 Leaks:  0
 Freed:  367

cue/testdata/benchmarks/inlinedisjunction.txtar

@@ -24,7 +24,7 @@ Unifications: 16
 Conjuncts:    113
 Disjuncts:    58
 
-CloseIDElems: 10
+CloseIDElems: 22
 NumCloseIDs: 3
 -- out/eval --
 (struct){
@@ -66,7 +66,7 @@ diff old new
 +Conjuncts:    113
 +Disjuncts:    58
 +
-+CloseIDElems: 10
++CloseIDElems: 22
 +NumCloseIDs: 3
 -- out/eval/stats --
 Leaks:  0

cue/testdata/benchmarks/issue2176.txtar

@@ -68,13 +68,13 @@ Unifications: 377
 Conjuncts:    1333
 Disjuncts:    8
 
-CloseIDElems: 135
-NumCloseIDs: 537
+CloseIDElems: 20071
+NumCloseIDs: 535
 -- diff/-out/evalalpha/stats<==>+out/eval/stats --
 diff old new
 --- old
 +++ new
-@@ -1,9 +1,12 @@
+@@ -1,12 +1,12 @@
 -Leaks:  90
 -Freed:  746
 -Reused: 742
@@ -84,6 +84,9 @@ diff old new
 -Unifications: 828
 -Conjuncts:    3306
 -Disjuncts:    1475
+-
+-CloseIDElems: 0
+-NumCloseIDs: 12
 +Leaks:  449
 +Freed:  0
 +Reused: 0
@@ -94,8 +97,8 @@ diff old new
 +Conjuncts:    1333
 +Disjuncts:    8
 +
-+CloseIDElems: 135
-+NumCloseIDs: 537
++CloseIDElems: 20071
++NumCloseIDs: 535
 -- out/eval/stats --
 Leaks:  90
 Freed:  746
@@ -106,6 +109,9 @@ Retain: 1363
 Unifications: 828
 Conjuncts:    3306
 Disjuncts:    1475
+
+CloseIDElems: 0
+NumCloseIDs: 12
 -- out/evalalpha --
 (struct){
   #Datastream: (#struct){

cue/testdata/benchmarks/issue3307.txtar

@@ -32,6 +32,9 @@ Retain: 72
 Unifications: 21
 Conjuncts:    3
 Disjuncts:    93
+
+CloseIDElems: 0
+NumCloseIDs: 1
 -- out/eval --
 (_|_){
   // [incomplete] incomplete bool: _

cue/testdata/benchmarks/issue3514.txtar

@@ -46,8 +46,8 @@ Unifications: 69
 Conjuncts:    7053
 Disjuncts:    2386
 
-CloseIDElems: 15
-NumCloseIDs: 6876
+CloseIDElems: 82
+NumCloseIDs: 5733
 -- diff/-out/evalalpha/stats<==>+out/eval/stats --
 diff old new
 --- old
@@ -72,8 +72,8 @@ diff old new
 +Conjuncts:    7053
 +Disjuncts:    2386
 +
-+CloseIDElems: 15
-+NumCloseIDs: 6876
++CloseIDElems: 82
++NumCloseIDs: 5733
 -- out/eval/stats --
 Leaks:  27
 Freed:  31092

cue/testdata/benchmarks/issue3633.txtar

@@ -10,8 +10,8 @@ Unifications: 667
 Conjuncts:    2680
 Disjuncts:    1955
 
-CloseIDElems: 4233
-NumCloseIDs: 55
+CloseIDElems: 7225
+NumCloseIDs: 53
 -- diff/-out/evalalpha/stats<==>+out/eval/stats --
 diff old new
 --- old
@@ -34,8 +34,8 @@ diff old new
 +Conjuncts:    2680
 +Disjuncts:    1955
 +
-+CloseIDElems: 4233
-+NumCloseIDs: 55
++CloseIDElems: 7225
++NumCloseIDs: 53
 -- out/eval/stats --
 Leaks:  0
 Freed:  3595

cue/testdata/benchmarks/issue572.txtar

@@ -9,6 +9,44 @@ unified: {
 		"\(entry)": true
 	}
 }
+-- out/evalalpha/stats --
+Leaks:  3001
+Freed:  0
+Reused: 0
+Allocs: 3001
+Retain: 0
+
+Unifications: 3001
+Conjuncts:    7498
+Disjuncts:    0
+
+CloseIDElems: 4500
+NumCloseIDs: 4499
+-- diff/-out/evalalpha/stats<==>+out/eval/stats --
+diff old new
+--- old
++++ new
+@@ -1,12 +1,12 @@
+-Leaks:  0
+-Freed:  3001
+-Reused: 2998
+-Allocs: 3
++Leaks:  3001
++Freed:  0
++Reused: 0
++Allocs: 3001
+ Retain: 0
+ 
+ Unifications: 3001
+ Conjuncts:    7498
+-Disjuncts:    3001
+-
+-CloseIDElems: 0
+-NumCloseIDs: 1
++Disjuncts:    0
++
++CloseIDElems: 4500
++NumCloseIDs: 4499
 -- out/eval/stats --
 Leaks:  0
 Freed:  3001
@@ -19,6 +57,9 @@ Retain: 0
 Unifications: 3001
 Conjuncts:    7498
 Disjuncts:    3001
+
+CloseIDElems: 0
+NumCloseIDs: 1
 -- out/eval --
 (struct){
   _entries(:p): (#list){

cue/testdata/benchmarks/listdedup.txtar

@@ -43,8 +43,8 @@ Unifications: 40
 Conjuncts:    98
 Disjuncts:    24
 
-CloseIDElems: 1224
-NumCloseIDs: 22
+CloseIDElems: 956
+NumCloseIDs: 18
 -- out/evalalpha --
 (struct){
   A: (#struct){ |((#struct){
@@ -486,8 +486,8 @@ diff old new
 +Conjuncts:    98
 +Disjuncts:    24
 +
-+CloseIDElems: 1224
-+NumCloseIDs: 22
++CloseIDElems: 956
++NumCloseIDs: 18
 -- out/eval/stats --
 Leaks:  0
 Freed:  24096

cue/testdata/benchmarks/mergeddisjunction.txtar

@@ -50,6 +50,9 @@ Retain: 0
 Unifications: 99
 Conjuncts:    530
 Disjuncts:    283
+
+CloseIDElems: 0
+NumCloseIDs: 2
 -- out/eval --
 (struct){
   list: (#list){

cue/testdata/benchmarks/sharefg.txtar

@@ -109,6 +109,43 @@ weaveExtraField: {
 	f2: f&{ ref: f1 }
 	f1: { out: 2 }
 }
+-- out/evalalpha/stats --
+Leaks:  452
+Freed:  0
+Reused: 0
+Allocs: 452
+Retain: 0
+
+Unifications: 452
+Conjuncts:    845
+Disjuncts:    0
+
+CloseIDElems: 4039
+NumCloseIDs: 238
+-- diff/-out/evalalpha/stats<==>+out/eval/stats --
+diff old new
+--- old
++++ new
+@@ -1,9 +1,12 @@
+-Leaks:  112
+-Freed:  340
+-Reused: 308
+-Allocs: 144
+-Retain: 428
++Leaks:  452
++Freed:  0
++Reused: 0
++Allocs: 452
++Retain: 0
+ 
+ Unifications: 452
+-Conjuncts:    853
+-Disjuncts:    720
++Conjuncts:    845
++Disjuncts:    0
++
++CloseIDElems: 4039
++NumCloseIDs: 238
 -- out/eval/stats --
 Leaks:  112
 Freed:  340

cue/testdata/benchmarks/typocheck.txtar

@@ -31,13 +31,13 @@ Unifications: 703
 Conjuncts:    1305
 Disjuncts:    0
 
-CloseIDElems: 31712
-NumCloseIDs: 210
+CloseIDElems: 45090
+NumCloseIDs: 212
 -- diff/-out/evalalpha/stats<==>+out/eval/stats --
 diff old new
 --- old
 +++ new
-@@ -1,9 +1,12 @@
+@@ -1,12 +1,12 @@
 -Leaks:  0
 -Freed:  700
 -Reused: 594
@@ -47,6 +47,9 @@ diff old new
 -Unifications: 700
 -Conjuncts:    1302
 -Disjuncts:    807
+-
+-CloseIDElems: 0
+-NumCloseIDs: 4
 +Leaks:  703
 +Freed:  0
 +Reused: 0
@@ -57,8 +60,8 @@ diff old new
 +Conjuncts:    1305
 +Disjuncts:    0
 +
-+CloseIDElems: 31712
-+NumCloseIDs: 210
++CloseIDElems: 45090
++NumCloseIDs: 212
 -- out/eval/stats --
 Leaks:  0
 Freed:  700
@@ -69,6 +72,9 @@ Retain: 108
 Unifications: 700
 Conjuncts:    1302
 Disjuncts:    807
+
+CloseIDElems: 0
+NumCloseIDs: 4
 -- out/eval --
 (struct){
   out: (#struct){

cue/testdata/builtins/056_issue314.txtar

@@ -84,8 +84,8 @@ Unifications: 21
 Conjuncts:    24
 Disjuncts:    0
 
-CloseIDElems: 21
-NumCloseIDs: 4
+CloseIDElems: 18
+NumCloseIDs: 3
 -- diff/-out/evalalpha/stats<==>+out/eval/stats --
 diff old new
 --- old
@@ -110,8 +110,8 @@ diff old new
 +Conjuncts:    24
 +Disjuncts:    0
 +
-+CloseIDElems: 21
-+NumCloseIDs: 4
++CloseIDElems: 18
++NumCloseIDs: 3
 -- out/eval/stats --
 Leaks:  0
 Freed:  45

cue/testdata/builtins/matchn.txtar

@@ -345,6 +345,45 @@ issue3694: simple: {
 		uses?: string
 	})
 }
+-- out/evalalpha/stats --
+Leaks:  1034
+Freed:  0
+Reused: 0
+Allocs: 1034
+Retain: 0
+
+Unifications: 931
+Conjuncts:    1511
+Disjuncts:    16
+
+CloseIDElems: 1227
+NumCloseIDs: 483
+-- diff/-out/evalalpha/stats<==>+out/eval/stats --
+diff old new
+--- old
++++ new
+@@ -1,9 +1,12 @@
+-Leaks:  52
+-Freed:  951
+-Reused: 944
+-Allocs: 59
+-Retain: 129
+-
+-Unifications: 987
+-Conjuncts:    1635
+-Disjuncts:    1081
++Leaks:  1034
++Freed:  0
++Reused: 0
++Allocs: 1034
++Retain: 0
++
++Unifications: 931
++Conjuncts:    1511
++Disjuncts:    16
++
++CloseIDElems: 1227
++NumCloseIDs: 483
 -- out/eval/stats --
 Leaks:  52
 Freed:  951

cue/testdata/builtins/validators.txtar

@@ -221,6 +221,9 @@ Retain: 23
 Unifications: 181
 Conjuncts:    327
 Disjuncts:    202
+
+CloseIDElems: 0
+NumCloseIDs: 8
 -- out/evalalpha --
 Errors:
 issue3474.topValidator.failType.A: conflicting values >10 and {C:1} (mismatched types number and struct):