Procedural Constraints Database Support

deployment/hasura/metadata/actions.graphql

@@ -255,6 +255,7 @@ type ResourceSamplesResponse {
 type ConstraintResponse {
   success: String!
   constraintId: Int!,
+  constraintInvocationId: Int!,
   constraintRevision: Int!,
   constraintName: String!,
   errors: [UserCodeError!]!

deployment/hasura/metadata/databases/tables/merlin/constraints/constraint_definition.yaml

@@ -75,13 +75,13 @@ select_permissions:
 insert_permissions:
   - role: aerie_admin
     permission:
-      columns: [constraint_id, definition]
+      columns: [constraint_id, definition, type, uploaded_jar_id]
       check: {}
       set:
         author: "x-hasura-user-id"
   - role: user
     permission:
-      columns: [constraint_id, definition]
+      columns: [constraint_id, definition, type, uploaded_jar_id]
       check: {"_or":[{"metadata":{"public":{"_eq":true}}},{"metadata":{"owner":{"_eq":"X-Hasura-User-Id"}}}]}
       set:
         author: "x-hasura-user-id"

deployment/hasura/metadata/databases/tables/merlin/constraints/constraint_specification.yaml

@@ -34,11 +34,11 @@ select_permissions:
 insert_permissions:
   - role: aerie_admin
     permission:
-      columns: [plan_id, constraint_id, constraint_revision, enabled]
+      columns: [plan_id, constraint_id, constraint_revision, enabled, arguments]
       check: {}
   - role: user
     permission:
-      columns: [plan_id, constraint_id, constraint_revision, enabled]
+      columns: [plan_id, constraint_id, constraint_revision, enabled, arguments]
       check: { "_and": [
         # User is a plan owner/collaborator
         { "plan": { "_or": [ { "owner": { "_eq": "X-Hasura-User-Id" } },
@@ -50,11 +50,11 @@ insert_permissions:
 update_permissions:
   - role: aerie_admin
     permission:
-      columns: [constraint_revision, enabled]
+      columns: [constraint_revision, enabled, arguments]
       filter: {}
   - role: user
     permission:
-      columns: [constraint_revision, enabled]
+      columns: [constraint_revision, enabled, arguments]
       filter: { "plan": { "_or": [ { "owner": { "_eq": "X-Hasura-User-Id" } },{ "collaborators": { "collaborator": { "_eq": "X-Hasura-User-Id" }}}]}}
 delete_permissions:
   - role: aerie_admin

deployment/hasura/migrations/Aerie/12_procedural_constraints/down.sql

@@ -0,0 +1,63 @@
+-- remove procedural constraints
+
+alter table merlin.constraint_run
+  drop column arguments;
+
+alter table merlin.constraint_specification
+  drop column arguments;
+
+-- delete all procedural constraints from specs (cascade is restricted)
+delete from merlin.constraint_specification cs
+  using merlin.constraint_definition cd
+where cd.constraint_id = cs.constraint_id
+  and cd.type = 'JAR'::merlin.constraint_type;
+
+-- delete all procedural constraint metadata and definitions
+delete from merlin.constraint_metadata cm
+  using merlin.constraint_definition cd
+where cd.constraint_id = cm.id
+  and cd.type = 'JAR'::merlin.constraint_type;
+
+alter table merlin.constraint_definition
+  drop constraint constraint_procedure_has_uploaded_jar,
+  drop constraint check_constraint_definition_type_consistency,
+
+  alter column definition set not null,
+
+  drop column type,
+  drop column uploaded_jar_id,
+  drop column parameter_schema;
+
+drop type merlin.constraint_type;
+
+-- revert constraint invocations
+
+-- remove all but the first invocation of a constraint
+-- we need to revert to a more constrained primary key, so we need to delete data
+-- keeping the first invocation is just a convention
+delete
+from merlin.constraint_specification
+where invocation_id not in (select min(invocation_id)
+                                 from merlin.constraint_specification
+                                 group by plan_id, constraint_id);
+
+alter table merlin.constraint_specification
+  drop constraint constraint_specification_pkey,
+  add constraint constraint_specification_pkey
+    primary key (plan_id, constraint_id),
+
+  drop column invocation_id;
+
+-- similar action with constraint_runs
+delete
+from merlin.constraint_run
+where constraint_run.constraint_invocation_id not in (select min(constraint_invocation_id)
+                                 from merlin.constraint_run
+                                 group by constraint_id, constraint_revision, simulation_dataset_id);
+
+alter table merlin.constraint_run
+  drop constraint constraint_run_key,
+  add constraint constraint_run_key
+    primary key (constraint_id, constraint_revision, simulation_dataset_id),
+
+  drop column constraint_invocation_id;

deployment/hasura/migrations/Aerie/12_procedural_constraints/up.sql

@@ -0,0 +1,63 @@
+-- constraint invocation support
+alter table merlin.constraint_specification
+  add column invocation_id integer generated by default as identity,
+
+  drop constraint constraint_specification_pkey,
+  add constraint constraint_specification_pkey
+    primary key (invocation_id);
+
+comment on column merlin.constraint_specification.invocation_id is e''
+  'The id of a specific constraint invocation in the specification. Primary key.';
+
+-- update constraint_run PK
+alter table merlin.constraint_run
+  -- temp set as nullable so we can insert, made explictly not null below
+  add column constraint_invocation_id integer null,
+
+  drop constraint constraint_run_key;
+
+-- copy invocation_ids from spec
+update merlin.constraint_run cr
+set constraint_invocation_id = cs.invocation_id
+from merlin.constraint_specification cs
+where cs.constraint_id = cr.constraint_id;
+
+-- delete cached constraint runs that aren't referenced by a spec anymore (this isn't ideal)
+delete from merlin.constraint_run where constraint_invocation_id is null;
+
+alter table merlin.constraint_run
+  add constraint constraint_run_key
+    primary key (constraint_invocation_id, simulation_dataset_id);
+
+
+-- procedural constraints support
+create type merlin.constraint_type as enum ('EDSL', 'JAR');
+
+alter table merlin.constraint_definition
+  add column type merlin.constraint_type not null default 'EDSL',
+  add column uploaded_jar_id integer,
+  add column parameter_schema jsonb,
+
+  alter column definition drop not null,
+
+  add constraint constraint_procedure_has_uploaded_jar
+    foreign key (uploaded_jar_id)
+      references merlin.uploaded_file
+      on update cascade
+      on delete restrict,
+
+  add constraint check_constraint_definition_type_consistency
+    check (
+        (type = 'EDSL' and definition is not null and uploaded_jar_id is null)
+        or
+        (type = 'JAR' and uploaded_jar_id is not null and definition is null)
+      );
+
+alter table merlin.constraint_specification
+  add column arguments jsonb not null default '{}'::jsonb;
+
+alter table merlin.constraint_run
+  add column arguments jsonb not null;
+
+
+call migrations.mark_migration_applied('12');

deployment/postgres-init-db/sql/init_merlin.sql

@@ -10,6 +10,7 @@ begin;
   \ir types/merlin/merlin-arguments.sql
   \ir types/merlin/activity-directive-metadata.sql
   \ir types/merlin/plan-merge-types.sql
+  \ir types/merlin/constraint_type.sql
 
   ------------
   -- Tables

deployment/postgres-init-db/sql/tables/merlin/constraints/constraint_definition.sql

@@ -1,7 +1,11 @@
 create table merlin.constraint_definition(
   constraint_id integer not null,
   revision integer not null default 0,
-  definition text not null,
+
+  type merlin.constraint_type not null default 'EDSL',
+  definition text,
+  uploaded_jar_id integer,
+  parameter_schema jsonb,
   author text,
   created_at timestamptz not null default now(),
 
@@ -16,7 +20,18 @@ create table merlin.constraint_definition(
     foreign key (author)
     references permissions.users
     on update cascade
-    on delete set null
+    on delete set null,
+  constraint constraint_procedure_has_uploaded_jar
+    foreign key (uploaded_jar_id)
+      references merlin.uploaded_file
+      on update cascade
+      on delete restrict,
+  constraint check_constraint_definition_type_consistency
+    check (
+      (type = 'EDSL' and definition is not null and uploaded_jar_id is null)
+      or
+      (type = 'JAR' and uploaded_jar_id is not null and definition is null)
+    )
 );
 
 comment on table merlin.constraint_definition is e''

deployment/postgres-init-db/sql/tables/merlin/constraints/constraint_run.sql

@@ -2,6 +2,8 @@ create table merlin.constraint_run (
   constraint_id integer not null,
   constraint_revision integer not null,
   simulation_dataset_id integer not null,
+  constraint_invocation_id integer not null,
+  arguments jsonb not null,
 
   results jsonb not null default '{}',
 
@@ -10,7 +12,7 @@ create table merlin.constraint_run (
   requested_at timestamptz not null default now(),
 
   constraint constraint_run_key
-    primary key (constraint_id, constraint_revision, simulation_dataset_id),
+    primary key (constraint_invocation_id, simulation_dataset_id),
   constraint constraint_run_to_constraint_definition
     foreign key (constraint_id, constraint_revision)
       references merlin.constraint_definition

deployment/postgres-init-db/sql/tables/merlin/constraints/constraint_specification.sql

@@ -1,14 +1,16 @@
 create table merlin.constraint_specification(
+  invocation_id integer generated by default as identity,
   plan_id integer not null
     references merlin.plan
       on update cascade
       on delete cascade,
   constraint_id integer not null,
   constraint_revision integer, -- latest is NULL
   enabled boolean not null default true,
+  arguments jsonb not null default '{}'::jsonb,
 
   constraint constraint_specification_pkey
-    primary key (plan_id, constraint_id),
+    primary key (invocation_id),
   constraint plan_spec_constraint_exists
     foreign key (constraint_id)
       references merlin.constraint_metadata(id)

deployment/postgres-init-db/sql/types/merlin/constraint_type.sql

@@ -0,0 +1 @@
+create type merlin.constraint_type as enum ('EDSL', 'JAR');

e2e-tests/src/test/java/gov/nasa/jpl/aerie/e2e/ConstraintsTests.java

@@ -2,6 +2,7 @@
 
 import com.microsoft.playwright.Playwright;
 import gov.nasa.jpl.aerie.e2e.types.ConstraintError;
+import gov.nasa.jpl.aerie.e2e.types.ConstraintInvocationId;
 import gov.nasa.jpl.aerie.e2e.types.ConstraintRecord;
 import gov.nasa.jpl.aerie.e2e.types.ExternalDataset.ProfileInput;
 import gov.nasa.jpl.aerie.e2e.types.ExternalDataset.ProfileInput.ProfileSegmentInput;
@@ -32,6 +33,7 @@ public class ConstraintsTests {
   private int planId;
   private int activityId;
   private int constraintId;
+  private int invocationId;
 
   private final String constraintName = "fruit_equal_peel";
   private final String constraintDefinition =
@@ -74,11 +76,13 @@ void beforeEach() throws IOException, InterruptedException {
         "1h",
         Json.createObjectBuilder().add("biteSize", 1).build());
     // Insert the Constraint
-    constraintId = hasura.insertPlanConstraint(
+    final var insertResp = hasura.insertPlanConstraint(
         constraintName,
         planId,
         constraintDefinition,
         "");
+    constraintId = insertResp.id();
+    invocationId = insertResp.invocationId();
   }
 
   @AfterEach
@@ -156,7 +160,7 @@ void constraintsSucceedAgainstVariantWithUnits() throws IOException {
     // Check the Response
     final var constraintResponse = constraintsResponses.get(0);
     assertTrue(constraintResponse.success());
-    assertEquals(fruitConstraintId, constraintResponse.constraintId());
+    assertEquals(fruitConstraintId.id(), constraintResponse.constraintId());
     assertEquals(fruitConstraintName, constraintResponse.constraintName());
     // Check the Result
     assertTrue(constraintResponse.result().isPresent());
@@ -358,7 +362,7 @@ void constraintVersionLocking() throws IOException {
     hasura.awaitSimulation(planId);
 
     // Update the plan's constraint specification to use a specific version
-    hasura.updatePlanConstraintSpecVersion(planId, constraintId, 0);
+    hasura.updatePlanConstraintSpecVersion(constraintId, 0);
 
     // Update definition to have invalid syntax
     final int newRevision = hasura.updateConstraintDefinition(
@@ -376,7 +380,7 @@ void constraintVersionLocking() throws IOException {
     assertTrue(initConstraint.result().isPresent());
 
     // Update constraint spec to use invalid definition
-    hasura.updatePlanConstraintSpecVersion(planId, constraintId, newRevision);
+    hasura.updatePlanConstraintSpecVersion(invocationId, newRevision);
 
     // Check constraints -- should fail
     final var badDefinitionResults = hasura.checkConstraints(planId);
@@ -394,7 +398,7 @@ void constraintVersionLocking() throws IOException {
           badConstraint.errors().get(1).message());
 
     // Update constraint spec to use initial definition
-    hasura.updatePlanConstraintSpecVersion(planId, constraintId, 0);
+    hasura.updatePlanConstraintSpecVersion(invocationId, 0);
 
     // Check constraints -- should match
     assertEquals(initResults, hasura.checkConstraints(planId));
@@ -406,13 +410,16 @@ void constraintIgnoreDisabled() throws IOException {
     hasura.awaitSimulation(planId);
     // Add a problematic constraint to the spec, then disable it
     final String problemConstraintName = "bad constraint";
-    final int problemConstraintId = hasura.insertPlanConstraint(
+    final var problemConstraintId = hasura.insertPlanConstraint(
         problemConstraintName,
         planId,
         "error :-(",
         "constraint that shouldn't compile");
+
+    final var invocationId = problemConstraintId.invocationId();
+
     try {
-      hasura.updatePlanConstraintSpecEnabled(planId, problemConstraintId, false);
+      hasura.updatePlanConstraintSpecEnabled(invocationId, false);
 
       // Check constraints -- Validate that only the enabled constraint is included
       final var initResults = hasura.checkConstraints(planId);
@@ -421,7 +428,7 @@ void constraintIgnoreDisabled() throws IOException {
       assertTrue(initResults.get(0).success());
 
       // Enable disabled constraint
-      hasura.updatePlanConstraintSpecEnabled(planId, problemConstraintId, true);
+      hasura.updatePlanConstraintSpecEnabled(invocationId, true);
 
       // Check constraints -- Validate that the other constraint is present and a failure
       final var results = hasura.checkConstraints(planId);
@@ -431,7 +438,7 @@ void constraintIgnoreDisabled() throws IOException {
       assertTrue(results.get(0).success());
 
       final var problemResults = results.get(1);
-      assertEquals(problemConstraintId, problemResults.constraintId());
+      assertEquals(problemConstraintId.id(), problemResults.constraintId());
       assertFalse(problemResults.success());
       assertEquals(problemConstraintName, problemResults.constraintName());
       assertEquals(2, problemResults.errors().size());
@@ -442,7 +449,7 @@ void constraintIgnoreDisabled() throws IOException {
       assertEquals("Constraint 'bad constraint' compilation failed:\n TypeError: TS1109 Expression expected.",
           problemResults.errors().get(1).message());
     } finally {
-      hasura.deleteConstraint(problemConstraintId);
+      hasura.deleteConstraint(problemConstraintId.id());
     }
   }
 

e2e-tests/src/test/java/gov/nasa/jpl/aerie/e2e/types/ConstraintInvocationId.java

@@ -0,0 +1,3 @@
+package gov.nasa.jpl.aerie.e2e.types;
+
+public record ConstraintInvocationId(int id, int invocationId) {}