refactor external_source_event_type to a view

db-tests/src/test/java/gov/nasa/jpl/aerie/database/ExternalEventTests.java

@@ -0,0 +1,86 @@
+package gov.nasa.jpl.aerie.database;
+
+import gov.nasa.jpl.aerie.database.PlanCollaborationTests.Activity;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.postgresql.util.PGInterval;
+import org.postgresql.util.PGobject;
+
+import javax.swing.plaf.basic.BasicInternalFrameTitlePane;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.SQLException;
+import java.time.Duration;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+@SuppressWarnings("SqlSourceToSinkFlow")
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class ExternalEventTests {
+  private DatabaseTestHelper helper;
+  private MerlinDatabaseTestHelper merlinHelper;
+  private Connection connection;
+  private int planId;
+
+  @BeforeEach
+  void beforeEach() throws SQLException {
+    int fileId = merlinHelper.insertFileUpload();
+    int missionModelId = merlinHelper.insertMissionModel(fileId);
+    planId = merlinHelper.insertPlan(missionModelId);
+  }
+
+  @AfterEach
+  void afterEach() throws SQLException {
+    helper.clearSchema("merlin");
+  }
+
+  @BeforeAll
+  void beforeAll() throws SQLException, IOException, InterruptedException {
+    helper = new DatabaseTestHelper("external_event", "Activity Directive Changelog Tests");
+    connection = helper.connection();
+    merlinHelper = new MerlinDatabaseTestHelper(connection);
+  }
+
+  @AfterAll
+  void afterAll() throws SQLException, IOException, InterruptedException {
+    helper.close();
+    connection = null;
+    helper = null;
+  }
+
+  private record ExternalEvent(int id, String key, int event_type_id, int source_id, Date start_time, PGInterval duration, String properties){}
+  private record ExternalSource(int id, String key, int file_id, int source_type_id, int derivation_group_id, Date valid_at, Date start_time, Date end_time, Date created_at, String metadata){}
+  private record ExternalEventType(int id, String name){}
+  private record ExternalSourceType(int id, String name){}
+  private record DerivationGroup(int id, String name, int source_type_id){}
+  private record DerivedExternalEvent(int source_id, int file_id, int event_id, String event_key, int event_type_id, int derivation_group_id, Date start_time, String source_range, String valid_at){}
+
+  // to test:
+  //  - upload file - creates source, creates dg, creates events, creates source type, creates event types
+  //  - delete file - should auto clear everything. add two sources, check everything still there after first deletion, then it disappears after second deletion
+  //  - derived events cases:
+  //      + rule 1
+  //      + rule 2
+  //      + rule 3
+  //      + rule 4
+  // - test duplicate events by constraint
+  @Test
+  void uploadFile() throws SQLException {
+    try (final var statement = connection.createStatement()) {
+       final var res = statement.executeQuery(
+          // language=sql
+          """
+          SELECT * FROM merlin.external_event;
+          """
+      );
+      res.next();
+       System.out.println(res.getInt("id"));
+    }
+  }
+}

deployment/hasura/metadata/databases/tables/merlin/external_source_event_type.yaml

@@ -1,15 +1,17 @@
 table:
-  name: external_source_event_types
+  name: external_source_event_type
   schema: merlin
 configuration:
   custom_name: "external_source_event_type"
 object_relationships:
   - name: external_source
     using:
-      foreign_key_constraint_on: external_source_id
-  - name: external_event_type
-    using:
-      foreign_key_constraint_on: external_event_type_id
+      manual_configuration:
+        remote_table:
+          name: external_source
+          schema: merlin
+        column_mapping:
+          external_source_id: id
 select_permissions:
   - role: aerie_admin
     permission:
@@ -26,26 +28,3 @@ select_permissions:
       columns: '*'
       filter: {}
       allow_aggregations: true
-insert_permissions:
-  - role: aerie_admin
-    permission:
-      columns: [external_source_id, external_event_type_id]
-      check: {}
-  - role: user
-    permission:
-      columns: [external_source_id, external_event_type_id]
-      check: {}
-update_permissions:
-  - role: aerie_admin
-    permission:
-      columns: [external_source_id, external_event_type_id]
-      filter: {}
-#  - role: user # justified???
-#    permission:
-#      columns: [valid_at, metadata]
-#      filter: {}
-delete_permissions:
-  - role: aerie_admin
-    permission:
-      filter: {}
-  # user deletions too???

deployment/hasura/migrations/Aerie/10_derived_events/up.sql

@@ -78,33 +78,31 @@ ALTER VIEW IF EXISTS merlin.derived_events OWNER TO aerie;
 -- Create a view that contains a comprehensive listing of information for each derivation group
 CREATE OR REPLACE VIEW merlin.derivation_group_comp
  AS
- SELECT derivation_group.id,
-    derivation_group.name,
-    derivation_group.source_type_id,
-    array_agg(DISTINCT concat(sources.source_id, ', ', sources.key, ', ', sources.contained_events)) AS sources,
-    array_agg(DISTINCT sources.type) AS event_types,
-    count(DISTINCT c.event_key) AS derived_total
-   FROM merlin.derivation_group
-     JOIN ( SELECT external_source.id AS source_id,
-            external_source.key,
-            count(a.event_key) AS contained_events,
-            unnest(b.types) AS type,
-            external_source.derivation_group_id,
-            external_source.valid_at
-           FROM merlin.external_source
-             JOIN ( SELECT external_event.source_id,
-                    external_event.key AS event_key
-                   FROM merlin.external_event) a ON a.source_id = external_source.id
-             JOIN ( SELECT external_event.source_id,
-                    array_agg(external_event_type.name) AS types
-                   FROM merlin.external_event
-                     JOIN merlin.external_event_type ON external_event_type.id = external_event.event_type_id
-                  GROUP BY external_event.source_id) b ON b.source_id = external_source.id
-          GROUP BY external_source.id, external_source.key, external_source.derivation_group_id, external_source.valid_at, b.types) sources ON sources.derivation_group_id = derivation_group.id
-     JOIN ( SELECT derived_events.event_key,
-            derived_events.derivation_group_id
-           FROM merlin.derived_events) c ON c.derivation_group_id = derivation_group.id
-  GROUP BY derivation_group.id, derivation_group.name, derivation_group.source_type_id;
+SELECT derivation_group.id,
+   derivation_group.name,
+   derivation_group.source_type_id,
+   array_agg(DISTINCT concat(sources.source_id, ', ', sources.key, ', ', sources.contained_events)) AS sources,
+   array_agg(DISTINCT sources.type) AS event_types,
+   count(DISTINCT c.event_key) AS derived_total
+  FROM merlin.derivation_group
+    JOIN ( SELECT external_source.id AS source_id,
+           external_source.key,
+           count(a.event_key) AS contained_events,
+           unnest(b.types) AS type,
+           external_source.derivation_group_id,
+           external_source.valid_at
+          FROM merlin.external_source
+            JOIN ( SELECT external_event.source_id,
+                   external_event.key AS event_key
+                  FROM merlin.external_event) a ON a.source_id = external_source.id
+            JOIN ( SELECT external_source_id AS source_id,
+                   event_types as types
+                  FROM merlin.external_source_event_type) b ON b.source_id = external_source.id
+         GROUP BY external_source.id, external_source.key, external_source.derivation_group_id, external_source.valid_at, b.types) sources ON sources.derivation_group_id = derivation_group.id
+    JOIN ( SELECT derived_events.event_key,
+           derived_events.derivation_group_id
+          FROM merlin.derived_events) c ON c.derivation_group_id = derivation_group.id
+ GROUP BY derivation_group.id, derivation_group.name, derivation_group.source_type_id;
 
 ALTER VIEW IF EXISTS merlin.derivation_group_comp OWNER TO aerie;
 COMMENT ON VIEW merlin.derivation_group_comp

deployment/hasura/migrations/Aerie/9_external_events/down.sql

@@ -4,6 +4,5 @@ DROP TABLE merlin.external_event CASCADE;
 DROP TABLE merlin.plan_derivation_group CASCADE;
 DROP TABLE merlin.external_source_type CASCADE;
 DROP TABLE merlin.external_event_type CASCADE;
-DROP TABLE merlin.external_source_event_types CASCADE;
 
 call migrations.mark_migration_rolled_back('9');

deployment/hasura/migrations/Aerie/9_external_events/up.sql

@@ -260,37 +260,18 @@ ALTER TABLE ONLY merlin.external_event
 
 
 -- Create table mapping external sources to their contained event types
-CREATE TABLE merlin.external_source_event_types (
-    id integer NOT NULL,
-    external_source_id integer NOT NULL,
-    external_event_type_id integer NOT NULL
-);
-
-COMMENT ON TABLE merlin.external_source_event_types IS 'A table detailing the event types that a given external_source has.';
-
--- Ensure the id is serial.
-CREATE SEQUENCE merlin.external_source_event_types_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
-ALTER SEQUENCE merlin.external_source_event_types_id_seq OWNED BY merlin.external_source_event_types.id;
-ALTER TABLE ONLY merlin.external_source_event_types ALTER COLUMN id SET DEFAULT nextval('merlin.external_source_event_types_id_seq'::regclass);
-
--- Set primary key
-ALTER TABLE ONLY merlin.external_source_event_types
-    ADD CONSTRAINT external_source_event_type_pkey PRIMARY KEY (id);
-
--- Add key for external_event_type_id
-ALTER TABLE ONLY merlin.external_source_event_types
-    ADD CONSTRAINT external_event_type_id FOREIGN KEY (external_event_type_id) REFERENCES merlin.external_event_type(id);
-
--- Add key for external_source_id
-ALTER TABLE ONLY merlin.external_source_event_types
-    ADD CONSTRAINT external_source_id FOREIGN KEY (external_source_id) REFERENCES merlin.external_source(id);
+CREATE OR REPLACE VIEW merlin.external_source_event_type
+ AS
+ SELECT external_source.id AS external_source_id,
+    array_agg(DISTINCT external_event.event_type_id) AS event_type_ids,
+    array_agg(DISTINCT external_event_type.name) AS event_types
+   FROM merlin.external_source
+     JOIN merlin.external_event ON external_event.source_id = external_source.id
+     JOIN merlin.external_event_type ON external_event_type.id = external_event.event_type_id
+  GROUP BY external_source.id;
+
+ALTER VIEW IF EXISTS merlin.external_source_event_type OWNER TO aerie;
+COMMENT ON VIEW merlin.external_source_event_type
+    IS 'Should be deprecated with the introduction of strict external source schemas, dictating allowable event types for given source types. But for now, this will do.';
 
 call migrations.mark_migration_applied('9');

deployment/postgres-init-db/sql/merlin/tables/external_event.sql

@@ -0,0 +1,5 @@
+create table external_event(
+  id integer generated always as identity check ( id > 0 ),
+  name text not null
+
+);