Merge pull request #1475 from betagouv/fix/policy

Update policy engine

.github/workflows/analysis-test-integration.yml

@@ -0,0 +1,21 @@
+name: 'Analysis - Integration test'
+
+on:
+  pull_request:
+    branches: [dev]
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  e2e:
+    name: Test integration
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Start
+        run: sh integration.sh
+

README.md

@@ -76,9 +76,9 @@ bash e2e.sh
 
 ```shell
 # local e2e
+vi .env (set your database to 'test')
 docker-compose up api (worker is started too)
 docker-compose up dashboard
-vi .env (set your database to 'test')
 docker-compose run api yarn workspace @pdc/proxy ilos seed
 cd tests
 yarn

api/db/migrations/20200325000000-update-trip-views.js

@@ -6,8 +6,8 @@
  */
 var { createMigration } = require('../helpers/createMigration');
 var { setup, up, down } = createMigration([
-  'trip/202003250000000_update_trip_view',
-  'trip/202003250000000_update_export_view',
+  'trip/20200325000000_update_trip_view',
+  'trip/20200325000000_update_export_view',
 ], __dirname);
 
 exports.setup = setup;

api/db/migrations/20200325000000-update_trip_views.js

@@ -6,8 +6,8 @@
  */
 var { createMigration } = require('../helpers/createMigration');
 var { setup, up, down } = createMigration([
-  'trip/202003250000000_update_trip_view',
-  'trip/202003250000000_update_export_view',
+  'trip/20200325000000_update_trip_view',
+  'trip/20200325000000_update_export_view',
 ], __dirname);
 
 exports.setup = setup;

api/db/migrations/20210901000000-update_policy_tables.js

@@ -0,0 +1,17 @@
+'use strict';
+/**
+ * Cast all foreign keys *_id as integer to match PostgreSQL types
+ * Current type is 'varchar' as fkeys were migrated from MongoDB
+ * as a toString() of ObjectID objects.
+ */
+var { createMigration } = require('../helpers/createMigration');
+
+var { setup, up, down } = createMigration([
+    'policy/20210901000000_update_policy_meta',
+    'policy/20210901000000_update_policy_trip_view',
+    'trip/20210901000000_add_territory_index'
+], __dirname);
+
+exports.setup = setup;
+exports.up = up;
+exports.down = down;

api/db/migrations/20210930135857-create_policy_trips_view.js

@@ -0,0 +1,8 @@
+'use strict';
+
+var { createMigration } = require('../helpers/createMigration');
+var { setup, up, down } = createMigration(['policy/20210930135857_create_policy_trips_view'], __dirname);
+
+exports.setup = setup;
+exports.up = up;
+exports.down = down;

api/db/migrations/20211005110112-fix_hydrate_from_policy.js

@@ -0,0 +1,8 @@
+'use strict';
+
+var { createMigration } = require('../helpers/createMigration');
+var { setup, up, down } = createMigration(['trip/20211005110112-fix_hydrate_from_policy'], __dirname);
+
+exports.setup = setup;
+exports.up = up;
+exports.down = down;

api/db/migrations/policy/20210901000000_update_policy_meta.down.sql

@@ -0,0 +1,7 @@
+ALTER TABLE policy.policy_metas
+    DROP COLUMN datetime,
+    ALTER COLUMN value TYPE JSON
+     USING value::text::json;
+DROP INDEX IF EXISTS policy.policy_meta_id_key;
+CREATE UNIQUE INDEX policy_meta_unique_key ON policy.policy_metas (policy_id, key);
+

api/db/migrations/policy/20210901000000_update_policy_meta.up.sql

@@ -0,0 +1,10 @@
+ALTER TABLE policy.policy_metas
+    ADD COLUMN datetime timestamp,
+    ALTER COLUMN value TYPE INT 
+     USING CASE 
+        WHEN value::text = '{}' THEN 0 
+        ELSE value::text::int
+     END;
+DROP INDEX IF EXISTS policy.policy_meta_unique_key;
+CREATE INDEX policy_meta_id_key ON policy.policy_metas (policy_id, key);
+CREATE INDEX policy_meta_incentive ON policy.policy_metas (datetime);

api/db/migrations/policy/20210901000000_update_policy_trip_view.down.sql

@@ -0,0 +1,80 @@
+CREATE EXTENSION IF NOT EXISTS intarray;
+DROP MATERIALIZED VIEW IF EXISTS policy.trips;
+
+CREATE MATERIALIZED VIEW policy.trips AS (
+  SELECT
+    cp._id as carpool_id,
+    cp.status as carpool_status,
+    cp.trip_id as trip_id,
+    tsc.value[1] as start_insee,
+    tec.value[1] as end_insee,
+    cp.operator_id::int as operator_id,
+    cp.operator_class as operator_class,
+    cp.datetime as datetime, 
+    cp.seats as seats,
+    cp.cost as cost,
+    cp.is_driver as is_driver,
+    (CASE WHEN cp.distance IS NOT NULL THEN cp.distance ELSE (cp.meta::json->>'calc_distance')::int END) as distance,
+    (CASE WHEN cp.duration IS NOT NULL THEN cp.duration ELSE (cp.meta::json->>'calc_duration')::int END) as duration,
+    id.identity_uuid as identity_uuid,
+    id.has_travel_pass as has_travel_pass,
+    id.is_over_18 as is_over_18,
+    ats || cp.start_territory_id as start_territory_id,
+    ate || cp.end_territory_id as end_territory_id,
+    ap.applicable_policies as applicable_policies,
+    pp.processed_policies as processed_policies,
+    (ap.applicable_policies - pp.processed_policies) as processable_policies
+  FROM carpool.carpools as cp
+  LEFT JOIN territory.get_ancestors(ARRAY[cp.start_territory_id]) as ats ON TRUE
+  LEFT JOIN territory.get_ancestors(ARRAY[cp.end_territory_id]) as ate ON TRUE,
+  LATERAL (
+    SELECT
+      array_agg(value) as value
+    FROM territory.territory_codes
+    WHERE territory_id = cp.start_territory_id
+    AND type = 'insee'
+  ) as tsc,
+  LATERAL (
+    SELECT
+      array_agg(value) as value
+    FROM territory.territory_codes
+    WHERE territory_id = cp.end_territory_id
+    AND type = 'insee'
+  ) as tec,
+  -- Find all policies that appliable to carpool
+  LATERAL (
+    SELECT
+      COALESCE(array_agg(pp._id), ARRAY[]::int[]) as applicable_policies
+    FROM policy.policies as pp
+    WHERE
+      pp.territory_id = any(cp.start_territory_id || ats || ate || cp.end_territory_id)
+      AND pp.start_date <= cp.datetime
+      AND pp.end_date >= cp.datetime
+      AND pp.status = 'active'
+  ) as ap,
+  -- Find all already processed policies
+  LATERAL (
+    SELECT
+      COALESCE(array_agg(pi.policy_id), ARRAY[]::int[]) as processed_policies
+    FROM policy.incentives as pi
+    WHERE
+      pi.carpool_id::int = cp._id
+  ) as pp,
+  -- Find identity relative data
+  LATERAL (
+    SELECT
+      (CASE WHEN ci.travel_pass_user_id IS NOT NULL THEN true ELSE false END) as has_travel_pass,
+      (CASE WHEN ci.over_18 IS NOT NULL THEN ci.over_18 ELSE null END) as is_over_18,
+      ci.uuid as identity_uuid
+    FROM carpool.identities as ci
+    WHERE
+      cp.identity_id = ci._id
+  ) as id
+  WHERE cp.datetime >= (NOW() - interval '140 days') AND cp.datetime < (NOW() - interval '5 days')
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS trips_carpool_id_idx ON policy.trips (carpool_id);
+CREATE INDEX IF NOT EXISTS trips_datetime_idx ON policy.trips (datetime);
+CREATE INDEX IF NOT EXISTS trips_trip_id_idx ON policy.trips (trip_id);
+CREATE INDEX IF NOT EXISTS trips_applicable_policies_idx ON policy.trips (applicable_policies);
+CREATE INDEX IF NOT EXISTS trips_processable_policies_idx ON policy.trips (processable_policies);

api/db/migrations/policy/20210901000000_update_policy_trip_view.up.sql

@@ -0,0 +1,28 @@
+CREATE EXTENSION IF NOT EXISTS intarray;
+DROP MATERIALIZED VIEW IF EXISTS policy.trips;
+
+CREATE VIEW policy.trips AS (
+  SELECT
+    cp._id as carpool_id,
+    cp.status as carpool_status,
+    cp.trip_id as trip_id,
+    cp.acquisition_id as acquisition_id,
+    cp.operator_id::int as operator_id,
+    cp.operator_class as operator_class,
+    cp.datetime as datetime, 
+    cp.seats as seats,
+    cp.cost as cost,
+    cp.is_driver as is_driver,
+    (CASE WHEN cp.distance IS NOT NULL THEN cp.distance ELSE (cp.meta::json->>'calc_distance')::int END) as distance,
+    (CASE WHEN cp.duration IS NOT NULL THEN cp.duration ELSE (cp.meta::json->>'calc_duration')::int END) as duration,
+    (CASE WHEN ci.travel_pass_user_id IS NOT NULL THEN true ELSE false END) as has_travel_pass,
+    (CASE WHEN ci.over_18 IS NOT NULL THEN ci.over_18 ELSE null END) as is_over_18,
+    ci.uuid as identity_uuid,
+    ats || cp.start_territory_id as start_territory_id,
+    ate || cp.end_territory_id as end_territory_id
+  FROM carpool.carpools as cp
+  LEFT JOIN territory.get_ancestors(ARRAY[cp.start_territory_id]) as ats ON TRUE
+  LEFT JOIN territory.get_ancestors(ARRAY[cp.end_territory_id]) as ate ON TRUE
+  LEFT JOIN carpool.identities as ci
+   ON cp.identity_id = ci._id
+);

api/db/migrations/policy/20210930135857_create_policy_trips_view.down.sql

@@ -0,0 +1,28 @@
+CREATE EXTENSION IF NOT EXISTS intarray;
+DROP VIEW IF EXISTS policy.trips;
+
+CREATE VIEW policy.trips AS (
+  SELECT
+    cp._id as carpool_id,
+    cp.status as carpool_status,
+    cp.trip_id as trip_id,
+    cp.acquisition_id as acquisition_id,
+    cp.operator_id::int as operator_id,
+    cp.operator_class as operator_class,
+    cp.datetime as datetime, 
+    cp.seats as seats,
+    cp.cost as cost,
+    cp.is_driver as is_driver,
+    (CASE WHEN cp.distance IS NOT NULL THEN cp.distance ELSE (cp.meta::json->>'calc_distance')::int END) as distance,
+    (CASE WHEN cp.duration IS NOT NULL THEN cp.duration ELSE (cp.meta::json->>'calc_duration')::int END) as duration,
+    (CASE WHEN ci.travel_pass_user_id IS NOT NULL THEN true ELSE false END) as has_travel_pass,
+    (CASE WHEN ci.over_18 IS NOT NULL THEN ci.over_18 ELSE null END) as is_over_18,
+    ci.uuid as identity_uuid,
+    ats || cp.start_territory_id as start_territory_id,
+    ate || cp.end_territory_id as end_territory_id
+  FROM carpool.carpools as cp
+  LEFT JOIN territory.get_ancestors(ARRAY[cp.start_territory_id]) as ats ON TRUE
+  LEFT JOIN territory.get_ancestors(ARRAY[cp.end_territory_id]) as ate ON TRUE
+  LEFT JOIN carpool.identities as ci
+   ON cp.identity_id = ci._id
+);

api/db/migrations/policy/20210930135857_create_policy_trips_view.up.sql

@@ -0,0 +1,29 @@
+CREATE EXTENSION IF NOT EXISTS intarray;
+DROP VIEW IF EXISTS policy.trips;
+
+CREATE VIEW policy.trips AS (
+  SELECT
+    cp._id as carpool_id,
+    cp.status as carpool_status,
+    cp.trip_id as trip_id,
+    cp.acquisition_id as acquisition_id,
+    cp.operator_id::int as operator_id,
+    cp.operator_class as operator_class,
+    cp.datetime as datetime, 
+    cp.start_territory_id as start_territory_id,
+    cp.end_territory_id as end_territory_id,
+    cp.seats as seats,
+    cp.cost as cost,
+    cp.is_driver as is_driver,
+    (CASE WHEN cp.distance IS NOT NULL THEN cp.distance ELSE (cp.meta::json->>'calc_distance')::int END) as distance,
+    (CASE WHEN cp.duration IS NOT NULL THEN cp.duration ELSE (cp.meta::json->>'calc_duration')::int END) as duration,
+    (CASE WHEN ci.travel_pass_user_id IS NOT NULL THEN true ELSE false END) as has_travel_pass,
+    (CASE WHEN ci.over_18 IS NOT NULL THEN ci.over_18 ELSE null END) as is_over_18,
+    ci.uuid as identity_uuid,
+    tcs.value AS start_insee,
+    tce.value AS end_insee
+  FROM carpool.carpools as cp
+  LEFT JOIN carpool.identities as ci ON cp.identity_id = ci._id
+  LEFT JOIN territory.territory_codes tcs ON cp.start_territory_id = tcs.territory_id AND tcs.type::text = 'insee'::text
+  LEFT JOIN territory.territory_codes tce ON cp.end_territory_id = tce.territory_id AND tce.type::text = 'insee'::text
+);

api/db/migrations/trip/20210901000000_add_territory_index.down.sql

@@ -0,0 +1,2 @@
+DROP INDEX IF EXISTS trip.start_territory_id_idx;
+DROP INDEX IF EXISTS trip.end_territory_id_idx;

api/db/migrations/trip/20210901000000_add_territory_index.up.sql

@@ -0,0 +1,2 @@
+CREATE INDEX start_territory_id_idx ON carpool.carpools (start_territory_id);
+CREATE INDEX end_territory_id_idx ON carpool.carpools (end_territory_id);