rsc: Don't track unnecessary visible files (#1612)

rust/entity/src/job.rs

@@ -58,8 +58,6 @@ pub enum Relation {
     OutputFile,
     #[sea_orm(has_many = "super::output_symlink::Entity")]
     OutputSymlink,
-    #[sea_orm(has_many = "super::visible_file::Entity")]
-    VisibleFile,
 }
 
 impl Related<super::job_use::Entity> for Entity {
@@ -86,10 +84,4 @@ impl Related<super::output_symlink::Entity> for Entity {
     }
 }
 
-impl Related<super::visible_file::Entity> for Entity {
-    fn to() -> RelationDef {
-        Relation::VisibleFile.def()
-    }
-}
-
 impl ActiveModelBehavior for ActiveModel {}

rust/entity/src/mod.rs

@@ -12,4 +12,3 @@ pub mod local_blob_store;
 pub mod output_dir;
 pub mod output_file;
 pub mod output_symlink;
-pub mod visible_file;

rust/entity/src/prelude.rs

@@ -10,4 +10,3 @@ pub use super::local_blob_store::Entity as LocalBlobStore;
 pub use super::output_dir::Entity as OutputDir;
 pub use super::output_file::Entity as OutputFile;
 pub use super::output_symlink::Entity as OutputSymlink;
-pub use super::visible_file::Entity as VisibleFile;

rust/migration/src/m20220101_000002_create_table.rs

@@ -70,35 +70,6 @@ impl MigrationTrait for Migration {
             )
             .await?;
 
-        // Here we implicitly create two indexes, one on the job_id and one on the
-        // primary key.
-        manager
-            .create_table(
-                Table::create()
-                    .table(VisibleFile::Table)
-                    .col(
-                        ColumnDef::new(VisibleFile::Id)
-                            .uuid()
-                            .not_null()
-                            .primary_key()
-                            .default(SimpleExpr::FunctionCall(PgFunc::gen_random_uuid())),
-                    )
-                    .col(ColumnDef::new(VisibleFile::Path).string().not_null())
-                    .col(ColumnDef::new(VisibleFile::Hash).string().not_null())
-                    .col(ColumnDef::new(VisibleFile::JobId).uuid().not_null())
-                    .foreign_key(
-                        ForeignKeyCreateStatement::new()
-                            .name("fk-visible_file-job")
-                            .from_tbl(VisibleFile::Table)
-                            .from_col(VisibleFile::JobId)
-                            .to_tbl(Job::Table)
-                            .to_col(Job::Id)
-                            .on_delete(ForeignKeyAction::Cascade),
-                    )
-                    .to_owned(),
-            )
-            .await?;
-
         manager
             .create_table(
                 Table::create()
@@ -209,13 +180,6 @@ impl MigrationTrait for Migration {
                     .to_owned(),
             )
             .await?;
-        manager
-            .drop_table(
-                sea_query::Table::drop()
-                    .table(VisibleFile::Table)
-                    .to_owned(),
-            )
-            .await?;
         manager
             .drop_table(sea_query::Table::drop().table(Job::Table).to_owned())
             .await?;
@@ -251,15 +215,6 @@ enum OutputFile {
     BlobId,
 }
 
-#[derive(DeriveIden)]
-enum VisibleFile {
-    Table,
-    Id,
-    Path,
-    Hash,
-    JobId,
-}
-
 // Only output jobs are ever added. We only index by Hash
 // and the primary key. The hash must be unique has is the
 // hash of all input content including visible files. We do

rust/migration/src/m20231117_162713_add_created_at.rs

@@ -25,7 +25,6 @@ impl MigrationTrait for Migration {
           (OutputDir::Table, OutputDir::CreatedAt),
           (OutputSymlink::Table, OutputSymlink::CreatedAt),
           (OutputFile::Table, OutputFile::CreatedAt),
-          (VisibleFile::Table, VisibleFile::CreatedAt),
           (ApiKey::Table, ApiKey::CreatedAt),
           (JobUses::Table, JobUses::CreatedAt),
           (Blob::Table, Blob::CreatedAt),
@@ -54,7 +53,6 @@ impl MigrationTrait for Migration {
           (OutputDir::Table, OutputDir::CreatedAt),
           (OutputSymlink::Table, OutputSymlink::CreatedAt),
           (OutputFile::Table, OutputFile::CreatedAt),
-          (VisibleFile::Table, VisibleFile::CreatedAt),
           (ApiKey::Table, ApiKey::CreatedAt),
           (JobUses::Table, JobUses::CreatedAt)
         } as (t, c) in {
@@ -95,12 +93,6 @@ enum OutputFile {
     CreatedAt,
 }
 
-#[derive(DeriveIden)]
-enum VisibleFile {
-    Table,
-    CreatedAt,
-}
-
 #[derive(DeriveIden)]
 enum ApiKey {
     Table,

rust/rsc/.config.json

@@ -3,7 +3,7 @@
   "server_address": "0.0.0.0:3002",
   "connection_pool_timeout": 60,
   "standalone": false,
-  "active_store": "bfa63b02-d41f-4eff-b0e3-f42bafb33a3d",
+  "active_store": "3446d287-1d6f-439f-bc8a-9e73ab34065d",
   "log_directory": null,
   "blob_eviction": {
     "tick_rate": 60,

rust/rsc/src/bin/rsc/add_job.rs

@@ -1,7 +1,7 @@
 use crate::types::AddJobPayload;
 use axum::{http::StatusCode, Json};
 
-use entity::{job, output_dir, output_file, output_symlink, visible_file};
+use entity::{job, output_dir, output_file, output_symlink};
 
 use sea_orm::{ActiveModelTrait, ActiveValue::*, DatabaseConnection, DbErr, TransactionTrait};
 use std::sync::Arc;
@@ -18,7 +18,6 @@ pub async fn add_job(
     let hash = payload.hash();
     tracing::info!(hash);
 
-    let vis = payload.visible_files;
     let output_files = payload.output_files;
     let output_symlinks = payload.output_symlinks;
     let output_dirs = payload.output_dirs;
@@ -51,20 +50,6 @@ pub async fn add_job(
                 let job = insert_job.save(txn).await?;
                 let job_id = job.id.unwrap();
 
-                database::create_many_visible_files(
-                    txn,
-                    vis.into_iter()
-                        .map(|vis_file| visible_file::ActiveModel {
-                            id: NotSet,
-                            created_at: NotSet,
-                            path: Set(vis_file.path),
-                            hash: Set(vis_file.hash),
-                            job_id: Set(job_id),
-                        })
-                        .collect(),
-                )
-                .await?;
-
                 database::create_many_output_files(
                     txn,
                     output_files

rust/rsc/src/database.rs

@@ -2,11 +2,10 @@ use chrono::NaiveDateTime;
 use data_encoding::BASE64;
 use entity::prelude::{
     ApiKey, Blob, BlobStore, Job, JobHistory, LocalBlobStore, OutputDir, OutputFile, OutputSymlink,
-    VisibleFile,
 };
 use entity::{
     api_key, blob, blob_store, job, job_history, local_blob_store, output_dir, output_file,
-    output_symlink, visible_file,
+    output_symlink,
 };
 use itertools::Itertools;
 use migration::OnConflict;
@@ -462,39 +461,6 @@ where
     Ok(count)
 }
 
-// --------------------------------------------------
-// ----------         Visible File         ----------
-// --------------------------------------------------
-
-// ----------            Create            ----------
-pub async fn create_many_visible_files<T: ConnectionTrait>(
-    db: &T,
-    visible_files: Vec<visible_file::ActiveModel>,
-) -> Result<(), DbErr> {
-    if visible_files.len() == 0 {
-        return Ok(());
-    }
-
-    let chunked: Vec<Vec<visible_file::ActiveModel>> = visible_files
-        .into_iter()
-        .chunks((MAX_SQLX_PARAMS / 5).into())
-        .into_iter()
-        .map(|chunk| chunk.collect())
-        .collect();
-
-    for chunk in chunked {
-        VisibleFile::insert_many(chunk).exec(db).await?;
-    }
-
-    Ok(())
-}
-
-// ----------             Read             ----------
-
-// ----------            Update            ----------
-
-// ----------            Delete            ----------
-
 // --------------------------------------------------
 // ----------         Output File          ----------
 // --------------------------------------------------