Support for SQLite and set it as default DB

common/src/config.d.ts

@@ -76,11 +76,15 @@ export namespace Configuration {
     db: string;
   }
 
+  export interface SqliteDbConfig {
+    provider: 'sqlite';
+  }
+
   export interface Main {
     port: number; // The listen port.
     libraryPath: string; // Path to the source library.
     cachePath: string; // Path to store thumbnails and cached media.
-    database?: Mongo; // Not passed back to the APIs.
+    database?: Mongo | SqliteDbConfig; // Not passed back to the APIs.
     transcoder: Transcoder;
     // Enable pHash generation for stills as part of import.
     enablePhash: boolean;

common/src/search.d.ts

@@ -30,8 +30,9 @@ export interface SubsetConstraints {
   // Arrays
   tags?: ArrayFilter;
   actors?: ArrayFilter;
-  type?: ArrayFilter;
-  autoTags?: ArrayFilter;
+  type?: StringFilter;
+  // Only currently searched by Full-text, but exists: false is used.
+  autoTags?: StringFilter;
 
   // Strings
   hash?: StringFilter;
@@ -62,13 +63,13 @@ export interface SubsetConstraints {
   // If not set defaults to whatever is sensible for the sortBy field.
   sortDirection?: 'ASC' | 'DESC';
   // Checks that metadata exists
-  indexed?: boolean;
+  indexed?: BooleanFilter;
   // Checks that metadata.qualityCache.0 exists or type != video
-  cached?: boolean;
+  cached?: BooleanFilter;
   // Checks if a string exists
-  phashed?: boolean;
+  phashed?: BooleanFilter;
   // Returns media that have potential clones. clones.length > 0 & exists
-  hasClones?: boolean;
+  hasClones?: BooleanFilter;
   // Return a random set of this size.
   sample?: number;
   // Return the first number of matches

env-sqlite.sh

@@ -0,0 +1,6 @@
+export PORT=8444
+export DATABASE=sqlite
+export CACHE_PATH=$(pwd)/data/cache
+export DATA_PATH=$(pwd)/data/raw
+
+mkdir -p "${CACHE_PATH}" "${DATA_PATH}"

server/copy.sh

@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+rm -rf ./dist/database/sqlite/migrations
+cp -R ./src/database/sqlite/migrations ./dist/database/sqlite/migrations

server/package.json

@@ -11,6 +11,7 @@
     "@tensorflow/tfjs-node": "^3.3.0",
     "ajv": "^8.6.0",
     "better-ajv-errors": "^1.1.2",
+    "better-sqlite3": "^9.2.2",
     "body-parser": "^1.18.3",
     "compression": "^1.7.1",
     "deepmerge": "^4.0.0",
@@ -24,25 +25,27 @@
     "once": "^1.4.0",
     "path-is-inside": "^1.0.2",
     "pngjs": "^6.0.0",
-    "proper-job": "^1.4.6",
+    "proper-job": "1.4.14",
     "rimraf": "^3.0.0",
     "saslprep": "^1.0.3",
     "socket.io": "^4.4.1",
     "strip-json-comments": "^3.0.1",
+    "uuid": "^9.0.1",
     "walk": "^2.3.9",
     "wrappy": "^1.0.2"
   },
   "scripts": {
     "start": "node dist/index.js",
-    "build": "tsc -p . && ./create-schemas.sh",
+    "build": "tsc -p . && ./create-schemas.sh && ./copy.sh",
     "clean": "rm -rf dist/* *.tsbuildinfo ./.schema-build-info",
     "lint": "eslint --ext .ts ./src",
     "fix": "eslint --fix --ext .ts ./src",
-    "test": "mocha --recursive --timeout 30000 dist/test"
+    "test": "TEST_MODE=1 mocha --recursive --timeout 30000 dist/test"
   },
   "author": "SimplyBoo",
   "license": "MIT",
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.8",
     "@types/chai": "^4.2.11",
     "@types/compression": "^1.0.1",
     "@types/express": "^4.17.1",
@@ -57,6 +60,7 @@
     "@types/pngjs": "^6.0.0",
     "@types/rimraf": "^3.0.0",
     "@types/socket.io": "^3.0.2",
+    "@types/uuid": "^9.0.7",
     "@types/walk": "^2.3.0",
     "@typescript-eslint/eslint-plugin": "^6.12.0",
     "@typescript-eslint/parser": "^6.12.0",

server/src/config.ts

@@ -50,6 +50,7 @@ const DEFAULTS: any = {
   // Enabled pHash generation during auto-import.
   enablePhash: false,
   enableTensorFlow: true,
+  database: { provider: 'sqlite' },
   transcoder: {
     // Copy data to speedup setup.
     maxCopyEnabled: true,

server/src/database/index.ts

@@ -1,7 +1,9 @@
+import Path from 'path';
 import Config from '../config';
 import type { Database } from '../types';
 
 import { MongoConnector } from './mongodb';
+import { SqliteConnector } from './sqlite';
 
 export async function setup(): Promise<Database> {
   const dbConfig = Config.get().database;
@@ -11,6 +13,11 @@ export async function setup(): Promise<Database> {
   switch (dbConfig.provider as string) {
     case 'mongodb':
       return MongoConnector.init();
+    case 'sqlite':
+      return SqliteConnector.init({
+        filename: Path.resolve(Config.get().cachePath, 'vimtur.db'),
+        libraryPath: Config.get().libraryPath,
+      });
     default:
       throw new Error(`Unsupported database type: ${dbConfig.provider}`);
   }

server/src/database/mongodb/index.ts

@@ -654,6 +654,18 @@ export class MongoConnector extends Database {
     await this.db.collection<BaseMedia>('media').updateOne({ hash }, { $pull: { actors: actor } });
   }
 
+  public async testCleanup(): Promise<void> {
+    if (!process.env.TEST_MODE) {
+      throw new Error('testCleanup called outside of test mode');
+    }
+    await this.db.collection('media').deleteMany({});
+    await this.db.collection('media.deleted').deleteMany({});
+    await this.db.collection('tags').deleteMany({});
+    await this.db.collection('actors').deleteMany({});
+    await this.db.collection('playlists').deleteMany({});
+    await this.db.collection('config').deleteMany({});
+  }
+
   public async subsetFields(constraints: SubsetConstraints, fields?: SubsetFields | 'all'): Promise<BaseMedia[]> {
     const mediaCollection = this.db.collection<BaseMedia>('media');
 

server/src/database/mongodb/utils.ts

@@ -142,7 +142,7 @@ export function createNumberFilter(field: string, options?: NumberFilter): objec
       filters.push({ $or: [{ [field]: { $lte: 0 } }, { [field]: { $exists: false } }] });
     } else {
       filters.push({
-        [field]: { $gte: options.max },
+        [field]: { $lte: options.max },
       });
     }
   }