Make uploads a two-step process. First we get a signed upload url whe…

…re we post the file to, next we create an entry with the file's information.

src/backend/Database.ts

@@ -190,13 +190,8 @@ export class Database implements Syncable {
             root: mutation.root
           })
         )
-      case MutationType.FileUpload:
-        return this.store(
-          EntryRow({
-            entryId: mutation.entryId
-          }).delete(),
-          EntryRow().insert(mutation.entry)
-        )
+      case MutationType.Upload:
+        return
       default:
         throw unreachable(mutation)
     }

src/backend/Handler.ts

@@ -24,6 +24,10 @@ const ResolveBody: Type<Connection.ResolveParams> = object({
   }).optional
 })
 
+const PrepareBody = object({
+  filename: string
+})
+
 function createRouter(
   auth: Auth.Server,
   createApi: (context: Connection.Context) => Connection
@@ -108,29 +112,13 @@ function createRouter(
       .map(respond),
 
     matcher
-      .post(Connection.routes.media())
+      .post(Connection.routes.prepareUpload())
       .map(context)
-      .map(router.parseFormData)
-      .map(async ({ctx, body}) => {
+      .map(router.parseJson)
+      .map(({ctx, body}) => {
         const api = createApi(ctx)
-        const workspace = String(body.get('workspace'))
-        const root = String(body.get('root'))
-        const castOrUndefined = <T>(cast: (x: unknown) => T, value: unknown) =>
-          value !== null ? cast(value) : undefined
-        return ctx.logger.result(
-          api.uploadFile({
-            workspace,
-            root,
-            buffer: await (body.get('buffer') as File).arrayBuffer(),
-            parentId: String(body.get('parentId')) || undefined,
-            path: String(body.get('path')),
-            preview: castOrUndefined(String, body.get('preview')),
-            averageColor: castOrUndefined(String, body.get('averageColor')),
-            thumbHash: castOrUndefined(String, body.get('thumbHash')),
-            width: castOrUndefined(Number, body.get('width')),
-            height: castOrUndefined(Number, body.get('height'))
-          })
-        )
+        const {filename} = PrepareBody(body)
+        return ctx.logger.result(api.prepareUpload(filename))
       })
       .map(respond)
   ).recover(router.reportError)

src/backend/Media.ts

@@ -1,14 +1,10 @@
 import {Connection, EntryRow} from 'alinea/core'
 
 export interface Media {
-  upload(
-    params: Connection.MediaUploadParams,
+  prepareUpload(
+    file: string,
     ctx: Connection.Context
-  ): Promise<string>
-  download(
-    params: Connection.DownloadParams,
-    ctx: Connection.Context
-  ): Promise<Connection.Download>
+  ): Promise<Connection.UploadResponse>
   delete(
     params: Connection.DeleteParams,
     ctx: Connection.Context

src/backend/Server.ts

@@ -1,24 +1,7 @@
-import {
-  Config,
-  Connection,
-  Entry,
-  EntryPhase,
-  Workspace,
-  createId
-} from 'alinea/core'
-import {entryFilepath} from 'alinea/core/EntryFilenames'
+import {Config, Connection} from 'alinea/core'
 import {EntryRecord} from 'alinea/core/EntryRecord'
 import {Graph} from 'alinea/core/Graph'
 import {Mutation, MutationType} from 'alinea/core/Mutation'
-import {generateKeyBetween} from 'alinea/core/util/FractionalIndexing'
-import {
-  basename,
-  dirname,
-  extname,
-  join,
-  normalize
-} from 'alinea/core/util/Paths'
-import {slugify} from 'alinea/core/util/Slugs'
 import {Database} from './Database.js'
 import {History, Revision} from './History.js'
 import {Media} from './Media.js'
@@ -28,7 +11,6 @@ import {Store} from './Store.js'
 import {Target} from './Target.js'
 import {ChangeSetCreator} from './data/ChangeSet.js'
 import {AlineaMeta} from './db/AlineaMeta.js'
-import {createContentHash} from './util/ContentHash.js'
 
 export interface PreviewOptions {
   preview?: boolean
@@ -89,110 +71,9 @@ export class Server implements Connection {
     return previews.sign({sub: user.sub})
   }
 
-  async uploadFile({
-    workspace: workspaceName,
-    root,
-    parentId,
-    ...file
-  }: Connection.UploadParams): Promise<Media.File> {
-    const {media, config} = this.options
-    const entryId = createId()
-    const dir = dirname(file.path)
-    const extension = extname(file.path).toLowerCase()
-    const name = basename(file.path, extension)
-    const fileName = `${slugify(name)}.${createId()}${extension}`
-    const workspace = Workspace.data(config.workspaces[workspaceName])
-    const prefix = workspace.mediaDir && normalize(workspace.mediaDir)
-    const fileLocation = join(prefix, dir, fileName)
-
-    let location = await media.upload(
-      {fileLocation, buffer: file.buffer},
-      this.context
-    )
-
-    // We'll strip the media dir off the location we received. We don't want
-    // this information to be saved to disk because it would be impractical
-    // to ever refactor to another directory.
-    if (prefix && location.startsWith(prefix))
-      location = location.slice(prefix.length)
-
-    const contentHash = await createContentHash(
-      EntryPhase.Published,
-      new Uint8Array(file.buffer)
-    )
-    const parent = await this.graph.preferDraft.maybeGet(
-      Entry({entryId: parentId}).select({
-        level: Entry.level,
-        entryId: Entry.entryId,
-        url: Entry.url,
-        path: Entry.path,
-        parentPaths({parents}) {
-          return parents().select(Entry.path)
-        }
-      })
-    )
-    const prev = await this.graph.preferDraft.maybeGet(
-      Entry({parent: parentId})
-    )
-    const entryLocation = {
-      workspace: workspaceName,
-      root,
-      locale: null,
-      path: basename(file.path.toLowerCase()),
-      phase: EntryPhase.Published
-    }
-
-    const filePath = entryFilepath(
-      config,
-      entryLocation,
-      parent ? parent.parentPaths.concat(parent.path) : []
-    )
-
-    const parentDir = dirname(filePath)
-
-    const entry: Media.File = {
-      ...entryLocation,
-      parent: parent?.entryId ?? null,
-      entryId,
-      type: 'MediaFile',
-      url: (parent ? parent.url : '/') + file.path.toLowerCase(),
-      title: basename(file.path, extension),
-      seeded: false,
-      modifiedAt: Date.now(),
-      searchableText: '',
-      index: generateKeyBetween(null, prev?.index ?? null),
-      i18nId: entryId,
-
-      level: parent ? parent.level + 1 : 0,
-      parentDir: parentDir,
-      filePath,
-      childrenDir: filePath.slice(0, -extension.length),
-      contentHash,
-      active: true,
-      main: true,
-      data: {
-        title: basename(file.path, extension),
-        location,
-        extension: extension.toLowerCase(),
-        size: file.buffer.byteLength,
-        hash: contentHash,
-        width: file.width,
-        height: file.height,
-        averageColor: file.averageColor,
-        thumbHash: file.thumbHash,
-        preview: file.preview
-      }
-    }
-    const entryFile = join(workspace.source, entry.root, entry.filePath)
-    await this.mutate([
-      {
-        type: MutationType.FileUpload,
-        entryId: entry.entryId,
-        file: entryFile,
-        entry
-      }
-    ])
-    return entry
+  prepareUpload(file: string): Promise<Connection.UploadResponse> {
+    const {media} = this.options
+    return media.prepareUpload(file, this.context)
   }
 
   // History

src/backend/data/ChangeSet.ts

@@ -6,21 +6,22 @@ import {
   DiscardDraftMutation,
   EditMutation,
   FileRemoveMutation,
-  FileUploadMutation,
   MoveMutation,
   Mutation,
   MutationType,
   OrderMutation,
   PublishMutation,
-  RemoveEntryMutation
+  RemoveEntryMutation,
+  UploadMutation
 } from 'alinea/core/Mutation'
 import {JsonLoader} from '../loader/JsonLoader.js'
 
 export enum ChangeType {
   Write = 'write',
   Rename = 'rename',
   Patch = 'patch',
-  Delete = 'delete'
+  Delete = 'delete',
+  Upload = 'upload'
 }
 export interface WriteChange {
   type: ChangeType.Write
@@ -41,7 +42,17 @@ export interface DeleteChange {
   type: ChangeType.Delete
   file: string
 }
-export type Change = WriteChange | RenameChange | PatchChange | DeleteChange
+export interface UploadChange {
+  type: ChangeType.Upload
+  file: string
+  fileId: string
+}
+export type Change =
+  | WriteChange
+  | RenameChange
+  | PatchChange
+  | DeleteChange
+  | UploadChange
 export type ChangeWithMeta = {
   changes: Array<Change>
   meta: Mutation
@@ -185,12 +196,10 @@ export class ChangeSetCreator {
     return result
   }
 
-  fileUploadChanges(mutation: FileUploadMutation): Array<Change> {
-    return this.editChanges({
-      ...mutation,
-      previousFile: mutation.file,
-      type: MutationType.Edit
-    })
+  fileUploadChanges(mutation: UploadMutation): Array<Change> {
+    return [
+      {type: ChangeType.Upload, file: mutation.file, fileId: mutation.entryId}
+    ]
   }
 
   fileRemoveChanges({file: entryFile}: FileRemoveMutation): Array<Change> {
@@ -215,7 +224,7 @@ export class ChangeSetCreator {
         return this.orderChanges(mutation)
       case MutationType.Move:
         return this.moveChanges(mutation)
-      case MutationType.FileUpload:
+      case MutationType.Upload:
         return this.fileUploadChanges(mutation)
       case MutationType.FileRemove:
         return this.fileRemoveChanges(mutation)

src/backend/router/NodeHandler.ts

@@ -23,7 +23,7 @@ async function writeReadableStreamToWritable(
   }
 }
 
-async function apply(response: Response, to: http.ServerResponse) {
+export async function respondTo(to: http.ServerResponse, response: Response) {
   to.statusCode = response.status
   response.headers.forEach((value, key) => to.setHeader(key, value))
   if (response.body) {
@@ -34,7 +34,7 @@ async function apply(response: Response, to: http.ServerResponse) {
 
 const skipHeaders = new Set(['transfer-encoding', 'connection', 'keep-alive'])
 
-function fromNodeRequest(request: http.IncomingMessage) {
+export function fromNodeRequest(request: http.IncomingMessage) {
   const headers = new Headers()
   for (const key of Object.keys(request.headers)) {
     if (skipHeaders.has(key)) continue
@@ -79,7 +79,7 @@ export function nodeHandler(
     const request = fromNodeRequest(req)
     const result = await handler(request)
     if (result) {
-      await apply(result, res)
+      await respondTo(res, result)
     } else if (next) {
       next()
     } else {

src/backend/router/Router.ts

@@ -12,6 +12,8 @@ function callHandler<In, Out>(handler: Handler<In, Out>, input: In) {
   return typeof handler === 'function' ? handler(input) : handler.handle(input)
 }
 
+export type HttpHandler = (input: Request) => Promise<Response>
+
 export class Route<In, Out> {
   constructor(public handle: Handle<In, Out>) {}
   map<T>(next: Handle<Out, T>): Route<In, T>

src/cli/Generate.ts

@@ -11,14 +11,15 @@ import {fillCache} from './generate/FillCache.js'
 import {GenerateContext} from './generate/GenerateContext.js'
 import {generateDashboard} from './generate/GenerateDashboard.js'
 import {loadCMS} from './generate/LoadConfig.js'
+import {LocalData} from './generate/LocalData.js'
 import {dirname} from './util/Dirname.js'
 import {findConfigFile} from './util/FindConfigFile.js'
 
 const __dirname = dirname(import.meta.url)
 const require = createRequire(import.meta.url)
 const alineaPackageDir = path.dirname(require.resolve('alinea/package.json'))
 
-export type GenerateOptions = {
+export interface GenerateOptions {
   cwd?: string
   staticDir?: string
   configFile?: string
@@ -27,6 +28,7 @@ export type GenerateOptions = {
   wasmCache?: boolean
   quiet?: boolean
   onAfterGenerate?: (env?: Record<string, string>) => void
+  dashboardUrl?: string
 }
 
 function generatePackage(context: GenerateContext, config: Config) {
@@ -60,6 +62,7 @@ export async function* generate(options: GenerateOptions): AsyncGenerator<
   {
     cms: CMS
     store: Store
+    localData: LocalData
   },
   void
 > {
@@ -107,8 +110,20 @@ export async function* generate(options: GenerateOptions): AsyncGenerator<
     try {
       const cms = await loadCMS(context.outDir)
       cms.exportStore(context.outDir, new Uint8Array())
-      for await (const _ of fillCache(context, store, cms, nextBuild)) {
-        yield {cms, store}
+      const fileData = new LocalData({
+        config: cms,
+        fs: fs.promises,
+        rootDir,
+        dashboardUrl: options.dashboardUrl
+      })
+      for await (const _ of fillCache(
+        context,
+        fileData,
+        store,
+        cms,
+        nextBuild
+      )) {
+        yield {cms, store, localData: fileData}
         // For debug reasons write out db
         if (process.env.NODE_ENV === 'development')
           fs.writeFileSync(