Skip to content

Commit

Permalink
chore: update
Browse files Browse the repository at this point in the history
  • Loading branch information
@isnolan
isnolan committed May 5, 2024
1 parent 8ef6cf7 commit ce0c095
Show file tree
Hide file tree
Showing 7 changed files with 139 additions and 146 deletions.
5 changes: 1 addition & 4 deletions apps/bodhi-service/src/modules/files/dto/queue.dto.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ export class ExtractQueueDto {
id: number;

@ApiProperty()
mimetype: string;
mimeType: string;

@ApiProperty()
folderPath: string;
Expand All @@ -30,7 +30,4 @@ export class ExtractQueueDto {
export class CleanQueueDto {
@ApiProperty()
id: number;

@ApiProperty()
user_id: number;
}
29 changes: 13 additions & 16 deletions apps/bodhi-service/src/modules/files/files.controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,14 @@ import { RequestWithUser } from '@/core/common/request.interface';
import { JwtOrApiKeyGuard } from '../auth/guard/mixed.guard';
import { FileDto, UploadFileReq } from './dto/upload.dto';
import { FilesService } from './files.service';
import { FileService } from './service';

@ApiTags('files')
@ApiBearerAuth()
@UseGuards(JwtOrApiKeyGuard)
@ApiSecurity('api-key', [])
@Controller('files')
export class FilesController {
constructor(
private readonly file: FileService,
private readonly service: FilesService,
) {}
constructor(private readonly service: FilesService) {}

@Get()
@ApiOperation({ summary: 'Get Files', description: 'Get Files' })
Expand All @@ -31,11 +27,11 @@ export class FilesController {
const { user_id, client_user_id = '' } = req.user; // from jwt or apikey

try {
const rows = await this.service.findActiveFilesByUserId(user_id, client_user_id);
const rows = await this.service.findActiveByUserId(user_id, client_user_id);
return rows.map((item) => {
const url = `https://s.alidraft.com${item.path}`;
const { name, size, mimetype, expires_at } = item;
const id = this.file.encodeId(item.id);
const id = this.service.encodeId(item.id);
return { id, name, size, mimetype, url, expires_at };
});
} catch (err) {
Expand All @@ -52,12 +48,12 @@ export class FilesController {
async upload(@Req() req: RequestWithUser, @UploadedFiles() files, @Body() body: UploadFileReq) {
const { user_id, client_user_id = '' } = req.user; // from jwt or apikey
const { purpose } = body;
const expires_at = new Date(Date.now() + 1000 * 60 * 60 * 24 * 30); // 30 days
const expires_at = new Date(Date.now() + 1000 * 60 * 60 * 24 * 15); // 15 days

try {
// 计算并检查hash
return Promise.all(
files.map((file) => {
files.map((file: Express.Multer.File) => {
const hashhex = createHash('md5');
hashhex.update(file.buffer);
const hash = hashhex.digest('hex');
Expand All @@ -75,14 +71,14 @@ export class FilesController {
}

@Get(':id')
@ApiOperation({ summary: 'Get file detail', description: 'Get file detail' })
@ApiOperation({ summary: 'Find file detail', description: 'Find file detail' })
@ApiResponse({ status: 200, description: 'success', type: FileDto })
async get(@Req() req: RequestWithUser, @Param('id') file_id: string): Promise<FileDto> {
async find(@Req() req: RequestWithUser, @Param('id') file_id: string): Promise<FileDto> {
const { user_id, client_user_id = '' } = req.user; // from jwt or apikey

try {
const id = this.file.decodeId(file_id);
const file = await this.service.findActiveById(id, user_id, client_user_id);
const id = this.service.decodeId(file_id);
const file = await this.service.findById(id, user_id, client_user_id);
const url = `https://s.alidraft.com${file.path}`;
delete file.path;

Expand All @@ -99,10 +95,11 @@ export class FilesController {
const { user_id, client_user_id = '' } = req.user; // from jwt or apikey

try {
const id = this.file.decodeId(file_id);
const file = await this.service.findActiveById(id, user_id, client_user_id);
const id = this.service.decodeId(file_id);
const file = await this.service.findById(id, user_id, client_user_id);
if (file && ['active', 'expired', 'created'].includes(file.state)) {
return this.service.delete(id, user_id, client_user_id);
this.service.delete(id, user_id);
return;
}
throw new NotFoundException();
} catch (err) {
Expand Down
35 changes: 22 additions & 13 deletions apps/bodhi-service/src/modules/files/files.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { InjectQueue } from '@nestjs/bull';
import { Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Queue } from 'bull';
import Hashids from 'hashids';
import * as mime from 'mime-types';
import * as moment from 'moment-timezone';

Expand All @@ -12,6 +13,7 @@ import { FileService } from './service';

@Injectable()
export class FilesService {
private readonly hashids: Hashids;
private readonly storage: Storage;

constructor(
Expand All @@ -20,30 +22,38 @@ export class FilesService {
private readonly file: FileService,
private readonly config: ConfigService,
) {
this.hashids = new Hashids('bodhi-files', 10);
this.storage = new Storage({ credentials: this.config.get('gcloud') });
}

async findActiveFilesByUserId(user_id: number, client_user_id?: string) {
encodeId(id: number) {
return this.hashids.encode(id);
}

decodeId(id: string) {
return this.hashids.decode(id)[0] as number;
}

async findActiveByUserId(user_id: number, client_user_id?: string) {
return this.file.findActiveByUserId(user_id, client_user_id);
}

async findActiveById(id: number, user_id: number, client_user_id?: string) {
return this.file.findActive(id, user_id, client_user_id);
async findById(id: number, user_id: number, client_user_id?: string) {
return this.file.find(id, user_id, client_user_id);
}

async delete(id: number, user_id: number, client_user_id?: string) {
this.queue.add('clean', { id, user_id });
return this.file.delete(id, user_id, client_user_id);
async delete(id: number, user_id: number) {
this.queue.add('file-clean', { id });
return this.file.delete(id, user_id);
}

async uploadFile(file: Express.Multer.File, opts: Partial<File>, purpose: string): Promise<FileDto> {
const { hash, name, mimetype, size } = opts;
// 检查是否已经上传
let f = await this.file.findActiveByHash(hash);
if (f) {
const id = this.file.encodeId(f.id);
const url = `https://s.alidraft.com${f.path}`;
return { id, name, size, mimetype, url, expires_at: f.expires_at };
return { id: this.encodeId(f.id), name, size, mimetype, url, expires_at: f.expires_at };
}

// 初次上传
Expand All @@ -58,16 +68,15 @@ export class FilesService {
await this.storage.bucket(bucket).file(filePath).save(file.buffer);

// file extract
if (mimetype.includes('pdf') && purpose === 'file-extract') {
this.queue.add('extract', { id: f.id, mimetype, folderPath, filePath });
if (purpose === 'file-extract') {
this.queue.add('file-extract', { id: f.id, mimeType: mimetype, folderPath, filePath });
}

// 更新文件
this.file.update(f.id, { path: filePath, state: FileState.ACTIVE });

const id = this.file.encodeId(f.id);
const url = `https://s.alidraft.com/${filePath}`;
return { id, name, url, size, mimetype, expires_at: f.expires_at } as FileDto;
return { id: this.encodeId(f.id), name, url, size, mimetype, expires_at: f.expires_at } as FileDto;
} catch (err) {
console.warn(err);

Expand All @@ -94,7 +103,7 @@ export class FilesService {

// 新建下载任务
file = await this.file.create({ file_id, name });
this.queue.add('download', { id: file.id, file_id, name, url });
this.queue.add('file-download', { id: file.id, file_id, name, url });

return { id: file.id, name, url };
}
Expand Down
35 changes: 25 additions & 10 deletions apps/bodhi-service/src/modules/files/process/clean.processor.ts
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { Storage } from '@google-cloud/storage';
import { Process, Processor } from '@nestjs/bull';
import { InjectQueue, Process, Processor } from '@nestjs/bull';
import { ConfigService } from '@nestjs/config';
import { Job } from 'bull';
import { GoogleAuth } from 'google-auth-library';
import { Cron, CronExpression } from '@nestjs/schedule';
import { Job, Queue } from 'bull';
import path from 'path';

import { CleanQueueDto } from '../dto/queue.dto';
Expand All @@ -11,26 +11,41 @@ import { FileService } from '../service/file.service';
@Processor('bodhi')
export class CleanProcessor {
private readonly storage: Storage;
private readonly auth: GoogleAuth;

constructor(
@InjectQueue('bodhi')
private readonly queue: Queue,
private readonly config: ConfigService,
private readonly file: FileService,
) {
this.storage = new Storage({ credentials: this.config.get('gcloud') });
}

@Process('clean')
/**
* Auto clean expired files
*/
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
async handleDailyCleanExpired() {
const files = await this.file.findExpired();
for (const file of files) {
this.queue.add('file-clean', { id: file.id });
}
}

/**
* Clean file from GCP Storage
* @param job
*/
@Process('file-clean')
async clean(job: Job<CleanQueueDto>) {
console.log(`[file]progress:clean`, job.data);
const { id, user_id } = job.data;
const file = await this.file.findActive(id, user_id);
console.log(`[files]progress:clean`, job.data);
const { id } = job.data;
try {
if (file.mimetype === 'application/pdf') {
const file = await this.file.find(id);
if (file && file.mimetype === 'application/pdf') {
const { bucket } = this.config.get('gcloud');
const prefix = path.dirname(file.path);
this.storage.bucket(bucket).deleteFiles({ prefix, force: true });
console.log(`->deleted`, prefix);
}
} catch (err) {
console.warn(`[file]progress:clean`, err.message);
Expand Down
74 changes: 35 additions & 39 deletions apps/bodhi-service/src/modules/files/process/download.processor.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,48 +18,44 @@ export class DownloadProcessor {
private readonly file: FileService,
) {}

@Process('download')
@Process('file-download')
async expired(job: Job<FileQuqueDto>) {
console.log(`[file]process`, job.data);
console.log(`[file]processs`, job.data);
const { id, url } = job.data;
/* eslint no-async-promise-executor: */
return new Promise(async (resolve, reject) => {
try {
// download & upload
const { buffer, mimetype, size } = await fetch(url, {
headers: {
'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
'sec-ch-ua-mobile': '?0',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'sec-ch-ua-platform': '"macOS"',
},
agent: new HttpsProxyAgent(this.config.get('proxy')),
}).then((res) => {
const mimetype = res.headers.get('Content-Type') as string;
return res.buffer().then((buffer) => ({ buffer, mimetype, size: buffer.length }));
});
console.log(`[file]process`, mimetype, size, url);
try {
// download & upload
const { buffer, mimetype, size } = await fetch(url, {
headers: {
'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
'sec-ch-ua-mobile': '?0',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'sec-ch-ua-platform': '"macOS"',
},
agent: new HttpsProxyAgent(this.config.get('proxy')),
}).then((res) => {
const mimetype = res.headers.get('Content-Type') as string;
return res.buffer().then((buffer) => ({ buffer, mimetype, size: buffer.length }));
});
console.log(`[files]process`, mimetype, size, url);

// Upload to oss
const hashhex = createHash('md5');
hashhex.update(buffer);
const hash = hashhex.digest('hex');
const ext = mime.extension(mimetype);
const path = `/attachments/${moment.tz('Asia/Shanghai').format('YYYYMM')}/${uuidv4()}.${ext}`;
this.file.update(id, { hash, path, size, mimetype });
// Upload to oss
const hashhex = createHash('md5');
hashhex.update(buffer);
const hash = hashhex.digest('hex');
const ext = mime.extension(mimetype);
const path = `/attachments/${moment.tz('Asia/Shanghai').format('YYYYMM')}/${uuidv4()}.${ext}`;
this.file.update(id, { hash, path, size, mimetype });

// const { res }: any = await putStream(path, { buffer, size });
// console.log(`[file]process`, res.statusMessage);
// if (res.statusMessage === 'OK') {
// this.file.updateState(id, FileState.ACTIVE);
// resolve(res);
// }
// reject(res.statusMessage);
} catch (err) {
console.warn(`[file]process`, err.message);
reject(err);
}
});
// const { res }: any = await putStream(path, { buffer, size });
// console.log(`[file]process`, res.statusMessage);
// if (res.statusMessage === 'OK') {
// this.file.updateState(id, FileState.ACTIVE);
// resolve(res);
// }
// reject(res.statusMessage);
} catch (err) {
console.warn(`[files]process`, err.message);
}
}
}
Loading

0 comments on commit ce0c095

Please sign in to comment.