= ({ className }) => {
);
+ const mitLink = showMitExperimentLink && (
+
+ {formatMessage({ id: 'mit.title' })}
+
+ );
return (
= ({ className }) => {
.register {
color: #7f7a7a;
font-size: 20px;
- line-height: 25x;
+ line-height: 25px;
padding: 7px 12px;
border-radius: ${style.borderRadiusBase};
${clickEffect()};
@@ -134,6 +140,7 @@ export const Header: FC = ({ className }) => {
{currentUser.token ? (
+ {mitLink}
= ({ className }) => {
) : (
)}
diff --git a/src/components/MitPreprocess/TranslateCompanion.tsx b/src/components/MitPreprocess/TranslateCompanion.tsx
new file mode 100644
index 0000000..6cf2e3b
--- /dev/null
+++ b/src/components/MitPreprocess/TranslateCompanion.tsx
@@ -0,0 +1,320 @@
+import { FC } from '../../interfaces';
+import { RefObject, useRef, useState } from 'react';
+import { FilePond } from 'react-filepond';
+import { css } from '@emotion/core';
+import { Button } from '../Button';
+import { createMoeflowProjectZip, LPFile } from './moeflow-packager';
+import { FailureResults } from '../../apis';
+import { measureImgSize } from '@jokester/ts-commonutil/lib/frontend/measure-img';
+import { clamp } from 'lodash-es';
+import { BBox, mitPreprocess, TextQuad } from '../../apis/mit_preprocess';
+import { ResourcePool } from '@jokester/ts-commonutil/lib/concurrency/resource-pool';
+
+const MAX_FILE_COUNT = 30;
+
+function getQuadCenter(q: TextQuad) {
+ const xs = q.pts.flatMap((pt) => pt.map((p) => p[0]));
+ const ys = q.pts.flatMap((pt) => pt.map((p) => p[1]));
+ const minX = Math.min(...xs);
+ const maxX = Math.max(...xs);
+ const minY = Math.min(...ys);
+ const maxY = Math.max(...ys);
+ return {
+ x: (minX + maxX) / 2,
+ y: (minY + maxY) / 2,
+ };
+}
+
+function buildLpFile(
+ img: File,
+ size: { width: number; height: number },
+ textQuads: TextQuad[],
+): LPFile {
+ const labels = textQuads
+ .sort((a, b) => {
+ // sort : top=>bottom , right=>left
+ const ca = getQuadCenter(a);
+ const cb = getQuadCenter(b);
+ return Math.sign(ca.y - cb.y) || Math.sign(cb.x - ca.x);
+ })
+ .map((q) => {
+ const { x, y } = getQuadCenter(q);
+ return {
+ x: clamp(x / size.width, 0, 1),
+ y: clamp(y / size.height, 0, 1),
+ position_type: 1,
+ translation: `${q.raw_text}\n${q.translated}`,
+ };
+ });
+ console.debug('labels', labels);
+ return {
+ file_name: img.name,
+ labels,
+ };
+}
+
+async function translateWithTask(
+ text: string,
+ targetLang = 'CHT',
+): Promise {
+ const task = await mitPreprocess.createTranslateTask({
+ query: text,
+ target_lang: targetLang,
+ translator: 'gpt4',
+ });
+ const result = await mitPreprocess.waitTranslateTask(task.data.task_id);
+ return result[0] || '';
+}
+
+async function* startTranslateFile(
+ image: File,
+ running: RefObject,
+): AsyncGenerator<{
+ progress?: string;
+ failed?: FailureResults;
+ detectTextResult?: unknown;
+ ocrResult?: unknown;
+ translateResult?: unknown;
+ result?: LPFile;
+}> {
+ let uploaded;
+ yield { progress: 'uploading' };
+ try {
+ uploaded = await mitPreprocess.uploadImg(image);
+ } catch (e: unknown) {
+ yield {
+ failed: e as FailureResults,
+ };
+ return;
+ }
+ yield { progress: 'extracting text lines' };
+ const { filename } = uploaded.data;
+
+ let detectTextResult;
+ try {
+ const task = await mitPreprocess.createImgTask(
+ filename,
+ 'mit_detect_text',
+ {},
+ );
+ detectTextResult = await mitPreprocess.waitImgTask<{
+ textlines: {
+ prob: number;
+ pts: number[][];
+ text: string;
+ // textlines: any[]; // FIXME why did server return this?
+ }[];
+ }>(task.data.task_id);
+ } catch (e: unknown) {
+ yield {
+ failed: e as FailureResults,
+ };
+ return;
+ }
+
+ yield { progress: 'recognizing text lines' };
+ let ocrResult;
+ try {
+ const created = await mitPreprocess.createImgTask(filename, 'mit_ocr', {
+ regions: detectTextResult.textlines,
+ });
+ ocrResult = await mitPreprocess.waitImgTask<
+ {
+ pts: BBox;
+ text: string;
+ textlines: Array;
+ }[]
+ >(created.data.task_id);
+ console.debug('ocrResult', ocrResult);
+ } catch (e: unknown) {
+ yield {
+ failed: e as FailureResults,
+ };
+ return;
+ }
+
+ yield { progress: 'translating' };
+ let translateResult: string[];
+ try {
+ const limiter = ResourcePool.multiple([1, 2, 3, 4]);
+ translateResult = await Promise.all(
+ ocrResult.map((textBlock) =>
+ limiter.use(() => translateWithTask(textBlock.text)),
+ ),
+ );
+ } catch (e: unknown) {
+ yield {
+ failed: e as FailureResults,
+ };
+ return;
+ }
+
+ const textQuads: TextQuad[] = ocrResult.map((textBlock, i) => ({
+ pts: textBlock.pts,
+ raw_text: textBlock.text,
+ translated: translateResult[i] ?? '',
+ }));
+
+ const lpFile = buildLpFile(image, await measureImgSize(image), textQuads);
+
+ yield {
+ result: lpFile,
+ };
+}
+
+async function translateFile(image: File, imageIndex: number): Promise {
+ try {
+ for await (const fileProgress of startTranslateFile(image, {
+ current: true,
+ })) {
+ console.debug(
+ `translating file #${imageIndex} / ${image.name}`,
+ 'step',
+ fileProgress,
+ );
+ if (fileProgress.result) {
+ return fileProgress.result;
+ } else if (fileProgress.failed) {
+ throw fileProgress.failed;
+ } // else: continue
+ }
+ } catch (e) {
+ console.error(`failed translating file #${imageIndex} / ${image.name}`, e);
+ return {
+ file_name: image.name,
+ labels: [],
+ };
+ }
+ throw new Error(`should not be here`);
+}
+
+async function startOcr(
+ files: File[],
+ onProgress?: (finished: number, total: number) => void,
+): Promise {
+ const limiter = ResourcePool.multiple([1, 2]);
+
+ const translations = await Promise.all(
+ files.map((f, i) =>
+ limiter.use(async () => {
+ const lpFile = await translateFile(f, i);
+ onProgress?.(i + 1, files.length);
+ return lpFile;
+ }),
+ ),
+ );
+ const zipBlob = await createMoeflowProjectZip(
+ {
+ name: `${files[0]!.name}`,
+ intro: `这是由<萌翻+Mit demo>生成的项目. https://moeflow-mit-poc.voxscape.io/temp/mit-preprocess`,
+ default_role: 'supporter',
+ allow_apply_type: 3,
+ application_check_type: 1,
+ is_need_check_application: true,
+ source_language: 'ja',
+ output_language: 'zh-TW',
+ },
+ translations.map((lp, i) => ({ lp, image: files[i] })),
+ );
+ return new File(
+ [zipBlob],
+ `moeflow-project-${Date.now()}-${files[0]!.name}.zip`,
+ );
+}
+
+interface DemoWorkingState {
+ nonce: string;
+ numPages: number;
+ finished: number;
+}
+
+export const DemoOcrFiles: FC<{}> = (props) => {
+ const [working, setWorking] = useState(null);
+ const [origFiles, setOrigFiles] = useState(() => []);
+ const [error, setError] = useState(null);
+ const [translated, setTranslated] = useState(null);
+ const filePondRef = useRef(null);
+
+ const onStartOcr = async (files: File[]) => {
+ try {
+ const initState = {
+ nonce: `${Math.random()}`,
+ numPages: files.length,
+ finished: 0,
+ };
+ setWorking(initState);
+ setTranslated(
+ await startOcr(files, (finished, total) =>
+ setWorking((s) =>
+ s?.nonce === initState.nonce
+ ? {
+ ...s,
+ finished: Math.max(s.finished, finished),
+ numPages: total,
+ }
+ : s,
+ ),
+ ),
+ );
+ } catch (e: any) {
+ alert(e?.message || 'error');
+ console.error(e);
+ } finally {
+ setWorking(null);
+ }
+ };
+ return (
+
+ 0}
+ ref={(value) => (filePondRef.current = value)}
+ css={css`
+ display: none;
+ `}
+ allowMultiple
+ acceptedFileTypes={['image/*', '.png', '.jpg']}
+ onupdatefiles={(_files) => {
+ const files = _files.map((f) => f.file) as File[];
+ console.debug('onaddfile', files);
+ if (!(files.length > 0 && files.length <= MAX_FILE_COUNT)) {
+ setError(`一次最多只能上传${MAX_FILE_COUNT}张图片`);
+ setOrigFiles([]);
+ filePondRef.current!.removeFiles();
+ } else {
+ setOrigFiles(files);
+ setError(null);
+ }
+ }}
+ />
+
+
+
+
+ );
+};
diff --git a/src/components/MitPreprocess/moeflow-packager.ts b/src/components/MitPreprocess/moeflow-packager.ts
new file mode 100644
index 0000000..5da4389
--- /dev/null
+++ b/src/components/MitPreprocess/moeflow-packager.ts
@@ -0,0 +1,80 @@
+import * as zip from '@zip.js/zip.js';
+
+export interface LPLabel {
+ x: number; // normalized
+ y: number; // normalized
+ position_type: number; // int , always 1 ?
+ translation: string; // singleline
+}
+
+export interface LPFile {
+ file_name: string; // img filename (basename)
+ labels: LPLabel[];
+}
+
+function serializeIntoLabelplusFormat(files: LPFile[]): string[] {
+ return files.flatMap((file) => [
+ `>>>>[${file.file_name}]<<<<`,
+ ...file.labels.flatMap((l, labelIndex) => [
+ `----[${labelIndex}]----[${l.x},${l.y},${l.position_type}]`,
+ l.translation,
+ ]),
+ ]);
+}
+
+type LANG_CODE = 'ja' | 'en' | 'zh-CN' | 'zh-TW';
+
+interface MoeflowProjectMeta {
+ name: string;
+ intro: string;
+ default_role: 'supporter';
+ allow_apply_type: 3;
+ application_check_type: 1;
+ is_need_check_application: boolean;
+ // create_time: string;
+ // edit_time: string;
+ source_language: 'ja';
+ // target_languages: LANG_CODE[];
+ // output_id: string;
+ output_language: LANG_CODE;
+}
+
+export interface MoeflowImageFile {
+ lp: LPFile;
+ image: Blob;
+}
+
+/**
+ * see moeflow-backend "TeamProjectImportAPI"
+ * @return a zip file for importing into moeflow-backend
+ */
+export async function createMoeflowProjectZip(
+ meta: MoeflowProjectMeta,
+ files: MoeflowImageFile[],
+): Promise {
+ const zipWriter = new zip.ZipWriter(new zip.BlobWriter('application/zip'), {
+ bufferedWrite: true,
+ level: 9,
+ });
+
+ {
+ const translationsTxt =
+ serializeIntoLabelplusFormat(files.map((f) => f.lp)).join('\n') + '\n';
+ const blob = new Blob([translationsTxt], { type: 'text/plain' });
+ await zipWriter.add('translations.txt', new zip.BlobReader(blob));
+ }
+
+ for (const f of files) {
+ await zipWriter.add(
+ `images/${f.lp.file_name}`,
+ new zip.BlobReader(f.image),
+ );
+ }
+
+ await zipWriter.add(
+ 'project.json',
+ new zip.TextReader(JSON.stringify(meta, null, 2)),
+ );
+
+ return zipWriter.close();
+}
diff --git a/src/configs.tsx b/src/configs.tsx
index 3f3c14d..da86138 100644
--- a/src/configs.tsx
+++ b/src/configs.tsx
@@ -3,6 +3,11 @@ const configs = {
// 所有API请求的baseURL。
// 如本地开发时backend URL不匹配，可在vite server中配置反向代理
baseURL: process.env.REACT_APP_BASE_URL,
+ /**
+ * TODO think how to deal with manga-image-translator backend
+ */
+ mitBackendURL: process.env.MIT_BACKEND_URL,
+ mitUiEnabled: !!process.env.MIT_BACKEND_URL,
/** 默认值 */
default: {
team: {
diff --git a/src/locales/en-us.json b/src/locales/en-us.json
index eb68960..234984a 100644
--- a/src/locales/en-us.json
+++ b/src/locales/en-us.json
@@ -6,5 +6,12 @@
"imageTranslator.imageViewerZoomPanel.fixWidth": "Fix Width",
"imageTranslator.imageViewerZoomPanel.originSize": "Origin Size",
"imageTranslator.imageViewerZoomPanel.zoomIn": "Zoom In",
- "imageTranslator.imageViewerZoomPanel.zoomOut": "Zoom Out"
+ "imageTranslator.imageViewerZoomPanel.zoomOut": "Zoom Out",
+ "mit.title": "mit assistant",
+ "mit.desc": "Auto translate based on manga-image-translator. Translated file can be imported as Moeflow project.",
+ "mit.step1PickImageFiles": "Please pick up to 30 image files",
+ "mit.startPreprocess": "Start",
+ "mit.preprocessStepLines": "Locating characters",
+ "mit.preprocessStepOcr": "Recognizing text",
+ "mit.preprocessStepTranslate": "Translating"
}
diff --git a/src/locales/zh-cn.json b/src/locales/zh-cn.json
index bce7e93..ad3eec0 100644
--- a/src/locales/zh-cn.json
+++ b/src/locales/zh-cn.json
@@ -364,5 +364,12 @@
"translation.copyToMyTranslation": "复制到我的翻译",
"translation.copyToMyTranslationNotEmptyTip": "您的翻译已有内容，您确定要覆盖吗？",
"translation.best": "最佳翻译",
- "imageTranslator.proofreaderShowSource": "显示原文"
+ "imageTranslator.proofreaderShowSource": "显示原文",
+ "mit.title": "mit翻译辅助",
+ "mit.desc": "基于manga-image-translator的自动翻译。请选择图上文字并机翻，",
+ "mit.step1PickImageFiles": "请选择图片文件 (最多30个)",
+ "mit.startPreprocess": "开始自动翻译",
+ "mit.preprocessStepLines": "定位文字",
+ "mit.preprocessStepOcr": "识别文字",
+ "mit.preprocessStepTranslate": "机翻"
}
diff --git a/src/pages/mit/MitPreprocessDemo.tsx b/src/pages/mit/MitPreprocessDemo.tsx
new file mode 100644
index 0000000..839efeb
--- /dev/null
+++ b/src/pages/mit/MitPreprocessDemo.tsx
@@ -0,0 +1,24 @@
+import { css } from '@emotion/core';
+import React from 'react';
+import { useIntl } from 'react-intl';
+import { useHistory } from 'react-router-dom';
+import { useTitle } from '../../hooks';
+import { FC } from '../../interfaces';
+import { DemoOcrFiles } from '../../components/MitPreprocess/TranslateCompanion';
+
+/** 模板的属性接口 */
+interface TmpProps {}
+/**
+ * 模板
+ */
+export const MitPreprocessDemo: FC = () => {
+ const history = useHistory();
+ const { formatMessage } = useIntl();
+ useTitle();
+
+ return (
+
+
+
+ );
+};
diff --git a/vite.config.mts b/vite.config.mts
index 3ff4060..396d3f8 100644
--- a/vite.config.mts
+++ b/vite.config.mts
@@ -37,6 +37,10 @@ export default defineConfig({
'process.env.REACT_APP_BASE_URL': JSON.stringify(
process.env.REACT_APP_BASE_URL ?? '/api/',
),
+ // works as feature flag
+ 'process.env.MIT_BACKEND_URL': JSON.stringify(
+ process.env.MIT_BACKEND_URL ?? null,
+ ),
},
resolve: {
alias: {},