fix: fix flashtext

pyproject.toml

@@ -56,7 +56,6 @@ elasticsearch = "~8.5.3"
 pymongo = "~3.12.0"
 spacy = "~3.4.1"
 dacite = "~1.6.0"
-flashtext = "~2.7"
 langid = "~1.1.6"
 influxdb-client = "~1.34.0"
 jsonschema = "~4.4.0"

robotoff/prediction/category/matcher.py

@@ -5,14 +5,13 @@
 import re
 from typing import Iterable, Optional
 
-from flashtext import KeywordProcessor
-
 from robotoff import settings
 from robotoff.products import ProductDataset
 from robotoff.taxonomy import TaxonomyType, get_taxonomy
 from robotoff.types import Prediction, PredictionType, ServerType
 from robotoff.utils import dump_json, get_logger, load_json
 from robotoff.utils.text import (
+    KeywordProcessor,
     get_lemmatizing_nlp,
     strip_accents_v1,
     strip_consecutive_spaces,

robotoff/prediction/category/neural/keras_category_classifier_3_0/preprocessing.py

@@ -6,11 +6,11 @@
 from typing import Optional
 
 import numpy as np
-from flashtext import KeywordProcessor
 
 from robotoff import settings
 from robotoff.taxonomy import Taxonomy, fetch_taxonomy
 from robotoff.types import JSONType
+from robotoff.utils.text import KeywordProcessor
 
 from .text_utils import fold, get_tag
 

robotoff/prediction/ocr/brand.py

@@ -1,13 +1,11 @@
 import functools
 from typing import Iterable, Optional, Union
 
-from flashtext import KeywordProcessor
-
 from robotoff import settings
 from robotoff.brands import get_brand_blacklist, keep_brand_from_taxonomy
 from robotoff.types import Prediction, PredictionType
 from robotoff.utils import get_logger, text_file_iter
-from robotoff.utils.text import get_tag
+from robotoff.utils.text import KeywordProcessor, get_tag
 
 from .dataclass import OCRResult, get_match_bounding_box, get_text
 from .utils import generate_keyword_processor

robotoff/prediction/ocr/image_flag.py

@@ -1,11 +1,10 @@
 import functools
 from typing import Optional, Union
 
-from flashtext import KeywordProcessor
-
 from robotoff import settings
 from robotoff.types import Prediction, PredictionType
 from robotoff.utils import text_file_iter
+from robotoff.utils.text import KeywordProcessor
 
 from .dataclass import OCRResult, SafeSearchAnnotationLikelihood, get_text
 

robotoff/prediction/ocr/label.py

@@ -2,11 +2,10 @@
 import re
 from typing import Iterable, Optional, Union
 
-from flashtext import KeywordProcessor
-
 from robotoff import settings
 from robotoff.types import Prediction, PredictionType
 from robotoff.utils import get_logger, text_file_iter
+from robotoff.utils.text import KeywordProcessor
 
 from .dataclass import OCRField, OCRRegex, OCRResult, get_match_bounding_box, get_text
 from .utils import generate_keyword_processor

robotoff/prediction/ocr/location.py

@@ -5,13 +5,11 @@
 from pathlib import Path
 from typing import BinaryIO, Iterable, Optional, Union
 
-from flashtext import KeywordProcessor
-
 from robotoff import settings
 from robotoff.types import Prediction, PredictionType
 from robotoff.utils import get_logger
 from robotoff.utils.cache import CachedStore
-from robotoff.utils.text import strip_accents_v1
+from robotoff.utils.text import KeywordProcessor, strip_accents_v1
 
 from .dataclass import OCRResult
 

robotoff/prediction/ocr/packager_code.py

@@ -1,12 +1,11 @@
 import re
 from typing import Optional, Union
 
-from flashtext import KeywordProcessor
-
 from robotoff import settings
 from robotoff.types import Prediction, PredictionType
 from robotoff.utils import text_file_iter
 from robotoff.utils.cache import CachedStore
+from robotoff.utils.text import KeywordProcessor
 
 from .dataclass import OCRField, OCRRegex, OCRResult, get_match_bounding_box, get_text
 from .utils import generate_keyword_processor

robotoff/prediction/ocr/utils.py

@@ -1,6 +1,6 @@
 from typing import Callable, Iterable, Optional
 
-from flashtext import KeywordProcessor
+from robotoff.utils.text import KeywordProcessor
 
 
 def generate_keyword_processor(

robotoff/utils/text.py → robotoff/utils/text/__init__.py

@@ -6,6 +6,7 @@
 
 from robotoff.utils import get_logger
 
+from .flashtext import KeywordProcessor  # noqa: F401
 from .fold_to_ascii import fold, fold_without_insertion_deletion
 
 logger = get_logger(__name__)