From 4ffc9bba6fe0d32bd197b49bdbf3af1151e7686c Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Mon, 16 Sep 2024 14:05:48 -0400 Subject: [PATCH 01/17] Minor detection improvements --- surya/layout.py | 23 ++++++++++---------- surya/postprocessing/heatmap.py | 37 +++++++++++++++------------------ surya/schema.py | 20 +++++++++++------- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/surya/layout.py b/surya/layout.py index 89f2a65..7afc54b 100644 --- a/surya/layout.py +++ b/surya/layout.py @@ -89,14 +89,12 @@ def get_regions_from_detection_result(detection_result: TextDetectionResult, hea max_x = max(max_x, max_x_box) max_y = max(max_y, max_y_box) - bbox.polygon[0][0] = min_x - bbox.polygon[0][1] = min_y - bbox.polygon[1][0] = max_x - bbox.polygon[1][1] = min_y - bbox.polygon[2][0] = max_x - bbox.polygon[2][1] = max_y - bbox.polygon[3][0] = min_x - bbox.polygon[3][1] = max_y + bbox.polygon = [ + [min_x, min_y], + [max_x, min_y], + [max_x, max_y], + [min_x, max_y] + ] if bbox_idx in box_lines and bbox.label in ["Picture"]: bbox.label = "Figure" @@ -104,17 +102,18 @@ def get_regions_from_detection_result(detection_result: TextDetectionResult, hea new_boxes.append(bbox) # Merge tables together (sometimes one column is detected as a separate table) - for i in range(5): # Up to 5 rounds of merging + mergeable_types = ["Table", "Picture", "Figure"] + for ftype in mergeable_types: to_remove = set() for bbox_idx, bbox in enumerate(new_boxes): - if bbox.label != "Table" or bbox_idx in to_remove: + if bbox.label != ftype or bbox_idx in to_remove: continue for bbox_idx2, bbox2 in enumerate(new_boxes): - if bbox2.label != "Table" or bbox_idx2 in to_remove or bbox_idx == bbox_idx2: + if bbox2.label != ftype or bbox_idx2 in to_remove or bbox_idx == bbox_idx2: continue - if bbox.intersection_pct(bbox2) > 0: + if bbox.intersection_pct(bbox2, x_margin=.05) > 0: bbox.merge(bbox2) to_remove.add(bbox_idx2) diff --git a/surya/postprocessing/heatmap.py b/surya/postprocessing/heatmap.py index c1ed38c..d618186 100644 --- a/surya/postprocessing/heatmap.py +++ b/surya/postprocessing/heatmap.py @@ -82,7 +82,6 @@ def detect_boxes(linemap, text_threshold, low_text): det = [] confidences = [] max_confidence = 0 - segmap = np.zeros_like(labels, dtype=np.uint8) for k in range(1, label_count): # size filtering @@ -90,39 +89,37 @@ def detect_boxes(linemap, text_threshold, low_text): if size < 10: continue - mask = labels == k - selected_linemap = linemap[mask] - - # thresholding - if np.max(selected_linemap) < text_threshold: - continue - # make segmentation map x, y, w, h = stats[k, [cv2.CC_STAT_LEFT, cv2.CC_STAT_TOP, cv2.CC_STAT_WIDTH, cv2.CC_STAT_HEIGHT]] try: - niter = int(np.sqrt(min(w, h)) * 2) + niter = int(np.sqrt(min(w, h))) except ValueError: - # Overflow in sqrt term niter = 0 buffer = 1 - sx, sy = max(0, x - niter), max(0, y - niter) + sx, sy = max(0, x - niter - buffer), max(0, y - niter - buffer) ex, ey = min(img_w, x + w + niter + buffer), min(img_h, y + h + niter + buffer) - segmap.fill(0) - segmap[mask] = 1 + mask = (labels[sy:ey, sx:ex] == k) + selected_linemap = linemap[sy:ey, sx:ex][mask] + line_max = np.max(selected_linemap) + + # thresholding + if line_max < text_threshold: + continue + + segmap = mask.astype(np.uint8) ksize = buffer + niter kernel = cv2.getStructuringElement(cv2.MORPH_RECT,(ksize, ksize)) - - # Doesn't work well without the zero start (ie, you can't trim the map tightly around the detected region) - selected_segmap = segmap[0:ey, 0:ex] - selected_segmap[sy:ey, sx:ex] = cv2.dilate(segmap[sy:ey, sx:ex], kernel) + selected_segmap = cv2.dilate(segmap, kernel) # make box indices = np.nonzero(selected_segmap) - np_contours = np.column_stack((indices[1], indices[0])) + x_inds = indices[1] + sx + y_inds = indices[0] + sy + np_contours = np.column_stack((x_inds, y_inds)) rectangle = cv2.minAreaRect(np_contours) box = cv2.boxPoints(rectangle) @@ -139,8 +136,8 @@ def detect_boxes(linemap, text_threshold, low_text): box = np.roll(box, 4-startidx, 0) box = np.array(box) - confidence = np.mean(selected_linemap[selected_linemap > low_text]) - max_confidence = max(max_confidence, confidence) + confidence = line_max + max_confidence = max(max_confidence, line_max) confidences.append(confidence) det.append(box) diff --git a/surya/schema.py b/surya/schema.py index 88d42cb..e50ed4b 100644 --- a/surya/schema.py +++ b/surya/schema.py @@ -71,19 +71,23 @@ def merge(self, other): y2 = max(self.bbox[3], other.bbox[3]) self.polygon = [[x1, y1], [x2, y1], [x2, y2], [x1, y2]] - def intersection_area(self, other, margin=0): - x_overlap = max(0, min(self.bbox[2], other.bbox[2] - margin) - max(self.bbox[0], other.bbox[0] + margin)) - y_overlap = max(0, min(self.bbox[3], other.bbox[3] - margin) - max(self.bbox[1], other.bbox[1] + margin)) + def intersection_area(self, other, x_margin=0, y_margin=0): + x_overlap = max(0, min(self.bbox[2] + x_margin, other.bbox[2] + x_margin) - max(self.bbox[0] - x_margin, other.bbox[0] - x_margin)) + y_overlap = max(0, min(self.bbox[3] + y_margin, other.bbox[3] + y_margin) - max(self.bbox[1] - y_margin, other.bbox[1] - y_margin)) return x_overlap * y_overlap - def intersection_pct(self, other, margin=0): - assert 0 <= margin <= 1 + def intersection_pct(self, other, x_margin=0, y_margin=0): + assert 0 <= x_margin <= 1 + assert 0 <= y_margin <= 1 if self.area == 0: return 0 - if margin: - margin = int(min(self.width, other.width) * margin) - intersection = self.intersection_area(other, margin) + if x_margin: + x_margin = int(min(self.width, other.width) * x_margin) + if y_margin: + y_margin = int(min(self.height, other.height) * y_margin) + + intersection = self.intersection_area(other, x_margin, y_margin) return intersection / self.area From c734a2e2141308226f66b1dfb64f616bd4321148 Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Mon, 16 Sep 2024 17:03:21 -0400 Subject: [PATCH 02/17] New table rec method --- ocr_app.py | 55 +++++++++++- surya/model/table_rec/model.py | 15 ++++ surya/model/table_rec/processor.py | 32 +++++++ surya/postprocessing/heatmap.py | 19 +++-- surya/schema.py | 12 +++ surya/settings.py | 5 ++ surya/tables.py | 132 +++++++++++++++++++++++++++++ 7 files changed, 262 insertions(+), 8 deletions(-) create mode 100644 surya/model/table_rec/model.py create mode 100644 surya/model/table_rec/processor.py create mode 100644 surya/tables.py diff --git a/ocr_app.py b/ocr_app.py index ff05cc6..4cf72c6 100644 --- a/ocr_app.py +++ b/ocr_app.py @@ -10,15 +10,19 @@ from surya.model.recognition.processor import load_processor as load_rec_processor from surya.model.ordering.processor import load_processor as load_order_processor from surya.model.ordering.model import load_model as load_order_model +from surya.model.table_rec.model import load_model as load_table_model +from surya.model.table_rec.processor import load_processor as load_table_processor from surya.ordering import batch_ordering -from surya.postprocessing.heatmap import draw_polys_on_image +from surya.postprocessing.heatmap import draw_polys_on_image, draw_bboxes_on_image from surya.ocr import run_ocr from surya.postprocessing.text import draw_text_on_image from PIL import Image from surya.languages import CODE_TO_LANGUAGE from surya.input.langs import replace_lang_with_code -from surya.schema import OCRResult, TextDetectionResult, LayoutResult, OrderResult +from surya.schema import OCRResult, TextDetectionResult, LayoutResult, OrderResult, TableResult from surya.settings import settings +from surya.tables import batch_table_recognition + @st.cache_resource() def load_det_cached(): @@ -40,6 +44,11 @@ def load_order_cached(): return load_order_model(), load_order_processor() +@st.cache_resource() +def load_table_cached(): + return load_table_model(), load_table_processor() + + def text_detection(img) -> (Image.Image, TextDetectionResult): pred = batch_text_detection([img], det_model, det_processor)[0] polygons = [p.polygon for p in pred.bboxes] @@ -66,6 +75,39 @@ def order_detection(img) -> (Image.Image, OrderResult): return order_img, pred +def table_recognition(img) -> (Image.Image, List[TableResult]): + _, layout_pred = layout_detection(img) + layout_bboxes = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] + table_imgs = [] + for table_bbox in layout_bboxes: + table_imgs.append(img.crop(table_bbox)) + table_boxes = batch_text_detection(table_imgs, det_model, det_processor) + table_bboxes = [table_box.bboxes for table_box in table_boxes] + table_preds = batch_table_recognition(table_imgs, table_bboxes, table_model, table_processor) + table_img = img.copy() + for results, table_bbox in zip(table_preds, layout_bboxes): + adjusted_bboxes = [] + labels = [] + for item in results.bboxes: + adjusted_bboxes.append([ + item.bbox[0] + table_bbox[0], + item.bbox[1] + table_bbox[1], + item.bbox[2] + table_bbox[0], + item.bbox[3] + table_bbox[1] + ]) + labels.append(f"{item.row_id} / {item.col_id}") + for item in results.unused_bboxes: + adjusted_bboxes.append([ + item.bbox[0] + table_bbox[0], + item.bbox[1] + table_bbox[1], + item.bbox[2] + table_bbox[0], + item.bbox[3] + table_bbox[1] + ]) + labels.append("Unused") + table_img = draw_bboxes_on_image(adjusted_bboxes, table_img, labels=labels) + return table_img, table_preds + + # Function for OCR def ocr(img, langs: List[str]) -> (Image.Image, OCRResult): replace_lang_with_code(langs) @@ -108,6 +150,7 @@ def page_count(pdf_file): rec_model, rec_processor = load_rec_cached() layout_model, layout_processor = load_layout_cached() order_model, order_processor = load_order_cached() +table_model, table_processor = load_table_cached() st.markdown(""" @@ -144,6 +187,7 @@ def page_count(pdf_file): text_rec = st.sidebar.button("Run OCR") layout_det = st.sidebar.button("Run Layout Analysis") order_det = st.sidebar.button("Run Reading Order") +table_rec = st.sidebar.button("Run Table Rec") if pil_image is None: st.stop() @@ -180,5 +224,12 @@ def page_count(pdf_file): st.image(order_img, caption="Reading Order", use_column_width=True) st.json(pred.model_dump(), expanded=True) + +if table_rec: + table_img, pred = table_recognition(pil_image) + with col1: + st.image(table_img, caption="Table Recognition", use_column_width=True) + st.json([p.model_dump() for p in pred], expanded=True) + with col2: st.image(pil_image, caption="Uploaded Image", use_column_width=True) \ No newline at end of file diff --git a/surya/model/table_rec/model.py b/surya/model/table_rec/model.py new file mode 100644 index 0000000..b73607a --- /dev/null +++ b/surya/model/table_rec/model.py @@ -0,0 +1,15 @@ +from transformers import TableTransformerForObjectDetection + +from surya.settings import settings + + +def load_model(checkpoint=settings.TABLE_REC_MODEL_CHECKPOINT, device=settings.TORCH_DEVICE_MODEL, dtype=settings.MODEL_DTYPE): + model = TableTransformerForObjectDetection.from_pretrained(checkpoint, torch_dtype=dtype) + + model = model.to(device) + model = model.eval() + + print(f"Loaded table model {checkpoint} on device {device} with dtype {dtype}") + + return model + diff --git a/surya/model/table_rec/processor.py b/surya/model/table_rec/processor.py new file mode 100644 index 0000000..0f00268 --- /dev/null +++ b/surya/model/table_rec/processor.py @@ -0,0 +1,32 @@ +import cv2 +import torch +import torch.nn.functional as F +from surya.settings import settings +import numpy as np + + +def load_processor(): + def _processor(imgs): + mean_vals = [0.485, 0.456, 0.406] + std_vals = [0.229, 0.224, 0.225] + imgs = [np.array(img) for img in imgs] + imgs = [_normalize(_resize(img), mean_vals, std_vals) for img in imgs] + pixel_values = torch.stack(imgs, dim=0) + return pixel_values + return _processor + + +def _resize(image, interpolation=cv2.INTER_LANCZOS4): + max_height, max_width = settings.TABLE_REC_IMAGE_SIZE["height"], settings.TABLE_REC_IMAGE_SIZE["width"] + resized_image = cv2.resize(image, (max_width, max_height), interpolation=interpolation) + resized_image = resized_image.transpose(2, 0, 1).astype(np.float32) + resized_image = torch.from_numpy(resized_image) + resized_image /= 255.0 + return resized_image + + +def _normalize(tensor, mean, std): + mean = torch.tensor(mean).view(-1, 1, 1) + std = torch.tensor(std).view(-1, 1, 1) + tensor = (tensor - mean) / std + return tensor \ No newline at end of file diff --git a/surya/postprocessing/heatmap.py b/surya/postprocessing/heatmap.py index d618186..5a9d551 100644 --- a/surya/postprocessing/heatmap.py +++ b/surya/postprocessing/heatmap.py @@ -172,13 +172,20 @@ def get_and_clean_boxes(textmap, processor_size, image_size, text_threshold=None return bboxes -def draw_bboxes_on_image(bboxes, image, labels=None): - draw = ImageDraw.Draw(image) - - for bbox in bboxes: - draw.rectangle(bbox, outline="red", width=1) - return image +def draw_bboxes_on_image(bboxes, image, labels=None): + polys = [] + for bb in bboxes: + # Clockwise polygon + poly = [ + [bb[0], bb[1]], + [bb[2], bb[1]], + [bb[2], bb[3]], + [bb[0], bb[3]] + ] + polys.append(poly) + + return draw_polys_on_image(polys, image, labels) def draw_polys_on_image(corners, image, labels=None, box_padding=-1, label_offset=1, label_font_size=10): diff --git a/surya/schema.py b/surya/schema.py index e50ed4b..3d5b8e1 100644 --- a/surya/schema.py +++ b/surya/schema.py @@ -165,3 +165,15 @@ class LayoutResult(BaseModel): class OrderResult(BaseModel): bboxes: List[OrderBox] image_bbox: List[float] + + +class TableCell(PolygonBox): + row_id: int + col_id: int + text: str + cell_id: int + + +class TableResult(BaseModel): + cells: List[TableCell] + image_bbox: List[float] diff --git a/surya/settings.py b/surya/settings.py index 33c4bdd..ea02faf 100644 --- a/surya/settings.py +++ b/surya/settings.py @@ -71,6 +71,11 @@ def TORCH_DEVICE_MODEL(self) -> str: ORDER_BATCH_SIZE: Optional[int] = None # Defaults to 4 for CPU/MPS, 32 otherwise ORDER_BENCH_DATASET_NAME: str = "vikp/order_bench" + # Table Rec + TABLE_REC_MODEL_CHECKPOINT: str = "microsoft/table-structure-recognition-v1.1-all" + TABLE_REC_IMAGE_SIZE: Dict = {"height": 1000, "width": 1000} + TABLE_REC_MIN_SCORE: float = 0.5 + # Tesseract (for benchmarks only) TESSDATA_PREFIX: Optional[str] = None diff --git a/surya/tables.py b/surya/tables.py new file mode 100644 index 0000000..3e33ade --- /dev/null +++ b/surya/tables.py @@ -0,0 +1,132 @@ +from typing import List, Optional + +import torch +from PIL import Image +from tqdm import tqdm + +from surya.schema import PolygonBox, TextLine, TableResult, Bbox, TableCell +from surya.settings import settings + + +def get_batch_size(): + batch_size = settings.ORDER_BATCH_SIZE + if batch_size is None: + batch_size = 8 + if settings.TORCH_DEVICE_MODEL == "mps": + batch_size = 8 + if settings.TORCH_DEVICE_MODEL == "cuda": + batch_size = 32 + return batch_size + + +def rescale_boxes(pred_bboxes, image_size): + cx, cy, w, h = pred_bboxes.unbind(-1) + img_h, img_w = image_size + x0 = (cx - 0.5 * w) * img_w + x1 = (cx + 0.5 * w) * img_w + y0 = (cy - 0.5 * h) * img_h + y1 = (cy + 0.5 * h) * img_h + return torch.stack([x0, y0, x1, y1], dim=-1) + + +def sort_table_blocks(cells, tolerance=5) -> list: + vertical_groups = {} + for idx, cell in enumerate(cells): + cell.cell_id = idx # Save id before sorting + bbox = cell.bbox + group_key = round((bbox[1] + bbox[3]) / 2 / tolerance) + if group_key not in vertical_groups: + vertical_groups[group_key] = [] + vertical_groups[group_key].append(cell) + + # Sort each group horizontally and flatten the groups into a single list + sorted_rows = [] + for idx, (_, group) in enumerate(sorted(vertical_groups.items())): + sorted_group = sorted(group, key=lambda x: x.bbox[0]) # sort by x within each row + for cell in sorted_group: + cell.row_id = idx + # TODO: if too few cells in row, merge with row above + sorted_rows.append(sorted_group) + + return sorted_rows + + +def post_process(results, img_size, id2label): + m = results.logits.softmax(-1).max(-1) + pred_labels = list(m.indices.detach().cpu().numpy()) + pred_scores = list(m.values.detach().cpu().numpy()) + pred_bboxes = results.pred_boxes.detach().cpu() + batch_columns = [] + for pred_label, pred_score, pred_bbox in zip(pred_labels, pred_scores, pred_bboxes): + pred_bbox = [elem.tolist() for elem in rescale_boxes(pred_bbox, img_size)] + + columns = [] + for label, score, bbox in zip(pred_label, pred_score, pred_bbox): + class_label = id2label.get(int(label), "unknown") + score = float(score) + if class_label == "table column" and score > settings.TABLE_REC_MIN_SCORE: + columns.append(Bbox(bbox=[float(elem) for elem in bbox])) + columns = sorted(columns, key=lambda x: x.bbox[0]) + batch_columns.append(columns) + return batch_columns + + +def batch_table_recognition(images: List, cells: List[List[PolygonBox]], model, processor, text_lines: Optional[List[List[TextLine]]] = None, batch_size: Optional[int] = None, min_text_assign_score=.2) -> List[TableResult]: + assert all([isinstance(image, Image.Image) for image in images]) + if batch_size is None: + batch_size = get_batch_size() + + all_results = [] + for i in tqdm(range(0, len(images), batch_size), desc="Recognizing tables"): + batch_images = images[i:i + batch_size] + batch_images = [image.convert("RGB") for image in batch_images] # also copies the images + image_bboxes = [[0, 0, img.size[0], img.size[1]] for img in batch_images] + batch_cells = cells[i:i + batch_size] + batch_text_lines = text_lines[i:i + batch_size] if text_lines is not None else None + + pixel_values = processor(batch_images) + pixel_values = pixel_values.to(model.device).to(model.dtype) + + with torch.no_grad(): + outputs = model(pixel_values=pixel_values) + + batch_columns = post_process(outputs, img_size=(settings.RECOGNITION_IMAGE_SIZE["height"], settings.RECOGNITION_IMAGE_SIZE["width"]), id2label=model.config.id2label) + + # Assign cells to columns + results = [] + for columns, cells, image_bbox in zip(batch_columns, batch_cells, image_bboxes): + rows = sort_table_blocks(cells) + result = [] + for idx, row in enumerate(rows): + for cell in row: + cell.col_id = -1 + for col_idx, column in enumerate(columns): + if column.bbox[0] <= cell.bbox[0]: + cell.col_id = col_idx + result.append(TableCell( + row_id=cell.row_id, + cell_id=cell.cell_id, + text="", + col_id=cell.col_id, + polygon=cell.polygon + )) + results.append(TableResult(cells=result, image_bbox=image_bbox)) + + if batch_text_lines is not None: + # Assign text to cells + for text_line, result in zip(batch_text_lines, results): + for text in text_line: + cell_assignment = None + max_intersect = None + for cell_idx, cell in result.cells: + if max_intersect is None or text.intersection_pct(cell) > max_intersect: + max_intersect = text.intersection_pct(cell) + cell_assignment = cell_idx + if max_intersect > min_text_assign_score: + result.cells[cell_assignment].text += text.text + " " + + all_results.extend(results) + return all_results + + + From ad9fbf8420d4e954f46d0e438be106be56d39b8e Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Fri, 20 Sep 2024 15:33:48 -0400 Subject: [PATCH 03/17] Add new table rec model --- ocr_app.py | 12 +- surya/layout.py | 2 +- surya/model/table_rec/config.py | 5 + surya/model/table_rec/decoder.py | 94 ++++++++++++ surya/model/table_rec/encoderdecoder.py | 107 ++++++++++++++ surya/model/table_rec/model.py | 31 +++- surya/model/table_rec/processor.py | 174 ++++++++++++++++++---- surya/schema.py | 4 +- surya/settings.py | 8 +- surya/tables.py | 187 ++++++++++-------------- table_recognition.py | 81 ++++++++++ 11 files changed, 547 insertions(+), 158 deletions(-) create mode 100644 surya/model/table_rec/config.py create mode 100644 surya/model/table_rec/decoder.py create mode 100644 surya/model/table_rec/encoderdecoder.py create mode 100644 table_recognition.py diff --git a/ocr_app.py b/ocr_app.py index 4cf72c6..1f8e749 100644 --- a/ocr_app.py +++ b/ocr_app.py @@ -82,13 +82,13 @@ def table_recognition(img) -> (Image.Image, List[TableResult]): for table_bbox in layout_bboxes: table_imgs.append(img.crop(table_bbox)) table_boxes = batch_text_detection(table_imgs, det_model, det_processor) - table_bboxes = [table_box.bboxes for table_box in table_boxes] + table_bboxes = [[tb.bbox for tb in table_box.bboxes] for table_box in table_boxes] table_preds = batch_table_recognition(table_imgs, table_bboxes, table_model, table_processor) table_img = img.copy() for results, table_bbox in zip(table_preds, layout_bboxes): adjusted_bboxes = [] labels = [] - for item in results.bboxes: + for item in results.cells: adjusted_bboxes.append([ item.bbox[0] + table_bbox[0], item.bbox[1] + table_bbox[1], @@ -96,14 +96,6 @@ def table_recognition(img) -> (Image.Image, List[TableResult]): item.bbox[3] + table_bbox[1] ]) labels.append(f"{item.row_id} / {item.col_id}") - for item in results.unused_bboxes: - adjusted_bboxes.append([ - item.bbox[0] + table_bbox[0], - item.bbox[1] + table_bbox[1], - item.bbox[2] + table_bbox[0], - item.bbox[3] + table_bbox[1] - ]) - labels.append("Unused") table_img = draw_bboxes_on_image(adjusted_bboxes, table_img, labels=labels) return table_img, table_preds diff --git a/surya/layout.py b/surya/layout.py index 7afc54b..070e000 100644 --- a/surya/layout.py +++ b/surya/layout.py @@ -113,7 +113,7 @@ def get_regions_from_detection_result(detection_result: TextDetectionResult, hea if bbox2.label != ftype or bbox_idx2 in to_remove or bbox_idx == bbox_idx2: continue - if bbox.intersection_pct(bbox2, x_margin=.05) > 0: + if bbox.intersection_pct(bbox2, x_margin=.25) > .1: bbox.merge(bbox2) to_remove.add(bbox_idx2) diff --git a/surya/model/table_rec/config.py b/surya/model/table_rec/config.py new file mode 100644 index 0000000..02e5d5b --- /dev/null +++ b/surya/model/table_rec/config.py @@ -0,0 +1,5 @@ +from transformers import MBartConfig + + +class TableRecDecoderConfig(MBartConfig): + pass \ No newline at end of file diff --git a/surya/model/table_rec/decoder.py b/surya/model/table_rec/decoder.py new file mode 100644 index 0000000..77f983d --- /dev/null +++ b/surya/model/table_rec/decoder.py @@ -0,0 +1,94 @@ +import copy +from dataclasses import dataclass +from typing import Optional, List, Tuple, Union + +import torch +from torch import nn +from transformers import MBartForCausalLM, MBartPreTrainedModel +from transformers.utils import ModelOutput + +from surya.model.ordering.decoder import MBartOrderDecoderWrapper +from surya.model.table_rec.config import TableRecDecoderConfig + + +@dataclass +class TableRecDecoderOutput(ModelOutput): + loss: Optional[torch.FloatTensor] = None + row_logits: torch.FloatTensor = None + col_logits: torch.FloatTensor = None + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None + hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None + attentions: Optional[Tuple[torch.FloatTensor, ...]] = None + cross_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None + + +class TableRecDecoder(MBartForCausalLM): + config_class = TableRecDecoderConfig + _tied_weights_keys = [] + + def __init__(self, config, **kwargs): + config = copy.deepcopy(config) + config.is_decoder = True + config.is_encoder_decoder = False + MBartPreTrainedModel.__init__(self, config) + self.model = MBartOrderDecoderWrapper(config) + + self.row_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + self.col_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def forward( + self, + input_boxes: torch.LongTensor = None, + input_boxes_mask: Optional[torch.Tensor] = None, + input_boxes_counts: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **kwargs + ) -> Union[Tuple, TableRecDecoderOutput]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model.decoder( + input_boxes=input_boxes, + input_boxes_mask=input_boxes_mask, + input_boxes_counts=input_boxes_counts, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + head_mask=head_mask, + cross_attn_head_mask=cross_attn_head_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + row_logits = self.row_head(outputs[0]) + col_logits = self.col_head(outputs[0]) + + return TableRecDecoderOutput( + loss=None, + col_logits=col_logits, + row_logits=row_logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + cross_attentions=outputs.cross_attentions, + ) \ No newline at end of file diff --git a/surya/model/table_rec/encoderdecoder.py b/surya/model/table_rec/encoderdecoder.py new file mode 100644 index 0000000..dda5990 --- /dev/null +++ b/surya/model/table_rec/encoderdecoder.py @@ -0,0 +1,107 @@ +from dataclasses import dataclass +from typing import Optional, Tuple, List, Union + +import torch +from transformers import VisionEncoderDecoderModel +from transformers.modeling_outputs import BaseModelOutput +from transformers.utils import ModelOutput + + +@dataclass +class TableRecOutput(ModelOutput): + loss: Optional[torch.FloatTensor] = None + row_logits: torch.FloatTensor = None + col_logits: torch.FloatTensor = None + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None + decoder_hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None + decoder_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None + cross_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None + encoder_last_hidden_state: Optional[torch.FloatTensor] = None + encoder_hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None + encoder_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None + + +class TableRecVisionEncoderDecoderModel(VisionEncoderDecoderModel): + def forward( + self, + pixel_values: Optional[torch.FloatTensor] = None, + decoder_input_boxes: torch.LongTensor = None, + # Shape (batch_size, num_boxes, 4), all coords scaled 0 - 1000, with 1001 as padding + decoder_input_boxes_mask: torch.LongTensor = None, # Shape (batch_size, num_boxes), 0 if padding, 1 otherwise + decoder_input_boxes_counts: torch.LongTensor = None, # Shape (batch_size), number of boxes in each image + encoder_outputs: Optional[Tuple[torch.FloatTensor]] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[List[List[int]]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **kwargs, + ) -> Union[Tuple[torch.FloatTensor], TableRecOutput]: + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + kwargs_encoder = {argument: value for argument, value in kwargs.items() if not argument.startswith("decoder_")} + + kwargs_decoder = { + argument[len("decoder_") :]: value for argument, value in kwargs.items() if argument.startswith("decoder_") + } + + if encoder_outputs is None: + if pixel_values is None: + raise ValueError("You have to specify pixel_values") + + encoder_outputs = self.encoder( + pixel_values=pixel_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + **kwargs_encoder, + ) + elif isinstance(encoder_outputs, tuple): + encoder_outputs = BaseModelOutput(*encoder_outputs) + + encoder_hidden_states = encoder_outputs[0] + + # optionally project encoder_hidden_states + if ( + self.encoder.config.hidden_size != self.decoder.config.hidden_size + and self.decoder.config.cross_attention_hidden_size is None + ): + encoder_hidden_states = self.enc_to_dec_proj(encoder_hidden_states) + + # else: + encoder_attention_mask = None + + # Decode + decoder_outputs = self.decoder( + input_boxes=decoder_input_boxes, + input_boxes_mask=decoder_input_boxes_mask, + input_boxes_counts=decoder_input_boxes_counts, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + inputs_embeds=decoder_inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + use_cache=use_cache, + past_key_values=past_key_values, + return_dict=return_dict, + labels=labels, + **kwargs_decoder, + ) + + if not return_dict: + return decoder_outputs + encoder_outputs + + return TableRecOutput( + loss=decoder_outputs.loss, + row_logits=decoder_outputs.row_logits, + col_logits=decoder_outputs.col_logits, + past_key_values=decoder_outputs.past_key_values, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + ) diff --git a/surya/model/table_rec/model.py b/surya/model/table_rec/model.py index b73607a..e2b9df2 100644 --- a/surya/model/table_rec/model.py +++ b/surya/model/table_rec/model.py @@ -1,15 +1,34 @@ -from transformers import TableTransformerForObjectDetection +from transformers import VisionEncoderDecoderConfig, AutoModel, AutoModelForCausalLM +from surya.model.ordering.config import VariableDonutSwinConfig +from surya.model.ordering.encoder import VariableDonutSwinModel +from surya.model.table_rec.config import TableRecDecoderConfig +from surya.model.table_rec.decoder import TableRecDecoder +from surya.model.table_rec.encoderdecoder import TableRecVisionEncoderDecoderModel from surya.settings import settings def load_model(checkpoint=settings.TABLE_REC_MODEL_CHECKPOINT, device=settings.TORCH_DEVICE_MODEL, dtype=settings.MODEL_DTYPE): - model = TableTransformerForObjectDetection.from_pretrained(checkpoint, torch_dtype=dtype) + config = VisionEncoderDecoderConfig.from_pretrained(checkpoint) - model = model.to(device) - model = model.eval() + decoder_config = vars(config.decoder) + decoder = TableRecDecoderConfig(**decoder_config) + config.decoder = decoder + + encoder_config = vars(config.encoder) + encoder = VariableDonutSwinConfig(**encoder_config) + config.encoder = encoder - print(f"Loaded table model {checkpoint} on device {device} with dtype {dtype}") + # Get transformers to load custom model + AutoModel.register(TableRecDecoderConfig, TableRecDecoder) + AutoModelForCausalLM.register(TableRecDecoderConfig, TableRecDecoder) + AutoModel.register(VariableDonutSwinConfig, VariableDonutSwinModel) - return model + model = TableRecVisionEncoderDecoderModel.from_pretrained(checkpoint, config=config, torch_dtype=dtype) + assert isinstance(model.decoder, TableRecDecoder) + assert isinstance(model.encoder, VariableDonutSwinModel) + model = model.to(device) + model = model.eval() + print(f"Loaded reading order model {checkpoint} on device {device} with dtype {dtype}") + return model \ No newline at end of file diff --git a/surya/model/table_rec/processor.py b/surya/model/table_rec/processor.py index 0f00268..4b6196c 100644 --- a/surya/model/table_rec/processor.py +++ b/surya/model/table_rec/processor.py @@ -1,32 +1,156 @@ -import cv2 -import torch -import torch.nn.functional as F -from surya.settings import settings +from copy import deepcopy +from typing import List, Dict, Optional, Union + import numpy as np +from PIL import Image +from torch import TensorType +from transformers import DonutImageProcessor, BatchFeature +from transformers.image_utils import ChannelDimension, ImageInput, PILImageResampling, make_list_of_images, valid_images + +from surya.settings import settings + + +def load_processor(checkpoint=settings.TABLE_REC_MODEL_CHECKPOINT): + processor = TableRecImageProcessor.from_pretrained(checkpoint) + processor.size = settings.TABLE_REC_IMAGE_SIZE + box_size = 1024 + max_tokens = 384 + processor.token_sep_id = max_tokens + box_size + 1 + processor.token_pad_id = max_tokens + box_size + 2 + processor.token_unused_id = max_tokens + 3 # This is a label, so don't add box size + processor.token_row_id = max_tokens + box_size + 4 + processor.token_eos_id = max_tokens + box_size + 5 + processor.max_boxes = settings.TABLE_REC_MAX_BOXES - 1 + processor.box_size = {"height": box_size, "width": box_size} + return processor + + +class TableRecImageProcessor(DonutImageProcessor): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.patch_size = kwargs.get("patch_size", (4, 4)) + + def process_inner(self, images: List[np.ndarray]): + images = [img.transpose(2, 0, 1) for img in images] # convert to CHW format + + assert images[0].shape[0] == 3 # RGB input images, channel dim last + + # Convert to float32 for rescale/normalize + images = [img.astype(np.float32) for img in images] + + # Rescale and normalize + images = [ + self.rescale(img, scale=self.rescale_factor, input_data_format=ChannelDimension.FIRST) + for img in images + ] + images = [ + self.normalize(img, mean=self.image_mean, std=self.image_std, input_data_format=ChannelDimension.FIRST) + for img in images + ] + + return images + + def process_boxes(self, boxes): + padded_boxes = [] + box_masks = [] + box_counts = [] + for b in boxes: + # Left pad for generation + padded_b = deepcopy(b) + padded_b.insert(0, [self.token_row_id] * 4) # special token for max col/row + padded_boxes.append(padded_b) + + max_boxes = max(len(b) for b in padded_boxes) + for i in range(len(padded_boxes)): + pad_len = max_boxes - len(padded_boxes[i]) + box_len = len(padded_boxes[i]) + box_mask = [0] * pad_len + [1] * box_len + padded_box = [[self.token_pad_id] * 4] * pad_len + padded_boxes[i] + padded_boxes[i] = padded_box + box_masks.append(box_mask) + box_counts.append([pad_len, max_boxes]) + + return padded_boxes, box_masks, box_counts + + def resize_img_and_boxes(self, img, boxes): + orig_dim = img.size + new_size = (self.size["width"], self.size["height"]) + img.thumbnail(new_size, Image.Resampling.LANCZOS) # Shrink largest dimension to fit new size + img = img.resize(new_size, Image.Resampling.LANCZOS) # Stretch smaller dimension to fit new size + + img = np.asarray(img, dtype=np.uint8) + + width, height = orig_dim + box_width, box_height = self.box_size["width"], self.box_size["height"] + for box in boxes: + # Rescale to 0-1024 + box[0] = box[0] / width * box_width + box[1] = box[1] / height * box_height + box[2] = box[2] / width * box_width + box[3] = box[3] / height * box_height + + if box[0] < 0: + box[0] = 0 + if box[1] < 0: + box[1] = 0 + if box[2] > box_width: + box[2] = box_width + if box[3] > box_height: + box[3] = box_height + + return img, boxes + def preprocess( + self, + images: ImageInput, + boxes: List[List[int]], + do_resize: bool = None, + size: Dict[str, int] = None, + resample: PILImageResampling = None, + do_thumbnail: bool = None, + do_align_long_axis: bool = None, + do_pad: bool = None, + random_padding: bool = False, + do_rescale: bool = None, + rescale_factor: float = None, + do_normalize: bool = None, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> Image.Image: + images = make_list_of_images(images) -def load_processor(): - def _processor(imgs): - mean_vals = [0.485, 0.456, 0.406] - std_vals = [0.229, 0.224, 0.225] - imgs = [np.array(img) for img in imgs] - imgs = [_normalize(_resize(img), mean_vals, std_vals) for img in imgs] - pixel_values = torch.stack(imgs, dim=0) - return pixel_values - return _processor + if not valid_images(images): + raise ValueError( + "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " + "torch.Tensor, tf.Tensor or jax.ndarray." + ) + new_images = [] + new_boxes = [] + for img, box in zip(images, boxes): + if len(box) > self.max_boxes: + raise ValueError(f"Too many boxes, max is {self.max_boxes}") + img, box = self.resize_img_and_boxes(img, box) + new_images.append(img) + new_boxes.append(box) -def _resize(image, interpolation=cv2.INTER_LANCZOS4): - max_height, max_width = settings.TABLE_REC_IMAGE_SIZE["height"], settings.TABLE_REC_IMAGE_SIZE["width"] - resized_image = cv2.resize(image, (max_width, max_height), interpolation=interpolation) - resized_image = resized_image.transpose(2, 0, 1).astype(np.float32) - resized_image = torch.from_numpy(resized_image) - resized_image /= 255.0 - return resized_image + images = new_images + boxes = new_boxes + # Convert to numpy for later processing steps + images = [np.array(image) for image in images] -def _normalize(tensor, mean, std): - mean = torch.tensor(mean).view(-1, 1, 1) - std = torch.tensor(std).view(-1, 1, 1) - tensor = (tensor - mean) / std - return tensor \ No newline at end of file + images = self.process_inner(images) + boxes, box_mask, box_counts = self.process_boxes(boxes) + data = { + "pixel_values": images, + "input_boxes": boxes, + "input_boxes_mask": box_mask, + "input_boxes_counts": box_counts, + } + return BatchFeature(data=data, tensor_type=return_tensors) \ No newline at end of file diff --git a/surya/schema.py b/surya/schema.py index 3d5b8e1..fb88922 100644 --- a/surya/schema.py +++ b/surya/schema.py @@ -167,11 +167,9 @@ class OrderResult(BaseModel): image_bbox: List[float] -class TableCell(PolygonBox): +class TableCell(Bbox): row_id: int col_id: int - text: str - cell_id: int class TableResult(BaseModel): diff --git a/surya/settings.py b/surya/settings.py index ea02faf..bc0cb1b 100644 --- a/surya/settings.py +++ b/surya/settings.py @@ -61,7 +61,7 @@ def TORCH_DEVICE_MODEL(self) -> str: RECOGNITION_ENCODER_BATCH_DIVISOR: int = 2 # Divisor for batch size in decoder # Layout - LAYOUT_MODEL_CHECKPOINT: str = "vikp/surya_layout3" + LAYOUT_MODEL_CHECKPOINT: str = "vikp/layout5" LAYOUT_BENCH_DATASET_NAME: str = "vikp/publaynet_bench" # Ordering @@ -72,9 +72,9 @@ def TORCH_DEVICE_MODEL(self) -> str: ORDER_BENCH_DATASET_NAME: str = "vikp/order_bench" # Table Rec - TABLE_REC_MODEL_CHECKPOINT: str = "microsoft/table-structure-recognition-v1.1-all" - TABLE_REC_IMAGE_SIZE: Dict = {"height": 1000, "width": 1000} - TABLE_REC_MIN_SCORE: float = 0.5 + TABLE_REC_MODEL_CHECKPOINT: str = "vikp/table_rec" + TABLE_REC_IMAGE_SIZE: Dict = {"height": 1024, "width": 1024} + TABLE_REC_MAX_BOXES: int = 384 # Tesseract (for benchmarks only) TESSDATA_PREFIX: Optional[str] = None diff --git a/surya/tables.py b/surya/tables.py index 3e33ade..1e10841 100644 --- a/surya/tables.py +++ b/surya/tables.py @@ -1,11 +1,13 @@ -from typing import List, Optional - +from copy import deepcopy +from typing import List import torch from PIL import Image -from tqdm import tqdm -from surya.schema import PolygonBox, TextLine, TableResult, Bbox, TableCell +from surya.model.ordering.encoderdecoder import OrderVisionEncoderDecoderModel +from surya.schema import TableResult, TableCell from surya.settings import settings +from tqdm import tqdm +import numpy as np def get_batch_size(): @@ -19,114 +21,81 @@ def get_batch_size(): return batch_size -def rescale_boxes(pred_bboxes, image_size): - cx, cy, w, h = pred_bboxes.unbind(-1) - img_h, img_w = image_size - x0 = (cx - 0.5 * w) * img_w - x1 = (cx + 0.5 * w) * img_w - y0 = (cy - 0.5 * h) * img_h - y1 = (cy + 0.5 * h) * img_h - return torch.stack([x0, y0, x1, y1], dim=-1) - - -def sort_table_blocks(cells, tolerance=5) -> list: - vertical_groups = {} - for idx, cell in enumerate(cells): - cell.cell_id = idx # Save id before sorting - bbox = cell.bbox - group_key = round((bbox[1] + bbox[3]) / 2 / tolerance) - if group_key not in vertical_groups: - vertical_groups[group_key] = [] - vertical_groups[group_key].append(cell) - - # Sort each group horizontally and flatten the groups into a single list - sorted_rows = [] - for idx, (_, group) in enumerate(sorted(vertical_groups.items())): - sorted_group = sorted(group, key=lambda x: x.bbox[0]) # sort by x within each row - for cell in sorted_group: - cell.row_id = idx - # TODO: if too few cells in row, merge with row above - sorted_rows.append(sorted_group) - - return sorted_rows - - -def post_process(results, img_size, id2label): - m = results.logits.softmax(-1).max(-1) - pred_labels = list(m.indices.detach().cpu().numpy()) - pred_scores = list(m.values.detach().cpu().numpy()) - pred_bboxes = results.pred_boxes.detach().cpu() - batch_columns = [] - for pred_label, pred_score, pred_bbox in zip(pred_labels, pred_scores, pred_bboxes): - pred_bbox = [elem.tolist() for elem in rescale_boxes(pred_bbox, img_size)] - - columns = [] - for label, score, bbox in zip(pred_label, pred_score, pred_bbox): - class_label = id2label.get(int(label), "unknown") - score = float(score) - if class_label == "table column" and score > settings.TABLE_REC_MIN_SCORE: - columns.append(Bbox(bbox=[float(elem) for elem in bbox])) - columns = sorted(columns, key=lambda x: x.bbox[0]) - batch_columns.append(columns) - return batch_columns - - -def batch_table_recognition(images: List, cells: List[List[PolygonBox]], model, processor, text_lines: Optional[List[List[TextLine]]] = None, batch_size: Optional[int] = None, min_text_assign_score=.2) -> List[TableResult]: +def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model: OrderVisionEncoderDecoderModel, processor, batch_size=None) -> List[TableResult]: assert all([isinstance(image, Image.Image) for image in images]) + assert len(images) == len(bboxes) if batch_size is None: batch_size = get_batch_size() - all_results = [] - for i in tqdm(range(0, len(images), batch_size), desc="Recognizing tables"): - batch_images = images[i:i + batch_size] - batch_images = [image.convert("RGB") for image in batch_images] # also copies the images - image_bboxes = [[0, 0, img.size[0], img.size[1]] for img in batch_images] - batch_cells = cells[i:i + batch_size] - batch_text_lines = text_lines[i:i + batch_size] if text_lines is not None else None - - pixel_values = processor(batch_images) - pixel_values = pixel_values.to(model.device).to(model.dtype) - - with torch.no_grad(): - outputs = model(pixel_values=pixel_values) - - batch_columns = post_process(outputs, img_size=(settings.RECOGNITION_IMAGE_SIZE["height"], settings.RECOGNITION_IMAGE_SIZE["width"]), id2label=model.config.id2label) - - # Assign cells to columns - results = [] - for columns, cells, image_bbox in zip(batch_columns, batch_cells, image_bboxes): - rows = sort_table_blocks(cells) - result = [] - for idx, row in enumerate(rows): - for cell in row: - cell.col_id = -1 - for col_idx, column in enumerate(columns): - if column.bbox[0] <= cell.bbox[0]: - cell.col_id = col_idx - result.append(TableCell( - row_id=cell.row_id, - cell_id=cell.cell_id, - text="", - col_id=cell.col_id, - polygon=cell.polygon - )) - results.append(TableResult(cells=result, image_bbox=image_bbox)) - - if batch_text_lines is not None: - # Assign text to cells - for text_line, result in zip(batch_text_lines, results): - for text in text_line: - cell_assignment = None - max_intersect = None - for cell_idx, cell in result.cells: - if max_intersect is None or text.intersection_pct(cell) > max_intersect: - max_intersect = text.intersection_pct(cell) - cell_assignment = cell_idx - if max_intersect > min_text_assign_score: - result.cells[cell_assignment].text += text.text + " " - - all_results.extend(results) - return all_results - + output_order = [] + for i in tqdm(range(0, len(images), batch_size), desc="Finding reading order"): + batch_bboxes = deepcopy(bboxes[i:i+batch_size]) + batch_images = images[i:i+batch_size] + batch_images = [image.convert("RGB") for image in batch_images] # also copies the images + orig_sizes = [image.size for image in batch_images] + model_inputs = processor(images=batch_images, boxes=batch_bboxes) + + batch_pixel_values = model_inputs["pixel_values"] + batch_bboxes = model_inputs["input_boxes"] + batch_bbox_mask = model_inputs["input_boxes_mask"] + batch_bbox_counts = model_inputs["input_boxes_counts"] + + batch_bboxes = torch.from_numpy(np.array(batch_bboxes, dtype=np.int32)).to(model.device) + batch_bbox_mask = torch.from_numpy(np.array(batch_bbox_mask, dtype=np.int32)).to(model.device) + batch_pixel_values = torch.tensor(np.array(batch_pixel_values), dtype=model.dtype).to(model.device) + batch_bbox_counts = torch.tensor(np.array(batch_bbox_counts), dtype=torch.long).to(model.device) + + col_predictions = [] + row_predictions = [] + max_rows = [] + max_cols = [] + with torch.inference_mode(): + return_dict = model( + pixel_values=batch_pixel_values, + decoder_input_boxes=batch_bboxes, + decoder_input_boxes_mask=batch_bbox_mask, + decoder_input_boxes_counts=batch_bbox_counts, + encoder_outputs=None, + past_key_values=None, + ) + row_logits = return_dict["row_logits"].detach() + col_logits = return_dict["col_logits"].detach() + + for z in range(len(batch_images)): + box_start_idx = batch_bbox_counts[z][0] + row_preds = row_logits[z][box_start_idx:].argmax(dim=-1) + max_row = row_preds[0] + row_preds = row_preds[1:] + + col_preds = col_logits[z][box_start_idx:].argmax(dim=-1) + max_col = col_preds[0] + col_preds = col_preds[1:] + + row_predictions.append(row_preds) + col_predictions.append(col_preds) + max_rows.append(max_row) + max_cols.append(max_col) + + assert len(row_predictions) == len(col_predictions) == len(max_rows) == len(max_cols) == len(batch_images) + for j, (row_pred, col_pred, max_row, max_col) in enumerate(zip(row_predictions, col_predictions, max_rows, max_cols)): + row_bboxes = bboxes[i+j] + + orig_size = orig_sizes[j] + out_data = [] + assert len(row_pred) == len(col_pred) == len(row_bboxes) + for z, (row_idx, col_idx, bbox) in enumerate(zip(row_pred, col_pred, row_bboxes)): + cell = TableCell( + bbox=bbox, + col_id=col_idx, + row_id=row_idx + ) + out_data.append(cell) + + result = TableResult( + cells=out_data, + image_bbox=[0, 0, orig_size[0], orig_size[1]], + ) + output_order.append(result) + return output_order \ No newline at end of file diff --git a/table_recognition.py b/table_recognition.py new file mode 100644 index 0000000..c83d2e3 --- /dev/null +++ b/table_recognition.py @@ -0,0 +1,81 @@ +import os +import argparse +import copy +import json +from collections import defaultdict + +from surya.detection import batch_text_detection +from surya.input.load import load_from_folder, load_from_file +from surya.layout import batch_layout_detection +from surya.model.detection.model import load_model as load_det_model, load_processor as load_det_processor +from surya.model.ordering.model import load_model +from surya.model.ordering.processor import load_processor +from surya.ordering import batch_ordering +from surya.postprocessing.heatmap import draw_polys_on_image +from surya.settings import settings + + +def main(): + parser = argparse.ArgumentParser(description="Find reading order of an input file or folder (PDFs or image).") + parser.add_argument("input_path", type=str, help="Path to pdf or image file or folder to find reading order in.") + parser.add_argument("--results_dir", type=str, help="Path to JSON file with layout results.", default=os.path.join(settings.RESULT_DIR, "surya")) + parser.add_argument("--max", type=int, help="Maximum number of pages to process.", default=None) + parser.add_argument("--images", action="store_true", help="Save images of detected layout bboxes.", default=False) + args = parser.parse_args() + + model = load_model() + processor = load_processor() + + layout_model = load_det_model(checkpoint=settings.LAYOUT_MODEL_CHECKPOINT) + layout_processor = load_det_processor(checkpoint=settings.LAYOUT_MODEL_CHECKPOINT) + + det_model = load_det_model() + det_processor = load_det_processor() + + if os.path.isdir(args.input_path): + images, names = load_from_folder(args.input_path, args.max) + folder_name = os.path.basename(args.input_path) + else: + images, names = load_from_file(args.input_path, args.max) + folder_name = os.path.basename(args.input_path).split(".")[0] + + line_predictions = batch_text_detection(images, det_model, det_processor) + layout_predictions = batch_layout_detection(images, layout_model, layout_processor, line_predictions) + table_boxes = [] + for layout_pred in layout_predictions: + bbox = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] + table_boxes.append(bbox) + + order_predictions = batch_ordering(images, bboxes, model, processor) + result_path = os.path.join(args.results_dir, folder_name) + os.makedirs(result_path, exist_ok=True) + + if args.images: + for idx, (image, layout_pred, order_pred, name) in enumerate(zip(images, layout_predictions, order_predictions, names)): + polys = [l.polygon for l in order_pred.bboxes] + labels = [str(l.position) for l in order_pred.bboxes] + bbox_image = draw_polys_on_image(polys, copy.deepcopy(image), labels=labels, label_font_size=20) + bbox_image.save(os.path.join(result_path, f"{name}_{idx}_order.png")) + + predictions_by_page = defaultdict(list) + for idx, (layout_pred, pred, name, image) in enumerate(zip(layout_predictions, order_predictions, names, images)): + out_pred = pred.model_dump() + for bbox, layout_bbox in zip(out_pred["bboxes"], layout_pred.bboxes): + bbox["label"] = layout_bbox.label + + out_pred["page"] = len(predictions_by_page[name]) + 1 + predictions_by_page[name].append(out_pred) + + # Sort in reading order + for name in predictions_by_page: + for page_preds in predictions_by_page[name]: + page_preds["bboxes"] = sorted(page_preds["bboxes"], key=lambda x: x["position"]) + + with open(os.path.join(result_path, "results.json"), "w+", encoding="utf-8") as f: + json.dump(predictions_by_page, f, ensure_ascii=False) + + print(f"Wrote results to {result_path}") + + +if __name__ == "__main__": + main() \ No newline at end of file From 95f2eb62d108588f9a1a63b532539587bbb842b3 Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Fri, 20 Sep 2024 22:05:21 -0400 Subject: [PATCH 04/17] Add bbox sorting --- surya/tables.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/surya/tables.py b/surya/tables.py index 1e10841..9b673ac 100644 --- a/surya/tables.py +++ b/surya/tables.py @@ -21,21 +21,39 @@ def get_batch_size(): return batch_size +def sort_bboxes(bboxes, tolerance=1): + vertical_groups = {} + for block in bboxes: + group_key = round(block[1] / tolerance) * tolerance + if group_key not in vertical_groups: + vertical_groups[group_key] = [] + vertical_groups[group_key].append(block) + + # Sort each group horizontally and flatten the groups into a single list + sorted_page_blocks = [] + for _, group in sorted(vertical_groups.items()): + sorted_group = sorted(group, key=lambda x: x[0]) + sorted_page_blocks.extend(sorted_group) + + return sorted_page_blocks + + def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model: OrderVisionEncoderDecoderModel, processor, batch_size=None) -> List[TableResult]: assert all([isinstance(image, Image.Image) for image in images]) assert len(images) == len(bboxes) if batch_size is None: batch_size = get_batch_size() - output_order = [] for i in tqdm(range(0, len(images), batch_size), desc="Finding reading order"): - batch_bboxes = deepcopy(bboxes[i:i+batch_size]) + batch_list_bboxes = deepcopy(bboxes[i:i+batch_size]) + batch_list_bboxes = [sort_bboxes(page_bboxes) for page_bboxes in batch_list_bboxes] # Sort bboxes before passing in + batch_images = images[i:i+batch_size] batch_images = [image.convert("RGB") for image in batch_images] # also copies the images orig_sizes = [image.size for image in batch_images] - model_inputs = processor(images=batch_images, boxes=batch_bboxes) + model_inputs = processor(images=batch_images, boxes=deepcopy(batch_list_bboxes)) batch_pixel_values = model_inputs["pixel_values"] batch_bboxes = model_inputs["input_boxes"] @@ -79,9 +97,7 @@ def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model max_cols.append(max_col) assert len(row_predictions) == len(col_predictions) == len(max_rows) == len(max_cols) == len(batch_images) - for j, (row_pred, col_pred, max_row, max_col) in enumerate(zip(row_predictions, col_predictions, max_rows, max_cols)): - row_bboxes = bboxes[i+j] - + for j, (row_pred, col_pred, max_row, max_col, row_bboxes) in enumerate(zip(row_predictions, col_predictions, max_rows, max_cols, batch_list_bboxes)): orig_size = orig_sizes[j] out_data = [] assert len(row_pred) == len(col_pred) == len(row_bboxes) From 31df896bae1ca2344f682dd023fde81b1bce4a2a Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Fri, 27 Sep 2024 11:37:17 -0400 Subject: [PATCH 05/17] Initial table rec ar model --- ocr_app.py | 23 +- poetry.lock | 134 +++- pyproject.toml | 1 + surya/detection.py | 4 +- surya/input/pdflines.py | 65 ++ surya/model/table_rec/config.py | 253 +++++++- surya/model/table_rec/decoder.py | 789 ++++++++++++++++++++++-- surya/model/table_rec/encoderdecoder.py | 170 ++--- surya/model/table_rec/model.py | 40 +- surya/model/table_rec/processor.py | 282 ++++++--- surya/postprocessing/text.py | 6 +- surya/settings.py | 7 +- surya/tables.py | 116 +++- 13 files changed, 1583 insertions(+), 307 deletions(-) create mode 100644 surya/input/pdflines.py diff --git a/ocr_app.py b/ocr_app.py index 1f8e749..de5f25d 100644 --- a/ocr_app.py +++ b/ocr_app.py @@ -4,6 +4,7 @@ import pypdfium2 import streamlit as st from surya.detection import batch_text_detection +from surya.input.pdflines import get_page_text_lines, get_table_blocks from surya.layout import batch_layout_detection from surya.model.detection.model import load_model, load_processor from surya.model.recognition.model import load_model as load_rec_model @@ -75,17 +76,24 @@ def order_detection(img) -> (Image.Image, OrderResult): return order_img, pred -def table_recognition(img) -> (Image.Image, List[TableResult]): +def table_recognition(img, filepath, page_idx: int, use_pdf_boxes: bool) -> (Image.Image, List[TableResult]): _, layout_pred = layout_detection(img) - layout_bboxes = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] + layout_tables = [l for l in layout_pred.bboxes if l.label == "Table"] + layout_tables_bboxes = [l.bbox for l in layout_tables] + table_imgs = [] - for table_bbox in layout_bboxes: + for table_bbox in layout_tables_bboxes: table_imgs.append(img.crop(table_bbox)) - table_boxes = batch_text_detection(table_imgs, det_model, det_processor) - table_bboxes = [[tb.bbox for tb in table_box.bboxes] for table_box in table_boxes] + if use_pdf_boxes: + page_text = get_page_text_lines(filepath, page_idx, img.size) + table_texts = get_table_blocks(layout_tables, page_text, img.size) + table_bboxes = [[tb["bbox"] for tb in table_text] for table_text in table_texts] + else: + table_boxes = batch_text_detection(table_imgs, det_model, det_processor) + table_bboxes = [[tb.bbox for tb in table_box.bboxes] for table_box in table_boxes] table_preds = batch_table_recognition(table_imgs, table_bboxes, table_model, table_processor) table_img = img.copy() - for results, table_bbox in zip(table_preds, layout_bboxes): + for results, table_bbox in zip(table_preds, layout_tables_bboxes): adjusted_bboxes = [] labels = [] for item in results.cells: @@ -180,6 +188,7 @@ def page_count(pdf_file): layout_det = st.sidebar.button("Run Layout Analysis") order_det = st.sidebar.button("Run Reading Order") table_rec = st.sidebar.button("Run Table Rec") +use_pdf_boxes = st.sidebar.checkbox("PDF table boxes", value=True, help="Table recognition only: Use the bounding boxes from the PDF file vs text detection model.") if pil_image is None: st.stop() @@ -218,7 +227,7 @@ def page_count(pdf_file): if table_rec: - table_img, pred = table_recognition(pil_image) + table_img, pred = table_recognition(pil_image, in_file, page_number - 1, use_pdf_boxes) with col1: st.image(table_img, caption="Table Recognition", use_column_width=True) st.json([p.model_dump() for p in pred], expanded=True) diff --git a/poetry.lock b/poetry.lock index 28087ff..d0cd8ee 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1333,6 +1333,17 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "joblib" +version = "1.4.2" +description = "Lightweight pipelining with Python functions" +optional = false +python-versions = ">=3.8" +files = [ + {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"}, + {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, +] + [[package]] name = "json5" version = "0.9.25" @@ -2315,11 +2326,11 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.21.0", markers = "python_version == \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\" and python_version < \"3.11\""}, {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\" and python_version < \"3.11\""}, {version = ">=1.19.3", markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and python_version >= \"3.8\" and python_version < \"3.10\" or python_version > \"3.9\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_system != \"Darwin\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, - {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] [[package]] @@ -2385,8 +2396,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.22.4", markers = "python_version < \"3.11\""}, - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -2443,6 +2454,23 @@ files = [ qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] testing = ["docopt", "pytest"] +[[package]] +name = "pdftext" +version = "0.3.10" +description = "Extract structured text from pdfs quickly" +optional = false +python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,!=3.8.*,>=3.9" +files = [ + {file = "pdftext-0.3.10-py3-none-any.whl", hash = "sha256:99bd900d0d0692df06719c07ce10a859750ade3eb7f10c543f637118417497f9"}, + {file = "pdftext-0.3.10.tar.gz", hash = "sha256:90de726e818fb5683a0616cabb1a75a32a7224e873c3058006c93da6e440c66c"}, +] + +[package.dependencies] +pydantic = ">=2.7.1,<3.0.0" +pydantic-settings = ">=2.2.1,<3.0.0" +pypdfium2 = ">=4.29.0,<5.0.0" +scikit-learn = ">=1.4.2,<2.0.0" + [[package]] name = "pexpect" version = "4.9.0" @@ -3823,6 +3851,93 @@ tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "hypothesis (>=6.70.2)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "safetensors[numpy]", "setuptools-rust (>=1.5.2)"] torch = ["safetensors[numpy]", "torch (>=1.10)"] +[[package]] +name = "scikit-learn" +version = "1.5.2" +description = "A set of python modules for machine learning and data mining" +optional = false +python-versions = ">=3.9" +files = [ + {file = "scikit_learn-1.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:299406827fb9a4f862626d0fe6c122f5f87f8910b86fe5daa4c32dcd742139b6"}, + {file = "scikit_learn-1.5.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:2d4cad1119c77930b235579ad0dc25e65c917e756fe80cab96aa3b9428bd3fb0"}, + {file = "scikit_learn-1.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c412ccc2ad9bf3755915e3908e677b367ebc8d010acbb3f182814524f2e5540"}, + {file = "scikit_learn-1.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a686885a4b3818d9e62904d91b57fa757fc2bed3e465c8b177be652f4dd37c8"}, + {file = "scikit_learn-1.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:c15b1ca23d7c5f33cc2cb0a0d6aaacf893792271cddff0edbd6a40e8319bc113"}, + {file = "scikit_learn-1.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:03b6158efa3faaf1feea3faa884c840ebd61b6484167c711548fce208ea09445"}, + {file = "scikit_learn-1.5.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:1ff45e26928d3b4eb767a8f14a9a6efbf1cbff7c05d1fb0f95f211a89fd4f5de"}, + {file = "scikit_learn-1.5.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f763897fe92d0e903aa4847b0aec0e68cadfff77e8a0687cabd946c89d17e675"}, + {file = "scikit_learn-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8b0ccd4a902836493e026c03256e8b206656f91fbcc4fde28c57a5b752561f1"}, + {file = "scikit_learn-1.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:6c16d84a0d45e4894832b3c4d0bf73050939e21b99b01b6fd59cbb0cf39163b6"}, + {file = "scikit_learn-1.5.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f932a02c3f4956dfb981391ab24bda1dbd90fe3d628e4b42caef3e041c67707a"}, + {file = "scikit_learn-1.5.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:3b923d119d65b7bd555c73be5423bf06c0105678ce7e1f558cb4b40b0a5502b1"}, + {file = "scikit_learn-1.5.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f60021ec1574e56632be2a36b946f8143bf4e5e6af4a06d85281adc22938e0dd"}, + {file = "scikit_learn-1.5.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:394397841449853c2290a32050382edaec3da89e35b3e03d6cc966aebc6a8ae6"}, + {file = "scikit_learn-1.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:57cc1786cfd6bd118220a92ede80270132aa353647684efa385a74244a41e3b1"}, + {file = "scikit_learn-1.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:757c7d514ddb00ae249832fe87100d9c73c6ea91423802872d9e74970a0e40b9"}, + {file = "scikit_learn-1.5.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:52788f48b5d8bca5c0736c175fa6bdaab2ef00a8f536cda698db61bd89c551c1"}, + {file = "scikit_learn-1.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:643964678f4b5fbdc95cbf8aec638acc7aa70f5f79ee2cdad1eec3df4ba6ead8"}, + {file = "scikit_learn-1.5.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca64b3089a6d9b9363cd3546f8978229dcbb737aceb2c12144ee3f70f95684b7"}, + {file = "scikit_learn-1.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:3bed4909ba187aca80580fe2ef370d9180dcf18e621a27c4cf2ef10d279a7efe"}, + {file = "scikit_learn-1.5.2.tar.gz", hash = "sha256:b4237ed7b3fdd0a4882792e68ef2545d5baa50aca3bb45aa7df468138ad8f94d"}, +] + +[package.dependencies] +joblib = ">=1.2.0" +numpy = ">=1.19.5" +scipy = ">=1.6.0" +threadpoolctl = ">=3.1.0" + +[package.extras] +benchmark = ["matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "pandas (>=1.1.5)"] +build = ["cython (>=3.0.10)", "meson-python (>=0.16.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)"] +docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pydata-sphinx-theme (>=0.15.3)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)", "sphinx (>=7.3.7)", "sphinx-copybutton (>=0.5.2)", "sphinx-design (>=0.5.0)", "sphinx-design (>=0.6.0)", "sphinx-gallery (>=0.16.0)", "sphinx-prompt (>=1.4.0)", "sphinx-remove-toctrees (>=1.0.0.post1)", "sphinxcontrib-sass (>=0.3.4)", "sphinxext-opengraph (>=0.9.1)"] +examples = ["matplotlib (>=3.3.4)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)"] +install = ["joblib (>=1.2.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)", "threadpoolctl (>=3.1.0)"] +maintenance = ["conda-lock (==2.5.6)"] +tests = ["black (>=24.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.9)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.2.1)", "scikit-image (>=0.17.2)"] + +[[package]] +name = "scipy" +version = "1.13.1" +description = "Fundamental algorithms for scientific computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "scipy-1.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:20335853b85e9a49ff7572ab453794298bcf0354d8068c5f6775a0eabf350aca"}, + {file = "scipy-1.13.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d605e9c23906d1994f55ace80e0125c587f96c020037ea6aa98d01b4bd2e222f"}, + {file = "scipy-1.13.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfa31f1def5c819b19ecc3a8b52d28ffdcc7ed52bb20c9a7589669dd3c250989"}, + {file = "scipy-1.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26264b282b9da0952a024ae34710c2aff7d27480ee91a2e82b7b7073c24722f"}, + {file = "scipy-1.13.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:eccfa1906eacc02de42d70ef4aecea45415f5be17e72b61bafcfd329bdc52e94"}, + {file = "scipy-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:2831f0dc9c5ea9edd6e51e6e769b655f08ec6db6e2e10f86ef39bd32eb11da54"}, + {file = "scipy-1.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:27e52b09c0d3a1d5b63e1105f24177e544a222b43611aaf5bc44d4a0979e32f9"}, + {file = "scipy-1.13.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:54f430b00f0133e2224c3ba42b805bfd0086fe488835effa33fa291561932326"}, + {file = "scipy-1.13.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e89369d27f9e7b0884ae559a3a956e77c02114cc60a6058b4e5011572eea9299"}, + {file = "scipy-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a78b4b3345f1b6f68a763c6e25c0c9a23a9fd0f39f5f3d200efe8feda560a5fa"}, + {file = "scipy-1.13.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:45484bee6d65633752c490404513b9ef02475b4284c4cfab0ef946def50b3f59"}, + {file = "scipy-1.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:5713f62f781eebd8d597eb3f88b8bf9274e79eeabf63afb4a737abc6c84ad37b"}, + {file = "scipy-1.13.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5d72782f39716b2b3509cd7c33cdc08c96f2f4d2b06d51e52fb45a19ca0c86a1"}, + {file = "scipy-1.13.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:017367484ce5498445aade74b1d5ab377acdc65e27095155e448c88497755a5d"}, + {file = "scipy-1.13.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:949ae67db5fa78a86e8fa644b9a6b07252f449dcf74247108c50e1d20d2b4627"}, + {file = "scipy-1.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de3ade0e53bc1f21358aa74ff4830235d716211d7d077e340c7349bc3542e884"}, + {file = "scipy-1.13.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2ac65fb503dad64218c228e2dc2d0a0193f7904747db43014645ae139c8fad16"}, + {file = "scipy-1.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:cdd7dacfb95fea358916410ec61bbc20440f7860333aee6d882bb8046264e949"}, + {file = "scipy-1.13.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:436bbb42a94a8aeef855d755ce5a465479c721e9d684de76bf61a62e7c2b81d5"}, + {file = "scipy-1.13.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:8335549ebbca860c52bf3d02f80784e91a004b71b059e3eea9678ba994796a24"}, + {file = "scipy-1.13.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d533654b7d221a6a97304ab63c41c96473ff04459e404b83275b60aa8f4b7004"}, + {file = "scipy-1.13.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:637e98dcf185ba7f8e663e122ebf908c4702420477ae52a04f9908707456ba4d"}, + {file = "scipy-1.13.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a014c2b3697bde71724244f63de2476925596c24285c7a637364761f8710891c"}, + {file = "scipy-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:392e4ec766654852c25ebad4f64e4e584cf19820b980bc04960bca0b0cd6eaa2"}, + {file = "scipy-1.13.1.tar.gz", hash = "sha256:095a87a0312b08dfd6a6155cbbd310a8c51800fc931b8c0b84003014b874ed3c"}, +] + +[package.dependencies] +numpy = ">=1.22.4,<2.3" + +[package.extras] +dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyle", "pydevtool", "rich-click", "ruff", "types-psutil", "typing_extensions"] +doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.12.0)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0)", "sphinx-design (>=0.4.0)"] +test = ["array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] + [[package]] name = "send2trash" version = "1.8.3" @@ -4043,6 +4158,17 @@ docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] test = ["pre-commit", "pytest (>=7.0)", "pytest-timeout"] typing = ["mypy (>=1.6,<2.0)", "traitlets (>=5.11.1)"] +[[package]] +name = "threadpoolctl" +version = "3.5.0" +description = "threadpoolctl" +optional = false +python-versions = ">=3.8" +files = [ + {file = "threadpoolctl-3.5.0-py3-none-any.whl", hash = "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"}, + {file = "threadpoolctl-3.5.0.tar.gz", hash = "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107"}, +] + [[package]] name = "tinycss2" version = "1.3.0" @@ -4819,4 +4945,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13,!=3.9.7" -content-hash = "d250e5223075069c0561f95e970624731feb7ddc20f1bc7b8ef6dd826a8f3085" +content-hash = "6c8c5d3130ab1c4a3b6f83846f6cb7f7b951bbc49ee153f181f2b88beee40615" diff --git a/pyproject.toml b/pyproject.toml index 7f26923..95bad5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ opencv-python = "^4.9.0.80" tabulate = "^0.9.0" filetype = "^1.2.0" ftfy = "^6.1.3" +pdftext = "^0.3.10" [tool.poetry.group.dev.dependencies] jupyter = "^1.0.0" diff --git a/surya/detection.py b/surya/detection.py index a808635..370c62b 100644 --- a/surya/detection.py +++ b/surya/detection.py @@ -32,8 +32,6 @@ def batch_detection(images: List, model: EfficientViTForSemanticSegmentation, pr batch_size = get_batch_size() heatmap_count = model.config.num_labels - images = [image.convert("RGB") for image in images] # also copies the images - orig_sizes = [image.size for image in images] splits_per_image = [get_total_splits(size, processor) for size in orig_sizes] @@ -55,7 +53,7 @@ def batch_detection(images: List, model: EfficientViTForSemanticSegmentation, pr all_preds = [] for batch_idx in tqdm(range(len(batches)), desc="Detecting bboxes"): batch_image_idxs = batches[batch_idx] - batch_images = convert_if_not_rgb([images[j] for j in batch_image_idxs]) + batch_images = [images[j].convert("RGB") for j in batch_image_idxs] split_index = [] split_heights = [] diff --git a/surya/input/pdflines.py b/surya/input/pdflines.py new file mode 100644 index 0000000..d59a497 --- /dev/null +++ b/surya/input/pdflines.py @@ -0,0 +1,65 @@ +from pdftext.extraction import dictionary_output + +from surya.postprocessing.text import sort_text_lines +from surya.schema import PolygonBox + + +def get_page_text_lines(filepath, page_idx, out_size): + full_text = dictionary_output(filepath, sort=False, page_range=[page_idx], keep_chars=True)[0] + text_bbox = full_text["bbox"] + text_w_scale = out_size[0] / text_bbox[2] + text_h_scale = out_size[1] / text_bbox[3] + for block in full_text["blocks"]: + for line in block["lines"]: + line["bbox"] = [line["bbox"][0] * text_w_scale, line["bbox"][1] * text_h_scale, + line["bbox"][2] * text_w_scale, line["bbox"][3] * text_h_scale] + for span in line["spans"]: + for char in span["chars"]: + char["bbox"] = [char["bbox"][0] * text_w_scale, char["bbox"][1] * text_h_scale, + char["bbox"][2] * text_w_scale, char["bbox"][3] * text_h_scale] + return full_text + + +def get_table_blocks(tables, full_text, img_size): + # Returns coordinates relative to input table, not full image + table_texts = [] + for table in tables: + table_text = [] + for block in full_text["blocks"]: + for line in block["lines"]: + line_poly = PolygonBox(polygon=[ + [line["bbox"][0], line["bbox"][1]], + [line["bbox"][2], line["bbox"][1]], + [line["bbox"][2], line["bbox"][3]], + [line["bbox"][0], line["bbox"][3]] + ]) + if line_poly.intersection_pct(table) < 0.8: + continue + curr_span = None + curr_box = None + for span in line["spans"]: + for char in span["chars"]: + if curr_span is None: + curr_span = char["char"] + curr_box = char["bbox"] + elif (char["bbox"][0] - curr_box[2]) / img_size[0] < 0.01: + curr_span += char["char"] + curr_box = [min(curr_box[0], char["bbox"][0]), min(curr_box[1], char["bbox"][1]), + max(curr_box[2], char["bbox"][2]), max(curr_box[3], char["bbox"][3])] + else: + table_text.append({"text": curr_span, "bbox": curr_box}) + curr_span = char["char"] + curr_box = char["bbox"] + if curr_span is not None: + table_text.append({"text": curr_span, "bbox": curr_box}) + # Adjust to be relative to input table + for item in table_text: + item["bbox"] = [ + item["bbox"][0] - table.bbox[0], + item["bbox"][1] - table.bbox[1], + item["bbox"][2] - table.bbox[0], + item["bbox"][3] - table.bbox[1] + ] + table_text = sort_text_lines(table_text) + table_texts.append(table_text) + return table_texts \ No newline at end of file diff --git a/surya/model/table_rec/config.py b/surya/model/table_rec/config.py index 02e5d5b..77120f0 100644 --- a/surya/model/table_rec/config.py +++ b/surya/model/table_rec/config.py @@ -1,5 +1,252 @@ -from transformers import MBartConfig +from transformers import PretrainedConfig +from surya.settings import settings +BOX_DIM = 1024 +SPECIAL_TOKENS = 7 +MAX_ROWS = 384 -class TableRecDecoderConfig(MBartConfig): - pass \ No newline at end of file + +class SuryaTableRecConfig(PretrainedConfig): + model_type = "vision-encoder-decoder" + is_composition = True + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + encoder_config = kwargs.pop("encoder") + decoder_config = kwargs.pop("decoder") + text_enc_config = kwargs.pop("text_encoder") + + self.encoder = encoder_config + self.decoder = decoder_config + self.text_encoder = text_enc_config + self.is_encoder_decoder = True + + if isinstance(decoder_config, dict): + self.decoder_start_token_id = decoder_config["bos_token_id"] + self.pad_token_id = decoder_config["pad_token_id"] + self.eos_token_id = decoder_config["eos_token_id"] + else: + self.decoder_start_token_id = decoder_config.bos_token_id + self.pad_token_id = decoder_config.pad_token_id + self.eos_token_id = decoder_config.eos_token_id + + +class DonutSwinTableRecConfig(PretrainedConfig): + model_type = "donut-swin" + + attribute_map = { + "num_attention_heads": "num_heads", + "num_hidden_layers": "num_layers", + } + + def __init__( + self, + image_size=(settings.TABLE_REC_IMAGE_SIZE["width"], settings.TABLE_REC_IMAGE_SIZE["height"]), + patch_size=4, + num_channels=3, + embed_dim=128, + depths=[2, 2, 14, 2], + num_heads=[4, 8, 16, 32], + num_kv_heads=[4, 8, 16, 32], + window_size=8, + mlp_ratio=4.0, + qkv_bias=True, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + drop_path_rate=0.1, + hidden_act="gelu", + use_absolute_embeddings=True, + initializer_range=0.02, + layer_norm_eps=1e-5, + encoder_length=1024, + **kwargs, + ): + super().__init__(**kwargs) + + self.image_size = image_size + self.patch_size = patch_size + self.num_channels = num_channels + self.embed_dim = embed_dim + self.depths = depths + self.num_layers = len(depths) + self.num_heads = num_heads + self.num_kv_heads = num_kv_heads + self.window_size = window_size + self.mlp_ratio = mlp_ratio + self.qkv_bias = qkv_bias + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.drop_path_rate = drop_path_rate + self.hidden_act = hidden_act + self.use_absolute_embeddings = use_absolute_embeddings + self.layer_norm_eps = layer_norm_eps + self.initializer_range = initializer_range + # we set the hidden_size attribute in order to make Swin work with VisionEncoderDecoderModel + # this indicates the channel dimension after the last stage of the model + self.hidden_size = int(embed_dim * 2 ** (len(depths) - 1)) + self.encoder_length = encoder_length + + +class SuryaTableRecDecoderConfig(PretrainedConfig): + model_type = "surya_tablerec" + + def __init__( + self, + num_hidden_layers=3, + vocab_size=settings.TABLE_REC_MAX_ROWS + SPECIAL_TOKENS, + hidden_size=512, + intermediate_size=4 * 512, + encoder_hidden_size=1024, + num_attention_heads=8, + lru_width=None, + attention_window_size=16, + conv1d_width=4, + logits_soft_cap=30.0, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=0, + eos_token_id=1, + bos_token_id=2, + hidden_activation="gelu_pytorch_tanh", + rope_theta=10000.0, + block_types=("attention",), + cross_attn_layers=(0, 1, 2, 3), + encoder_cross_attn_layers=(0, 1, 2, 3), + self_attn_layers=(0, 1, 2, 3), + global_attn_layers=(0, 1, 2, 3), + attention_dropout=0.0, + num_key_value_heads=4, + attention_bias=False, + w_init_variance_scale=0.01, + init_std=0.02, + tie_word_embeddings=False, + aux_heads=0, # How many n-token-ahead heads to add + causal=True, + **kwargs, + ): + self.num_hidden_layers = num_hidden_layers + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_attention_heads = num_attention_heads + self.lru_width = lru_width if lru_width is not None else hidden_size + self.attention_window_size = attention_window_size + self.conv1d_width = conv1d_width + self.logits_soft_cap = logits_soft_cap + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.block_types = list(block_types) + self.hidden_activation = hidden_activation + self.head_dim = self.hidden_size // self.num_attention_heads + self.num_key_value_heads = num_key_value_heads if num_key_value_heads is not None else num_attention_heads + if self.num_key_value_heads > self.num_attention_heads: + raise ValueError("The number of `num_key_value_heads` must be smaller than `num_attention_heads`") + self.cross_attn_layers = cross_attn_layers + self.self_attn_layers = self_attn_layers + self.global_attn_layers = global_attn_layers + self.attention_dropout = attention_dropout + self.attention_bias = attention_bias + self.w_init_variance_scale = w_init_variance_scale + self.final_w_init_variance_scale = 2.0 / self.num_hidden_layers + self.init_std = init_std + self.tie_word_embeddings = tie_word_embeddings + self.aux_heads = aux_heads + self.encoder_hidden_size=encoder_hidden_size + self.causal = causal + self.encoder_cross_attn_layers = encoder_cross_attn_layers + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + **kwargs, + ) + + @property + def layers_block_type(self): + return (self.block_types * 100)[: self.num_hidden_layers] + + +class SuryaTableRecTextEncoderConfig(PretrainedConfig): + model_type = "surya_tablerec" + + def __init__( + self, + num_hidden_layers=4, + vocab_size=settings.TABLE_REC_MAX_ROWS + SPECIAL_TOKENS, + hidden_size=1024, + intermediate_size=4 * 1024, + encoder_hidden_size=1024, + num_attention_heads=16, + lru_width=None, + attention_window_size=16, + conv1d_width=4, + logits_soft_cap=30.0, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=0, + eos_token_id=1, + bos_token_id=2, + hidden_activation="gelu_pytorch_tanh", + rope_theta=10000.0, + block_types=("attention",), + cross_attn_layers=(0, 1, 2, 3, 4, 5), + self_attn_layers=(0, 1, 2, 3, 4, 5), + global_attn_layers=(0, 1, 2, 3, 4, 5), + attention_dropout=0.0, + num_key_value_heads=16, + attention_bias=False, + w_init_variance_scale=0.01, + init_std=0.02, + tie_word_embeddings=False, + causal=False, + max_width=BOX_DIM + SPECIAL_TOKENS, + max_height=BOX_DIM + SPECIAL_TOKENS, + max_position_embeddings=1024, + **kwargs, + ): + self.num_hidden_layers = num_hidden_layers + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_attention_heads = num_attention_heads + self.lru_width = lru_width if lru_width is not None else hidden_size + self.attention_window_size = attention_window_size + self.conv1d_width = conv1d_width + self.logits_soft_cap = logits_soft_cap + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.block_types = list(block_types) + self.hidden_activation = hidden_activation + self.head_dim = self.hidden_size // self.num_attention_heads + self.num_key_value_heads = num_key_value_heads if num_key_value_heads is not None else num_attention_heads + if self.num_key_value_heads > self.num_attention_heads: + raise ValueError("The number of `num_key_value_heads` must be smaller than `num_attention_heads`") + self.cross_attn_layers = cross_attn_layers + self.self_attn_layers = self_attn_layers + self.global_attn_layers = global_attn_layers + self.attention_dropout = attention_dropout + self.attention_bias = attention_bias + self.w_init_variance_scale = w_init_variance_scale + self.final_w_init_variance_scale = 2.0 / self.num_hidden_layers + self.init_std = init_std + self.tie_word_embeddings = tie_word_embeddings + self.encoder_hidden_size = encoder_hidden_size + self.causal = causal + self.max_width = max_width + self.max_height = max_height + self.max_position_embeddings = max_position_embeddings + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + **kwargs, + ) + + @property + def layers_block_type(self): + return (self.block_types * 100)[: self.num_hidden_layers] \ No newline at end of file diff --git a/surya/model/table_rec/decoder.py b/surya/model/table_rec/decoder.py index 77f983d..89758d4 100644 --- a/surya/model/table_rec/decoder.py +++ b/surya/model/table_rec/decoder.py @@ -1,94 +1,765 @@ -import copy from dataclasses import dataclass -from typing import Optional, List, Tuple, Union +from typing import Dict, Optional, Tuple, Union import torch +import torch.utils.checkpoint from torch import nn -from transformers import MBartForCausalLM, MBartPreTrainedModel from transformers.utils import ModelOutput -from surya.model.ordering.decoder import MBartOrderDecoderWrapper -from surya.model.table_rec.config import TableRecDecoderConfig +from surya.model.table_rec.config import SuryaTableRecDecoderConfig, SuryaTableRecTextEncoderConfig +from transformers import PreTrainedModel +from transformers.activations import ACT2FN +from transformers.modeling_attn_mask_utils import AttentionMaskConverter +from transformers.modeling_outputs import BaseModelOutputWithNoAttention, CausalLMOutput +from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS +from surya.settings import settings + +_MAX_SQRT_GRADIENT = 1000.0 @dataclass -class TableRecDecoderOutput(ModelOutput): - loss: Optional[torch.FloatTensor] = None - row_logits: torch.FloatTensor = None - col_logits: torch.FloatTensor = None - past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None - hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None - attentions: Optional[Tuple[torch.FloatTensor, ...]] = None - cross_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None +class TableRecModelOutput(ModelOutput): + row_logits: torch.Tensor + col_logits: torch.Tensor | None = None + hidden_states: torch.Tensor | None = None -class TableRecDecoder(MBartForCausalLM): - config_class = TableRecDecoderConfig - _tied_weights_keys = [] +class SuryaTableRecDecoderRMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-6): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.zeros(dim)) - def __init__(self, config, **kwargs): - config = copy.deepcopy(config) - config.is_decoder = True - config.is_encoder_decoder = False - MBartPreTrainedModel.__init__(self, config) - self.model = MBartOrderDecoderWrapper(config) + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) - self.row_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - self.col_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + def forward(self, x): + output = self._norm(x.float()) + # Llama does x.to(float16) * w whilst SuryaTableRecDecoder is (x * w).to(float16) + # See https://github.com/huggingface/transformers/pull/29402 + output = output * (1.0 + self.weight.float()) + return output.type_as(x) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.eps}" + + +ALL_LAYERNORM_LAYERS.append(SuryaTableRecDecoderRMSNorm) + + +class SuryaTableRecDecoderRotaryEmbedding(nn.Module): + def __init__(self, dim, base=10000, device=None): + super().__init__() + self.dim = dim + self.base = base + inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float() / self.dim)) + self.register_buffer("inv_freq", tensor=inv_freq, persistent=False) + + @torch.no_grad() + # Copied from transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding.forward with Gemma->SuryaTableRecDecoder + def forward(self, x, position_ids, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + self.inv_freq.to(x.device) + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() + sin = emb.sin() + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +# Copied from transformers.models.llama.modeling_llama.rotate_half +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +# Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb +def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +# Copied from transformers.models.llama.modeling_llama.repeat_kv +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +class SuryaTableRecDecoderSdpaCrossAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper + Modified for GQA + """ + + def __init__(self, config: SuryaTableRecDecoderConfig): + super().__init__() + self.config = config + self.attention_dropout = config.attention_dropout + self.hidden_size = config.hidden_size + self.num_attention_heads = config.num_attention_heads + self.head_dim = config.head_dim + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_attention_heads // self.num_key_value_heads + + self.q_proj = nn.Linear(self.hidden_size, self.num_attention_heads * self.head_dim, bias=config.attention_bias) + self.k_proj = nn.Linear(self.config.encoder_hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias) + self.v_proj = nn.Linear(self.config.encoder_hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias) + self.o_proj = nn.Linear(self.num_attention_heads * self.head_dim, self.hidden_size, bias=True) + self.rotary_emb = SuryaTableRecDecoderRotaryEmbedding( + self.head_dim, + base=config.rope_theta, + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + # Encoder attention mask currently ignored + + bsz, q_len, _ = hidden_states.size() + _, v_len, _ = encoder_hidden_states.size() + + query_states = self.q_proj(hidden_states) + query_states = query_states.view(bsz, q_len, self.num_attention_heads, self.head_dim).transpose(1, 2) + + if self.key_states is None: + key_states = self.k_proj(encoder_hidden_states) + value_states = self.v_proj(encoder_hidden_states) + key_states = key_states.view(bsz, v_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, v_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + if use_cache: + self._update_cache(key_states, value_states) + else: + key_states = self.key_states + value_states = self.value_states + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_output = torch.nn.functional.scaled_dot_product_attention( + query_states.contiguous(), + key_states.contiguous(), + value_states.contiguous(), + attn_mask=None, + dropout_p=self.attention_dropout if self.training else 0.0, + scale=self.head_dim**-0.5, + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.view(bsz, q_len, self.hidden_size) + attn_output = self.o_proj(attn_output) + return attn_output + + def _setup_cache(self, batch_size, device, dtype=None): + # Setup initial caches + self.value_states = None + self.key_states = None + + @torch.no_grad() + def _update_cache(self, key_states, value_states, **cache_kwargs): + self.value_states = value_states + self.key_states = key_states + + +class SuryaTableRecDecoderSdpaAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: SuryaTableRecDecoderConfig): + super().__init__() + self.config = config + self.attention_dropout = config.attention_dropout + self.hidden_size = config.hidden_size + self.num_attention_heads = config.num_attention_heads + self.head_dim = config.head_dim + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_attention_heads // self.num_key_value_heads + + self.q_proj = nn.Linear(self.hidden_size, self.num_attention_heads * self.head_dim, bias=config.attention_bias) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias) + self.o_proj = nn.Linear(self.num_attention_heads * self.head_dim, self.hidden_size, bias=True) + self.rotary_emb = SuryaTableRecDecoderRotaryEmbedding( + self.head_dim, + base=config.rope_theta, + ) + + def forward( + self, + hidden_states: torch.Tensor, + position_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + cache_position: Optional[torch.LongTensor] = None, + use_cache: bool = False, + window_attn: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + # Final is bsz, num_attention_heads, seq_len, head_dim + query_states = query_states.view(bsz, q_len, self.num_attention_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + cos, sin = self.rotary_emb(value_states, position_ids, seq_len=None) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if use_cache and hasattr(self, "key_states"): + cache_kwargs = {"cache_position": cache_position, "window_attn": window_attn} + key_states, value_states = self._update_cache(key_states, value_states, **cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + causal_mask = attention_mask + if attention_mask is not None: + # Mask is batch, head, seq_len, kv_len + causal_mask = causal_mask[:, :, :, :key_states.shape[-2]] + current_cache_position = cache_position[-1].item() if cache_position is not None else None + if current_cache_position and settings.RECOGNITION_STATIC_CACHE: + # Mask out future cache positions + position_mask = torch.ones_like(causal_mask, dtype=torch.bool, device=causal_mask.device) + position_mask[:, :, :, :current_cache_position + 1] = False + causal_mask = torch.where(position_mask, torch.finfo(causal_mask.dtype).min, causal_mask) + + attn_output = torch.nn.functional.scaled_dot_product_attention( + query_states.contiguous(), + key_states.contiguous(), + value_states.contiguous(), + attn_mask=causal_mask, + dropout_p=self.attention_dropout if self.training else 0.0, + scale=self.head_dim**-0.5, + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.view(bsz, q_len, self.hidden_size) + attn_output = self.o_proj(attn_output) + return attn_output + + def _setup_cache(self, batch_size, device, dtype=None): + if dtype is None and self.config.torch_dtype is not None: + dtype = self.config.torch_dtype + dtype = dtype if dtype is not None else torch.float32 + + # Setup initial caches + self.value_states = None + self.key_states = None + + if settings.RECOGNITION_STATIC_CACHE: + cache_shape = (batch_size, self.num_key_value_heads, settings.RECOGNITION_MAX_TOKENS, self.head_dim) + self.value_states = torch.zeros(cache_shape, dtype=dtype, device=device) + self.key_states = torch.zeros(cache_shape, dtype=dtype, device=device) + + def _update_static_cache(self, key_states, value_states, **cache_kwargs): + cache_position = cache_kwargs.get("cache_position") + k_out, v_out = self.key_states.to(key_states.device), self.value_states.to(value_states.device) + + k_out[:, :, cache_position] = key_states.to(k_out.dtype) + v_out[:, :, cache_position] = value_states.to(v_out.dtype) + + self.key_states, self.value_states = k_out, v_out + return k_out, v_out + + def _update_dynamic_cache(self, key_states, value_states, **cache_kwargs): + k_out = key_states + if self.key_states is not None: + k_out = torch.cat([self.key_states, key_states], dim=2) + + v_out = value_states + if self.value_states is not None: + v_out = torch.cat([self.value_states, value_states], dim=2) + + self.key_states, self.value_states = k_out, v_out + return k_out, v_out + + @torch.no_grad() + def _update_cache(self, key_states, value_states, **cache_kwargs): + if settings.RECOGNITION_STATIC_CACHE: + return self._update_static_cache(key_states, value_states, **cache_kwargs) + + return self._update_dynamic_cache(key_states, value_states, **cache_kwargs) + + +class SuryaTableRecDecoderMlp(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) + if config.hidden_activation is None: + config.hidden_activation = "gelu_pytorch_tanh" + hidden_activation = config.hidden_activation + self.act_fn = ACT2FN[hidden_activation] + + def forward(self, x): + return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + + +class SuryaTableRecDecoderLayer(nn.Module): + def __init__(self, config, layer_idx): + super().__init__() + super().__init__() + self.cross_pre_norm = SuryaTableRecDecoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.temporal_pre_norm = SuryaTableRecDecoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.temporal_block = None + if layer_idx in config.self_attn_layers: + self.temporal_block = SuryaTableRecDecoderSdpaAttention(config) + + self.cross_attn_block = None + if layer_idx in config.cross_attn_layers: + self.cross_attn_block = SuryaTableRecDecoderSdpaCrossAttention(config) + + self.window_attn = layer_idx not in config.global_attn_layers + self.channel_pre_norm = SuryaTableRecDecoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.mlp_block = SuryaTableRecDecoderMlp(config) + + def forward( + self, + activations: torch.Tensor, + position_ids: torch.Tensor, + attention_mask: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + encoder_attention_mask: torch.Tensor = None, + cache_position: torch.Tensor = None, + use_cache: bool = None, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + raw_activations = activations + + if self.cross_attn_block is not None: + # Do cross-attention on encoder outputs + cross_attn_inputs = self.cross_pre_norm(activations) + cross_attn_path = self.cross_attn_block( + cross_attn_inputs, encoder_hidden_states, attention_mask, encoder_attention_mask, use_cache=use_cache + ) + cross_attn_output = cross_attn_path + raw_activations + else: + cross_attn_output = raw_activations + + if self.temporal_block is not None: + inputs_normalized = self.temporal_pre_norm(cross_attn_output) # RMSNorm introduces slight slight differences + hidden_states = self.temporal_block( + inputs_normalized, position_ids, attention_mask, cache_position=cache_position, use_cache=use_cache, window_attn=self.window_attn + ) + + residual = hidden_states + raw_activations + else: + residual = cross_attn_output + + hidden_states = self.channel_pre_norm(residual) + hidden_states = self.mlp_block(hidden_states) + + hidden_states = hidden_states + residual + return hidden_states + + +class SuryaTableRecDecoderPreTrainedModel(PreTrainedModel): + config_class = SuryaTableRecDecoderConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["SuryaTableRecDecoderLayer"] + _skip_keys_device_placement = ["cache"] + _supports_flash_attn_2 = False + _supports_sdpa = False # we can't compare with eager for now + _supports_cache_class = True + _supports_quantized_cache = True + + def _init_weights(self, module): + if isinstance(module, SuryaTableRecDecoderSdpaAttention): + torch.nn.init.normal_(module.q_proj.weight, mean=0.0, std=self.config.init_std) + torch.nn.init.normal_(module.k_proj.weight, mean=0.0, std=self.config.init_std) + torch.nn.init.normal_(module.v_proj.weight, mean=0.0, std=self.config.init_std) + + torch.nn.init.normal_(module.o_proj.weight, mean=0.0, std=self.config.init_std) + elif isinstance(module, nn.Linear): + torch.nn.init.normal_(module.weight, mean=0.0, std=self.config.init_std) + if getattr(module, "bias", None) is not None: + torch.nn.init.zeros_(module.bias) + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=self.config.init_std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + def _setup_cache(self, config, batch, device, dtype): + layers = getattr(self, "model", self).layers + for layer in layers: + if layer.temporal_block: + layer.temporal_block._setup_cache(batch, device, dtype) + if layer.cross_attn_block: + layer.cross_attn_block._setup_cache(batch, device, dtype) + + def reset_cache(self, batch, device, dtype): + pass + + def _tie_weights(self): + pass + + def tie_weights(self): + pass + + +class LabelEmbedding(nn.Module): + def __init__(self, config): + super().__init__() + self.vocab_size = config.vocab_size + self.row_embed = nn.Embedding(config.vocab_size, config.hidden_size) + self.col_embed = nn.Embedding(config.vocab_size, config.hidden_size) + + def forward(self, labels: torch.LongTensor, input_box_counts: torch.LongTensor): + row_labels, col_labels = labels.unbind(dim=-1) + row_labels = torch.clamp(row_labels, 0, self.vocab_size - 1).long() + col_labels = torch.clamp(col_labels, 0, self.vocab_size - 1).long() + embedded = self.row_embed(row_labels) + self.col_embed(col_labels) # Embed the labels in as well + return embedded + +class BboxEmbedding(nn.Module): + def __init__(self, config, embed_positions=False): + super().__init__() + self.x1_embed = nn.Embedding(config.max_width, config.hidden_size) + self.y1_embed = nn.Embedding(config.max_height, config.hidden_size) + self.x2_embed = nn.Embedding(config.max_width, config.hidden_size) + self.y2_embed = nn.Embedding(config.max_height, config.hidden_size) + self.w_embed = nn.Embedding(config.max_width, config.hidden_size) + self.h_embed = nn.Embedding(config.max_height, config.hidden_size) + self.cx_embed = nn.Embedding(config.max_width, config.hidden_size) + self.cy_embed = nn.Embedding(config.max_height, config.hidden_size) + self.box_pos_embed = nn.Embedding(config.max_position_embeddings, config.hidden_size) + self.max_width = config.max_width + self.max_height = config.max_height + self.embed_positions = embed_positions + + def forward(self, boxes: torch.LongTensor, input_box_counts: torch.LongTensor): + x1, y1, x2, y2 = boxes.unbind(dim=-1) + x1 = torch.clamp(x1, 0, self.max_width - 1).long() + y1 = torch.clamp(y1, 0, self.max_height - 1).long() + x2 = torch.clamp(x2, 0, self.max_width - 1).long() + y2 = torch.clamp(y2, 0, self.max_height - 1).long() + + # Shape is (batch_size, num_boxes/seq len, d_model) + w = x2 - x1 + h = y2 - y1 + # Center x and y in torch long tensors + cx = (x1 + x2) / 2 + cy = (y1 + y2) / 2 + cx = cx.long() + cy = cy.long() + + w = torch.clamp(w, 0, self.max_width - 1).long() + h = torch.clamp(h, 0, self.max_height - 1).long() + cx = torch.clamp(cx, 0, self.max_width - 1).long() + cy = torch.clamp(cy, 0, self.max_height - 1).long() + + coord_embeds = self.x1_embed(x1) + self.y1_embed(y1) + self.x2_embed(x2) + self.y2_embed(y2) + embedded = coord_embeds + self.w_embed(w) + self.h_embed(h) + self.cx_embed(cx) + self.cy_embed(cy) + + # Add in positional embeddings for the boxes and labels + if self.embed_positions: + for j in range(embedded.shape[0]): + box_start = input_box_counts[j, 0] + box_end = input_box_counts[j, 1] - 1 # Skip the sep token + box_count = box_end - box_start + embedded[j, box_start:box_end] = embedded[j, box_start:box_end] + self.box_pos_embed.weight[:box_count] + + return embedded + + +class SuryaTableRecDecoderModel(SuryaTableRecDecoderPreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`SuryaTableRecDecoderDecoderLayer`] + + Args: + config: SuryaTableRecDecoderConfig + """ + + def __init__(self, config: SuryaTableRecDecoderConfig, embed_labels=False, embed_positions=True): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + self.causal = config.causal + + if embed_labels: + self.embed_tokens = LabelEmbedding(config) + else: + self.embed_tokens = BboxEmbedding(config, embed_positions=embed_positions) + + self.layers = nn.ModuleList( + [SuryaTableRecDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.final_norm = SuryaTableRecDecoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.gradient_checkpointing = False + + self.register_buffer( + "normalizer", torch.tensor(self.config.hidden_size**0.5, dtype=torch.float32), persistent=False + ) # Initialize weights and apply final processing self.post_init() + # Copied from transformers.models.llama.modeling_llama.LlamaModel.get_input_embeddings + def get_input_embeddings(self): + return self.embed_tokens + + # Copied from transformers.models.llama.modeling_llama.LlamaModel.set_input_embeddings + def set_input_embeddings(self, value): + self.embed_tokens = value + def forward( self, - input_boxes: torch.LongTensor = None, - input_boxes_mask: Optional[torch.Tensor] = None, - input_boxes_counts: Optional[torch.Tensor] = None, + input_ids: torch.LongTensor = None, + input_boxes_counts: torch.LongTensor = None, + position_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, encoder_hidden_states: Optional[torch.FloatTensor] = None, encoder_attention_mask: Optional[torch.FloatTensor] = None, - head_mask: Optional[torch.Tensor] = None, - cross_attn_head_mask: Optional[torch.Tensor] = None, - past_key_values: Optional[List[torch.FloatTensor]] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - labels: Optional[torch.LongTensor] = None, + cache_position: Optional[torch.LongTensor] = None, use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - **kwargs - ) -> Union[Tuple, TableRecDecoderOutput]: - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) + prefill: bool = False + ) -> Union[Tuple, BaseModelOutputWithNoAttention]: + use_cache = use_cache if use_cache is not None else self.config.use_cache return_dict = return_dict if return_dict is not None else self.config.use_return_dict - # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) - outputs = self.model.decoder( - input_boxes=input_boxes, - input_boxes_mask=input_boxes_mask, - input_boxes_counts=input_boxes_counts, + if self.gradient_checkpointing and self.training and use_cache: + use_cache = False + + inputs_embeds = self.embed_tokens(input_ids, input_boxes_counts) + hidden_states = inputs_embeds + + if use_cache and prefill: + self._setup_cache(self.config, hidden_states.shape[0], hidden_states.device, hidden_states.dtype) + + if cache_position is None: + cache_position = torch.arange(hidden_states.shape[1], device=hidden_states.device) + if position_ids is None: + position_ids = cache_position.unsqueeze(0) + + causal_mask = self._update_causal_mask(attention_mask, inputs_embeds, cache_position) + + all_hidden_states = () if output_hidden_states else None + for i, residual_block in enumerate(self.layers): + if output_hidden_states: + all_hidden_states += (hidden_states,) + if self.gradient_checkpointing and self.training: + hidden_states = self._gradient_checkpointing_func( + residual_block.__call__, hidden_states, position_ids, causal_mask, encoder_hidden_states, encoder_attention_mask, cache_position, use_cache + ) + else: + hidden_states = residual_block(hidden_states, position_ids, causal_mask, encoder_hidden_states, encoder_attention_mask, cache_position, use_cache) + + hidden_states = self.final_norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, all_hidden_states] if v is not None) + + return BaseModelOutputWithNoAttention( + last_hidden_state=hidden_states, + hidden_states=all_hidden_states, + ) + + # TODO: As of torch==2.2.0, the `attention_mask` passed to the model in `generate` is 2D and of dynamic length even when the static + # KV cache is used. This is an issue for torch.compile which then recaptures cudagraphs at each decode steps due to the dynamic shapes. + # (`recording cudagraph tree for symint key 13`, etc.), which is VERY slow. A workaround is `@torch.compiler.disable`, but this prevents using + # `fullgraph=True`. See more context in https://github.com/huggingface/transformers/pull/29114 + # Ignore copy + def _update_causal_mask(self, attention_mask, input_tensor, cache_position): + if not self.causal: + return None + + dtype, device = input_tensor.dtype, input_tensor.device + min_dtype = torch.finfo(dtype).min + sequence_length = input_tensor.shape[1] + target_length = max(settings.TABLE_REC_MAX_BOXES, sequence_length) + + diagonal = torch.full((sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device) + causal_mask = diagonal + if sequence_length != 1: + # Select the upper triangular part of the matrix, but unmask current token (the diagonal) + # triu will be the min_dtype, everything else is 0 (attended to) + causal_mask = torch.triu(diagonal, diagonal=1) + + causal_mask *= torch.arange(target_length, device=device) > cache_position.reshape(-1, 1) + causal_mask = causal_mask[None, None, :, :].expand(input_tensor.shape[0], 1, -1, -1) + if attention_mask is not None: + causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit + if attention_mask.dim() == 2: + # Mask positions in the causal mask that are masked in the attention mask + mask_length = attention_mask.shape[-1] + padding_mask = causal_mask[..., :mask_length].eq(0.0) * attention_mask[:, None, None, :].eq(0.0) + causal_mask[..., :mask_length] = causal_mask[..., :mask_length].masked_fill(padding_mask, min_dtype) + + if attention_mask is not None and attention_mask.device.type == "cuda": + # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when + # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. + # Details: https://github.com/pytorch/pytorch/issues/110213 + causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) + + return causal_mask + + +class SuryaTableRecDecoder(SuryaTableRecDecoderPreTrainedModel): + _tied_weights_keys = None + + def __init__(self, config, **kwargs): + super().__init__(config) + self.model = SuryaTableRecDecoderModel(config, embed_labels=True, embed_positions=False) + self.vocab_size = config.vocab_size + + self.row_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + self.col_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + # Ignore copy + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + cache_position: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + prefill: bool = False, + **kwargs + ) -> Union[Tuple, TableRecModelOutput]: + outputs = self.model( + input_ids=input_ids, + cache_position=cache_position, + attention_mask=attention_mask, encoder_hidden_states=encoder_hidden_states, encoder_attention_mask=encoder_attention_mask, - head_mask=head_mask, - cross_attn_head_mask=cross_attn_head_mask, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, + output_hidden_states=True, + return_dict=True, + prefill=prefill, ) - row_logits = self.row_head(outputs[0]) - col_logits = self.col_head(outputs[0]) + hidden_states = outputs[0] + row_logits = self.row_head(hidden_states) + col_logits = self.col_head(hidden_states) - return TableRecDecoderOutput( - loss=None, - col_logits=col_logits, + return TableRecModelOutput( row_logits=row_logits, - past_key_values=outputs.past_key_values, + col_logits=col_logits, hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - cross_attentions=outputs.cross_attentions, + ) + +@dataclass +class TextEncoderOutput(CausalLMOutput): + hidden_states: torch.FloatTensor = None + + +class SuryaTableRecTextEncoder(SuryaTableRecDecoderPreTrainedModel): + _tied_weights_keys = None + config_class = SuryaTableRecTextEncoderConfig + + def __init__(self, config, **kwargs): + super().__init__(config) + self.model = SuryaTableRecDecoderModel(config, embed_labels=False, embed_positions=True) + self.vocab_size = config.vocab_size + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + # Ignore copy + def forward( + self, + input_boxes: Optional[torch.LongTensor] = None, + input_boxes_counts: Optional[torch.LongTensor] = None, + cache_position: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + **kwargs + ) -> Union[Tuple, CausalLMOutput]: + outputs = self.model( + input_ids=input_boxes, + input_boxes_counts=input_boxes_counts, + cache_position=cache_position, + attention_mask=attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + use_cache=use_cache, + output_hidden_states=True, + return_dict=True, + ) + + return TextEncoderOutput( + hidden_states=outputs.last_hidden_state, ) \ No newline at end of file diff --git a/surya/model/table_rec/encoderdecoder.py b/surya/model/table_rec/encoderdecoder.py index dda5990..9e2b468 100644 --- a/surya/model/table_rec/encoderdecoder.py +++ b/surya/model/table_rec/encoderdecoder.py @@ -1,107 +1,135 @@ +import random from dataclasses import dataclass -from typing import Optional, Tuple, List, Union +from typing import Optional, Union, Tuple import torch -from transformers import VisionEncoderDecoderModel -from transformers.modeling_outputs import BaseModelOutput +from torch import nn +from torch.nn import CrossEntropyLoss +from transformers import PreTrainedModel, VisionEncoderDecoderConfig, PretrainedConfig +from transformers.modeling_outputs import Seq2SeqLMOutput, BaseModelOutput +from transformers.models.vision_encoder_decoder.modeling_vision_encoder_decoder import shift_tokens_right +from surya.model.table_rec.decoder import SuryaTableRecTextEncoder, SuryaTableRecDecoder +from surya.model.recognition.encoder import DonutSwinModel +import torch.nn.functional as F from transformers.utils import ModelOutput @dataclass class TableRecOutput(ModelOutput): - loss: Optional[torch.FloatTensor] = None row_logits: torch.FloatTensor = None col_logits: torch.FloatTensor = None - past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None - decoder_hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None - decoder_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None - cross_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None - encoder_last_hidden_state: Optional[torch.FloatTensor] = None - encoder_hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None - encoder_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None + decoder_hidden_states: Optional[Tuple[torch.FloatTensor]] = None -class TableRecVisionEncoderDecoderModel(VisionEncoderDecoderModel): +class TableRecEncoderDecoderModel(PreTrainedModel): + config_class = VisionEncoderDecoderConfig + base_model_prefix = "vision_encoder_decoder" + main_input_name = "pixel_values" + supports_gradient_checkpointing = True + _supports_param_buffer_assignment = False + + def __init__( + self, + config: Optional[PretrainedConfig] = None, + encoder: Optional[PreTrainedModel] = None, + text_encoder: Optional[PreTrainedModel] = None, + decoder: Optional[PreTrainedModel] = None, + ): + # initialize with config + # make sure input & output embeddings is not tied + config.tie_word_embeddings = False + config.decoder.tie_word_embeddings = False + super().__init__(config) + + if encoder is None: + encoder = DonutSwinModel(config.encoder) + + if text_encoder is None: + text_encoder = SuryaTableRecTextEncoder(config.text_encoder, attn_implementation=config._attn_implementation) + + if decoder is None: + decoder = SuryaTableRecDecoder(config.decoder, attn_implementation=config._attn_implementation) + + self.encoder = encoder + self.decoder = decoder + self.text_encoder = text_encoder + + # make sure that the individual model's config refers to the shared config + # so that the updates to the config will be synced + self.encoder.config = self.config.encoder + self.decoder.config = self.config.decoder + self.text_encoder.config = self.config.text_encoder + + def get_encoder(self): + return self.encoder + + def get_decoder(self): + return self.decoder + + def get_output_embeddings(self): + return self.decoder.get_output_embeddings() + + def set_output_embeddings(self, new_embeddings): + return self.decoder.set_output_embeddings(new_embeddings) + def forward( self, - pixel_values: Optional[torch.FloatTensor] = None, - decoder_input_boxes: torch.LongTensor = None, - # Shape (batch_size, num_boxes, 4), all coords scaled 0 - 1000, with 1001 as padding - decoder_input_boxes_mask: torch.LongTensor = None, # Shape (batch_size, num_boxes), 0 if padding, 1 otherwise - decoder_input_boxes_counts: torch.LongTensor = None, # Shape (batch_size), number of boxes in each image + decoder_input_ids: torch.LongTensor = None, + decoder_cache_position: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, encoder_outputs: Optional[Tuple[torch.FloatTensor]] = None, - past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, - decoder_inputs_embeds: Optional[torch.FloatTensor] = None, - labels: Optional[List[List[int]]] = None, use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, **kwargs, ) -> Union[Tuple[torch.FloatTensor], TableRecOutput]: - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - kwargs_encoder = {argument: value for argument, value in kwargs.items() if not argument.startswith("decoder_")} kwargs_decoder = { argument[len("decoder_") :]: value for argument, value in kwargs.items() if argument.startswith("decoder_") } - if encoder_outputs is None: - if pixel_values is None: - raise ValueError("You have to specify pixel_values") - - encoder_outputs = self.encoder( - pixel_values=pixel_values, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - **kwargs_encoder, - ) - elif isinstance(encoder_outputs, tuple): - encoder_outputs = BaseModelOutput(*encoder_outputs) - - encoder_hidden_states = encoder_outputs[0] - - # optionally project encoder_hidden_states - if ( - self.encoder.config.hidden_size != self.decoder.config.hidden_size - and self.decoder.config.cross_attention_hidden_size is None - ): - encoder_hidden_states = self.enc_to_dec_proj(encoder_hidden_states) - - # else: - encoder_attention_mask = None - # Decode decoder_outputs = self.decoder( - input_boxes=decoder_input_boxes, - input_boxes_mask=decoder_input_boxes_mask, - input_boxes_counts=decoder_input_boxes_counts, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - inputs_embeds=decoder_inputs_embeds, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, + input_labels=decoder_input_ids, + input_boxes_counts=None, + cache_position=decoder_cache_position, + attention_mask=decoder_attention_mask, + encoder_hidden_states=encoder_outputs, + encoder_attention_mask=None, use_cache=use_cache, - past_key_values=past_key_values, - return_dict=return_dict, - labels=labels, **kwargs_decoder, ) - if not return_dict: - return decoder_outputs + encoder_outputs - return TableRecOutput( - loss=decoder_outputs.loss, row_logits=decoder_outputs.row_logits, col_logits=decoder_outputs.col_logits, - past_key_values=decoder_outputs.past_key_values, decoder_hidden_states=decoder_outputs.hidden_states, - decoder_attentions=decoder_outputs.attentions, - cross_attentions=decoder_outputs.cross_attentions, - encoder_last_hidden_state=encoder_outputs.last_hidden_state, - encoder_hidden_states=encoder_outputs.hidden_states, - encoder_attentions=encoder_outputs.attentions, ) + + def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor): + return shift_tokens_right(labels, self.config.pad_token_id, self.config.decoder_start_token_id) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, use_cache=None, encoder_outputs=None, **kwargs + ): + decoder_inputs = self.decoder.prepare_inputs_for_generation(input_ids, past_key_values=past_key_values) + decoder_attention_mask = decoder_inputs["attention_mask"] if "attention_mask" in decoder_inputs else None + input_dict = { + "attention_mask": attention_mask, + "decoder_attention_mask": decoder_attention_mask, + "decoder_input_ids": decoder_inputs["input_ids"], + "encoder_outputs": encoder_outputs, + "past_key_values": decoder_inputs["past_key_values"], + "use_cache": use_cache, + } + return input_dict + + def resize_token_embeddings(self, *args, **kwargs): + raise NotImplementedError( + "Resizing the embedding layers via the VisionEncoderDecoderModel directly is not supported.Please use the" + " respective methods of the wrapped decoder object (model.decoder.resize_token_embeddings(...))" + ) + + def _reorder_cache(self, past_key_values, beam_idx): + # apply decoder cache reordering here + return self.decoder._reorder_cache(past_key_values, beam_idx) \ No newline at end of file diff --git a/surya/model/table_rec/model.py b/surya/model/table_rec/model.py index e2b9df2..7ffb828 100644 --- a/surya/model/table_rec/model.py +++ b/surya/model/table_rec/model.py @@ -1,34 +1,34 @@ -from transformers import VisionEncoderDecoderConfig, AutoModel, AutoModelForCausalLM - -from surya.model.ordering.config import VariableDonutSwinConfig -from surya.model.ordering.encoder import VariableDonutSwinModel -from surya.model.table_rec.config import TableRecDecoderConfig -from surya.model.table_rec.decoder import TableRecDecoder -from surya.model.table_rec.encoderdecoder import TableRecVisionEncoderDecoderModel +from surya.model.recognition.encoder import DonutSwinModel +from surya.model.table_rec.config import SuryaTableRecConfig, SuryaTableRecDecoderConfig, DonutSwinTableRecConfig, \ + SuryaTableRecTextEncoderConfig +from surya.model.table_rec.decoder import SuryaTableRecDecoder, SuryaTableRecTextEncoder +from surya.model.table_rec.encoderdecoder import TableRecEncoderDecoderModel from surya.settings import settings def load_model(checkpoint=settings.TABLE_REC_MODEL_CHECKPOINT, device=settings.TORCH_DEVICE_MODEL, dtype=settings.MODEL_DTYPE): - config = VisionEncoderDecoderConfig.from_pretrained(checkpoint) - decoder_config = vars(config.decoder) - decoder = TableRecDecoderConfig(**decoder_config) + config = SuryaTableRecConfig.from_pretrained(checkpoint) + decoder_config = config.decoder + decoder = SuryaTableRecDecoderConfig(**decoder_config) config.decoder = decoder - encoder_config = vars(config.encoder) - encoder = VariableDonutSwinConfig(**encoder_config) + encoder_config = config.encoder + encoder = DonutSwinTableRecConfig(**encoder_config) config.encoder = encoder - # Get transformers to load custom model - AutoModel.register(TableRecDecoderConfig, TableRecDecoder) - AutoModelForCausalLM.register(TableRecDecoderConfig, TableRecDecoder) - AutoModel.register(VariableDonutSwinConfig, VariableDonutSwinModel) + text_encoder_config = config.text_encoder + text_encoder = SuryaTableRecTextEncoderConfig(**text_encoder_config) + config.text_encoder = text_encoder + + model = TableRecEncoderDecoderModel.from_pretrained(checkpoint, config=config, torch_dtype=dtype) - model = TableRecVisionEncoderDecoderModel.from_pretrained(checkpoint, config=config, torch_dtype=dtype) - assert isinstance(model.decoder, TableRecDecoder) - assert isinstance(model.encoder, VariableDonutSwinModel) + assert isinstance(model.decoder, SuryaTableRecDecoder) + assert isinstance(model.encoder, DonutSwinModel) + assert isinstance(model.text_encoder, SuryaTableRecTextEncoder) model = model.to(device) model = model.eval() - print(f"Loaded reading order model {checkpoint} on device {device} with dtype {dtype}") + + print(f"Loaded recognition model {checkpoint} on device {device} with dtype {dtype}") return model \ No newline at end of file diff --git a/surya/model/table_rec/processor.py b/surya/model/table_rec/processor.py index 4b6196c..b96438e 100644 --- a/surya/model/table_rec/processor.py +++ b/surya/model/table_rec/processor.py @@ -1,88 +1,185 @@ -from copy import deepcopy -from typing import List, Dict, Optional, Union +from typing import Dict, Union, Optional, List, Iterable +import cv2 +import torch +from torch import TensorType +from transformers import DonutImageProcessor, DonutProcessor +from transformers.image_processing_utils import BatchFeature +from transformers.image_transforms import pad, normalize +from transformers.image_utils import PILImageResampling, ImageInput, ChannelDimension, make_list_of_images, get_image_size import numpy as np from PIL import Image -from torch import TensorType -from transformers import DonutImageProcessor, BatchFeature -from transformers.image_utils import ChannelDimension, ImageInput, PILImageResampling, make_list_of_images, valid_images - +import PIL +from surya.model.recognition.tokenizer import Byt5LangTokenizer from surya.settings import settings +from surya.model.table_rec.config import BOX_DIM -def load_processor(checkpoint=settings.TABLE_REC_MODEL_CHECKPOINT): - processor = TableRecImageProcessor.from_pretrained(checkpoint) - processor.size = settings.TABLE_REC_IMAGE_SIZE - box_size = 1024 - max_tokens = 384 - processor.token_sep_id = max_tokens + box_size + 1 - processor.token_pad_id = max_tokens + box_size + 2 - processor.token_unused_id = max_tokens + 3 # This is a label, so don't add box size - processor.token_row_id = max_tokens + box_size + 4 - processor.token_eos_id = max_tokens + box_size + 5 - processor.max_boxes = settings.TABLE_REC_MAX_BOXES - 1 - processor.box_size = {"height": box_size, "width": box_size} +def load_processor(): + processor = SuryaProcessor() + processor.image_processor.train = False + processor.image_processor.max_size = settings.TABLE_REC_IMAGE_SIZE + + processor.token_pad_id = 0 + processor.token_eos_id = 1 + processor.token_bos_id = 2 + processor.token_row_id = 3 + processor.token_unused_id = 4 + processor.box_size = (BOX_DIM, BOX_DIM) return processor -class TableRecImageProcessor(DonutImageProcessor): - def __init__(self, *args, **kwargs): +class SuryaImageProcessor(DonutImageProcessor): + def __init__(self, *args, max_size=None, train=False, **kwargs): super().__init__(*args, **kwargs) self.patch_size = kwargs.get("patch_size", (4, 4)) + self.max_size = max_size + self.train = train + + @classmethod + def numpy_resize(cls, image: np.ndarray, size, interpolation=cv2.INTER_LANCZOS4): + max_width, max_height = size["width"], size["height"] + + resized_image = cv2.resize(image, (max_width, max_height), interpolation=interpolation) + resized_image = resized_image.transpose(2, 0, 1) + + return resized_image def process_inner(self, images: List[np.ndarray]): - images = [img.transpose(2, 0, 1) for img in images] # convert to CHW format + assert images[0].shape[2] == 3 # RGB input images, channel dim last - assert images[0].shape[0] == 3 # RGB input images, channel dim last + # This also applies the right channel dim format, to channel x height x width + images = [SuryaImageProcessor.numpy_resize(img, self.max_size, self.resample) for img in images] + assert images[0].shape[0] == 3 # RGB input images, channel dim first # Convert to float32 for rescale/normalize images = [img.astype(np.float32) for img in images] - # Rescale and normalize + # Pads with 255 (whitespace) + # Pad to max size to improve performance + max_size = self.max_size images = [ - self.rescale(img, scale=self.rescale_factor, input_data_format=ChannelDimension.FIRST) - for img in images + SuryaImageProcessor.pad_image( + image=image, + size=max_size, + input_data_format=ChannelDimension.FIRST, + pad_value=settings.RECOGNITION_PAD_VALUE + ) + for image in images ] + # Rescale and normalize + for idx in range(len(images)): + images[idx] = images[idx] * self.rescale_factor images = [ - self.normalize(img, mean=self.image_mean, std=self.image_std, input_data_format=ChannelDimension.FIRST) + SuryaImageProcessor.normalize(img, mean=self.image_mean, std=self.image_std, input_data_format=ChannelDimension.FIRST) for img in images ] return images - def process_boxes(self, boxes): - padded_boxes = [] - box_masks = [] - box_counts = [] - for b in boxes: - # Left pad for generation - padded_b = deepcopy(b) - padded_b.insert(0, [self.token_row_id] * 4) # special token for max col/row - padded_boxes.append(padded_b) - - max_boxes = max(len(b) for b in padded_boxes) - for i in range(len(padded_boxes)): - pad_len = max_boxes - len(padded_boxes[i]) - box_len = len(padded_boxes[i]) - box_mask = [0] * pad_len + [1] * box_len - padded_box = [[self.token_pad_id] * 4] * pad_len + padded_boxes[i] - padded_boxes[i] = padded_box - box_masks.append(box_mask) - box_counts.append([pad_len, max_boxes]) + def preprocess( + self, + images: ImageInput, + do_resize: bool = None, + size: Dict[str, int] = None, + resample: PILImageResampling = None, + do_thumbnail: bool = None, + do_align_long_axis: bool = None, + do_pad: bool = None, + random_padding: bool = False, + do_rescale: bool = None, + rescale_factor: float = None, + do_normalize: bool = None, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> PIL.Image.Image: + images = make_list_of_images(images) + + # Convert to numpy for later processing steps + images = [np.array(img) for img in images] + images = self.process_inner(images) + + data = {"pixel_values": images} + return BatchFeature(data=data, tensor_type=return_tensors) + + @classmethod + def pad_image( + cls, + image: np.ndarray, + size: Dict[str, int], + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + pad_value: float = 0.0, + ) -> np.ndarray: + output_height, output_width = size["height"], size["width"] + input_height, input_width = get_image_size(image, channel_dim=input_data_format) + + delta_width = output_width - input_width + delta_height = output_height - input_height - return padded_boxes, box_masks, box_counts + assert delta_width >= 0 and delta_height >= 0 - def resize_img_and_boxes(self, img, boxes): - orig_dim = img.size - new_size = (self.size["width"], self.size["height"]) - img.thumbnail(new_size, Image.Resampling.LANCZOS) # Shrink largest dimension to fit new size - img = img.resize(new_size, Image.Resampling.LANCZOS) # Stretch smaller dimension to fit new size + pad_top = delta_height // 2 + pad_left = delta_width // 2 - img = np.asarray(img, dtype=np.uint8) + pad_bottom = delta_height - pad_top + pad_right = delta_width - pad_left - width, height = orig_dim - box_width, box_height = self.box_size["width"], self.box_size["height"] + padding = ((pad_top, pad_bottom), (pad_left, pad_right)) + return pad(image, padding, data_format=data_format, input_data_format=input_data_format, constant_values=pad_value) + + @classmethod + def align_long_axis( + cls, + image: np.ndarray, + size: Dict[str, int], + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + ) -> np.ndarray: + input_height, input_width = image.shape[:2] + output_height, output_width = size["height"], size["width"] + + if (output_width < output_height and input_width > input_height) or ( + output_width > output_height and input_width < input_height + ): + image = np.rot90(image, 3) + + return image + + @classmethod + def normalize( + cls, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + return normalize( + image, mean=mean, std=std, data_format=data_format, input_data_format=input_data_format, **kwargs + ) + + +class SuryaProcessor(DonutProcessor): + def __init__(self, image_processor=None, tokenizer=None, train=False, **kwargs): + image_processor = SuryaImageProcessor.from_pretrained(settings.RECOGNITION_MODEL_CHECKPOINT) + if image_processor is None: + raise ValueError("You need to specify an `image_processor`.") + + tokenizer = Byt5LangTokenizer() + super().__init__(image_processor, tokenizer) + self.current_processor = self.image_processor + self._in_target_context_manager = False + + def resize_boxes(self, img, boxes): + width, height = img.size + box_width, box_height = self.box_size for box in boxes: # Rescale to 0-1024 box[0] = box[0] / width * box_width @@ -99,58 +196,39 @@ def resize_img_and_boxes(self, img, boxes): if box[3] > box_height: box[3] = box_height - return img, boxes + return boxes - def preprocess( - self, - images: ImageInput, - boxes: List[List[int]], - do_resize: bool = None, - size: Dict[str, int] = None, - resample: PILImageResampling = None, - do_thumbnail: bool = None, - do_align_long_axis: bool = None, - do_pad: bool = None, - random_padding: bool = False, - do_rescale: bool = None, - rescale_factor: float = None, - do_normalize: bool = None, - image_mean: Optional[Union[float, List[float]]] = None, - image_std: Optional[Union[float, List[float]]] = None, - return_tensors: Optional[Union[str, TensorType]] = None, - data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, - input_data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> Image.Image: - images = make_list_of_images(images) + def __call__(self, *args, **kwargs): + images = kwargs.pop("images", []) + boxes = kwargs.pop("boxes", []) + assert len(images) == len(boxes) - if not valid_images(images): - raise ValueError( - "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " - "torch.Tensor, tf.Tensor or jax.ndarray." - ) + if len(args) > 0: + images = args[0] + args = args[1:] - new_images = [] new_boxes = [] - for img, box in zip(images, boxes): - if len(box) > self.max_boxes: - raise ValueError(f"Too many boxes, max is {self.max_boxes}") - img, box = self.resize_img_and_boxes(img, box) - new_images.append(img) - new_boxes.append(box) + max_len = max([len(b) for b in boxes]) + 1 + box_masks = [] + box_ends = [] + for i in range(len(boxes)): + nb = self.resize_boxes(images[i], boxes[i]) + nb.insert(0, [self.token_row_id] * 4) # Insert special token for max rows/cols - images = new_images - boxes = new_boxes + pad_length = max_len - len(nb) + box_mask = [1] * len(nb) + [0] * (pad_length) + box_ends.append(len(nb)) + nb = nb + [[self.token_pad_id] * 4] * pad_length - # Convert to numpy for later processing steps - images = [np.array(image) for image in images] + new_boxes.append(nb) + box_masks.append(box_mask) - images = self.process_inner(images) - boxes, box_mask, box_counts = self.process_boxes(boxes) - data = { - "pixel_values": images, - "input_boxes": boxes, - "input_boxes_mask": box_mask, - "input_boxes_counts": box_counts, - } - return BatchFeature(data=data, tensor_type=return_tensors) \ No newline at end of file + box_ends = torch.tensor(box_ends, dtype=torch.long) + box_starts = torch.tensor([0] * len(boxes), dtype=torch.long) + box_ranges = torch.stack([box_starts, box_ends], dim=1) + + inputs = self.image_processor(images, *args, **kwargs) + inputs["input_boxes"] = torch.tensor(new_boxes, dtype=torch.long) + inputs["input_boxes_mask"] = torch.tensor(box_masks, dtype=torch.long) + inputs["input_boxes_counts"] = box_ranges + return inputs diff --git a/surya/postprocessing/text.py b/surya/postprocessing/text.py index fea9c3e..520a0ae 100644 --- a/surya/postprocessing/text.py +++ b/surya/postprocessing/text.py @@ -10,12 +10,12 @@ from surya.postprocessing.math.latex import is_latex -def sort_text_lines(lines: List[TextLine], tolerance=1.25): +def sort_text_lines(lines: List[TextLine] | List[dict], tolerance=1.25): # Sorts in reading order. Not 100% accurate, this should only # be used as a starting point for more advanced sorting. vertical_groups = {} for line in lines: - group_key = round(line.bbox[1] / tolerance) * tolerance + group_key = round(getattr(line, 'bbox', line.get("bbox", None))[1] / tolerance) * tolerance if group_key not in vertical_groups: vertical_groups[group_key] = [] vertical_groups[group_key].append(line) @@ -23,7 +23,7 @@ def sort_text_lines(lines: List[TextLine], tolerance=1.25): # Sort each group horizontally and flatten the groups into a single list sorted_lines = [] for _, group in sorted(vertical_groups.items()): - sorted_group = sorted(group, key=lambda x: x.bbox[0]) + sorted_group = sorted(group, key=lambda x: getattr(x, 'bbox', x.get("bbox", None))[0]) sorted_lines.extend(sorted_group) return sorted_lines diff --git a/surya/settings.py b/surya/settings.py index bc0cb1b..4b02533 100644 --- a/surya/settings.py +++ b/surya/settings.py @@ -72,9 +72,10 @@ def TORCH_DEVICE_MODEL(self) -> str: ORDER_BENCH_DATASET_NAME: str = "vikp/order_bench" # Table Rec - TABLE_REC_MODEL_CHECKPOINT: str = "vikp/table_rec" - TABLE_REC_IMAGE_SIZE: Dict = {"height": 1024, "width": 1024} - TABLE_REC_MAX_BOXES: int = 384 + TABLE_REC_MODEL_CHECKPOINT: str = "vikp/table_rec_ar3" + TABLE_REC_IMAGE_SIZE: Dict = {"height": 512, "width": 512} + TABLE_REC_MAX_BOXES: int = 512 + TABLE_REC_MAX_ROWS: int = 384 # Tesseract (for benchmarks only) TESSDATA_PREFIX: Optional[str] = None diff --git a/surya/tables.py b/surya/tables.py index 9b673ac..1ef6162 100644 --- a/surya/tables.py +++ b/surya/tables.py @@ -8,6 +8,7 @@ from surya.settings import settings from tqdm import tqdm import numpy as np +from surya.model.table_rec.config import SPECIAL_TOKENS def get_batch_size(): @@ -51,6 +52,7 @@ def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model batch_images = images[i:i+batch_size] batch_images = [image.convert("RGB") for image in batch_images] # also copies the images + current_batch_size = len(batch_images) orig_sizes = [image.size for image in batch_images] model_inputs = processor(images=batch_images, boxes=deepcopy(batch_list_bboxes)) @@ -65,42 +67,92 @@ def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model batch_pixel_values = torch.tensor(np.array(batch_pixel_values), dtype=model.dtype).to(model.device) batch_bbox_counts = torch.tensor(np.array(batch_bbox_counts), dtype=torch.long).to(model.device) - col_predictions = [] - row_predictions = [] + # Setup inputs for the decoder + batch_decoder_input = [[[model.config.decoder.bos_token_id] * 2] for _ in range(current_batch_size)] + batch_decoder_input = torch.tensor(np.stack(batch_decoder_input, axis=0), dtype=torch.long, device=model.device) + inference_token_count = batch_decoder_input.shape[1] + + col_predictions = None + row_predictions = None + max_tokens = min(batch_bbox_counts[:, 1].max().item(), settings.TABLE_REC_MAX_BOXES) + decoder_position_ids = torch.ones_like(batch_decoder_input[0, :, 0], dtype=torch.int64, device=model.device).cumsum(0) - 1 + model.decoder.model._setup_cache(model.config, batch_size, model.device, model.dtype) + model.text_encoder.model._setup_cache(model.config, batch_size, model.device, model.dtype) + + with torch.inference_mode(): + encoder_hidden_states = model.encoder(pixel_values=batch_pixel_values).last_hidden_state + text_encoder_hidden_states = model.text_encoder( + input_boxes=batch_bboxes, + input_boxes_counts=batch_bbox_counts, + cache_position=None, + attention_mask=batch_bbox_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=None, + use_cache=False + ).hidden_states + + token_count = 0 + while token_count < max_tokens: + is_prefill = token_count == 0 + return_dict = model.decoder( + input_ids=batch_decoder_input, + encoder_hidden_states=text_encoder_hidden_states, + cache_position=decoder_position_ids, + use_cache=True, + prefill=is_prefill + ) + + decoder_position_ids = decoder_position_ids[-1:] + 1 + row_logits = return_dict["row_logits"].detach() + col_logits = return_dict["col_logits"].detach() + + row_preds = torch.argmax(row_logits[:, -1], dim=-1).unsqueeze(1) + col_preds = torch.argmax(col_logits[:, -1], dim=-1).unsqueeze(1) + + if col_predictions is None: + col_predictions = col_preds + else: + col_predictions = torch.cat([col_predictions, col_preds], dim=1) + + if row_predictions is None: + row_predictions = row_preds + else: + row_predictions = torch.cat([row_predictions, row_preds], dim=1) + + batch_decoder_input = torch.cat([row_preds, col_preds], dim=1).unsqueeze(1) + + token_count += inference_token_count + inference_token_count = batch_decoder_input.shape[1] + + # Get raw values without special tokens + row_predictions -= SPECIAL_TOKENS + col_predictions -= SPECIAL_TOKENS + + ls_row_predictions = [] + ls_col_predictions = [] max_rows = [] max_cols = [] - with torch.inference_mode(): - return_dict = model( - pixel_values=batch_pixel_values, - decoder_input_boxes=batch_bboxes, - decoder_input_boxes_mask=batch_bbox_mask, - decoder_input_boxes_counts=batch_bbox_counts, - encoder_outputs=None, - past_key_values=None, - ) - row_logits = return_dict["row_logits"].detach() - col_logits = return_dict["col_logits"].detach() - - for z in range(len(batch_images)): - box_start_idx = batch_bbox_counts[z][0] - row_preds = row_logits[z][box_start_idx:].argmax(dim=-1) - max_row = row_preds[0] - row_preds = row_preds[1:] - - col_preds = col_logits[z][box_start_idx:].argmax(dim=-1) - max_col = col_preds[0] - col_preds = col_preds[1:] - - row_predictions.append(row_preds) - col_predictions.append(col_preds) - max_rows.append(max_row) - max_cols.append(max_col) - - assert len(row_predictions) == len(col_predictions) == len(max_rows) == len(max_cols) == len(batch_images) - for j, (row_pred, col_pred, max_row, max_col, row_bboxes) in enumerate(zip(row_predictions, col_predictions, max_rows, max_cols, batch_list_bboxes)): + for z in range(current_batch_size): + box_end_idx = batch_bbox_counts[z][1] + row_preds = row_predictions[z][:box_end_idx] + max_row = row_preds[0].item() + row_preds = row_preds[1:].tolist() + + col_preds = col_predictions[z][:box_end_idx] + max_col = col_preds[0].item() + col_preds = col_preds[1:].tolist() + + ls_row_predictions.append(row_preds) + ls_col_predictions.append(col_preds) + max_rows.append(max_row) + max_cols.append(max_col) + + assert len(ls_row_predictions) == len(ls_col_predictions) == len(max_rows) == len(max_cols) == len(batch_images) + for j, (row_pred, col_pred, max_row, max_col, row_bboxes) in enumerate(zip(ls_row_predictions, ls_col_predictions, max_rows, max_cols, batch_list_bboxes)): orig_size = orig_sizes[j] out_data = [] - assert len(row_pred) == len(col_pred) == len(row_bboxes) + # They either match up, or there are too many bboxes passed in + assert (len(row_pred) == len(col_pred) == len(row_bboxes)) or len(row_bboxes) > settings.TABLE_REC_MAX_BOXES for z, (row_idx, col_idx, bbox) in enumerate(zip(row_pred, col_pred, row_bboxes)): cell = TableCell( bbox=bbox, From 9989a7961512ad02f995f46298bd4f4f4d0e30e4 Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Mon, 30 Sep 2024 19:15:39 -0400 Subject: [PATCH 06/17] Ar version --- ocr_app.py | 5 +- surya/model/table_rec/config.py | 8 + surya/model/table_rec/decoder.py | 62 ++++++-- surya/model/table_rec/processor.py | 17 ++- surya/postprocessing/heatmap.py | 4 +- surya/schema.py | 8 + surya/settings.py | 4 +- surya/tables.py | 228 ++++++++++++++++++++++------- 8 files changed, 260 insertions(+), 76 deletions(-) diff --git a/ocr_app.py b/ocr_app.py index de5f25d..c8136cc 100644 --- a/ocr_app.py +++ b/ocr_app.py @@ -104,7 +104,7 @@ def table_recognition(img, filepath, page_idx: int, use_pdf_boxes: bool) -> (Ima item.bbox[3] + table_bbox[1] ]) labels.append(f"{item.row_id} / {item.col_id}") - table_img = draw_bboxes_on_image(adjusted_bboxes, table_img, labels=labels) + table_img = draw_bboxes_on_image(adjusted_bboxes, table_img, labels=labels, label_font_size=12) return table_img, table_preds @@ -182,6 +182,7 @@ def page_count(pdf_file): pil_image = get_page_image(in_file, page_number) else: pil_image = Image.open(in_file).convert("RGB") + page_number = None text_det = st.sidebar.button("Run Text Detection") text_rec = st.sidebar.button("Run OCR") @@ -227,7 +228,7 @@ def page_count(pdf_file): if table_rec: - table_img, pred = table_recognition(pil_image, in_file, page_number - 1, use_pdf_boxes) + table_img, pred = table_recognition(pil_image, in_file, page_number - 1 if page_number else None, use_pdf_boxes) with col1: st.image(table_img, caption="Table Recognition", use_column_width=True) st.json([p.model_dump() for p in pred], expanded=True) diff --git a/surya/model/table_rec/config.py b/surya/model/table_rec/config.py index 77120f0..b8a4ced 100644 --- a/surya/model/table_rec/config.py +++ b/surya/model/table_rec/config.py @@ -123,6 +123,10 @@ def __init__( tie_word_embeddings=False, aux_heads=0, # How many n-token-ahead heads to add causal=True, + max_classes=2 + SPECIAL_TOKENS, + max_width=1024 + SPECIAL_TOKENS, + max_height=1024 + SPECIAL_TOKENS, + out_box_size=1024, **kwargs, ): self.num_hidden_layers = num_hidden_layers @@ -156,6 +160,10 @@ def __init__( self.encoder_hidden_size=encoder_hidden_size self.causal = causal self.encoder_cross_attn_layers = encoder_cross_attn_layers + self.max_classes = max_classes + self.max_width = max_width + self.max_height = max_height + self.out_box_size = out_box_size super().__init__( pad_token_id=pad_token_id, diff --git a/surya/model/table_rec/decoder.py b/surya/model/table_rec/decoder.py index 89758d4..262183c 100644 --- a/surya/model/table_rec/decoder.py +++ b/surya/model/table_rec/decoder.py @@ -19,8 +19,8 @@ @dataclass class TableRecModelOutput(ModelOutput): - row_logits: torch.Tensor - col_logits: torch.Tensor | None = None + bbox_logits: torch.Tensor + class_logits: torch.Tensor | None = None hidden_states: torch.Tensor | None = None @@ -442,14 +442,42 @@ class LabelEmbedding(nn.Module): def __init__(self, config): super().__init__() self.vocab_size = config.vocab_size - self.row_embed = nn.Embedding(config.vocab_size, config.hidden_size) - self.col_embed = nn.Embedding(config.vocab_size, config.hidden_size) + self.x1_embed = nn.Embedding(config.max_width, config.hidden_size) + self.y1_embed = nn.Embedding(config.max_height, config.hidden_size) + self.x2_embed = nn.Embedding(config.max_width, config.hidden_size) + self.y2_embed = nn.Embedding(config.max_height, config.hidden_size) + self.w_embed = nn.Embedding(config.max_width, config.hidden_size) + self.h_embed = nn.Embedding(config.max_height, config.hidden_size) + self.cx_embed = nn.Embedding(config.max_width, config.hidden_size) + self.cy_embed = nn.Embedding(config.max_height, config.hidden_size) + self.class_embed = nn.Embedding(config.max_classes, config.hidden_size) + self.max_width = config.max_width + self.max_height = config.max_height + self.max_classes = config.max_classes def forward(self, labels: torch.LongTensor, input_box_counts: torch.LongTensor): - row_labels, col_labels = labels.unbind(dim=-1) - row_labels = torch.clamp(row_labels, 0, self.vocab_size - 1).long() - col_labels = torch.clamp(col_labels, 0, self.vocab_size - 1).long() - embedded = self.row_embed(row_labels) + self.col_embed(col_labels) # Embed the labels in as well + cx, cy, w, h, class_ = labels.to(torch.long).unbind(dim=-1) + # Shape is (batch_size, num_boxes/seq len, d_model) + x1 = (cx - w // 2).long() + y1 = (cy - h // 2).long() + x2 = (cx + w // 2).long() + y2 = (cy + h // 2).long() + x1 = torch.clamp(x1, 0, self.max_width - 1) + y1 = torch.clamp(y1, 0, self.max_height - 1) + x2 = torch.clamp(x2, 0, self.max_width - 1) + y2 = torch.clamp(y2, 0, self.max_height - 1) + + class_ = torch.clamp(class_, 0, self.max_classes - 1).long() + + w = torch.clamp(w, 0, self.max_width - 1).long() + h = torch.clamp(h, 0, self.max_height - 1).long() + cx = torch.clamp(cx, 0, self.max_width - 1).long() + cy = torch.clamp(cy, 0, self.max_height - 1).long() + + coord_embeds = self.x1_embed(x1) + self.y1_embed(y1) + self.x2_embed(x2) + self.y2_embed(y2) + class_embeds = self.class_embed(class_) + embedded = coord_embeds + self.w_embed(w) + self.h_embed(h) + self.cx_embed(cx) + self.cy_embed(cy) + class_embeds + return embedded @@ -649,8 +677,9 @@ def __init__(self, config, **kwargs): self.model = SuryaTableRecDecoderModel(config, embed_labels=True, embed_positions=False) self.vocab_size = config.vocab_size - self.row_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - self.col_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + self.bbox_head = nn.Linear(config.hidden_size, config.max_width * 4, bias=False) + self.class_head = nn.Linear(config.hidden_size, config.max_classes, bias=False) + self.max_width = config.max_width # Initialize weights and apply final processing self.post_init() @@ -698,15 +727,16 @@ def forward( ) hidden_states = outputs[0] - row_logits = self.row_head(hidden_states) - col_logits = self.col_head(hidden_states) + bbox_logits = self.bbox_head(hidden_states) + class_logits = self.class_head(hidden_states) + bsz, seq_len = class_logits.shape[:2] + bbox_logits = bbox_logits.view(bsz, seq_len, 4, self.max_width) return TableRecModelOutput( - row_logits=row_logits, - col_logits=col_logits, - hidden_states=outputs.hidden_states, + bbox_logits=bbox_logits, + class_logits=class_logits, + hidden_states=hidden_states, ) - @dataclass class TextEncoderOutput(CausalLMOutput): hidden_states: torch.FloatTensor = None diff --git a/surya/model/table_rec/processor.py b/surya/model/table_rec/processor.py index b96438e..d1fb5d2 100644 --- a/surya/model/table_rec/processor.py +++ b/surya/model/table_rec/processor.py @@ -1,3 +1,4 @@ +import math from typing import Dict, Union, Optional, List, Iterable import cv2 @@ -12,7 +13,7 @@ import PIL from surya.model.recognition.tokenizer import Byt5LangTokenizer from surya.settings import settings -from surya.model.table_rec.config import BOX_DIM +from surya.model.table_rec.config import BOX_DIM, SPECIAL_TOKENS def load_processor(): @@ -26,6 +27,7 @@ def load_processor(): processor.token_row_id = 3 processor.token_unused_id = 4 processor.box_size = (BOX_DIM, BOX_DIM) + processor.special_token_count = SPECIAL_TOKENS return processor @@ -176,6 +178,8 @@ def __init__(self, image_processor=None, tokenizer=None, train=False, **kwargs): super().__init__(image_processor, tokenizer) self.current_processor = self.image_processor self._in_target_context_manager = False + self.max_input_boxes = kwargs.get("max_input_boxes", 256) + self.extra_input_boxes = kwargs.get("extra_input_boxes", 64) def resize_boxes(self, img, boxes): width, height = img.size @@ -207,15 +211,24 @@ def __call__(self, *args, **kwargs): images = args[0] args = args[1:] + for i in range(len(boxes)): + if len(boxes[i]) > self.max_input_boxes: + downsample_ratio = math.ceil(len(boxes[i]) / self.max_input_boxes) + boxes[i] = boxes[i][::downsample_ratio] + new_boxes = [] max_len = max([len(b) for b in boxes]) + 1 box_masks = [] box_ends = [] for i in range(len(boxes)): nb = self.resize_boxes(images[i], boxes[i]) + nb = [[b + self.special_token_count for b in box] for box in nb] # shift up + nb.insert(0, [self.token_row_id] * 4) # Insert special token for max rows/cols + for _ in range(self.extra_input_boxes): + nb.append([self.token_unused_id] * 4) - pad_length = max_len - len(nb) + pad_length = max(max_len, self.max_input_boxes + self.extra_input_boxes) - len(nb) box_mask = [1] * len(nb) + [0] * (pad_length) box_ends.append(len(nb)) nb = nb + [[self.token_pad_id] * 4] * pad_length diff --git a/surya/postprocessing/heatmap.py b/surya/postprocessing/heatmap.py index 5a9d551..6905c1b 100644 --- a/surya/postprocessing/heatmap.py +++ b/surya/postprocessing/heatmap.py @@ -173,7 +173,7 @@ def get_and_clean_boxes(textmap, processor_size, image_size, text_threshold=None -def draw_bboxes_on_image(bboxes, image, labels=None): +def draw_bboxes_on_image(bboxes, image, labels=None, label_font_size=10): polys = [] for bb in bboxes: # Clockwise polygon @@ -185,7 +185,7 @@ def draw_bboxes_on_image(bboxes, image, labels=None): ] polys.append(poly) - return draw_polys_on_image(polys, image, labels) + return draw_polys_on_image(polys, image, labels, label_font_size=label_font_size) def draw_polys_on_image(corners, image, labels=None, box_padding=-1, label_offset=1, label_font_size=10): diff --git a/surya/schema.py b/surya/schema.py index fb88922..d6d4e7b 100644 --- a/surya/schema.py +++ b/surya/schema.py @@ -123,6 +123,14 @@ def area(self): def polygon(self): return [[self.bbox[0], self.bbox[1]], [self.bbox[2], self.bbox[1]], [self.bbox[2], self.bbox[3]], [self.bbox[0], self.bbox[3]]] + def intersection_pct(self, other): + if self.area == 0: + return 0 + + x_overlap = max(0, min(self.bbox[2], other.bbox[2]) - max(self.bbox[0], other.bbox[0])) + y_overlap = max(0, min(self.bbox[3], other.bbox[3]) - max(self.bbox[1], other.bbox[1])) + intersection = x_overlap * y_overlap + return intersection / self.area class LayoutBox(PolygonBox): label: str diff --git a/surya/settings.py b/surya/settings.py index 4b02533..3e3e5ba 100644 --- a/surya/settings.py +++ b/surya/settings.py @@ -72,8 +72,8 @@ def TORCH_DEVICE_MODEL(self) -> str: ORDER_BENCH_DATASET_NAME: str = "vikp/order_bench" # Table Rec - TABLE_REC_MODEL_CHECKPOINT: str = "vikp/table_rec_ar3" - TABLE_REC_IMAGE_SIZE: Dict = {"height": 512, "width": 512} + TABLE_REC_MODEL_CHECKPOINT: str = "vikp/table_rec_ar" + TABLE_REC_IMAGE_SIZE: Dict = {"height": 640, "width": 640} TABLE_REC_MAX_BOXES: int = 512 TABLE_REC_MAX_ROWS: int = 384 diff --git a/surya/tables.py b/surya/tables.py index 1ef6162..97678de 100644 --- a/surya/tables.py +++ b/surya/tables.py @@ -1,10 +1,11 @@ +from collections import defaultdict from copy import deepcopy from typing import List import torch from PIL import Image from surya.model.ordering.encoderdecoder import OrderVisionEncoderDecoderModel -from surya.schema import TableResult, TableCell +from surya.schema import TableResult, TableCell, Bbox from surya.settings import settings from tqdm import tqdm import numpy as np @@ -39,6 +40,63 @@ def sort_bboxes(bboxes, tolerance=1): return sorted_page_blocks +def cx_cy_to_corners(pred): + w = pred[2] / 2 + h = pred[3] / 2 + x1 = pred[0] - w + y1 = pred[1] - h + x2 = pred[0] + w + y2 = pred[1] + h + + return [x1, y1, x2, y2] + + +def corners_to_cx_cy(pred): + x = (pred[0] + pred[2]) / 2 + y = (pred[1] + pred[3]) / 2 + w = pred[2] - pred[0] + h = pred[3] - pred[1] + + return [x, y, w, h] + + +def snap_to_bboxes(rc_box, input_boxes, used_cells_row, used_cells_col, row=True, threshold=.2): + sel_bboxes = [] + for cell_idx, cell in enumerate(input_boxes): + rc_corner_bbox = cx_cy_to_corners(rc_box) + intersection_pct = Bbox(bbox=cell).intersection_pct(Bbox(bbox=rc_corner_bbox)) + + if row: + if cell_idx not in used_cells_row: + if intersection_pct > threshold: + sel_bboxes.append(cell) + used_cells_row.add(cell_idx) + else: + if cell_idx not in used_cells_col: + if intersection_pct > threshold: + sel_bboxes.append(cell) + used_cells_col.add(cell_idx) + + if len(sel_bboxes) == 0: + return rc_box, used_cells_row, used_cells_col + + new_bbox = [ + min([b[0] for b in sel_bboxes]), + min([b[1] for b in sel_bboxes]), + max([b[2] for b in sel_bboxes]), + max([b[3] for b in sel_bboxes]) + ] + new_bbox = [ + max(new_bbox[0], rc_corner_bbox[0]), + max(new_bbox[1], rc_corner_bbox[1]), + min(new_bbox[2], rc_corner_bbox[2]), + min(new_bbox[3], rc_corner_bbox[3]) + ] + cx_cy_box = corners_to_cx_cy(new_bbox) + return cx_cy_box, used_cells_row, used_cells_col + + + def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model: OrderVisionEncoderDecoderModel, processor, batch_size=None) -> List[TableResult]: assert all([isinstance(image, Image.Image) for image in images]) assert len(images) == len(bboxes) @@ -68,17 +126,19 @@ def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model batch_bbox_counts = torch.tensor(np.array(batch_bbox_counts), dtype=torch.long).to(model.device) # Setup inputs for the decoder - batch_decoder_input = [[[model.config.decoder.bos_token_id] * 2] for _ in range(current_batch_size)] + batch_decoder_input = [[[model.config.decoder.bos_token_id] * 5] for _ in range(current_batch_size)] batch_decoder_input = torch.tensor(np.stack(batch_decoder_input, axis=0), dtype=torch.long, device=model.device) inference_token_count = batch_decoder_input.shape[1] - col_predictions = None - row_predictions = None max_tokens = min(batch_bbox_counts[:, 1].max().item(), settings.TABLE_REC_MAX_BOXES) decoder_position_ids = torch.ones_like(batch_decoder_input[0, :, 0], dtype=torch.int64, device=model.device).cumsum(0) - 1 model.decoder.model._setup_cache(model.config, batch_size, model.device, model.dtype) model.text_encoder.model._setup_cache(model.config, batch_size, model.device, model.dtype) + batch_predictions = [[] for _ in range(current_batch_size)] + used_cells_row = [set() for _ in range(current_batch_size)] + used_cells_col = [set() for _ in range(current_batch_size)] + with torch.inference_mode(): encoder_hidden_states = model.encoder(pixel_values=batch_pixel_values).last_hidden_state text_encoder_hidden_states = model.text_encoder( @@ -92,6 +152,8 @@ def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model ).hidden_states token_count = 0 + all_done = torch.zeros(current_batch_size, dtype=torch.bool, device=model.device) + while token_count < max_tokens: is_prefill = token_count == 0 return_dict = model.decoder( @@ -103,67 +165,129 @@ def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model ) decoder_position_ids = decoder_position_ids[-1:] + 1 - row_logits = return_dict["row_logits"].detach() - col_logits = return_dict["col_logits"].detach() + box_logits = return_dict["bbox_logits"][:, -1, :].detach() + rowcol_logits = return_dict["class_logits"][:, -1, :].detach() + + rowcol_preds = torch.argmax(rowcol_logits, dim=-1) + box_preds = torch.argmax(box_logits, dim=-1) - row_preds = torch.argmax(row_logits[:, -1], dim=-1).unsqueeze(1) - col_preds = torch.argmax(col_logits[:, -1], dim=-1).unsqueeze(1) + done = (rowcol_preds == processor.tokenizer.eos_id) | (rowcol_preds == processor.tokenizer.pad_id) + done = done + all_done = all_done | done - if col_predictions is None: - col_predictions = col_preds - else: - col_predictions = torch.cat([col_predictions, col_preds], dim=1) + if all_done.all(): + break - if row_predictions is None: - row_predictions = row_preds - else: - row_predictions = torch.cat([row_predictions, row_preds], dim=1) + for batch_idx, (box_pred, class_pred, ucr, ucc, bboxes, status) in enumerate(zip(box_preds, rowcol_preds, used_cells_row, used_cells_col, batch_list_bboxes, all_done)): + if status: + continue + class_pred = class_pred.item() - SPECIAL_TOKENS + nb = processor.resize_boxes(batch_images[batch_idx], deepcopy(bboxes)) + new_bbox, ucr, ucc = snap_to_bboxes(box_pred.tolist(), nb, ucr, ucc, row=class_pred == 0) + new_bbox = torch.tensor(new_bbox, dtype=torch.long, device=model.device) + box_preds[batch_idx] = new_bbox - batch_decoder_input = torch.cat([row_preds, col_preds], dim=1).unsqueeze(1) + used_cells_row[batch_idx] = ucr + used_cells_col[batch_idx] = ucc + + batch_decoder_input = torch.cat([box_preds.unsqueeze(1), rowcol_preds.unsqueeze(1).unsqueeze(1)], dim=-1) + + for j, (pred, status) in enumerate(zip(batch_decoder_input, all_done)): + if not status: + batch_predictions[j].append(pred[0].tolist()) token_count += inference_token_count inference_token_count = batch_decoder_input.shape[1] + """ + for j, (preds, bboxes, orig_size) in enumerate(zip(batch_predictions, batch_list_bboxes, orig_sizes)): + out_data = [] + # They either match up, or there are too many bboxes passed in + img_w, img_h = orig_size + # cx, cy to corners + for i, pred in enumerate(preds): + scale_w = img_w / model.config.decoder.out_box_size + scale_h = img_h / model.config.decoder.out_box_size + class_ = int(pred[4] - SPECIAL_TOKENS) + pred = cx_cy_to_corners(pred) - # Get raw values without special tokens - row_predictions -= SPECIAL_TOKENS - col_predictions -= SPECIAL_TOKENS - - ls_row_predictions = [] - ls_col_predictions = [] - max_rows = [] - max_cols = [] - for z in range(current_batch_size): - box_end_idx = batch_bbox_counts[z][1] - row_preds = row_predictions[z][:box_end_idx] - max_row = row_preds[0].item() - row_preds = row_preds[1:].tolist() - - col_preds = col_predictions[z][:box_end_idx] - max_col = col_preds[0].item() - col_preds = col_preds[1:].tolist() - - ls_row_predictions.append(row_preds) - ls_col_predictions.append(col_preds) - max_rows.append(max_row) - max_cols.append(max_col) - - assert len(ls_row_predictions) == len(ls_col_predictions) == len(max_rows) == len(max_cols) == len(batch_images) - for j, (row_pred, col_pred, max_row, max_col, row_bboxes) in enumerate(zip(ls_row_predictions, ls_col_predictions, max_rows, max_cols, batch_list_bboxes)): - orig_size = orig_sizes[j] + preds[i] = [pred[0] * scale_w, pred[1] * scale_h, pred[2] * scale_w, pred[3] * scale_h, class_] + + rows = [p[:4] for p in preds if p[4] == 0] + cols = [p[:4] for p in preds if p[4] == 1] + + for cell in bboxes: + max_intersection = 0 + row_pred = -1 + for row_idx, row in enumerate(rows): + intersection_pct = Bbox(bbox=cell).intersection_pct(Bbox(bbox=row)) + if intersection_pct > max_intersection: + max_intersection = intersection_pct + row_pred = row_idx + + max_intersection = 0 + col_pred = -1 + for col_idx, col in enumerate(cols): + intersection_pct = Bbox(bbox=cell).intersection_pct(Bbox(bbox=col)) + if intersection_pct > max_intersection: + max_intersection = intersection_pct + col_pred = col_idx + + cell = TableCell( + bbox=cell, + col_id=col_pred, + row_id=row_pred + ) + out_data.append(cell) + + result = TableResult( + cells=out_data, + image_bbox=[0, 0, img_w, img_h], + ) + + output_order.append(result) + """ + for j, (preds, bboxes, orig_size) in enumerate(zip(batch_predictions, batch_list_bboxes, orig_sizes)): out_data = [] # They either match up, or there are too many bboxes passed in - assert (len(row_pred) == len(col_pred) == len(row_bboxes)) or len(row_bboxes) > settings.TABLE_REC_MAX_BOXES - for z, (row_idx, col_idx, bbox) in enumerate(zip(row_pred, col_pred, row_bboxes)): - cell = TableCell( - bbox=bbox, - col_id=col_idx, - row_id=row_idx - ) - out_data.append(cell) + img_w, img_h = orig_size + # cx, cy to corners + for i, pred in enumerate(preds): + width_scaler = img_w / model.config.decoder.out_box_size + height_scaler = img_h / model.config.decoder.out_box_size + w = pred[2] / 2 + h = pred[3] / 2 + x1 = pred[0] - w + y1 = pred[1] - h + x2 = pred[0] + w + y2 = pred[1] + h + class_ = int(pred[4] - SPECIAL_TOKENS) + + preds[i] = [x1 * width_scaler, y1 * height_scaler, x2 * width_scaler, y2 * height_scaler, class_] + + rows = [p[:4] for p in preds if p[4] == 0] + cols = [p[:4] for p in preds if p[4] == 1] + + for row_idx, row in enumerate(rows): + cell = TableCell( + bbox=row, + col_id=-1, + row_id=row_idx + ) + out_data.append(cell) + + for col_idx, col in enumerate(cols): + cell = TableCell( + bbox=col, + col_id=col_idx, + row_id=-1 + ) + out_data.append(cell) result = TableResult( cells=out_data, - image_bbox=[0, 0, orig_size[0], orig_size[1]], + image_bbox=[0, 0, img_w, img_h], ) + output_order.append(result) + return output_order \ No newline at end of file From b3fe5ae8d95b695b42b9552aaf8979ddc38f594f Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Tue, 1 Oct 2024 22:36:57 -0400 Subject: [PATCH 07/17] More memory efficient detection --- surya/detection.py | 37 +++++++++++++++++++---------------- surya/layout.py | 48 ++++++++++++++++++++++++++++++++-------------- surya/settings.py | 4 ++-- surya/tables.py | 10 +++++----- 4 files changed, 62 insertions(+), 37 deletions(-) diff --git a/surya/detection.py b/surya/detection.py index 370c62b..a5b4a28 100644 --- a/surya/detection.py +++ b/surya/detection.py @@ -1,4 +1,4 @@ -from typing import List, Tuple +from typing import List, Tuple, Generator import torch import numpy as np @@ -26,7 +26,12 @@ def get_batch_size(): return batch_size -def batch_detection(images: List, model: EfficientViTForSemanticSegmentation, processor, batch_size=None) -> Tuple[List[List[np.ndarray]], List[Tuple[int, int]]]: +def batch_detection( + images: List, + model: EfficientViTForSemanticSegmentation, + processor, + batch_size=None +) -> Generator[Tuple[List[List[np.ndarray]], List[Tuple[int, int]]], None, None]: assert all([isinstance(image, Image.Image) for image in images]) if batch_size is None: batch_size = get_batch_size() @@ -50,7 +55,6 @@ def batch_detection(images: List, model: EfficientViTForSemanticSegmentation, pr if len(current_batch) > 0: batches.append(current_batch) - all_preds = [] for batch_idx in tqdm(range(len(batches)), desc="Detecting bboxes"): batch_image_idxs = batches[batch_idx] batch_images = [images[j].convert("RGB") for j in batch_image_idxs] @@ -96,11 +100,7 @@ def batch_detection(images: List, model: EfficientViTForSemanticSegmentation, pr heatmaps[k] = np.vstack([heatmaps[k], pred_heatmaps[k]]) preds[idx] = heatmaps - all_preds.extend(preds) - - assert len(all_preds) == len(images) - assert all([len(pred) == heatmap_count for pred in all_preds]) - return all_preds, orig_sizes + yield preds, [orig_sizes[j] for j in batch_image_idxs] def parallel_get_lines(preds, orig_sizes): @@ -123,16 +123,21 @@ def parallel_get_lines(preds, orig_sizes): def batch_text_detection(images: List, model, processor, batch_size=None) -> List[TextDetectionResult]: - preds, orig_sizes = batch_detection(images, model, processor, batch_size=batch_size) + detection_generator = batch_detection(images, model, processor, batch_size=batch_size) + results = [] - if settings.IN_STREAMLIT or len(images) < settings.DETECTOR_MIN_PARALLEL_THRESH: # Ensures we don't parallelize with streamlit, or with very few images - for i in range(len(images)): - result = parallel_get_lines(preds[i], orig_sizes[i]) - results.append(result) - else: - max_workers = min(settings.DETECTOR_POSTPROCESSING_CPU_WORKERS, len(images)) + max_workers = min(settings.DETECTOR_POSTPROCESSING_CPU_WORKERS, len(images)) + parallelize = not settings.IN_STREAMLIT and len(images) >= settings.DETECTOR_MIN_PARALLEL_THRESH + + if parallelize: with ProcessPoolExecutor(max_workers=max_workers) as executor: - results = list(executor.map(parallel_get_lines, preds, orig_sizes)) + for preds, orig_sizes in detection_generator: + batch_results = list(executor.map(parallel_get_lines, preds, orig_sizes)) + results.extend(batch_results) + else: + for preds, orig_sizes in detection_generator: + for pred, orig_size in zip(preds, orig_sizes): + results.append(parallel_get_lines(pred, orig_size)) return results diff --git a/surya/layout.py b/surya/layout.py index 070e000..4702168 100644 --- a/surya/layout.py +++ b/surya/layout.py @@ -181,23 +181,43 @@ def parallel_get_regions(heatmaps: List[np.ndarray], orig_size, id2label, detect def batch_layout_detection(images: List, model, processor, detection_results: Optional[List[TextDetectionResult]] = None, batch_size=None) -> List[LayoutResult]: - preds, orig_sizes = batch_detection(images, model, processor, batch_size=batch_size) + layout_generator = batch_detection(images, model, processor, batch_size=batch_size) id2label = model.config.id2label results = [] - if settings.IN_STREAMLIT or len(images) < settings.DETECTOR_MIN_PARALLEL_THRESH: # Ensures we don't parallelize with streamlit or too few images - for i in range(len(images)): - result = parallel_get_regions(preds[i], orig_sizes[i], id2label, detection_results[i] if detection_results else None) - results.append(result) - else: - futures = [] - max_workers = min(settings.DETECTOR_POSTPROCESSING_CPU_WORKERS, len(images)) - with ProcessPoolExecutor(max_workers=max_workers) as executor: - for i in range(len(images)): - future = executor.submit(parallel_get_regions, preds[i], orig_sizes[i], id2label, detection_results[i] if detection_results else None) - futures.append(future) + max_workers = min(settings.DETECTOR_POSTPROCESSING_CPU_WORKERS, len(images)) + parallelize = not settings.IN_STREAMLIT and len(images) >= settings.DETECTOR_MIN_PARALLEL_THRESH - for future in futures: - results.append(future.result()) + if parallelize: + with ProcessPoolExecutor(max_workers=max_workers) as executor: + img_idx = 0 + for preds, orig_sizes in layout_generator: + futures = [] + for pred, orig_size in zip(preds, orig_sizes): + future = executor.submit( + parallel_get_regions, + pred, + orig_size, + id2label, + detection_results[img_idx] if detection_results else None + ) + + futures.append(future) + img_idx += 1 + + for future in futures: + results.append(future.result()) + else: + img_idx = 0 + for preds, orig_sizes in layout_generator: + for pred, orig_size in zip(preds, orig_sizes): + results.append(parallel_get_regions( + pred, + orig_size, + id2label, + detection_results[img_idx] if detection_results else None + )) + + img_idx += 1 return results \ No newline at end of file diff --git a/surya/settings.py b/surya/settings.py index 3e3e5ba..7f0063d 100644 --- a/surya/settings.py +++ b/surya/settings.py @@ -61,7 +61,7 @@ def TORCH_DEVICE_MODEL(self) -> str: RECOGNITION_ENCODER_BATCH_DIVISOR: int = 2 # Divisor for batch size in decoder # Layout - LAYOUT_MODEL_CHECKPOINT: str = "vikp/layout5" + LAYOUT_MODEL_CHECKPOINT: str = "vikp/layout6" LAYOUT_BENCH_DATASET_NAME: str = "vikp/publaynet_bench" # Ordering @@ -72,7 +72,7 @@ def TORCH_DEVICE_MODEL(self) -> str: ORDER_BENCH_DATASET_NAME: str = "vikp/order_bench" # Table Rec - TABLE_REC_MODEL_CHECKPOINT: str = "vikp/table_rec_ar" + TABLE_REC_MODEL_CHECKPOINT: str = "vikp/table_rec_ar2" TABLE_REC_IMAGE_SIZE: Dict = {"height": 640, "width": 640} TABLE_REC_MAX_BOXES: int = 512 TABLE_REC_MAX_ROWS: int = 384 diff --git a/surya/tables.py b/surya/tables.py index 97678de..f3a00be 100644 --- a/surya/tables.py +++ b/surya/tables.py @@ -60,7 +60,7 @@ def corners_to_cx_cy(pred): return [x, y, w, h] -def snap_to_bboxes(rc_box, input_boxes, used_cells_row, used_cells_col, row=True, threshold=.2): +def snap_to_bboxes(rc_box, input_boxes, used_cells_row, used_cells_col, row=True, row_threshold=.2, col_threshold=.2): sel_bboxes = [] for cell_idx, cell in enumerate(input_boxes): rc_corner_bbox = cx_cy_to_corners(rc_box) @@ -68,12 +68,12 @@ def snap_to_bboxes(rc_box, input_boxes, used_cells_row, used_cells_col, row=True if row: if cell_idx not in used_cells_row: - if intersection_pct > threshold: + if intersection_pct > row_threshold: sel_bboxes.append(cell) used_cells_row.add(cell_idx) else: if cell_idx not in used_cells_col: - if intersection_pct > threshold: + if intersection_pct > col_threshold: sel_bboxes.append(cell) used_cells_col.add(cell_idx) @@ -183,7 +183,8 @@ def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model continue class_pred = class_pred.item() - SPECIAL_TOKENS nb = processor.resize_boxes(batch_images[batch_idx], deepcopy(bboxes)) - new_bbox, ucr, ucc = snap_to_bboxes(box_pred.tolist(), nb, ucr, ucc, row=class_pred == 0) + is_row = class_pred == 0 + new_bbox, ucr, ucc = snap_to_bboxes(box_pred.tolist(), nb, ucr, ucc, row=is_row) new_bbox = torch.tensor(new_bbox, dtype=torch.long, device=model.device) box_preds[batch_idx] = new_bbox @@ -266,7 +267,6 @@ def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model rows = [p[:4] for p in preds if p[4] == 0] cols = [p[:4] for p in preds if p[4] == 1] - for row_idx, row in enumerate(rows): cell = TableCell( bbox=row, From 3e2b86c3ccddfe6905daa9b702f397c064e1523f Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Thu, 3 Oct 2024 14:39:55 -0400 Subject: [PATCH 08/17] Add table parsing script --- detect_layout.py | 4 +- detect_text.py | 4 +- ocr_app.py | 7 +- ocr_text.py | 4 +- reading_order.py | 4 +- surya/input/load.py | 19 ++++-- surya/input/pdflines.py | 50 ++++++++------ surya/layout.py | 10 ++- surya/schema.py | 6 +- surya/settings.py | 2 +- surya/tables.py | 144 ++++++++++++++++++++++------------------ table_recognition.py | 105 +++++++++++++++++++++-------- 12 files changed, 223 insertions(+), 136 deletions(-) diff --git a/detect_layout.py b/detect_layout.py index 8e791b7..3a54f81 100644 --- a/detect_layout.py +++ b/detect_layout.py @@ -27,10 +27,10 @@ def main(): det_processor = load_processor() if os.path.isdir(args.input_path): - images, names = load_from_folder(args.input_path, args.max) + images, names, _ = load_from_folder(args.input_path, args.max) folder_name = os.path.basename(args.input_path) else: - images, names = load_from_file(args.input_path, args.max) + images, names, _ = load_from_file(args.input_path, args.max) folder_name = os.path.basename(args.input_path).split(".")[0] line_predictions = batch_text_detection(images, det_model, det_processor) diff --git a/detect_text.py b/detect_text.py index e2ecc4d..e7b0e5a 100644 --- a/detect_text.py +++ b/detect_text.py @@ -28,10 +28,10 @@ def main(): processor = load_processor(checkpoint=checkpoint) if os.path.isdir(args.input_path): - images, names = load_from_folder(args.input_path, args.max) + images, names, _ = load_from_folder(args.input_path, args.max) folder_name = os.path.basename(args.input_path) else: - images, names = load_from_file(args.input_path, args.max) + images, names, _ = load_from_file(args.input_path, args.max) folder_name = os.path.basename(args.input_path).split(".")[0] start = time.time() diff --git a/ocr_app.py b/ocr_app.py index c8136cc..df7b366 100644 --- a/ocr_app.py +++ b/ocr_app.py @@ -78,11 +78,10 @@ def order_detection(img) -> (Image.Image, OrderResult): def table_recognition(img, filepath, page_idx: int, use_pdf_boxes: bool) -> (Image.Image, List[TableResult]): _, layout_pred = layout_detection(img) - layout_tables = [l for l in layout_pred.bboxes if l.label == "Table"] - layout_tables_bboxes = [l.bbox for l in layout_tables] + layout_tables = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] table_imgs = [] - for table_bbox in layout_tables_bboxes: + for table_bbox in layout_tables: table_imgs.append(img.crop(table_bbox)) if use_pdf_boxes: page_text = get_page_text_lines(filepath, page_idx, img.size) @@ -93,7 +92,7 @@ def table_recognition(img, filepath, page_idx: int, use_pdf_boxes: bool) -> (Ima table_bboxes = [[tb.bbox for tb in table_box.bboxes] for table_box in table_boxes] table_preds = batch_table_recognition(table_imgs, table_bboxes, table_model, table_processor) table_img = img.copy() - for results, table_bbox in zip(table_preds, layout_tables_bboxes): + for results, table_bbox in zip(table_preds, layout_tables): adjusted_bboxes = [] labels = [] for item in results.cells: diff --git a/ocr_text.py b/ocr_text.py index 5b5bd65..b22ec95 100644 --- a/ocr_text.py +++ b/ocr_text.py @@ -30,10 +30,10 @@ def main(): args = parser.parse_args() if os.path.isdir(args.input_path): - images, names = load_from_folder(args.input_path, args.max, args.start_page) + images, names, _ = load_from_folder(args.input_path, args.max, args.start_page) folder_name = os.path.basename(args.input_path) else: - images, names = load_from_file(args.input_path, args.max, args.start_page) + images, names, _ = load_from_file(args.input_path, args.max, args.start_page) folder_name = os.path.basename(args.input_path).split(".")[0] if args.lang_file: diff --git a/reading_order.py b/reading_order.py index cc30ad2..4277a8a 100644 --- a/reading_order.py +++ b/reading_order.py @@ -33,10 +33,10 @@ def main(): det_processor = load_det_processor() if os.path.isdir(args.input_path): - images, names = load_from_folder(args.input_path, args.max) + images, names, _ = load_from_folder(args.input_path, args.max) folder_name = os.path.basename(args.input_path) else: - images, names = load_from_file(args.input_path, args.max) + images, names, _ = load_from_file(args.input_path, args.max) folder_name = os.path.basename(args.input_path).split(".")[0] line_predictions = batch_text_detection(images, det_model, det_processor) diff --git a/surya/input/load.py b/surya/input/load.py index aa8f1a1..304fde4 100644 --- a/surya/input/load.py +++ b/surya/input/load.py @@ -1,5 +1,6 @@ import PIL +from surya.input.pdflines import get_page_text_lines from surya.input.processing import open_pdf, get_page_images import os import filetype @@ -26,15 +27,20 @@ def load_pdf(pdf_path, max_pages=None, start_page=None): page_indices = list(range(start_page, last_page)) images = get_page_images(doc, page_indices) + text_lines = get_page_text_lines( + pdf_path, + page_indices, + [i.size for i in images] + ) doc.close() names = [get_name_from_path(pdf_path) for _ in page_indices] - return images, names + return images, names, text_lines def load_image(image_path): image = Image.open(image_path).convert("RGB") name = get_name_from_path(image_path) - return [image], [name] + return [image], [name], [None] def load_from_file(input_path, max_pages=None, start_page=None): @@ -51,21 +57,24 @@ def load_from_folder(folder_path, max_pages=None, start_page=None): images = [] names = [] + text_lines = [] for path in image_paths: extension = filetype.guess(path) if extension and extension.extension == "pdf": - image, name = load_pdf(path, max_pages, start_page) + image, name, text_line = load_pdf(path, max_pages, start_page) images.extend(image) names.extend(name) + text_lines.extend(text_line) else: try: - image, name = load_image(path) + image, name, text_line = load_image(path) images.extend(image) names.extend(name) + text_lines.extend(text_line) except PIL.UnidentifiedImageError: print(f"Could not load image {path}") continue - return images, names + return images, names, text_lines def load_lang_file(lang_path, names): diff --git a/surya/input/pdflines.py b/surya/input/pdflines.py index d59a497..dbed72b 100644 --- a/surya/input/pdflines.py +++ b/surya/input/pdflines.py @@ -4,26 +4,34 @@ from surya.schema import PolygonBox -def get_page_text_lines(filepath, page_idx, out_size): - full_text = dictionary_output(filepath, sort=False, page_range=[page_idx], keep_chars=True)[0] - text_bbox = full_text["bbox"] - text_w_scale = out_size[0] / text_bbox[2] - text_h_scale = out_size[1] / text_bbox[3] - for block in full_text["blocks"]: - for line in block["lines"]: - line["bbox"] = [line["bbox"][0] * text_w_scale, line["bbox"][1] * text_h_scale, - line["bbox"][2] * text_w_scale, line["bbox"][3] * text_h_scale] - for span in line["spans"]: - for char in span["chars"]: - char["bbox"] = [char["bbox"][0] * text_w_scale, char["bbox"][1] * text_h_scale, - char["bbox"][2] * text_w_scale, char["bbox"][3] * text_h_scale] - return full_text +def get_page_text_lines(filepath: str, page_idxs: list, out_sizes: list): + assert len(page_idxs) == len(out_sizes) + pages_text = dictionary_output(filepath, sort=False, page_range=page_idxs, keep_chars=True) + for full_text, out_size in zip(pages_text, out_sizes): + text_bbox = full_text["bbox"] + text_w_scale = out_size[0] / text_bbox[2] + text_h_scale = out_size[1] / text_bbox[3] + for block in full_text["blocks"]: + for line in block["lines"]: + line["bbox"] = [line["bbox"][0] * text_w_scale, line["bbox"][1] * text_h_scale, + line["bbox"][2] * text_w_scale, line["bbox"][3] * text_h_scale] + for span in line["spans"]: + for char in span["chars"]: + char["bbox"] = [char["bbox"][0] * text_w_scale, char["bbox"][1] * text_h_scale, + char["bbox"][2] * text_w_scale, char["bbox"][3] * text_h_scale] + return pages_text -def get_table_blocks(tables, full_text, img_size): +def get_table_blocks(tables: list, full_text: list, img_size: list, table_thresh=.8): # Returns coordinates relative to input table, not full image table_texts = [] for table in tables: + table_poly = PolygonBox(polygon=[ + [table[0], table[1]], + [table[2], table[1]], + [table[2], table[3]], + [table[0], table[3]] + ]) table_text = [] for block in full_text["blocks"]: for line in block["lines"]: @@ -33,7 +41,7 @@ def get_table_blocks(tables, full_text, img_size): [line["bbox"][2], line["bbox"][3]], [line["bbox"][0], line["bbox"][3]] ]) - if line_poly.intersection_pct(table) < 0.8: + if line_poly.intersection_pct(table_poly) < table_thresh: continue curr_span = None curr_box = None @@ -42,7 +50,7 @@ def get_table_blocks(tables, full_text, img_size): if curr_span is None: curr_span = char["char"] curr_box = char["bbox"] - elif (char["bbox"][0] - curr_box[2]) / img_size[0] < 0.01: + elif (char["bbox"][0] - curr_box[2]) / img_size[0] < 0.01 and (char["bbox"][1] - curr_box[1]) / img_size[1] < 0.01: curr_span += char["char"] curr_box = [min(curr_box[0], char["bbox"][0]), min(curr_box[1], char["bbox"][1]), max(curr_box[2], char["bbox"][2]), max(curr_box[3], char["bbox"][3])] @@ -55,10 +63,10 @@ def get_table_blocks(tables, full_text, img_size): # Adjust to be relative to input table for item in table_text: item["bbox"] = [ - item["bbox"][0] - table.bbox[0], - item["bbox"][1] - table.bbox[1], - item["bbox"][2] - table.bbox[0], - item["bbox"][3] - table.bbox[1] + item["bbox"][0] - table[0], + item["bbox"][1] - table[1], + item["bbox"][2] - table[0], + item["bbox"][3] - table[1] ] table_text = sort_text_lines(table_text) table_texts.append(table_text) diff --git a/surya/layout.py b/surya/layout.py index 4702168..9f7dd4d 100644 --- a/surya/layout.py +++ b/surya/layout.py @@ -12,7 +12,7 @@ def get_regions_from_detection_result(detection_result: TextDetectionResult, heatmaps: List[np.ndarray], orig_size, id2label, segment_assignment, vertical_line_width=20) -> List[LayoutBox]: logits = np.stack(heatmaps, axis=0) - vertical_line_bboxes = [line for line in detection_result.vertical_lines] + vertical_line_bboxes = detection_result.vertical_lines line_bboxes = detection_result.bboxes # Scale back to processor size @@ -38,6 +38,8 @@ def get_regions_from_detection_result(detection_result: TextDetectionResult, hea detected_boxes = [] for heatmap_idx in range(1, len(id2label)): # Skip the blank class heatmap = logits[heatmap_idx] + if np.max(heatmap) < settings.DETECTOR_BLANK_THRESHOLD: + continue bboxes = get_detected_boxes(heatmap) bboxes = [bbox for bbox in bboxes if bbox.area > 25] for bb in bboxes: @@ -150,10 +152,14 @@ def get_regions(heatmaps: List[np.ndarray], orig_size, id2label, segment_assignm heatmap = heatmaps[i] assert heatmap.shape == segment_assignment.shape heatmap[segment_assignment != i] = 0 # zero out where another segment is + + # Skip processing empty labels + if np.max(heatmap) < settings.DETECTOR_BLANK_THRESHOLD: + continue + bbox = get_and_clean_boxes(heatmap, list(reversed(heatmap.shape)), orig_size) for bb in bbox: bboxes.append(LayoutBox(polygon=bb.polygon, label=id2label[i])) - heatmaps.append(heatmap) bboxes = keep_largest_boxes(bboxes) return bboxes diff --git a/surya/schema.py b/surya/schema.py index d6d4e7b..8b9204f 100644 --- a/surya/schema.py +++ b/surya/schema.py @@ -176,10 +176,12 @@ class OrderResult(BaseModel): class TableCell(Bbox): - row_id: int - col_id: int + row_id: int | None = None + col_id: int | None = None class TableResult(BaseModel): cells: List[TableCell] + rows: List[TableCell] + cols: List[TableCell] image_bbox: List[float] diff --git a/surya/settings.py b/surya/settings.py index 7f0063d..5a8f1ea 100644 --- a/surya/settings.py +++ b/surya/settings.py @@ -72,7 +72,7 @@ def TORCH_DEVICE_MODEL(self) -> str: ORDER_BENCH_DATASET_NAME: str = "vikp/order_bench" # Table Rec - TABLE_REC_MODEL_CHECKPOINT: str = "vikp/table_rec_ar2" + TABLE_REC_MODEL_CHECKPOINT: str = "vikp/table_rec_ar3" TABLE_REC_IMAGE_SIZE: Dict = {"height": 640, "width": 640} TABLE_REC_MAX_BOXES: int = 512 TABLE_REC_MAX_ROWS: int = 384 diff --git a/surya/tables.py b/surya/tables.py index f3a00be..ac6536e 100644 --- a/surya/tables.py +++ b/surya/tables.py @@ -62,8 +62,8 @@ def corners_to_cx_cy(pred): def snap_to_bboxes(rc_box, input_boxes, used_cells_row, used_cells_col, row=True, row_threshold=.2, col_threshold=.2): sel_bboxes = [] + rc_corner_bbox = cx_cy_to_corners(rc_box) for cell_idx, cell in enumerate(input_boxes): - rc_corner_bbox = cx_cy_to_corners(rc_box) intersection_pct = Bbox(bbox=cell).intersection_pct(Bbox(bbox=rc_corner_bbox)) if row: @@ -97,15 +97,15 @@ def snap_to_bboxes(rc_box, input_boxes, used_cells_row, used_cells_col, row=True -def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model: OrderVisionEncoderDecoderModel, processor, batch_size=None) -> List[TableResult]: +def batch_table_recognition(images: List, input_bboxes: List[List[List[float]]], model: OrderVisionEncoderDecoderModel, processor, batch_size=None) -> List[TableResult]: assert all([isinstance(image, Image.Image) for image in images]) - assert len(images) == len(bboxes) + assert len(images) == len(input_bboxes) if batch_size is None: batch_size = get_batch_size() output_order = [] - for i in tqdm(range(0, len(images), batch_size), desc="Finding reading order"): - batch_list_bboxes = deepcopy(bboxes[i:i+batch_size]) + for i in tqdm(range(0, len(images), batch_size), desc="Recognizing tables"): + batch_list_bboxes = deepcopy(input_bboxes[i:i+batch_size]) batch_list_bboxes = [sort_bboxes(page_bboxes) for page_bboxes in batch_list_bboxes] # Sort bboxes before passing in batch_images = images[i:i+batch_size] @@ -199,62 +199,14 @@ def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model token_count += inference_token_count inference_token_count = batch_decoder_input.shape[1] - """ - for j, (preds, bboxes, orig_size) in enumerate(zip(batch_predictions, batch_list_bboxes, orig_sizes)): - out_data = [] - # They either match up, or there are too many bboxes passed in - img_w, img_h = orig_size - # cx, cy to corners - for i, pred in enumerate(preds): - scale_w = img_w / model.config.decoder.out_box_size - scale_h = img_h / model.config.decoder.out_box_size - class_ = int(pred[4] - SPECIAL_TOKENS) - pred = cx_cy_to_corners(pred) - - preds[i] = [pred[0] * scale_w, pred[1] * scale_h, pred[2] * scale_w, pred[3] * scale_h, class_] - - rows = [p[:4] for p in preds if p[4] == 0] - cols = [p[:4] for p in preds if p[4] == 1] - - for cell in bboxes: - max_intersection = 0 - row_pred = -1 - for row_idx, row in enumerate(rows): - intersection_pct = Bbox(bbox=cell).intersection_pct(Bbox(bbox=row)) - if intersection_pct > max_intersection: - max_intersection = intersection_pct - row_pred = row_idx - - max_intersection = 0 - col_pred = -1 - for col_idx, col in enumerate(cols): - intersection_pct = Bbox(bbox=cell).intersection_pct(Bbox(bbox=col)) - if intersection_pct > max_intersection: - max_intersection = intersection_pct - col_pred = col_idx - - cell = TableCell( - bbox=cell, - col_id=col_pred, - row_id=row_pred - ) - out_data.append(cell) - result = TableResult( - cells=out_data, - image_bbox=[0, 0, img_w, img_h], - ) - - output_order.append(result) - """ for j, (preds, bboxes, orig_size) in enumerate(zip(batch_predictions, batch_list_bboxes, orig_sizes)): - out_data = [] - # They either match up, or there are too many bboxes passed in img_w, img_h = orig_size + width_scaler = img_w / model.config.decoder.out_box_size + height_scaler = img_h / model.config.decoder.out_box_size + # cx, cy to corners for i, pred in enumerate(preds): - width_scaler = img_w / model.config.decoder.out_box_size - height_scaler = img_h / model.config.decoder.out_box_size w = pred[2] / 2 h = pred[3] / 2 x1 = pred[0] - w @@ -265,26 +217,88 @@ def batch_table_recognition(images: List, bboxes: List[List[List[float]]], model preds[i] = [x1 * width_scaler, y1 * height_scaler, x2 * width_scaler, y2 * height_scaler, class_] - rows = [p[:4] for p in preds if p[4] == 0] - cols = [p[:4] for p in preds if p[4] == 1] - for row_idx, row in enumerate(rows): + # Get rows and columns + bb_rows = [p[:4] for p in preds if p[4] == 0] + bb_cols = [p[:4] for p in preds if p[4] == 1] + + rows = [] + cols = [] + for row_idx, row in enumerate(bb_rows): cell = TableCell( bbox=row, - col_id=-1, row_id=row_idx ) - out_data.append(cell) + rows.append(cell) - for col_idx, col in enumerate(cols): + for col_idx, col in enumerate(bb_cols): cell = TableCell( bbox=col, col_id=col_idx, - row_id=-1 ) - out_data.append(cell) + cols.append(cell) + + # Assign cells to rows/columns + cells = [] + for cell in bboxes: + max_intersection = 0 + row_pred = None + for row_idx, row in enumerate(rows): + intersection_pct = Bbox(bbox=cell).intersection_pct(row) + if intersection_pct > max_intersection: + max_intersection = intersection_pct + row_pred = row_idx + + max_intersection = 0 + col_pred = None + for col_idx, col in enumerate(cols): + intersection_pct = Bbox(bbox=cell).intersection_pct(col) + if intersection_pct > max_intersection: + max_intersection = intersection_pct + col_pred = col_idx + + cells.append( + TableCell( + bbox=cell, + row_id=row_pred, + col_id=col_pred + ) + ) + + for cell in cells: + if cell.row_id is None: + closest_row = None + closest_row_dist = None + for cell2 in cells: + if cell2.row_id is None: + continue + cell_y_center = (cell.bbox[1] + cell.bbox[3]) / 2 + cell2_y_center = (cell2.bbox[1] + cell2.bbox[3]) / 2 + y_dist = abs(cell_y_center - cell2_y_center) + if closest_row_dist is None or y_dist < closest_row_dist: + closest_row = cell2.row_id + closest_row_dist = y_dist + cell.row_id = closest_row + + if cell.col_id is None: + closest_col = None + closest_col_dist = None + for cell2 in cells: + if cell2.col_id is None: + continue + cell_x_center = (cell.bbox[0] + cell.bbox[2]) / 2 + cell2_x_center = (cell2.bbox[0] + cell2.bbox[2]) / 2 + x_dist = abs(cell2_x_center - cell_x_center) + if closest_col_dist is None or x_dist < closest_col_dist: + closest_col = cell2.col_id + closest_col_dist = x_dist + + cell.col_id = closest_col + result = TableResult( - cells=out_data, + cells=cells, + rows=rows, + cols=cols, image_bbox=[0, 0, img_w, img_h], ) diff --git a/table_recognition.py b/table_recognition.py index c83d2e3..1d7bfbc 100644 --- a/table_recognition.py +++ b/table_recognition.py @@ -1,3 +1,4 @@ +import pypdfium2 as pdfium # Needs to be on top to avoid warning import os import argparse import copy @@ -6,12 +7,13 @@ from surya.detection import batch_text_detection from surya.input.load import load_from_folder, load_from_file +from surya.input.pdflines import get_table_blocks from surya.layout import batch_layout_detection from surya.model.detection.model import load_model as load_det_model, load_processor as load_det_processor -from surya.model.ordering.model import load_model -from surya.model.ordering.processor import load_processor -from surya.ordering import batch_ordering -from surya.postprocessing.heatmap import draw_polys_on_image +from surya.model.table_rec.model import load_model as load_model +from surya.model.table_rec.processor import load_processor +from surya.tables import batch_table_recognition +from surya.postprocessing.heatmap import draw_polys_on_image, draw_bboxes_on_image from surya.settings import settings @@ -21,6 +23,7 @@ def main(): parser.add_argument("--results_dir", type=str, help="Path to JSON file with layout results.", default=os.path.join(settings.RESULT_DIR, "surya")) parser.add_argument("--max", type=int, help="Maximum number of pages to process.", default=None) parser.add_argument("--images", action="store_true", help="Save images of detected layout bboxes.", default=False) + parser.add_argument("--detect_boxes", action="store_true", help="Detect table boxes.", default=False) args = parser.parse_args() model = load_model() @@ -33,46 +36,92 @@ def main(): det_processor = load_det_processor() if os.path.isdir(args.input_path): - images, names = load_from_folder(args.input_path, args.max) + images, names, text_lines = load_from_folder(args.input_path, args.max) folder_name = os.path.basename(args.input_path) else: - images, names = load_from_file(args.input_path, args.max) + images, names, text_lines = load_from_file(args.input_path, args.max) folder_name = os.path.basename(args.input_path).split(".")[0] + pnums = [] + prev_name = None + for i, name in enumerate(names): + if prev_name is None or prev_name != name: + pnums.append(0) + else: + pnums.append(pnums[-1] + 1) + line_predictions = batch_text_detection(images, det_model, det_processor) layout_predictions = batch_layout_detection(images, layout_model, layout_processor, line_predictions) table_boxes = [] - for layout_pred in layout_predictions: - bbox = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] - table_boxes.append(bbox) + table_cells = [] + table_cells_text = [] - order_predictions = batch_ordering(images, bboxes, model, processor) + table_imgs = [] + table_counts = [] + for layout_pred, text_line, img in zip(layout_predictions, text_lines, images): + # The bbox for the entire table + bbox = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] + # Number of tables per page + table_counts.append(len(bbox)) + + if len(bbox) == 0: + continue + + table_boxes.extend(bbox) + + page_table_imgs = [img.crop(bb) for bb in bbox] + table_imgs.extend(page_table_imgs) + + # The text cells inside each table + if text_line is None or args.detect_boxes: + cell_bboxes = batch_text_detection(page_table_imgs, det_model, det_processor) + cell_bboxes = [[tb.bbox for tb in table_box.bboxes] for table_box in cell_bboxes] + cell_text = [[None for tb in table_box.bboxes] for table_box in cell_bboxes] + table_cells_text.extend(cell_text) + table_cells.extend(cell_bboxes) + else: + table_texts = get_table_blocks(bbox, text_line, img.size) + table_cells.extend( + [[tb["bbox"] for tb in table_text] for table_text in table_texts] + ) + table_cells_text.extend( + [[tb["text"] for tb in table_text] for table_text in table_texts] + ) + + table_preds = batch_table_recognition(table_imgs, table_cells, model, processor) result_path = os.path.join(args.results_dir, folder_name) os.makedirs(result_path, exist_ok=True) if args.images: - for idx, (image, layout_pred, order_pred, name) in enumerate(zip(images, layout_predictions, order_predictions, names)): - polys = [l.polygon for l in order_pred.bboxes] - labels = [str(l.position) for l in order_pred.bboxes] - bbox_image = draw_polys_on_image(polys, copy.deepcopy(image), labels=labels, label_font_size=20) - bbox_image.save(os.path.join(result_path, f"{name}_{idx}_order.png")) - - predictions_by_page = defaultdict(list) - for idx, (layout_pred, pred, name, image) in enumerate(zip(layout_predictions, order_predictions, names, images)): - out_pred = pred.model_dump() - for bbox, layout_bbox in zip(out_pred["bboxes"], layout_pred.bboxes): - bbox["label"] = layout_bbox.label + pass - out_pred["page"] = len(predictions_by_page[name]) + 1 - predictions_by_page[name].append(out_pred) + img_idx = 0 + prev_count = 0 + table_predictions = defaultdict(list) + for i in range(sum(table_counts)): + while i >= prev_count + table_counts[img_idx]: + prev_count += table_counts[img_idx] + img_idx += 1 + + pred = table_preds[i] + orig_name = names[img_idx] + pnum = pnums[img_idx] + table_img = table_imgs[i] + + out_pred = pred.model_dump() + out_pred["page"] = pnum + 1 + table_idx = i - prev_count + out_pred["table_idx"] = table_idx + table_predictions[orig_name].append(out_pred) - # Sort in reading order - for name in predictions_by_page: - for page_preds in predictions_by_page[name]: - page_preds["bboxes"] = sorted(page_preds["bboxes"], key=lambda x: x["position"]) + if args.images: + boxes = [l.bbox for l in pred.cells] + labels = [f"{l.row_id}/{l.col_id}" for l in pred.cells] + bbox_image = draw_bboxes_on_image(boxes, table_img, labels=labels, label_font_size=20) + bbox_image.save(os.path.join(result_path, f"{name}_page{pnum + 1}_table{table_idx}_table.png")) with open(os.path.join(result_path, "results.json"), "w+", encoding="utf-8") as f: - json.dump(predictions_by_page, f, ensure_ascii=False) + json.dump(table_predictions, f, ensure_ascii=False) print(f"Wrote results to {result_path}") From 7489c14c9a964b1dc5cb83895b9715b5c28c75c7 Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Thu, 3 Oct 2024 16:03:23 -0400 Subject: [PATCH 09/17] Mostly handle rotation --- ocr_app.py | 2 +- surya/input/pdflines.py | 32 ++++++++++++++++++++++++++------ surya/postprocessing/heatmap.py | 10 +++++----- table_recognition.py | 18 +++++++++++++++--- 4 files changed, 47 insertions(+), 15 deletions(-) diff --git a/ocr_app.py b/ocr_app.py index df7b366..5071f88 100644 --- a/ocr_app.py +++ b/ocr_app.py @@ -84,7 +84,7 @@ def table_recognition(img, filepath, page_idx: int, use_pdf_boxes: bool) -> (Ima for table_bbox in layout_tables: table_imgs.append(img.crop(table_bbox)) if use_pdf_boxes: - page_text = get_page_text_lines(filepath, page_idx, img.size) + page_text = get_page_text_lines(filepath, [page_idx], [img.size])[0] table_texts = get_table_blocks(layout_tables, page_text, img.size) table_bboxes = [[tb["bbox"] for tb in table_text] for table_text in table_texts] else: diff --git a/surya/input/pdflines.py b/surya/input/pdflines.py index dbed72b..6cb4c76 100644 --- a/surya/input/pdflines.py +++ b/surya/input/pdflines.py @@ -4,13 +4,21 @@ from surya.schema import PolygonBox -def get_page_text_lines(filepath: str, page_idxs: list, out_sizes: list): +def rotate_90(bbox: list) -> list: + return [bbox[1], bbox[0], bbox[3], bbox[2]] + + + + + +def get_page_text_lines(filepath: str, page_idxs: list, out_sizes: list) -> list: assert len(page_idxs) == len(out_sizes) pages_text = dictionary_output(filepath, sort=False, page_range=page_idxs, keep_chars=True) for full_text, out_size in zip(pages_text, out_sizes): - text_bbox = full_text["bbox"] - text_w_scale = out_size[0] / text_bbox[2] - text_h_scale = out_size[1] / text_bbox[3] + width = full_text["width"] + height = full_text["height"] + text_w_scale = out_size[0] / width + text_h_scale = out_size[1] / height for block in full_text["blocks"]: for line in block["lines"]: line["bbox"] = [line["bbox"][0] * text_w_scale, line["bbox"][1] * text_h_scale, @@ -22,7 +30,7 @@ def get_page_text_lines(filepath: str, page_idxs: list, out_sizes: list): return pages_text -def get_table_blocks(tables: list, full_text: list, img_size: list, table_thresh=.8): +def get_table_blocks(tables: list, full_text: dict, img_size: list, table_thresh=.8): # Returns coordinates relative to input table, not full image table_texts = [] for table in tables: @@ -33,6 +41,7 @@ def get_table_blocks(tables: list, full_text: list, img_size: list, table_thresh [table[0], table[3]] ]) table_text = [] + rotation = full_text["rotation"] for block in full_text["blocks"]: for line in block["lines"]: line_poly = PolygonBox(polygon=[ @@ -47,10 +56,21 @@ def get_table_blocks(tables: list, full_text: list, img_size: list, table_thresh curr_box = None for span in line["spans"]: for char in span["chars"]: + same_span = False + if curr_span: + if rotation == 90: + same_span = (char["bbox"][0] - curr_box[0]) / img_size[0] < 0.01 and abs(char["bbox"][1] - curr_box[3]) / img_size[1] < 0.01 + elif rotation == 180: + same_span = (char["bbox"][2] - curr_box[0]) / img_size[0] < 0.01 and (char["bbox"][1] - curr_box[1]) / img_size[1] < 0.01 + elif rotation == 270: + same_span = (char["bbox"][0] - curr_box[0]) / img_size[0] < 0.01 and abs(char["bbox"][3] - curr_box[1]) / img_size[1] < 0.01 + else: + same_span = (char["bbox"][0] - curr_box[2]) / img_size[0] < 0.01 and (char["bbox"][1] - curr_box[1]) / img_size[1] < 0.01 + if curr_span is None: curr_span = char["char"] curr_box = char["bbox"] - elif (char["bbox"][0] - curr_box[2]) / img_size[0] < 0.01 and (char["bbox"][1] - curr_box[1]) / img_size[1] < 0.01: + elif same_span: curr_span += char["char"] curr_box = [min(curr_box[0], char["bbox"][0]), min(curr_box[1], char["bbox"][1]), max(curr_box[2], char["bbox"][2]), max(curr_box[3], char["bbox"][3])] diff --git a/surya/postprocessing/heatmap.py b/surya/postprocessing/heatmap.py index 6905c1b..2013dad 100644 --- a/surya/postprocessing/heatmap.py +++ b/surya/postprocessing/heatmap.py @@ -173,7 +173,7 @@ def get_and_clean_boxes(textmap, processor_size, image_size, text_threshold=None -def draw_bboxes_on_image(bboxes, image, labels=None, label_font_size=10): +def draw_bboxes_on_image(bboxes, image, labels=None, label_font_size=10, color='red'): polys = [] for bb in bboxes: # Clockwise polygon @@ -185,10 +185,10 @@ def draw_bboxes_on_image(bboxes, image, labels=None, label_font_size=10): ] polys.append(poly) - return draw_polys_on_image(polys, image, labels, label_font_size=label_font_size) + return draw_polys_on_image(polys, image, labels, label_font_size=label_font_size, color=color) -def draw_polys_on_image(corners, image, labels=None, box_padding=-1, label_offset=1, label_font_size=10): +def draw_polys_on_image(corners, image, labels=None, box_padding=-1, label_offset=1, label_font_size=10, color='red'): draw = ImageDraw.Draw(image) font_path = get_font_path() label_font = ImageFont.truetype(font_path, label_font_size) @@ -196,7 +196,7 @@ def draw_polys_on_image(corners, image, labels=None, box_padding=-1, label_offse for i in range(len(corners)): poly = corners[i] poly = [(int(p[0]), int(p[1])) for p in poly] - draw.polygon(poly, outline='red', width=1) + draw.polygon(poly, outline=color, width=1) if labels is not None: label = labels[i] @@ -215,7 +215,7 @@ def draw_polys_on_image(corners, image, labels=None, box_padding=-1, label_offse draw.text( text_position, label, - fill="red", + fill=color, font=label_font ) diff --git a/table_recognition.py b/table_recognition.py index 1d7bfbc..d56e1f1 100644 --- a/table_recognition.py +++ b/table_recognition.py @@ -13,7 +13,7 @@ from surya.model.table_rec.model import load_model as load_model from surya.model.table_rec.processor import load_processor from surya.tables import batch_table_recognition -from surya.postprocessing.heatmap import draw_polys_on_image, draw_bboxes_on_image +from surya.postprocessing.heatmap import draw_bboxes_on_image from surya.settings import settings @@ -50,6 +50,8 @@ def main(): else: pnums.append(pnums[-1] + 1) + prev_name = name + line_predictions = batch_text_detection(images, det_model, det_processor) layout_predictions = batch_layout_detection(images, layout_model, layout_processor, line_predictions) table_boxes = [] @@ -117,8 +119,18 @@ def main(): if args.images: boxes = [l.bbox for l in pred.cells] labels = [f"{l.row_id}/{l.col_id}" for l in pred.cells] - bbox_image = draw_bboxes_on_image(boxes, table_img, labels=labels, label_font_size=20) - bbox_image.save(os.path.join(result_path, f"{name}_page{pnum + 1}_table{table_idx}_table.png")) + bbox_image = draw_bboxes_on_image(boxes, copy.deepcopy(table_img), labels=labels, label_font_size=20) + bbox_image.save(os.path.join(result_path, f"{name}_page{pnum + 1}_table{table_idx}_cells.png")) + + rows = [l.bbox for l in pred.rows] + cols = [l.bbox for l in pred.cols] + row_labels = [f"Row {l.row_id}" for l in pred.rows] + col_labels = [f"Col {l.col_id}" for l in pred.cols] + + rc_image = copy.deepcopy(table_img) + rc_image = draw_bboxes_on_image(rows, rc_image, labels=row_labels, label_font_size=20, color="blue") + rc_image = draw_bboxes_on_image(cols, rc_image, labels=col_labels, label_font_size=20, color="red") + rc_image.save(os.path.join(result_path, f"{name}_page{pnum + 1}_table{table_idx}_rc.png")) with open(os.path.join(result_path, "results.json"), "w+", encoding="utf-8") as f: json.dump(table_predictions, f, ensure_ascii=False) From 4e89fe7fe25e06176911333ce54e132fb60cd0e6 Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Thu, 3 Oct 2024 18:17:26 -0400 Subject: [PATCH 10/17] Output text with bboxes --- ocr_app.py | 7 +++--- surya/schema.py | 5 ++++ surya/tables.py | 57 +++++++++++++++++++++++++++++++------------- table_recognition.py | 24 +++++++++---------- 4 files changed, 60 insertions(+), 33 deletions(-) diff --git a/ocr_app.py b/ocr_app.py index 5071f88..385042a 100644 --- a/ocr_app.py +++ b/ocr_app.py @@ -85,11 +85,10 @@ def table_recognition(img, filepath, page_idx: int, use_pdf_boxes: bool) -> (Ima table_imgs.append(img.crop(table_bbox)) if use_pdf_boxes: page_text = get_page_text_lines(filepath, [page_idx], [img.size])[0] - table_texts = get_table_blocks(layout_tables, page_text, img.size) - table_bboxes = [[tb["bbox"] for tb in table_text] for table_text in table_texts] + table_bboxes = get_table_blocks(layout_tables, page_text, img.size) else: - table_boxes = batch_text_detection(table_imgs, det_model, det_processor) - table_bboxes = [[tb.bbox for tb in table_box.bboxes] for table_box in table_boxes] + ocr_results = run_ocr(table_imgs, [None] * len(table_imgs), det_model, det_processor, rec_model, rec_processor) + table_bboxes = [[{"bbox": tb.bbox, "text": tb.text} for tb in ocr_result.text_lines] for ocr_result in ocr_results] table_preds = batch_table_recognition(table_imgs, table_bboxes, table_model, table_processor) table_img = img.copy() for results, table_bbox in zip(table_preds, layout_tables): diff --git a/surya/schema.py b/surya/schema.py index 8b9204f..880a77e 100644 --- a/surya/schema.py +++ b/surya/schema.py @@ -123,6 +123,10 @@ def area(self): def polygon(self): return [[self.bbox[0], self.bbox[1]], [self.bbox[2], self.bbox[1]], [self.bbox[2], self.bbox[3]], [self.bbox[0], self.bbox[3]]] + @property + def center(self): + return [(self.bbox[0] + self.bbox[2]) / 2, (self.bbox[1] + self.bbox[3]) / 2] + def intersection_pct(self, other): if self.area == 0: return 0 @@ -178,6 +182,7 @@ class OrderResult(BaseModel): class TableCell(Bbox): row_id: int | None = None col_id: int | None = None + text: str | None = None class TableResult(BaseModel): diff --git a/surya/tables.py b/surya/tables.py index ac6536e..09fefd9 100644 --- a/surya/tables.py +++ b/surya/tables.py @@ -1,6 +1,6 @@ from collections import defaultdict from copy import deepcopy -from typing import List +from typing import List, Dict import torch from PIL import Image @@ -26,7 +26,7 @@ def get_batch_size(): def sort_bboxes(bboxes, tolerance=1): vertical_groups = {} for block in bboxes: - group_key = round(block[1] / tolerance) * tolerance + group_key = round(block["bbox"][1] / tolerance) * tolerance if group_key not in vertical_groups: vertical_groups[group_key] = [] vertical_groups[group_key].append(block) @@ -34,7 +34,7 @@ def sort_bboxes(bboxes, tolerance=1): # Sort each group horizontally and flatten the groups into a single list sorted_page_blocks = [] for _, group in sorted(vertical_groups.items()): - sorted_group = sorted(group, key=lambda x: x[0]) + sorted_group = sorted(group, key=lambda x: x["bbox"][0]) sorted_page_blocks.extend(sorted_group) return sorted_page_blocks @@ -96,17 +96,30 @@ def snap_to_bboxes(rc_box, input_boxes, used_cells_row, used_cells_col, row=True return cx_cy_box, used_cells_row, used_cells_col +def is_rotated(rows, cols): + # Determine if the table is rotated by looking at row and column width / height ratios + # Rows should have a >1 ratio, cols <1 + widths = sum([r.width for r in rows]) + heights = sum([c.height for c in cols]) + r_ratio = widths / heights -def batch_table_recognition(images: List, input_bboxes: List[List[List[float]]], model: OrderVisionEncoderDecoderModel, processor, batch_size=None) -> List[TableResult]: + widths = sum([c.width for c in cols]) + heights = sum([r.height for r in rows]) + c_ratio = widths / heights + + return r_ratio * 2 < c_ratio + +def batch_table_recognition(images: List, table_cells: List[List[Dict]], model: OrderVisionEncoderDecoderModel, processor, batch_size=None) -> List[TableResult]: assert all([isinstance(image, Image.Image) for image in images]) - assert len(images) == len(input_bboxes) + assert len(images) == len(table_cells) if batch_size is None: batch_size = get_batch_size() output_order = [] for i in tqdm(range(0, len(images), batch_size), desc="Recognizing tables"): - batch_list_bboxes = deepcopy(input_bboxes[i:i+batch_size]) - batch_list_bboxes = [sort_bboxes(page_bboxes) for page_bboxes in batch_list_bboxes] # Sort bboxes before passing in + batch_table_cells = deepcopy(table_cells[i:i+batch_size]) + batch_table_cells = [sort_bboxes(page_bboxes) for page_bboxes in batch_table_cells] # Sort bboxes before passing in + batch_list_bboxes = [[block["bbox"] for block in page] for page in batch_table_cells] batch_images = images[i:i+batch_size] batch_images = [image.convert("RGB") for image in batch_images] # also copies the images @@ -200,7 +213,7 @@ def batch_table_recognition(images: List, input_bboxes: List[List[List[float]]], token_count += inference_token_count inference_token_count = batch_decoder_input.shape[1] - for j, (preds, bboxes, orig_size) in enumerate(zip(batch_predictions, batch_list_bboxes, orig_sizes)): + for j, (preds, input_cells, orig_size) in enumerate(zip(batch_predictions, batch_table_cells, orig_sizes)): img_w, img_h = orig_size width_scaler = img_w / model.config.decoder.out_box_size height_scaler = img_h / model.config.decoder.out_box_size @@ -239,11 +252,11 @@ def batch_table_recognition(images: List, input_bboxes: List[List[List[float]]], # Assign cells to rows/columns cells = [] - for cell in bboxes: + for cell in input_cells: max_intersection = 0 row_pred = None for row_idx, row in enumerate(rows): - intersection_pct = Bbox(bbox=cell).intersection_pct(row) + intersection_pct = Bbox(bbox=cell["bbox"]).intersection_pct(row) if intersection_pct > max_intersection: max_intersection = intersection_pct row_pred = row_idx @@ -251,19 +264,21 @@ def batch_table_recognition(images: List, input_bboxes: List[List[List[float]]], max_intersection = 0 col_pred = None for col_idx, col in enumerate(cols): - intersection_pct = Bbox(bbox=cell).intersection_pct(col) + intersection_pct = Bbox(bbox=cell["bbox"]).intersection_pct(col) if intersection_pct > max_intersection: max_intersection = intersection_pct col_pred = col_idx cells.append( TableCell( - bbox=cell, + bbox=cell["bbox"], + text=cell.get("text"), row_id=row_pred, col_id=col_pred ) ) + rotated = is_rotated(rows, cols) for cell in cells: if cell.row_id is None: closest_row = None @@ -271,8 +286,12 @@ def batch_table_recognition(images: List, input_bboxes: List[List[List[float]]], for cell2 in cells: if cell2.row_id is None: continue - cell_y_center = (cell.bbox[1] + cell.bbox[3]) / 2 - cell2_y_center = (cell2.bbox[1] + cell2.bbox[3]) / 2 + if rotated: + cell_y_center = cell.center[0] + cell2_y_center = cell2.center[0] + else: + cell_y_center = cell.center[1] + cell2_y_center = cell2.center[1] y_dist = abs(cell_y_center - cell2_y_center) if closest_row_dist is None or y_dist < closest_row_dist: closest_row = cell2.row_id @@ -285,8 +304,13 @@ def batch_table_recognition(images: List, input_bboxes: List[List[List[float]]], for cell2 in cells: if cell2.col_id is None: continue - cell_x_center = (cell.bbox[0] + cell.bbox[2]) / 2 - cell2_x_center = (cell2.bbox[0] + cell2.bbox[2]) / 2 + if rotated: + cell_x_center = cell.center[1] + cell2_x_center = cell2.center[1] + else: + cell_x_center = cell.center[0] + cell2_x_center = cell2.center[0] + x_dist = abs(cell2_x_center - cell_x_center) if closest_col_dist is None or x_dist < closest_col_dist: closest_col = cell2.col_id @@ -294,7 +318,6 @@ def batch_table_recognition(images: List, input_bboxes: List[List[List[float]]], cell.col_id = closest_col - result = TableResult( cells=cells, rows=rows, diff --git a/table_recognition.py b/table_recognition.py index d56e1f1..8178a34 100644 --- a/table_recognition.py +++ b/table_recognition.py @@ -12,6 +12,9 @@ from surya.model.detection.model import load_model as load_det_model, load_processor as load_det_processor from surya.model.table_rec.model import load_model as load_model from surya.model.table_rec.processor import load_processor +from surya.model.recognition.model import load_model as load_rec_model +from surya.model.recognition.processor import load_processor as load_rec_processor +from surya.ocr import run_ocr from surya.tables import batch_table_recognition from surya.postprocessing.heatmap import draw_bboxes_on_image from surya.settings import settings @@ -56,10 +59,15 @@ def main(): layout_predictions = batch_layout_detection(images, layout_model, layout_processor, line_predictions) table_boxes = [] table_cells = [] - table_cells_text = [] table_imgs = [] table_counts = [] + + rec_model, rec_processor = None, None + if args.detect_boxes or any([tl is None for tl in text_lines]): + rec_model = load_rec_model() + rec_processor = load_rec_processor() + for layout_pred, text_line, img in zip(layout_predictions, text_lines, images): # The bbox for the entire table bbox = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] @@ -76,19 +84,11 @@ def main(): # The text cells inside each table if text_line is None or args.detect_boxes: - cell_bboxes = batch_text_detection(page_table_imgs, det_model, det_processor) - cell_bboxes = [[tb.bbox for tb in table_box.bboxes] for table_box in cell_bboxes] - cell_text = [[None for tb in table_box.bboxes] for table_box in cell_bboxes] - table_cells_text.extend(cell_text) + ocr_results = run_ocr(page_table_imgs, [None] * len(page_table_imgs), det_model, det_processor, rec_model, rec_processor) + cell_bboxes = [[{"bbox": tb.bbox, "text": tb.text} for tb in ocr_result.text_lines] for ocr_result in ocr_results] table_cells.extend(cell_bboxes) else: - table_texts = get_table_blocks(bbox, text_line, img.size) - table_cells.extend( - [[tb["bbox"] for tb in table_text] for table_text in table_texts] - ) - table_cells_text.extend( - [[tb["text"] for tb in table_text] for table_text in table_texts] - ) + table_cells.extend(get_table_blocks(bbox, text_line, img.size)) table_preds = batch_table_recognition(table_imgs, table_cells, model, processor) result_path = os.path.join(args.results_dir, folder_name) From 50a85892837059710206ec88416cfeb84aa8cb57 Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Fri, 4 Oct 2024 16:48:38 -0400 Subject: [PATCH 11/17] Add table rec benchmark --- .github/workflows/tests.yml | 9 +-- README.md | 66 ++++++++++++++++++++-- benchmark/table_recognition.py | 88 ++++++++++++++++++++++++++++++ ocr_app.py | 30 +++++----- pyproject.toml | 4 +- scripts/verify_benchmark_scores.py | 10 ++++ surya/benchmark/metrics.py | 49 +++++++++++++++++ surya/input/pdflines.py | 7 --- surya/postprocessing/text.py | 4 +- surya/settings.py | 6 +- surya/tables.py | 10 ++-- table_recognition.py | 34 ++++++------ 12 files changed, 262 insertions(+), 55 deletions(-) create mode 100644 benchmark/table_recognition.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6620ab3..9edbe6e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,10 +36,11 @@ jobs: run: | poetry run python benchmark/layout.py --max 5 poetry run python scripts/verify_benchmark_scores.py results/benchmark/layout_bench/results.json --bench_type layout - - name: Run ordering benchmark text + - name: Run ordering benchmark run: | poetry run python benchmark/ordering.py --max 5 poetry run python scripts/verify_benchmark_scores.py results/benchmark/order_bench/results.json --bench_type ordering - - - + - name: Run table recognition benchmark + run: | + poetry run python benchmark/table_recognition.py --max 5 + poetry run python scripts/verify_benchmark_scores.py results/benchmark/table_rec_bench/results.json --bench_type table_recognition \ No newline at end of file diff --git a/README.md b/README.md index 3067cdb..93d2aa0 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Surya is a document OCR toolkit that does: - Line-level text detection in any language - Layout analysis (table, image, header, etc detection) - Reading order detection +- Table recognition (detecting rows/columns) It works on a range of documents (see [usage](#usage) and [benchmarks](#benchmarks) for more details). @@ -272,6 +273,43 @@ processor = load_processor() order_predictions = batch_ordering([image], [bboxes], model, processor) ``` +## Table Recognition + +This command will write out a json file with the detected table cells and row/column ids, along with row/column bounding boxes. + +```shell +surya_table DATA_PATH +``` + +- `DATA_PATH` can be an image, pdf, or folder of images/pdfs +- `--images` will save images of the pages and detected table cells + rows and columns (optional) +- `--max` specifies the maximum number of pages to process if you don't want to process everything +- `--results_dir` specifies the directory to save results to instead of the default +- `--detect_boxes` specifies if cells should be detected. By default, they're pulled out of the PDF, but this is not always possible. +- `--skip_table_detection` tells table recognition not to detect tables first. Use this if your image is already cropped to a table. + +The `results.json` file will contain a json dictionary where the keys are the input filenames without extensions. Each value will be a list of dictionaries, one per page of the input document. Each page dictionary contains: + +- `cells` - detected table cells + - `bbox` - the axis-aligned rectangle for the text line in (x1, y1, x2, y2) format. (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner. + - `row_id` - the id of the row this cell belongs to. + - `col_id` - the id of the column this cell belongs to. + - `text` - if text could be pulled out of the pdf, the text of this cell. +- `rows` - detected table rows + - `bbox` - the bounding box of the table row + - `row_id` - the id of the row +- `cols` - detected table columns + - `bbox` - the bounding box of the table column + - `col_id`- the id of the column +- `page` - the page number in the file +- `table_idx` - the index of the table on the page (sorted in vertical order) +- `image_bbox` - the bbox for the image in (x1, y1, x2, y2) format. (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner. All line bboxes will be contained within this bbox. + +**Performance tips** + +Setting the `TABLE_REC_BATCH_SIZE` env var properly will make a big difference when using a GPU. Each batch item will use `150MB` of VRAM, so very high batch sizes are possible. The default is a batch size `64`, which will use about 10GB of VRAM. Depending on your CPU core count, it might help, too - the default CPU batch size is `8`. + + # Limitations - This is specialized for document OCR. It will likely not work on photos or other images. @@ -381,10 +419,18 @@ I benchmarked the layout analysis on [Publaynet](https://github.com/ibm-aur-nlp/ **Methodology** -I benchmarked the layout analysis on the layout dataset from [here](https://www.icst.pku.edu.cn/cpdp/sjzy/), which was not in the training data. Unfortunately, this dataset is fairly noisy, and not all the labels are correct. It was very hard to find a dataset annotated with reading order and also layout information. I wanted to avoid using a cloud service for the ground truth. +I benchmarked the reading order on the layout dataset from [here](https://www.icst.pku.edu.cn/cpdp/sjzy/), which was not in the training data. Unfortunately, this dataset is fairly noisy, and not all the labels are correct. It was very hard to find a dataset annotated with reading order and also layout information. I wanted to avoid using a cloud service for the ground truth. The accuracy is computed by finding if each pair of layout boxes is in the correct order, then taking the % that are correct. +## Table Recognition + +.93 penalized row iou (out of 1), and .86 penalized column iou. Took .05 seconds per image on an A10. + +**Methodology** + +The benchmark uses a subset of [Fintabnet](https://developer.ibm.com/exchanges/data/all/fintabnet/) from IBM. It has labeled rows and columns. After table recognition is run, the predicted rows and columns are compared to the ground truth. There is an additional penalty for predicting too many or too few rows/columns. + ## Running your own benchmarks You can benchmark the performance of surya on your machine. @@ -396,7 +442,7 @@ You can benchmark the performance of surya on your machine. This will evaluate tesseract and surya for text line detection across a randomly sampled set of images from [doclaynet](https://huggingface.co/datasets/vikp/doclaynet_bench). -``` +```shell python benchmark/detection.py --max 256 ``` @@ -409,7 +455,7 @@ python benchmark/detection.py --max 256 This will evaluate surya and optionally tesseract on multilingual pdfs from common crawl (with synthetic data for missing languages). -``` +```shell python benchmark/recognition.py --tesseract ``` @@ -425,7 +471,7 @@ python benchmark/recognition.py --tesseract This will evaluate surya on the publaynet dataset. -``` +```shell python benchmark/layout.py ``` @@ -435,7 +481,7 @@ python benchmark/layout.py **Reading Order** -``` +```shell python benchmark/ordering.py ``` @@ -443,6 +489,16 @@ python benchmark/ordering.py - `--debug` will render images with detected text - `--results_dir` will let you specify a directory to save results to instead of the default one +**Table Recognition** + +```shell +python benchmark/table_recognition.py +``` + +- `--max` controls how many images to process for the benchmark +- `--debug` will render images with detected text +- `--results_dir` will let you specify a directory to save results to instead of the default one + # Training Text detection was trained on 4x A6000s for 3 days. It used a diverse set of images as training data. It was trained from scratch using a modified efficientvit architecture for semantic segmentation. diff --git a/benchmark/table_recognition.py b/benchmark/table_recognition.py new file mode 100644 index 0000000..a834eac --- /dev/null +++ b/benchmark/table_recognition.py @@ -0,0 +1,88 @@ +import argparse +import collections +import copy +import json + +from surya.input.processing import convert_if_not_rgb +from surya.model.table_rec.model import load_model +from surya.model.table_rec.processor import load_processor +from surya.tables import batch_table_recognition +from surya.settings import settings +from surya.benchmark.metrics import rank_accuracy, penalized_iou_score +import os +import time +import datasets + + +def main(): + parser = argparse.ArgumentParser(description="Benchmark surya table recognition model.") + parser.add_argument("--results_dir", type=str, help="Path to JSON file with benchmark results.", default=os.path.join(settings.RESULT_DIR, "benchmark")) + parser.add_argument("--max", type=int, help="Maximum number of images to run benchmark on.", default=None) + args = parser.parse_args() + + model = load_model() + processor = load_processor() + + pathname = "table_rec_bench" + # These have already been shuffled randomly, so sampling from the start is fine + split = "train" + if args.max is not None: + split = f"train[:{args.max}]" + dataset = datasets.load_dataset(settings.TABLE_REC_BENCH_DATASET_NAME, split=split) + images = list(dataset["image"]) + images = convert_if_not_rgb(images) + bboxes = list(dataset["bboxes"]) + + start = time.time() + bboxes = [[{"bbox": b, "text": None} for b in bb] for bb in bboxes] + table_rec_predictions = batch_table_recognition(images, bboxes, model, processor) + surya_time = time.time() - start + + folder_name = os.path.basename(pathname).split(".")[0] + result_path = os.path.join(args.results_dir, folder_name) + os.makedirs(result_path, exist_ok=True) + + page_metrics = collections.OrderedDict() + mean_col_iou = 0 + mean_row_iou = 0 + for idx, pred in enumerate(table_rec_predictions): + row = dataset[idx] + pred_row_boxes = [p.bbox for p in pred.rows] + pred_col_bboxes = [p.bbox for p in pred.cols] + actual_row_bboxes = row["rows"] + actual_col_bboxes = row["cols"] + row_score = penalized_iou_score(pred_row_boxes, actual_row_bboxes) + col_score = penalized_iou_score(pred_col_bboxes, actual_col_bboxes) + page_results = { + "row_score": row_score, + "col_score": col_score, + "row_count": len(actual_row_bboxes), + "col_count": len(actual_col_bboxes) + } + + mean_col_iou += col_score + mean_row_iou += row_score + + page_metrics[idx] = page_results + + mean_col_iou /= len(table_rec_predictions) + mean_row_iou /= len(table_rec_predictions) + + out_data = { + "time": surya_time, + "mean_row_iou": mean_row_iou, + "mean_col_iou": mean_col_iou, + "page_metrics": page_metrics + } + + with open(os.path.join(result_path, "results.json"), "w+") as f: + json.dump(out_data, f, indent=4) + + print(f"Mean penalized row iou is {mean_row_iou:.2f}. Mean penalized column iou is {mean_col_iou:.2f}.") + print(f"Took {surya_time / len(images):.2f} seconds per image, and {surya_time:.1f} seconds total.") + print("Mean iou is the average of the iou scores for each row or column, with penalties for too many/few predictions.") + print(f"Wrote results to {result_path}") + + +if __name__ == "__main__": + main() diff --git a/ocr_app.py b/ocr_app.py index 385042a..7851a8a 100644 --- a/ocr_app.py +++ b/ocr_app.py @@ -62,7 +62,7 @@ def layout_detection(img) -> (Image.Image, LayoutResult): pred = batch_layout_detection([img], layout_model, layout_processor, [det_pred])[0] polygons = [p.polygon for p in pred.bboxes] labels = [p.label for p in pred.bboxes] - layout_img = draw_polys_on_image(polygons, img.copy(), labels=labels) + layout_img = draw_polys_on_image(polygons, img.copy(), labels=labels, label_font_size=18) return layout_img, pred @@ -72,25 +72,28 @@ def order_detection(img) -> (Image.Image, OrderResult): pred = batch_ordering([img], [bboxes], order_model, order_processor)[0] polys = [l.polygon for l in pred.bboxes] positions = [str(l.position) for l in pred.bboxes] - order_img = draw_polys_on_image(polys, img.copy(), labels=positions, label_font_size=20) + order_img = draw_polys_on_image(polys, img.copy(), labels=positions, label_font_size=18) return order_img, pred -def table_recognition(img, filepath, page_idx: int, use_pdf_boxes: bool) -> (Image.Image, List[TableResult]): - _, layout_pred = layout_detection(img) - layout_tables = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] +def table_recognition(img, filepath, page_idx: int, use_pdf_boxes: bool, skip_table_detection: bool) -> (Image.Image, List[TableResult]): + if skip_table_detection: + layout_tables = [(0, 0, img.size[0], img.size[1])] + table_imgs = [img] + else: + _, layout_pred = layout_detection(img) + layout_tables = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] + table_imgs = [img.crop(tb) for tb in layout_tables] - table_imgs = [] - for table_bbox in layout_tables: - table_imgs.append(img.crop(table_bbox)) if use_pdf_boxes: page_text = get_page_text_lines(filepath, [page_idx], [img.size])[0] table_bboxes = get_table_blocks(layout_tables, page_text, img.size) else: - ocr_results = run_ocr(table_imgs, [None] * len(table_imgs), det_model, det_processor, rec_model, rec_processor) - table_bboxes = [[{"bbox": tb.bbox, "text": tb.text} for tb in ocr_result.text_lines] for ocr_result in ocr_results] + det_results = batch_text_detection(table_imgs, det_model, det_processor) + table_bboxes = [[{"bbox": tb.bbox, "text": None} for tb in det_result.bboxes] for det_result in det_results] table_preds = batch_table_recognition(table_imgs, table_bboxes, table_model, table_processor) table_img = img.copy() + for results, table_bbox in zip(table_preds, layout_tables): adjusted_bboxes = [] labels = [] @@ -102,7 +105,7 @@ def table_recognition(img, filepath, page_idx: int, use_pdf_boxes: bool) -> (Ima item.bbox[3] + table_bbox[1] ]) labels.append(f"{item.row_id} / {item.col_id}") - table_img = draw_bboxes_on_image(adjusted_bboxes, table_img, labels=labels, label_font_size=12) + table_img = draw_bboxes_on_image(adjusted_bboxes, table_img, labels=labels, label_font_size=18) return table_img, table_preds @@ -123,7 +126,7 @@ def open_pdf(pdf_file): @st.cache_data() -def get_page_image(pdf_file, page_num, dpi=96): +def get_page_image(pdf_file, page_num, dpi=settings.IMAGE_DPI): doc = open_pdf(pdf_file) renderer = doc.render( pypdfium2.PdfBitmap.to_pil, @@ -188,6 +191,7 @@ def page_count(pdf_file): order_det = st.sidebar.button("Run Reading Order") table_rec = st.sidebar.button("Run Table Rec") use_pdf_boxes = st.sidebar.checkbox("PDF table boxes", value=True, help="Table recognition only: Use the bounding boxes from the PDF file vs text detection model.") +skip_table_detection = st.sidebar.checkbox("Skip table detection", value=False, help="Table recognition only: Skip table detection and treat the whole image/page as a table.") if pil_image is None: st.stop() @@ -226,7 +230,7 @@ def page_count(pdf_file): if table_rec: - table_img, pred = table_recognition(pil_image, in_file, page_number - 1 if page_number else None, use_pdf_boxes) + table_img, pred = table_recognition(pil_image, in_file, page_number - 1 if page_number else None, use_pdf_boxes, skip_table_detection) with col1: st.image(table_img, caption="Table Recognition", use_column_width=True) st.json([p.model_dump() for p in pred], expanded=True) diff --git a/pyproject.toml b/pyproject.toml index 95bad5d..f856bc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "surya-ocr" -version = "0.5.0" +version = "0.5.1" description = "OCR, layout, reading order, and line detection in 90+ languages" authors = ["Vik Paruchuri "] readme = "README.md" @@ -17,6 +17,7 @@ include = [ "run_ocr_app.py", "detect_layout.py", "reading_order.py", + "table_recognition.py" ] [tool.poetry.dependencies] @@ -51,6 +52,7 @@ surya_ocr = "ocr_text:main" surya_layout = "detect_layout:main" surya_gui = "run_ocr_app:run_app" surya_order = "reading_order:main" +surya_table = "table_recognition:main" [build-system] requires = ["poetry-core"] diff --git a/scripts/verify_benchmark_scores.py b/scripts/verify_benchmark_scores.py index 4e8db55..4677305 100644 --- a/scripts/verify_benchmark_scores.py +++ b/scripts/verify_benchmark_scores.py @@ -27,6 +27,14 @@ def verify_order(data): raise ValueError("Scores do not meet the required threshold") +def verify_table_rec(data): + row_score = data["mean_row_iou"] + col_score = data["mean_col_iou"] + + if row_score < 0.75 or col_score < 0.75: + raise ValueError("Scores do not meet the required threshold") + + def verify_scores(file_path, bench_type): with open(file_path, 'r') as file: data = json.load(file) @@ -39,6 +47,8 @@ def verify_scores(file_path, bench_type): verify_layout(data) elif bench_type == "ordering": verify_order(data) + elif bench_type == "table_recognition": + verify_table_rec(data) else: raise ValueError("Invalid benchmark type") diff --git a/surya/benchmark/metrics.py b/surya/benchmark/metrics.py index afcb417..d0c1b33 100644 --- a/surya/benchmark/metrics.py +++ b/surya/benchmark/metrics.py @@ -4,6 +4,7 @@ import numpy as np from concurrent.futures import ProcessPoolExecutor + def intersection_area(box1, box2): x_left = max(box1[0], box2[0]) y_top = max(box1[1], box2[1]) @@ -15,6 +16,54 @@ def intersection_area(box1, box2): return (x_right - x_left) * (y_bottom - y_top) +def box_area(box): + return (box[2] - box[0]) * (box[3] - box[1]) + + +def calculate_iou(box1, box2): + intersection = intersection_area(box1, box2) + union = box_area(box1) + box_area(box2) - intersection + + if union == 0: + return 0 + return intersection / union + + +def match_boxes(preds, references): + num_actual = len(references) + num_predicted = len(preds) + + iou_matrix = np.zeros((num_actual, num_predicted)) + for i, actual in enumerate(references): + for j, pred in enumerate(preds): + iou_matrix[i, j] = calculate_iou(actual, pred) + + sorted_indices = np.argsort(iou_matrix, axis=None)[::-1] + sorted_ious = iou_matrix.flatten()[sorted_indices] + actual_indices, predicted_indices = np.unravel_index(sorted_indices, iou_matrix.shape) + + assigned_actual = set() + assigned_pred = set() + + matches = [] + for idx, iou in zip(zip(actual_indices, predicted_indices), sorted_ious): + i, j = idx + if i not in assigned_actual and j not in assigned_pred: + matches.append((i, j, iou_matrix[i, j])) + assigned_actual.add(i) + assigned_pred.add(j) + + unassigned_actual = set(range(num_actual)) - assigned_actual + unassigned_pred = set(range(num_predicted)) - assigned_pred + matches.extend([(i, None, 0.0) for i in unassigned_actual]) + matches.extend([(None, j, 0.0) for j in unassigned_pred]) + + return matches + +def penalized_iou_score(preds, references): + matches = match_boxes(preds, references) + iou = sum([match[2] for match in matches]) / len(matches) + return iou def intersection_pixels(box1, box2): x_left = max(box1[0], box2[0]) diff --git a/surya/input/pdflines.py b/surya/input/pdflines.py index 6cb4c76..8f36aef 100644 --- a/surya/input/pdflines.py +++ b/surya/input/pdflines.py @@ -4,13 +4,6 @@ from surya.schema import PolygonBox -def rotate_90(bbox: list) -> list: - return [bbox[1], bbox[0], bbox[3], bbox[2]] - - - - - def get_page_text_lines(filepath: str, page_idxs: list, out_sizes: list) -> list: assert len(page_idxs) == len(out_sizes) pages_text = dictionary_output(filepath, sort=False, page_range=page_idxs, keep_chars=True) diff --git a/surya/postprocessing/text.py b/surya/postprocessing/text.py index 520a0ae..542a80c 100644 --- a/surya/postprocessing/text.py +++ b/surya/postprocessing/text.py @@ -15,7 +15,7 @@ def sort_text_lines(lines: List[TextLine] | List[dict], tolerance=1.25): # be used as a starting point for more advanced sorting. vertical_groups = {} for line in lines: - group_key = round(getattr(line, 'bbox', line.get("bbox", None))[1] / tolerance) * tolerance + group_key = round(line.bbox[1] if isinstance(line, TextLine) else line["bbox"][1] / tolerance) * tolerance if group_key not in vertical_groups: vertical_groups[group_key] = [] vertical_groups[group_key].append(line) @@ -23,7 +23,7 @@ def sort_text_lines(lines: List[TextLine] | List[dict], tolerance=1.25): # Sort each group horizontally and flatten the groups into a single list sorted_lines = [] for _, group in sorted(vertical_groups.items()): - sorted_group = sorted(group, key=lambda x: getattr(x, 'bbox', x.get("bbox", None))[0]) + sorted_group = sorted(group, key=lambda x: x.bbox[0] if isinstance(x, TextLine) else x["bbox"][0]) sorted_lines.extend(sorted_group) return sorted_lines diff --git a/surya/settings.py b/surya/settings.py index 5a8f1ea..bc751c7 100644 --- a/surya/settings.py +++ b/surya/settings.py @@ -61,7 +61,7 @@ def TORCH_DEVICE_MODEL(self) -> str: RECOGNITION_ENCODER_BATCH_DIVISOR: int = 2 # Divisor for batch size in decoder # Layout - LAYOUT_MODEL_CHECKPOINT: str = "vikp/layout6" + LAYOUT_MODEL_CHECKPOINT: str = "vikp/surya_layout4" LAYOUT_BENCH_DATASET_NAME: str = "vikp/publaynet_bench" # Ordering @@ -72,10 +72,12 @@ def TORCH_DEVICE_MODEL(self) -> str: ORDER_BENCH_DATASET_NAME: str = "vikp/order_bench" # Table Rec - TABLE_REC_MODEL_CHECKPOINT: str = "vikp/table_rec_ar3" + TABLE_REC_MODEL_CHECKPOINT: str = "vikp/surya_tablerec" TABLE_REC_IMAGE_SIZE: Dict = {"height": 640, "width": 640} TABLE_REC_MAX_BOXES: int = 512 TABLE_REC_MAX_ROWS: int = 384 + TABLE_REC_BATCH_SIZE: Optional[int] = None + TABLE_REC_BENCH_DATASET_NAME: str = "vikp/fintabnet_bench" # Tesseract (for benchmarks only) TESSDATA_PREFIX: Optional[str] = None diff --git a/surya/tables.py b/surya/tables.py index 09fefd9..978842c 100644 --- a/surya/tables.py +++ b/surya/tables.py @@ -13,13 +13,13 @@ def get_batch_size(): - batch_size = settings.ORDER_BATCH_SIZE + batch_size = settings.TABLE_REC_BATCH_SIZE if batch_size is None: batch_size = 8 if settings.TORCH_DEVICE_MODEL == "mps": batch_size = 8 if settings.TORCH_DEVICE_MODEL == "cuda": - batch_size = 32 + batch_size = 64 return batch_size @@ -100,11 +100,11 @@ def is_rotated(rows, cols): # Determine if the table is rotated by looking at row and column width / height ratios # Rows should have a >1 ratio, cols <1 widths = sum([r.width for r in rows]) - heights = sum([c.height for c in cols]) + heights = sum([c.height for c in cols]) + 1 r_ratio = widths / heights widths = sum([c.width for c in cols]) - heights = sum([r.height for r in rows]) + heights = sum([r.height for r in rows]) + 1 c_ratio = widths / heights return r_ratio * 2 < c_ratio @@ -199,7 +199,7 @@ def batch_table_recognition(images: List, table_cells: List[List[Dict]], model: is_row = class_pred == 0 new_bbox, ucr, ucc = snap_to_bboxes(box_pred.tolist(), nb, ucr, ucc, row=is_row) new_bbox = torch.tensor(new_bbox, dtype=torch.long, device=model.device) - box_preds[batch_idx] = new_bbox + #box_preds[batch_idx] = new_bbox used_cells_row[batch_idx] = ucr used_cells_col[batch_idx] = ucc diff --git a/table_recognition.py b/table_recognition.py index 8178a34..e468705 100644 --- a/table_recognition.py +++ b/table_recognition.py @@ -27,6 +27,7 @@ def main(): parser.add_argument("--max", type=int, help="Maximum number of pages to process.", default=None) parser.add_argument("--images", action="store_true", help="Save images of detected layout bboxes.", default=False) parser.add_argument("--detect_boxes", action="store_true", help="Detect table boxes.", default=False) + parser.add_argument("--skip_table_detection", action="store_true", help="Tables are already cropped, so don't re-detect tables.", default=False) args = parser.parse_args() model = load_model() @@ -63,29 +64,30 @@ def main(): table_imgs = [] table_counts = [] - rec_model, rec_processor = None, None - if args.detect_boxes or any([tl is None for tl in text_lines]): - rec_model = load_rec_model() - rec_processor = load_rec_processor() - for layout_pred, text_line, img in zip(layout_predictions, text_lines, images): - # The bbox for the entire table - bbox = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] - # Number of tables per page - table_counts.append(len(bbox)) + # The table may already be cropped + if args.skip_table_detection: + table_imgs.append(img) + table_counts.append(1) + table_boxes.append(layout_pred.image_bbox) + else: + # The bbox for the entire table + bbox = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] + # Number of tables per page + table_counts.append(len(bbox)) - if len(bbox) == 0: - continue + if len(bbox) == 0: + continue - table_boxes.extend(bbox) + table_boxes.extend(bbox) - page_table_imgs = [img.crop(bb) for bb in bbox] - table_imgs.extend(page_table_imgs) + page_table_imgs = [img.crop(bb) for bb in bbox] + table_imgs.extend(page_table_imgs) # The text cells inside each table if text_line is None or args.detect_boxes: - ocr_results = run_ocr(page_table_imgs, [None] * len(page_table_imgs), det_model, det_processor, rec_model, rec_processor) - cell_bboxes = [[{"bbox": tb.bbox, "text": tb.text} for tb in ocr_result.text_lines] for ocr_result in ocr_results] + det_results = batch_text_detection(page_table_imgs, det_model, det_processor,) + cell_bboxes = [[{"bbox": tb.bbox, "text": None} for tb in det_result.bboxes] for det_result in det_results] table_cells.extend(cell_bboxes) else: table_cells.extend(get_table_blocks(bbox, text_line, img.size)) From 1d7954ddcc4cafb40dfe5618d6a8d27279b374bf Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Fri, 4 Oct 2024 19:51:33 -0400 Subject: [PATCH 12/17] Add tatr benchmark --- benchmark/table_recognition.py | 58 ++++++++++++-- scripts/verify_benchmark_scores.py | 4 +- surya/benchmark/metrics.py | 10 ++- surya/benchmark/tatr.py | 117 +++++++++++++++++++++++++++++ surya/tables.py | 15 ---- 5 files changed, 177 insertions(+), 27 deletions(-) create mode 100644 surya/benchmark/tatr.py diff --git a/benchmark/table_recognition.py b/benchmark/table_recognition.py index a834eac..008f43d 100644 --- a/benchmark/table_recognition.py +++ b/benchmark/table_recognition.py @@ -6,9 +6,10 @@ from surya.input.processing import convert_if_not_rgb from surya.model.table_rec.model import load_model from surya.model.table_rec.processor import load_processor -from surya.tables import batch_table_recognition +from surya.tables import batch_table_recognition, get_batch_size from surya.settings import settings from surya.benchmark.metrics import rank_accuracy, penalized_iou_score +from surya.benchmark.tatr import load_tatr, batch_inference_tatr import os import time import datasets @@ -18,6 +19,7 @@ def main(): parser = argparse.ArgumentParser(description="Benchmark surya table recognition model.") parser.add_argument("--results_dir", type=str, help="Path to JSON file with benchmark results.", default=os.path.join(settings.RESULT_DIR, "benchmark")) parser.add_argument("--max", type=int, help="Maximum number of images to run benchmark on.", default=None) + parser.add_argument("--tatr", action="store_true", help="Run table transformer.", default=False) args = parser.parse_args() model = load_model() @@ -68,21 +70,65 @@ def main(): mean_col_iou /= len(table_rec_predictions) mean_row_iou /= len(table_rec_predictions) - out_data = { + out_data = {"surya": { "time": surya_time, "mean_row_iou": mean_row_iou, "mean_col_iou": mean_col_iou, "page_metrics": page_metrics - } + }} + + if args.tatr: + tatr_model = load_tatr() + start = time.time() + tatr_predictions = batch_inference_tatr(tatr_model, images, 1) + tatr_time = time.time() - start + + page_metrics = collections.OrderedDict() + mean_col_iou = 0 + mean_row_iou = 0 + for idx, pred in enumerate(tatr_predictions): + row = dataset[idx] + pred_row_boxes = [p["bbox"] for p in pred["rows"]] + pred_col_bboxes = [p["bbox"] for p in pred["cols"]] + actual_row_bboxes = row["rows"] + actual_col_bboxes = row["cols"] + row_score = penalized_iou_score(pred_row_boxes, actual_row_bboxes) + col_score = penalized_iou_score(pred_col_bboxes, actual_col_bboxes) + page_results = { + "row_score": row_score, + "col_score": col_score, + "row_count": len(actual_row_bboxes), + "col_count": len(actual_col_bboxes) + } + + mean_col_iou += col_score + mean_row_iou += row_score + + page_metrics[idx] = page_results + + mean_col_iou /= len(tatr_predictions) + mean_row_iou /= len(tatr_predictions) + + out_data["tatr"] = { + "time": tatr_time, + "mean_row_iou": mean_row_iou, + "mean_col_iou": mean_col_iou, + "page_metrics": page_metrics + } + with open(os.path.join(result_path, "results.json"), "w+") as f: json.dump(out_data, f, indent=4) - print(f"Mean penalized row iou is {mean_row_iou:.2f}. Mean penalized column iou is {mean_col_iou:.2f}.") - print(f"Took {surya_time / len(images):.2f} seconds per image, and {surya_time:.1f} seconds total.") + print(f"Surya mean penalized row iou is {out_data['surya']['mean_row_iou']:.2f}. Mean penalized column iou is {out_data['surya']['mean_col_iou']:.2f}.") + if args.tatr: + print(f"TATR mean penalized row iou is {out_data['tatr']['mean_row_iou']:.2f}. Mean penalized column iou is {out_data['tatr']['mean_col_iou']:.2f}.") + print(f"Surya took {surya_time / len(images):.2f} seconds per image, and {surya_time:.1f} seconds total.") + if args.tatr: + print(f"TATR took {tatr_time / len(images):.2f} seconds per image, and {tatr_time:.1f} seconds total.") print("Mean iou is the average of the iou scores for each row or column, with penalties for too many/few predictions.") print(f"Wrote results to {result_path}") if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/scripts/verify_benchmark_scores.py b/scripts/verify_benchmark_scores.py index 4677305..956aaae 100644 --- a/scripts/verify_benchmark_scores.py +++ b/scripts/verify_benchmark_scores.py @@ -28,8 +28,8 @@ def verify_order(data): def verify_table_rec(data): - row_score = data["mean_row_iou"] - col_score = data["mean_col_iou"] + row_score = data["surya"]["mean_row_iou"] + col_score = data["surya"]["mean_col_iou"] if row_score < 0.75 or col_score < 0.75: raise ValueError("Scores do not meet the required threshold") diff --git a/surya/benchmark/metrics.py b/surya/benchmark/metrics.py index d0c1b33..32439d3 100644 --- a/surya/benchmark/metrics.py +++ b/surya/benchmark/metrics.py @@ -20,9 +20,11 @@ def box_area(box): return (box[2] - box[0]) * (box[3] - box[1]) -def calculate_iou(box1, box2): +def calculate_iou(box1, box2, box1_only=False): intersection = intersection_area(box1, box2) - union = box_area(box1) + box_area(box2) - intersection + union = box_area(box1) + if not box1_only: + union += box_area(box2) - intersection if union == 0: return 0 @@ -36,7 +38,7 @@ def match_boxes(preds, references): iou_matrix = np.zeros((num_actual, num_predicted)) for i, actual in enumerate(references): for j, pred in enumerate(preds): - iou_matrix[i, j] = calculate_iou(actual, pred) + iou_matrix[i, j] = calculate_iou(actual, pred, box1_only=True) sorted_indices = np.argsort(iou_matrix, axis=None)[::-1] sorted_ious = iou_matrix.flatten()[sorted_indices] @@ -55,7 +57,7 @@ def match_boxes(preds, references): unassigned_actual = set(range(num_actual)) - assigned_actual unassigned_pred = set(range(num_predicted)) - assigned_pred - matches.extend([(i, None, 0.0) for i in unassigned_actual]) + matches.extend([(i, None, -1.0) for i in unassigned_actual]) matches.extend([(None, j, 0.0) for j in unassigned_pred]) return matches diff --git a/surya/benchmark/tatr.py b/surya/benchmark/tatr.py new file mode 100644 index 0000000..6c9d9f6 --- /dev/null +++ b/surya/benchmark/tatr.py @@ -0,0 +1,117 @@ +import torch +from transformers import DetrFeatureExtractor, AutoModelForObjectDetection +from surya.settings import settings + +from PIL import Image +import numpy as np + + +class MaxResize(object): + def __init__(self, max_size=800): + self.max_size = max_size + + def __call__(self, image): + width, height = image.size + current_max_size = max(width, height) + scale = self.max_size / current_max_size + resized_image = image.resize((int(round(scale * width)), int(round(scale * height)))) + + return resized_image + + +def to_tensor(image): + # Convert PIL Image to NumPy array + np_image = np.array(image).astype(np.float32) + + # Rearrange dimensions to [C, H, W] format + np_image = np_image.transpose((2, 0, 1)) + + # Normalize to [0.0, 1.0] + np_image /= 255.0 + + return torch.from_numpy(np_image) + + +def normalize(tensor, mean, std): + for t, m, s in zip(tensor, mean, std): + t.sub_(m).div_(s) + return tensor + + +def structure_transform(image): + image = MaxResize(1000)(image) + tensor = to_tensor(image) + normalized_tensor = normalize(tensor, [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) + return normalized_tensor + + +def box_cxcywh_to_xyxy(x): + x_c, y_c, w, h = x.unbind(-1) + b = [(x_c - 0.5 * w), (y_c - 0.5 * h), (x_c + 0.5 * w), (y_c + 0.5 * h)] + return torch.stack(b, dim=1) + + +def rescale_bboxes(out_bbox, size): + width, height = size + boxes = box_cxcywh_to_xyxy(out_bbox) + boxes = boxes * torch.tensor([width, height, width, height], dtype=torch.float32) + return boxes + + +def outputs_to_objects(outputs, img_sizes, id2label): + m = outputs.logits.softmax(-1).max(-1) + batch_labels = list(m.indices.detach().cpu().numpy()) + batch_scores = list(m.values.detach().cpu().numpy()) + batch_bboxes = outputs['pred_boxes'].detach().cpu() + + batch_objects = [] + for i in range(len(img_sizes)): + pred_bboxes = [elem.tolist() for elem in rescale_bboxes(batch_bboxes[i], img_sizes[i])] + pred_scores = batch_scores[i] + pred_labels = batch_labels[i] + + objects = [] + for label, score, bbox in zip(pred_labels, pred_scores, pred_bboxes): + class_label = id2label[int(label)] + if not class_label == 'no object': + objects.append({ + 'label': class_label, + 'score': float(score), + 'bbox': [float(elem) for elem in bbox]} + ) + + rows = [] + cols = [] + for i, cell in enumerate(objects): + if cell["label"] == "table column": + cols.append(cell) + + if cell["label"] == "table row": + rows.append(cell) + batch_objects.append({ + "rows": rows, + "cols": cols + }) + + return batch_objects + + +def load_tatr(): + return AutoModelForObjectDetection.from_pretrained("microsoft/table-transformer-structure-recognition-v1.1-all").to(settings.TORCH_DEVICE_MODEL) + + +def batch_inference_tatr(model, images, batch_size): + device = model.device + rows_cols = [] + for i in range(0, len(images), batch_size): + batch_images = images[i:i + batch_size] + pixel_values = torch.stack([structure_transform(img) for img in batch_images], dim=0).to(device) + + # forward pass + with torch.no_grad(): + outputs = model(pixel_values) + + id2label = model.config.id2label + id2label[len(model.config.id2label)] = "no object" + rows_cols.extend(outputs_to_objects(outputs, [img.size for img in batch_images], id2label)) + return rows_cols \ No newline at end of file diff --git a/surya/tables.py b/surya/tables.py index 978842c..63471fa 100644 --- a/surya/tables.py +++ b/surya/tables.py @@ -149,8 +149,6 @@ def batch_table_recognition(images: List, table_cells: List[List[Dict]], model: model.text_encoder.model._setup_cache(model.config, batch_size, model.device, model.dtype) batch_predictions = [[] for _ in range(current_batch_size)] - used_cells_row = [set() for _ in range(current_batch_size)] - used_cells_col = [set() for _ in range(current_batch_size)] with torch.inference_mode(): encoder_hidden_states = model.encoder(pixel_values=batch_pixel_values).last_hidden_state @@ -191,19 +189,6 @@ def batch_table_recognition(images: List, table_cells: List[List[Dict]], model: if all_done.all(): break - for batch_idx, (box_pred, class_pred, ucr, ucc, bboxes, status) in enumerate(zip(box_preds, rowcol_preds, used_cells_row, used_cells_col, batch_list_bboxes, all_done)): - if status: - continue - class_pred = class_pred.item() - SPECIAL_TOKENS - nb = processor.resize_boxes(batch_images[batch_idx], deepcopy(bboxes)) - is_row = class_pred == 0 - new_bbox, ucr, ucc = snap_to_bboxes(box_pred.tolist(), nb, ucr, ucc, row=is_row) - new_bbox = torch.tensor(new_bbox, dtype=torch.long, device=model.device) - #box_preds[batch_idx] = new_bbox - - used_cells_row[batch_idx] = ucr - used_cells_col[batch_idx] = ucc - batch_decoder_input = torch.cat([box_preds.unsqueeze(1), rowcol_preds.unsqueeze(1).unsqueeze(1)], dim=-1) for j, (pred, status) in enumerate(zip(batch_decoder_input, all_done)): From a07fd356ceb98a82d23329e4eeb7a6f482295a26 Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Mon, 7 Oct 2024 11:07:28 -0400 Subject: [PATCH 13/17] Add highres option for PIL images --- ocr_app.py | 24 ++++++++++++++---- ocr_text.py | 4 +-- surya/benchmark/metrics.py | 5 +++- surya/input/load.py | 14 ++++++----- surya/model/table_rec/processor.py | 2 +- surya/settings.py | 3 ++- table_recognition.py | 40 ++++++++++++++++++------------ 7 files changed, 60 insertions(+), 32 deletions(-) diff --git a/ocr_app.py b/ocr_app.py index 7851a8a..be053ae 100644 --- a/ocr_app.py +++ b/ocr_app.py @@ -76,14 +76,26 @@ def order_detection(img) -> (Image.Image, OrderResult): return order_img, pred -def table_recognition(img, filepath, page_idx: int, use_pdf_boxes: bool, skip_table_detection: bool) -> (Image.Image, List[TableResult]): +def table_recognition(img, highres_image, filepath, page_idx: int, use_pdf_boxes: bool, skip_table_detection: bool) -> (Image.Image, List[TableResult]): if skip_table_detection: layout_tables = [(0, 0, img.size[0], img.size[1])] table_imgs = [img] else: _, layout_pred = layout_detection(img) layout_tables = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] - table_imgs = [img.crop(tb) for tb in layout_tables] + table_imgs = [] + width_scale = highres_image.size[0] / img.size[0] + height_scale = highres_image.size[1] / img.size[1] + for tb in layout_tables: + highres_bbox = [ + int(tb[0] * width_scale), + int(tb[1] * height_scale), + int(tb[2] * width_scale), + int(tb[3] * height_scale) + ] + table_imgs.append( + highres_image.crop(highres_bbox) + ) if use_pdf_boxes: page_text = get_page_text_lines(filepath, [page_idx], [img.size])[0] @@ -180,9 +192,11 @@ def page_count(pdf_file): page_count = page_count(in_file) page_number = st.sidebar.number_input(f"Page number out of {page_count}:", min_value=1, value=1, max_value=page_count) - pil_image = get_page_image(in_file, page_number) + pil_image = get_page_image(in_file, page_number, settings.IMAGE_DPI) + pil_image_highres = get_page_image(in_file, page_number, dpi=settings.IMAGE_DPI_HIGHRES) else: pil_image = Image.open(in_file).convert("RGB") + pil_image_highres = pil_image page_number = None text_det = st.sidebar.button("Run Text Detection") @@ -213,7 +227,7 @@ def page_count(pdf_file): # Run OCR if text_rec: - rec_img, pred = ocr(pil_image, languages) + rec_img, pred = ocr(pil_image_highres, languages) with col1: st.image(rec_img, caption="OCR Result", use_column_width=True) json_tab, text_tab = st.tabs(["JSON", "Text Lines (for debugging)"]) @@ -230,7 +244,7 @@ def page_count(pdf_file): if table_rec: - table_img, pred = table_recognition(pil_image, in_file, page_number - 1 if page_number else None, use_pdf_boxes, skip_table_detection) + table_img, pred = table_recognition(pil_image, pil_image_highres, in_file, page_number - 1 if page_number else None, use_pdf_boxes, skip_table_detection) with col1: st.image(table_img, caption="Table Recognition", use_column_width=True) st.json([p.model_dump() for p in pred], expanded=True) diff --git a/ocr_text.py b/ocr_text.py index b22ec95..cc6688d 100644 --- a/ocr_text.py +++ b/ocr_text.py @@ -30,10 +30,10 @@ def main(): args = parser.parse_args() if os.path.isdir(args.input_path): - images, names, _ = load_from_folder(args.input_path, args.max, args.start_page) + images, names, _ = load_from_folder(args.input_path, args.max, args.start_page, settings.IMAGE_DPI_HIGHRES) folder_name = os.path.basename(args.input_path) else: - images, names, _ = load_from_file(args.input_path, args.max, args.start_page) + images, names, _ = load_from_file(args.input_path, args.max, args.start_page, settings.IMAGE_DPI_HIGHRES) folder_name = os.path.basename(args.input_path).split(".")[0] if args.lang_file: diff --git a/surya/benchmark/metrics.py b/surya/benchmark/metrics.py index 32439d3..1582769 100644 --- a/surya/benchmark/metrics.py +++ b/surya/benchmark/metrics.py @@ -51,7 +51,10 @@ def match_boxes(preds, references): for idx, iou in zip(zip(actual_indices, predicted_indices), sorted_ious): i, j = idx if i not in assigned_actual and j not in assigned_pred: - matches.append((i, j, iou_matrix[i, j])) + iou_val = iou_matrix[i, j] + if iou_val > .95: # Account for rounding on box edges + iou_val = 1.0 + matches.append((i, j, iou_val)) assigned_actual.add(i) assigned_pred.add(j) diff --git a/surya/input/load.py b/surya/input/load.py index 304fde4..a065906 100644 --- a/surya/input/load.py +++ b/surya/input/load.py @@ -2,17 +2,19 @@ from surya.input.pdflines import get_page_text_lines from surya.input.processing import open_pdf, get_page_images +from surya.settings import settings import os import filetype from PIL import Image import json + def get_name_from_path(path): return os.path.basename(path).split(".")[0] -def load_pdf(pdf_path, max_pages=None, start_page=None): +def load_pdf(pdf_path, max_pages=None, start_page=None, dpi=settings.IMAGE_DPI): doc = open_pdf(pdf_path) last_page = len(doc) @@ -26,7 +28,7 @@ def load_pdf(pdf_path, max_pages=None, start_page=None): last_page = min(start_page + max_pages, last_page) page_indices = list(range(start_page, last_page)) - images = get_page_images(doc, page_indices) + images = get_page_images(doc, page_indices, dpi=dpi) text_lines = get_page_text_lines( pdf_path, page_indices, @@ -43,15 +45,15 @@ def load_image(image_path): return [image], [name], [None] -def load_from_file(input_path, max_pages=None, start_page=None): +def load_from_file(input_path, max_pages=None, start_page=None, dpi=settings.IMAGE_DPI): input_type = filetype.guess(input_path) if input_type.extension == "pdf": - return load_pdf(input_path, max_pages, start_page) + return load_pdf(input_path, max_pages, start_page, dpi=dpi) else: return load_image(input_path) -def load_from_folder(folder_path, max_pages=None, start_page=None): +def load_from_folder(folder_path, max_pages=None, start_page=None, dpi=settings.IMAGE_DPI): image_paths = [os.path.join(folder_path, image_name) for image_name in os.listdir(folder_path) if not image_name.startswith(".")] image_paths = [ip for ip in image_paths if not os.path.isdir(ip)] @@ -61,7 +63,7 @@ def load_from_folder(folder_path, max_pages=None, start_page=None): for path in image_paths: extension = filetype.guess(path) if extension and extension.extension == "pdf": - image, name, text_line = load_pdf(path, max_pages, start_page) + image, name, text_line = load_pdf(path, max_pages, start_page, dpi=dpi) images.extend(image) names.extend(name) text_lines.extend(text_line) diff --git a/surya/model/table_rec/processor.py b/surya/model/table_rec/processor.py index d1fb5d2..ed013a7 100644 --- a/surya/model/table_rec/processor.py +++ b/surya/model/table_rec/processor.py @@ -217,7 +217,7 @@ def __call__(self, *args, **kwargs): boxes[i] = boxes[i][::downsample_ratio] new_boxes = [] - max_len = max([len(b) for b in boxes]) + 1 + max_len = max([len(b) for b in boxes]) + 1 + self.extra_input_boxes box_masks = [] box_ends = [] for i in range(len(boxes)): diff --git a/surya/settings.py b/surya/settings.py index bc751c7..5ce4da7 100644 --- a/surya/settings.py +++ b/surya/settings.py @@ -10,7 +10,8 @@ class Settings(BaseSettings): # General TORCH_DEVICE: Optional[str] = None - IMAGE_DPI: int = 192 + IMAGE_DPI: int = 96 # Used for detection, layout, reading order + IMAGE_DPI_HIGHRES: int = 192 # Used for OCR, table rec IN_STREAMLIT: bool = False # Whether we're running in streamlit ENABLE_EFFICIENT_ATTENTION: bool = True # Usually keep True, but if you get CUDA errors, setting to False can help diff --git a/table_recognition.py b/table_recognition.py index e468705..566030f 100644 --- a/table_recognition.py +++ b/table_recognition.py @@ -12,9 +12,6 @@ from surya.model.detection.model import load_model as load_det_model, load_processor as load_det_processor from surya.model.table_rec.model import load_model as load_model from surya.model.table_rec.processor import load_processor -from surya.model.recognition.model import load_model as load_rec_model -from surya.model.recognition.processor import load_processor as load_rec_processor -from surya.ocr import run_ocr from surya.tables import batch_table_recognition from surya.postprocessing.heatmap import draw_bboxes_on_image from surya.settings import settings @@ -40,10 +37,12 @@ def main(): det_processor = load_det_processor() if os.path.isdir(args.input_path): - images, names, text_lines = load_from_folder(args.input_path, args.max) + images, _, _ = load_from_folder(args.input_path, args.max) + highres_images, names, text_lines = load_from_folder(args.input_path, args.max, dpi=settings.IMAGE_DPI_HIGHRES) folder_name = os.path.basename(args.input_path) else: - images, names, text_lines = load_from_file(args.input_path, args.max) + images, _, _ = load_from_file(args.input_path, args.max) + highres_images, names, text_lines = load_from_file(args.input_path, args.max, dpi=settings.IMAGE_DPI_HIGHRES) folder_name = os.path.basename(args.input_path).split(".")[0] pnums = [] @@ -58,18 +57,18 @@ def main(): line_predictions = batch_text_detection(images, det_model, det_processor) layout_predictions = batch_layout_detection(images, layout_model, layout_processor, line_predictions) - table_boxes = [] table_cells = [] table_imgs = [] table_counts = [] - for layout_pred, text_line, img in zip(layout_predictions, text_lines, images): + for layout_pred, text_line, img, highres_img in zip(layout_predictions, text_lines, images, highres_images): # The table may already be cropped if args.skip_table_detection: - table_imgs.append(img) + table_imgs.append(highres_img) table_counts.append(1) - table_boxes.append(layout_pred.image_bbox) + page_table_imgs = [highres_img] + highres_bbox = [[0, 0, highres_img.size[0], highres_img.size[1]]] else: # The bbox for the entire table bbox = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] @@ -79,26 +78,35 @@ def main(): if len(bbox) == 0: continue - table_boxes.extend(bbox) + width_scaler = highres_img.size[0] / img.size[0] + height_scaler = highres_img.size[1] / img.size[1] + page_table_imgs = [] + highres_bbox = [] + for bb in bbox: + highres_bb = [ + int(bb[0] * width_scaler), + int(bb[1] * height_scaler), + int(bb[2] * width_scaler), + int(bb[3] * height_scaler), + ] + page_table_imgs.append(highres_img.crop(highres_bb)) + highres_bbox.append(highres_bb) - page_table_imgs = [img.crop(bb) for bb in bbox] table_imgs.extend(page_table_imgs) # The text cells inside each table - if text_line is None or args.detect_boxes: + table_blocks = get_table_blocks(highres_bbox, text_line, highres_img.size) if text_line is not None else None + if text_line is None or args.detect_boxes or len(table_blocks) == 0: det_results = batch_text_detection(page_table_imgs, det_model, det_processor,) cell_bboxes = [[{"bbox": tb.bbox, "text": None} for tb in det_result.bboxes] for det_result in det_results] table_cells.extend(cell_bboxes) else: - table_cells.extend(get_table_blocks(bbox, text_line, img.size)) + table_cells.extend(table_blocks) table_preds = batch_table_recognition(table_imgs, table_cells, model, processor) result_path = os.path.join(args.results_dir, folder_name) os.makedirs(result_path, exist_ok=True) - if args.images: - pass - img_idx = 0 prev_count = 0 table_predictions = defaultdict(list) From 7240d72c4bd0778c24822311fa90ffd547f7693f Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Mon, 7 Oct 2024 13:13:25 -0400 Subject: [PATCH 14/17] Add highres image option, automatic text detection --- ocr_app.py | 37 ++++++++++++++++------------- surya/settings.py | 2 +- surya/tables.py | 56 -------------------------------------------- table_recognition.py | 2 +- 4 files changed, 23 insertions(+), 74 deletions(-) diff --git a/ocr_app.py b/ocr_app.py index be053ae..ffd4ed5 100644 --- a/ocr_app.py +++ b/ocr_app.py @@ -76,17 +76,18 @@ def order_detection(img) -> (Image.Image, OrderResult): return order_img, pred -def table_recognition(img, highres_image, filepath, page_idx: int, use_pdf_boxes: bool, skip_table_detection: bool) -> (Image.Image, List[TableResult]): +def table_recognition(img, highres_img, filepath, page_idx: int, use_pdf_boxes: bool, skip_table_detection: bool) -> (Image.Image, List[TableResult]): if skip_table_detection: - layout_tables = [(0, 0, img.size[0], img.size[1])] + layout_tables = [(0, 0, highres_img.size[0], highres_img.size[1])] table_imgs = [img] else: _, layout_pred = layout_detection(img) - layout_tables = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] + layout_tables_lowres = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] table_imgs = [] - width_scale = highres_image.size[0] / img.size[0] - height_scale = highres_image.size[1] / img.size[1] - for tb in layout_tables: + width_scale = highres_img.size[0] / img.size[0] + height_scale = highres_img.size[1] / img.size[1] + layout_tables = [] + for tb in layout_tables_lowres: highres_bbox = [ int(tb[0] * width_scale), int(tb[1] * height_scale), @@ -94,13 +95,14 @@ def table_recognition(img, highres_image, filepath, page_idx: int, use_pdf_boxes int(tb[3] * height_scale) ] table_imgs.append( - highres_image.crop(highres_bbox) + highres_img.crop(highres_bbox) ) + layout_tables.append(highres_bbox) - if use_pdf_boxes: - page_text = get_page_text_lines(filepath, [page_idx], [img.size])[0] - table_bboxes = get_table_blocks(layout_tables, page_text, img.size) - else: + page_text = get_page_text_lines(filepath, [page_idx], [highres_img.size])[0] + table_bboxes = get_table_blocks(layout_tables, page_text, highres_img.size) + + if not use_pdf_boxes or any(len(tb) == 0 for tb in table_bboxes): det_results = batch_text_detection(table_imgs, det_model, det_processor) table_bboxes = [[{"bbox": tb.bbox, "text": None} for tb in det_result.bboxes] for det_result in det_results] table_preds = batch_table_recognition(table_imgs, table_bboxes, table_model, table_processor) @@ -109,15 +111,18 @@ def table_recognition(img, highres_image, filepath, page_idx: int, use_pdf_boxes for results, table_bbox in zip(table_preds, layout_tables): adjusted_bboxes = [] labels = [] + width_scale = img.size[0] / highres_img.size[0] + height_scale = img.size[1] / highres_img.size[1] + for item in results.cells: adjusted_bboxes.append([ - item.bbox[0] + table_bbox[0], - item.bbox[1] + table_bbox[1], - item.bbox[2] + table_bbox[0], - item.bbox[3] + table_bbox[1] + (item.bbox[0] + table_bbox[0]) * width_scale, + (item.bbox[1] + table_bbox[1]) * height_scale, + (item.bbox[2] + table_bbox[0]) * width_scale, + (item.bbox[3] + table_bbox[1]) * height_scale ]) labels.append(f"{item.row_id} / {item.col_id}") - table_img = draw_bboxes_on_image(adjusted_bboxes, table_img, labels=labels, label_font_size=18) + table_img = draw_bboxes_on_image(adjusted_bboxes, table_img, labels=labels, label_font_size=12) return table_img, table_preds diff --git a/surya/settings.py b/surya/settings.py index 5ce4da7..260f669 100644 --- a/surya/settings.py +++ b/surya/settings.py @@ -73,7 +73,7 @@ def TORCH_DEVICE_MODEL(self) -> str: ORDER_BENCH_DATASET_NAME: str = "vikp/order_bench" # Table Rec - TABLE_REC_MODEL_CHECKPOINT: str = "vikp/surya_tablerec" + TABLE_REC_MODEL_CHECKPOINT: str = "vikp/table_rec_ar4" TABLE_REC_IMAGE_SIZE: Dict = {"height": 640, "width": 640} TABLE_REC_MAX_BOXES: int = 512 TABLE_REC_MAX_ROWS: int = 384 diff --git a/surya/tables.py b/surya/tables.py index 63471fa..424f25b 100644 --- a/surya/tables.py +++ b/surya/tables.py @@ -40,62 +40,6 @@ def sort_bboxes(bboxes, tolerance=1): return sorted_page_blocks -def cx_cy_to_corners(pred): - w = pred[2] / 2 - h = pred[3] / 2 - x1 = pred[0] - w - y1 = pred[1] - h - x2 = pred[0] + w - y2 = pred[1] + h - - return [x1, y1, x2, y2] - - -def corners_to_cx_cy(pred): - x = (pred[0] + pred[2]) / 2 - y = (pred[1] + pred[3]) / 2 - w = pred[2] - pred[0] - h = pred[3] - pred[1] - - return [x, y, w, h] - - -def snap_to_bboxes(rc_box, input_boxes, used_cells_row, used_cells_col, row=True, row_threshold=.2, col_threshold=.2): - sel_bboxes = [] - rc_corner_bbox = cx_cy_to_corners(rc_box) - for cell_idx, cell in enumerate(input_boxes): - intersection_pct = Bbox(bbox=cell).intersection_pct(Bbox(bbox=rc_corner_bbox)) - - if row: - if cell_idx not in used_cells_row: - if intersection_pct > row_threshold: - sel_bboxes.append(cell) - used_cells_row.add(cell_idx) - else: - if cell_idx not in used_cells_col: - if intersection_pct > col_threshold: - sel_bboxes.append(cell) - used_cells_col.add(cell_idx) - - if len(sel_bboxes) == 0: - return rc_box, used_cells_row, used_cells_col - - new_bbox = [ - min([b[0] for b in sel_bboxes]), - min([b[1] for b in sel_bboxes]), - max([b[2] for b in sel_bboxes]), - max([b[3] for b in sel_bboxes]) - ] - new_bbox = [ - max(new_bbox[0], rc_corner_bbox[0]), - max(new_bbox[1], rc_corner_bbox[1]), - min(new_bbox[2], rc_corner_bbox[2]), - min(new_bbox[3], rc_corner_bbox[3]) - ] - cx_cy_box = corners_to_cx_cy(new_bbox) - return cx_cy_box, used_cells_row, used_cells_col - - def is_rotated(rows, cols): # Determine if the table is rotated by looking at row and column width / height ratios # Rows should have a >1 ratio, cols <1 diff --git a/table_recognition.py b/table_recognition.py index 566030f..c23c18e 100644 --- a/table_recognition.py +++ b/table_recognition.py @@ -96,7 +96,7 @@ def main(): # The text cells inside each table table_blocks = get_table_blocks(highres_bbox, text_line, highres_img.size) if text_line is not None else None - if text_line is None or args.detect_boxes or len(table_blocks) == 0: + if text_line is None or args.detect_boxes or any(len(tb) == 0 for tb in table_blocks): det_results = batch_text_detection(page_table_imgs, det_model, det_processor,) cell_bboxes = [[{"bbox": tb.bbox, "text": None} for tb in det_result.bboxes] for det_result in det_results] table_cells.extend(cell_bboxes) From e4fea86b44f5157eaa25d29da5a6aec27f533d32 Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Tue, 8 Oct 2024 11:08:05 -0400 Subject: [PATCH 15/17] Fix processor inconsistency --- README.md | 13 +- benchmark/table_recognition.py | 21 +- detect_layout.py | 1 + ocr_app.py | 21 +- ocr_text.py | 10 +- poetry.lock | 217 ++++++++++----------- pyproject.toml | 2 +- static/images/benchmark_tablerec_acc.png | Bin 0 -> 25781 bytes static/images/benchmark_tablerec_speed.png | Bin 0 -> 23075 bytes static/images/table_rec.png | Bin 0 -> 2049668 bytes surya/input/load.py | 24 +-- surya/input/processing.py | 2 +- surya/model/table_rec/processor.py | 11 +- surya/ocr.py | 10 +- surya/postprocessing/heatmap.py | 8 +- surya/postprocessing/util.py | 4 + surya/settings.py | 4 +- surya/tables.py | 4 +- table_recognition.py | 14 +- 19 files changed, 188 insertions(+), 178 deletions(-) create mode 100644 static/images/benchmark_tablerec_acc.png create mode 100644 static/images/benchmark_tablerec_speed.png create mode 100644 static/images/table_rec.png diff --git a/README.md b/README.md index 93d2aa0..c44a38c 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,11 @@ It works on a range of documents (see [usage](#usage) and [benchmarks](#benchmar |:------------------------------------------------------------------:|:--------------------------------------------------------------------------:| | ![New York Times Article Layout](static/images/excerpt_layout.png) | ![New York Times Article Reading Order](static/images/excerpt_reading.jpg) | +| Table Recognition | | +|:-----------------------------------------:|:----------------:| +| ![Table Rec](static/images/table_rec.png) | | + + Surya is named for the [Hindu sun god](https://en.wikipedia.org/wiki/Surya), who has universal vision. ## Community @@ -425,7 +430,12 @@ The accuracy is computed by finding if each pair of layout boxes is in the corre ## Table Recognition -.93 penalized row iou (out of 1), and .86 penalized column iou. Took .05 seconds per image on an A10. +| Model | Row Intersection | Col Intersection | Time Per Image | +|-------------------|--------------------|--------------------|------------------| +| Surya | 0.96 | 0.92 | 0.03 | +| Table transformer | 0.72 | 0.84 | 0.02 | + +Higher is better for intersection, which the percentage of the actual row/column overlapped by the predictions. **Methodology** @@ -498,6 +508,7 @@ python benchmark/table_recognition.py - `--max` controls how many images to process for the benchmark - `--debug` will render images with detected text - `--results_dir` will let you specify a directory to save results to instead of the default one +- `--tatr` specifies whether to also run table transformer # Training diff --git a/benchmark/table_recognition.py b/benchmark/table_recognition.py index 008f43d..921aa20 100644 --- a/benchmark/table_recognition.py +++ b/benchmark/table_recognition.py @@ -3,6 +3,8 @@ import copy import json +from tabulate import tabulate + from surya.input.processing import convert_if_not_rgb from surya.model.table_rec.model import load_model from surya.model.table_rec.processor import load_processor @@ -120,13 +122,18 @@ def main(): with open(os.path.join(result_path, "results.json"), "w+") as f: json.dump(out_data, f, indent=4) - print(f"Surya mean penalized row iou is {out_data['surya']['mean_row_iou']:.2f}. Mean penalized column iou is {out_data['surya']['mean_col_iou']:.2f}.") - if args.tatr: - print(f"TATR mean penalized row iou is {out_data['tatr']['mean_row_iou']:.2f}. Mean penalized column iou is {out_data['tatr']['mean_col_iou']:.2f}.") - print(f"Surya took {surya_time / len(images):.2f} seconds per image, and {surya_time:.1f} seconds total.") - if args.tatr: - print(f"TATR took {tatr_time / len(images):.2f} seconds per image, and {tatr_time:.1f} seconds total.") - print("Mean iou is the average of the iou scores for each row or column, with penalties for too many/few predictions.") + table = [ + ["Model", "Row Intersection", "Col Intersection", "Time Per Image"], + ["Surya", f"{out_data['surya']['mean_row_iou']:.2f}", f"{out_data['surya']['mean_col_iou']:.2f}", + f"{surya_time / len(images):.2f}"], + ["Table transformer", f"{out_data['tatr']['mean_row_iou']:.2f}", f"{out_data['tatr']['mean_col_iou']:.2f}", + f"{tatr_time / len(images):.2f}"] + ] + + print(tabulate(table, headers="firstrow", tablefmt="github")) + + print("Intersection is the average of the intersection % between each actual row/column, and the predictions. With penalties for too many/few predictions.") + print("Note that table transformers is unbatched, since the example code in the repo is unbatched.") print(f"Wrote results to {result_path}") diff --git a/detect_layout.py b/detect_layout.py index 3a54f81..39cf2ef 100644 --- a/detect_layout.py +++ b/detect_layout.py @@ -1,3 +1,4 @@ +import pypdfium2 # Causes a warning if not the top import import argparse import copy import json diff --git a/ocr_app.py b/ocr_app.py index ffd4ed5..539d88c 100644 --- a/ocr_app.py +++ b/ocr_app.py @@ -23,6 +23,7 @@ from surya.schema import OCRResult, TextDetectionResult, LayoutResult, OrderResult, TableResult from surya.settings import settings from surya.tables import batch_table_recognition +from surya.postprocessing.util import rescale_bboxes, rescale_bbox @st.cache_resource() @@ -79,21 +80,14 @@ def order_detection(img) -> (Image.Image, OrderResult): def table_recognition(img, highres_img, filepath, page_idx: int, use_pdf_boxes: bool, skip_table_detection: bool) -> (Image.Image, List[TableResult]): if skip_table_detection: layout_tables = [(0, 0, highres_img.size[0], highres_img.size[1])] - table_imgs = [img] + table_imgs = [highres_img] else: _, layout_pred = layout_detection(img) layout_tables_lowres = [l.bbox for l in layout_pred.bboxes if l.label == "Table"] table_imgs = [] - width_scale = highres_img.size[0] / img.size[0] - height_scale = highres_img.size[1] / img.size[1] layout_tables = [] for tb in layout_tables_lowres: - highres_bbox = [ - int(tb[0] * width_scale), - int(tb[1] * height_scale), - int(tb[2] * width_scale), - int(tb[3] * height_scale) - ] + highres_bbox = rescale_bbox(tb, img.size, highres_img.size) table_imgs.append( highres_img.crop(highres_bbox) ) @@ -105,6 +99,7 @@ def table_recognition(img, highres_img, filepath, page_idx: int, use_pdf_boxes: if not use_pdf_boxes or any(len(tb) == 0 for tb in table_bboxes): det_results = batch_text_detection(table_imgs, det_model, det_processor) table_bboxes = [[{"bbox": tb.bbox, "text": None} for tb in det_result.bboxes] for det_result in det_results] + table_preds = batch_table_recognition(table_imgs, table_bboxes, table_model, table_processor) table_img = img.copy() @@ -122,14 +117,14 @@ def table_recognition(img, highres_img, filepath, page_idx: int, use_pdf_boxes: (item.bbox[3] + table_bbox[1]) * height_scale ]) labels.append(f"{item.row_id} / {item.col_id}") - table_img = draw_bboxes_on_image(adjusted_bboxes, table_img, labels=labels, label_font_size=12) + table_img = draw_bboxes_on_image(adjusted_bboxes, table_img, labels=labels, label_font_size=10) return table_img, table_preds # Function for OCR -def ocr(img, langs: List[str]) -> (Image.Image, OCRResult): +def ocr(img, highres_img, langs: List[str]) -> (Image.Image, OCRResult): replace_lang_with_code(langs) - img_pred = run_ocr([img], [langs], det_model, det_processor, rec_model, rec_processor)[0] + img_pred = run_ocr([img], [langs], det_model, det_processor, rec_model, rec_processor, highres_images=[highres_img])[0] bboxes = [l.bbox for l in img_pred.text_lines] text = [l.text for l in img_pred.text_lines] @@ -232,7 +227,7 @@ def page_count(pdf_file): # Run OCR if text_rec: - rec_img, pred = ocr(pil_image_highres, languages) + rec_img, pred = ocr(pil_image, pil_image_highres, languages) with col1: st.image(rec_img, caption="OCR Result", use_column_width=True) json_tab, text_tab = st.tabs(["JSON", "Text Lines (for debugging)"]) diff --git a/ocr_text.py b/ocr_text.py index cc6688d..be24f6f 100644 --- a/ocr_text.py +++ b/ocr_text.py @@ -30,10 +30,12 @@ def main(): args = parser.parse_args() if os.path.isdir(args.input_path): - images, names, _ = load_from_folder(args.input_path, args.max, args.start_page, settings.IMAGE_DPI_HIGHRES) + images, names, _ = load_from_folder(args.input_path, args.max, args.start_page) + highres_images, _, _ = load_from_folder(args.input_path, args.max, args.start_page, settings.IMAGE_DPI_HIGHRES) folder_name = os.path.basename(args.input_path) else: - images, names, _ = load_from_file(args.input_path, args.max, args.start_page, settings.IMAGE_DPI_HIGHRES) + images, names, _ = load_from_file(args.input_path, args.max, args.start_page) + highres_images, _, _ = load_from_file(args.input_path, args.max, args.start_page, settings.IMAGE_DPI_HIGHRES) folder_name = os.path.basename(args.input_path).split(".")[0] if args.lang_file: @@ -60,7 +62,7 @@ def main(): os.makedirs(result_path, exist_ok=True) start = time.time() - predictions_by_image = run_ocr(images, image_langs, det_model, det_processor, rec_model, rec_processor) + predictions_by_image = run_ocr(images, image_langs, det_model, det_processor, rec_model, rec_processor, highres_images=highres_images) if args.debug: print(f"OCR took {time.time() - start:.2f} seconds") max_chars = max([len(l.text) for p in predictions_by_image for l in p.text_lines]) @@ -70,7 +72,7 @@ def main(): for idx, (name, image, pred, langs) in enumerate(zip(names, images, predictions_by_image, image_langs)): bboxes = [l.bbox for l in pred.text_lines] pred_text = [l.text for l in pred.text_lines] - page_image = draw_text_on_image(bboxes, pred_text, image.size, langs, has_math="_math" in langs) + page_image = draw_text_on_image(bboxes, pred_text, image.size, langs, has_math="_math" in langs if langs else False) page_image.save(os.path.join(result_path, f"{name}_{idx}_text.png")) out_preds = defaultdict(list) diff --git a/poetry.lock b/poetry.lock index d0cd8ee..86b3f5c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -605,6 +605,23 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "coloredlogs" +version = "15.0.1" +description = "Colored terminal output for Python's logging module" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934"}, + {file = "coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0"}, +] + +[package.dependencies] +humanfriendly = ">=9.1" + +[package.extras] +cron = ["capturer (>=2.4)"] + [[package]] name = "comm" version = "0.2.2" @@ -803,6 +820,17 @@ files = [ {file = "filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb"}, ] +[[package]] +name = "flatbuffers" +version = "24.3.25" +description = "The FlatBuffers serialization format for Python" +optional = false +python-versions = "*" +files = [ + {file = "flatbuffers-24.3.25-py2.py3-none-any.whl", hash = "sha256:8dbdec58f935f3765e4f7f3cf635ac3a77f83568138d6a2311f524ec96364812"}, + {file = "flatbuffers-24.3.25.tar.gz", hash = "sha256:de2ec5b203f21441716617f38443e0a8ebf3d25bf0d9c0bb0ce68fa00ad546a4"}, +] + [[package]] name = "fqdn" version = "1.5.1" @@ -1148,6 +1176,20 @@ testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gr torch = ["safetensors", "torch"] typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] +[[package]] +name = "humanfriendly" +version = "10.0" +description = "Human friendly output for text interfaces using Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477"}, + {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, +] + +[package.dependencies] +pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_version >= \"3.8\""} + [[package]] name = "idna" version = "3.7" @@ -1333,17 +1375,6 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] -[[package]] -name = "joblib" -version = "1.4.2" -description = "Lightweight pipelining with Python functions" -optional = false -python-versions = ">=3.8" -files = [ - {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"}, - {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, -] - [[package]] name = "json5" version = "0.9.25" @@ -2307,6 +2338,48 @@ files = [ {file = "nvidia_nvtx_cu12-12.1.105-py3-none-win_amd64.whl", hash = "sha256:65f4d98982b31b60026e0e6de73fbdfc09d08a96f4656dd3665ca616a11e1e82"}, ] +[[package]] +name = "onnxruntime" +version = "1.19.2" +description = "ONNX Runtime is a runtime accelerator for Machine Learning models" +optional = false +python-versions = "*" +files = [ + {file = "onnxruntime-1.19.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:84fa57369c06cadd3c2a538ae2a26d76d583e7c34bdecd5769d71ca5c0fc750e"}, + {file = "onnxruntime-1.19.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bdc471a66df0c1cdef774accef69e9f2ca168c851ab5e4f2f3341512c7ef4666"}, + {file = "onnxruntime-1.19.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e3a4ce906105d99ebbe817f536d50a91ed8a4d1592553f49b3c23c4be2560ae6"}, + {file = "onnxruntime-1.19.2-cp310-cp310-win32.whl", hash = "sha256:4b3d723cc154c8ddeb9f6d0a8c0d6243774c6b5930847cc83170bfe4678fafb3"}, + {file = "onnxruntime-1.19.2-cp310-cp310-win_amd64.whl", hash = "sha256:17ed7382d2c58d4b7354fb2b301ff30b9bf308a1c7eac9546449cd122d21cae5"}, + {file = "onnxruntime-1.19.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:d863e8acdc7232d705d49e41087e10b274c42f09e259016a46f32c34e06dc4fd"}, + {file = "onnxruntime-1.19.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c1dfe4f660a71b31caa81fc298a25f9612815215a47b286236e61d540350d7b6"}, + {file = "onnxruntime-1.19.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a36511dc07c5c964b916697e42e366fa43c48cdb3d3503578d78cef30417cb84"}, + {file = "onnxruntime-1.19.2-cp311-cp311-win32.whl", hash = "sha256:50cbb8dc69d6befad4746a69760e5b00cc3ff0a59c6c3fb27f8afa20e2cab7e7"}, + {file = "onnxruntime-1.19.2-cp311-cp311-win_amd64.whl", hash = "sha256:1c3e5d415b78337fa0b1b75291e9ea9fb2a4c1f148eb5811e7212fed02cfffa8"}, + {file = "onnxruntime-1.19.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:68e7051bef9cfefcbb858d2d2646536829894d72a4130c24019219442b1dd2ed"}, + {file = "onnxruntime-1.19.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d2d366fbcc205ce68a8a3bde2185fd15c604d9645888703785b61ef174265168"}, + {file = "onnxruntime-1.19.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:477b93df4db467e9cbf34051662a4b27c18e131fa1836e05974eae0d6e4cf29b"}, + {file = "onnxruntime-1.19.2-cp312-cp312-win32.whl", hash = "sha256:9a174073dc5608fad05f7cf7f320b52e8035e73d80b0a23c80f840e5a97c0147"}, + {file = "onnxruntime-1.19.2-cp312-cp312-win_amd64.whl", hash = "sha256:190103273ea4507638ffc31d66a980594b237874b65379e273125150eb044857"}, + {file = "onnxruntime-1.19.2-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:636bc1d4cc051d40bc52e1f9da87fbb9c57d9d47164695dfb1c41646ea51ea66"}, + {file = "onnxruntime-1.19.2-cp38-cp38-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5bd8b875757ea941cbcfe01582970cc299893d1b65bd56731e326a8333f638a3"}, + {file = "onnxruntime-1.19.2-cp38-cp38-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b2046fc9560f97947bbc1acbe4c6d48585ef0f12742744307d3364b131ac5778"}, + {file = "onnxruntime-1.19.2-cp38-cp38-win32.whl", hash = "sha256:31c12840b1cde4ac1f7d27d540c44e13e34f2345cf3642762d2a3333621abb6a"}, + {file = "onnxruntime-1.19.2-cp38-cp38-win_amd64.whl", hash = "sha256:016229660adea180e9a32ce218b95f8f84860a200f0f13b50070d7d90e92956c"}, + {file = "onnxruntime-1.19.2-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:006c8d326835c017a9e9f74c9c77ebb570a71174a1e89fe078b29a557d9c3848"}, + {file = "onnxruntime-1.19.2-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df2a94179a42d530b936f154615b54748239c2908ee44f0d722cb4df10670f68"}, + {file = "onnxruntime-1.19.2-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fae4b4de45894b9ce7ae418c5484cbf0341db6813effec01bb2216091c52f7fb"}, + {file = "onnxruntime-1.19.2-cp39-cp39-win32.whl", hash = "sha256:dc5430f473e8706fff837ae01323be9dcfddd3ea471c900a91fa7c9b807ec5d3"}, + {file = "onnxruntime-1.19.2-cp39-cp39-win_amd64.whl", hash = "sha256:38475e29a95c5f6c62c2c603d69fc7d4c6ccbf4df602bd567b86ae1138881c49"}, +] + +[package.dependencies] +coloredlogs = "*" +flatbuffers = "*" +numpy = ">=1.21.6" +packaging = "*" +protobuf = "*" +sympy = "*" + [[package]] name = "opencv-python" version = "4.10.0.84" @@ -2456,20 +2529,20 @@ testing = ["docopt", "pytest"] [[package]] name = "pdftext" -version = "0.3.10" +version = "0.3.12" description = "Extract structured text from pdfs quickly" optional = false python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,!=3.8.*,>=3.9" files = [ - {file = "pdftext-0.3.10-py3-none-any.whl", hash = "sha256:99bd900d0d0692df06719c07ce10a859750ade3eb7f10c543f637118417497f9"}, - {file = "pdftext-0.3.10.tar.gz", hash = "sha256:90de726e818fb5683a0616cabb1a75a32a7224e873c3058006c93da6e440c66c"}, + {file = "pdftext-0.3.12-py3-none-any.whl", hash = "sha256:4903349d5be23984dcb417d6d37611fec9595cf65b535fc54c609f8bc702b847"}, + {file = "pdftext-0.3.12.tar.gz", hash = "sha256:a82cccff535c97f806f8f32708bf336e19d1005b7b4a8e08cc44a37522a5cd62"}, ] [package.dependencies] +onnxruntime = ">=1.19.2,<2.0.0" pydantic = ">=2.7.1,<3.0.0" pydantic-settings = ">=2.2.1,<3.0.0" pypdfium2 = ">=4.29.0,<5.0.0" -scikit-learn = ">=1.4.2,<2.0.0" [[package]] name = "pexpect" @@ -3044,6 +3117,20 @@ files = [ {file = "pypdfium2-4.30.0.tar.gz", hash = "sha256:48b5b7e5566665bc1015b9d69c1ebabe21f6aee468b509531c3c8318eeee2e16"}, ] +[[package]] +name = "pyreadline3" +version = "3.5.4" +description = "A python implementation of GNU readline." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyreadline3-3.5.4-py3-none-any.whl", hash = "sha256:eaf8e6cc3c49bcccf145fc6067ba8643d1df34d604a1ec0eccbf7a18e6d3fae6"}, + {file = "pyreadline3-3.5.4.tar.gz", hash = "sha256:8d57d53039a1c75adba8e50dd3d992b28143480816187ea5efbd5c78e6c885b7"}, +] + +[package.extras] +dev = ["build", "flake8", "mypy", "pytest", "twine"] + [[package]] name = "pytesseract" version = "0.3.10" @@ -3851,93 +3938,6 @@ tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "hypothesis (>=6.70.2)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "safetensors[numpy]", "setuptools-rust (>=1.5.2)"] torch = ["safetensors[numpy]", "torch (>=1.10)"] -[[package]] -name = "scikit-learn" -version = "1.5.2" -description = "A set of python modules for machine learning and data mining" -optional = false -python-versions = ">=3.9" -files = [ - {file = "scikit_learn-1.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:299406827fb9a4f862626d0fe6c122f5f87f8910b86fe5daa4c32dcd742139b6"}, - {file = "scikit_learn-1.5.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:2d4cad1119c77930b235579ad0dc25e65c917e756fe80cab96aa3b9428bd3fb0"}, - {file = "scikit_learn-1.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c412ccc2ad9bf3755915e3908e677b367ebc8d010acbb3f182814524f2e5540"}, - {file = "scikit_learn-1.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a686885a4b3818d9e62904d91b57fa757fc2bed3e465c8b177be652f4dd37c8"}, - {file = "scikit_learn-1.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:c15b1ca23d7c5f33cc2cb0a0d6aaacf893792271cddff0edbd6a40e8319bc113"}, - {file = "scikit_learn-1.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:03b6158efa3faaf1feea3faa884c840ebd61b6484167c711548fce208ea09445"}, - {file = "scikit_learn-1.5.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:1ff45e26928d3b4eb767a8f14a9a6efbf1cbff7c05d1fb0f95f211a89fd4f5de"}, - {file = "scikit_learn-1.5.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f763897fe92d0e903aa4847b0aec0e68cadfff77e8a0687cabd946c89d17e675"}, - {file = "scikit_learn-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8b0ccd4a902836493e026c03256e8b206656f91fbcc4fde28c57a5b752561f1"}, - {file = "scikit_learn-1.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:6c16d84a0d45e4894832b3c4d0bf73050939e21b99b01b6fd59cbb0cf39163b6"}, - {file = "scikit_learn-1.5.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f932a02c3f4956dfb981391ab24bda1dbd90fe3d628e4b42caef3e041c67707a"}, - {file = "scikit_learn-1.5.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:3b923d119d65b7bd555c73be5423bf06c0105678ce7e1f558cb4b40b0a5502b1"}, - {file = "scikit_learn-1.5.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f60021ec1574e56632be2a36b946f8143bf4e5e6af4a06d85281adc22938e0dd"}, - {file = "scikit_learn-1.5.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:394397841449853c2290a32050382edaec3da89e35b3e03d6cc966aebc6a8ae6"}, - {file = "scikit_learn-1.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:57cc1786cfd6bd118220a92ede80270132aa353647684efa385a74244a41e3b1"}, - {file = "scikit_learn-1.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:757c7d514ddb00ae249832fe87100d9c73c6ea91423802872d9e74970a0e40b9"}, - {file = "scikit_learn-1.5.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:52788f48b5d8bca5c0736c175fa6bdaab2ef00a8f536cda698db61bd89c551c1"}, - {file = "scikit_learn-1.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:643964678f4b5fbdc95cbf8aec638acc7aa70f5f79ee2cdad1eec3df4ba6ead8"}, - {file = "scikit_learn-1.5.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca64b3089a6d9b9363cd3546f8978229dcbb737aceb2c12144ee3f70f95684b7"}, - {file = "scikit_learn-1.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:3bed4909ba187aca80580fe2ef370d9180dcf18e621a27c4cf2ef10d279a7efe"}, - {file = "scikit_learn-1.5.2.tar.gz", hash = "sha256:b4237ed7b3fdd0a4882792e68ef2545d5baa50aca3bb45aa7df468138ad8f94d"}, -] - -[package.dependencies] -joblib = ">=1.2.0" -numpy = ">=1.19.5" -scipy = ">=1.6.0" -threadpoolctl = ">=3.1.0" - -[package.extras] -benchmark = ["matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "pandas (>=1.1.5)"] -build = ["cython (>=3.0.10)", "meson-python (>=0.16.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)"] -docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pydata-sphinx-theme (>=0.15.3)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)", "sphinx (>=7.3.7)", "sphinx-copybutton (>=0.5.2)", "sphinx-design (>=0.5.0)", "sphinx-design (>=0.6.0)", "sphinx-gallery (>=0.16.0)", "sphinx-prompt (>=1.4.0)", "sphinx-remove-toctrees (>=1.0.0.post1)", "sphinxcontrib-sass (>=0.3.4)", "sphinxext-opengraph (>=0.9.1)"] -examples = ["matplotlib (>=3.3.4)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)"] -install = ["joblib (>=1.2.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)", "threadpoolctl (>=3.1.0)"] -maintenance = ["conda-lock (==2.5.6)"] -tests = ["black (>=24.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.9)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.2.1)", "scikit-image (>=0.17.2)"] - -[[package]] -name = "scipy" -version = "1.13.1" -description = "Fundamental algorithms for scientific computing in Python" -optional = false -python-versions = ">=3.9" -files = [ - {file = "scipy-1.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:20335853b85e9a49ff7572ab453794298bcf0354d8068c5f6775a0eabf350aca"}, - {file = "scipy-1.13.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d605e9c23906d1994f55ace80e0125c587f96c020037ea6aa98d01b4bd2e222f"}, - {file = "scipy-1.13.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfa31f1def5c819b19ecc3a8b52d28ffdcc7ed52bb20c9a7589669dd3c250989"}, - {file = "scipy-1.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26264b282b9da0952a024ae34710c2aff7d27480ee91a2e82b7b7073c24722f"}, - {file = "scipy-1.13.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:eccfa1906eacc02de42d70ef4aecea45415f5be17e72b61bafcfd329bdc52e94"}, - {file = "scipy-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:2831f0dc9c5ea9edd6e51e6e769b655f08ec6db6e2e10f86ef39bd32eb11da54"}, - {file = "scipy-1.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:27e52b09c0d3a1d5b63e1105f24177e544a222b43611aaf5bc44d4a0979e32f9"}, - {file = "scipy-1.13.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:54f430b00f0133e2224c3ba42b805bfd0086fe488835effa33fa291561932326"}, - {file = "scipy-1.13.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e89369d27f9e7b0884ae559a3a956e77c02114cc60a6058b4e5011572eea9299"}, - {file = "scipy-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a78b4b3345f1b6f68a763c6e25c0c9a23a9fd0f39f5f3d200efe8feda560a5fa"}, - {file = "scipy-1.13.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:45484bee6d65633752c490404513b9ef02475b4284c4cfab0ef946def50b3f59"}, - {file = "scipy-1.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:5713f62f781eebd8d597eb3f88b8bf9274e79eeabf63afb4a737abc6c84ad37b"}, - {file = "scipy-1.13.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5d72782f39716b2b3509cd7c33cdc08c96f2f4d2b06d51e52fb45a19ca0c86a1"}, - {file = "scipy-1.13.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:017367484ce5498445aade74b1d5ab377acdc65e27095155e448c88497755a5d"}, - {file = "scipy-1.13.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:949ae67db5fa78a86e8fa644b9a6b07252f449dcf74247108c50e1d20d2b4627"}, - {file = "scipy-1.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de3ade0e53bc1f21358aa74ff4830235d716211d7d077e340c7349bc3542e884"}, - {file = "scipy-1.13.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2ac65fb503dad64218c228e2dc2d0a0193f7904747db43014645ae139c8fad16"}, - {file = "scipy-1.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:cdd7dacfb95fea358916410ec61bbc20440f7860333aee6d882bb8046264e949"}, - {file = "scipy-1.13.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:436bbb42a94a8aeef855d755ce5a465479c721e9d684de76bf61a62e7c2b81d5"}, - {file = "scipy-1.13.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:8335549ebbca860c52bf3d02f80784e91a004b71b059e3eea9678ba994796a24"}, - {file = "scipy-1.13.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d533654b7d221a6a97304ab63c41c96473ff04459e404b83275b60aa8f4b7004"}, - {file = "scipy-1.13.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:637e98dcf185ba7f8e663e122ebf908c4702420477ae52a04f9908707456ba4d"}, - {file = "scipy-1.13.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a014c2b3697bde71724244f63de2476925596c24285c7a637364761f8710891c"}, - {file = "scipy-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:392e4ec766654852c25ebad4f64e4e584cf19820b980bc04960bca0b0cd6eaa2"}, - {file = "scipy-1.13.1.tar.gz", hash = "sha256:095a87a0312b08dfd6a6155cbbd310a8c51800fc931b8c0b84003014b874ed3c"}, -] - -[package.dependencies] -numpy = ">=1.22.4,<2.3" - -[package.extras] -dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyle", "pydevtool", "rich-click", "ruff", "types-psutil", "typing_extensions"] -doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.12.0)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0)", "sphinx-design (>=0.4.0)"] -test = ["array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] - [[package]] name = "send2trash" version = "1.8.3" @@ -4158,17 +4158,6 @@ docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] test = ["pre-commit", "pytest (>=7.0)", "pytest-timeout"] typing = ["mypy (>=1.6,<2.0)", "traitlets (>=5.11.1)"] -[[package]] -name = "threadpoolctl" -version = "3.5.0" -description = "threadpoolctl" -optional = false -python-versions = ">=3.8" -files = [ - {file = "threadpoolctl-3.5.0-py3-none-any.whl", hash = "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"}, - {file = "threadpoolctl-3.5.0.tar.gz", hash = "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107"}, -] - [[package]] name = "tinycss2" version = "1.3.0" @@ -4945,4 +4934,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13,!=3.9.7" -content-hash = "6c8c5d3130ab1c4a3b6f83846f6cb7f7b951bbc49ee153f181f2b88beee40615" +content-hash = "58776d15cd2b20d1a735106b587c2595e18fd521dc91ef732192610e8e93d8ff" diff --git a/pyproject.toml b/pyproject.toml index f856bc0..0fef8ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ opencv-python = "^4.9.0.80" tabulate = "^0.9.0" filetype = "^1.2.0" ftfy = "^6.1.3" -pdftext = "^0.3.10" +pdftext = "^0.3.12" [tool.poetry.group.dev.dependencies] jupyter = "^1.0.0" diff --git a/static/images/benchmark_tablerec_acc.png b/static/images/benchmark_tablerec_acc.png new file mode 100644 index 0000000000000000000000000000000000000000..103b49e57f27127fd3460f47f6d4ee58ef3abc6f GIT binary patch literal 25781 zcmeIbXIPbK)+Kx_TP+o}EF%U$MG=)C3L*%osBnmq5d=grkU^3XwNxl0cnqi{k(@=z zSrHQ|8H6JsDw2ar5D@s*K2%rLd~d(=&NXw*{FrI3>TU~9c%J*Y@4eSvYwg|V_o^w( znz3XCgTa`^RNSS>U`(!OFn&_|c`AMrt8CVY|JmxeTi;R3_K2g4>G8u1RZ~YhD_ch^ z^MicOhmSj$+uCdrmK5H+f$x~3qn(44h=}!{FA%mpepG~a^(rTP$aFi!eGUx998>zw zq-eP)a|Xj~C3Dvf?Gu4r^)Bw02gk?1v@p${``tRf{I_2X-NU0*!zvR;R~$^;V_p`T zs&n~X-JFQnJvxR7RiUXnwKb$;FBl|S^empfwWh~+?0DW4{!ezo=H<^uou$*W8fsjP zT9)N*D;ddZ@GlbOlE)P=7zJ(n%dDi8LD|vZC zo{6;c-Zjd0wy20yb@iY0-sD2*EQYUn_q)so`Wx`iaI?P&>-82c`6)9OUN1UW_N_!N zWcC8ttk%|J_!c39l-8@&UW4^6saCaOEemg`jSY8*Twc6!=f;g2+l(%#Y8EKDW@cr@ z?hV~(QFgCLF7}Or)3?v6^op{wGQ6U~>RpCLoNhw-Qrny_1`|@lA6nns`fO+4wwlY(X3EZzieoo$_O z*Yr5cspF=UQ+r=V#GMa2Lft++zZ!Y2`@(|tvB#TV-q~!<5@b(1H4TsT)V0kPm5Pn- z-!6AIC>m%wD)~JJ~CjvFQK|mxbR9qfMM2Hv}TOv>e0dcah+Ye z&OW`oG^#{yeE*j(U+&0y3?H%`!Ee-}e_=UxzuPKglK11Z|JL#0jwFl9=&LsGGFB9d zR_407c#I9Tm^b(o9lRP96}8{UC`jYDw@c4px@v%g!2`4NA3l62PhQCw-*eW6VaFQi zVt0Hfx@eH=nwlWAV#SV&3xza&W-r&=--54NB58Mb#=;G2=ggTS(_g7|N7DWtSK)m_ zVOd$@q`cwS<1KH5OiWC~t?RV0=qmOBQEEY&HHk(Ge))wDZ@D3CWbp84iF2L(6+CY2 zPx@ScVWUqyJtA^)a=Z8JY5)8>Agy`B!i%{0(39Uj*9v=emq%QbcUoiAFjl2!V*cE3 z#mf-6{G6N|$F4f@`b-CfRSH+MzkDcaZflDP-0IMN`)F{o+xTkp@v%|YrgMv~uTq$O z($$q7!EkqW4{LeNjrDqc?Qu!#)~;3L;o(`g_wFevm+ttPi-gxoNg0d}m3Z+puU@^H z_L141tlSpdo#L5>3-uTosCsa?SUXaMo9%`zFhfCZNdTU=66*?&jD;op!QqQbB&^or zCQhC_8KawEP?upJR`1FT>7va#d%4tFw}DRA&$o|SeUlsi;h+zI#gCQptzNT6 zX?Uc3=v}6RGHcJ3W#(=r*Pe=g?@Dj1j8I;*K*&HFM_>jQmo`@43Cm;Fqu}V6vQ95? zgSxu-#0g&ca{??|<%Qo^SE+SjLj z`ll(CiAK^!xvu@ELyEsP7ds7h8&xHl$Q2E-({1$%y=OU%4m%kJR9WcYdG_H+$=I=( znpsgJuWoO;aOo0#z^A6B(43RsV)s2fV&1JId;HVyj|&T%ySftGhrXIuMr*vtd!`$w zqZO&5(N-QokMqL&L+dhn`(Y8=AFbbLr)!TUeuVKQUh2T%9|Uaq87dJbA2m z70z%>OR$?-dE@xr9M}F>y7xJ9KT2&*;?$&=Yb6+_tJP;YX`P#~Amw;5O&Yk*Q?=rO>9(k=9yzS&>&mSk4)sJ3D*otR>;!zn{p=&bEB=``jhc&hgf~c&-&Vs;LewYS<6kSt%|(#yUyHMnm6P zY65KY1Z{0?Jx2R$EE;k$7m66iBQPtnaL~2G_RQ*Hr>i5t>qIEsaBO-!#RI`w>g3mO zr5o!aG{W|nzrAnTUi&Z#A!G$J7F+p3&*fDi34MdjPn2(N(CBH%ZSLrZ!$R#%Fi0)` z^6}Y?O-I#y+xvXGh2eSjvb5kp2I7=XwjnAw>K#`byQb*&swS-p|l{o z-!R=K0THI7-lZ}1T-?=F3KqC4eA6AIqtz}hF4naVc3ajYRy{0{wH$f1YURrLJUof` zq7DQHM7^xwlb^Y}>Qallea=65vg@3W&-x7;RIqC+?}%G&HZP~Oi_=RoeiKTNz?}{E*O*wJcE#FS zyuBa#_48}Gx^){jVp%Rfm@@2knS0Y7d7s%j0TR{}SA*odjK@bB$F(0GQN;x*kByBf zDJt$gaU$cx$B+GAi$&Ipita@SEXQiyU9Xq;yre{=s^l;d>&x4lqGl`-4*c)~;p^DD z4ErUUj$EquwZK8I!hXH(=f~&bGFU11T{{Jb5OJWYSR_9lXCO%0MGZGsjpg8#VjMVl z@E+n$G(v~LyfrGNNH&|@zuVR4y6Mps3k@O%S|DQ3o#uOvI`*|{s0K;tWIK12BbYaT z{v0W&71@e}e$&_2x50D7nyI;c`>Ef~oS~!b@x95%wId;2J62ne`CzmL$E!ZuWdWYW z(cqJNu-@1G{r#Mtx7Zqge0*-Eb8(tw&70nD-yT*u;r`@V8;`yT zoW{dObjF8nX>>nixm4CYW5tWsy41M(2LLFd$3MR^uld~7WxZ?HE`GL+L5E76SJ}PY z*4E6I{vx)6o{{0$Gka(XlFoBV9=DhqL3 zBkuBMyW7IKb4}+9?|&;1R9#XHV8?3e?zX8(a~rggXMM>ZAMH8)%)f8zm1SGYCB4RS z>_5NaUL_?}7pqlk~E|s z=aH=^dha%U_;4`gSlMGm6%~~)qnVzgSxr4XwtcN-g*Ic@$6QE4*#p1-{xep>qH=Ba z;q|)l$Ig0t*F~0u$c1!Dd)kf;j$m)jVm<|uu(Gr)4?o}3+H zxeLr#SXEWEn6{O34vGcSR1PL`F zgFSjm{(N#O26Alv{*Ib-qbYo&^#K7wh7!xS9_Lpbp{)tjlu%Hxy(-bj&}R;B_4h9y zfi;wio;+!N|L9D6Y3LbY&vq#KajhXI9`Ou&RQ}UDhT%bMF~!!rx`tHT@RsIc&~+;NkV_ zym_fdUM~^1ZP-8bwfUitNxc3>cGX(u%?BfU+9$L;Mk z6crU?*@|`Q<)fblv&W^I+S+b+{%WgmWvDztdCAt};d#UDab}JF;cjsyLDEr3n-P+l z-$$*d&EdIjx_F&&&WQ^n4h|02`Uc%L9VlR^25;MkbF~*3lHd!K3?ACVQZiwMXl;0J@*oFzZs8dH#;0abc z_qY*YKuKIQP!DJs^7P8`I1{hY{?n&LkVSmov42P!pKM+rZeG5t_@+?B%Ny&(9b5Na zUM{VRNT`GO#p^Z#(0cOgr?TIs%+w4FDIfiI#mZ1=WbkES;i}rI#e+bv$7AK<8K?G9 zVQPl-Q+iP(FB)H>gPKFdDHFTi^7S3@EW~jm-`26w5$kW)#w=3}oWA#(W;b+=gv<=u z{NUulg9l?LbuyJ>6kgnR`uzN=JZcom>ihQxQU!r>I2tG+tzqkyE$Kep0+N#Y`Qu|F zll|?(T{a|ybY1gvJhv;&s+MPHboT7oH>)hFP?ub_uv4#72Lufq7*+0)$p_B9ckkZj z6*opTBMO-7R!axdfvk!}xfTd&KB>Mo!gsj%rVf&%xY_ex5RsYK zFEy9AH}$=d?g`IQd%;?riiGK7!{&PQ=n>V8Q)e%WJ~M4j?9OvDPz{&M^C}#{U4D8# zH+%3CGSPK@Hsc6hYVh#Lu~J?u%@GwXOP9}s?cMd+QuZIt-jT2#)@M&*DA=oS zVMyM_VGUKM)-OhZD1vi}-G;h6MVHrtTJTCT2yFd#cUG&FR^?Ag^%&d<|wm$f%&=mB7* zPyj5ZRDCh9Z|p3vs$rUyD-gF+%bOsDx{^`Wo?x{$X~&+tYvFw^_dG|34brS`6K7&L z0tm#a*K0&B0AXSQI^ji{Z zXThPPU9Wm*e}n@4L=9I?Vu;R^sb?JcZCbrtKsZ)9Rx9cXfyJVyPh(NL8Khd=$QvCn z%>DXueHu^jj0NkWfmMcE!+ah4+jS5oRW&p;1P2-b1k?cBwDk3Fx%Rb&`uY80l;y~j zcIhVE?$}noI*PaB41kISPQ#sTCm$e2N3Gm_VeN(uuLPaE&z_COeb1ORO9vP##bd-7 zC+WFOX_R?IWE*OJ4J<`fn8F;Ye5nq@$~%4i#O!LwHDvYGStgkS-&xdWrDi#I#g>-t zrEhNZ7z{e{>Q*SeXFbjf;`N=4hrHcAJg6qUfB*jco-OJfM@RTGGBS=K!MS#4wrFQL znuDcS=QZ|&NWD685BZ6iJTUhgFUz9Z8cJq=N`=~4{Fb31wB#4&c7H30#NEO@Q? za97_WXc~(nv_E4n!tq|*F3}D*1hw{n z2&+6heV${>o8`?VAts2P58Q-LvTy*7VOz%<=Vk#P93I&iu5w%K!uj*bLW`Cyi+5=_ zseogrYh@LGXN#>cvgUFbS8bp!oswV~q7o8}vUTymbnt9+?(LdR#RITf`sl4bK0GpA z;D8k11NNc5R|9|^_!d`CgO_*i+<6@dLC7eJ)|*NSo~%5C3f*}9P=L>K=gzgEDk;a} z_E#I))(S68(uuoi?EYmp1y68+t^>74sQ|&HwWD4wo5t&YU@3bgut7zXdJSp;&LKo} ziZx76T&Eq)r>CcPnODvjh20J0vi9nPxNRrDN_M%Wy7pUubgAVL{F$NtY95c!a)wxt z8!z_&-=Lh>E738}WpCn=@+>|T`-wS;X1bSqvx}lwBEWomGhT-s$P- zW#P{eza*$w`+zYB{v|}`jdItx$;JI2{->|QoF!fbGzXR#- zz*by-6C}M9B#U=IK)_WwFVBxdEg@S_ted63E4sMoa-PS?5>)hC2tfZdWya#UbLTqy zijNGiZ7*Ysus=q!8uCYYxsNP1AGcb~?J?ZPbap>CbCKE?d&@zk;bj$HC8JU4a|;=! zZKk3JX>~FFTB0t+{Lrr5yK8SUyB?~FxeZv+VtZp*ugbVNhoRhm&|DrK zo9{zI+@O>8*xB7L1G{txd}-QYKpHMv+a5Evdfj*~vaK18?vn2AZs*3qYXa&^sLuK^ zIywv0NEoXBr$~4=J~W<>j$X_8F0dMNmo2k;bw|8-F>(VzpLlQF@zRoQ-!3(wG&|wu zwiH#?W}KHp%OvdzM7atjK^c{$TI#KmlziLU*M~&)+_W-A>zO{v?{?cU8Keh`$~e8A z)5VhCu9_|q-Y-gTrX}wk7?1$|ymbG*=*Nzve9zW2mTVM2+|m-+k$5V8{_&50EXKd9 z541|6=me5kf>_RZm^d$CIJZynHk%)y2ht9M?>$6yYj1zQIDf5-Y$vGXjTaUQS!85o zt#NaI`_Vm#b@tpjZq$5UysqyU@2wUVo_Jh%>?Z~z{2}6!T<=wfBenNU&sfwXCX9{@ zZn|4DZ~nY_Vc=iPkrZOCC^HyGi2XmM(uD})JlTEuwiA1iadbfd%MN|ICSY9RP$CnB z{SaQ9H=Y23NdQ1SDSu2VS9wye$gWmsB)1J3a!ywLK?Od$vb>_NtzrQ$FS40)8JmkC zx_olXBz!>v%GqHQgHEVN5v-z-MJPE+q{ndqq=d_lnr3;*0>YeO4>dM*Kpj0ds z#cm244VOCM-i~U=>zCQNxNHICaCxi4XUUWj@Cu7r01ns-B0#vDKO{Z#43f0}aGHCw zkz1B6${<9QH~=tEI?eL_a;|;ROSag=`y0D53IA$qU(a&sNd(onZriqPsv~{<{WtjW z{+&n>Zt=sMTNzga$D{S=w<*@#o5aU&l@$vuyojrc#p8*}dIjYn6nD@u*yxp>ib7CL zFL8ev!h3m22)^}2Nr^h5(CP_&hKFhJ@UZt1MeA>$U&or{=NTD-{*kb42yB-ENsn}M zr+y|a%_)Hu^5?oY9euNwkFOw62pF6gJXEoo`;T_MP5ui0A{Wg1cYdWQ^m4vK=eiX( zqSgfES~URhv&+^0AM_XB{h#mk<(kRGRayJ+Xjk2vU>Qm)g5Ylj6F)XJt(Y^%S3rXY zFFdtbNkyd+d{KEe7oQFS*l{N(kIpW$=u)Fp3!TyLAAOyz!6fM-b5z8vq>sJ=+5&N~ z3=DaB_WbvuIKB%OFJ61G=<9Q~VomAtMT??9e$__$^2o%3muUtH1vgrbA{C5|pa%jv zE1VtzkyzLniHSzp7NEt3u}xBe6Z)h$H!xdq#yf(Q{{A(G&T$uP@HTD5s)K_MOXng-g2Q=I9FN#x;#Ug-@INqWQEKRu9$^84)?_ub+nd+ z&6qJm$w_6+`;brAl@~5w)&WoZsjba?i-?wN6m9zLPoJ%Be!e#Fi_ss?{V!ea1)c{< zb$(l?0)| zXnK4ORPKUVJlpgk=;$EN1NWj{=&VlG-*Dl|m7Dy#6%>jOCbyEn09qD@q7PYyfGO@i zo!tm%tes(Jx-UT&=o#tOW}Rj@htoqbt;Flz`)J-a*KNX{x^!|*P3!&g_e z9igG%k%4B@!pYSJ%DRCR0{YOz;qxnv=9f3Di=hB00B>DIg^q_9hOpI&gs*`xnrdFL zroXdR3(^Krs)6<=M7e=9@zcxL_rR|@*JZ_P;n9~%Q+3}9Oq z@Q5RE4K@2;@b&fQCuSX5i-wt5IR6V^U6O}z89LZ9PMvQxP&Uzi4063BI%gu9@!SJM zrAXEXJ~{^R5KE>DYJ9Ux_kN!=2V37Xe&R;Jr5I#6#_*F6N&xt`-+r?yRtb<$MWP_8 z!rk3{v%_bV<#L`s?B`5Z|9RrW%L^(hv`B;~4V9<6CGiz`#{Xh*WAaN;F7^Z9Kc@ZC zhK2^`3gpiAcbN|XFT&2A{fEbwqDAdsEUm}%OgtiRvoUxu1hw&G=W}z%;3e~}{Bi^p zIF_qm)z8zXmjSI^mv*sXmbXSX{Oo>Ru<7N68!pPjx= zH$EDvzYTtZ$Mm$$g?V_v9wA4fYNHsPE!T#vhm|y)1O>9 zXZHOhEa3cw3nLM8O>o89x5W-`MycK3S|*Qpa)Y15S(ly$9c*B#$aBBFk)GWZhBX1> zem^^9KZ~#>G-pkGwhl@Rx`T#XH|ego2V^hW_gtPf{u2W3wU7`J1Y!{~BV6b>734pK z974n~s@h!uIc-L78rS9U*40ViJl#-k2B5S2(+vA;-hW22mr z<<)RJ?HJp}JSx7hKUT}-3w*B~n>?5EfcO?fsfVPEoEld$e8ORLP(bQYi{IdMByY=pq>pDHM4J-h_>nblVFG@eO z8Lh0WE)+jSDGJ_|6%NpjP#2Bzh82#rkHhVd`gVQ2o-l+WW!4(?;CQUqOVsV~FKDCi zU9f1;QTAtCXmd|b5<+V%@Hz>t2;Nl(K9IIrD6i4JWeGP(jHj z`(GxG&)g?PMKK7A7cN{VN8YWxZz@lu)iYn-bi?l4!3HgGyFfC+M5?zp=I84s83%ok z=s`U0?X`nkYJuA&&045$4L?7>Vcvm9zY&Ga-&SyXGUZ)Lv+N&us4wd4jh1e)sRYx* z&lDCG9_}rilM3w-`i?sG4<&w^^m;H579XCRza!_B=Q%cX6j5mlQq%|U8)umZmjx?E~2&JzB5{d@mjrAHIBkIU=eAI=cU@%^lV6!_1qz^v%IrUiC zPKxw6)~lIBoRi|ZV7+cDq_Rq+rdUW^9iLuYGsySMCF&C)>KI78w5}AJdIJCg1gg>S z<(;GGxz=Z-IpC-FXO!_tNayTGSZ0U;YR~S; z)51Bo?jq`oL|5iIjeBP$Y~Dpv6hqioOnZ%T>!n31IJ$jlBQIKB-KMRJeB4ocG$hrq zp{hz}>+w&1{LL?}3DnBe1N7FQ!0)W15XTNV^V@I4rnN@+8<#8adYD<}JG5LO%HsX| z158bPvp57ErdQ)b4-hRf<3GNo4Iu(QNCW!6-##dnE& zM6yoluT&;y2eOH|UJ@uloE?&OQCNcdR+u(>S*fqw*jgyKWc64g=b3|y5QP$I6a_@j~}0Vu37AOcupdEq}G+-l+5uX)UVvkOceezyIVks(P7g6m`D=jS8X zr{HLs$USO!sD5pp4p27wGzctk>VAU(~lS-fP)op@^;d3so2;YCoI z0sk;bt-+qME7q)=~iGD^{!kJD+T+9BVt+U4P-?MJ-?z>z+_t5V<`Dc@{$F z>nji{n~31PW~Fy`kX>etS|BA;!s28-(aho6mH^3pKGFbU4S&Gk+DT<>Hrqx@LP8gW zCp-u35gQLZU8`_q8JVgo6AZVk=I4K2T&&Du%{p)-S>J52x3w$28!|`;@haF$7BIlj zcaeJIW9hy(H>QfE3ERaDhyb^sh+=CAu&Z=2K6;~cUs1#NZM12CttcduVXde2(Y zc97&0_U4?hXY=jPcInZ@9<<6$7^LvCM&;IB7-X~{kyijyhJg-`%6NTcxinD7ejK9P z5>{{Tf?!(*ug>Uje+7<>*8V7XugI7N8PIX;$I0xg&wKPBVv#wB)Oklx%&$Y_8YK!0 z42b#y<5Z1`188s~!jKZA4wZhNe=uFUQfmnFsUnn<{DH`kYWCt+_Is;0(9S({5)5>( z%W7hE;^r@18i#b$2D42yXx5;-e4h!kUcstcNSzOkmLy_jV!)I_VAP`>FSG@6q;+GS z5yDd?aa*`r%#2B_$T{iY(feo` z1p+01;RKv&eM0A`&6d}>80bR{K{ZIa%*s`(hytXN7imrlUVw_h!BW>tZX&P77iFN9 zMbZYTTeaCF@v8A}4f}%|Rx`{ON+;U8`s1OFHQ6-gCE)4Y8QX7Gu#cxNNYyg!6R`-V z6rRla^jz`#>1B$%e}I7h66U-=uhh1$x zZ!=oxcgb`ya9}A`41D-XSSJ)37IF}H!92+2_dp}JA{7(LFV9}~J{0yb?dZKLGBdUf z@BRSS8-CO;fsz7u9gZCd$H&Ip;z^p#?d`D}_C44(P__7v7s2p&6-?c`ckfUk7dUc- zQwjG!0NMV_yZyi8Gx(KV7tw;epT4-szhMExe9H$5SH1U}`~KIwtn`upD}Gp(h|^1 zi}tcGADpfQzy5lUd(#n$@i0QF*$*L(#2^ol%8!b&uk&F^EUsWq*K&%I3_h7~!&WtH zY(zjZ@Qkm2TC^KBf*Qmx#65uYy5vJXLT0>MkMEKj`yRGL+~Rt>C(K7qkchH5kAk7T z^JZV&7o?8sz?rbrWe?!Lx*GH4ik#r6xeF1p75d3l)5TeCgNeqjZ4uLqjb&j`gYUIG zCY9qs_FLE{>vv}rthoK~P_|PsiM62UNGuEvyqZ%G@<9hk!D*u z+nXEf{oy$!5+r*7H@!|&RJ6zmS~>yt7>JgvZ~*dlWDi1uq$_1+@Q=@WR9GkvZsCm_ zeG)81E9+ge>g(&NrUHk|aTOdk3uXJ7u@C2> zw#ROGn%|k#`4^3dI-<5bJ8;?zSBH~`qT%~GGoH2?{lwU29~gJ_>JwLI#hzbJr7X1< z&s~2aI^{?)m$>W1&f@-)jh#sIf1_vr{R>)bQRdA)VaE0~8y}10u4k0nJgMqS#IJuS z|Ib(6|5t?ZPgms!AuW@q74}wT7Eorz1&9*}H=FDv9Zle5LWt3W?+T{eRnWZ+SOiUH zEt4#5o5FdWpT^c@StcI+I)`7dc4Ozm2k4)W3*yBf`QSONi=3Fjx2pI<>kKk@z%-$A zcl)I50dV>qZ<7zHt>>aIJ9X(zQ*bv*$+!J>0_3LxFBzygpZ=2W-&RQ;|4XAhnq9qwC5Ot+e^*L`5`_;!S3zoewI7wSNZ%Po= zYiuZrKL#owr8#hktC{$M<2E-qFE-mg+sB3pn?jYcNbaXj1I{m>Yx4*?DwmT9;o~!3 zc0F5&iA?Fyd2ixxxt7hYN7Y0HP0G;{9;`mM_X;MkAS)cye~a$V^s?tr53j8Z9JFGu)ba9_hJgR`gxWdsUH7$n+r(Bq^ z;Yj6l&gXi6v{QG19nJdsSJ;W$AiS~o7YOgyN5yPsLp??4AUZI6(9s`~J=Hh(dSW)g z@jM54f(K4C`RnMp^ukkIFPcha;Gn6`u$O0{FJvv%@v$0V-q34fAaK0=`Rkog*Lp_zp7n`BSe-05NTENqr#z-FVmxofqJeKA z&54!Th?5`UF>t$Z3US!z2O^h0Xwx*C`r(4+Bv=RmXVyZG`P$pN7ongJ{Q^=sCY(=t zb@Ov^F_gqLg`$WZi6;5N=qX_5VCAWfczI30S6c!F8~pN7FfEY69}fk8r}kdnVJN)fkF4SK(g7J(!aY zf*>fAD8VxUI?~N^A1crJ2wJ`CZL&OS#I#iGf@nPYI+RZrI@}?Pf;EqYq#Q=pHDcy~ z68fGlwS8VyC0^tN)CGanHXslE0Cy}WzS2H?t0O8#Wfq-;1snAD3JcF}oNd#BlL>K7 z8}*fu8~zJVGvtF59$6XoUsduU1zJadGe6hC?gSWQj(zWK5=`6#Q0C+Ip7XHnruy)5 z>q4|K$~?XsVW0|^11o6PXF!fx^P>6^i_~@SPTbk*5DDx>pb2=S4LqQ-xZ@lMMUd6* ztD0mxtAd!=oUn)USXq+GvS1o>BbmhB-X8wQfcl4-LJO7e$h*K(`U%Al;9Fg4h>V*q z91DVp*T6SqW@gqJDH+2ePIdnD>C-`hs;6Cnl>p(qTzekpliz&m75^}K+H7h@0o@t} zC7=~VT3QVPb$QNmSpKO6W!<`$)o_F@Sg;@hMRsI{8X`Nqg)6cf`iT+%_o%wjZUT=6 zV=Gf@5g_rW=H~vl(!8PvPC*)9GU|wzMyX2YZs} z!)Vihy^F&p-{vgDxk$+XBoth?CEmXdU2KPXa*)xaVAe+CLL^RYG{Qb5IBUC9ib=EH z>Y`>y*q^CMG9;2*6kLdLAS_)!Uf#l!%_$)mfv=lNak39l^T;rckTzIj_O}PuHe1(K zhIaLLvGsuVRqc%(9UUK};jY_}!kmuraxN%*8lot)R(>Y=$0SM)Ca>MN@ynMuP73Cm zhg97-{<|pWrG8>Q(=@YdpkxH;a0&;;?JI?$($u&BGZGFMncAR)DY6K6gZHfbdsKOL zi8~DY$KvtX>CPojg|#9t!O7BG`GND%&!1zhDN>M`7G=mwxWcQAQ3j!eXpAYENG4a2 zOw@y*?76}=I17b@>tmS;T*ko*DP&%Secw1rR zcwmq~4QU7ycS$jl`_X%bTby&hek-;jyuqOBF+Ms-7D31YWPHXgt2cU$=PucN?7*JO zoDVfqMC?_jR;mX%@KjetP2te{8GM-7bQ7Ya+_5c zIs&4%iX?JSw1#XXmp<5`HvWO(2kkN}hl!mB5k(TLPQ1Pbi@YEx(TKA8`%{`Zf9>5y z&k3o9B`i^pMW3Y$2?V*EZk2eIX~ADA&m!OT-hwNHHFN(Chy)zd*< zfTXe3o!Ikc>wNnHyQ#GXpLnE_bGWugHucKXqtMYpeFZB}#DrYJwi#osRu0tt6McoU z8yih<-C|)ATU~5{kNhC|K3+w0KIW9lvnIE!)ou7-dR%0oL}oH#^T`rCkK1Wt6EsO;6XZjON!-5={RXE@lpCoTDdyLpN_Ewtuy%w$A!-4qj@!Pc#ZEmPeQMBA9}Iv?ds$Oc^Y@2|y}l1+ zVv&S(vi^JoE10z6AgsKJpHSzxUIPmWsNM+&7&~|NY%6vVp_nkRF-&2nxWi3;iMz=csO0Jm-1}@`L087&SJ6|sl16R(QjJSSWN34-)e#JVbY4f;tsfTk_}P-5aW zDflEr;bWky;RU(D4@AGWCds4*XJ2LgYMiGd5w|`vM(ECSeJ=z;-lk5OI@JOZD9!w0 z(N2!zf$@G^*`f8y<;&v9oI-#h7P7fUPP~8wR)P>yX{H^%L`j(UroNwaN1 z`mZxnip~_%P3@ew@>SSN0tI6SiXVPCpF$mLJ_sa`;xMef{p$p6a~(qdA#MrSE*9I6 z06KVud*~yuL>4_#IRS{hKOqJ}r;354Km0!hKBhh(;fNt8ZGZ<8iZmCW7F zxhJu8xMff)q%?scH}8G_h!qZZkU4q)N$04TIZ?EQ9hY=8+^~>1IoJLwJ!&syl0s84 zbE3Qqa|>^A6j)(30SK}-pX-KyPH^2iC5WqfFy=#+)igp;6ZGe;ZBPx(M&$=Udw+Cj z#~)U^Kx(>dD(=|Kbk^7mvR4G9OvoELQ;!L`9> z95tU)DsDgG_qnV%f23QQ9fEfWqL5@eqEf(*HVh~&x}2@Fed^86rS(9SOfR`8>6I%V z+XN#lgZ|r$UWBNH6R+8W*I1_-tY*R|QHH!l3ih96x}rm?+HpY2zxK%qzEnWfHdJFO z)v`|Q+GNIl6V0htPN_Uw;OA+-To5xE$NB-{U;T=UmHp6s^IowX@Tc-h{{{A+9O0sDA)ALGiP; zeK`>9@QQd&z_ORohr}BHS z4U;%Z1pP2nsc7P2mP5*nfU@}KqJ72&Pd@aF;s;1)#Q`uoay!W7pP{Otc&Dk%&#WTA!c9uf>SK)*@+Gj_RUwZPkmw7`zUqB)VL z3iPoQda3?OoRW^%y4h*P&q1ih<1PsiBCz-ygX)P|G!aAgH1{?)3#bL|#m3-gQd7)K zk(|4jgAfZEI~L6n4?89fXdze^_OoXSkY}&+lh6kF*N-2)_^@spzDbx^UoKi-v^P`p zN-UJcu^->m*$*=zyT+l}^=Qq+S@3SgV@)xy&_)*~Z4FpMbs>wcW=cDE?gd=ujrkpI z0D|c2M!KSoH+(}yN&}+YYLtygXL9A9oXseDOJi}Q-?iz;D?dx zK>Upw!ah5<_z3!g4jBD?Ew_B9R3BV2*9o;ooZ=ukq}L%6!^(5_67hZRz!q@J2I*{T9{9hqrguj6nUjHN#EKv+(7 zZycImu7+G{Xjlj00|Y1nEY`9;Gk%YVF2!`{?gm)hYGC1f|JSl z5|-sfjToofHhRVfL7ZEU-T)lUGzplUApt;;6|en@Kv0Rz2h4oPjmA!pAyP$!OXe*d z&r@FWQ9>d9?tx(g)mbsNBdYAH{6wA+*;Y7(3))VI``7c>BvF+2s7?em2&o1nm9$GW zko#&TcGWsmMfLDWnZTSte+IH#-XO<`0Y{LeeG%D^8LP;=vD1dUYnIg;d}gBgBY|82 za1|11h@K3x5nG*!qu_*M!TH3**q87vK0R$?AUnxI2ycSANNyi1As1;e=8EK)C@xF~ zZV#=%Ma!3u7EtHVh~fmPfh{=b7y`Bq8Be*|2K`i^@%|owX2TbV`jwR`VvwQ;_1RHL zgLBblQ)inP%l;p$y#B|NDIxX0l4s|3M= zYzu!vq=>DK==)=maVo`{Ede#CIOVTAmiq&SzawaL!q4`|Z;HNL7cX4UL`?ye>uAR$ zb?T7#JG{gQnQ*Yv5r~cJL6PB$k5o?NfA1E%)?gGYaQVeS-ny+ZhqKzZy$eg)pyScJ zh*!~^mvUHT*r8m==y{})ljKmOBDiA1R}(>urW*iMH4dIcA7xdJ~4^X@hqaL0FU8X`fgTEq#`+mEn5mv*(Z96d6D z+|8uSXV=3Tu<;=$JTt_WcehPcDgQYh{|^BD-@l+0|Epm6_j>%d>+#k1=l`h2M`m>j zgt*^>fK8KfXfO`tz=CGZnN$>MclHf+^gsTIw0cIs2byL%saqf0~^?e5J+_XahRXa&BzcrwNlK3mk#AcICW8 z4&!lZ8q5Ge(3r?D2&A1R0bd;1-0^*C6%sZLIWZt=$;u`82B@_L7Mf0D&NpwH-nJwQ zD$HH*<5T|ekGHAQ{_&4ioNx0w)LuEEi1D>k2HmJD&ePK~O%K=3n>}%6%U9Nmh9iBReCmx*}b;f{UCm!o?LdRIbgYcGrm2chOXTj@NnPLBty7;J8 zuWAN0i(x!X*IhZQYqu+Fz;PnKR@(fM627C1RP}aOLN$nssSvoqw+u0_Ptak7<*8AI z93LUt{D_FriPbKH8=3f4;2tFwjZ;FhwM177D;(UCB?iinO$s*W>@e^>`t-(oy|i$bb15#6nS! zaIaxnyyuTE&jhpuIfu#IzdO|Jk4g!ncoGqRsRx`Usene*-0z7(nOytCa-u|`sbv-z zK?MW%;GdSLoA`+TjwK2{-R&s3D=m|9BSE>FW3ml-R3p`bhR?3&6ol^|QLvl$>#r$r zsG{%bZS`lE{Ln9834K#A@!`pDN8s5)m)*8)`%$4P+Yg~&LiNv%1i!G32~o7}kglz) z<&W2#ZVtPapn?krKys70*93fPxLlYLmINrUKA2P{q$33 z-Q=Hsg4c3vehoSnRW1~VY;{;u)C|8Q4U1Z1DWZS>{VLsAe5*a%j2Q2Kw|74@>BmT~ zm#o=Z?$-Stps|LAB8+Xbyt@TV!F@T)d<1=fnQ59RXu(;PCyViYBc~-f;1RgVlG|p4 zIF~V}#iC${tFRP>B3z(((}dwHlLpBIDz}Q$3JA|{F>=pT#m<1!kLYZ&bKtT zM$Wa2p$TqMLhTd$FK6r%rv`THka@=}cj6QO&%`&h1$N!^r%_o^S=%^|zHK$q5eaSR z{M80$LbJ-C>r&ehEaZYrYF?t@DBxd|I6@ov1@rQ7rp3FAWK1$yPyScPXeuRzoR8zX z4ObpW?JAfyNy#Do5W_lXhLzQs2r7_-g~w}FKw=Olh`a>WDj4g87Kj+rGu1+kRo?N- zDK8n^=PX{?ZM6wVDDOi~JBa&vZg^VwbPH#vFnsxqdL|$&-=;dGW2p8_{cjjiD7Pt$^9bg~-KyuMSEzt)@X$KSl^>}aW z{+Er7jiFA|)vITGvgo>NpiqPX#>7OTp}iQBzdRs_g~wq9XyBGE2*qX$#6xp=END=) zZ7B7W2d)II2)7HURdZv3RinTA86==etVXx*;HQ~C@n~epiE({TKZe<4)FnpBFt#0v z9zV32z!F@My}5!M>RW_*8}2+RU}-`V=>0f)H*UJUBFdNa z=Z4isj>9&OKYEzF^)5Tpgwok!npH8*p1wr#<@zjuN(@`X7@)XcikD;%kX~ z8gdDPu{CT5pJ{m@b?~o+`^nmgkmFZUo|&3N5E}PeL86#mzHhSgv^u{Bm$0L zJ(Ff2Q3oJI8X7x7lN4x7D%-g{e<`jn0=DSq=uc!3A578(7*HLAAsyJ=AzBxvHDERm zO@E^SnwbAZBRx7TWHBnRF5Nb1+1BIFvA(3Cpcj-!zhl(jzX#+KY(A+~r4Ns)v-5q! z^T$=meXbfPsex@lA5OM(IP(Qw_Xzb${#?IZ>N_e}YK_b``JISK%x2LimkOC}VdAIf z1Qml7j$l%|Gum`<9UAA+nA{JcO_5cdWRgUxyMFRP^Tdy^;nMIF&+kpsNc-gU{ajuu zz@)vUoob;&+87-tv`3QNm#h=GSeoKP&5a5A$vbJ11)f%Cnhe@bY1|?<4e5_GH;h_( z$kRniAA&**S_spcK|aReiFRPZL^)g`ksu!9K=#lWFn+-`YxY2V(ZYZ{^n=Hncn<9b zC!s-;d{P^1ad2sj1|0(oe574WBc{6egUt{!q2WA0l8OKe2cU19=5-(L`XMwZQGZi3 zz%b}NH3_*k=k?&i@FcPOl<9WS)s+C5KN=^}*QgL&?aW(I4Z9b*+5lJ;3+!)$_==Q7+0wW&36dL@guow$!{DnsEL z2JStE$3!VL(J=jupV|IA_gc)$XoW6aMkCh_bH10K|A_>NiyvtNSl3}RA7c1MOi&7L z&zr;;-+&0Qk)In+>g4<5@NJ6Gmyyw>4g)eVv;j}qV)4_)7MlLb{v73)XO4Cp%p)-k z$Q$2(e7MgX^9B*;4cT~Vm@SAyHmmUa$Tp!Dl2{Py+aTzt5^}(-_~z>jkC8MZH;f(m^5q#XyBfpJHW4KkkPqA^ z&zie*X~01bv|T=3Wa4gGBjMV2EDWI#&*3TT8dl$jhnGNnP_ZAqPIDUl%r+QhRbp9g z!4ySvPFBH%(0$8j|NcwJ$#}Md`Gqu+@*O5CrKw==T*iiq%rGgaZEc$x?1$5{76%%O zq=C%?ei#60J=1GYvJq`Y)J`VU1KtsV`WTp6Ryc?Z8Vrcz?N31p(F-%F>e3o|kf7r5 zr`c5~hxwrvu^lm}1LrVvTH-?(1&G2xd7P$)`B~P_uJUi*TxpYwsDkS;gexhUCMQwr zA#4?BwcD@(79aHc+-G;=%dFWd?QNdR5R)Fd`L3q60*XVJ?`KUpf`){OOvyu@lZ8YkyCmT|*kEK#iUlvm0UVfvczXC`=KDWA7ofn!D&DjzuUW->(zq z8RLl+hu(pKqB9cXKH_ydCfA%taJjZdRdX3bxb2CsU~RKI3~`kRZbX(O80Q+~x(!yl z_2iN~vmUM9m;ylhO#O5X7V%m|XO{wwl{%L1@x<+=iBtx}CqvDoY{dKI)VE5zWnf2%eMX;}i#}+eR z*I{_fcm+(y37Fl(X$d4Z1+0`CYF&SAasJ^36MOh%a8S7zuSD3G)&LQmJzB!VW(vb? z3*x|6W5k{i?J1bv9uhJ|){aJlGmV?2@wUINR+3CrLk*D4bAU-z77fD%211&7)qSJf z!(os-C!Fp{Ohz8~_y}5ycJjg9kc0sD9bTl}-n`QsVi3>w-~F#!74eQ}vbUqV6 zAC7G6(mmL)F1~sM2ug{7(MTY=KmOU$YJO6gQ2|`{UY0gIZsaD3F7LYk={WG$b`U9HMre!TkfL3v=I@&z(O%91jX3>2*;l`hhyc zW_=(38iG~_S#T4}*LoT<# z*{7M$m=Pw-T!NraaS6zv1H;BD-~|!)7^2J07s5rU}GC4 z(ws7!I!nmrIN$#DT4G~uFfNe2mFATKreg>P)e1VmTQoKUZ`Z^`5}L+{%Vpb~?H{4> z&OLb}iD|ZtiO4UOh;-z#gneT*lcrvhJvd$tZj{C;kcS~sN=5q$l|jb%+EnH2{oj8i%QbTqOR*S_8)&z(Fj zh_OUC(^v_bHbC?27x`C?`+vYl0g_9xU-fWGD5#Ksk)}WMn*$8fQzrL54O*ew!jq-G zM5ZPj*AxV(K>+ls1|nBKx=q)SVZYinA4}DSR>o>#g~blPL_KJV>%Cro2#oV>c~q7( z(Uy9+$#aWBncUX9ckd?41#7H3f0s#u zob-JYutYQsn`TAe$aesE)Bo)QnnBVarj&3NUHFClwxeq|taCw%tuAWtJUn&KR74f4eTA&V!zIMd0fyFxFOjAraag zqkvD;tU@#UXg+`6o#0nxKOdMT|?axzA89UluuyMR# ze&&1U^Y#wrHr5A(4htRJ^Szm)qpgFuu&~viZxFJvHx=gDvE2zDvds2`jsu0V#+dwn zQM`1VIfbHgi+bz_b=RXx~hAz$Ly|-?t_y>ig?-BsMS>! zA)K{uKHhVB%%Z(u_EbQ+*Q}e)9!^2Mtp1OF_G!QUcI={?TW@`WTBQ&h#ai)-HN~e+ zP&eZQr=J}Dx9QLYz9+4FSF*9Oy*iWHeQ7DTt;|)?uIeb+>%aV>GSHmaIXFo7@u6f| zwP<#f1h_bxXRCX86}$-GN!cpv`?9Vsy;`bpX4sE&kM@u511;G@oLc7Lr|usze|}|| zxck_JOI9g)!`C$*Ya5r^9dQ|aEbi3HCnePu^m!4bQNH_Y%FOv32E+36`(n1{!pV=D zwj6OjX>Dz-6yN)?zrE0?;+{yImXT+yl;@OLWtf=T$GfHmG?Q^7MNBgbS*} zCEX9&zI~Q7MpaO#^;xv!LduOzqT-JJB9;wi_;pde#NCIinqz)hx2tWqT9EqF*Q;t< zYzmVdx+?EFucRD(S1QiHR?)j_WBtDS?z@o}n{47QG|kU)Wx9SobM@-go{^q9rN@IW z%EQFeu+)uX11%cqCZ`xy8#ZiEN-P!Q6w;4xNY)eBySF;yY))?Oy+h7ahtKcjp62G3 z78lD&x{Xxd+A14#>sI8h2#?yP*hSB*uHLn#JTsDykn$YCmF(WV`&C28{rgGh%R+im zOSlCbBl`OGX@W|$yxUOzxOI2f>dl8Ej|}ESAAK7VlcV_q2BqFgsdFoow~m>iD}VTWszyo z@L0#9K0feGYisAwP|6QKecdzO?w#Fdy59HE$-58n`LZ1R>Zh-*-ZD}qT*!Xd+R!k3 zr&3sau2WxWYpa2mmzPD|qZ_S5kN0v38>*i<^Kjet?I}$|CLGSn?cg59e__vk!KJzK)RckgNP zWty5HF=g$UJ5Sx~Zl9lX39Q^iIqh+$haAbTmM#_8woQh|dnR?TaNdhr+sjM}zI!*J zE#JNGhG^T9KAZKvxVJ}6eGR)09ny@7ikj+8N;zo%;lvhcFZ~0il^LU+y$Nw6Dc-Y+ z0g_{PbTchtQwryFakYY{W!>Czb+az)Muh+R)6yh`V1C_vx6CtV&jtqv$G$7NcEnK$ zdsE+>X*oLDXw=u{K9p}gkLXU&N->-&oS&8+9cnjP!6j559Uzs6=x%js_oT((1+p08 zUNbpgui6;1QN+Tm?$N1@l5QDF;Svct=_XAtHXSs(#VM#8bLYm#*Qi4-<39 zuxdGE^(IH9vo!cNFG5F2?a@ht|Ku?ifX6Ys8NRedRq9nZ9E z4EZoIZa4hlmA#V_)1gW-E-Y*_;ECwtX|jh9mNuP&na+!oZ&slIh?2A}f+i{XEr zLtrD9kpAA4MSHnDM^vI^udPO+m&X;}pc^on$MjOqT;2NW&i?b)TU`SrhJQ9r)%KsA z9^ZsFwPf3f@@`O;p3N{QEa7%9zje-^N4l+nbJHeE^(4)T!=6(a2JJXOo7&pivT|2* zOPHP5kK1ysaA5im75NgYvXGIA3QLKw81y8qP~cW@UpNhD=$BY z5H>kz*-+ka=;H9XBK&#Ha7k&Mi^1{p&usV|Q6c%Ew%-nXxY zF1dYsk*$-HNUrmMg?6&0Ug5s8FV@ri=Vqr{n(rO5vKT+Ym&mq)6E8)0&rRN&HK`0U zKarMCAD^7lVK@)?I}bEBwMmau%gW=MHq%{NbDey7wu6f4tZzVcPczM;-Vy=MbGQ?|A3`X ze{^yKj&ePHw#hUi*myk&q~wU0#j|q~ie~G5rDjLeBq!efw11^+oN`1~{=EljO{&qO5U4L&M(2l!z>= zmPZ*G2dSzuGMDGZvI`SbV-=i5F*iVtZ!QahivR2T7BM^Aaa)3TJmx6V(%FveT`5czAe7`IDZT=vXz- zo?;}e@mRZNYz{G`g1mO2Jptv_vc165tv|JdRE{R84rDo}{?{t_daFx{bAG+@1pzpX z*%i;`n}Vwfc8fg+nf$Us^5cy?UYaR}k|fkh ztEw8k=f3bH>pxYki&JiVc4>)5lBOVII;^ig;VhR&O24py*NFDfqduscak#s9q`St) zcDReu^71{lLlY6x29sgrykzI&M{F{~G1S{QXKNA+^W7}o#Rz%@?2^)<*qysySeuHl#) zIr*2Dmv5RnTXe&N$C8=o#QaS681tUV(j(VjeCFesjaPbMD8goDd`H? zJYMSmp(8-ri}|fay{!(yzLbxb_t@60Td)88GppRy9On_s%elj}V0wcIEG?Z>u$g@ji({Tcv04Kz=L6G# zfSM|tkwjavW=jh-lQfpD-u&c#Yg@W$K&z{k=Ao*pDpAkifsYmU6=|p;=8Xm3^IjW# zjYT&-br~|GhH~AzEhL=gG&eKZ*PzXCwn{c&XeR4r7nrPP42$8YD!a5@j7y#c@MO~z z3*|L3GAjP6#vmEPo zDb*9GmW|`>JOB3GyLVAI6r>EIAZd&bwvG;$30sbI*Vwzb6n-r5F7R*LyK1o$bZ+}3QnDm`{y_k|?_oi+YWuojef>h5YKNd*4``4vB zc@~922DyBQXNQmFsg@C94v+S5`dP7!2+Yr29Q`6q9~LwS|7EZy`Wo4dfj3XnpM6$( zAl~`usmFM$OKDk|K($wA_*-eC_g?RRTjn{|WSTXuqoXtR_NO)M8#g8b%1}dx+?A=+ z8YDuEAKxJ7^w!2IW_e9cPTrHaXn)BIb+@)D-E;>P<&a;C9yT{w<=t#BH``Tpe5yUa zNz2vEEfL>X`s&s0{t>I^&!2zj?M(#IXV1+Ce8llRdZ{F5i=>KpMG6BJUS})L|Z4{ZIpe} zrX-*=mASb&f_H<$!|7L7a5Yc)Zx&NUC40*4`gW1z=ik;XU$H{X**VK=ZqgL1brM&s z3cSOl8ys1+@!*NipFfjQfYv0&zO&RE$w}eFiR%8Q^wF2w14zbl8+_Wu&clF zL9SyBYC(qEXz$|CZK4-mpT=3&($>D?;pv%qzKrx;Cx2SHrr=}OTXas?EOXqlFDL8+RkPE?v4rFd$OlE@5F| z&(FnNG@+%AR5$#E$#Ag=7SK{!BQ+6;>dTIe$M;wOWhWZu=b}o-e){z3%(RrrxpQVE zH@9R}aZ+qoh?z4TDU=Iy6EB}wPJY>51YlJ8;E1XJ`hCyi@mnf%eX(fk*4!6KDFqhe z6FZCc`p?ge7rww*SS2kjO%RGJcI-y<@c>TOb9VrxfPO!hAE;XY!w)~mKeg}lM^r?g zExGvuwZ+Qw`qaxOz@QteB)<69U8}kGiGA&=Vr^44M3;Pp2$2lSFT>& z&{-BL%CGs@?8=oZ>#L;a^Xw3=dJ!`-9%tYG_KQ5usQ++h*(U6(sPo_(^ZLg+b%QFJ zz{UFjJ&h+OCf1V=04T7@n;9-+yQy;r|GkORuXyZ4deU7H%aC{P-@pIV@qA_U+wDbd zsLjzw!vyTb9d-r069bMvb9}3JN`2cP!^<-TppHV?xO{S$* z$ESBiY*RKZSry%F`HX?X=m-wFjrE)J?%LHddSBvV+S>K&&2mtuC8aJP6Url>S_Ono zcbFQIUdL_4G=06Vao%Wyp8VD29R1whUsG~K_W0WQg{}ovadr9X)tVhIZV*6a8hYkB zZd?TCSkirL&j@W?x*2d3WyZsgUtsp0h~*=cZcEfE7JYDpBus5%P2VxpSo& zA4#G3P|=F}&`K>bdw=_7U7~jCLnPnyG8UooM#k5;`^{4aa!~d4=~5A=x(j~Tr4)Ah z^yypaW#qRhhihu0kCWo5?{p+;`fFe(mD(4fvL9W35rnj^Qs~tS+^n05^3~8r9JmP*vsSeG@g4-HJl*rQg~}E>iD( zspK2F2)TS} zAiLe#OrmIS5Edux{4eKXw-CgMxrZ?aLLA^x7Em7>8uMngijkPoG$>h2kN)HvEhLH5$Aqe*T>Y`J9`SI2h|9X*4LL57w)D& zSOt4~dtPcnLc-K!R|Lnpb=#?=*b~MCXe&uGS+}<(JLSxoGb7;m*!#quODt?`&P{Gf zw{8T-)XbME=2Hh#vRG)WveA!!s6Ol+RudA_-E9`UTTMQ~bL=j!dCfyPROO#8|MZiU z=f}ziNoajF5>(~F!^7`Mdl%e9x0oJxMeq0clOy#C4ZN~&{s@gwX1l^&WmxTZS*P~- z^}fkA=!Bxqy1Kf)&UbNg(kS$r&9HqZ)A{kE1?W3B^ng|^gktFmcUQv3=%$<8$g*jt zgKfZD?%De8U9xZ-t&gJ62S0d_jMYEELY6IKC2~x9S3+EPEnz)A-nSvJs27RPs^!Iv zO+eBDlCvXqV4?1U$BYJ!W^WOD=HdNIwWQ0SKHJizWgwXV52t{0=}ppGxBf8N*BHuR zFi;jUESUU^0T4u|XQxIx-@P-|3;XfQ#|mdp(23Cnp3^{Wic(Ti*Kgcl0{!^Sz9D-W zbfIu|?EYZ>xZ!M{(R?i^Hb=n69lX3|QzJcY-BH&_OlDqaQUy5z5WyI!5N$W>f1N7F zT?lT>qA4xb(DfbLXiwd76ozc8B)x2U)`g~rDFsss?&E_@MdADyoPrwvx+~N$?p%Pr z#GIVNH+ZBCaU=+YB8_Tbpo-Z%j$MLs1ncU6IoP*sNyh#Hc-A$<0SezcV0x248mre& ze)#p*U$Hi~5zOcN>auN&W`T_rz=)Ox3ydN_EpT`rMF`{Gd}zgWb!u3xdw{p3Ex`)+ z#fBOZ?)0E(M!iC3(p8+Ak&=}le0&WAGSm^Vj zqESHfo}Rb}p)7}=CilGHv5~O7{OOZzD+re?Vep3;9IwBWd$AOvlB>Hf55YIp2M}0e}^iUu4IQ9SK06S^JPsE40Xp@ZcFgkjLzV6N1Uwg(K z@5h3+B59RGG0<9)X;rvZzH%NPNxWFNmcH(q=wfX_avEBYBm9DH-;Rz|3{57}1=YFQIrYK4>Uu+$sC<`+4a_G@Qn@{lWI`XL7|~>gspWMYLnA_x|!UR z&qG_c?m+~>?*(+Xpx`Mm!3rl&W(bBje|Wj}Y0iv|OaUspZk}uEie5X4qWVAT2RQBJMu3Y*FO7A4lye*-OIfZqOe6Ce1=D6rWoE1$4D@8F>H4I_dPL zk~67h#n(BqW5V^$qj@JPgP&iSQ&nMx!;~Zo4Mi?%%l@Q&S8mWERh$0Qb7tc6y}icI zPjNarG^7}%ppbXtibDee0@N%6g^ZjfT|fO;ac7?_w2oWZxl9q4(W9IBc3ks_Q=6%jdTNzWZle7+-O^rksbtIl+r89t9;8? z0PrpU;>C+E2;11$K-p@U2JJOE(4vK|kI-zA<8Pv|%be>wfDn7@;sQu>vwnU2Q-?0Y zFL#ayaBEbBOLlg2YzGA)32sT;vHNtC=bA0zF}RfzG?XD;aPg$t(dr94-B%%^koUB3 z5gZ&GJE-^eo%KZ_`2hMMGV*&;h{V0;@~q0`{5W3$;~9C525&rQ_Pw&QGR{lW;F@oN zW5}VzFKh*+mucOqdu|73pu)(FPGQ59wAD1l}X%2qMj4-89HaZq-Jd zGYLPBTY8DQHs0o*;x*N?3!GkukoJ>jBUfjChr3HEBcxLZ*C8gRgVsR}ipnThie7An z^Q@ugsD_ADvpRlI7q1dcf(XfpL)^sBFdUUhJ@t$nIik%ucF}?(A3uKN?B-!-j{)(m zPRlT@;(j1;Q58rm%gP%#u@n3R8aj)%yo(Dh@8=hrrojG>%uNsV^`#Uxe5sPQ5gkMh z96^^x)}S7APtx||H@ly9h2Q2y+Najk)L1mqA2|2FjzX4Rg$@t^5p^JK_8khoSM{sg zyEt$GRZ`O@fh_9MtE7^^aadGG{s0NH2Jzj0^N8{DuLzq<@Yd*u+sX62tKr-5b-L|UC_yGJw>MspPO=DdsPR$(L`K<*>#a+Y;#B4emP4^(FT-28Cg$fgGFlElKfU0k}n#8AQBtC_d^MP5^qES%PJ-FLOlqstLWVZ&aEiZMnmsy7j zOhgPKxX;5HFN(yfRQWo1_4;{z? zr@*@LkjI2=OBF!CJxCLCL-TXWHU*Q9k|hyLsILTm?w}%*G=N>@H7GDWa^Qg48i`Ln zR_@S4FHBEOPEu)vASM)5tz}C7BN7cn%HZ8CB&5{tJ)dvrJ|NO^GM`rxG%1!fHK`GO zTK;&O8d3=%n4G$+e{dh`uR#U=fK)}`IP|AFtUfz8HyyRS8(Vc6T?`=og%TsAK$QPS zmzFG5$n~I3F6Hrv#Gz%MkYwMr&>z12I!xNzuu5_~j06|r2v$UmsE&mo(}Wa~V$%eZRiXp6ez`Ln|%pr(Ua`@%;tA<}2suPA3-H+5%cl6>GYu@kO2GinZ7aPt|XjbL+@FqmXn_$d#lED%s_#x z0FiRw2DhZ{+O=y@5*mcOGl(#oz;V+;eKKb7C6eLXzh*(htN(sNmN6;2i#v!tHK9w%{Zkkot>RM5%V)~UbCZGEIpB};X2tv z!fdFjsya@5GEUY>m!BRRVD$2Q$7}xKWgvK=z&5u&K|*^UFe$GFePrdA&Jl;^QtsF> zK5=nf%ZA59kH|2u+nGP{-VfcZVk*y&+aGIfQhp~2L}LQbf*bfU6rNaIG2>{VX@sXH z(OI#42hW$Xqs|tMP0*#2aBd4`$Ml%-Dze~4wXl$c3k3`|bozJ%3O6CokzhUy4cTP> z+!8_ZIk_-=e?53L&ykwr5OZC+^90o6Bk=)AD3gg=DdAtQ-W+cleo-Eum)k3a%u6DG z#6FSep;l!PAr^V=h-3f5C)#@;YzLDb0!<#0sWO4HP_W?etP zS1m1ickkYnE9fuUaQV1rPycrmikz*Pg%&}xj~K3>9}+!u$|3O5d;8>}3n~Y;{ugp+ zVWs}Y$^7$$w;}xfIt|Z^RO~#o$~a}Wki%kY0EmI8tbzH1h{ME_aVGUJV^1UnYy~VA zXy0C;3BbEVB{*5#=`%racMow}JxJN&cJHr%fTAbKTwR{1678$6Z#}ck4w4F?UtTbo z8d#KS;ARjXL;dzXNd(W8h6uXXX`Nm0{k%~{G~#rrfh!6#SpK6prz(pCd&axa&{Iyz zKzqWU|GdCruKo{PkAItFXvqRGnE&bdisj3e1p>Mun&_aWdqCGXpM{~M>`dzG{6|U; zEJ`O?jf}Oj(wbKEOB(w6VZ==ZZG!}ZfS|uPDWN%#HKAzCg3yZ z=|o^%agT{7=t{Qs44)haIW6Kb@o7`y5x78LXMmQA2B~q?4LgCa0;X!4c2FqS-&WVt zD}`{Yha=2MOWR7$^h$+l@<8B>;C@gASj}y(fN@EFg`v$X)H7b?I|Gsp7LKAvqkK0e+EPsobaU4nwi)pOus zuZ|TWt^CfzlnkbK|ege3oj9wjFw|1>@qY({%`lpPRy+Ki` z1Kt7e;L@rK`b`Kj6X~m}2jHq39v)6}XMLirpi*I;1fU(<0$KnwL`f$uoTjEGfV)6m z@Mp?Ul29T-d4b(d0zcLPt<%AES-hg{AYAN6_zrvQ-+FxrJRWrKULu4gFtY8=h1r0o zXla6Jw1?=fnB;jo!b_WNkZ^#-SP3Rb@ZUrX<0HgpxLQPlq9u8jc zojYcFdj`;q*xOCEU-1AVngKW74LIGfTPna8nP-ZZi&($Wra8_`*m=%PcF|(0t4}k| zA_kODgRv#|LqJ7B*X7%_>o~w&?ayns_YZzyZO&U0v01U!9UN8gda>^c2aS95mFhO~ zZ*ocI#FdCY|4GUJr>5lppK9m%Ej@XVWd(K8juA~`HILL|V!8urBN+;faeeghHAr%I zH3Z1{q3loyG|5#{Q{&jMApx?5IeJs@SdAoi!n;Kx0lp;Rf{BNuQlL}Zy=zxV|AIC) zQ6^A((j|afl2{6fpbQ35YakRM1#>_jT!D>u_MN?+B+vRDg_vzu2Z6j2^`F`|4sfkq z8};+g-vGn5IvTTn9^@#On>XWjX&Qp|>d~d-wYSc1KlOH!{tP_3)e!o&-4JWhAoap% z#^3Os^?Q7;Hm`0a%_9h0Ubs||tf-^~CXUD(o5j*zulS0*FEYlyJJ6UG6cS>xqLr9c zK(>%x6}VkMLL%?4cey@dz%m6FXFTt2blqTHX}m1a$S9A=N=m0Mi%HbA=DWL)9y1Z0 zt=IH`Xe(F*S`75l1kh~7hTpR;ZAWlXlso8F=$H*q{RsF+>!(1(gaq6{?q6RzF+7Yc z+ygG54!6%l1EM#LR4`JnrrdgZE$J3)ZJ&TK=0I)a@tSydZ+6{x->DaRc|K(@1Yq?+ z>PfSLhFC|einnfkOT4I1{)r(>DeSPZ=nIel^`P6qMH4azy%@-$nN>^{L3h;n`oX$c zP!3!dpMm!iQ-6l#=HTLryZGfJF+C6+UtK*2yjV2aX3p+HGzUa90ORZS_R><4^W=^n z7eI{%g+h2@l4wH>-KdZSn2?+3XgU+XW|MSm6$UY*7oZh2RTV`G>Jep5ceK z|6uJO1PAW`gNNz{!mljKp&g#I(iiC)&gcL@5 z)B}s1Cc0W#IZ#g)eGSQ39M!R9i=_AOSH&T{I(VB#JA;CVc%(NQ(e@uW0J7~vN5|Ps zMI|LAoH;vo8ye{v7!6Hf|-_CR5jLT=c)@nYivq^Ygi*WEA@$elX-;vHQG?IjzOxIHMWP0|d2ktnWZT55qfzFx!q7`|OD^G+58LAn0?UA3Tr*D-DQm*8%T zyDt=p7$FP-h=@3NL~&^O;kwPzZVudjyq{aA0RkrRUBc$VMB-M@wl+xCPL)AWhw#EM z{$@=7^FPoFFKN#=!X+#2+|LI9HwxX|Chzl$QxUARLeXOH5o`4h2@8At;lsJ}=gz&f z$ZWQPYBzp8Ij3+x0Wy3_5^>XvhJoW+>rHB z>@ac?(TfQ^W-Twt+J>Nwt5Xca(I-QZzN@l=^;+!pqNr)2)V?qFtB7X_U_PqF@5;)A z4fEompjrYd!mIfU(uHZZC%i?)nk==1vg0mtHn9~Qw<7|k^(MbW*7^M;jV}kOb615` znkcb1i^d)2iPq-`0``64pI=dJm8mOYUdwyY-8~8QG*Ku05#;uRsHNy-4p=M{qht2S z_*_OFDnyb(x0MJZlp^iWh1Ze9rp%Ep+hMe>=>&zr3ZVtjwcSiVI@+NqS-c>`KGJ|xZtXt@n zk9u!40Q>>^(ma)ub?lB(KYnt0C!qPb8YP+uat=+Xo;wPlBXa*4K^C9yLo3H(gEL~0 z@6GAy=>sx{$rn(vk6e>=LADr$veDdTw5%DXyweaiKWngBiKEN9K6Q4IIVggiX1k>4 zYrq;E#B>zv@96Z2f#8nHcGidOpcNfxMTc4gn$4-FcE{B#SI8h!BK8km6kK~I@;6v- z3_2k?;)8Ir?1O!N_nStTnKvl8Lxv-ZdAw98-@A$H~Vm~fRk|?m4ehewr zmn!VJCYx=%9!f1O21;BaXboBnK*R$n&s^6e7UdQB6(g$4QTgtEIl!!_0*rbQxrIi2 z6_eA`T<(_^Zb==<08}yIHHch+0k3!@))O=$79b08{PjNR{lr}3oD^#G8M8M;x-M2_ zXMITQ4}mh|YP9z>_IC^p8t!3R2v%M8p5bBGf6URrHhQim#b_kJdqx&JDj4zy<|T-& z%6m>tIPcTtaIYzIBFaF$dJ2#($X&I25K%+? z9(a!#v`?`30faQ39WR_Gd^AC!q`@ZiG$x+P=^Myp`+WCn)Ybat4D)EH_IEV?%(ek5 zS?foUll_-{sl9B}G#vOFk2pVuPbC_YHLS^{Q8JTdX}7Sz=di$#hUDR4C*l_7zUa^@ z6lp{!gaf7EFX(*OF<_&J0+|LhaIOUCb|*-Y|Yu2o_O}`**{VU?5G4QDJUx5E$X4XYf+tg3-9U|2(-k;>~x1<2%#1=xdP$Tx&fCITW@ zir&|$XLC`b2*pQc#_-8Z^d;r-TnlHR=w0$v95o|jiM+b=>f*EHG?)IPDd&Y0zG2hra*k*9-nMN8wf*LJtT!kfFeD(gsJ}Y zDxw>}RYa|Y&J&5=hQlS9^$S6KtA%66##(GFfBShY$?DLH9WUNWJI{o9y8=a!M($fs z`$?p$0INtxUa>pu=nlOL>8J)SkWt_|1=?WV7bP@j%~Mx=eL4017UlwQ=P{v2zJL6C ziSP848AqHG(_1&VA2!aDdJx4@*%xvswF>}QF=USlHlDN^#9og+q7KUULo!rtYqG+@}Y^mOzKgsQ5DkS_dt zFlZ&JQz4Vz=0%%qfxpmVh%l5$B7%VB z%>GQ=2tq&xf=6gE1g#(|pc}tW7zD6fS$}=+#IQa1N{tl5+2KkdAyBnJQu8wo`Eyf! zghnf3D}QiA3#u^4e!<$-l3jwm@fx!v2h zM_11Q`ly11B`1i424pVOnyl&49mHHiN*LrO`o;c5OTH#qAgd8zXiMV47xmczs38y% zz31mpRYF%5J!vsX)rOQ}^dXg~VfZC7EJ8boX3P_yjrbo9f`DE;3ztJOg4+^!g%$$} zR{-1uIPDtPba<&`@?ZxQ2LIfTsYKgC)3dE{0S-l@GJ3(Lh}D!N3~=GxR1%S$+-EUw zD963)PixqR`RjIeDzJ#5-Y#$5wPQyyy^t3-ZIC&i=>cdCrX>xAmdBtT(uY~m75N&} z#VmHZ7rLYd8W-!{XK?folLp-U1I@Fr^dniwj8hdA1v0!rbIO)@VT?;1qr|uz1DHjt z3Bv#jw%O!xr&jAO(3ZpsMdn9VZ&^nI(smkq;{p@~j8u2(nziZ1!?uwFUD#rD#ixiN z0S6i2Xh3HC7tCowcd*g;h5VwADsW)_kH0P^O%rCY)xor46h@3O047zS_AsbhO=K|) zPgpm?8tTP#PhPTk@#bD5b+`m+F`$TzL>xgz>EaZyE(n&P+=LERmRAr(=2TF)PPAeu zfLyOXzLc@}i>ihzZ3h*jMc~)b&)BluUc_?(e@%8K8oS+?DXFP@;?S9T1~`;q!2?4- zD74o4g`}I!h42KJ8^L%c@hI}!dS^z%TVIb1geF}dw%clWnt~oYu+po@L)ic_R!5V~ zYYrevJR?MdCO89mqXwzvp!b|d%N^6ouo^UGjxhaGYoVO*8iemy^UXKaSUy6z0CUMf zc4$sLdyR}*!*`hopdsqCP(Nk@(f?$(d(|OZTZ`K&s-_s`J7Po*&72b2eN1B!$=#|S z4wje2o!%=RXJg_8vTOsgt%ynadSb znQSuvP^9xis?P8E!h$ew1NOX#;K}mbf<0+$pBodE9IaO~VXALC_ou{)YGLmzrp573 zr{I;AlCNw`wcr`MzU*!#{q4Q$_m*u5@1aBSC4>Ti3g+5kp2=8%u6N2?K#~ekn{@Nv zz$@DB@#O{qYz!+-zpD#wvS;&Mw%t;$pCUJqCo)hdr+ahs3l|_u&RV(fo|}Gt0;+`@ z4w$^L-N$-E&`!D4toE1sDeq z9RoEq*m6xY|Ljz+4yK(i>t$*gF@dei6IrO^ewL5h#M+{asQV!E_ffNBmV+Kb;(x zT_YI-)>;Nag%Qp4r&=3ImFK2l(t$qh=pZsTlgR% z!i|CQzmAz=WCT)zJZgfBPaUvaXcvwN*mrYdOa`DM#v3kdOw8XM%;{JS`&V^&^TL(a zT=(#tfYl@8LeqZmb?|Y1lMN%M;F6!@zIg(+hqX*GxJkm;U^s?(6QR` zh3|?H7(SVeadglZ8bBO+_o4Ekz{CMX>D>W7gdSIj*j9G&MDCsjxM&!(1Nn zGKZqd!jHbj=`Nfdupy(37lyD3@cR%qB3R46bs9#7cy5OyY5a+U1w+u`-g6g;F`dlz zpnyx@fnk{Muu4Q=u*<3Uw>AQptEb+N@N8 zZNLHKy2N_Gj8C8KVX<+Pb#&Y!(SJ5G!ou*L5wwN9HA+DBV&hc3zwfMkD#ssWhix}R z(+JXuQc0sUkTq+kBEtF-YZrV!BIQp$Zgo-=YngteZL4sE|JeYkiWV9gn(H#;RZ)Oc zAsq6Y^_zS>*D;5hpKmp8>F(}+UzbI`P-5i*WyS%M33UgW+CdjHHccW63(wC!89oSe zPA}RzlZJ(5M0O@YcK46MayY|Htm@rDGu6G>P00l(nZ79xl$?XwAC3KC_FN2uz3lB| zT?c!hY!uA)ZG*V;DliovkGP?fNWFay-cF~>pr!QO-{z36hI6UV7h;b8xKYx;yIZ; z!ILR0tIEsOgZVWGN+MoGS`2ZHVH^Y_6e{r5`>DI*p(+SYJS$~4rcaM6&xS1H9*s>` zq)^y@jY=?hVDNn4?B(oq0sMFIm6N7gwx{GOt|;B_sjR4Af~kd6I$3x|U!NV4X;4rQ z;4+n(+9ynfDMo9+=nmQiSPiSJ!w|?xAaWWv6POHy>`Vq*@DQJ3^QdW@dOVdNw6L(y zfPsI&MFKp5Lc!-(2MSzbl{$&Y^I(`!#R&Yy~y6&->pTa&5K}QDesv;@BX68vkRPBg^m~i^~5e z7L|Y70>~KfH8#WCESVA|5EJfSn31aJ%J3Kvro|G~q1cOc%|0!mb+Y5X+W`REiO3H` zuK_#2{{8!x5By;TfS~|f=Y90i92ousDbqZ4GD6A#YM?Jf_&6weWkp4@$d9=&yR~X# zCcYMoh!uG@+Z{JLIV3(Ue)uN;4F|~M0+2D{(Fl;oju0A=5W3ZK;IYUfwTNgeVpe_h z_AccHu-?fas7~Zclu2T?ia4**Yyxj4>*b(QVr*O{|jR6Vom0tdrDVkahJB6 z>@2$f%ID^m4*yForD=*s!u2#dW@k?n4A?IWe=LWViqx%)=A;hjm#7FFjjVxXN@QB4 z#6(YBTyR8$u(!82!x{e=zV2Nh} zWTzb?YY8Ez9#8pL(9}`LTj`RRfrg_GKD=+gwXK-bnsYjLl1gkosOyfn6q#&7oWS~= z)d;pI<>JDDJuLFCi$dCH3Wgc6xcyV;?#-btX>C!X{f}GvsE|#_lRL}L4`m5a$3k}tX-G4h7cje!hjJweH z1q03k=m0S4TTi4=Jo{qD&Yk30a!MsetA5rcicn{m%gOe$U(#K0myp6>|(iewPTF~}&(mMw!^UO+5r-+lKGOmCBw=t^1Q zl?5;-Lql@%^5n4_|IY*Mlx*R+YyW$_#$8BGZLS@wP->L%{64~~z*|M0=%*J!EN39E zh(JhuG?)+3M8qDP^;kf0YhXC@J+R^%+zc@&k%B@4QE>V4<%8#5ehd9A@#s8+XVU5; zr)Ff5iF!;rlIpaw=rkyfQOHxhtT{kp8p8}VT2ECxu?iaVKdfd{4Be^|I;8*gKBAT6 zI+^~R5aqua@SOeE=mJOQWfY%nhYw~Zr7Pxmp8VwO%!Vo6JGXlJm(vseO_MsG-aLe9 zel!&H0NAwPoU`(TOh6oFE8fu1xa@{u#|nb+>q?ec$JSloUs~0(2DF;#%s7nnIk%vV z5`f~P7a?8thbP_SV`pJ5YJkC)%?~Q>IsCd{uTPnqrzX%k>iy8>$>&7k)0CZ*63_ zzDbB;@cu3UtjJi(#wHfE5w7MA$+9L63gr{mfy{GnRyz*95xeMal&=}*I|z4364Y}A zW#w#3%+P6J$UAEKtFOM&PoEF=hHV}8aS?_T_9aWo+c16rm<9ArxkkVdbV}$On%eY| zH6MsM$b&OS?T|aHM}p+2Ikq0ef=jE&uPCr^c#!8zk=BP4kZu0aov2_^0dsM^G^ zfv4gnkw=$6?9GxQk12vUDFRO?c^V%&e-$uV#L41`F~RL*Xc_ER5SUB1k?tr_L~O$Y ymf0h+-eJmN!B)(d95xlUfIt)u{EsH{NpJr{pCd{`4&Nq+fhwnTj4t!z<^Kz4JPBO@ literal 0 HcmV?d00001 diff --git a/static/images/table_rec.png b/static/images/table_rec.png new file mode 100644 index 0000000000000000000000000000000000000000..3def4d52ab5e22428f5c652dd83ed10912c9d39f GIT binary patch literal 2049668 zcmZU)1z21?(=fb?yA-#@-KFSarNyN<#R_b3ExP#PEk#Rlx6G z_x-%@`&|Ej<~k=iNhUd&Br}tloR6>76>zaAu>b%7u9D(QEdT%w3;>|GW1u~ybmO)& z0svT*wsLZ>mE`2;Ub{M5**aJP0E!Xa+XmO$ayf*RBD;I}D~gmq2Gj2V*d<2ns?5|1|(_r&`Q{o{@`-oKC>; zk%#sW$)PPzA$)Roa9u3z>Av(J>F$=3ZA-6dx*^}yB`i{wrpO=vS zQ$>UL$swX!dDUt*L1R-+rJwIq1&xHX_0ve1FN+WcCwGnP@%)n+-LZJ7R__g8j;Z2k zACf6`ZQh@vJS+)%+;3D{e|Qw&&1RP7RQiY$H#(BFo6TxAv_afE;Jbd}=Vxt^w{iU9 zV|_~!pQI{iROK@Hw)+j?N%gahunaLOmca1u2k=T;G(s7q0vPrc9nlyJ-=bRvV**IE zp$MJ({4ppcwUe4qTI|&e2%Si@0t8d^mz#~JuqPfsbz%Pd%o>fyiPVd!62UfH4?sgU zH7%|5^L6YNAbcDJ@Ig&bs=6D%hL7ZOEu_FPkam*G8o&p`_yVIGLurB&qyRb1&@*6E z2&6VTg3w?^0`w#Ro+6M!1|=yZ(Tt`KJnFwo=au?jGEWqab7-=UAwI~rX;EvP&p zB`b0+f_|VjwB;a6gbI`x^)(h7hcc~YSYCe0x7Rvg4ov13W%49B%9^_0&~&JZu|2Sv zW6iJmt`H^2eyp1kUWvtlY(#NEf3HUoOA6%vBg$=6@F@-1BTb(%jsF zzdvO&#ZQ&}E9GQp#OZc?0WGgf4D#z%@*2f~pAy@pt0v#zU z${5Tpm|~#}oxm4VtK#M`K6xGbv}cPS%?BoCNBZfw-%9w3M}6a4Q^>5`s9dPrn^?}-s>Q;V&L)^RH&Bwu`9*_cSsSFA z`)#&3wD{dO2R*TGuCFW=Ns8?9_DTkHhqV_gd}M30gY(F>Qi$;^kv9RG43;vuC04~f zS}MAnIufP7@`dxtiw|=^Wm#pWxdHMf@~<@2zH!-!*M9Jc@=3n2Od8?p;aVzHtERMC z%sAJC)P*ObY2?e~Yp%!ikg278t*}bl5*{WUW;L|o#x6rO9IU;tGnw*vZsb@u?=W%@ zJlAU_Jr!QdUDH*oZ*w$r_zN^WK7BkxxOXtom~*O}T)bbHUTl=-tLJ49o41|YJHt6H zQFIynRUI-mVQX2V{`*5)T>z|2Conr8_8}$cyX=<^pAh*F*6^2dP3E7?Kbl9Zeu&Q+ z_sdU|KbOak#_VxlCtlC&G3lXIQ&qE26;7*3!AZ4CsZw=Q+gGDaj~@mPC#CA9E(-Ui zO%6{EU8XEo!nmoqwYWIA%8fdc)RX)MC6ij+_}q-QFt&7t8?9eeSr}NpnKn9qqg@tK zHe*z8(4J*4Boc5~*J$F_J}>&aV4k|t`(240l}+p+x_J50Qc~pocS^Eq<4!q`qp_fRYUdSW#b^&280Qc_U!|oOU`BKW{tMs zv^JPdn}*=H$#bj|tk>2@*Y`E|-!e`nHb%}JhMXK+&)Ht_)W+@nq!Y;;mFo`JXSUO#r% z4!ZS|Y`B%)x>|jo(7Kpd$4m)L2@Sz~JC}RFabd&2`QB>{Bcgw<-@E^OEy*Rq`>@4g z$?UVQ*87RqRlX+|8>_^lUnaij`?0iVJhD8es}(qh7}^-B{f7Fe`asa680pswWOUghei%a46#nbsw~R~J-Mg|Q9l-U$3cHI=;noUDK2UW?&43hamAKKmTC;NaPp3*S0p$QVbn%43h+5Ud!S2<(3 ziqYl1Px^WH{<8Cjxv`|8gnHqMNAyS9y;G9YSJ#)qO^&5^g!hb(y;*7(>7I@^rf}{C zLmO=a!z&L#>(l9Mhd_>7TR)YFq%4M_yjG(hMyN(dPNxfcMx|OAMhor!4;Nd{43%X2 zZ)sG#jM+Wd$`+Is*caJ3v3cwlEoAtNO!a938 zqAKY;;sH!?H4XK(k4Ym2c<+NUw>-qo(wW_pZxc%awf zlTLS?Pu#4^eZpH5J#cHN?I<>X%UJsZfg#Eud9r5HIo{kG9OFSCiWb&carP)o^dbB&|Q!?+hNz+*Oj=0y{{mCw8xxTt-;akVXE%&m<`5(=@`n_+{ zeKLH~DkR=Ix30S8mwA~U&826Fj4q!vdh81CU(`y_d-)v6FSZ6ck4w(^AKvGp@8U0k zhkQgYZ2mxo)`M9GG?cULq|0uG@9O1K6Ao0fHQruUf3S|d5xeINBzt`DpgR(~_TJ4| zGFyyW9NRT-U!~$DZ6k-W`-xp}JmM!xd$tSv`mbqJNH;53D4;EUSc+<2yi-3l8TO61 zq{%nW*N}eaM}BL7P&(EAV|}w-&ddK6tv4yK+n@W^>Y?n9@7QJUI>q|>i@FbzjaWGZ zVF{lBP$Oi37#;wtN{)a`{m*m)lA1rZ>a!~iaILRKBoP4jL#jb+${zr=?9@AMfU+q- zwNyUd6i`>6ynluVZmiJ2AiZ|Ab;0c-xXB**j{NxLyU7Eu|KlW*_pkQ3Kp>nG$8WwO z@b!}l8)<2vWTmDC;CPB-08oIG0Mw@#@adKUQvGjS9>@kj{wEy?0En;!p!}zd`cwG# zihH{M;`|#Te+~zrKfOJBx}iBp|5F+boP+#7aTNEbJbS8HB=F)=Y-J^@|<0q&;~+-}}Z?&eT# zCpX4_EBT*#URt_Ao~Z9`>+D4Lw_bA#XAgHt28O>4{qObfcUnSi|F0z{xBqnOse`=)5Fhv-g#TaG|26qvs0ROq`dm!- zzfu1y>;H?Y>t^XH=j`~@sJqnv4cLE>|2y+Ph+y8oeg7|7{7dM6QlBU-g$3sQ-%*po zvJm-*gto`kU$Zh7mq?A;Qt!UumC(A+WfE1ju=hd;=Z(o8t|%U(%$*IiHv%H7VE%5@NEe%spZDzwzt3UN4Ckf|A4D1tbZ-r zg@$zA@lR2^;eHWMN^{+9s>`lbITlM_`h9(Cm31^#kzTK%ar7`1`}JTdJmF$y3N`Kf z{WCha22T2;2|hT!tYEkhFp!dW&Ci{KPJ19*pEiX6*IM10Qd9|I4(kR(@d6Od}vHX=lR8-o?SMG*(?LS8*YrMqQ z#D2%2T^U(ryIBXiWv2!)>2w$e4nNUYRxbB!m@#f7TleeN)DDtM^RJYB7pj$`)p(aJ zDc`?VCvP96%R3{@qCBUG%QVqf*~JQkSgBgKpI#r{1hAjGSQ*(M0*=?hn8ub9b#=8& zOfr?e%uvKcTD?2_!`p*FJ`0I5>9N{L{?zm?er`1I@}QnrFgjK~!msYsOI0S)HRDDsSC^>cMqU7pAp;%!ywgedzp@4e^@_iCrN+N8FXIsoUu`rR&$2 z%ReWr4zNUV|AlupFi?gP1t@@CQc@D?O_>yagZ8}_6x-`j&uT`|z4-A}2~fSD1o2rl z{}O>f)Bs~&%_A&1M1ipb1Imk-POfitVCc7M(R|uN_>4AMrXbU8^tudYC@+G!kvw~@ zPPN3-9-~E zxEnoCVl0>2ncyrQQY;7@1}?KuSS4w?Sg~F?nd-h_POo=}=QgU}an5LWi#h799Hv_7 zKwlUei*pBFK`##2K~Oo0Kcm0MRJ^OQb5Xo3j&0~p`a=@5C8J0+urmwPk^jQ0e4zTb zE7%|C1pW3%W&$e>O_lE@Xs*^iR=s#&a1epC*#eII+GV8~r&aoFO*!|S>x21a@_A-5G3JNcX~YvRB^i;7}Bn?-!{? z63CAGpBuv4nPqQPOHOQ~b1l10az_<_Swi9TjcZXRno_DKUoizo+DXEahm0B;Szo%T ztn(C1FK{+t*u!$0ONX$vQ{e_9)^~ZxHJ@IWb<=&mbXZ%Sa?5n(FS!4%U9>UY72&$G znY=lcE4ReJ9Nw5#x8jMpZ~LvN>}bV4Q2XNn(Pa4yNs1P-?93ONhE9P{E+#;a*h1(N zr}!?F4$tCAJ0+}dFjZdswF?57ihWTgB-g9LkdXA1 z(!ulNbeUfsHu5UXbo)8wtpHz}X>4+kmIw0e4=_hGZwamZ2l%WhSIm;XaO#)$(LP=& zH2Rzu-yK*>Cucw2ku}|9g8)3G(;4i6vJ`IJDnL8 zD>j!O{wz<)6uWs!9c>?sM_6Ogat;KJaJ<=D)PH?@^}fEwv#Cu-djALv(E1KZV{!C@KlWfNwPeD{v6APuO@Lm1MM$F8X|9XON(m#M!b`n zM*2L?vf&s$LOqP?CglFyhhg)%GP-jZ}ZMM_Bw%d@rxn1XZ&iqx!u-Z! zUwAFq!fnmQZB~l113oW5Y}LC`nu4+eZoVH{^2Izuyk>MnP-on;j1vqw(nohxFLU(1 z@3)1gk=o3Ey-qiXw%4JpaRi zL~)5L&_wf?X;w5_Xf~MKp4fYeGO|$HPCiaa8etx8pw_L9CGzV~oFS{xlRu+dHw+S0 zwqGgc(rnY-O{TdCij4@jE50d}PxY+oKoKL}L9~LP!;go%jC3&@Y5N7b`JYKf*!8(n zXEJ~++N_6BlW^B589g}TAu?!Clf8WS@5(;S1ENER!qd70``NPG#F~Z96{h0XwZ42g z%=lm0%f#Ek7}Jo!ku`1^27i&RZD0WOu1i%0eB2lJ0>)(RzHEv-XxC$iS9EKZbY$}u zZCe#Xx`d7u5f({eMGl1iJIFN5L(+xW_xW$Gqbj9;7YW)(+t=1#H_0sw*ct5~R!_oD zreVn7BbAw!1KmR;`6PmZcUIcuCAPPI!sL@leUVkiachbyQKQ||pvwp*8QiVXc=8tlc23CD*O?{R z-a=*{%#%E$ns^zU6-#vJElbx9^Z$O($uHPFM(W;sWHl1b75-r>t|KY0GA~Ml0?HH; zcvpY{xnx0>f6G9owU#yRqE7EZOA&|TTM^e3x3GSQ^5_8nf+#BZAmJ6=da026cLB_0 zZHJ=$>cZdi`7j=fNyrvzfb6k76LDF5hH9t1#t7KUz< z5CakMKPY;(6p^|_LuSQgeVTE2H?>DBR;9$BBr%r;ZYV9y6cR11Se*q5_^X3Fv22WR zlZrfY-P+d%epNy*)kj=0l94?c{s>saF1s6-1B0dVd>PL{k(|14edorH&6HznBWeLR zSZ|cBcsow#ako2HTa6K;EhTntD~Aaf zO^W@A>Ugn)iVUfZo#FNitH$1McPU$%Ijm4)2GvZ>lXUxVUE=7*q`F%=M z#Y~^GHS$d`@nf0bF-%~wU(U<)RGvql_rm4b*||yBQFmlV@qK*WImlzVF#+vA6_4|% zK2fUTiBemuQWAeDrSg|jI-SI^)cA-i+|Nnb>KcY^a4~QV`vWccD~~v#+$#Z4{9siz zwypaWTI$*sTR~)@L62TS*^B)a5!#=aZZSlB_YUe@ueyg?)ZB2Op}vFrIHz<0YO7N@UFCZc* z#40`Y#t|||ymtdk^s;ANuFF7;|57Nqt19A^$gj{3Y?3mP1|Fvg?$&Vlafq z&^W&=(ul{qYG9Q_O>G$&c#j?K{Tj$AFmsph-=_eQ3t{y4#qfO6iaN6IUzlER_DGzxln%YpyXU;e$su|v*{5oNB1IaNTmg;R;D zqgFGU9(1Cm&$ME5z@9GeA&vu1as$6UlSTa9^t3 z3#5d7RD>noeGJ8L^;!^rZyfXZ;O6aj5hXM|rHFh%mIDPMEU0;_FiUr=Me{}Vu9FiJ zl{`EWbepLcLPSZbtbyTo||n1o^I(zgJl+}NOP|B2BrC&e|LG4%*bB0?!Kl;2tS8Ow*8 z_RS*r`Up&MK*H0twOYFRtc8|ij-dfai)zf;2B0y8T}6Hl<8_Jm77nUe9g{`-5B_3T zp914&V(*pHL>#|7+ku&anBVvP6ozI*p*--vm{k;SHlVjWIC&BePfW9F8ZJaj>(`Sh z;j12SJHOobJ>eAwl&>PGT)@nfo3|)wYc7%!BgwHyqk8p$JX`osBO#1Y!#0O0=1piY z&DqsYYahRp4{oMeya7GsMpx)|BSlx-Q>;?hN#rX(UKbCwFRiuGoH%z?8_s^C!RLUw&H-q(X=S;rXX)sJ2~p55La$ zH^riOS`CcFBa`ReQ#P3+%3n%F7?Q8+SG-{^(X03jwXFSFQrxQh*g;)$wVf&dBJ=rr zp*~srD)jM}W25hV+Rx*vHqSqXwuZIJ#e4gO&wS~r&eN$W5;v|4%y8J)6{K*2(NW^z z?i=0MyZ$$-H_grO+}(A00HQ(_M!Y2=PH_W!dpmZva7-Mmai&Qr6-j^t#o>N3cHpgZ zq>s?#M{C?~ye(V^+kA(Ol37$X11%{M zt@A5>oqZ(w_?vS?A2Nasb{REY>|G1+F{6Ls>R?R9#I(B`WbO8AI7!fGo+i4e&EaNM zH$T5nIpG|wif6-4BG3TK!DSUq&Ik#?#yq-APYa4?fVw^uh-=xuQ zk9Fl=kytsQGVY$VkwlS3-r0zAPrca^D#SNQ>q7gX*KDPId#A(~5$#r*`LDXC`tKrn z^^04b`mgA3c}{=^&H^H4QQ#0sKrB6sqKiaVKu?ngqH2ii9HIrF-Q@F``8BO*vWR61 zpnKhJ7p*g+qk_7#=7*Wpo9euTJN)W{f>`v55Qm9apy|k?yP@*aSlCXoyqTF`z#T3r zEVs4f!AqUF+ZFAI3d(Y#I6TtVAouChsrZ)sI~?ufq%ZP3Vk{0Ec%$PZAwhzm+xgK{ ztNY7wfk}7FzQ;=A+!_+;e5Hww@m~0>#V985AuQu6J23A4D3Gqg&vgnJyt1pMCC`hG z1nN7^z0cs0xv9tL|FjnRqgU^ z__Y0sZ?k{y&@0k~o_D1Tu|#xi)!cNt%I|(ab!cLvycOR~CQ_5SVfRdP@b5X*cAUEb z!4$NBxFg6GTJuv(=k~u8nZa6;M}PGqS)3KZHrqbYAwLfDq+cHBP=ou%7H3oy!tT5A zyKuQoOLgtDu|hEiT>Dyk;o8w`6t`K?-rxU-hnzm46-l=zwf5bj7RuUK&XyiNnXG~+ z;nyh{as)IqP7jx_M?^?iajr>^-Ptw~Gt^TVFht(V=uEtX&dA6pASy~H#~c1`J|=V4 z3%(AC7AiAl*Tn$VlGiK2Xc7=MV!k&u8Q=%TrD;f&_o>xf!~1-pirgZF)0*_KUf3Dn zpuz3*E7*G>7xr2V{LWC=Ax;lbR$8i)HD&#Hx-9_c`bDmH7Slmh@ZFna&xB?yl#}&n z&K*J5&ScS@Ocy#}rjZ;jB5}UCU6l;$?NuoJ?w{}z0}41`n9Y@tGYnmFO&XwzKz`Nf zm2rD0^tyl&Ud;(fUz7bJvJ&3`jnP1e;Z({Yr~_P9GmIa%sCFCujg*0&KUY`zKSs1| zO?SHKM~~!gS8!(*ckck-4)yxkP7I{KpqEmKmY>fL;iSTeo>RuI%_L?aW|H0~ebt4n zeT+F-`2nj7d+F<2M=3B(F)uA;w#kT`u#{=yQfo2_g1Ztj_vcfb>y8-F9lGaO&88okwL$7T@f z;>HvN+5cxVTInYQx8}A0W=pVU`(LDHNC(Qs&`7W-#DWqif3_E`jvD=G@e^7z+_QyK z3@&6=8VEftyu)lUR7PJ^Ft%NR?To4I?^Xq6f)U%QDrI>f77o(JPKbG1e!g5*C#U(* zQfpH^oXFyQZ~ErIh|94kXxC-Y0xH>F0>Gt_{N0OxBhnWq+ z;jk7NQ0)6qesjs_=;(>i#Duf2tMUN_j2fo2=xl5H6;%_b;S&2>;|}tANzec2xL6|r z@Bw%MFB#2h7jWiN#Zku3!Pm%jyCK8bYqC0DU=hn`&Z_EtPkZ4cqPKUZStgB4wd8>%z$8}0 zP$n$bn|k!T(m0$sFa2AvrUnrlm=E&t=H=3dFOuW+C;R z{DPgtDup)$(l#2fr`@X$(}qAy*#^knhp%NQdj7a_J<7-uTe+{!G8>{SgR8a|Sp+0~~XRKMZGP4A(p zcp6gk@3w-60eYvF4k%2>$)AY@Vp2072fZSD5$p7&zPVp1X>(qD@Z zt(&$7&@>%CYpbU-*5&j>F3^F;y{d#j$zc&b&?vcz;l1aO78$l}zAIc@TxEUzj2rS?;pDluytL5CG4hQ)Gp_6%JAAu_2@znlT-F%(a(KZ-mc2I;+XMPb7t|!Oc zskL(Varse-2mvtA4z7EKFgtN|RymIkWQdh>58Tiki{}!+ot4ZySMUF16aReSqtO(l z0S26Lc^)jLjvq_BGXA&|U-M^C?E7M_VsoU-jevW3GvM3+votRJ_6r0yB(G;bhllP4W3Yv-Z;)yQ(-ggpenIBo_Se-m7BQcajq3i^2x2<^6@^6B)L=q$7iD{>GSiU`DP*(832NDPMiC@xJPT=t6Q_z?E{yjf6nNQe)`63D@)~3^SpvGxY+}74Ho`Z?onhH}L ze6$~`FDA=* zh`VF|tNZ178N%v2Gvf4mn;xEzOJ9bw0+c3zfx;ZOl5Jx)Iv-rxM>@~mv=knIgte^B zTuonot&tGN9g0EQqC%vMIF>^MWboHl`O#jlBibLHEqNY&ym@anw7W)rc9!byVSx^i z;^X_~>F8r$YE*C98}#)Yqr1VWOveI;Fv=suztxpOtk}u8jVm3P<dcbJyfCkE#eN7YnrbHtHW?`dKgBdyRnr3GgXE z?y)L%l|nqI@n>w{m6p1N^J1-$sa#ft9ug>nk#3Z1KHZS`#}1X$KCTO_x3px6|9_lG ze_cu`{wpuTTU> z-Ld@g+uA9D93u6HIMAh9vh|am3Z#1HTqVMb7?0AfS{$}&4;ZL;HhBE zZtL4>4;EgAPfJgpxfE1fE)F;{;C%4$MEYH?eZ}Cqb>?jL0A0@qU=&FLxd^6%8+gS! zoPCNhRekF5 z`m2w8(dHJD;kF0%;Y-v-a`pp58#Xh^Dm*1IOg%<>jp&ez;S&ALhs=3#r&u!}QY$!p zb-Lr&-$bQks>0#t8OA5%F@>hhHxasvodGrHa!O-|Oxxg(n%Bptyp?QgDz{I#f~^Lup9 zaD2wqMeXbo{A3cgsH|;aTQ^lxUI`&y(Zmmdt`DHi`E1a@Kmag79Y?tCr>Nmlst(LN zjD=%?Q9er0PKTTKfVnsZ?-+_i3Lljo3h@OmTOg$?oQi+2mZ$31;1y)xN7Un+SRk%c z96G9&*!8#|&aPKRer_NECnUmL)IyZ!5Cwcd4LbWx=K|-`L__2ReKh}uQ4`hC*O9@3 zLIS-nOg8--Vlb!FrSM$O*RxZ`xc(YmGr(d`jwy&HC07^rXvdGi^p=~}< zE{r&`-ToqaDj-oPL0*zgIdOQ;8pnhLUpZf@_@(ks58iG&WuNy8O&%(e${3!d5k+XY zh6@x{>jHXb6O*A}@`rcZBPLbY@>mcy->aQmGFNPm);g~Whc~VAFoP{CIF7a4fjIQaOXs^Vwkqwh{Tk~fCBKe^uA zp;B;14>Cm!o|8*sxVrhJ$!MKwmbPrWJv_ADVmRtnm=n(b7Vh`lh$?=8_my_l5sH@6 z%d@fdyV{x;{yyB^;j{7vb=2=63nYBA*gQC5@ETB;IzfJTjS#F6l=R7y3wWA%qhGm( zT3@FSo$Rd2i3Qv>EHu0~F$6l*bgF#zm&H&2{hJb(`RjFypH$;nLSb>9Xyz~4j1r(| zI;?m1y-!fUaaavN)E5<|ugcfV1ch3SbTKL%iNP5oTo;)Cu_6DpBR_2?Eye+? z*LaAO0OtbJ`>TH6e#O9PMNR0!WQA1;zj37db>dLZ``yg9QsP+kQ;V46UT$bBK?R#0 z)4E#t_^@ccC1=0gxGDOVTFiPuEM_T3VekEMc$?1+B$Xj5ICQe5O{C^H1XZ#@v0x!b z@rYk&FcKwlJD+TDVpeZ#9EgD@8bEHhqS=uc%nTq!l6jR56X-LNZM>eiN9xHT8}+p$ zrt4t)Z1Y(kQ_+AW1|`js?_{bYM|&nTF$tr8Gt9`0VJ0Ois22tVo8^2f`>qBA9g~0l z#Atin6P;ERN?(=*_GZvnVV8Brij?AyAIqRZ0b_`cjn!E5;wS-dZLa8EciKy5iZRwZ zEosGt$|xE6QeAkWzQU;D%kFXfV%^7;Ko!=%f|)Uvu+bAuwYNAP#%CBc=LE;0{Jw@k zBRh(V#G}ff0l^81N`4#g1`l$y17#q?{Ibx-sF5W#-oki$NgIcZj>kr0i$(}9Fj}$K?5WQiHP3@KRh#qkc6C_ zoRAFOmYAaBjDA}u?kVPo!aq}Gs#J%7dqFYG)SQM!tprijGj>1VK{9?*Nmd7V7!yXF zKU_E<0Ycd@^B(jG-y8EzkEqKl_0eA0sM8_K=Xs!{b&h%#jEFmjkoXW8wWLCQ zv{tdk6f+dRo#>7*w9P(|4r*hTOHxU`{w_p60S@a9bN~CRkxU7xmG^SwH&wW|hG+eT z&}qIRiv8^?+&44266c zeDwCX&&ul=vV^;Me(u{bhbMa@`|#~v!s&3>&JchV%@F&Zt`K=NBv2M1q%4?7a#51f zu(+aLBM67mBa{n<3_VK@EtQmz^ywGjMFim4Nt9 z5_H^AimU=%$r{CQR%BN~K_J!70cfy(#?Y#~jyw>j_jDIz(26~RFcgj#&5I&Iei#WK zL?-{ah9jY)hAn|${XavOM&X8OuQBT>a?olkzWz!@O;`_91(MB zQ{=cccGvgrozUefw+M$$@&#BJ`X=DSHb<{Zsz-;iz0u7f*E~Zeg1;^OIBePWZp$O7 zutUObkb024mRl_QDopA;->zn4mpW)yfral?%9JrP^?DY)vl1_<7>;?698D*m-3Wiw zC!t@}!$Ma?I|ituIzEPWeM(;fBM~vw`OQt@@%Z~PtGB-2D$ZY&sv*BcP5_a*+mC)3rHk?%!nieMA~U6ywQvw>3-1s3*k zDz(UdIIWiaQWGN(BkKLqi<|-+4{b6k5yzL3Wj^f)(0DhC;itz4H(_+>EG7F+H8L1W zE1h%WALGZr{&>E2N(9Tp-}4TmQnxs0`Pi-pv;1N{XDR9qVsC_qCax1pKG-}TfEA!zIGOOEZ{w5q|0CIrQhD%Cm4}2uvd#X)!54EWU!IF) zy~f9<Qtc-_mzGjMqC|_|uB+ zgfR4IHsk&`s7IOih{M_T!1km~+R?7{3~6@Q8?q6du!$^W1nVZee7Hu}D-cJeVOeL@4ZFx4B zCz(+GJE=UZRWYe8=s3rG=tDL&z0Q;n4N0`xj<%n`4Iz+?(&|&Q5Mv6<> zIsE){1+bQx+>e37uz&d{nUwsNbDv76?MzfJrZ@uK^s3q$S}LcGLrG$D*Wh)T6<$^x zo*UGIL#h14yN__T5Gx!5qyU*e>GRVm{Af$tqnY>s{G_>M}1xxx6M%$irdsQWt!l|a2uu;3YP>IcpVi}s1fUMw zJ?G0JpD{M3UKW*iRV^^jUSD?_qHo}Xty1rrE5!wEie(W^J#BcykfdXA1*o~1uqK7zNix;1G=`1OALUs%`^82sU5&+xGITv<+Fa-i?#B zJc%s|Nb@f0L_{}7Er$L8!T?438+tIuJwVzZBA9cm`vLjZl9u$9OWWtVTn(C)_^_@MCp#u)WY+3`y>xA)ia-eb;MrG;m`g5#NPOpI(_z z6ke+}0$qLLWn=<9V_KI-%8y;>AFSK-AZf`->!>um*O*Y>D0m+~Z2vObbHIP9`cGNC z<+ksU688&5KHx1m;OQr#EE2>l$FSNkC3bp=fb()wSvXNSOh}%P@+EYboO)+bmN%CB zNfG zSrm-ivqJ+rPBu^6A%QU&neHeD(=5Cpy+tAARdod-zfcw^Wr}4M_};ANIMabtimouA z(g+d%mE4aqf)B}xhZV6z-xnTh;Zk*vX;NEbG7ucB7%1jT;a;I2mBUZJNc2Lowne=O z=;@U%CsVUG4m6~{&memXgz6X>rSrJvTz1PvrBh3z{}2t9>0H$})#OE=koVptqN9J^ ziHK)ip-_Q`WKm`iJ@E*o707W)c+wtpA=g`q$Fw1bz8>SX%fS-Nn@CQftQQG$FWEFj zzru7{07t17xH?$p6sFH{lUeotZi_mE4!c$G5*SB^Uzd0-#RI2G_+t&?JM`u2slI=n zk{sRTw!!XNaBPye8BK;Ispe&Nj9PEqpiU5xgR$>Rc3R-I>?@Lz&qnx|6ZoXk+vKm{ z<004#Jj!$abD~Q|9e0+f-0W*|b!hXBP~6i&gV04FNKEI;HuL4=khW&bkS(1C?#6BU z000~y&dkYC@(vZusR!3KM>Xv$*#r8Sk*tY!a^WmB(JQqORP=I8l7?sg#(*R9E?pym zlmIjs#nH)DrEAm_N!tLzEJpNRfZlTf5e%Xq0-8R8I3$%&whwtFy4-ZkA6;;j$h{2$ zP;z=P;L*yZA4+o+FyMG?Uo_~fE^N?oC@>^DlJC-Iz1l9@4uBYyNV$|?I7C2X<%eZS zIc$SNZSeyne@^^F}~NqDSDl4f4SX`Tfcsi8pk!*oUgdY^X$Gr;z+a{fmx;J|F7l9QhaD zAbwYX{IwMMu&;QQvUE?T=+#enMD3TW=q+ot7bE=CGe*CETS~T2bEKs)FB$McnBQ+~ zL4jisomPpRRoTiAiB?9@i)VZWVR7ozfeH%o6(}_**gS9BN>`+NUxoYSMWD+DO%|y$ zo=w@(NWAgHCDj#T-1S902|EWzD4<5&c>!;h8{PC<+3;xt-VEv@bLGq^Kp`L4f|0mR z*Po2#W$E>=XLzD^n^!!>OX0+F%JExEt-dGg(vR%P@wZVq7_CB$W)|FA7(S= zA7N~QE(8>l6y3S*=qAXZEnUyUIp>)xJt1)M9=XN928rrhKu0c@0^ct6s*ldp;QC9N zz!^Q-*QFF8b$;!laLDyVt9|1?s{_7Z3kSVd<96D3p61dUER-YYzrCs%cc>M%pdna@ zec5koCM?94xvcZ#$acS|nObMA2V!#RJI?cz)E~v6l?s=lU-5_fJ;QImMKdM3e0T53 zg0zf^AFBt2ro`zP03(yPOR3yE;RLY4B&e;ts~R~9C`+0*b~HOtG)ja7z)ksDLvXJQ z8KH>;m(&09xoEW{*5oeeM<^PRPZXWO^S0m$`~rli*x3sf#53u!V*{QJ1J~mr+d1sY zb@=Gxt39By3&{cb04~EZdo-t~VhL>=STZ~@Pe^@={x0OrwE-SQDqBYQlU{2hicu40 zp8tTEk4BU=neqaHJ?MnYW)6s*uXip{Kx-A)Y5jrSL+)eb@(Br?QggSWf=+Bh|7}D! zeSMv%Y3=N<*-|b7&lbm-GPHpsfnLG}=KW+%WU=-~R~lRnxOE(sSOdhCdKENR!YNZm zs_lYaaUbqwG89cqSK~upddSo$f|xsEc9O@~;Q64eAT>^^cc=T#eR>-Wk&uJ5Q0rwm z@^8Er3*Nx|>m~iS(+h^j=i~|eF-h>0!_8>{*I9wQLkzf_oSG|FcZ25*Z(zJ&;I$x_ zA#HP|q(TY1r&<(nNe=Tu)xz*G!(iE2#)e!c7>R5WMxEfBH5|c=3sOxCWV43ktU}@x2h zhu}-a;jb_&Q%rRF$Zn+qKvT3wL7Q!2a7g1au>9#1>0fZm1^Q-LUo~sa*gX9XI{|R! zNbbs~n||}OJSMfmZ+C?Fu%8Cond8YnrvuN|qa<8zL{+L5q4rbStK5rg1mCP$x;&i? z>;t$;{XeGOGOX!0>>nQ8jAqnG>6Vny-2w_o4(XDH(OuFdAV^4ul(fVE=`H~!r9&9q z@ZaygpXWI4z1h2M$By&*p6B_gs~U@FrrolPat60R*$fPZo$8<9SDTeY5A4b0FBkLx z?$ucEO|g$_W8w^u6siumUh$!p4&d4UZv&8OmCklwQ#Nd-RA`;pd(W92; ztm5=);xNedE|>(#Lkv{7b`Y(e`%L;f-VDX_Ta9u?9G3b#BnijX&ni^Rl<&))=P`kjsiVHV_UZis*m{#b1B_M=7$9=$+7 z5dh$MgX0>Dm@NR9Sh(oL-7gmryFn=!8wsH7x;5g3j}}v{0`3z#DlCKW+vt(G4=7=Y z<5KGa_}ZgSHCp?MPch(l@j!%FJQ`x1aI;3#tZz2N_s4*MnYX3Jp)&jE%1H;8|5~8A zg8+B{r8*w4&af^J)7-!Q&%z%Df`{V~;;&m&?_MCQuY>@`pG!Wd>Sy?HGf6*R6`GfV zPcT4adf`;ShDrGXg1QGs=wC1}y2?%_lM`HKGk$ReQo+OHh+4&Z0pN>rhILtOT2yd) zBpw}h>9QOl2`YFwdoz5MZmVy~5D1of%}0hL%sg8?Rtr^!iT%?4;EhRV4Mupmg(|Y` z5TirPu%HTvs1U5f_s3~Cd%29?772M58DbQHIyJ$yJ; z3Now7LCV)1Cx3YVr_~iB7%Rc-W#&C}FFH1H2%tjjXl?_Nvb`Jc?I+%Z*L2Lvv}2Z@ z=X^wk=&!%XW+ZLya8qdhyZwXuw0D!=^Mb+tdZo52^+}?|DjXFo!J@T^8;>RcdM+E= zkcFcu0e99_5)+18wQF;q{{&k;S2;!1Lnn=0M%6&HBl-+1So##be?zCJf>a23i3vm+ zr$9RKg;7u}9}qRG1?^s+Lw|IP;|hjlS5#`4ZT~^%z14US3(w%lc&cv1#}EhyFv&=B zYxr#*$~sEl$ZuiC`Kque;(?7W*QYRQ?1MV@nXM`QC_Gep6RO{Y;b8+8QEnw<1q24J ztC2b%^Qd<>rgeexYLdOWQ_y;#pc^?WX~Ilegav==Nk8tmJOENjG!jwYFHslv3G+*p zK`lp}tZR6c`S4mbyjniMHR{)5MWOI#GEkw;jyoYdT&c!CIUz(@H1(4KY%+McRGJ~tvcf@Y)tODnVb|A&0j@M)0^nqR z8Xf(khWFzGr7i(NI`N9Y@?l>oMlEr4Y>hZ6qjDEx&^n9=9ZjrR-!e!>zS}pc3@Z`^ zf(<3&UN_VL{PpLC-aP#{5st=`l-g?|lIKMhJneIaG;N02Bh^FV8;H~ilx^0cN66+@Yzw}nEmhEZbMmHXG`kETm?G;eTJo3vNGXg*v!cDs4 z^Y{F3v?2bSC=mWcI~5gfB?iNu4uf$1Ug^ka zD`aejP?K{i4|d7?@~^1fV5+>w#ZCN|IJZ3%SDq)pL7(_xbn(_Y*(7v`*v@7az*YLL zTJO%nFxYdu0U}#dF@Dm)X1p3`P`SP$>l-Z)<;|SAK(tjJQ3g17vTLo-V@v6Ch~y<% ztScl36KURkv1FDPGsQA=#kf`4i+zV{D#X8rBWE<-7alU`vldjAM|okl{vIu{+oTRI zke&lRP1*FO88Bm|-@Q>yQuSvxMHLh*ev3R@_IrV{C&XwlB3Fx*tsb|0JSMHa@99u< z`8RInkxw@MqXjNIdkn4Csh_Tk@JX9o2E$_euq_8L=r9rdZEv$I<3q3{g$$o?AfdV; z$`TaTjl+HbuoItbhD8LRgi-|;AwviZ0jCQ)Z+8BLKxW?P4k6DM*;iqMf&W9oLGy&+ zu?=XDop<}^dS9*y(<`~|I)%IfSEZLwDZaR#*AN(3v|}Fq=rbAi&Rh%`bOV4T5iyJ5 z%fKH2lZg%>{PzGXH8sl_3`=R0q%LDoYi51QnbGh8^5go-003~3NKjme0S6|8FYU+P zvM%6uID@eiLwWu0Brqm`k%xb#PAUYS%m`zU{s+q4-lbUwm7nE%s7ejP#I^O-K1dN1 z3zCDpg(Gwcz!!doo&HJHe|{*BSg=%tOtWLkW1=Ln0OD{u;C&iJaP%IWFdQM)x>sQY zLaaZbi-o2U3wJ54Uoe4w0Yqf6eXS(BYC7iAZ_%~tXyBn>PRk-oL41brSY`Wl1{K-7 zri+g@RxD4phlDyGv;K{ zS2Nbo`K*0Aa(So#6_;ghjx&0dJC&d%Eku2)nHWG%`E>pK3GaDVyW#o~XiJaa9}z@d z40N1m+nkvSIXouM5Ce-cHWssag&GK9+9=$%pk(HkrPou2wI-tl89=A#2>n@9m z0t5}EZQ^jd$_RRJe*%Ss1_La)2TOkS)vAngMVFw){t`d#hKntPyQPL8$A69Tz93vn z2^5q>Jz8_BV|%pUt$yP{7SeijMq)Cz+l4SPJGv!^ONm8Ai6+ilz{5AHH3)$c#pEbSqX0oL>XyJpr6J#- z?LlT9c@kJUHTVgky&LQE8pBxd-RH#YGHe_R z0lneb*|l?V(c_z24#AQ)Vzd{x681@0t#7qQVpv_wrO!x+YIY%;YvcbibSKmZn>&2H zGo&e$@y#%Xxc5;Ot)!m@TdWXFQS;NTsfNMEg)r3Srqt@nM^9d0Qu$={J9qE7Nq1Gv zqHyvFPc9)WfHyFSeu%64!I1FjuX!D^C2)a~mXvAQv&xuA5R2GEp^V>3&kzjjp`ca% z@+7#iRg5psE3m~$a#OU~rv&9yrPjnIvj=PBk?Pgn>L!JR!B^?@V9flrj|4h8|HzGfpENaDF~8d9u@j0+b3@Uax==Z4ih$uQE}dEN9?> zM16Xg&t61ie}=8{T&Fl%lHKwn&1|uCdaF$61ojxN40MP(>n#{2l@}CMEL4Ce8^E7L ztK*}cpXv{w(uW^@1sBMnojC|Mx2SrNcJS9B3L_W*gH~&O@@)=joo<@GerS2S)59G` zfX92H)&1e6{Pm5~*CQ!69jL2mTq$YARBdoI3&XN%zU1I4taCK5xVI=%<^nk^+@vc2 z!?*mEa0m0?t1uLFDW#|lVjR~9GZG*eGKGPyfjwchj7|t+F_hyYK4adKtp)B(LnnB{ z2(PsGR8heuL^43VOF>P8oW#djR0u$jA01iQqRAe805CEVD+$W3zC&o)s{U(y<($m4 z3IIf*@(ZR00-L`qEHyVbi+oAi0>Vsm|3lR!SNhQ4c%(UH5D2n*53b%l9srWwy)Co+ zU$1vvx-e9m{X9p7*V?(YdAtSFGpg2YiEdqBtv!G9?noK0igRQTFOplWXQ$PJ_I_zD z1Pc~F$`gXY7NiZ|ETnzf=97enTa+Nt{MOKKhwVrToADb2m_!>A{q(7WKK==$_|ZRoxy9DI^r2xtKVlhd(R&oeaghBg5rw)E2wcQO^-n2_xtA` zL03-l4|^yC#k+ssO|qwsi+v?PENRGnm>g0t1FsXmBny0 z;EOM44Stzjhjg@jpo?QH2V^V_bXf;5qKYDa4Cv^UADGg9{au#!wlVrI1eGx zfhp3PV|l2KWetkMIDXtb5e$1!|0dyWmzOX@FtpKAyp0RfXu?L>#2(b~vta1hgxY4B zV1&3|{&6WL@Gbt$ZzB?7Ku&?1-os2B9^b73xXzL0ceqgl?5Vo!gz2DBVg=$a@EUpm1!@()&LZJ3aQb8{y!SE6m z<62f$l$Z}{86`feptTfF3x4Uz;#gL;Nxdarc#YO4A!s2+8YGRCCBGixPC5lOR6`|1 zhk&F$O2|t#H#Jq1VNnuo;lZYUFmu|s`N6DEAmwZmFW5;*6W36Z`xSPVM^O&=`S7e6 z)-7bHf=X?%&a;SGE%ta1nEBx0;t-;S#?Eha1*9r=QBbh>^zN}pS?KK?;G~nA@f*L6b;FYVx7IJ()260v z!cZ_@)QF^6JnOPstUVk6)?h-e*5dBsv^Tet(7hmYA{k>SG9_Z`1d1J*XKY58gz|HB z6|jj*z+J*0H3cWx-ynWS?656SHI#ybR-Yft+WAb8o|cNmc=EW6e>o-Yha$1ejgi!| z_D-+Q72zx;zSM8Y5P&@yO9Jogi@y;s%dkjmk*ezqx9S^On5%XTlh45X;exg3dtLgQ zJ<5jnD@Fiua%^jIJbW-ZM5vw<=6_jQsh7@pV+8M$$yv_~uP?C^4U^)BK!Lr##XJu-68;Rba=Lo3bKvt53(@~b~yI-2|_ z==E)7V-^5NebPp?>4%gA^2eA&)SZtrBccMBCIRk3P0d4}ue|q_6BGUfKj*{)j%>}o zU1Bd4hhX7fe{RfO7m|H+%8?FoAItsuHv|PCeZPPlQr6o}IIg8);bZdV;Q9lJ8H~2U$r4ZEUNtY zz0XyW@Wa5CpxL%r(!bNpTNFKEIHke}!nbHuT|R&t}T-2o8*hc25qw zjyHsrwceL6_3La^s*!EEN*MOV0~Ex8iX*{aYvDUn>}!7K{6eMI{esg%@bZTWOBW5} zz@CnrFI8zOm;s$Pi!yw8NMg#CRT``F*?CgyjEP5Z~g2?yO%AZcYHn<#y37?fk;f4d?WxE@%>EB=q(zee1Z_R<-s>WL11hF+ipLh470%{z{>nS>}