From d56d58d163d1e067968d9e63980b49ebd889026c Mon Sep 17 00:00:00 2001 From: Brian Ray Date: Tue, 14 Aug 2018 13:02:12 -0400 Subject: [PATCH] experimental python 3 support --- mm3/__init__.py | 21 + mm3/color_converter.py | 84 + mm3/composer_base.py | 48 + mm3/composer_xls.py | 243 ++ mm3/config_base.py | 33 + mm3/contrib/__init__.py | 0 mm3/contrib/django/__init__.py | 0 mm3/contrib/django/data_model.py | 158 + mm3/contrib/django/grid.py | 26 + mm3/contrib/prettytable/__init__.py | 0 mm3/contrib/prettytable/composers.py | 34 + mm3/document_base.py | 86 + mm3/document_writers.py | 69 + mm3/grid_base.py | 83 + mm3/lib/__init__.py | 0 mm3/lib/font_data/__init__.py | 4 + mm3/lib/font_data/core.py | 58 + mm3/lib/font_data/decorators.py | 29 + mm3/lib/font_data/tests.py | 18 + mm3/lib/xldate/__init__.py | 2 + mm3/lib/xldate/convert.py | 61 + mm3/lib/xldate/tests.py | 27 + mm3/lib/xlwt_0_7_2/BIFFRecords.py | 2397 ++++++++++++++ mm3/lib/xlwt_0_7_2/Bitmap.py | 262 ++ mm3/lib/xlwt_0_7_2/Cell.py | 243 ++ mm3/lib/xlwt_0_7_2/Column.py | 46 + mm3/lib/xlwt_0_7_2/CompoundDoc.py | 516 +++ mm3/lib/xlwt_0_7_2/ExcelFormula.py | 43 + mm3/lib/xlwt_0_7_2/ExcelFormulaLexer.py | 128 + mm3/lib/xlwt_0_7_2/ExcelFormulaParser.py | 677 ++++ mm3/lib/xlwt_0_7_2/ExcelMagic.py | 862 +++++ mm3/lib/xlwt_0_7_2/Formatting.py | 261 ++ mm3/lib/xlwt_0_7_2/Row.py | 253 ++ mm3/lib/xlwt_0_7_2/Style.py | 593 ++++ mm3/lib/xlwt_0_7_2/UnicodeUtils.py | 81 + mm3/lib/xlwt_0_7_2/Utils.py | 196 ++ mm3/lib/xlwt_0_7_2/Workbook.py | 636 ++++ mm3/lib/xlwt_0_7_2/Worksheet.py | 1297 ++++++++ mm3/lib/xlwt_0_7_2/__init__.py | 16 + mm3/lib/xlwt_0_7_2/antlr.py | 2874 +++++++++++++++++ mm3/lib/xlwt_0_7_2/licences.py | 82 + mm3/model_base.py | 178 + mm3/serializer_base.py | 25 + mm3/style_base.py | 42 + tests3/api_tests.py | 13 + tests3/author.bmp | Bin 0 -> 225648 bytes tests3/basic_tests.py | 215 ++ tests3/customize_tests.py | 73 + tests3/django_tests.py | 44 + tests3/dummy_django_project/__init__.py | 0 tests3/dummy_django_project/manage.py | 10 + .../marmir_test/__init__.py | 0 .../marmir_test/models.py | 36 + tests3/dummy_django_project/settings.py | 160 + tests3/dup_type_fix_tests.py | 60 + tests3/formula_test.py | 33 + tests3/gdata_tests.py | 53 + tests3/model_base_tests.py | 13 + tests3/more_data_tests.py | 58 + tests3/multi_sheets.py | 65 + tests3/pretty_table_tests.py | 53 + tests3/psycopg2_tests.py | 81 + tests3/xlrd_helper.py | 30 + 63 files changed, 13789 insertions(+) create mode 100644 mm3/__init__.py create mode 100644 mm3/color_converter.py create mode 100644 mm3/composer_base.py create mode 100644 mm3/composer_xls.py create mode 100644 mm3/config_base.py create mode 100644 mm3/contrib/__init__.py create mode 100644 mm3/contrib/django/__init__.py create mode 100644 mm3/contrib/django/data_model.py create mode 100644 mm3/contrib/django/grid.py create mode 100644 mm3/contrib/prettytable/__init__.py create mode 100644 mm3/contrib/prettytable/composers.py create mode 100644 mm3/document_base.py create mode 100644 mm3/document_writers.py create mode 100644 mm3/grid_base.py create mode 100644 mm3/lib/__init__.py create mode 100644 mm3/lib/font_data/__init__.py create mode 100644 mm3/lib/font_data/core.py create mode 100644 mm3/lib/font_data/decorators.py create mode 100644 mm3/lib/font_data/tests.py create mode 100644 mm3/lib/xldate/__init__.py create mode 100644 mm3/lib/xldate/convert.py create mode 100644 mm3/lib/xldate/tests.py create mode 100644 mm3/lib/xlwt_0_7_2/BIFFRecords.py create mode 100644 mm3/lib/xlwt_0_7_2/Bitmap.py create mode 100644 mm3/lib/xlwt_0_7_2/Cell.py create mode 100644 mm3/lib/xlwt_0_7_2/Column.py create mode 100644 mm3/lib/xlwt_0_7_2/CompoundDoc.py create mode 100644 mm3/lib/xlwt_0_7_2/ExcelFormula.py create mode 100644 mm3/lib/xlwt_0_7_2/ExcelFormulaLexer.py create mode 100644 mm3/lib/xlwt_0_7_2/ExcelFormulaParser.py create mode 100644 mm3/lib/xlwt_0_7_2/ExcelMagic.py create mode 100644 mm3/lib/xlwt_0_7_2/Formatting.py create mode 100644 mm3/lib/xlwt_0_7_2/Row.py create mode 100644 mm3/lib/xlwt_0_7_2/Style.py create mode 100644 mm3/lib/xlwt_0_7_2/UnicodeUtils.py create mode 100644 mm3/lib/xlwt_0_7_2/Utils.py create mode 100644 mm3/lib/xlwt_0_7_2/Workbook.py create mode 100644 mm3/lib/xlwt_0_7_2/Worksheet.py create mode 100644 mm3/lib/xlwt_0_7_2/__init__.py create mode 100644 mm3/lib/xlwt_0_7_2/antlr.py create mode 100644 mm3/lib/xlwt_0_7_2/licences.py create mode 100644 mm3/model_base.py create mode 100644 mm3/serializer_base.py create mode 100644 mm3/style_base.py create mode 100644 tests3/api_tests.py create mode 100644 tests3/author.bmp create mode 100644 tests3/basic_tests.py create mode 100644 tests3/customize_tests.py create mode 100644 tests3/django_tests.py create mode 100644 tests3/dummy_django_project/__init__.py create mode 100644 tests3/dummy_django_project/manage.py create mode 100644 tests3/dummy_django_project/marmir_test/__init__.py create mode 100644 tests3/dummy_django_project/marmir_test/models.py create mode 100644 tests3/dummy_django_project/settings.py create mode 100644 tests3/dup_type_fix_tests.py create mode 100755 tests3/formula_test.py create mode 100644 tests3/gdata_tests.py create mode 100644 tests3/model_base_tests.py create mode 100644 tests3/more_data_tests.py create mode 100644 tests3/multi_sheets.py create mode 100644 tests3/pretty_table_tests.py create mode 100644 tests3/psycopg2_tests.py create mode 100644 tests3/xlrd_helper.py diff --git a/mm3/__init__.py b/mm3/__init__.py new file mode 100644 index 0000000..c3f4e9d --- /dev/null +++ b/mm3/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +__version__ = "0.1.3" +__author__ = [ + "Brian Ray ", +] +__license__ = "TBD" + +from .document_base import * +from . import config_base +from . import model_base +import logging +logging.basicConfig() + + +Date = model_base.DateFieldType +URL = model_base.URLFieldType +Image = model_base.ImageFieldType +Formula = model_base.FormulaFieldType + +Config = config_base.ConfigBase diff --git a/mm3/color_converter.py b/mm3/color_converter.py new file mode 100644 index 0000000..9712727 --- /dev/null +++ b/mm3/color_converter.py @@ -0,0 +1,84 @@ +from .lib.font_data.decorators import memoized + +excel_color_dict = {} + +excel_color_dict['0, 0, 0'] = 0x08 +excel_color_dict['255, 255, 255'] = 0x09 +excel_color_dict['255, 0, 0'] = 0x0A +excel_color_dict['0, 255, 0'] = 0x0B +excel_color_dict['0, 0, 255'] = 0x0C +excel_color_dict['255, 255, 0'] = 0x0D +excel_color_dict['255, 0, 255'] = 0x0E +excel_color_dict['0, 255, 255'] = 0x0F +excel_color_dict['128, 0, 0'] = 0x10 +excel_color_dict['0, 128, 0'] = 0x11 +excel_color_dict['0, 0, 128'] = 0x12 +excel_color_dict['128, 128, 0'] = 0x13 +excel_color_dict['128, 0, 128'] = 0x14 +excel_color_dict['0, 128, 128'] = 0x15 +excel_color_dict['192, 192, 192'] = 0x16 +excel_color_dict['128, 128, 128'] = 0x17 +excel_color_dict['153, 153, 255'] = 0x18 +excel_color_dict['153, 51, 102'] = 0x1A +excel_color_dict['255, 255, 204'] = 0x1C +excel_color_dict['204, 255, 255'] = 0x1D +excel_color_dict['102, 0, 102'] = 0x1E +excel_color_dict['255, 128, 128'] = 0x1F +excel_color_dict['0, 102, 204'] = 0x28 +excel_color_dict['204, 204, 255'] = 0x29 +excel_color_dict['0, 204, 255'] = 0x2A +excel_color_dict['204, 255, 204'] = 0x2B +excel_color_dict['255, 255, 153'] = 0x2C +excel_color_dict['153, 204, 255'] = 0x2D +excel_color_dict['255, 153, 204'] = 0x2E +excel_color_dict['204, 153, 255'] = 0x2F +excel_color_dict['255, 204, 153'] = 0x30 +excel_color_dict['51, 102, 255'] = 0x31 +excel_color_dict['51, 204, 204'] = 0x32 +excel_color_dict['153, 204, 0'] = 0x33 +excel_color_dict['255, 204, 0'] = 0x34 +excel_color_dict['255, 153, 0'] = 0x35 +excel_color_dict['255, 102, 0'] = 0x36 +excel_color_dict['102, 102, 153'] = 0x37 +excel_color_dict['150, 150, 150'] = 0x38 +excel_color_dict['0, 51, 102'] = 0x39 +excel_color_dict['51, 153, 102'] = 0x3A +excel_color_dict['0, 51, 0'] = 0x3B +excel_color_dict['51, 51, 0'] = 0x3C +excel_color_dict['153, 51, 0'] = 0x3D +excel_color_dict['51, 51, 153'] = 0x3E +excel_color_dict['51, 51, 51'] = 0x3F + + +@memoized +def rgb(c): + split = (c[0:2], c[2:4], c[4:6]) + out = [] + for x in split: + out.append(int(x,16)) + return out + +@memoized +def get_closest_rgb_match(hex): + hex = hex.replace("#",'').strip() + color_dict = excel_color_dict + orig_rgb = rgb(hex) + new_color = '' + min_distance = 195075 + orig_r = orig_rgb[0] + orig_g = orig_rgb[1] + orig_b = orig_rgb[2] + for key in color_dict.keys(): + new_r = int(key.split(',')[0]) + new_g = int(key.split(',')[1]) + new_b = int(key.split(',')[2]) + r_distance = orig_r - new_r + g_distance = orig_g - new_g + b_distance = orig_b - new_b + current_distance = (r_distance * r_distance) + (g_distance * g_distance) + (b_distance * b_distance) + if current_distance < min_distance: + min_distance = current_distance + new_color = key + return color_dict.get(new_color) + + diff --git a/mm3/composer_base.py b/mm3/composer_base.py new file mode 100644 index 0000000..0cbeee8 --- /dev/null +++ b/mm3/composer_base.py @@ -0,0 +1,48 @@ + +import logging +from .model_base import HeaderFieldType + +log = logging.getLogger(__name__) + + +class ComposerBase(object): + """ Used by Composers """ + def run(self): + raise Exception("Overwrite run() in subclass") + + def __init__(self, data_model, grid, document): + self.data_model = data_model + self.grid = grid + self.document = document + self.row_id = 0 + self.col_id = 0 + + def row(self, row): + self.col_id = 0 + self.start_new_row(self.row_id) + for cell in row: + self.write_cell(self.row_id, self.col_id, cell) + self.col_id += 1 + self.end_row(self.row_id) + self.row_id += 1 + + def iterate_grid(self): + for row in self.grid.grid_data: + self.row(row) + + def write_header(self): + i = 0 + for header in self.data_model.field_titles: + cell = HeaderFieldType(data=header) + log.info(cell.__dict__) + self.write_cell(0, i, cell) + i += 1 + self.row_id += 1 + + def set_option(self, x): + log.warn("%s not supported with this composer %s" % (x, self.__class__.__name__)) + + def finish(self): + """ Things we do after we are done """ + for key in [x for x in dir(self.document.config) if not x.startswith("_")]: + self.set_option(key) diff --git a/mm3/composer_xls.py b/mm3/composer_xls.py new file mode 100644 index 0000000..f7f3b2d --- /dev/null +++ b/mm3/composer_xls.py @@ -0,0 +1,243 @@ +import re +from .composer_base import ComposerBase +from . import lib.xlwt_0_7_2 as xlwt +from .lib.font_data.core import get_string_width +from .lib.xldate.convert import to_excel_from_C_codes +import logging +import io +from . import model_base +from . import style_base +from . import color_converter + +log = logging.getLogger(__name__) + + +def get_string_width_from_style(char_string, style): + if type(char_string) not in (str, str): + return 0 + point_size = style.font.height / 0x14 # convert back to points + font_name = style.font.name + if not font_name: + font_name = 'Arial' + return int(get_string_width(font_name, point_size, char_string) * 50) + + +class styleXLS(style_base.StyleBase): + + font_points = 12 + + def get_pattern(self): + pattern = xlwt.Pattern() + # see issue #27 https://github.com/brianray/mm/issues/27 + if not self.background_color: + return pattern + pattern.pattern = 1 + color = color_converter.get_closest_rgb_match(self.background_color) + pattern.pattern_fore_colour = color + return pattern + + def get_font_color(self): + color = 0 + if self.color: + color = color_converter.get_closest_rgb_match(self.color) + return color + + def get_border(self): + border = xlwt.Borders() + if False: # TODO borders + border.left = border.right = border.top = border.bottom = 3000 + if self.border_color: + color = color_converter.get_closest_rgb_match(self.border_color) + border.top_color = color + border.bottom_color = color + border.left_color = color + border.right_color = color + return border + + def is_bold(self): + if self.font_style == 'bold': + return True + return False + + def get_font_points(self): + if self.font_size: + return self.font_size + return 12 # TODO: default from config? + + def get_font_name(self): + if not self.font_family: + return 'Arial' + return self.font_family + + def get_text_align(self): + text_align = xlwt.Alignment() + # HORZ - (0-General, 1-Left, 2-Center, 3-Right, 4-Filled, 5-Justified, 6-CenterAcrossSel, 7-Distributed) + horz = 0 + if self.text_align == 'center': + horz = 2 + elif self.text_align == 'right': + horz = 3 + elif self.text_align == 'left': + horz = 1 # left + elif self.text_align is not None: + log.warn("Unknown text_align %s" % self.text_align) + + text_align.horz = horz + return text_align + + +class ComposerXLS(ComposerBase): + + def convert_style(self, stylestr): + in_style = styleXLS() + in_style.style_from_string(stylestr) + + style = xlwt.XFStyle() + fnt1 = xlwt.Font() + fnt1.name = in_style.get_font_name() + fnt1.bold = in_style.is_bold() + fnt1.height = in_style.get_font_points() * 0x14 + fnt1.colour_index = in_style.get_font_color() + style.font = fnt1 + style.alignment = in_style.get_text_align() + style.pattern = in_style.get_pattern() + style.borders = in_style.get_border() + + return style + + def cell_to_value(self, cell, row_id): + + if self.document.config.headers and row_id == 0: + css_like_style = self.document.config.header_style + elif len(self.document.config.row_styles) == 0: + css_like_style = '' + else: + style_index = row_id % len(self.document.config.row_styles) + css_like_style = self.document.config.row_styles[style_index] + + style = self.convert_style(css_like_style) + + if type(cell) == model_base.HeaderFieldType: + style = self.convert_style(self.document.config.header_style) + return cell.data, style + + elif type(cell) in (model_base.IntFieldType, model_base.StringFieldType): + return cell.data, style + + elif type(cell) == model_base.DateTimeFieldType: + style.num_format_str = self.document.config.get('datetime_format', 'M/D/YY h:mm') + return cell.data, style + elif type(cell) == model_base.DateFieldType: + num_string_format = self.document.config.get('date_format', 'M/D/YY') + if cell.format: + num_string_format = to_excel_from_C_codes(cell.format, self.document.config) + style.num_format_str = num_string_format + return cell.data, style + + else: + return cell.data, style + + def start_new_row(self, id): + pass + + def end_row(self, id): + pass + + def write_cell(self, row_id, col_id, cell): + + value, style = self.cell_to_value(cell, row_id) + if type(cell) == model_base.ImageFieldType: + if cell.width: + self.sheet.col(col_id).width = cell.width * 256 + if cell.height: + self.sheet.col(col_id).height = cell.height * 256 + self.sheet.insert_bitmap(value, row_id, col_id) + + elif type(cell) == model_base.URLFieldType: + self.sheet.write( + row_id, + col_id, + xlwt.Formula('HYPERLINK("%s";"%s")' % (value, cell.displayname)), + style + ) + elif type(cell) == model_base.FormulaFieldType: + self.sheet.write( + row_id, + col_id, + xlwt.Formula(re.sub('^=', '', value)), + style + ) + + else: + # most cases + self.sheet.write(row_id, col_id, value, style) + self.done_write_cell(row_id, col_id, cell, value, style) + + def done_write_cell(self, row_id, col_id, cell, value, style): + + if self.document.config.get('adjust_all_col_width', False): + + current_width = self.sheet.col_width(col_id) + 0x0d00 + log.info("current width is %s" % current_width) + new_width = None + + if type(cell) == model_base.StringFieldType: + new_width = get_string_width_from_style(value, style) + + elif type(cell) == model_base.DateTimeFieldType: + new_width = 6550 # todo: different date formats + + elif type(cell) == model_base.URLFieldType: + new_width = get_string_width_from_style(cell.displayname, style) + + if new_width and new_width > current_width: + log.info("setting col #%s form width %s to %s" % (col_id, current_width, new_width)) + col = self.sheet.col(col_id) + if new_width > 65535: # USHRT_MAX + new_width = 65534 + current_width = new_width + col.width = new_width + + def set_option(self, key): + + val = getattr(self.document.config, key) + if key == 'freeze_col' and val and val > 0: + self.sheet.panes_frozen = True + self.sheet.vert_split_pos = val + + elif key == 'freeze_row' and val and val > 0: + self.sheet.panes_frozen = True + self.sheet.horz_split_pos = val + + else: + + log.info("Nothing to be done for %s" % key) + + return + log.info("Set option %s" % key) + + def run(self, child=None): + top = False + if not child: + self.w = xlwt.Workbook(style_compression=2) + top = True + else: + self.w = child + self.sheet = self.w.add_sheet(self.document.name or "Sheet 1") + if self.document.config.headers: + self.write_header() + self.iterate_grid() + self.finish() + + # process any childern + for doc_child in self.document.children: + doc_child.writestr(child=self.w) + + if top: + # write the file to string + output = io.StringIO() + self.w.save(output) + contents = output.getvalue() + output.close() + + return contents diff --git a/mm3/config_base.py b/mm3/config_base.py new file mode 100644 index 0000000..b191ad5 --- /dev/null +++ b/mm3/config_base.py @@ -0,0 +1,33 @@ + + +class ConfigBase(object): + """ Holds the configuration """ + + def get(self,key,default=None): + if hasattr(self,key): + return getattr(self,key) + return default + + def __init__(self, config_dict=None): + if config_dict: + self.set_dict(config_dict) + + def set_dict(self, config_dict): + for k,v in list(config_dict.items()): + setattr(self, k, v) + + + # default settings + headers = True + header_style = "color: #ffffff; font-family: arial; background-color: #0000B3; font-size: 12pt; text-align: center" + freeze_col = 0 + freeze_row = 1 + row_styles = ( + "color: #000000; font-family: arial; background-color: #666666; border-color: #ff0000", + "color: #000000; font-family: arial; background-color: #FFFFFF" # Alternate + ) + adjust_all_col_width = True + datetime_format = 'M/D/YY h:mm:ss' + date_format = 'M/D/YY' + time_format = "h:mm:ss" + INGORE_DATA_MISMATCH = True diff --git a/mm3/contrib/__init__.py b/mm3/contrib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mm3/contrib/django/__init__.py b/mm3/contrib/django/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mm3/contrib/django/data_model.py b/mm3/contrib/django/data_model.py new file mode 100644 index 0000000..966e267 --- /dev/null +++ b/mm3/contrib/django/data_model.py @@ -0,0 +1,158 @@ +import logging +from mm import model_base +from django.db import models + +log = logging.getLogger(__name__) + + +class DjangoDataModel(object): + """ Data Model creates a list of system defined data types in self.field_headers""" + + def __init__(self, data, order=None, column_types=None): + """ constructor takes data as a tuple or list""" + + self.field_headers = [] + self.field_titles = [] + if data.count() == 0: + raise Exception("Can not make spreadsheets with an empty set") + first_data = data[0] + + header_types = {} + for f in first_data._meta.fields: # TODO: + obj._meta.many_to_many + header_types[f.name] = self.figure_out_type(f) + + #TODO: 'if order' sort the list + + # assign data + for verbose_name, field_type_class in list(header_types.items()): + + # we add it to the 'class' so to be + # used in every instance + self.field_titles.append(verbose_name) + self.field_headers.append(field_type_class) + log.info("created field type %s for %s" % (field_type_class, verbose_name)) + + def __serialize(obj): + obj._meta.fields + + def _string_types(self): + return [ + models.CharField, + models.CommaSeparatedIntegerField, + models.IPAddressField, + models.SlugField, + models.TextField, + models.EmailField, + models.FilePathField, + models.GenericIPAddressField, + ] + + def _int_types(self): + return [ + models.AutoField, + models.IntegerField, + models.PositiveIntegerField, + models.PositiveSmallIntegerField, + models.SmallIntegerField, + models.BigIntegerField, + + ] + + def _bool_types(self): + return [ + models.BooleanField, + models.NullBooleanField, + ] + + def _date_types(self): + return [ + models.DateField, + ] + + def _time_types(self): + return [ + models.TimeField + ] + + def _datetime_types(self): + return [ + models.DateTimeField, + ] + + def _decimal_types(self): + return [ + models.DecimalField, + ] + + def _float_types(self): + return [ + models.FloatField, + ] + + def _none_types(self): + l = [ + models.NullBooleanField, + models.FileField, + models.ImageField, + ] + if "BinaryField" in dir(models): + l.append(models.BinaryField) + return l + + def _url_types(self): + return [ + models.URLField + ] + + def type_mapping(self): + return ( + (self._string_types, model_base.StringFieldType), + (self._int_types, model_base.IntFieldType), + (self._bool_types, model_base.BoolFieldType), + (self._date_types, model_base.DateFieldType), + (self._time_types, model_base.TimeFieldType), + (self._datetime_types, model_base.DateTimeFieldType), + (self._decimal_types, model_base.DecimalFieldType), + (self._float_types, model_base.FloatFieldType), + (self._none_types, model_base.NoneFieldType), + (self._url_types, model_base.URLFieldType), + ) + + def figure_out_type(self, item): + """ + +This is how django stores types in sqlite3: + +"AutoField" integer NOT NULL PRIMARY KEY, +"BigInteger" bigint NOT NULL, +"BinaryField" BLOB NOT NULL, +"BooleanField" bool NOT NULL, +"CharField" varchar(50) NOT NULL, +"CommaSeparatedIntegerField" varchar(25) NOT NULL, +"DateField" date NOT NULL, +"DateTimeField" datetime NOT NULL, +"DecimalField" decimal NOT NULL, +"EmailField" varchar(75) NOT NULL, +"FileField" varchar(100) NOT NULL, +"FilePathField" varchar(100) NOT NULL, +"FloatField" real NOT NULL, +"ImageField" varchar(100) NOT NULL, +"IntegerField" integer NOT NULL, +"IPAddressField" char(15) NOT NULL, +"GenericIPAddressField" char(39) NOT NULL, +"NullBooleanField" bool, +"PositiveIntegerField" integer unsigned NOT NULL, +"PositiveSmallIntegerField" smallint unsigned NOT NULL, +"SlugField" varchar(50) NOT NULL, +"SmallIntegerField" smallint NOT NULL, +"TextField" text NOT NULL, +"TimeField" time NOT NULL, +"URLField" varchar(200) NOT NULL + """ + item_type = type(item) + for func, mm_type in self.type_mapping(): + if item_type in func(): + return mm_type + + log.warn("Returning None type for type %s" % item_type) + return model_base.NoneFieldType diff --git a/mm3/contrib/django/grid.py b/mm3/contrib/django/grid.py new file mode 100644 index 0000000..cedb246 --- /dev/null +++ b/mm3/contrib/django/grid.py @@ -0,0 +1,26 @@ +import logging +log = logging.getLogger(__name__) + + +class DjangoGrid(object): + + def populate(self, indata, config=None): + for required in ('row_count', 'col_count', 'headers'): + if not hasattr(self, required): + raise Exception("missing required attribute to Grid: %s" % required) + # create a grid + self.grid_data = [[None] * self.col_count for i in range(self.row_count)] + + # now populate + # this is pass one + # want to do as much processing here as we can + # we populate left to right, top to bottom + for row_id in range(self.row_count): + row = indata[row_id] + for col_id in range(self.col_count): + field_type_class = self.headers[col_id] + data = getattr(row, self.titles[col_id]) + self.grid_data[row_id][col_id] = field_type_class(data) + + #Ilport pdb; pdb.set_trace() + log.info("populated grid %sX%s" % (self.row_count, self.col_count)) diff --git a/mm3/contrib/prettytable/__init__.py b/mm3/contrib/prettytable/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mm3/contrib/prettytable/composers.py b/mm3/contrib/prettytable/composers.py new file mode 100644 index 0000000..f97fb59 --- /dev/null +++ b/mm3/contrib/prettytable/composers.py @@ -0,0 +1,34 @@ +from mm.composer_base import ComposerBase +import logging +log = logging.getLogger(__name__) +pretty_table = False +try: + + import prettytable # NOQA + pretty_table = True +except ImportError: + pass + + +class ComposerPrettyTable(ComposerBase): + + def write_header(self): + self.pt.field_names = self.data_model.field_titles + + def row(self, row): + self.pt.add_row([cell.data for cell in row]) + + def run(self, child=None): + if not pretty_table: + raise Exception("Module 'prettytable' required for text table output") + self.pt = prettytable.PrettyTable() + if self.document.config.headers: + self.write_header() + self.iterate_grid() + self.finish() + + # process any childern + for doc_child in self.document.children: + doc_child.writestr(child=self.pt) + + return self.pt diff --git a/mm3/document_base.py b/mm3/document_base.py new file mode 100644 index 0000000..3c11b20 --- /dev/null +++ b/mm3/document_base.py @@ -0,0 +1,86 @@ +from .document_writers import DocumentWriter +from .model_base import DataModel +from .config_base import ConfigBase +from .serializer_base import Serializer +from .grid_base import GridBase +import logging + +log = logging.getLogger(__name__) + + +class Document(DocumentWriter): + """ + Document reporesents the abstact view you interact with in order to send + data for your document and ultimately get output. + """ + def __init__( + self, + data, + data_model_class=None, + grid_class=None, + serializer_class=None, + config=None, + config_dict=None, + order=None, + column_types=None): + """ + data -- a dict or a list of data you wish to use for a the + spreadsheet + data_model -- (optional) fields defenitions + grid_model -- (optional) takes data_model and data and fills grid + serializer_class -- (optional) class to use to serialize raw data + config -- (optional) Configuration (ConfigBase) instance + config_dict -- (optional) a dictionary of key/values of settings + order -- (optional) also headers + column_types -- (optional) a dictionary of column types; e.g. column_name1 is a date column: {'column_name1': mm.Date) + + """ + self.data = data + self.config = config + self.name = None + self.children = [] + if not self.config: + self.config = ConfigBase() + if config_dict: + self.config.set_dict(config_dict) + + # make a data model if one does not exist + self.data_model_class = data_model_class + if not data_model_class: + self.data_model_class = DataModel + + self.data_model = self.data_model_class(data, order=order, column_types=column_types) + + # grid base + if not grid_class: + grid_class = GridBase + + # Serialize the data + # we look at it here once and only once + # we look at it again when we write + # goal to pass over data no more than twice, if possible + if not serializer_class: + serializer_class = Serializer + serializer = serializer_class( + self.data_model, + self.data, + self.config, + grid_class=grid_class + ) + + # returns a grid instance + self.grid = serializer.serialize() + + log.info("Documnet Created") + + def set_composer_class(self, composer_class): + self.composer_class = composer_class + + def set_composer(self, composer): + self.composer = composer + + def set_name(self, name): + self.name = name + + def add_child(self, document): + self.children.append(document) diff --git a/mm3/document_writers.py b/mm3/document_writers.py new file mode 100644 index 0000000..416e41a --- /dev/null +++ b/mm3/document_writers.py @@ -0,0 +1,69 @@ +from .composer_xls import ComposerXLS +from mm.contrib.prettytable.composers import ComposerPrettyTable, pretty_table + +import os +import tempfile +import logging + +log = logging.getLogger(__name__) + + +class DocumentWriter(object): + "runs a composer" + + composer_class = None + composer = None + + def writestr(self, child=False): + composer_class = self.composer_class + if not composer_class: + # default format is XLS + composer_class = ComposerXLS + log.info("Setting output format to XLS") + self.composer = composer_class(self.data_model, self.grid, self) + return self.composer.run(child=child) + + def write(self, filename): + ext = os.path.splitext(filename)[-1].lower() + if ext == "xls": + self.composer = ComposerXLS(self.data_model, self.grid, self) + log.info("Setting output format to XLS, based on file extension") + elif ext == "txt" and pretty_table: + self.composer = ComposerPrettyTable(self.data_model, self.grid, self) + log.info("Setting output format to TXT, based on file extension") + + with open(filename, "wb") as f: + f.write(self.writestr()) + + log.info("wrote file: %s" % filename) + + def write_gdata(self, name, username, password, auth_token=None): + try: + import gdata + import gdata.docs.service + except ImportError: + raise Exception("Must install package 'gdata' to use write_gdata()") + + tmp_file, tmp_file_path = tempfile.mkstemp() + self.write(tmp_file_path) + + gd_client = gdata.docs.service.DocsService() + gd_client.ssl = True + if not auth_token: + gd_client.ClientLogin( + username, + password, + "marmir-1.0") + else: + #TODO: use the token + raise Exception("oauth not yet supported") + + ms = gdata.MediaSource( + file_path=tmp_file_path, + content_type='application/vnd.ms-excel') + entry = gd_client.Upload(ms, name) # NOQA + + #cleanup + os.unlink(tmp_file_path) + + return gd_client.GetClientLoginToken() diff --git a/mm3/grid_base.py b/mm3/grid_base.py new file mode 100644 index 0000000..66959f5 --- /dev/null +++ b/mm3/grid_base.py @@ -0,0 +1,83 @@ +import logging +from .model_base import is_custom_mm_type +log = logging.getLogger(__name__) + + +class GridBase(object): + + def populate(self, indata, config): + for required in ('row_count', 'col_count', 'headers', 'titles'): + if not hasattr(self, required): + raise Exception("missing required attribute to Grid: %s" % + required) + # create a grid + self.grid_data = [[None] * self.col_count for i in range(self.row_count)] + + using_lists = False # support for lists #2 + if type(indata[0]) != dict: + using_lists = True + + # now populate + # this is pass one + # want to do as much processing here as we can + # we populate left to right, top to bottom + n_missing = 0 + for row_id in range(self.row_count): + for col_id in range(self.col_count): + field_type_class = self.headers[col_id] + + # headers from seelf.data_model.field_headers, sorted + if using_lists: + try: + # direct data access lists + data = indata[row_id][col_id] + except IndexError: + log.warning('No index found in row %d column %d' % + (row_id, col_id)) + if config.INGORE_DATA_MISMATCH: + data = '' + n_missing += 1 + + else: + if len(indata[row_id]) > self.col_count: + raise Exception("Data mismatch: Row %d has %d more columns than row 1" % + ((row_id + 1), len(indata[row_id])-self.col_count)) + try: + #direct data access dicts + data = indata[row_id][self.titles[col_id]] + except IndexError: + log.warning('No index found in row %d column %d' % + (row_id, col_id)) + n_missing += 1 + if config.INGORE_DATA_MISMATCH: + data = '' + except KeyError: + log.warning('No key found in row %d column %d' % + (row_id, col_id)) + n_missing += 1 + if config.INGORE_DATA_MISMATCH: + data = '' + + if is_custom_mm_type(data): + # explicit type + try: + self.grid_data[row_id][col_id] = data + except IndexError: + log.warning('No index found in row %d column %d' % + (row_id, col_id)) + n_missing += 1 + if config.INGORE_DATA_MISMATCH: + data = '' + else: + # wrap in type from headers + try: + self.grid_data[row_id][col_id] = field_type_class(data) + except IndexError: + log.warning('No index found in row %d column %d' % + (row_id, col_id)) + n_missing += 1 + if config.INGORE_DATA_MISMATCH: + data = '' + log.info("populated grid %sX%s" % (self.row_count, self.col_count)) + if n_missing > 0: + log.info('%d missing items' % n_missing) diff --git a/mm3/lib/__init__.py b/mm3/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mm3/lib/font_data/__init__.py b/mm3/lib/font_data/__init__.py new file mode 100644 index 0000000..fd40910 --- /dev/null +++ b/mm3/lib/font_data/__init__.py @@ -0,0 +1,4 @@ + + + + diff --git a/mm3/lib/font_data/core.py b/mm3/lib/font_data/core.py new file mode 100644 index 0000000..6fb8453 --- /dev/null +++ b/mm3/lib/font_data/core.py @@ -0,0 +1,58 @@ +import os.path as path +from .decorators import memoized +try: + import pickle as pickle +except: + import pickle + +LATEST_FONT_DATA = "font_data_ms_fonts.bin" + +@memoized +def get_font_data(): + this_file_path = path.abspath(__file__) + data_path = path.join( path.dirname(this_file_path), LATEST_FONT_DATA) + pkl_file = open(data_path, 'rb') + FONT_DATA = pickle.load(pkl_file) + pkl_file.close() + return FONT_DATA + + +@memoized +def get_character_data(font_name, char): + font_data = get_font_data() + font_name = font_name.replace("-","_").replace(" ","_") + if font_name not in font_data: + font_name = font_name.capitalize() + if font_name not in font_data: + raise Exception("no font data for font %s" % font_name) + font_set = font_data[font_name] + if char not in font_set['values']: + return font_set['default_width'], {} # empty kerns + return font_set['values'][char] + +@memoized +def get_character_width(font_name, char): + width, kerns = get_character_data(font_name, char) + return width + +@memoized +def get_kern_offset(font_name, char1, char2): + width, kerns = get_character_data(font_name, char1) + if char2 in kerns: + return kerns[char2] + return 0 + +@memoized +def get_string_width(font_name, point_size, char_string): + out_width_256 = 0 + current_pos = 0 + str_length = len(char_string) + for char in char_string: + out_width_256 += get_character_width(font_name, char) + if current_pos != (str_length-1): + out_width_256 += get_kern_offset(font_name, char, char_string[current_pos+1]) + current_pos += 1 + return out_width_256 * ( point_size / 256.0 ) + +if __name__ == "__main__": + get_character_data('Arial', 'A') diff --git a/mm3/lib/font_data/decorators.py b/mm3/lib/font_data/decorators.py new file mode 100644 index 0000000..ed18d2c --- /dev/null +++ b/mm3/lib/font_data/decorators.py @@ -0,0 +1,29 @@ + +import collections +import functools + +class memoized(object): + '''Decorator. Caches a function's return value each time it is called. + If called later with the same arguments, the cached value is returned + (not reevaluated). + ''' + def __init__(self, func): + self.func = func + self.cache = {} + def __call__(self, *args): + if not isinstance(args, collections.Hashable): + # uncacheable. a list, for instance. + # better to not cache than blow up. + return self.func(*args) + if args in self.cache: + return self.cache[args] + else: + value = self.func(*args) + self.cache[args] = value + return value + def __repr__(self): + '''Return the function's docstring.''' + return self.func.__doc__ + def __get__(self, obj, objtype): + '''Support instance methods.''' + return functools.partial(self.__call__, obj) diff --git a/mm3/lib/font_data/tests.py b/mm3/lib/font_data/tests.py new file mode 100644 index 0000000..10dfe91 --- /dev/null +++ b/mm3/lib/font_data/tests.py @@ -0,0 +1,18 @@ +import unittest +from . import core + + +class test_font_data(unittest.TestCase): + + def test_checkwidth1(self): + + width = core.get_string_width('Arial', 11, 'hello world') + self.assertEqual(width, 52.421875) + + def test_checkwidth2(self): + width = core.get_string_width('Times New Roman', 23 ,'The quick brown fox jumps over the lazy dog') + self.assertEqual(width, 420.46875) + + + + diff --git a/mm3/lib/xldate/__init__.py b/mm3/lib/xldate/__init__.py new file mode 100644 index 0000000..139597f --- /dev/null +++ b/mm3/lib/xldate/__init__.py @@ -0,0 +1,2 @@ + + diff --git a/mm3/lib/xldate/convert.py b/mm3/lib/xldate/convert.py new file mode 100644 index 0000000..464bf36 --- /dev/null +++ b/mm3/lib/xldate/convert.py @@ -0,0 +1,61 @@ +import re + + +class UnsupportedFormatCodeException(Exception): + pass + +def to_excel_from_C_codes(cdate_str, config): + """ + ref http://office.microsoft.com/en-us/excel-help/create-a-custom-number-format-HP010342372.aspx + and http://docs.python.org/2/library/datetime.html + + """ + pairs = ( + + ('%a', None, "abbreviated weekday names require special function in excel"), # not supported + ('%A', None, "weeday naes require a special funtion in excel"), # " + ('%b', 'mmm'), # month as an abbreviation (Jan to Dec). + ('%B', 'mmmm'), # month as a full name (January to December) + ('%c', config.get('datetime_format', 'M/D/YY h:mm:ss') ), # date and time representation. + ('%d', 'dd'), + ('%f', '[ss].00'), # Microsecond as a decimal number [0,999999], zero-padded on the left || Elapsed time (seconds and hundredths) 3735.80 [ss].00 + ('%H', 'hh'), # Hours 00-23 hh + ('%I', None, "AM or PM required for 12 hour clock"), # AM or PM required + ('%j', None, "Day of year not supported in Excel"), + ('%m', 'mm'), + ('%M', 'mm'), + ('%p', 'AM/PM'), # + ('%S', 'ss'), + ('%U', None, "Excel has no support for week number"), + ('%w', None, "Excel has no support for week day"), + ('%W', None, "Excel has no support for Week Number of year"), + ('%x', config.get('datetime_format', 'M/D/YY') ), # date + ('%X', config.get('time_format', 'h:mm:ss') ), # time + ('%y', 'yy'), # year as a two-digit number. + ('%Y', 'yyyy'), # year as a four-digit number. + ('%z', None, "Excel has no time zone support"), + ('%Z', None, "Excel has no time zone support"), + ('%%', r'\%'), +) + + original_str = cdate_str + for t in pairs: + if not t[1] and cdate_str.find(t[0]) > -1: + reason = "Excel does not support" + if len(t) > 2: + reason = t[2] + raise UnsupportedFormatCodeException("Could not replace %s (%s) found in %s" % (t[0], + reason, + original_str)) + elif not t[1]: + continue + + cdate_str = re.sub(t[0], t[1], cdate_str) + + return cdate_str + + + + + + diff --git a/mm3/lib/xldate/tests.py b/mm3/lib/xldate/tests.py new file mode 100644 index 0000000..7f3786b --- /dev/null +++ b/mm3/lib/xldate/tests.py @@ -0,0 +1,27 @@ +import unittest +from . import convert + + +class TestsConvert(unittest.TestCase): + + def test_formats(self): + tests = ( + ("%b %d %H:%M:%S %Y", "mmm dd hh:mm:ss yyyy"), # Jul 08 08:08:10 2011 + ('%b|%B|%c|%d|%f|%H|%m|%M|%p|%S|%x|%X|%y|%Y|%%','mmm|mmmm|M/D/YY h:mm:ss|dd|[ss].00|hh|mm|mm|AM/PM|ss|M/D/YY|h:mm:ss|yy|yyyy|\%'), + ) + for test in tests: + excel = convert.to_excel_from_C_codes(test[0],{}) + self.assertEqual(test[1], excel) + + + tests = ( + ('%a', '%A', '%I', '%j', '%U', '%w', '%W', '%z', '%Z',) + ) + for test in tests: + self.assertRaises(convert.UnsupportedFormatCodeException, + convert.to_excel_from_C_codes, test, {}) + +if __name__ == "__main__": + unittest.main() + + diff --git a/mm3/lib/xlwt_0_7_2/BIFFRecords.py b/mm3/lib/xlwt_0_7_2/BIFFRecords.py new file mode 100644 index 0000000..a31a458 --- /dev/null +++ b/mm3/lib/xlwt_0_7_2/BIFFRecords.py @@ -0,0 +1,2397 @@ +# -*- coding: cp1252 -*- +from struct import pack +from .UnicodeUtils import upack1, upack2 +import sys + +class SharedStringTable(object): + _SST_ID = 0x00FC + _CONTINUE_ID = 0x003C + + def __init__(self, encoding): + self.encoding = encoding + self._str_indexes = {} + self._tally = [] + self._add_calls = 0 + # Following 3 attrs are used for temporary storage in the + # get_biff_record() method and methods called by it. The pseudo- + # initialisation here is for documentation purposes only. + self._sst_record = None + self._continues = None + self._current_piece = None + + def add_str(self, s): + if self.encoding != 'ascii' and not isinstance(s, str): + s = str(s, self.encoding) + self._add_calls += 1 + if s not in self._str_indexes: + idx = len(self._str_indexes) + self._str_indexes[s] = idx + self._tally.append(1) + else: + idx = self._str_indexes[s] + self._tally[idx] += 1 + return idx + + def del_str(self, idx): + # This is called when we are replacing the contents of a string cell. + assert self._tally[idx] > 0 + self._tally[idx] -= 1 + self._add_calls -= 1 + + def str_index(self, s): + return self._str_indexes[s] + + def get_biff_record(self): + self._sst_record = '' + self._continues = [None, None] + self._current_piece = pack(' 0x2020: # limit for BIFF7/8 + chunks = [] + pos = 0 + while pos < len(data): + chunk_pos = pos + 0x2020 + chunk = data[pos:chunk_pos] + chunks.append(chunk) + pos = chunk_pos + continues = pack('<2H', self._REC_ID, len(chunks[0])) + chunks[0] + for chunk in chunks[1:]: + continues += pack('<2H%ds'%len(chunk), 0x003C, len(chunk), chunk) + # 0x003C -- CONTINUE record id + return continues + else: + return self.get_rec_header() + data + + +class Biff8BOFRecord(BiffRecord): + """ + Offset Size Contents + 0 2 Version, contains 0600H for BIFF8 and BIFF8X + 2 2 Type of the following data: + 0005H = Workbook globals + 0006H = Visual Basic module + 0010H = Worksheet + 0020H = Chart + 0040H = Macro sheet + 0100H = Workspace file + 4 2 Build identifier + 6 2 Build year + 8 4 File history flags + 12 4 Lowest Excel version that can read all records in this file + """ + _REC_ID = 0x0809 + # stream types + BOOK_GLOBAL = 0x0005 + VB_MODULE = 0x0006 + WORKSHEET = 0x0010 + CHART = 0x0020 + MACROSHEET = 0x0040 + WORKSPACE = 0x0100 + + def __init__(self, rec_type): + version = 0x0600 + build = 0x0DBB + year = 0x07CC + file_hist_flags = 0x00 + ver_can_read = 0x06 + + self._rec_data = pack('<4H2I', version, rec_type, build, year, file_hist_flags, ver_can_read) + + +class InteraceHdrRecord(BiffRecord): + _REC_ID = 0x00E1 + + def __init__(self): + self._rec_data = pack('BB', 0xB0, 0x04) + + +class InteraceEndRecord(BiffRecord): + _REC_ID = 0x00E2 + + def __init__(self): + self._rec_data = '' + + +class MMSRecord(BiffRecord): + _REC_ID = 0x00C1 + + def __init__(self): + self._rec_data = pack('> 15 + c = low_15 | high_15 + passwd_hash ^= c + passwd_hash ^= len(plaintext) + passwd_hash ^= 0xCE4B + return passwd_hash + + def __init__(self, passwd = ""): + self._rec_data = pack('