From 6ada4d468a6b33e2f6af8a3c0af4fc71ca4cc603 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:27:32 -0500 Subject: [PATCH 01/76] Add basic output formatter class functionality. --- mycli/main.py | 111 ++-- mycli/myclirc | 2 +- mycli/packages/expanded.py | 2 +- mycli/packages/tabulate.py | 1078 ------------------------------------ mycli/sqlcompleter.py | 11 +- setup.py | 1 + tests/test_main.py | 28 +- tests/test_tabulate.py | 22 - tests/utils.py | 17 +- 9 files changed, 81 insertions(+), 1191 deletions(-) delete mode 100644 mycli/packages/tabulate.py delete mode 100644 tests/test_tabulate.py diff --git a/mycli/main.py b/mycli/main.py index ba344395..b9a3cebe 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -35,8 +35,6 @@ from pygments.token import Token from configobj import ConfigObj, ConfigObjError -from .packages.tabulate import tabulate, table_formats -from .packages.expanded import expanded_table from .packages.special.main import (COMMANDS, NO_QUERY) import mycli.packages.special as special from .sqlcompleter import SQLCompleter @@ -49,6 +47,7 @@ open_mylogin_cnf, read_config_file, read_config_files, str_to_bool) from .key_bindings import mycli_bindings +from .output_formatter import OutputFormatter from .encodingutils import utf8tounicode from .lexer import MyCliLexer from .__init__ import __version__ @@ -144,6 +143,8 @@ def __init__(self, sqlexecute=None, prompt=None, self.completion_refresher = CompletionRefresher() + self.formatter = OutputFormatter() + self.logger = logging.getLogger(__name__) self.initialize_logging() @@ -191,9 +192,9 @@ def register_special_commands(self): '\\R', 'Change prompt format.', aliases=('\\R',), case_sensitive=True) def change_table_format(self, arg, **_): - if not arg in table_formats(): + if not arg in self.formatter.supported_formats(): msg = "Table type %s not yet implemented. Allowed types:" % arg - for table_type in table_formats(): + for table_type in self.formatter.supported_formats(): msg += "\n\t%s" % table_type yield (None, None, None, msg) else: @@ -533,9 +534,8 @@ def one_iteration(document=None): else: max_width = None - formatted = format_output(title, cur, headers, - status, self.table_format, - special.is_expanded_output(), max_width) + formatted = self.format_output(title, cur, headers, + status, special.is_expanded_output(), max_width) output.extend(formatted) end = time() @@ -715,15 +715,55 @@ def get_prompt(self, string): string = string.replace('\\_', ' ') return string - def run_query(self, query, table_format=None, new_line=True): + def run_query(self, query, new_line=True): """Runs query""" results = self.sqlexecute.run(query) for result in results: title, cur, headers, status = result - output = format_output(title, cur, headers, None, table_format) + output = self.format_output(title, cur, headers, None) for line in output: click.echo(line, nl=new_line) + def format_output(self, title, cur, headers, status, expanded=False, + max_width=None): + output = [] + if title: # Only print the title if it's not None. + output.append(title) + if cur: + headers = [utf8tounicode(x) for x in headers] + + if expanded: + output.append(self.formatter.format_output(cur, headers, + 'expanded')) + elif self.table_format == 'csv': + content = StringIO() + writer = csv.writer(content) + writer.writerow(headers) + + for row in cur: + row = ['null' if val is None else str(val) for val in row] + writer.writerow(row) + + output.append(content.getvalue()) + content.close() + else: + rows = list(cur) + formatted = self.formatter.format_output(rows, headers, + self.table_format) + if (self.table_format != 'expanded' and + max_width and rows and + content_exceeds_width(rows[0], max_width) and + headers): + output.append(self.formatter.format_output(cur, headers, + 'expanded')) + else: + output.append(formatted) + if status: # Only print the status if it's not None. + output.append(status) + + return output + + @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') @click.option('-P', '--port', envvar='MYSQL_TCP_PORT', type=int, help='Port number to use for connection. Honors ' @@ -818,12 +858,11 @@ def cli(database, user, host, port, socket, password, dbname, # --execute argument if execute: try: - table_format = None - if table: - table_format = mycli.table_format - elif csv: + table_format = 'tsv' + if csv: table_format = 'csv' - mycli.run_query(execute, table_format=table_format) + mycli.table_format = table_format or mycli.table_format + mycli.run_query(execute) exit(0) except Exception as e: click.secho(str(e), err=True, fg='red') @@ -844,58 +883,22 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - table_format = None + table_format = 'tsv' new_line = True if csv: table_format = 'csv' new_line = False - elif table: - table_format = mycli.table_format - mycli.run_query(stdin_text, table_format=table_format, new_line=new_line) + mycli.table_format = table_format or mycli.table_format + + mycli.run_query(stdin_text, new_line=new_line) exit(0) except Exception as e: click.secho(str(e), err=True, fg='red') exit(1) -def format_output(title, cur, headers, status, table_format, expanded=False, max_width=None): - output = [] - if title: # Only print the title if it's not None. - output.append(title) - if cur: - headers = [utf8tounicode(x) for x in headers] - table_format = 'tsv' if table_format is None else table_format - - if expanded: - output.append(expanded_table(cur, headers)) - elif table_format == 'csv': - content = StringIO() - writer = csv.writer(content) - writer.writerow(headers) - - for row in cur: - row = ['null' if val is None else str(val) for val in row] - writer.writerow(row) - - output.append(content.getvalue()) - content.close() - else: - rows = list(cur) - tabulated, frows = tabulate(rows, headers, tablefmt=table_format, - missingval='') - if (max_width and rows and - content_exceeds_width(frows[0], max_width) and - headers): - output.append(expanded_table(rows, headers)) - else: - output.append(tabulated) - if status: # Only print the status if it's not None. - output.append(status) - - return output - def content_exceeds_width(row, width): # Account for 3 characters between each column separator_space = (len(row)*3) diff --git a/mycli/myclirc b/mycli/myclirc index baf50510..ff6e8f73 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -66,7 +66,7 @@ less_chatty = False login_path_as_host = False # Cause result sets to be displayed vertically if they are too wide for the current window, -# and using normal tabular format otherwise. (This applies to statements terminated by ; or \G.) +# and using normal tabular format otherwise. (This applies to statements terminated by ; or \G.) auto_vertical_output = False # Custom colors for the completion menu, toolbar, etc. diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 128e9c69..245e4ca7 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,4 +1,4 @@ -from .tabulate import _text_type +from tabulate import _text_type import binascii def pad(field, total, char=u" "): diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py deleted file mode 100644 index fa971826..00000000 --- a/mycli/packages/tabulate.py +++ /dev/null @@ -1,1078 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Pretty-print tabular data.""" - -from __future__ import print_function -from __future__ import unicode_literals -from collections import namedtuple -from decimal import Decimal -from platform import python_version_tuple -from wcwidth import wcswidth -import re -import binascii - - -if python_version_tuple()[0] < "3": - from itertools import izip_longest - from functools import partial - _none_type = type(None) - _int_type = int - _long_type = long - _float_type = float - _text_type = unicode - _binary_type = str - - def _is_file(f): - return isinstance(f, file) - -else: - from itertools import zip_longest as izip_longest - from functools import reduce, partial - _none_type = type(None) - _int_type = int - _long_type = int - _float_type = float - _text_type = str - _binary_type = bytes - - import io - def _is_file(f): - return isinstance(f, io.IOBase) - - -__all__ = ["tabulate", "tabulate_formats", "simple_separated_format"] -__version__ = "0.7.4" - - -MIN_PADDING = 2 - - -Line = namedtuple("Line", ["begin", "hline", "sep", "end"]) - - -DataRow = namedtuple("DataRow", ["begin", "sep", "end"]) - - -# A table structure is suppposed to be: -# -# --- lineabove --------- -# headerrow -# --- linebelowheader --- -# datarow -# --- linebewteenrows --- -# ... (more datarows) ... -# --- linebewteenrows --- -# last datarow -# --- linebelow --------- -# -# TableFormat's line* elements can be -# -# - either None, if the element is not used, -# - or a Line tuple, -# - or a function: [col_widths], [col_alignments] -> string. -# -# TableFormat's *row elements can be -# -# - either None, if the element is not used, -# - or a DataRow tuple, -# - or a function: [cell_values], [col_widths], [col_alignments] -> string. -# -# padding (an integer) is the amount of white space around data values. -# -# with_header_hide: -# -# - either None, to display all table elements unconditionally, -# - or a list of elements not to be displayed if the table has column headers. -# -TableFormat = namedtuple("TableFormat", ["lineabove", "linebelowheader", - "linebetweenrows", "linebelow", - "headerrow", "datarow", - "padding", "with_header_hide", "with_align"]) - - -def _pipe_segment_with_colons(align, colwidth): - """Return a segment of a horizontal line with optional colons which - indicate column's alignment (as in `pipe` output format).""" - w = colwidth - if align in ["right", "decimal"]: - return ('-' * (w - 1)) + ":" - elif align == "center": - return ":" + ('-' * (w - 2)) + ":" - elif align == "left": - return ":" + ('-' * (w - 1)) - else: - return '-' * w - - -def _pipe_line_with_colons(colwidths, colaligns): - """Return a horizontal line with optional colons to indicate column's - alignment (as in `pipe` output format).""" - segments = [_pipe_segment_with_colons(a, w) for a, w in zip(colaligns, colwidths)] - return "|" + "|".join(segments) + "|" - - -def _mediawiki_row_with_attrs(separator, cell_values, colwidths, colaligns): - alignment = { "left": '', - "right": 'align="right"| ', - "center": 'align="center"| ', - "decimal": 'align="right"| ' } - # hard-coded padding _around_ align attribute and value together - # rather than padding parameter which affects only the value - values_with_attrs = [' ' + alignment.get(a, '') + c + ' ' - for c, a in zip(cell_values, colaligns)] - colsep = separator*2 - return (separator + colsep.join(values_with_attrs)).rstrip() - - -def _html_row_with_attrs(celltag, cell_values, colwidths, colaligns): - alignment = { "left": '', - "right": ' style="text-align: right;"', - "center": ' style="text-align: center;"', - "decimal": ' style="text-align: right;"' } - values_with_attrs = ["<{0}{1}>{2}".format(celltag, alignment.get(a, ''), c) - for c, a in zip(cell_values, colaligns)] - return "" + "".join(values_with_attrs).rstrip() + "" - - -def _latex_line_begin_tabular(colwidths, colaligns, booktabs=False): - alignment = { "left": "l", "right": "r", "center": "c", "decimal": "r" } - tabular_columns_fmt = "".join([alignment.get(a, "l") for a in colaligns]) - return "\n".join(["\\begin{tabular}{" + tabular_columns_fmt + "}", - "\\toprule" if booktabs else "\hline"]) - -LATEX_ESCAPE_RULES = {r"&": r"\&", r"%": r"\%", r"$": r"\$", r"#": r"\#", - r"_": r"\_", r"^": r"\^{}", r"{": r"\{", r"}": r"\}", - r"~": r"\textasciitilde{}", "\\": r"\textbackslash{}", - r"<": r"\ensuremath{<}", r">": r"\ensuremath{>}"} - - -def _latex_row(cell_values, colwidths, colaligns): - def escape_char(c): - return LATEX_ESCAPE_RULES.get(c, c) - escaped_values = ["".join(map(escape_char, cell)) for cell in cell_values] - rowfmt = DataRow("", "&", "\\\\") - return _build_simple_row(escaped_values, rowfmt) - - -_table_formats = {"simple": - TableFormat(lineabove=Line("", "-", " ", ""), - linebelowheader=Line("", "-", " ", ""), - linebetweenrows=None, - linebelow=Line("", "-", " ", ""), - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, - with_header_hide=["lineabove", "linebelow"], with_align=True), - "plain": - TableFormat(lineabove=None, linebelowheader=None, - linebetweenrows=None, linebelow=None, - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None, with_align=True), - "grid": - TableFormat(lineabove=Line("+", "-", "+", "+"), - linebelowheader=Line("+", "=", "+", "+"), - linebetweenrows=Line("+", "-", "+", "+"), - linebelow=Line("+", "-", "+", "+"), - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None, with_align=True), - "fancy_grid": - TableFormat(lineabove=Line("╒", "═", "╤", "╕"), - linebelowheader=Line("╞", "═", "╪", "╡"), - linebetweenrows=Line("├", "─", "┼", "┤"), - linebelow=Line("╘", "═", "╧", "╛"), - headerrow=DataRow("│", "│", "│"), - datarow=DataRow("│", "│", "│"), - padding=1, with_header_hide=None, with_align=True), - "pipe": - TableFormat(lineabove=_pipe_line_with_colons, - linebelowheader=_pipe_line_with_colons, - linebetweenrows=None, - linebelow=None, - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, - with_header_hide=["lineabove"], with_align=True), - "orgtbl": - TableFormat(lineabove=None, - linebelowheader=Line("|", "-", "+", "|"), - linebetweenrows=None, - linebelow=None, - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None, with_align=True), - "psql": - TableFormat(lineabove=Line("+", "-", "+", "+"), - linebelowheader=Line("|", "-", "+", "|"), - linebetweenrows=None, - linebelow=Line("+", "-", "+", "+"), - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None, with_align=True), - "rst": - TableFormat(lineabove=Line("", "=", " ", ""), - linebelowheader=Line("", "=", " ", ""), - linebetweenrows=None, - linebelow=Line("", "=", " ", ""), - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None, with_align=True), - "mediawiki": - TableFormat(lineabove=Line("{| class=\"wikitable\" style=\"text-align: left;\"", - "", "", "\n|+ \n|-"), - linebelowheader=Line("|-", "", "", ""), - linebetweenrows=Line("|-", "", "", ""), - linebelow=Line("|}", "", "", ""), - headerrow=partial(_mediawiki_row_with_attrs, "!"), - datarow=partial(_mediawiki_row_with_attrs, "|"), - padding=0, with_header_hide=None, with_align=True), - "html": - TableFormat(lineabove=Line("", "", "", ""), - linebelowheader=None, - linebetweenrows=None, - linebelow=Line("
", "", "", ""), - headerrow=partial(_html_row_with_attrs, "th"), - datarow=partial(_html_row_with_attrs, "td"), - padding=0, with_header_hide=None, with_align=False), - "latex": - TableFormat(lineabove=_latex_line_begin_tabular, - linebelowheader=Line("\\hline", "", "", ""), - linebetweenrows=None, - linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), - headerrow=_latex_row, - datarow=_latex_row, - padding=1, with_header_hide=None, with_align=False), - "latex_booktabs": - TableFormat(lineabove=partial(_latex_line_begin_tabular, booktabs=True), - linebelowheader=Line("\\midrule", "", "", ""), - linebetweenrows=None, - linebelow=Line("\\bottomrule\n\\end{tabular}", "", "", ""), - headerrow=_latex_row, - datarow=_latex_row, - padding=1, with_header_hide=None, with_align=False), - "tsv": - TableFormat(lineabove=None, linebelowheader=None, - linebetweenrows=None, linebelow=None, - headerrow=DataRow("", "\t", ""), - datarow=DataRow("", "\t", ""), - padding=0, with_header_hide=None, with_align=False)} - - -tabulate_formats = list(sorted(_table_formats.keys())) - - -_invisible_codes = re.compile(r"\x1b\[\d*m|\x1b\[\d*\;\d*\;\d*m") # ANSI color codes -_invisible_codes_bytes = re.compile(b"\x1b\[\d*m|\x1b\[\d*\;\d*\;\d*m") # ANSI color codes - - -def simple_separated_format(separator): - """Construct a simple TableFormat with columns separated by a separator. - - >>> tsv = simple_separated_format("\\t") ; \ - print(tabulate([["foo", 1], ["spam", 23]], tablefmt=tsv)[0].replace('\\t', r'\\t')) - foo\\t1 - spam\\t23 - """ - return TableFormat(None, None, None, None, - headerrow=DataRow('', separator, ''), - datarow=DataRow('', separator, ''), - padding=0, with_header_hide=None, - with_align=False) - - -def _isconvertible(conv, string): - try: - n = conv(string) - return True - except (ValueError, TypeError): - return False - - -def _isnumber(string): - """ - >>> _isnumber("123.45") - True - >>> _isnumber("123") - True - >>> _isnumber("spam") - False - """ - return _isconvertible(float, string) - - -def _isint(string): - """ - >>> _isint("123") - True - >>> _isint("123.45") - False - """ - return type(string) is _int_type or type(string) is _long_type or \ - (isinstance(string, _binary_type) or isinstance(string, _text_type)) and \ - _isconvertible(int, string) - - -def _type(string, has_invisible=True): - """The least generic type (type(None), int, float, str, unicode). - - >>> _type(None) is type(None) - True - >>> _type("foo") is type("") - True - >>> _type("1") is type(1) - True - >>> _type('\x1b[31m42\x1b[0m') is type(42) - True - >>> _type('\x1b[31m42\x1b[0m') is type(42) - True - - """ - - if has_invisible and \ - (isinstance(string, _text_type) or isinstance(string, _binary_type)): - string = _strip_invisible(string) - - if string is None: - return _none_type - if isinstance(string, (bool, Decimal,)): - return _text_type - elif hasattr(string, "isoformat"): # datetime.datetime, date, and time - return _text_type - elif _isint(string): - return int - elif _isnumber(string): - return float - elif isinstance(string, _binary_type): - return _binary_type - else: - return _text_type - - -def _afterpoint(string): - """Symbols after a decimal point, -1 if the string lacks the decimal point. - - >>> _afterpoint("123.45") - 2 - >>> _afterpoint("1001") - -1 - >>> _afterpoint("eggs") - -1 - >>> _afterpoint("123e45") - 2 - - """ - if _isnumber(string): - if _isint(string): - return -1 - else: - pos = string.rfind(".") - pos = string.lower().rfind("e") if pos < 0 else pos - if pos >= 0: - return len(string) - pos - 1 - else: - return -1 # no point - else: - return -1 # not a number - - -def _padleft(width, s, has_invisible=True): - """Flush right. - - >>> _padleft(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430' - True - - """ - lwidth = width - wcswidth(_strip_invisible(s) if has_invisible else s) - return ' ' * lwidth + s - - -def _padright(width, s, has_invisible=True): - """Flush left. - - >>> _padright(6, '\u044f\u0439\u0446\u0430') == '\u044f\u0439\u0446\u0430 ' - True - - """ - rwidth = width - wcswidth(_strip_invisible(s) if has_invisible else s) - return s + ' ' * rwidth - - -def _padboth(width, s, has_invisible=True): - """Center string. - - >>> _padboth(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430 ' - True - - """ - xwidth = width - wcswidth(_strip_invisible(s) if has_invisible else s) - lwidth = xwidth // 2 - rwidth = 0 if xwidth <= 0 else lwidth + xwidth % 2 - return ' ' * lwidth + s + ' ' * rwidth - - -def _strip_invisible(s): - "Remove invisible ANSI color codes." - if isinstance(s, _text_type): - return re.sub(_invisible_codes, "", s) - else: # a bytestring - return re.sub(_invisible_codes_bytes, "", s) - - -def _visible_width(s): - """Visible width of a printed string. ANSI color codes are removed. - - >>> _visible_width('\x1b[31mhello\x1b[0m'), _visible_width("world") - (5, 5) - - """ - if isinstance(s, _text_type) or isinstance(s, _binary_type): - return wcswidth(_strip_invisible(s)) - else: - return wcswidth(_text_type(s)) - - -def _align_column(strings, alignment, minwidth=0, has_invisible=True): - """[string] -> [padded_string] - - >>> list(map(str,_align_column(["12.345", "-1234.5", "1.23", "1234.5", "1e+234", "1.0e234"], "decimal"))) - [' 12.345 ', '-1234.5 ', ' 1.23 ', ' 1234.5 ', ' 1e+234 ', ' 1.0e234'] - - >>> list(map(str,_align_column(['123.4', '56.7890'], None))) - ['123.4', '56.7890'] - - """ - if alignment == "right": - padfn = _padleft - elif alignment == "center": - padfn = _padboth - elif alignment == "decimal": - decimals = [_afterpoint(s) for s in strings] - maxdecimals = max(decimals) - strings = [s + (maxdecimals - decs) * " " - for s, decs in zip(strings, decimals)] - padfn = _padleft - elif not alignment: - return strings - else: - padfn = _padright - - if has_invisible: - width_fn = _visible_width - else: - width_fn = wcswidth - - maxwidth = max(max(map(width_fn, strings)), minwidth) - padded_strings = [padfn(maxwidth, s, has_invisible) for s in strings] - return padded_strings - - -def _more_generic(type1, type2): - types = { _none_type: 0, int: 1, float: 2, _binary_type: 3, _text_type: 4 } - invtypes = { 4: _text_type, 3: _binary_type, 2: float, 1: int, 0: _none_type } - moregeneric = max(types.get(type1, 4), types.get(type2, 4)) - return invtypes[moregeneric] - - -def _column_type(strings, has_invisible=True): - """The least generic type all column values are convertible to. - - >>> _column_type(["1", "2"]) is _int_type - True - >>> _column_type(["1", "2.3"]) is _float_type - True - >>> _column_type(["1", "2.3", "four"]) is _text_type - True - >>> _column_type(["four", '\u043f\u044f\u0442\u044c']) is _text_type - True - >>> _column_type([None, "brux"]) is _text_type - True - >>> _column_type([1, 2, None]) is _int_type - True - >>> import datetime as dt - >>> _column_type([dt.datetime(1991,2,19), dt.time(17,35)]) is _text_type - True - - """ - types = [_type(s, has_invisible) for s in strings ] - return reduce(_more_generic, types, int) - - -def _format(val, valtype, floatfmt, missingval=""): - u"""Format a value accoding to its type. - - Unicode is supported: - - >>> hrow = ['\u0431\u0443\u043a\u0432\u0430', '\u0446\u0438\u0444\u0440\u0430'] ; \ - tbl = [['\u0430\u0437', 2], ['\u0431\u0443\u043a\u0438', 4]] ; \ - print(tabulate(tbl, headers=hrow)[0]) - буква цифра - ------- ------- - аз 2 - буки 4 - """ - if val is None: - return missingval - - if valtype in [int, _text_type]: - return "{0}".format(val) - elif valtype is _binary_type: - try: - return _text_type(val, "ascii") - except UnicodeDecodeError: - return _text_type('0x' + binascii.hexlify(val).decode('ascii')) - except TypeError: - return _text_type(val) - elif valtype is float: - return format(float(val), floatfmt) - else: - return "{0}".format(val) - - -def _align_header(header, alignment, width): - if alignment == "left": - return _padright(width, header) - elif alignment == "center": - return _padboth(width, header) - elif not alignment: - return "{0}".format(header) - else: - return _padleft(width, header) - - -def _normalize_tabular_data(tabular_data, headers): - """Transform a supported data type to a list of lists, and a list of headers. - - Supported tabular data types: - - * list-of-lists or another iterable of iterables - - * list of named tuples (usually used with headers="keys") - - * list of dicts (usually used with headers="keys") - - * list of OrderedDicts (usually used with headers="keys") - - * 2D NumPy arrays - - * NumPy record arrays (usually used with headers="keys") - - * dict of iterables (usually used with headers="keys") - - * pandas.DataFrame (usually used with headers="keys") - - The first row can be used as headers if headers="firstrow", - column indices can be used as headers if headers="keys". - - """ - - if hasattr(tabular_data, "keys") and hasattr(tabular_data, "values"): - # dict-like and pandas.DataFrame? - if hasattr(tabular_data.values, "__call__"): - # likely a conventional dict - keys = tabular_data.keys() - rows = list(izip_longest(*tabular_data.values())) # columns have to be transposed - elif hasattr(tabular_data, "index"): - # values is a property, has .index => it's likely a pandas.DataFrame (pandas 0.11.0) - keys = tabular_data.keys() - vals = tabular_data.values # values matrix doesn't need to be transposed - names = tabular_data.index - rows = [[v]+list(row) for v,row in zip(names, vals)] - else: - raise ValueError("tabular data doesn't appear to be a dict or a DataFrame") - - if headers == "keys": - headers = list(map(_text_type,keys)) # headers should be strings - - else: # it's a usual an iterable of iterables, or a NumPy array - rows = list(tabular_data) - - if (headers == "keys" and - hasattr(tabular_data, "dtype") and - getattr(tabular_data.dtype, "names")): - # numpy record array - headers = tabular_data.dtype.names - elif (headers == "keys" - and len(rows) > 0 - and isinstance(rows[0], tuple) - and hasattr(rows[0], "_fields")): - # namedtuple - headers = list(map(_text_type, rows[0]._fields)) - elif (len(rows) > 0 - and isinstance(rows[0], dict)): - # dict or OrderedDict - uniq_keys = set() # implements hashed lookup - keys = [] # storage for set - if headers == "firstrow": - firstdict = rows[0] if len(rows) > 0 else {} - keys.extend(firstdict.keys()) - uniq_keys.update(keys) - rows = rows[1:] - for row in rows: - for k in row.keys(): - #Save unique items in input order - if k not in uniq_keys: - keys.append(k) - uniq_keys.add(k) - if headers == 'keys': - headers = keys - elif isinstance(headers, dict): - # a dict of headers for a list of dicts - headers = [headers.get(k, k) for k in keys] - headers = list(map(_text_type, headers)) - elif headers == "firstrow": - if len(rows) > 0: - headers = [firstdict.get(k, k) for k in keys] - headers = list(map(_text_type, headers)) - else: - headers = [] - elif headers: - raise ValueError('headers for a list of dicts is not a dict or a keyword') - rows = [[row.get(k) for k in keys] for row in rows] - elif headers == "keys" and len(rows) > 0: - # keys are column indices - headers = list(map(_text_type, range(len(rows[0])))) - - # take headers from the first row if necessary - if headers == "firstrow" and len(rows) > 0: - headers = list(map(_text_type, rows[0])) # headers should be strings - rows = rows[1:] - - headers = list(map(_text_type,headers)) - rows = list(map(list,rows)) - - # pad with empty headers for initial columns if necessary - if headers and len(rows) > 0: - nhs = len(headers) - ncols = len(rows[0]) - if nhs < ncols: - headers = [""]*(ncols - nhs) + headers - - return rows, headers - -def table_formats(): - return _table_formats.keys() - -def tabulate(tabular_data, headers=[], tablefmt="simple", - floatfmt="g", numalign="decimal", stralign="left", - missingval=""): - """Format a fixed width table for pretty printing. - - >>> print(tabulate([[1, 2.34], [-56, "8.999"], ["2", "10001"]])[0]) - --- --------- - 1 2.34 - -56 8.999 - 2 10001 - --- --------- - - The first required argument (`tabular_data`) can be a - list-of-lists (or another iterable of iterables), a list of named - tuples, a dictionary of iterables, an iterable of dictionaries, - a two-dimensional NumPy array, NumPy record array, or a Pandas' - dataframe. - - - Table headers - ------------- - - To print nice column headers, supply the second argument (`headers`): - - - `headers` can be an explicit list of column headers - - if `headers="firstrow"`, then the first row of data is used - - if `headers="keys"`, then dictionary keys or column indices are used - - Otherwise a headerless table is produced. - - If the number of headers is less than the number of columns, they - are supposed to be names of the last columns. This is consistent - with the plain-text format of R and Pandas' dataframes. - - >>> print(tabulate([["sex","age"],["Alice","F",24],["Bob","M",19]], - ... headers="firstrow")[0]) - sex age - ----- ----- ----- - Alice F 24 - Bob M 19 - - - Column alignment - ---------------- - - `tabulate` tries to detect column types automatically, and aligns - the values properly. By default it aligns decimal points of the - numbers (or flushes integer numbers to the right), and flushes - everything else to the left. Possible column alignments - (`numalign`, `stralign`) are: "right", "center", "left", "decimal" - (only for `numalign`), and None (to disable alignment). - - - Table formats - ------------- - - `floatfmt` is a format specification used for columns which - contain numeric data with a decimal point. - - `None` values are replaced with a `missingval` string: - - >>> print(tabulate([["spam", 1, None], - ... ["eggs", 42, 3.14], - ... ["other", None, 2.7]], missingval="?")[0]) - ----- -- ---- - spam 1 ? - eggs 42 3.14 - other ? 2.7 - ----- -- ---- - - Various plain-text table formats (`tablefmt`) are supported: - 'plain', 'simple', 'grid', 'pipe', 'orgtbl', 'rst', 'mediawiki', - 'latex', and 'latex_booktabs'. Variable `tabulate_formats` contains the list of - currently supported formats. - - "plain" format doesn't use any pseudographics to draw tables, - it separates columns with a double space: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "plain")[0]) - strings numbers - spam 41.9999 - eggs 451 - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="plain")[0]) - spam 41.9999 - eggs 451 - - "simple" format is like Pandoc simple_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "simple")[0]) - strings numbers - --------- --------- - spam 41.9999 - eggs 451 - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="simple")[0]) - ---- -------- - spam 41.9999 - eggs 451 - ---- -------- - - "grid" is similar to tables produced by Emacs table.el package or - Pandoc grid_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "grid")[0]) - +-----------+-----------+ - | strings | numbers | - +===========+===========+ - | spam | 41.9999 | - +-----------+-----------+ - | eggs | 451 | - +-----------+-----------+ - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="grid")[0]) - +------+----------+ - | spam | 41.9999 | - +------+----------+ - | eggs | 451 | - +------+----------+ - - "fancy_grid" draws a grid using box-drawing characters: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "fancy_grid")[0]) - ╒═══════════╤═══════════╕ - │ strings │ numbers │ - ╞═══════════╪═══════════╡ - │ spam │ 41.9999 │ - ├───────────┼───────────┤ - │ eggs │ 451 │ - ╘═══════════╧═══════════╛ - - "pipe" is like tables in PHP Markdown Extra extension or Pandoc - pipe_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "pipe")[0]) - | strings | numbers | - |:----------|----------:| - | spam | 41.9999 | - | eggs | 451 | - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="pipe")[0]) - |:-----|---------:| - | spam | 41.9999 | - | eggs | 451 | - - "orgtbl" is like tables in Emacs org-mode and orgtbl-mode. They - are slightly different from "pipe" format by not using colons to - define column alignment, and using a "+" sign to indicate line - intersections: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "orgtbl")[0]) - | strings | numbers | - |-----------+-----------| - | spam | 41.9999 | - | eggs | 451 | - - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="orgtbl")[0]) - | spam | 41.9999 | - | eggs | 451 | - - "rst" is like a simple table format from reStructuredText; please - note that reStructuredText accepts also "grid" tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "rst")[0]) - ========= ========= - strings numbers - ========= ========= - spam 41.9999 - eggs 451 - ========= ========= - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="rst")[0]) - ==== ======== - spam 41.9999 - eggs 451 - ==== ======== - - "mediawiki" produces a table markup used in Wikipedia and on other - MediaWiki-based sites: - - >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], ["eggs", "451.0"]], - ... headers="firstrow", tablefmt="mediawiki")[0]) - {| class="wikitable" style="text-align: left;" - |+ - |- - ! strings !! align="right"| numbers - |- - | spam || align="right"| 41.9999 - |- - | eggs || align="right"| 451 - |} - - "html" produces HTML markup: - - >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], ["eggs", "451.0"]], - ... headers="firstrow", tablefmt="html")[0]) - - - - -
stringsnumbers
spam41.9999
eggs451
- - "latex" produces a tabular environment of LaTeX document markup: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex")[0]) - \\begin{tabular}{ll} - \\hline - spam & 41.9999 \\\\ - eggs & 451 \\\\ - \\hline - \\end{tabular} - - "latex_booktabs" produces a tabular environment of LaTeX document markup - using the booktabs.sty package: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex_booktabs")[0]) - \\begin{tabular}{ll} - \\toprule - spam & 41.9999 \\\\ - eggs & 451 \\\\ - \\bottomrule - \end{tabular} - - Also returns a tuple of the raw rows pulled from tabular_data - """ - if tabular_data is None: - tabular_data = [] - list_of_lists, headers = _normalize_tabular_data(tabular_data, headers) - - # format rows and columns, convert numeric values to strings - cols = list(zip(*list_of_lists)) - coltypes = list(map(_column_type, cols)) - cols = [[_format(v, ct, floatfmt, missingval) for v in c] - for c,ct in zip(cols, coltypes)] - - # optimization: look for ANSI control codes once, - # enable smart width functions only if a control code is found - plain_text = '\n'.join(['\t'.join(map(_text_type, headers))] + \ - ['\t'.join(map(_text_type, row)) for row in cols]) - has_invisible = re.search(_invisible_codes, plain_text) - if has_invisible: - width_fn = _visible_width - else: - width_fn = wcswidth - - if not isinstance(tablefmt, TableFormat): - tablefmt = _table_formats.get(tablefmt, _table_formats["simple"]) - - if tablefmt.with_align: - # align columns - aligns = [numalign if ct in [int,float] else stralign for ct in coltypes] - else: - aligns = [False for ct in coltypes] - minwidths = [width_fn(h) + MIN_PADDING for h in headers] if headers else [0]*len(cols) - cols = [_align_column(c, a, minw, has_invisible) - for c, a, minw in zip(cols, aligns, minwidths)] - - if headers: - # align headers and add headers - t_cols = cols or [['']] * len(headers) - if tablefmt.with_align: - t_aligns = aligns or [stralign] * len(headers) - else: - t_aligns = [False for ct in coltypes] - minwidths = [max(minw, width_fn(c[0])) for minw, c in zip(minwidths, t_cols)] - headers = [_align_header(h, a, minw) - for h, a, minw in zip(headers, t_aligns, minwidths)] - rows = list(zip(*cols)) - else: - minwidths = [width_fn(c[0]) for c in cols] - rows = list(zip(*cols)) - - return _format_table(tablefmt, headers, rows, minwidths, aligns), rows - - -def _build_simple_row(padded_cells, rowfmt): - "Format row according to DataRow format without padding." - begin, sep, end = rowfmt - return (begin + sep.join(padded_cells) + end).rstrip() - - -def _build_row(padded_cells, colwidths, colaligns, rowfmt): - "Return a string which represents a row of data cells." - if not rowfmt: - return None - if hasattr(rowfmt, "__call__"): - return rowfmt(padded_cells, colwidths, colaligns) - else: - return _build_simple_row(padded_cells, rowfmt) - - -def _build_line(colwidths, colaligns, linefmt): - "Return a string which represents a horizontal line." - if not linefmt: - return None - if hasattr(linefmt, "__call__"): - return linefmt(colwidths, colaligns) - else: - begin, fill, sep, end = linefmt - cells = [fill*w for w in colwidths] - return _build_simple_row(cells, (begin, sep, end)) - - -def _pad_row(cells, padding): - if cells: - pad = " "*padding - padded_cells = [pad + cell + pad for cell in cells] - return padded_cells - else: - return cells - - -def _format_table(fmt, headers, rows, colwidths, colaligns): - """Produce a plain-text representation of the table.""" - lines = [] - hidden = fmt.with_header_hide if (headers and fmt.with_header_hide) else [] - pad = fmt.padding - headerrow = fmt.headerrow - - padded_widths = [(w + 2*pad) for w in colwidths] - padded_headers = _pad_row(headers, pad) - padded_rows = [_pad_row(row, pad) for row in rows] - - if fmt.lineabove and "lineabove" not in hidden: - lines.append(_build_line(padded_widths, colaligns, fmt.lineabove)) - - if padded_headers: - lines.append(_build_row(padded_headers, padded_widths, colaligns, headerrow)) - if fmt.linebelowheader and "linebelowheader" not in hidden: - lines.append(_build_line(padded_widths, colaligns, fmt.linebelowheader)) - - if padded_rows and fmt.linebetweenrows and "linebetweenrows" not in hidden: - # initial rows with a line below - for row in padded_rows[:-1]: - lines.append(_build_row(row, padded_widths, colaligns, fmt.datarow)) - lines.append(_build_line(padded_widths, colaligns, fmt.linebetweenrows)) - # the last row without a line below - lines.append(_build_row(padded_rows[-1], padded_widths, colaligns, fmt.datarow)) - else: - for row in padded_rows: - lines.append(_build_row(row, padded_widths, colaligns, fmt.datarow)) - - if fmt.linebelow and "linebelow" not in hidden: - lines.append(_build_line(padded_widths, colaligns, fmt.linebelow)) - - return "\n".join(lines) - - -def _main(): - """\ - Usage: tabulate [options] [FILE ...] - - Pretty-print tabular data. See also https://bitbucket.org/astanin/python-tabulate - - FILE a filename of the file with tabular data; - if "-" or missing, read data from stdin. - - Options: - - -h, --help show this message - -1, --header use the first row of data as a table header - -s REGEXP, --sep REGEXP use a custom column separator (default: whitespace) - -f FMT, --format FMT set output table format; supported formats: - plain, simple, grid, fancy_grid, pipe, orgtbl, - rst, mediawiki, html, latex, latex_booktabs, tsv - (default: simple) - """ - import getopt - import sys - import textwrap - usage = textwrap.dedent(_main.__doc__) - try: - opts, args = getopt.getopt(sys.argv[1:], - "h1f:s:", - ["help", "header", "format", "separator"]) - except getopt.GetoptError as e: - print(e) - print(usage) - sys.exit(2) - headers = [] - tablefmt = "simple" - sep = r"\s+" - for opt, value in opts: - if opt in ["-1", "--header"]: - headers = "firstrow" - elif opt in ["-f", "--format"]: - if value not in tabulate_formats: - print("%s is not a supported table format" % value) - print(usage) - sys.exit(3) - tablefmt = value - elif opt in ["-s", "--sep"]: - sep = value - elif opt in ["-h", "--help"]: - print(usage) - sys.exit(0) - files = [sys.stdin] if not args else args - for f in files: - if f == "-": - f = sys.stdin - if _is_file(f): - _pprint_file(f, headers=headers, tablefmt=tablefmt, sep=sep) - else: - with open(f) as fobj: - _pprint_file(fobj) - - -def _pprint_file(fobject, headers, tablefmt, sep): - rows = fobject.readlines() - table = [re.split(sep, r.rstrip()) for r in rows] - print(tabulate(table, headers, tablefmt)) - - -if __name__ == "__main__": - _main() diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index 41364756..656e5d53 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -1,13 +1,15 @@ from __future__ import print_function from __future__ import unicode_literals import logging +from re import compile, escape +from collections import Counter + from prompt_toolkit.completion import Completer, Completion + +from .output_formatter import OutputFormatter from .packages.completion_engine import suggest_type from .packages.parseutils import last_word from .packages.special.favoritequeries import favoritequeries -from re import compile, escape -from .packages.tabulate import table_formats -from collections import Counter _logger = logging.getLogger(__name__) @@ -57,7 +59,8 @@ def __init__(self, smart_completion=True): self.name_pattern = compile("^[_a-z][_a-z0-9\$]*$") self.special_commands = [] - self.table_formats = table_formats() + formatter = OutputFormatter() + self.table_formats = formatter.supported_formats() self.reset_completions() def escape_name(self, name): diff --git a/setup.py b/setup.py index 82cdef6e..3ecc21c4 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.5', 'pycryptodome >= 3', + 'tabulate >= 0.7.6', ] setup( diff --git a/tests/test_main.py b/tests/test_main.py index 14f1c10e..cb769a0b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,7 +3,7 @@ import click from click.testing import CliRunner -from mycli.main import (cli, confirm_destructive_query, format_output, +from mycli.main import (cli, confirm_destructive_query, is_destructive, query_starts_with, queries_start_with, thanks_picker, PACKAGE_ROOT) from utils import USER, HOST, PORT, PASSWORD, dbtest, run @@ -16,32 +16,6 @@ CLI_ARGS = ['--user', USER, '--host', HOST, '--port', PORT, '--password', PASSWORD, '_test_db'] -def test_format_output(): - results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], - 'test status', 'psql') - expected = ['Title', '+---------+---------+\n| head1 | head2 |\n|---------+---------|\n| abc | def |\n+---------+---------+', 'test status'] - assert results == expected - -def test_format_output_auto_expand(): - table_results = format_output('Title', [('abc', 'def')], - ['head1', 'head2'], 'test status', 'psql', - max_width=100) - table = ['Title', '+---------+---------+\n| head1 | head2 |\n|---------+---------|\n| abc | def |\n+---------+---------+', 'test status'] - assert table_results == table - - expanded_results = format_output('Title', [('abc', 'def')], - ['head1', 'head2'], 'test status', 'psql', - max_width=1) - expanded = ['Title', u'***************************[ 1. row ]***************************\nhead1 | abc\nhead2 | def\n', 'test status'] - assert expanded_results == expanded - -def test_format_output_no_table(): - results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], - 'test status', None) - - expected = ['Title', u'head1\thead2\nabc\tdef', 'test status'] - assert results == expected - @dbtest def test_execute_arg(executor): run(executor, 'create table test (a text)') diff --git a/tests/test_tabulate.py b/tests/test_tabulate.py deleted file mode 100644 index e0ddc407..00000000 --- a/tests/test_tabulate.py +++ /dev/null @@ -1,22 +0,0 @@ -from mycli.packages.tabulate import tabulate -from textwrap import dedent - - -def test_dont_strip_leading_whitespace(): - data = [[' abc']] - headers = ['xyz'] - tbl, _ = tabulate(data, headers, tablefmt='psql') - assert tbl == dedent(''' - +---------+ - | xyz | - |---------| - | abc | - +---------+ ''').strip() -def test_dont_add_whitespace(): - data = [[3, 4]] - headers = ['1', '2'] - tbl, _ = tabulate(data, headers, tablefmt='tsv') - assert tbl == dedent(''' - 1\t2 - 3\t4 - ''').strip() diff --git a/tests/utils.py b/tests/utils.py index c76cb7ad..5e9e0abb 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,8 +1,11 @@ -import pytest -import pymysql -from mycli.main import format_output, special from os import getenv +import pymysql +import pytest + +from mycli.main import MyCli, special +from mycli.output_formatter import OutputFormatter + PASSWORD = getenv('PYTEST_PASSWORD') USER = getenv('PYTEST_USER', 'root') HOST = getenv('PYTEST_HOST', 'localhost') @@ -37,8 +40,14 @@ def create_db(dbname): def run(executor, sql, join=False): " Return string output for the sql to be run " result = [] + + # TODO: this needs to go away. `run()` should not test formatted output. + # It should test raw results. + mycli = MyCli() + mycli.table_format = 'psql' for title, rows, headers, status in executor.run(sql): - result.extend(format_output(title, rows, headers, status, 'psql', special.is_expanded_output())) + result.extend(mycli.format_output(title, rows, headers, status, special.is_expanded_output())) + if join: result = '\n'.join(result) return result From 2bc4facb4ae978c7465dcb4128b895224348acb1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:30:40 -0500 Subject: [PATCH 02/76] Add missing class. --- mycli/output_formatter.py | 57 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 mycli/output_formatter.py diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py new file mode 100644 index 00000000..c939aa51 --- /dev/null +++ b/mycli/output_formatter.py @@ -0,0 +1,57 @@ +"""A generic output formatter interface.""" + +from __future__ import unicode_literals + +from tabulate import tabulate + +from .packages.expanded import expanded_table + + +def tabulate_wrapper(data, headers, table_format=None, missing_value=None): + """Wrap tabulate inside a standard function for OutputFormatter.""" + return tabulate(data, headers, tablefmt=table_format, + missingval=missing_value) + + +class OutputFormatter(object): + """A class with a standard interface for various formatting libraries.""" + + def __init__(self): + """Register the supported output formats.""" + self._output_formats = {} + + tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', + 'orgtbl', 'jira', 'psql', 'rst', 'tsv', + 'mediawiki', 'moinmoin', 'html', 'latex', + 'latex_booktabs', 'textile') + for tabulate_format in tabulate_formats: + self.register_output_format(tabulate_format, tabulate_wrapper, + table_format=tabulate_format) + + self.register_output_format('expanded', expanded_table) + + def register_output_format(self, name, function, **kwargs): + """Register a new output format. + + *function* should be a callable that accepts the following arguments: + - *headers*: A list of headers for the output data. + - *data*: The data that needs formatting. + - *kwargs*: Any other keyword arguments for controlling the output. + It should return the formatted output as a string. + """ + self._output_formats[name] = (function, kwargs) + + def supported_formats(self): + """Return the supported output format names.""" + return tuple(self._output_formats.keys()) + + def format_output(self, data, headers, format_name, **kwargs): + """Format the headers and data using a specific formatter. + + *format_name* must be a formatter available in `supported_formats()`. + + All keyword arguments are passed to the specified formatter. + """ + function, fkwargs = self._output_formats[format_name] + fkwargs.update(kwargs) + return function(data, headers, **fkwargs) From 3c783213fa0c2c63668a709644ddded28f2dfad7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:34:10 -0500 Subject: [PATCH 03/76] Make logic more readable. --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index b9a3cebe..46de4cbf 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -192,7 +192,7 @@ def register_special_commands(self): '\\R', 'Change prompt format.', aliases=('\\R',), case_sensitive=True) def change_table_format(self, arg, **_): - if not arg in self.formatter.supported_formats(): + if arg not in self.formatter.supported_formats(): msg = "Table type %s not yet implemented. Allowed types:" % arg for table_type in self.formatter.supported_formats(): msg += "\n\t%s" % table_type From 26ff1f3f23fac653de47e51d4494c494a1fe4cc3 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:37:31 -0500 Subject: [PATCH 04/76] Move CSV to output formatter. --- mycli/main.py | 17 ----------------- mycli/output_formatter.py | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 46de4cbf..389f176c 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -15,12 +15,6 @@ from random import choice from io import open -# support StringIO for Python 2 and 3 -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - import click import sqlparse from prompt_toolkit import CommandLineInterface, Application, AbortAction @@ -735,17 +729,6 @@ def format_output(self, title, cur, headers, status, expanded=False, if expanded: output.append(self.formatter.format_output(cur, headers, 'expanded')) - elif self.table_format == 'csv': - content = StringIO() - writer = csv.writer(content) - writer.writerow(headers) - - for row in cur: - row = ['null' if val is None else str(val) for val in row] - writer.writerow(row) - - output.append(content.getvalue()) - content.close() else: rows = list(cur) formatted = self.formatter.format_output(rows, headers, diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index c939aa51..5081a5d6 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -2,6 +2,12 @@ from __future__ import unicode_literals +import csv +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + from tabulate import tabulate from .packages.expanded import expanded_table @@ -13,6 +19,21 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value=None): missingval=missing_value) +def csv_wrapper(data, headers): + content = StringIO() + writer = csv.writer(content) + writer.writerow(headers) + + for row in data: + row = ['null' if val is None else str(val) for val in row] + writer.writerow(row) + + output = content.getvalue() + content.close() + + return output + + class OutputFormatter(object): """A class with a standard interface for various formatting libraries.""" @@ -28,6 +49,7 @@ def __init__(self): self.register_output_format(tabulate_format, tabulate_wrapper, table_format=tabulate_format) + self.register_output_format('csv', csv_wrapper) self.register_output_format('expanded', expanded_table) def register_output_format(self, name, function, **kwargs): From 5da417568292477e3603c8124ed733e7be0dd840 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:51:47 -0500 Subject: [PATCH 05/76] Simply format output. --- mycli/main.py | 27 ++++++++++++--------------- mycli/output_formatter.py | 2 +- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 389f176c..c3485dfc 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -720,27 +720,24 @@ def run_query(self, query, new_line=True): def format_output(self, title, cur, headers, status, expanded=False, max_width=None): + table_format = 'expanded' if expanded else self.table_format output = [] + if title: # Only print the title if it's not None. output.append(title) + if cur: headers = [utf8tounicode(x) for x in headers] - if expanded: - output.append(self.formatter.format_output(cur, headers, - 'expanded')) - else: - rows = list(cur) - formatted = self.formatter.format_output(rows, headers, - self.table_format) - if (self.table_format != 'expanded' and - max_width and rows and - content_exceeds_width(rows[0], max_width) and - headers): - output.append(self.formatter.format_output(cur, headers, - 'expanded')) - else: - output.append(formatted) + rows = list(cur) + formatted = self.formatter.format_output(rows, headers, table_format) + + if (table_format != 'expanded' and max_width and rows and + content_exceeds_width(rows[0], max_width) and headers): + formatted = self.formatter.format_output(rows, headers, 'expanded') + + output.append(formatted) + if status: # Only print the status if it's not None. output.append(status) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5081a5d6..5c2678a0 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -13,7 +13,7 @@ from .packages.expanded import expanded_table -def tabulate_wrapper(data, headers, table_format=None, missing_value=None): +def tabulate_wrapper(data, headers, table_format=None, missing_value=''): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, missingval=missing_value) From 8918a51db94b46ae9846946685860c51f2ad0061 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:55:39 -0500 Subject: [PATCH 06/76] Fix tsv/table logic. --- mycli/main.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index c3485dfc..cb40aaf0 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -838,10 +838,12 @@ def cli(database, user, host, port, socket, password, dbname, # --execute argument if execute: try: - table_format = 'tsv' - if csv: + table_format = None + if table: + table_format = mycli.table_format + elif csv: table_format = 'csv' - mycli.table_format = table_format or mycli.table_format + mycli.table_format = table_format or 'tsv' mycli.run_query(execute) exit(0) except Exception as e: @@ -863,14 +865,16 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - table_format = 'tsv' + table_format = None new_line = True if csv: table_format = 'csv' new_line = False + elif table: + table_format = mycli.table_format - mycli.table_format = table_format or mycli.table_format + mycli.table_format = table_format or 'tsv' mycli.run_query(stdin_text, new_line=new_line) exit(0) From 7b6398a14df84ebaa0801fe000b620a2c19a5949 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:59:01 -0500 Subject: [PATCH 07/76] Use missing_value keyword for csv wrapper. --- mycli/output_formatter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5c2678a0..5a90709a 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -19,13 +19,14 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value=''): missingval=missing_value) -def csv_wrapper(data, headers): +def csv_wrapper(data, headers, missing_value='null'): + """Wrap CSV formatting inside a standard function for OutputFormatter.""" content = StringIO() writer = csv.writer(content) writer.writerow(headers) for row in data: - row = ['null' if val is None else str(val) for val in row] + row = [missing_value if val is None else str(val) for val in row] writer.writerow(row) output = content.getvalue() From f5a305d45652d017e411c1f47cf9e87fe412b4f7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:05:22 -0500 Subject: [PATCH 08/76] Do not use tabulate for TSV formatting. --- mycli/output_formatter.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5a90709a..6b0d8f2a 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -19,10 +19,10 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value=''): missingval=missing_value) -def csv_wrapper(data, headers, missing_value='null'): +def csv_wrapper(data, headers, missing_value='null', delimiter=','): """Wrap CSV formatting inside a standard function for OutputFormatter.""" content = StringIO() - writer = csv.writer(content) + writer = csv.writer(content, delimiter=delimiter) writer.writerow(headers) for row in data: @@ -43,14 +43,15 @@ def __init__(self): self._output_formats = {} tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', - 'orgtbl', 'jira', 'psql', 'rst', 'tsv', - 'mediawiki', 'moinmoin', 'html', 'latex', - 'latex_booktabs', 'textile') + 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', + 'moinmoin', 'html', 'latex', 'latex_booktabs', + 'textile') for tabulate_format in tabulate_formats: self.register_output_format(tabulate_format, tabulate_wrapper, table_format=tabulate_format) self.register_output_format('csv', csv_wrapper) + self.register_output_format('tsv', csv_wrapper, delimiter='\t') self.register_output_format('expanded', expanded_table) def register_output_format(self, name, function, **kwargs): From 93c6306b16c6316e352c6f6307b45c49602258dd Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:07:11 -0500 Subject: [PATCH 09/76] Add space before expanded. --- mycli/output_formatter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 6b0d8f2a..29922b86 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -52,6 +52,7 @@ def __init__(self): self.register_output_format('csv', csv_wrapper) self.register_output_format('tsv', csv_wrapper, delimiter='\t') + self.register_output_format('expanded', expanded_table) def register_output_format(self, name, function, **kwargs): From 3a881284339f71033d29e78cef288589efcb5fe1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:25:42 -0500 Subject: [PATCH 10/76] Add preprocessor. --- mycli/output_formatter.py | 32 +++++++++++++++++++++++--------- mycli/packages/expanded.py | 2 +- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 29922b86..9cc60267 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -13,20 +13,23 @@ from .packages.expanded import expanded_table -def tabulate_wrapper(data, headers, table_format=None, missing_value=''): +def override_missing_value(data, missing_value='', **_): + """Override missing values in the data with *missing_value*.""" + return [[missing_value if v is None else v for v in row] for row in data] + + +def tabulate_wrapper(data, headers, table_format=None, **_): """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate(data, headers, tablefmt=table_format, - missingval=missing_value) + return tabulate(data, headers, tablefmt=table_format) -def csv_wrapper(data, headers, missing_value='null', delimiter=','): +def csv_wrapper(data, headers, delimiter=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" content = StringIO() writer = csv.writer(content, delimiter=delimiter) writer.writerow(headers) for row in data: - row = [missing_value if val is None else str(val) for val in row] writer.writerow(row) output = content.getvalue() @@ -48,12 +51,20 @@ def __init__(self): 'textile') for tabulate_format in tabulate_formats: self.register_output_format(tabulate_format, tabulate_wrapper, - table_format=tabulate_format) + table_format=tabulate_format, + preprocessor=override_missing_value, + missing_value='') - self.register_output_format('csv', csv_wrapper) - self.register_output_format('tsv', csv_wrapper, delimiter='\t') + self.register_output_format('csv', csv_wrapper, + preprocessor=override_missing_value, + missing_value='null') + self.register_output_format('tsv', csv_wrapper, delimiter='\t', + preprocessor=override_missing_value, + missing_value='null') - self.register_output_format('expanded', expanded_table) + self.register_output_format('expanded', expanded_table, + preprocessor=override_missing_value, + missing_value='') def register_output_format(self, name, function, **kwargs): """Register a new output format. @@ -79,4 +90,7 @@ def format_output(self, data, headers, format_name, **kwargs): """ function, fkwargs = self._output_formats[format_name] fkwargs.update(kwargs) + preprocessor = fkwargs.pop('preprocessor', None) + if preprocessor: + data = preprocessor(data, **fkwargs) return function(data, headers, **fkwargs) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 245e4ca7..2b0b7207 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -19,7 +19,7 @@ def format_field(value): except UnicodeDecodeError: return _text_type('0x' + binascii.hexlify(value).decode('ascii')) -def expanded_table(rows, headers): +def expanded_table(rows, headers, **_): header_len = max([len(x) for x in headers]) max_row_len = 0 results = [] From cff2ddd8b198d4d47d45ef67f5d7493d95b073be Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:33:41 -0500 Subject: [PATCH 11/76] Simplify missing value. --- mycli/output_formatter.py | 6 +++--- mycli/packages/expanded.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 9cc60267..9c3d7d02 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -18,9 +18,10 @@ def override_missing_value(data, missing_value='', **_): return [[missing_value if v is None else v for v in row] for row in data] -def tabulate_wrapper(data, headers, table_format=None, **_): +def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate(data, headers, tablefmt=table_format) + return tabulate(data, headers, tablefmt=table_format, + missingval=missing_value) def csv_wrapper(data, headers, delimiter=',', **_): @@ -52,7 +53,6 @@ def __init__(self): for tabulate_format in tabulate_formats: self.register_output_format(tabulate_format, tabulate_wrapper, table_format=tabulate_format, - preprocessor=override_missing_value, missing_value='') self.register_output_format('csv', csv_wrapper, diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 2b0b7207..b39d9507 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -35,7 +35,6 @@ def expanded_table(rows, headers, **_): max_row_len = row_len for header, value in zip(padded_headers, row): - if value is None: value = '' row_result.append(u"%s %s" % (header, value)) results.append('\n'.join(row_result)) From 5cbdf480e9cc3a65d6ac54619ec797a3cd4e41e0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:36:36 -0500 Subject: [PATCH 12/76] Disable number parsing. --- mycli/output_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 9c3d7d02..2d73e1a5 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -21,7 +21,7 @@ def override_missing_value(data, missing_value='', **_): def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, - missingval=missing_value) + missingval=missing_value, disable_numparse=True) def csv_wrapper(data, headers, delimiter=',', **_): From cced4902517706581ec93237a4424e7be5a3b2c0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:44:23 -0500 Subject: [PATCH 13/76] Remove unused imports. --- mycli/main.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index cb40aaf0..beeeb7d2 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -5,9 +5,7 @@ import os import os.path import sys -import csv import traceback -import socket import logging import threading from time import time @@ -27,9 +25,8 @@ ConditionalProcessor) from prompt_toolkit.history import FileHistory from pygments.token import Token -from configobj import ConfigObj, ConfigObjError -from .packages.special.main import (COMMANDS, NO_QUERY) +from .packages.special.main import NO_QUERY import mycli.packages.special as special from .sqlcompleter import SQLCompleter from .clitoolbar import create_toolbar_tokens_func @@ -38,8 +35,7 @@ from .clibuffer import CLIBuffer from .completion_refresher import CompletionRefresher from .config import (write_default_config, get_mylogin_cnf_path, - open_mylogin_cnf, read_config_file, - read_config_files, str_to_bool) + open_mylogin_cnf, read_config_files, str_to_bool) from .key_bindings import mycli_bindings from .output_formatter import OutputFormatter from .encodingutils import utf8tounicode From d0f1dbf441ae823d56cadca456b1ff35a35abd88 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:47:06 -0500 Subject: [PATCH 14/76] Remove tabulate license note. --- LICENSE.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 9a41a67d..8afaa265 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -27,9 +27,3 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------- - -This program also bundles with it python-tabulate -(https://pypi.python.org/pypi/tabulate) library. This library is licensed under -MIT License. - -------------------------------------------------------------------------------- From 369b5b0795d5391870f260c8563cb6e27e895308 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 1 Apr 2017 11:39:29 +0200 Subject: [PATCH 15/76] Fix tests/test_sqlexecute unit tests --- mycli/output_formatter.py | 17 +++++++++++++++++ tests/test_sqlexecute.py | 4 ++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 2d73e1a5..1397b54f 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import csv +import binascii try: from cStringIO import StringIO except ImportError: @@ -18,6 +19,21 @@ def override_missing_value(data, missing_value='', **_): return [[missing_value if v is None else v for v in row] for row in data] +def bytes_to_unicode(data, **_): + results = [] + for row in data: + result = [] + for v in row: + if isinstance(v, bytes): + try: + conv = v.decode('utf8') + except: + v = '0x' + binascii.hexlify(v).decode('ascii') + result.append(v) + results.append(result) + return results + + def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, @@ -52,6 +68,7 @@ def __init__(self): 'textile') for tabulate_format in tabulate_formats: self.register_output_format(tabulate_format, tabulate_wrapper, + preprocessor=bytes_to_unicode, table_format=tabulate_format, missing_value='') diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index e9929a70..d964f43b 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -27,9 +27,9 @@ def test_bools(executor): results = run(executor, '''select * from test''', join=True) assert results == dedent("""\ +-----+ - | a | + | a | |-----| - | 1 | + | 1 | +-----+ 1 row in set""") From ae3712f764aaf22aa5a2fe3c9a277b375f541bc5 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 1 Apr 2017 12:06:01 +0200 Subject: [PATCH 16/76] fix test_main unit tests --- tests/test_main.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index cb769a0b..29501f13 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -8,6 +8,8 @@ thanks_picker, PACKAGE_ROOT) from utils import USER, HOST, PORT, PASSWORD, dbtest, run +from textwrap import dedent + try: text_type = basestring except NameError: @@ -80,7 +82,7 @@ def test_batch_mode(executor): result = runner.invoke(cli, args=CLI_ARGS, input=sql) assert result.exit_code == 0 - assert 'count(*)\n3\na\nabc\n' in result.output + assert 'count(*)\n3\n\na\nabc\n' in result.output @dbtest def test_batch_mode_table(executor): @@ -95,10 +97,17 @@ def test_batch_mode_table(executor): runner = CliRunner() result = runner.invoke(cli, args=CLI_ARGS + ['-t'], input=sql) - expected = ( - '| count(*) |\n|------------|\n| 3 |\n+------------+\n' - '+-----+\n| a |\n|-----|\n| abc |\n+-----+' - ) + expected = (dedent("""\ + +------------+ + | count(*) | + |------------| + | 3 | + +------------+ + +-----+ + | a | + |-----| + | abc | + +-----+""")) assert result.exit_code == 0 assert expected in result.output From 3dbe5c01cb03fa42b6dc7718b3b4b8bfcfc5c92a Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 1 Apr 2017 12:39:37 +0200 Subject: [PATCH 17/76] use str for python2 delimiter --- mycli/output_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 1397b54f..178add2f 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -43,7 +43,7 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): def csv_wrapper(data, headers, delimiter=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" content = StringIO() - writer = csv.writer(content, delimiter=delimiter) + writer = csv.writer(content, delimiter=str(delimiter)) writer.writerow(headers) for row in data: From 434aea0d3f31cde2f76309b1eea429e1b97f6527 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 1 Apr 2017 16:37:06 +0200 Subject: [PATCH 18/76] add docstring to bytes_to_unicode function --- mycli/output_formatter.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 178add2f..2375c0dd 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """A generic output formatter interface.""" from __future__ import unicode_literals @@ -20,6 +21,12 @@ def override_missing_value(data, missing_value='', **_): def bytes_to_unicode(data, **_): + """Convert bytes that cannot be decoded to utf8 to hexlified string + >>> result = bytes_to_unicode([[b"\\xff", "abc", "✌"]]) + >>> print(" ".join(result[0])) + 0xff abc ✌ + """ + results = [] for row in data: result = [] From be0e2ed85917ec4015bbde46f132b85d511e4e45 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 10:00:13 -0500 Subject: [PATCH 19/76] Add headers to preprocessor. --- mycli/output_formatter.py | 44 +++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 2375c0dd..ba79e803 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -15,30 +15,34 @@ from .packages.expanded import expanded_table -def override_missing_value(data, missing_value='', **_): +def override_missing_value(data, headers, missing_value='', **_): """Override missing values in the data with *missing_value*.""" - return [[missing_value if v is None else v for v in row] for row in data] + return ([[missing_value if v is None else v for v in row] for row in data], + headers) -def bytes_to_unicode(data, **_): - """Convert bytes that cannot be decoded to utf8 to hexlified string - >>> result = bytes_to_unicode([[b"\\xff", "abc", "✌"]]) - >>> print(" ".join(result[0])) - 0xff abc ✌ +def bytes_to_hex(b): + """Convert bytes that cannot be decoded to utf8 to hexlified string. + + >>> print(bytes_to_hex(b"\\xff")) + 0xff + >>> print(bytes_to_hex('abc')) + abc + >>> print(bytes_to_hex('✌')) + ✌ """ + if isinstance(b, bytes): + try: + b.decode('utf8') + except: + b = '0x' + binascii.hexlify(b).decode('ascii') + return b - results = [] - for row in data: - result = [] - for v in row: - if isinstance(v, bytes): - try: - conv = v.decode('utf8') - except: - v = '0x' + binascii.hexlify(v).decode('ascii') - result.append(v) - results.append(result) - return results + +def bytes_to_unicode(data, headers, **_): + """Convert all *data* and *headers* to unicode.""" + return ([[bytes_to_hex(v) for v in row] for row in data], + [bytes_to_hex(h) for h in headers]) def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): @@ -116,5 +120,5 @@ def format_output(self, data, headers, format_name, **kwargs): fkwargs.update(kwargs) preprocessor = fkwargs.pop('preprocessor', None) if preprocessor: - data = preprocessor(data, **fkwargs) + data, headers = preprocessor(data, headers, **fkwargs) return function(data, headers, **fkwargs) From f4d05278d7984e7d3cdc9d0361bf3a9f89d89cfc Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:10:20 -0500 Subject: [PATCH 20/76] Move mycli table format to output formatter. --- mycli/main.py | 48 +++++++++++++++++++-------------------- mycli/output_formatter.py | 26 ++++++++++++++++++--- tests/utils.py | 1 - 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index beeeb7d2..fa715d26 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -106,7 +106,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.multi_line = c['main'].as_bool('multi_line') self.key_bindings = c['main']['key_bindings'] special.set_timing_enabled(c['main'].as_bool('timing')) - self.table_format = c['main']['table_format'] + self.formatter = OutputFormatter(format_name=c['main']['table_format']) self.syntax_style = c['main']['syntax_style'] self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] @@ -133,8 +133,6 @@ def __init__(self, sqlexecute=None, prompt=None, self.completion_refresher = CompletionRefresher() - self.formatter = OutputFormatter() - self.logger = logging.getLogger(__name__) self.initialize_logging() @@ -182,14 +180,16 @@ def register_special_commands(self): '\\R', 'Change prompt format.', aliases=('\\R',), case_sensitive=True) def change_table_format(self, arg, **_): - if arg not in self.formatter.supported_formats(): - msg = "Table type %s not yet implemented. Allowed types:" % arg + print('change table: ' + arg) + try: + self.formatter.set_format_name(arg) + yield (None, None, None, + 'Changed table type to {}'.format(arg)) + except ValueError: + msg = 'Table type {} not yet implemented. Allowed types:'.format(arg) for table_type in self.formatter.supported_formats(): - msg += "\n\t%s" % table_type + msg += "\n\t{}".format(table_type) yield (None, None, None, msg) - else: - self.table_format = arg - yield (None, None, None, "Changed table Type to %s" % self.table_format) def change_db(self, arg, **_): if arg is None: @@ -716,7 +716,7 @@ def run_query(self, query, new_line=True): def format_output(self, title, cur, headers, status, expanded=False, max_width=None): - table_format = 'expanded' if expanded else self.table_format + expanded = expanded or self.formatter.get_format_name() == 'expanded' output = [] if title: # Only print the title if it's not None. @@ -726,11 +726,13 @@ def format_output(self, title, cur, headers, status, expanded=False, headers = [utf8tounicode(x) for x in headers] rows = list(cur) - formatted = self.formatter.format_output(rows, headers, table_format) + formatted = self.formatter.format_output( + rows, headers, format_name='expanded' if expanded else None) - if (table_format != 'expanded' and max_width and rows and + if (not expanded and max_width and rows and content_exceeds_width(rows[0], max_width) and headers): - formatted = self.formatter.format_output(rows, headers, 'expanded') + formatted = self.formatter.format_output( + rows, headers, format_name='expanded') output.append(formatted) @@ -834,12 +836,11 @@ def cli(database, user, host, port, socket, password, dbname, # --execute argument if execute: try: - table_format = None - if table: - table_format = mycli.table_format - elif csv: - table_format = 'csv' - mycli.table_format = table_format or 'tsv' + if csv: + mycli.formatter.set_format_name('csv') + elif not table: + mycli.formatter.set_format_name('tsv') + mycli.run_query(execute) exit(0) except Exception as e: @@ -861,16 +862,13 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - table_format = None new_line = True if csv: - table_format = 'csv' + mycli.formatter.set_format_name('csv') new_line = False - elif table: - table_format = mycli.table_format - - mycli.table_format = table_format or 'tsv' + elif not table: + mycli.formatter.set_format_name('tsv') mycli.run_query(stdin_text, new_line=new_line) exit(0) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index ba79e803..ebb92b1d 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -3,8 +3,8 @@ from __future__ import unicode_literals -import csv import binascii +import csv try: from cStringIO import StringIO except ImportError: @@ -69,9 +69,10 @@ def csv_wrapper(data, headers, delimiter=',', **_): class OutputFormatter(object): """A class with a standard interface for various formatting libraries.""" - def __init__(self): + def __init__(self, format_name=None): """Register the supported output formats.""" self._output_formats = {} + self._format_name = None tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', @@ -94,6 +95,21 @@ def __init__(self): preprocessor=override_missing_value, missing_value='') + if format_name: + self.set_format_name(format_name) + + def set_format_name(self, format_name): + """Set the OutputFormatter's default format.""" + if format_name in self.supported_formats(): + self._format_name = format_name + else: + raise ValueError('unrecognized format_name: {}'.format( + format_name)) + + def get_format_name(self): + """Get the OutputFormatter's default format.""" + return self._format_name + def register_output_format(self, name, function, **kwargs): """Register a new output format. @@ -109,13 +125,17 @@ def supported_formats(self): """Return the supported output format names.""" return tuple(self._output_formats.keys()) - def format_output(self, data, headers, format_name, **kwargs): + def format_output(self, data, headers, format_name=None, **kwargs): """Format the headers and data using a specific formatter. *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. """ + format_name = format_name or self._format_name + if format_name not in self.supported_formats(): + raise ValueError('unrecognized format: {}'.format(format_name)) + function, fkwargs = self._output_formats[format_name] fkwargs.update(kwargs) preprocessor = fkwargs.pop('preprocessor', None) diff --git a/tests/utils.py b/tests/utils.py index 5e9e0abb..64006324 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -44,7 +44,6 @@ def run(executor, sql, join=False): # TODO: this needs to go away. `run()` should not test formatted output. # It should test raw results. mycli = MyCli() - mycli.table_format = 'psql' for title, rows, headers, status in executor.run(sql): result.extend(mycli.format_output(title, rows, headers, status, special.is_expanded_output())) From 90fd5fe54e05b5a9f8e33af35f71a5fcae8ebce6 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:20:34 -0500 Subject: [PATCH 21/76] Use context manager for StringIO. --- mycli/output_formatter.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index ebb92b1d..6a056489 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import binascii +import contextlib import csv try: from cStringIO import StringIO @@ -53,17 +54,14 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): def csv_wrapper(data, headers, delimiter=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" - content = StringIO() - writer = csv.writer(content, delimiter=str(delimiter)) - writer.writerow(headers) + with contextlib.closing(StringIO()) as content: + writer = csv.writer(content, delimiter=str(delimiter)) - for row in data: - writer.writerow(row) + writer.writerow(headers) + for row in data: + writer.writerow(row) - output = content.getvalue() - content.close() - - return output + return content.getvalue() class OutputFormatter(object): From 9f0ace286f60875d708018a41b10c7015cebcac7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:27:34 -0500 Subject: [PATCH 22/76] Remove register output format method. --- mycli/output_formatter.py | 47 +++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 6a056489..14339cb7 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -69,7 +69,21 @@ class OutputFormatter(object): def __init__(self, format_name=None): """Register the supported output formats.""" - self._output_formats = {} + self._output_formats = { + 'csv': (csv_wrapper, { + 'preprocessor': override_missing_value, + 'missing_value': '' + }), + 'tsv': (csv_wrapper, { + 'preprocessor': override_missing_value, + 'missing_value': '', + 'delimiter': '\t' + }), + 'expanded': (expanded_table, { + 'preprocessor': override_missing_value, + 'missing_value': '' + }) + } self._format_name = None tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', @@ -77,21 +91,11 @@ def __init__(self, format_name=None): 'moinmoin', 'html', 'latex', 'latex_booktabs', 'textile') for tabulate_format in tabulate_formats: - self.register_output_format(tabulate_format, tabulate_wrapper, - preprocessor=bytes_to_unicode, - table_format=tabulate_format, - missing_value='') - - self.register_output_format('csv', csv_wrapper, - preprocessor=override_missing_value, - missing_value='null') - self.register_output_format('tsv', csv_wrapper, delimiter='\t', - preprocessor=override_missing_value, - missing_value='null') - - self.register_output_format('expanded', expanded_table, - preprocessor=override_missing_value, - missing_value='') + self._output_formats[tabulate_format] = ( + tabulate_wrapper, {'preprocessor': bytes_to_unicode, + 'table_format': tabulate_format, + 'missing_value': ''} + ) if format_name: self.set_format_name(format_name) @@ -108,17 +112,6 @@ def get_format_name(self): """Get the OutputFormatter's default format.""" return self._format_name - def register_output_format(self, name, function, **kwargs): - """Register a new output format. - - *function* should be a callable that accepts the following arguments: - - *headers*: A list of headers for the output data. - - *data*: The data that needs formatting. - - *kwargs*: Any other keyword arguments for controlling the output. - It should return the formatted output as a string. - """ - self._output_formats[name] = (function, kwargs) - def supported_formats(self): """Return the supported output format names.""" return tuple(self._output_formats.keys()) From e7b93b8162b663ad745948ccbecb9f839c78d05f Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:31:52 -0500 Subject: [PATCH 23/76] Remove print statement. --- mycli/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index fa715d26..661dc65d 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -180,7 +180,6 @@ def register_special_commands(self): '\\R', 'Change prompt format.', aliases=('\\R',), case_sensitive=True) def change_table_format(self, arg, **_): - print('change table: ' + arg) try: self.formatter.set_format_name(arg) yield (None, None, None, From b58cad04f7dda089612c5da03c93b87aa5b221a6 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:34:31 -0500 Subject: [PATCH 24/76] Update list of table formats. --- mycli/myclirc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/myclirc b/mycli/myclirc index ff6e8f73..36e9e409 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -31,7 +31,8 @@ log_level = INFO timing = True # Table format. Possible values: psql, plain, simple, grid, fancy_grid, pipe, -# orgtbl, rst, mediawiki, html, latex, latex_booktabs, tsv. +# orgtbl, jira, rst, mediawiki, moinmoin, html, latex, latex_booktabs, +# textile, csv, tsv. # Recommended: psql, fancy_grid and grid. table_format = psql From cbd5ba0a58d699e402a1ef2c8b4665d3bfc375ae Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:43:03 -0500 Subject: [PATCH 25/76] Move bytes_to_hex to encoding utils package. --- mycli/encodingutils.py | 21 +++++++++++++++++++++ mycli/output_formatter.py | 20 +------------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index 29564d08..ed8f9a15 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -1,8 +1,10 @@ +import binascii import sys PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] == 3 + def unicode2utf8(arg): """ Only in Python 2. Psycopg2 expects the args as bytes not unicode. @@ -13,6 +15,7 @@ def unicode2utf8(arg): return arg.encode('utf-8') return arg + def utf8tounicode(arg): """ Only in Python 2. Psycopg2 returns the error message as utf-8. @@ -22,3 +25,21 @@ def utf8tounicode(arg): if PY2 and isinstance(arg, str): return arg.decode('utf-8') return arg + + +def bytes_to_hex(b): + """Convert bytes that cannot be decoded to utf8 to hexlified string. + + >>> print(bytes_to_hex(b"\\xff")) + 0xff + >>> print(bytes_to_hex('abc')) + abc + >>> print(bytes_to_hex('✌')) + ✌ + """ + if isinstance(b, bytes): + try: + b.decode('utf8') + except: + b = '0x' + binascii.hexlify(b).decode('ascii') + return b diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 14339cb7..a1319fd8 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals -import binascii import contextlib import csv try: @@ -13,6 +12,7 @@ from tabulate import tabulate +from .encodingutils import bytes_to_hex from .packages.expanded import expanded_table @@ -22,24 +22,6 @@ def override_missing_value(data, headers, missing_value='', **_): headers) -def bytes_to_hex(b): - """Convert bytes that cannot be decoded to utf8 to hexlified string. - - >>> print(bytes_to_hex(b"\\xff")) - 0xff - >>> print(bytes_to_hex('abc')) - abc - >>> print(bytes_to_hex('✌')) - ✌ - """ - if isinstance(b, bytes): - try: - b.decode('utf8') - except: - b = '0x' + binascii.hexlify(b).decode('ascii') - return b - - def bytes_to_unicode(data, headers, **_): """Convert all *data* and *headers* to unicode.""" return ([[bytes_to_hex(v) for v in row] for row in data], From ae648abb91f92645832c54675039ea3a7d8e0154 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:47:39 -0500 Subject: [PATCH 26/76] Fix encoding issue. --- mycli/encodingutils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index ed8f9a15..43ce6c7b 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + import binascii import sys From 6ce7e158ed969a6d381f10cf0d1551ae63c04fd7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 15:27:56 -0500 Subject: [PATCH 27/76] Do not rely on tabulate for text type. --- mycli/encodingutils.py | 13 ++++++++++--- mycli/packages/expanded.py | 11 ++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index 43ce6c7b..c5f32e9d 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -7,6 +7,13 @@ PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] == 3 +if PY2: + text_type = unicode + binary_type = str +else: + text_type = str + binary_type = bytes + def unicode2utf8(arg): """ @@ -14,7 +21,7 @@ def unicode2utf8(arg): In Python 3 the args are expected as unicode. """ - if PY2 and isinstance(arg, unicode): + if PY2 and isinstance(arg, text_type): return arg.encode('utf-8') return arg @@ -25,7 +32,7 @@ def utf8tounicode(arg): In Python 3 the errors are returned as unicode. """ - if PY2 and isinstance(arg, str): + if PY2 and isinstance(arg, binary_type): return arg.decode('utf-8') return arg @@ -40,7 +47,7 @@ def bytes_to_hex(b): >>> print(bytes_to_hex('✌')) ✌ """ - if isinstance(b, bytes): + if isinstance(b, binary_type): try: b.decode('utf8') except: diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index b39d9507..d79b9acd 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,6 +1,7 @@ -from tabulate import _text_type import binascii +from .encodingutils import binary_type, text_type + def pad(field, total, char=u" "): return field + (char * (total - len(field))) @@ -12,12 +13,12 @@ def get_separator(num, header_len, data_len): def format_field(value): # Returns the field as a text type, otherwise will hexify the string try: - if isinstance(value, bytes): - return _text_type(value, "ascii") + if isinstance(value, binary_type): + return text_type(value, "ascii") else: - return _text_type(value) + return text_type(value) except UnicodeDecodeError: - return _text_type('0x' + binascii.hexlify(value).decode('ascii')) + return text_type('0x' + binascii.hexlify(value).decode('ascii')) def expanded_table(rows, headers, **_): header_len = max([len(x) for x in headers]) From 4a65f7bb98654f058e0f3cae1c06cf2f66bd33e2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 15:28:53 -0500 Subject: [PATCH 28/76] Fix import. --- mycli/packages/expanded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index d79b9acd..d1108c2e 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,6 +1,6 @@ import binascii -from .encodingutils import binary_type, text_type +from mycli.encodingutils import binary_type, text_type def pad(field, total, char=u" "): return field + (char * (total - len(field))) From 02f5c521495eec0968d99fdf603be958968a2a5e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:24:03 -0500 Subject: [PATCH 29/76] Move pre-processing to output_formatter. --- mycli/encodingutils.py | 16 +++++++------- mycli/main.py | 2 -- mycli/output_formatter.py | 45 +++++++++++++++++++++++++------------- mycli/packages/expanded.py | 29 ++++++++++-------------- tests/test_expanded.py | 3 ++- 5 files changed, 52 insertions(+), 43 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index c5f32e9d..aac4aa07 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -37,19 +37,19 @@ def utf8tounicode(arg): return arg -def bytes_to_hex(b): - """Convert bytes that cannot be decoded to utf8 to hexlified string. +def bytes_to_string(b): + """Convert bytes to a string. Hexlify bytes that can't be decoded. - >>> print(bytes_to_hex(b"\\xff")) + >>> print(bytes_to_string(b"\\xff")) 0xff - >>> print(bytes_to_hex('abc')) + >>> print(bytes_to_string('abc')) abc - >>> print(bytes_to_hex('✌')) + >>> print(bytes_to_string('✌')) ✌ """ if isinstance(b, binary_type): try: - b.decode('utf8') - except: - b = '0x' + binascii.hexlify(b).decode('ascii') + return b.decode('utf8') + except UnicodeDecodeError: + return '0x' + binascii.hexlify(b).decode('ascii') return b diff --git a/mycli/main.py b/mycli/main.py index 661dc65d..faca8b37 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -722,8 +722,6 @@ def format_output(self, title, cur, headers, status, expanded=False, output.append(title) if cur: - headers = [utf8tounicode(x) for x in headers] - rows = list(cur) formatted = self.formatter.format_output( rows, headers, format_name='expanded' if expanded else None) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index a1319fd8..5474b41f 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -12,20 +12,34 @@ from tabulate import tabulate -from .encodingutils import bytes_to_hex +from . import encodingutils from .packages.expanded import expanded_table +def to_string(value): + """Convert *value* to a string.""" + if isinstance(value, encodingutils.binary_type): + return encodingutils.bytes_to_string(value) + else: + return encodingutils.text_type(value) + + +def convert_to_string(data, headers, **_): + """Convert all *data* and *headers* to strings.""" + return ([[to_string(v) for v in row] for row in data], + [to_string(h) for h in headers]) + + def override_missing_value(data, headers, missing_value='', **_): """Override missing values in the data with *missing_value*.""" return ([[missing_value if v is None else v for v in row] for row in data], headers) -def bytes_to_unicode(data, headers, **_): - """Convert all *data* and *headers* to unicode.""" - return ([[bytes_to_hex(v) for v in row] for row in data], - [bytes_to_hex(h) for h in headers]) +def bytes_to_string(data, headers, **_): + """Convert all *data* and *headers* to strings.""" + return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], + [encodingutils.bytes_to_string(h) for h in headers]) def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): @@ -53,16 +67,16 @@ def __init__(self, format_name=None): """Register the supported output formats.""" self._output_formats = { 'csv': (csv_wrapper, { - 'preprocessor': override_missing_value, + 'preprocessor': (override_missing_value, bytes_to_string), 'missing_value': '' }), 'tsv': (csv_wrapper, { - 'preprocessor': override_missing_value, + 'preprocessor': (override_missing_value, bytes_to_string), 'missing_value': '', 'delimiter': '\t' }), 'expanded': (expanded_table, { - 'preprocessor': override_missing_value, + 'preprocessor': (override_missing_value, convert_to_string), 'missing_value': '' }) } @@ -73,11 +87,11 @@ def __init__(self, format_name=None): 'moinmoin', 'html', 'latex', 'latex_booktabs', 'textile') for tabulate_format in tabulate_formats: - self._output_formats[tabulate_format] = ( - tabulate_wrapper, {'preprocessor': bytes_to_unicode, - 'table_format': tabulate_format, - 'missing_value': ''} - ) + self._output_formats[tabulate_format] = (tabulate_wrapper, { + 'preprocessor': (bytes_to_string, ), + 'table_format': tabulate_format, + 'missing_value': '' + }) if format_name: self.set_format_name(format_name) @@ -111,7 +125,8 @@ def format_output(self, data, headers, format_name=None, **kwargs): function, fkwargs = self._output_formats[format_name] fkwargs.update(kwargs) - preprocessor = fkwargs.pop('preprocessor', None) + preprocessor = fkwargs.get('preprocessor', None) if preprocessor: - data, headers = preprocessor(data, headers, **fkwargs) + for f in preprocessor: + data, headers = f(data, headers, **fkwargs) return function(data, headers, **fkwargs) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index d1108c2e..c3a8e9a5 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,42 +1,37 @@ -import binascii +"""Format data into a vertical, expanded table layout.""" -from mycli.encodingutils import binary_type, text_type +from __future__ import unicode_literals -def pad(field, total, char=u" "): + +def pad(field, total, char=' '): return field + (char * (total - len(field))) -def get_separator(num, header_len, data_len): - sep = u"***************************[ %d. row ]***************************\n" % (num + 1) +def get_separator(num, header_len, data_len): + sep = "***************************[ %d. row ]***************************\n" % (num + 1) return sep -def format_field(value): - # Returns the field as a text type, otherwise will hexify the string - try: - if isinstance(value, binary_type): - return text_type(value, "ascii") - else: - return text_type(value) - except UnicodeDecodeError: - return text_type('0x' + binascii.hexlify(value).decode('ascii')) def expanded_table(rows, headers, **_): + """Format *rows* and *headers* as an expanded table. + + The values in *rows* and *headers* must be strings. + """ header_len = max([len(x) for x in headers]) max_row_len = 0 results = [] - padded_headers = [pad(x, header_len) + u" |" for x in headers] + padded_headers = [pad(x, header_len) + ' |' for x in headers] header_len += 2 for row in rows: - row = [format_field(x) for x in row] row_len = max([len(x) for x in row]) row_result = [] if row_len > max_row_len: max_row_len = row_len for header, value in zip(padded_headers, row): - row_result.append(u"%s %s" % (header, value)) + row_result.append('{0} {1}'.format(header, value)) results.append('\n'.join(row_result)) diff --git a/tests/test_expanded.py b/tests/test_expanded.py index 9b2a6c5c..cec94069 100644 --- a/tests/test_expanded.py +++ b/tests/test_expanded.py @@ -1,7 +1,8 @@ from mycli.packages.expanded import expanded_table +from mycli.encodingutils import text_type def test_expanded_table_renders(): - input = [("hello", 123), ("world", 456)] + input = [("hello", text_type(123)), ("world", text_type(456))] expected = """***************************[ 1. row ]*************************** name | hello From 5fe9d68c9679fe7e40bbec719a05a8c4d9053dfb Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:46:23 -0500 Subject: [PATCH 30/76] Refactor expanded table. --- mycli/packages/expanded.py | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index c3a8e9a5..0b9e6ef1 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -3,13 +3,16 @@ from __future__ import unicode_literals -def pad(field, total, char=' '): - return field + (char * (total - len(field))) +def get_separator(num, header_len): + """Get a row separator.""" + sep = "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) + return sep -def get_separator(num, header_len, data_len): - sep = "***************************[ %d. row ]***************************\n" % (num + 1) - return sep +def format_row(headers, row): + """Format a row.""" + formatted_row = [' '.join(field) for field in zip(headers, row)] + return '\n'.join(formatted_row) def expanded_table(rows, headers, **_): @@ -18,26 +21,12 @@ def expanded_table(rows, headers, **_): The values in *rows* and *headers* must be strings. """ header_len = max([len(x) for x in headers]) - max_row_len = 0 - results = [] - - padded_headers = [pad(x, header_len) + ' |' for x in headers] - header_len += 2 - - for row in rows: - row_len = max([len(x) for x in row]) - row_result = [] - if row_len > max_row_len: - max_row_len = row_len - - for header, value in zip(padded_headers, row): - row_result.append('{0} {1}'.format(header, value)) - - results.append('\n'.join(row_result)) + padded_headers = ['{} |'.format(x.ljust(header_len)) for x in headers] + results = [format_row(padded_headers, row) for row in rows] output = [] for i, result in enumerate(results): - output.append(get_separator(i, header_len, max_row_len)) + output.append(get_separator(i, header_len + 2)) output.append(result) output.append('\n') From 2052ae45925ce10412b1b1cb26a8cab593d51e0e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:48:22 -0500 Subject: [PATCH 31/76] Remove extra variable. --- mycli/packages/expanded.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 0b9e6ef1..e4283d73 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -5,8 +5,7 @@ def get_separator(num, header_len): """Get a row separator.""" - sep = "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) - return sep + return "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) def format_row(headers, row): From 328a7673e13ee40f1518edd3231b8a93471ece32 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:50:12 -0500 Subject: [PATCH 32/76] Remove extra argument to get_separator. --- mycli/packages/expanded.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index e4283d73..83ad4276 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -3,8 +3,8 @@ from __future__ import unicode_literals -def get_separator(num, header_len): - """Get a row separator.""" +def get_separator(num): + """Get a row separator for row *num*.""" return "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) @@ -25,7 +25,7 @@ def expanded_table(rows, headers, **_): output = [] for i, result in enumerate(results): - output.append(get_separator(i, header_len + 2)) + output.append(get_separator(i)) output.append(result) output.append('\n') From 041eafbca08a1729698b59f6fee252ef96c686eb Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:52:09 -0500 Subject: [PATCH 33/76] Use named format strings for clarity. --- mycli/packages/expanded.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 83ad4276..3ec93829 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -5,7 +5,8 @@ def get_separator(num): """Get a row separator for row *num*.""" - return "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) + return "{divider}[ {n}. row ]{divider}\n".format( + divider='*' * 27, n=num + 1) def format_row(headers, row): From 3a10f5bb0c4ca5b4d099541591fd9618ce5e6d23 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:56:36 -0500 Subject: [PATCH 34/76] Move divider to format_row function. --- mycli/packages/expanded.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 3ec93829..2ab57994 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -11,7 +11,7 @@ def get_separator(num): def format_row(headers, row): """Format a row.""" - formatted_row = [' '.join(field) for field in zip(headers, row)] + formatted_row = [' | '.join(field) for field in zip(headers, row)] return '\n'.join(formatted_row) @@ -21,11 +21,11 @@ def expanded_table(rows, headers, **_): The values in *rows* and *headers* must be strings. """ header_len = max([len(x) for x in headers]) - padded_headers = ['{} |'.format(x.ljust(header_len)) for x in headers] - results = [format_row(padded_headers, row) for row in rows] + padded_headers = [x.ljust(header_len) for x in headers] + formatted_rows = [format_row(padded_headers, row) for row in rows] output = [] - for i, result in enumerate(results): + for i, result in enumerate(formatted_rows): output.append(get_separator(i)) output.append(result) output.append('\n') From 61bdbd98c5569b84a7b994bd67a457d4bfe62a2a Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 21:03:17 -0500 Subject: [PATCH 35/76] Make the expanded format test more readable. --- tests/test_expanded.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/test_expanded.py b/tests/test_expanded.py index cec94069..db503cba 100644 --- a/tests/test_expanded.py +++ b/tests/test_expanded.py @@ -1,14 +1,19 @@ +"""Test the vertical, expanded table formatter.""" +from textwrap import dedent + from mycli.packages.expanded import expanded_table from mycli.encodingutils import text_type + def test_expanded_table_renders(): - input = [("hello", text_type(123)), ("world", text_type(456))] - - expected = """***************************[ 1. row ]*************************** -name | hello -age | 123 -***************************[ 2. row ]*************************** -name | world -age | 456 -""" - assert expected == expanded_table(input, ["name", "age"]) + results = [('hello', text_type(123)), ('world', text_type(456))] + + expected = dedent("""\ + ***************************[ 1. row ]*************************** + name | hello + age | 123 + ***************************[ 2. row ]*************************** + name | world + age | 456 + """) + assert expected == expanded_table(results, ('name', 'age')) From 0f01a9d5767e1dc0d079633e806770099255af28 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 21:08:44 -0500 Subject: [PATCH 36/76] Update docstring to be clearer. --- mycli/output_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5474b41f..86db809c 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -37,7 +37,7 @@ def override_missing_value(data, headers, missing_value='', **_): def bytes_to_string(data, headers, **_): - """Convert all *data* and *headers* to strings.""" + """Convert all *data* and *headers* bytes to strings.""" return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], [encodingutils.bytes_to_string(h) for h in headers]) From 4b47e43eb8b7d332d64bd6c2f6575e34d3451b67 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Thu, 23 Mar 2017 12:58:37 +0100 Subject: [PATCH 37/76] Use pymysql default conversions (instead of only our conversions) This will convert a MySQL DECIMAL to the Python's decimal.Decimal type. Tabulate handles Decimal as a string (and not a number) and will not remove the padding (fixes Issue#375) --- changelog.md | 1 + mycli/sqlexecute.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 924d8efa..f87407e3 100644 --- a/changelog.md +++ b/changelog.md @@ -10,6 +10,7 @@ Bug Fixes: * Fix requirements and remove old compatibility code (Thanks: [Dick Marinus]) * Fix bug where mycli would not start due to the thanks/credit intro text. (Thanks: [Thomas Roten]). +* Use pymysql default conversions (issue #375). (Thanks: [Dick Marinus]). Internal Changes: ----------------- diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 786c8f0e..08030af4 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -4,7 +4,7 @@ from .packages import special from pymysql.constants import FIELD_TYPE from pymysql.converters import (convert_mysql_timestamp, convert_datetime, - convert_timedelta, convert_date) + convert_timedelta, convert_date, conversions) _logger = logging.getLogger(__name__) @@ -65,12 +65,13 @@ def connect(self, database=None, user=None, password=None, host=None, '\tlocal_infile: %r' '\tssl: %r', database, user, host, port, socket, charset, local_infile, ssl) - conv = { + conv = conversions.copy() + conv.update({ FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or obj), FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or obj), FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or obj), FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or obj), - } + }) conn = pymysql.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, From 5839c11d641f97660a6cded8e6a354ec54d72ce1 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sun, 2 Apr 2017 18:42:54 +0200 Subject: [PATCH 38/76] align floats on decimal point, quote columns with trailing/starting whitespace --- mycli/output_formatter.py | 97 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5474b41f..35e26674 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -15,6 +15,8 @@ from . import encodingutils from .packages.expanded import expanded_table +from decimal import Decimal + def to_string(value): """Convert *value* to a string.""" @@ -42,6 +44,87 @@ def bytes_to_string(data, headers, **_): [encodingutils.bytes_to_string(h) for h in headers]) +def intlen(value): + """Find (character) length + >>> intlen('11.1') + 2 + >>> intlen('11') + 2 + >>> intlen('1.1') + 1 + """ + pos = value.find('.') + if pos < 0: + pos = len(value) + return pos + +def align_decimals(data, headers, **_): + """Align decimals to decimal point + >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) + 1 + 11.1 + 1.1 + """ + pointpos = len(data[0]) * [0] + for row in data: + i = 0 + for v in row: + if isinstance(v, Decimal): + v = str(v) + pointpos[i] = max(intlen(v), pointpos[i]) + i += 1 + results = [] + for row in data: + i = 0 + result = [] + for v in row: + if isinstance(v, Decimal): + v = str(v) + result.append((pointpos[i]-intlen(v))*" "+v) + else: + result.append(v) + i += 1 + results.append(result) + return results, headers + + +def quote_whitespaces(data, headers, quotestyle="'", **_): + """Quote whitespace + >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) + ' before' + 'after ' + ' both ' + 'none' + >>> for i in quote_whitespaces([["abc"], ["def"], ["ghi"], ["jkl"]], [])[0]: print(i[0]) + abc + def + ghi + jkl + """ + """Convert all *data* and *headers* to strings.""" + quote = len(data[0])*[False] + for row in data: + i = 0 + for v in row: + v = encodingutils.text_type(v) + if v[0] == ' ' or v[-1] == ' ': + quote[i] = True + i += 1 + + results = [] + for row in data: + result = [] + i = 0 + for v in row: + if quote[i]: + result.append('{quotestyle}{value}{quotestyle}'.format(quotestyle=quotestyle, value=v)) + else: + result.append(v) + i += 1 + results.append(result) + return results, headers + + def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, @@ -88,7 +171,7 @@ def __init__(self, format_name=None): 'textile') for tabulate_format in tabulate_formats: self._output_formats[tabulate_format] = (tabulate_wrapper, { - 'preprocessor': (bytes_to_string, ), + 'preprocessor': (bytes_to_string, align_decimals, quote_whitespaces), 'table_format': tabulate_format, 'missing_value': '' }) @@ -118,6 +201,18 @@ def format_output(self, data, headers, format_name=None, **kwargs): *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. + >>> print(OutputFormatter().format_output( \ + [["abc", Decimal(1)], ["defg", Decimal('11.1')], ["hi", Decimal('1.1')]], \ + ["text", "numeric"], \ + "psql" \ + )) + +--------+-----------+ + | text | numeric | + |--------+-----------| + | abc | ' 1' | + | defg | '11.1' | + | hi | ' 1.1' | + +--------+-----------+ """ format_name = format_name or self._format_name if format_name not in self.supported_formats(): From 1db7f865b91f57baa19fd27234506d6974895801 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 13:24:47 -0500 Subject: [PATCH 39/76] Add basic output_formatter tests. --- tests/test_output_formatter.py | 82 ++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 tests/test_output_formatter.py diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py new file mode 100644 index 00000000..f331262e --- /dev/null +++ b/tests/test_output_formatter.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +"""Test the generic output formatter interface.""" + +from __future__ import unicode_literals + +from textwrap import dedent + +from mycli.output_formatter import (bytes_to_string, convert_to_string, + csv_wrapper, OutputFormatter, + override_missing_value, tabulate_wrapper, + to_string) + + +def test_to_string(): + """Test the *output_formatter.to_string()* function.""" + assert 'a' == to_string('a') + assert 'a' == to_string(b'a') + assert '1' == to_string(1) + assert '1.23' == to_string(1.23) + + +def test_convert_to_string(): + """Test the *output_formatter.convert_to_string()* function.""" + data = [[1, 'John'], [2, 'Jill']] + headers = [0, 'name'] + expected = ([['1', 'John'], ['2', 'Jill']], ['0', 'name']) + + assert expected == convert_to_string(data, headers) + + +def test_override_missing_values(): + """Test the *output_formatter.override_missing_values()* function.""" + data = [[1, None], [2, 'Jill']] + headers = [0, 'name'] + expected = ([[1, ''], [2, 'Jill']], [0, 'name']) + + assert expected == override_missing_value(data, headers, + missing_value='') + + +def test_bytes_to_string(): + """Test the *output_formatter.bytes_to_string()* function.""" + data = [[1, 'John'], [2, b'Jill']] + headers = [0, 'name'] + expected = ([[1, 'John'], [2, 'Jill']], [0, 'name']) + + assert expected == bytes_to_string(data, headers) + + +def test_tabulate_wrapper(): + """Test the *output_formatter.tabulate_wrapper()* function.""" + data = [['abc', 1], ['d', 456]] + headers = ['letters', 'number'] + output = tabulate_wrapper(data, headers, table_format='psql') + assert output == dedent('''\ + +-----------+----------+ + | letters | number | + |-----------+----------| + | abc | 1 | + | d | 456 | + +-----------+----------+''') + + +def test_csv_wrapper(): + """Test the *output_formatter.csv_wrapper()* function.""" + # Test comma-delimited output. + data = [['abc', 1], ['d', 456]] + headers = ['letters', 'number'] + output = csv_wrapper(data, headers) + assert output == dedent('''\ + letters,number\r\n\ + abc,1\r\n\ + d,456\r\n''') + + # Test tab-delimited output. + data = [['abc', 1], ['d', 456]] + headers = ['letters', 'number'] + output = csv_wrapper(data, headers, delimiter='\t') + assert output == dedent('''\ + letters\tnumber\r\n\ + abc\t1\r\n\ + d\t456\r\n''') From 86846167f1fd159a99b89f75d4e24d8290eff85e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 14:28:30 -0500 Subject: [PATCH 40/76] Add terminaltables. --- mycli/output_formatter.py | 33 ++++++++++++++++++++++++++++++--- setup.py | 1 + tests/test_output_formatter.py | 16 +++++++++++++++- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index bc275984..566a337f 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -5,18 +5,18 @@ import contextlib import csv +from decimal import Decimal try: from cStringIO import StringIO except ImportError: from io import StringIO from tabulate import tabulate +import terminaltables from . import encodingutils from .packages.expanded import expanded_table -from decimal import Decimal - def to_string(value): """Convert *value* to a string.""" @@ -143,6 +143,23 @@ def csv_wrapper(data, headers, delimiter=',', **_): return content.getvalue() +def terminal_tables_wrapper(data, headers, table_format=None, **_): + """Wrap terminaltables inside a standard function for OutputFormatter.""" + if table_format == 'ascii': + table = terminaltables.AsciiTable + elif table_format == 'single': + table = terminaltables.SingleTable + elif table_format == 'double': + table = terminaltables.DoubleTable + elif table_format == 'github': + table = terminaltables.GithubFlavoredMarkdownTable + else: + raise ValueError('unrecognized table format: {}'.format(table_format)) + + t = table([headers] + data) + return t.table + + class OutputFormatter(object): """A class with a standard interface for various formatting libraries.""" @@ -171,11 +188,21 @@ def __init__(self, format_name=None): 'textile') for tabulate_format in tabulate_formats: self._output_formats[tabulate_format] = (tabulate_wrapper, { - 'preprocessor': (bytes_to_string, align_decimals, quote_whitespaces), + 'preprocessor': (bytes_to_string, align_decimals), 'table_format': tabulate_format, 'missing_value': '' }) + terminal_tables_formats = ('ascii', 'single', 'double', 'github') + for terminal_tables_format in terminal_tables_formats: + self._output_formats[terminal_tables_format] = ( + terminal_tables_wrapper, { + 'preprocessor': (bytes_to_string, override_missing_value, + align_decimals), + 'table_format': terminal_tables_format, + 'missing_value': '' + }) + if format_name: self.set_format_name(format_name) diff --git a/setup.py b/setup.py index 3ecc21c4..21e55508 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ 'configobj >= 5.0.5', 'pycryptodome >= 3', 'tabulate >= 0.7.6', + 'terminaltables >= 3.0.0', ] setup( diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index f331262e..66ac4237 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -8,7 +8,7 @@ from mycli.output_formatter import (bytes_to_string, convert_to_string, csv_wrapper, OutputFormatter, override_missing_value, tabulate_wrapper, - to_string) + terminal_tables_wrapper, to_string) def test_to_string(): @@ -80,3 +80,17 @@ def test_csv_wrapper(): letters\tnumber\r\n\ abc\t1\r\n\ d\t456\r\n''') + + +def test_terminal_tables_wrapper(): + """Test the *output_formatter.terminal_tables_wrapper()* function.""" + data = [['abc', 1], ['d', 456]] + headers = ['letters', 'number'] + output = terminal_tables_wrapper(data, headers, table_format='ascii') + assert output == dedent('''\ + +---------+--------+ + | letters | number | + +---------+--------+ + | abc | 1 | + | d | 456 | + +---------+--------+''') From 4c13163941b64dab71bad9a91906351f606c0e06 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 14:43:49 -0500 Subject: [PATCH 41/76] Default myclirc to ascii table. --- mycli/myclirc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mycli/myclirc b/mycli/myclirc index 36e9e409..2f359927 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -30,11 +30,9 @@ log_level = INFO # Timing of sql statments and table rendering. timing = True -# Table format. Possible values: psql, plain, simple, grid, fancy_grid, pipe, -# orgtbl, jira, rst, mediawiki, moinmoin, html, latex, latex_booktabs, -# textile, csv, tsv. -# Recommended: psql, fancy_grid and grid. -table_format = psql +# Table format. Possible values: ascii, single, double, or github. +# Recommended: ascii +table_format = ascii # Syntax coloring style. Possible values (many support the "-dark" suffix): # manni, igor, xcode, vim, autumn, vs, rrt, native, perldoc, borland, tango, emacs, From dd41cf394602523331d5c57152a6f223a7c52d06 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 14:43:58 -0500 Subject: [PATCH 42/76] Fix table format tests. --- mycli/output_formatter.py | 16 ++++++++-------- tests/test_main.py | 14 +++++++------- tests/test_sqlexecute.py | 32 ++++++++++++++++---------------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 566a337f..5e330531 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -231,15 +231,15 @@ def format_output(self, data, headers, format_name=None, **kwargs): >>> print(OutputFormatter().format_output( \ [["abc", Decimal(1)], ["defg", Decimal('11.1')], ["hi", Decimal('1.1')]], \ ["text", "numeric"], \ - "psql" \ + "ascii" \ )) - +--------+-----------+ - | text | numeric | - |--------+-----------| - | abc | ' 1' | - | defg | '11.1' | - | hi | ' 1.1' | - +--------+-----------+ + +------+---------+ + | text | numeric | + +------+---------+ + | abc | 1 | + | defg | 11.1 | + | hi | 1.1 | + +------+---------+ """ format_name = format_name or self._format_name if format_name not in self.supported_formats(): diff --git a/tests/test_main.py b/tests/test_main.py index 29501f13..1271f18d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -48,7 +48,7 @@ def test_execute_arg_with_table(executor): sql = 'select * from test;' runner = CliRunner() result = runner.invoke(cli, args=CLI_ARGS + ['-e', sql] + ['--table']) - expected = '+-----+\n| a |\n|-----|\n| abc |\n+-----+\n' + expected = '+-----+\n| a |\n+-----+\n| abc |\n+-----+\n' assert result.exit_code == 0 assert expected in result.output @@ -98,14 +98,14 @@ def test_batch_mode_table(executor): result = runner.invoke(cli, args=CLI_ARGS + ['-t'], input=sql) expected = (dedent("""\ - +------------+ - | count(*) | - |------------| - | 3 | - +------------+ + +----------+ + | count(*) | + +----------+ + | 3 | + +----------+ +-----+ | a | - |-----| + +-----+ | abc | +-----+""")) diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index d964f43b..a9b5fbec 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -15,7 +15,7 @@ def test_conn(executor): assert results == dedent("""\ +-----+ | a | - |-----| + +-----+ | abc | +-----+ 1 row in set""") @@ -26,11 +26,11 @@ def test_bools(executor): run(executor, '''insert into test values(True)''') results = run(executor, '''select * from test''', join=True) assert results == dedent("""\ - +-----+ - | a | - |-----| - | 1 | - +-----+ + +---+ + | a | + +---+ + | 1 | + +---+ 1 row in set""") @dbtest @@ -41,7 +41,7 @@ def test_binary(executor): assert results == dedent("""\ +----------------------------------------------------------------------------------------------+ | geom | - |----------------------------------------------------------------------------------------------| + +----------------------------------------------------------------------------------------------+ | 0x00000000010200000002000000397f130a11185d4034f44f70b1de43400000000000185d40423ee8d9acde4340 | +----------------------------------------------------------------------------------------------+ 1 row in set""") @@ -140,7 +140,7 @@ def test_favorite_query(executor): > select * from test where a like 'a%' +-----+ | a | - |-----| + +-----+ | abc | +-----+""") @@ -163,13 +163,13 @@ def test_favorite_query_multiple_statement(executor): > select * from test where a like 'a%' +-----+ | a | - |-----| + +-----+ | abc | +-----+ > select * from test where a like 'd%' +-----+ | a | - |-----| + +-----+ | def | +-----+""") @@ -261,13 +261,13 @@ def test_favorite_query_multiline_statement(executor): > select * from test where a like 'a%' +-----+ | a | - |-----| + +-----+ | abc | +-----+ > select * from test where a like 'd%' +-----+ | a | - |-----| + +-----+ | def | +-----+""") @@ -282,7 +282,7 @@ def test_timestamp_null(executor): assert results == dedent("""\ +---------------------+ | a | - |---------------------| + +---------------------+ | 0000-00-00 00:00:00 | +---------------------+ 1 row in set""") @@ -295,7 +295,7 @@ def test_datetime_null(executor): assert results == dedent("""\ +---------------------+ | a | - |---------------------| + +---------------------+ | 0000-00-00 00:00:00 | +---------------------+ 1 row in set""") @@ -308,7 +308,7 @@ def test_date_null(executor): assert results == dedent("""\ +------------+ | a | - |------------| + +------------+ | 0000-00-00 | +------------+ 1 row in set""") @@ -321,7 +321,7 @@ def test_time_null(executor): assert results == dedent("""\ +----------+ | a | - |----------| + +----------+ | 00:00:00 | +----------+ 1 row in set""") From 145d5d2af0e5bf15fdaa3ffbf443eb2942244691 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 15:17:34 -0500 Subject: [PATCH 43/76] Add a multi-column test for align_decimals. --- mycli/output_formatter.py | 28 ++++++++++------------------ tests/test_output_formatter.py | 19 +++++++++++++++---- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5e330531..068f0246 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -67,23 +67,19 @@ def align_decimals(data, headers, **_): """ pointpos = len(data[0]) * [0] for row in data: - i = 0 - for v in row: + for i, v in enumerate(row): if isinstance(v, Decimal): - v = str(v) + v = encodingutils.text_type(v) pointpos[i] = max(intlen(v), pointpos[i]) - i += 1 results = [] for row in data: - i = 0 result = [] - for v in row: + for i, v in enumerate(row): if isinstance(v, Decimal): - v = str(v) - result.append((pointpos[i]-intlen(v))*" "+v) + v = encodingutils.text_type(v) + result.append((pointpos[i] - intlen(v)) * " " + v) else: result.append(v) - i += 1 results.append(result) return results, headers @@ -101,26 +97,22 @@ def quote_whitespaces(data, headers, quotestyle="'", **_): ghi jkl """ - """Convert all *data* and *headers* to strings.""" - quote = len(data[0])*[False] + quote = len(data[0]) * [False] for row in data: - i = 0 - for v in row: + for i, v in enumerate(row): v = encodingutils.text_type(v) if v[0] == ' ' or v[-1] == ' ': quote[i] = True - i += 1 results = [] for row in data: result = [] - i = 0 - for v in row: + for i, v in enumerate(row): if quote[i]: - result.append('{quotestyle}{value}{quotestyle}'.format(quotestyle=quotestyle, value=v)) + result.append('{quotestyle}{value}{quotestyle}'.format( + quotestyle=quotestyle, value=v)) else: result.append(v) - i += 1 results.append(result) return results, headers diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 66ac4237..045bd730 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -3,12 +3,14 @@ from __future__ import unicode_literals +from decimal import Decimal from textwrap import dedent -from mycli.output_formatter import (bytes_to_string, convert_to_string, - csv_wrapper, OutputFormatter, - override_missing_value, tabulate_wrapper, - terminal_tables_wrapper, to_string) +from mycli.output_formatter import (align_decimals, bytes_to_string, + convert_to_string, csv_wrapper, + OutputFormatter, override_missing_value, + tabulate_wrapper, terminal_tables_wrapper, + to_string) def test_to_string(): @@ -47,6 +49,15 @@ def test_bytes_to_string(): assert expected == bytes_to_string(data, headers) +def test_align_decimals(): + """Test the *output_formatter.align_decimals()* function.""" + data = [[Decimal('200'), Decimal('1')], [Decimal('1.00002'), Decimal('1.0')]] + headers = ['num1', 'num2'] + expected = ([['200', '1'], [' 1.00002', '1.0']], ['num1', 'num2']) + + assert expected == align_decimals(data, headers) + + def test_tabulate_wrapper(): """Test the *output_formatter.tabulate_wrapper()* function.""" data = [['abc', 1], ['d', 456]] From 6ea35bad05cee2df19e37fd19301b1129636e264 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 19:19:30 -0500 Subject: [PATCH 44/76] Format/idiomatic changes. --- mycli/output_formatter.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 068f0246..549ee4e8 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -101,18 +101,16 @@ def quote_whitespaces(data, headers, quotestyle="'", **_): for row in data: for i, v in enumerate(row): v = encodingutils.text_type(v) - if v[0] == ' ' or v[-1] == ' ': + if v.startswith(' ') or v.endswith(' '): quote[i] = True results = [] for row in data: result = [] for i, v in enumerate(row): - if quote[i]: - result.append('{quotestyle}{value}{quotestyle}'.format( - quotestyle=quotestyle, value=v)) - else: - result.append(v) + quotation = quotestyle if quote[i] else '' + result.append('{quotestyle}{value}{quotestyle}'.format( + quotestyle=quotation, value=v)) results.append(result) return results, headers @@ -192,8 +190,8 @@ def __init__(self, format_name=None): 'preprocessor': (bytes_to_string, override_missing_value, align_decimals), 'table_format': terminal_tables_format, - 'missing_value': '' - }) + 'missing_value': ''} + ) if format_name: self.set_format_name(format_name) @@ -220,10 +218,12 @@ def format_output(self, data, headers, format_name=None, **kwargs): *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. - >>> print(OutputFormatter().format_output( \ - [["abc", Decimal(1)], ["defg", Decimal('11.1')], ["hi", Decimal('1.1')]], \ - ["text", "numeric"], \ - "ascii" \ + + >>> print(OutputFormatter().format_output(\ + [["abc", Decimal(1)], ["defg", Decimal('11.1')],\ + ["hi", Decimal('1.1')]],\ + ["text", "numeric"],\ + "ascii"\ )) +------+---------+ | text | numeric | From c51cfac3c1458a07a2f9b8acb59b2ef2f9321fbb Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 19:19:53 -0500 Subject: [PATCH 45/76] Move format_output tests to tests module. --- mycli/output_formatter.py | 14 -------------- tests/test_output_formatter.py | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 549ee4e8..6656ec8c 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -218,20 +218,6 @@ def format_output(self, data, headers, format_name=None, **kwargs): *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. - - >>> print(OutputFormatter().format_output(\ - [["abc", Decimal(1)], ["defg", Decimal('11.1')],\ - ["hi", Decimal('1.1')]],\ - ["text", "numeric"],\ - "ascii"\ - )) - +------+---------+ - | text | numeric | - +------+---------+ - | abc | 1 | - | defg | 11.1 | - | hi | 1.1 | - +------+---------+ """ format_name = format_name or self._format_name if format_name not in self.supported_formats(): diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 045bd730..8ddeb888 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -105,3 +105,21 @@ def test_terminal_tables_wrapper(): | abc | 1 | | d | 456 | +---------+--------+''') + + +def test_output_formatter(): + """Test the *output_formatter.OutputFormatter* class.""" + data = [['abc', Decimal(1)], ['defg', Decimal('11.1')], + ['hi', Decimal('1.1')]] + headers = ['text', 'numeric'] + expected = dedent('''\ + +------+---------+ + | text | numeric | + +------+---------+ + | abc | 1 | + | defg | 11.1 | + | hi | 1.1 | + +------+---------+''') + + assert expected == OutputFormatter().format_output(data, headers, + format_name='ascii') From a36cf85b53f36221032acb6e2ee704e0900d3a0d Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 17:12:48 -0700 Subject: [PATCH 46/76] Refactor OutputFormatter to remove tight coupling. --- mycli/main.py | 7 +- mycli/output_formatter.py | 232 ------------------ mycli/output_formatter/__init__.py | 0 .../delimited_output_adapter.py | 22 ++ mycli/output_formatter/output_formatter.py | 91 +++++++ mycli/output_formatter/preprocessors.py | 98 ++++++++ mycli/output_formatter/tabulate_adapter.py | 14 ++ .../terminaltables_adapter.py | 24 ++ mycli/sqlcompleter.py | 6 +- 9 files changed, 255 insertions(+), 239 deletions(-) delete mode 100644 mycli/output_formatter.py create mode 100644 mycli/output_formatter/__init__.py create mode 100644 mycli/output_formatter/delimited_output_adapter.py create mode 100644 mycli/output_formatter/output_formatter.py create mode 100644 mycli/output_formatter/preprocessors.py create mode 100644 mycli/output_formatter/tabulate_adapter.py create mode 100644 mycli/output_formatter/terminaltables_adapter.py diff --git a/mycli/main.py b/mycli/main.py index faca8b37..0b948a85 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -37,7 +37,7 @@ from .config import (write_default_config, get_mylogin_cnf_path, open_mylogin_cnf, read_config_files, str_to_bool) from .key_bindings import mycli_bindings -from .output_formatter import OutputFormatter +from .output_formatter import output_formatter from .encodingutils import utf8tounicode from .lexer import MyCliLexer from .__init__ import __version__ @@ -106,7 +106,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.multi_line = c['main'].as_bool('multi_line') self.key_bindings = c['main']['key_bindings'] special.set_timing_enabled(c['main'].as_bool('timing')) - self.formatter = OutputFormatter(format_name=c['main']['table_format']) + self.formatter = output_formatter.OutputFormatter(format_name=c['main']['table_format']) self.syntax_style = c['main']['syntax_style'] self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] @@ -145,7 +145,8 @@ def __init__(self, sqlexecute=None, prompt=None, # Initialize completer. self.smart_completion = c['main'].as_bool('smart_completion') - self.completer = SQLCompleter(self.smart_completion) + self.completer = SQLCompleter(self.smart_completion, + supported_formats=self.formatter.supported_formats) self._completer_lock = threading.Lock() # Register custom special commands. diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py deleted file mode 100644 index 6656ec8c..00000000 --- a/mycli/output_formatter.py +++ /dev/null @@ -1,232 +0,0 @@ -# -*- coding: utf-8 -*- -"""A generic output formatter interface.""" - -from __future__ import unicode_literals - -import contextlib -import csv -from decimal import Decimal -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - -from tabulate import tabulate -import terminaltables - -from . import encodingutils -from .packages.expanded import expanded_table - - -def to_string(value): - """Convert *value* to a string.""" - if isinstance(value, encodingutils.binary_type): - return encodingutils.bytes_to_string(value) - else: - return encodingutils.text_type(value) - - -def convert_to_string(data, headers, **_): - """Convert all *data* and *headers* to strings.""" - return ([[to_string(v) for v in row] for row in data], - [to_string(h) for h in headers]) - - -def override_missing_value(data, headers, missing_value='', **_): - """Override missing values in the data with *missing_value*.""" - return ([[missing_value if v is None else v for v in row] for row in data], - headers) - - -def bytes_to_string(data, headers, **_): - """Convert all *data* and *headers* bytes to strings.""" - return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], - [encodingutils.bytes_to_string(h) for h in headers]) - - -def intlen(value): - """Find (character) length - >>> intlen('11.1') - 2 - >>> intlen('11') - 2 - >>> intlen('1.1') - 1 - """ - pos = value.find('.') - if pos < 0: - pos = len(value) - return pos - -def align_decimals(data, headers, **_): - """Align decimals to decimal point - >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) - 1 - 11.1 - 1.1 - """ - pointpos = len(data[0]) * [0] - for row in data: - for i, v in enumerate(row): - if isinstance(v, Decimal): - v = encodingutils.text_type(v) - pointpos[i] = max(intlen(v), pointpos[i]) - results = [] - for row in data: - result = [] - for i, v in enumerate(row): - if isinstance(v, Decimal): - v = encodingutils.text_type(v) - result.append((pointpos[i] - intlen(v)) * " " + v) - else: - result.append(v) - results.append(result) - return results, headers - - -def quote_whitespaces(data, headers, quotestyle="'", **_): - """Quote whitespace - >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) - ' before' - 'after ' - ' both ' - 'none' - >>> for i in quote_whitespaces([["abc"], ["def"], ["ghi"], ["jkl"]], [])[0]: print(i[0]) - abc - def - ghi - jkl - """ - quote = len(data[0]) * [False] - for row in data: - for i, v in enumerate(row): - v = encodingutils.text_type(v) - if v.startswith(' ') or v.endswith(' '): - quote[i] = True - - results = [] - for row in data: - result = [] - for i, v in enumerate(row): - quotation = quotestyle if quote[i] else '' - result.append('{quotestyle}{value}{quotestyle}'.format( - quotestyle=quotation, value=v)) - results.append(result) - return results, headers - - -def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): - """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate(data, headers, tablefmt=table_format, - missingval=missing_value, disable_numparse=True) - - -def csv_wrapper(data, headers, delimiter=',', **_): - """Wrap CSV formatting inside a standard function for OutputFormatter.""" - with contextlib.closing(StringIO()) as content: - writer = csv.writer(content, delimiter=str(delimiter)) - - writer.writerow(headers) - for row in data: - writer.writerow(row) - - return content.getvalue() - - -def terminal_tables_wrapper(data, headers, table_format=None, **_): - """Wrap terminaltables inside a standard function for OutputFormatter.""" - if table_format == 'ascii': - table = terminaltables.AsciiTable - elif table_format == 'single': - table = terminaltables.SingleTable - elif table_format == 'double': - table = terminaltables.DoubleTable - elif table_format == 'github': - table = terminaltables.GithubFlavoredMarkdownTable - else: - raise ValueError('unrecognized table format: {}'.format(table_format)) - - t = table([headers] + data) - return t.table - - -class OutputFormatter(object): - """A class with a standard interface for various formatting libraries.""" - - def __init__(self, format_name=None): - """Register the supported output formats.""" - self._output_formats = { - 'csv': (csv_wrapper, { - 'preprocessor': (override_missing_value, bytes_to_string), - 'missing_value': '' - }), - 'tsv': (csv_wrapper, { - 'preprocessor': (override_missing_value, bytes_to_string), - 'missing_value': '', - 'delimiter': '\t' - }), - 'expanded': (expanded_table, { - 'preprocessor': (override_missing_value, convert_to_string), - 'missing_value': '' - }) - } - self._format_name = None - - tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', - 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', - 'moinmoin', 'html', 'latex', 'latex_booktabs', - 'textile') - for tabulate_format in tabulate_formats: - self._output_formats[tabulate_format] = (tabulate_wrapper, { - 'preprocessor': (bytes_to_string, align_decimals), - 'table_format': tabulate_format, - 'missing_value': '' - }) - - terminal_tables_formats = ('ascii', 'single', 'double', 'github') - for terminal_tables_format in terminal_tables_formats: - self._output_formats[terminal_tables_format] = ( - terminal_tables_wrapper, { - 'preprocessor': (bytes_to_string, override_missing_value, - align_decimals), - 'table_format': terminal_tables_format, - 'missing_value': ''} - ) - - if format_name: - self.set_format_name(format_name) - - def set_format_name(self, format_name): - """Set the OutputFormatter's default format.""" - if format_name in self.supported_formats(): - self._format_name = format_name - else: - raise ValueError('unrecognized format_name: {}'.format( - format_name)) - - def get_format_name(self): - """Get the OutputFormatter's default format.""" - return self._format_name - - def supported_formats(self): - """Return the supported output format names.""" - return tuple(self._output_formats.keys()) - - def format_output(self, data, headers, format_name=None, **kwargs): - """Format the headers and data using a specific formatter. - - *format_name* must be a formatter available in `supported_formats()`. - - All keyword arguments are passed to the specified formatter. - """ - format_name = format_name or self._format_name - if format_name not in self.supported_formats(): - raise ValueError('unrecognized format: {}'.format(format_name)) - - function, fkwargs = self._output_formats[format_name] - fkwargs.update(kwargs) - preprocessor = fkwargs.get('preprocessor', None) - if preprocessor: - for f in preprocessor: - data, headers = f(data, headers, **fkwargs) - return function(data, headers, **fkwargs) diff --git a/mycli/output_formatter/__init__.py b/mycli/output_formatter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py new file mode 100644 index 00000000..04571e38 --- /dev/null +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -0,0 +1,22 @@ +import contextlib +import csv +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + +from .preprocessors import (override_missing_value, bytes_to_string) + +supported_formats = ('csv',) +delimiter_preprocessors = (override_missing_value, bytes_to_string) + +def delimiter_adapter(data, headers, delimiter=',', **_): + """Wrap CSV formatting inside a standard function for OutputFormatter.""" + with contextlib.closing(StringIO()) as content: + writer = csv.writer(content, delimiter=str(delimiter)) + + writer.writerow(headers) + for row in data: + writer.writerow(row) + + return content.getvalue() diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py new file mode 100644 index 00000000..e4b28023 --- /dev/null +++ b/mycli/output_formatter/output_formatter.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from ..packages.expanded import expanded_table +from .preprocessors import (override_missing_value, convert_to_string) +from .delimited_output_adapter import delimiter_adapter, delimiter_preprocessors +from .tabulate_adapter import (tabulate_adapter, + supported_formats as tabulate_formats, + preprocessors as tabulate_preprocessors) +from .terminaltables_adapter import (terminaltables_adapter, + preprocessors as terminaltables_preprocessors, + supported_formats as terminaltables_formats) +from collections import namedtuple + +OutputFormatHandler = namedtuple('OutputFormatHandler', + 'format_name preprocessors formatter formatter_args') + +"""A generic output formatter interface.""" +class OutputFormatter(object): + """A class with a standard interface for various formatting libraries.""" + + _output_formats = {} + + def __init__(self, format_name=None): + """Register the supported output formats.""" + self._format_name = format_name + + def set_format_name(self, format_name): + """Set the OutputFormatter's default format.""" + if format_name in self.supported_formats(): + self._format_name = format_name + else: + raise ValueError('unrecognized format_name: {}'.format( + format_name)) + + def get_format_name(self): + """Get the OutputFormatter's default format.""" + return self._format_name + + def supported_formats(self): + """Return the supported output format names.""" + return tuple(self._output_formats.keys()) + + @classmethod + def register_new_formatter(cls, format_name, handler, preprocessors=None, + kwargs=None): + """Register a new fomatter to format the output""" + cls._output_formats[format_name] = OutputFormatHandler(format_name, + preprocessors, handler, kwargs) + + def format_output(self, data, headers, format_name=None, **kwargs): + """Format the headers and data using a specific formatter. + + *format_name* must be a formatter available in `supported_formats()`. + + All keyword arguments are passed to the specified formatter. + """ + format_name = format_name or self._format_name + if format_name not in self.supported_formats(): + raise ValueError('unrecognized format: {}'.format(format_name)) + + (_, preprocessors, formatter, fkwargs) = self._output_formats[format_name] + fkwargs.update(kwargs) + if preprocessors: + for f in preprocessors: + data, headers = f(data, headers, **fkwargs) + return formatter(data, headers, **fkwargs) + +OutputFormatter.register_new_formatter('csv', + delimiter_adapter, + delimiter_preprocessors, {'missing_value': ''}) +OutputFormatter.register_new_formatter('tsv', + delimiter_adapter, + delimiter_preprocessors, + {'missing_value': '', 'delimiter': '\t'}) +OutputFormatter.register_new_formatter('expanded', + expanded_table, + (override_missing_value, convert_to_string), + {'missing_value': '', 'delimiter': '\t'}) + +for tabulate_format in tabulate_formats: + OutputFormatter.register_new_formatter(tabulate_format, + tabulate_adapter, + tabulate_preprocessors, + {'table_format': tabulate_format, 'missing_value': ''}) + +for terminaltables_format in terminaltables_formats: + OutputFormatter.register_new_formatter(terminaltables_format, + terminaltables_adapter, + terminaltables_preprocessors, + {'table_format': terminaltables_format, 'missing_value': ''}) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py new file mode 100644 index 00000000..b3a9373a --- /dev/null +++ b/mycli/output_formatter/preprocessors.py @@ -0,0 +1,98 @@ +from decimal import Decimal +from .. import encodingutils + +def to_string(value): + """Convert *value* to a string.""" + if isinstance(value, encodingutils.binary_type): + return encodingutils.bytes_to_string(value) + else: + return encodingutils.text_type(value) + + +def convert_to_string(data, headers, **_): + """Convert all *data* and *headers* to strings.""" + return ([[to_string(v) for v in row] for row in data], + [to_string(h) for h in headers]) + + +def override_missing_value(data, headers, missing_value='', **_): + """Override missing values in the data with *missing_value*.""" + return ([[missing_value if v is None else v for v in row] for row in data], + headers) + + +def bytes_to_string(data, headers, **_): + """Convert all *data* and *headers* bytes to strings.""" + return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], + [encodingutils.bytes_to_string(h) for h in headers]) + + +def intlen(value): + """Find (character) length + >>> intlen('11.1') + 2 + >>> intlen('11') + 2 + >>> intlen('1.1') + 1 + """ + pos = value.find('.') + if pos < 0: + pos = len(value) + return pos + +def align_decimals(data, headers, **_): + """Align decimals to decimal point + >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) + 1 + 11.1 + 1.1 + """ + pointpos = len(data[0]) * [0] + for row in data: + for i, v in enumerate(row): + if isinstance(v, Decimal): + v = encodingutils.text_type(v) + pointpos[i] = max(intlen(v), pointpos[i]) + results = [] + for row in data: + result = [] + for i, v in enumerate(row): + if isinstance(v, Decimal): + v = encodingutils.text_type(v) + result.append((pointpos[i] - intlen(v)) * " " + v) + else: + result.append(v) + results.append(result) + return results, headers + + +def quote_whitespaces(data, headers, quotestyle="'", **_): + """Quote whitespace + >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) + ' before' + 'after ' + ' both ' + 'none' + >>> for i in quote_whitespaces([["abc"], ["def"], ["ghi"], ["jkl"]], [])[0]: print(i[0]) + abc + def + ghi + jkl + """ + quote = len(data[0]) * [False] + for row in data: + for i, v in enumerate(row): + v = encodingutils.text_type(v) + if v.startswith(' ') or v.endswith(' '): + quote[i] = True + + results = [] + for row in data: + result = [] + for i, v in enumerate(row): + quotation = quotestyle if quote[i] else '' + result.append('{quotestyle}{value}{quotestyle}'.format( + quotestyle=quotation, value=v)) + results.append(result) + return results, headers diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py new file mode 100644 index 00000000..8c4599c1 --- /dev/null +++ b/mycli/output_formatter/tabulate_adapter.py @@ -0,0 +1,14 @@ +from tabulate import tabulate +from .preprocessors import (bytes_to_string, align_decimals) + +supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', + 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', + 'moinmoin', 'html', 'latex', 'latex_booktabs', + 'textile') + +preprocessors = (bytes_to_string, align_decimals) + +def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): + """Wrap tabulate inside a standard function for OutputFormatter.""" + return tabulate(data, headers, tablefmt=table_format, + missingval=missing_value, disable_numparse=True) diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py new file mode 100644 index 00000000..15981078 --- /dev/null +++ b/mycli/output_formatter/terminaltables_adapter.py @@ -0,0 +1,24 @@ +import terminaltables +from .preprocessors import (bytes_to_string, align_decimals, + override_missing_value) + +supported_formats = ('ascii', 'single', 'double', 'github') +preprocessors = (bytes_to_string, override_missing_value, align_decimals) + +def terminaltables_adapter(data, headers, table_format=None, **_): + """Wrap terminaltables inside a standard function for OutputFormatter.""" + + table_format_handler = { + 'ascii': terminaltables.AsciiTable, + 'single': terminaltables.SingleTable, + 'double': terminaltables.DoubleTable, + 'github': terminaltables.GithubFlavoredMarkdownTable, + } + + try: + table = table_format_handler[table_format] + except KeyError: + raise ValueError('unrecognized table format: {}'.format(table_format)) + + t = table([headers] + data) + return t.table diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index 656e5d53..f01afb1d 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -6,7 +6,6 @@ from prompt_toolkit.completion import Completer, Completion -from .output_formatter import OutputFormatter from .packages.completion_engine import suggest_type from .packages.parseutils import last_word from .packages.special.favoritequeries import favoritequeries @@ -50,7 +49,7 @@ class SQLCompleter(Completer): users = [] - def __init__(self, smart_completion=True): + def __init__(self, smart_completion=True, supported_formats=()): super(self.__class__, self).__init__() self.smart_completion = smart_completion self.reserved_words = set() @@ -59,8 +58,7 @@ def __init__(self, smart_completion=True): self.name_pattern = compile("^[_a-z][_a-z0-9\$]*$") self.special_commands = [] - formatter = OutputFormatter() - self.table_formats = formatter.supported_formats() + self.table_formats = supported_formats self.reset_completions() def escape_name(self, name): From 7a81b9e6bc173fed75fd2653a67f606942bd04d6 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 19:52:35 -0700 Subject: [PATCH 47/76] Update the tests. --- tests/test_output_formatter.py | 18 ++++++++++++------ tests/utils.py | 1 - 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 8ddeb888..484f52d8 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -6,12 +6,18 @@ from decimal import Decimal from textwrap import dedent -from mycli.output_formatter import (align_decimals, bytes_to_string, - convert_to_string, csv_wrapper, - OutputFormatter, override_missing_value, - tabulate_wrapper, terminal_tables_wrapper, - to_string) - +from mycli.output_formatter.preprocessors import (align_decimals, + bytes_to_string, + convert_to_string, + override_missing_value, + to_string) +from mycli.output_formatter.output_formatter import OutputFormatter +from mycli.output_formatter.delimited_output_adapter import (delimiter_adapter + as csv_wrapper) +from mycli.output_formatter.tabulate_adapter import (tabulate_adapter as + tabulate_wrapper) +from mycli.output_formatter.terminaltables_adapter import \ + (terminaltables_adapter as terminal_tables_wrapper) def test_to_string(): """Test the *output_formatter.to_string()* function.""" diff --git a/tests/utils.py b/tests/utils.py index 64006324..bff31045 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,7 +4,6 @@ import pytest from mycli.main import MyCli, special -from mycli.output_formatter import OutputFormatter PASSWORD = getenv('PYTEST_PASSWORD') USER = getenv('PYTEST_USER', 'root') From a96867c6fa06049d34d309927535de2851ea18da Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 3 Apr 2017 23:01:12 -0500 Subject: [PATCH 48/76] PEP8 edits. --- .../delimited_output_adapter.py | 3 +- mycli/output_formatter/output_formatter.py | 74 ++++++++++--------- mycli/output_formatter/preprocessors.py | 5 +- mycli/output_formatter/tabulate_adapter.py | 11 +-- .../terminaltables_adapter.py | 14 ++-- tests/test_output_formatter.py | 22 +++--- 6 files changed, 71 insertions(+), 58 deletions(-) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index 04571e38..04e9f252 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -5,11 +5,12 @@ except ImportError: from io import StringIO -from .preprocessors import (override_missing_value, bytes_to_string) +from .preprocessors import override_missing_value, bytes_to_string supported_formats = ('csv',) delimiter_preprocessors = (override_missing_value, bytes_to_string) + def delimiter_adapter(data, headers, delimiter=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" with contextlib.closing(StringIO()) as content: diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index e4b28023..8f698fc0 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -1,21 +1,25 @@ # -*- coding: utf-8 -*- +"""A generic output formatter interface.""" + from __future__ import unicode_literals +from collections import namedtuple -from ..packages.expanded import expanded_table +from mycli.packages.expanded import expanded_table from .preprocessors import (override_missing_value, convert_to_string) -from .delimited_output_adapter import delimiter_adapter, delimiter_preprocessors +from .delimited_output_adapter import (delimiter_adapter, + delimiter_preprocessors) from .tabulate_adapter import (tabulate_adapter, - supported_formats as tabulate_formats, - preprocessors as tabulate_preprocessors) -from .terminaltables_adapter import (terminaltables_adapter, - preprocessors as terminaltables_preprocessors, - supported_formats as terminaltables_formats) -from collections import namedtuple + supported_formats as tabulate_formats, + preprocessors as tabulate_preprocessors) +from .terminaltables_adapter import ( + terminaltables_adapter, preprocessors as terminaltables_preprocessors, + supported_formats as terminaltables_formats) + +OutputFormatHandler = namedtuple( + 'OutputFormatHandler', + 'format_name preprocessors formatter formatter_args') -OutputFormatHandler = namedtuple('OutputFormatHandler', - 'format_name preprocessors formatter formatter_args') -"""A generic output formatter interface.""" class OutputFormatter(object): """A class with a standard interface for various formatting libraries.""" @@ -43,10 +47,10 @@ def supported_formats(self): @classmethod def register_new_formatter(cls, format_name, handler, preprocessors=None, - kwargs=None): - """Register a new fomatter to format the output""" - cls._output_formats[format_name] = OutputFormatHandler(format_name, - preprocessors, handler, kwargs) + kwargs=None): + """Register a new formatter to format the output.""" + cls._output_formats[format_name] = OutputFormatHandler( + format_name, preprocessors, handler, kwargs) def format_output(self, data, headers, format_name=None, **kwargs): """Format the headers and data using a specific formatter. @@ -66,26 +70,28 @@ def format_output(self, data, headers, format_name=None, **kwargs): data, headers = f(data, headers, **fkwargs) return formatter(data, headers, **fkwargs) -OutputFormatter.register_new_formatter('csv', - delimiter_adapter, - delimiter_preprocessors, {'missing_value': ''}) -OutputFormatter.register_new_formatter('tsv', - delimiter_adapter, - delimiter_preprocessors, - {'missing_value': '', 'delimiter': '\t'}) -OutputFormatter.register_new_formatter('expanded', - expanded_table, - (override_missing_value, convert_to_string), - {'missing_value': '', 'delimiter': '\t'}) + +OutputFormatter.register_new_formatter('csv', delimiter_adapter, + delimiter_preprocessors, + {'missing_value': ''}) +OutputFormatter.register_new_formatter('tsv', delimiter_adapter, + delimiter_preprocessors, + {'missing_value': '', + 'delimiter': '\t'}) +OutputFormatter.register_new_formatter('expanded', expanded_table, + (override_missing_value, + convert_to_string), + {'missing_value': '', + 'delimiter': '\t'}) for tabulate_format in tabulate_formats: - OutputFormatter.register_new_formatter(tabulate_format, - tabulate_adapter, - tabulate_preprocessors, - {'table_format': tabulate_format, 'missing_value': ''}) + OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, + tabulate_preprocessors, + {'table_format': tabulate_format, + 'missing_value': ''}) for terminaltables_format in terminaltables_formats: - OutputFormatter.register_new_formatter(terminaltables_format, - terminaltables_adapter, - terminaltables_preprocessors, - {'table_format': terminaltables_format, 'missing_value': ''}) + OutputFormatter.register_new_formatter( + terminaltables_format, terminaltables_adapter, + terminaltables_preprocessors, + {'table_format': terminaltables_format, 'missing_value': ''}) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index b3a9373a..d6db3fd4 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -1,5 +1,7 @@ from decimal import Decimal -from .. import encodingutils + +from mycli import encodingutils + def to_string(value): """Convert *value* to a string.""" @@ -41,6 +43,7 @@ def intlen(value): pos = len(value) return pos + def align_decimals(data, headers, **_): """Align decimals to decimal point >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 8c4599c1..0db31ff9 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -1,13 +1,14 @@ from tabulate import tabulate -from .preprocessors import (bytes_to_string, align_decimals) -supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', - 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', - 'moinmoin', 'html', 'latex', 'latex_booktabs', - 'textile') +from .preprocessors import bytes_to_string, align_decimals + +supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', + 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', + 'html', 'latex', 'latex_booktabs', 'textile') preprocessors = (bytes_to_string, align_decimals) + def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py index 15981078..ac580517 100644 --- a/mycli/output_formatter/terminaltables_adapter.py +++ b/mycli/output_formatter/terminaltables_adapter.py @@ -1,19 +1,21 @@ import terminaltables + from .preprocessors import (bytes_to_string, align_decimals, - override_missing_value) + override_missing_value) supported_formats = ('ascii', 'single', 'double', 'github') preprocessors = (bytes_to_string, override_missing_value, align_decimals) + def terminaltables_adapter(data, headers, table_format=None, **_): """Wrap terminaltables inside a standard function for OutputFormatter.""" table_format_handler = { - 'ascii': terminaltables.AsciiTable, - 'single': terminaltables.SingleTable, - 'double': terminaltables.DoubleTable, - 'github': terminaltables.GithubFlavoredMarkdownTable, - } + 'ascii': terminaltables.AsciiTable, + 'single': terminaltables.SingleTable, + 'double': terminaltables.DoubleTable, + 'github': terminaltables.GithubFlavoredMarkdownTable, + } try: table = table_format_handler[table_format] diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 484f52d8..7c0890fd 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -2,22 +2,22 @@ """Test the generic output formatter interface.""" from __future__ import unicode_literals - from decimal import Decimal from textwrap import dedent from mycli.output_formatter.preprocessors import (align_decimals, - bytes_to_string, - convert_to_string, - override_missing_value, - to_string) + bytes_to_string, + convert_to_string, + override_missing_value, + to_string) from mycli.output_formatter.output_formatter import OutputFormatter -from mycli.output_formatter.delimited_output_adapter import (delimiter_adapter - as csv_wrapper) -from mycli.output_formatter.tabulate_adapter import (tabulate_adapter as - tabulate_wrapper) -from mycli.output_formatter.terminaltables_adapter import \ - (terminaltables_adapter as terminal_tables_wrapper) +from mycli.output_formatter.delimited_output_adapter import ( + delimiter_adapter as csv_wrapper) +from mycli.output_formatter.tabulate_adapter import ( + tabulate_adapter as tabulate_wrapper) +from mycli.output_formatter.terminaltables_adapter import ( + terminaltables_adapter as terminal_tables_wrapper) + def test_to_string(): """Test the *output_formatter.to_string()* function.""" From 3d9feb9241787eeecad974f7a5c1e5f9429ef604 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 21:14:52 -0700 Subject: [PATCH 49/76] Move expanded.py into output_formatter package. --- mycli/output_formatter/delimited_output_adapter.py | 2 +- mycli/{packages => output_formatter}/expanded.py | 0 mycli/output_formatter/output_formatter.py | 5 ++--- tests/test_expanded.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) rename mycli/{packages => output_formatter}/expanded.py (100%) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index 04e9f252..7111af6d 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -7,7 +7,7 @@ from .preprocessors import override_missing_value, bytes_to_string -supported_formats = ('csv',) +supported_formats = ('csv', 'tsv') delimiter_preprocessors = (override_missing_value, bytes_to_string) diff --git a/mycli/packages/expanded.py b/mycli/output_formatter/expanded.py similarity index 100% rename from mycli/packages/expanded.py rename to mycli/output_formatter/expanded.py diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 8f698fc0..1564026b 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals from collections import namedtuple -from mycli.packages.expanded import expanded_table +from .expanded import expanded_table from .preprocessors import (override_missing_value, convert_to_string) from .delimited_output_adapter import (delimiter_adapter, delimiter_preprocessors) @@ -81,8 +81,7 @@ def format_output(self, data, headers, format_name=None, **kwargs): OutputFormatter.register_new_formatter('expanded', expanded_table, (override_missing_value, convert_to_string), - {'missing_value': '', - 'delimiter': '\t'}) + {'missing_value': ''}) for tabulate_format in tabulate_formats: OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, diff --git a/tests/test_expanded.py b/tests/test_expanded.py index db503cba..7233e91c 100644 --- a/tests/test_expanded.py +++ b/tests/test_expanded.py @@ -1,7 +1,7 @@ """Test the vertical, expanded table formatter.""" from textwrap import dedent -from mycli.packages.expanded import expanded_table +from mycli.output_formatter.expanded import expanded_table from mycli.encodingutils import text_type From b5bda3f20f2cea84bc72aa6a28cbf1f31cfa67f7 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 21:26:33 -0700 Subject: [PATCH 50/76] Refactor the delimiter adapter to include tsv. --- mycli/output_formatter/delimited_output_adapter.py | 9 +++++++-- mycli/output_formatter/output_formatter.py | 14 +++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index 7111af6d..d3b5120b 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -11,10 +11,15 @@ delimiter_preprocessors = (override_missing_value, bytes_to_string) -def delimiter_adapter(data, headers, delimiter=',', **_): +def delimiter_adapter(data, headers, table_format=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" with contextlib.closing(StringIO()) as content: - writer = csv.writer(content, delimiter=str(delimiter)) + if table_format == 'csv': + writer = csv.writer(content, delimiter=',') + elif table_format == 'tsv': + writer = csv.writer(content, delimiter='\t') + else: + raise ValueError('Invalid table_format specified.') writer.writerow(headers) for row in data: diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 1564026b..a47029cc 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -7,6 +7,7 @@ from .expanded import expanded_table from .preprocessors import (override_missing_value, convert_to_string) from .delimited_output_adapter import (delimiter_adapter, + supported_formats as delimiter_formats, delimiter_preprocessors) from .tabulate_adapter import (tabulate_adapter, supported_formats as tabulate_formats, @@ -71,18 +72,17 @@ def format_output(self, data, headers, format_name=None, **kwargs): return formatter(data, headers, **fkwargs) -OutputFormatter.register_new_formatter('csv', delimiter_adapter, - delimiter_preprocessors, - {'missing_value': ''}) -OutputFormatter.register_new_formatter('tsv', delimiter_adapter, - delimiter_preprocessors, - {'missing_value': '', - 'delimiter': '\t'}) OutputFormatter.register_new_formatter('expanded', expanded_table, (override_missing_value, convert_to_string), {'missing_value': ''}) +for delimiter_format in delimiter_formats: + OutputFormatter.register_new_formatter(delimiter_format, delimiter_adapter, + delimiter_preprocessors, + {'table_format': delimiter_format, + 'missing_value': ''}) + for tabulate_format in tabulate_formats: OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, tabulate_preprocessors, From 2120b3653e3f8027ebfa16703737b65aa7d31f81 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 21:33:41 -0700 Subject: [PATCH 51/76] Fix failing tests. --- mycli/output_formatter/delimited_output_adapter.py | 2 +- tests/test_output_formatter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index d3b5120b..8036a5ff 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -11,7 +11,7 @@ delimiter_preprocessors = (override_missing_value, bytes_to_string) -def delimiter_adapter(data, headers, table_format=',', **_): +def delimiter_adapter(data, headers, table_format='csv', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" with contextlib.closing(StringIO()) as content: if table_format == 'csv': diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 7c0890fd..7949032f 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -92,7 +92,7 @@ def test_csv_wrapper(): # Test tab-delimited output. data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = csv_wrapper(data, headers, delimiter='\t') + output = csv_wrapper(data, headers, table_format='tsv') assert output == dedent('''\ letters\tnumber\r\n\ abc\t1\r\n\ From 549d9ad683a67ae16ef08fc92ba62c02252cc737 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 4 Apr 2017 18:57:56 -0500 Subject: [PATCH 52/76] Fix behave tests table formatting. --- tests/features/fixture_data/help_commands.txt | 2 +- tests/features/steps/crud_table.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/features/fixture_data/help_commands.txt b/tests/features/fixture_data/help_commands.txt index 1c5a08bf..ee3d4ca4 100644 --- a/tests/features/fixture_data/help_commands.txt +++ b/tests/features/fixture_data/help_commands.txt @@ -1,6 +1,6 @@ +-------------+-------------------+---------------------------------------------------------+ | Command | Shortcut | Description | -|-------------+-------------------+---------------------------------------------------------| ++-------------+-------------------+---------------------------------------------------------+ | \G | \G | Display results vertically. | | \dt | \dt [table] | List or describe tables. | | \e | \e | Edit command with editor. (uses $EDITOR) | diff --git a/tests/features/steps/crud_table.py b/tests/features/steps/crud_table.py index 72b4e080..a4ed0fe5 100644 --- a/tests/features/steps/crud_table.py +++ b/tests/features/steps/crud_table.py @@ -91,7 +91,7 @@ def step_see_data_selected(context): """ Wait to see select output. """ - wrappers.expect_exact(context, '+-----+\r\n| x |\r\n|-----|\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) + wrappers.expect_exact(context, '+-----+\r\n| x |\r\n+-----+\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) @then('we see record deleted') From 079d9f870debfed03a8ec0351859ddd0a0670df0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 5 Apr 2017 22:04:04 -0500 Subject: [PATCH 53/76] PEP8 changes per pep8radius request. --- mycli/encodingutils.py | 15 +++++++++------ mycli/main.py | 15 +++++++++------ mycli/output_formatter/expanded.py | 1 + mycli/output_formatter/output_formatter.py | 4 +++- mycli/output_formatter/preprocessors.py | 10 ++++++++-- mycli/sqlexecute.py | 12 ++++++------ tests/features/steps/crud_table.py | 3 ++- tests/test_output_formatter.py | 3 ++- tests/utils.py | 3 ++- 9 files changed, 42 insertions(+), 24 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index aac4aa07..1a8b5bbb 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -16,9 +16,10 @@ def unicode2utf8(arg): - """ - Only in Python 2. Psycopg2 expects the args as bytes not unicode. - In Python 3 the args are expected as unicode. + """Convert strings to UTF8-encoded bytes. + + Only in Python 2. In Python 3 the args are expected as unicode. + """ if PY2 and isinstance(arg, text_type): @@ -27,9 +28,10 @@ def unicode2utf8(arg): def utf8tounicode(arg): - """ - Only in Python 2. Psycopg2 returns the error message as utf-8. - In Python 3 the errors are returned as unicode. + """Convert UTF8-encoded bytes to strings. + + Only in Python 2. In Python 3 the errors are returned as strings. + """ if PY2 and isinstance(arg, binary_type): @@ -46,6 +48,7 @@ def bytes_to_string(b): abc >>> print(bytes_to_string('✌')) ✌ + """ if isinstance(b, binary_type): try: diff --git a/mycli/main.py b/mycli/main.py index aea9c730..64c58705 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -106,7 +106,8 @@ def __init__(self, sqlexecute=None, prompt=None, self.multi_line = c['main'].as_bool('multi_line') self.key_bindings = c['main']['key_bindings'] special.set_timing_enabled(c['main'].as_bool('timing')) - self.formatter = output_formatter.OutputFormatter(format_name=c['main']['table_format']) + self.formatter = output_formatter.OutputFormatter( + format_name=c['main']['table_format']) self.syntax_style = c['main']['syntax_style'] self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] @@ -146,7 +147,7 @@ def __init__(self, sqlexecute=None, prompt=None, # Initialize completer. self.smart_completion = c['main'].as_bool('smart_completion') self.completer = SQLCompleter(self.smart_completion, - supported_formats=self.formatter.supported_formats) + supported_formats=self.formatter.supported_formats) self._completer_lock = threading.Lock() # Register custom special commands. @@ -186,7 +187,8 @@ def change_table_format(self, arg, **_): yield (None, None, None, 'Changed table type to {}'.format(arg)) except ValueError: - msg = 'Table type {} not yet implemented. Allowed types:'.format(arg) + msg = 'Table type {} not yet implemented. Allowed types:'.format( + arg) for table_type in self.formatter.supported_formats(): msg += "\n\t{}".format(table_type) yield (None, None, None, msg) @@ -524,8 +526,9 @@ def one_iteration(document=None): else: max_width = None - formatted = self.format_output(title, cur, headers, - status, special.is_expanded_output(), max_width) + formatted = self.format_output(title, cur, headers, status, + special.is_expanded_output(), + max_width) output.extend(formatted) end = time() @@ -706,7 +709,7 @@ def get_prompt(self, string): return string def run_query(self, query, new_line=True): - """Runs query""" + """Runs *query*.""" results = self.sqlexecute.run(query) for result in results: title, cur, headers, status = result diff --git a/mycli/output_formatter/expanded.py b/mycli/output_formatter/expanded.py index 2ab57994..f77c1ee3 100644 --- a/mycli/output_formatter/expanded.py +++ b/mycli/output_formatter/expanded.py @@ -19,6 +19,7 @@ def expanded_table(rows, headers, **_): """Format *rows* and *headers* as an expanded table. The values in *rows* and *headers* must be strings. + """ header_len = max([len(x) for x in headers]) padded_headers = [x.ljust(header_len) for x in headers] diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index a47029cc..759fb879 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -59,12 +59,14 @@ def format_output(self, data, headers, format_name=None, **kwargs): *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. + """ format_name = format_name or self._format_name if format_name not in self.supported_formats(): raise ValueError('unrecognized format: {}'.format(format_name)) - (_, preprocessors, formatter, fkwargs) = self._output_formats[format_name] + (_, preprocessors, formatter, + fkwargs) = self._output_formats[format_name] fkwargs.update(kwargs) if preprocessors: for f in preprocessors: diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index d6db3fd4..010c1b14 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -30,13 +30,15 @@ def bytes_to_string(data, headers, **_): def intlen(value): - """Find (character) length + """Find (character) length. + >>> intlen('11.1') 2 >>> intlen('11') 2 >>> intlen('1.1') 1 + """ pos = value.find('.') if pos < 0: @@ -45,11 +47,13 @@ def intlen(value): def align_decimals(data, headers, **_): - """Align decimals to decimal point + """Align decimals to decimal point. + >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) 1 11.1 1.1 + """ pointpos = len(data[0]) * [0] for row in data: @@ -72,6 +76,7 @@ def align_decimals(data, headers, **_): def quote_whitespaces(data, headers, quotestyle="'", **_): """Quote whitespace + >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) ' before' 'after ' @@ -82,6 +87,7 @@ def quote_whitespaces(data, headers, quotestyle="'", **_): def ghi jkl + """ quote = len(data[0]) * [False] for row in data: diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 08030af4..4d5c0a09 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -4,7 +4,7 @@ from .packages import special from pymysql.constants import FIELD_TYPE from pymysql.converters import (convert_mysql_timestamp, convert_datetime, - convert_timedelta, convert_date, conversions) + convert_timedelta, convert_date, conversions) _logger = logging.getLogger(__name__) @@ -67,11 +67,11 @@ def connect(self, database=None, user=None, password=None, host=None, database, user, host, port, socket, charset, local_infile, ssl) conv = conversions.copy() conv.update({ - FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or obj), - FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or obj), - FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or obj), - FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or obj), - }) + FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or obj), + FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or obj), + FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or obj), + FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or obj), + }) conn = pymysql.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, diff --git a/tests/features/steps/crud_table.py b/tests/features/steps/crud_table.py index a4ed0fe5..b73ff0d6 100644 --- a/tests/features/steps/crud_table.py +++ b/tests/features/steps/crud_table.py @@ -91,7 +91,8 @@ def step_see_data_selected(context): """ Wait to see select output. """ - wrappers.expect_exact(context, '+-----+\r\n| x |\r\n+-----+\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) + wrappers.expect_exact( + context, '+-----+\r\n| x |\r\n+-----+\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) @then('we see record deleted') diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 7949032f..ac5cfc07 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -57,7 +57,8 @@ def test_bytes_to_string(): def test_align_decimals(): """Test the *output_formatter.align_decimals()* function.""" - data = [[Decimal('200'), Decimal('1')], [Decimal('1.00002'), Decimal('1.0')]] + data = [[Decimal('200'), Decimal('1')], [ + Decimal('1.00002'), Decimal('1.0')]] headers = ['num1', 'num2'] expected = ([['200', '1'], [' 1.00002', '1.0']], ['num1', 'num2']) diff --git a/tests/utils.py b/tests/utils.py index bff31045..b29e5e07 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -44,7 +44,8 @@ def run(executor, sql, join=False): # It should test raw results. mycli = MyCli() for title, rows, headers, status in executor.run(sql): - result.extend(mycli.format_output(title, rows, headers, status, special.is_expanded_output())) + result.extend(mycli.format_output(title, rows, headers, status, + special.is_expanded_output())) if join: result = '\n'.join(result) From 8bc0fc41db40bb86c35d159fbbab00bc34d5b092 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 5 Apr 2017 22:05:59 -0500 Subject: [PATCH 54/76] Add period for pep8radius. --- mycli/output_formatter/preprocessors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index 010c1b14..c2e221f8 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -75,7 +75,7 @@ def align_decimals(data, headers, **_): def quote_whitespaces(data, headers, quotestyle="'", **_): - """Quote whitespace + """Quote whitespace. >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) ' before' From 935fc568f2b33f641c37cbd743ec2a213b82dc81 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 10 Apr 2017 08:06:18 -0500 Subject: [PATCH 55/76] Do not reference results rows by index. --- mycli/output_formatter/preprocessors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index c2e221f8..8fe73bb7 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -55,7 +55,7 @@ def align_decimals(data, headers, **_): 1.1 """ - pointpos = len(data[0]) * [0] + pointpos = len(headers) * [0] for row in data: for i, v in enumerate(row): if isinstance(v, Decimal): @@ -89,7 +89,7 @@ def quote_whitespaces(data, headers, quotestyle="'", **_): jkl """ - quote = len(data[0]) * [False] + quote = len(headers) * [False] for row in data: for i, v in enumerate(row): v = encodingutils.text_type(v) From 5849ac9261a9c5c68a10bd5b3fceee88c2daf4b2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 10 Apr 2017 08:15:21 -0500 Subject: [PATCH 56/76] Add empty result tests. --- mycli/output_formatter/preprocessors.py | 24 ++----------------- tests/test_output_formatter.py | 31 ++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index 8fe73bb7..6f2e459c 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -47,14 +47,7 @@ def intlen(value): def align_decimals(data, headers, **_): - """Align decimals to decimal point. - - >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) - 1 - 11.1 - 1.1 - - """ + """Align decimals to decimal point.""" pointpos = len(headers) * [0] for row in data: for i, v in enumerate(row): @@ -75,20 +68,7 @@ def align_decimals(data, headers, **_): def quote_whitespaces(data, headers, quotestyle="'", **_): - """Quote whitespace. - - >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) - ' before' - 'after ' - ' both ' - 'none' - >>> for i in quote_whitespaces([["abc"], ["def"], ["ghi"], ["jkl"]], [])[0]: print(i[0]) - abc - def - ghi - jkl - - """ + """Quote leading/trailing whitespace.""" quote = len(headers) * [False] for row in data: for i, v in enumerate(row): diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index ac5cfc07..dcc26e62 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -8,6 +8,7 @@ from mycli.output_formatter.preprocessors import (align_decimals, bytes_to_string, convert_to_string, + quote_whitespaces, override_missing_value, to_string) from mycli.output_formatter.output_formatter import OutputFormatter @@ -56,7 +57,7 @@ def test_bytes_to_string(): def test_align_decimals(): - """Test the *output_formatter.align_decimals()* function.""" + """Test the *align_decimals()* function.""" data = [[Decimal('200'), Decimal('1')], [ Decimal('1.00002'), Decimal('1.0')]] headers = ['num1', 'num2'] @@ -65,6 +66,34 @@ def test_align_decimals(): assert expected == align_decimals(data, headers) +def test_align_decimals_empty_result(): + """Test *align_decimals()* with no results.""" + data = [] + headers = ['num1', 'num2'] + expected = ([], ['num1', 'num2']) + + assert expected == align_decimals(data, headers) + + +def test_quote_whitespaces(): + """Test the *quote_whitespaces()* function.""" + data = [[" before", "after "], [" both ", "none"]] + headers = ['h1', 'h2'] + expected = ([["' before'", "'after '"], ["' both '", "'none'"]], + ['h1', 'h2']) + + assert expected == quote_whitespaces(data, headers) + + +def test_quote_whitespaces_empty_result(): + """Test the *quote_whitespaces()* function with no results.""" + data = [] + headers = ['h1', 'h2'] + expected = ([], ['h1', 'h2']) + + assert expected == quote_whitespaces(data, headers) + + def test_tabulate_wrapper(): """Test the *output_formatter.tabulate_wrapper()* function.""" data = [['abc', 1], ['d', 456]] From 385f47892fbf2e8c8b0fdf3bde174a2128a3b255 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 14:49:03 -0500 Subject: [PATCH 57/76] Revert "Remove tabulate license note." This reverts commit d0f1dbf441ae823d56cadca456b1ff35a35abd88. --- LICENSE.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/LICENSE.txt b/LICENSE.txt index 8afaa265..9a41a67d 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -27,3 +27,9 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------- + +This program also bundles with it python-tabulate +(https://pypi.python.org/pypi/tabulate) library. This library is licensed under +MIT License. + +------------------------------------------------------------------------------- From 16738b1dbcb6b146175e3308352c113006b5b3ca Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:13:03 -0500 Subject: [PATCH 58/76] Add vendored tabulate version 0.8.0. --- mycli/output_formatter/tabulate_adapter.py | 11 +- mycli/packages/tabulate.py | 1423 ++++++++++++++++++++ setup.py | 1 - 3 files changed, 1428 insertions(+), 7 deletions(-) create mode 100644 mycli/packages/tabulate.py diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 0db31ff9..14e6c770 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -1,15 +1,14 @@ -from tabulate import tabulate - -from .preprocessors import bytes_to_string, align_decimals +from mycli.packages import tabulate +from .preprocessors import bytes_to_string, align_decimals, quote_whitespaces supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', 'html', 'latex', 'latex_booktabs', 'textile') -preprocessors = (bytes_to_string, align_decimals) +preprocessors = (bytes_to_string, align_decimals, quote_whitespaces) def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate(data, headers, tablefmt=table_format, - missingval=missing_value, disable_numparse=True) + return tabulate.tabulate(data, headers, tablefmt=table_format, + missingval=missing_value, disable_numparse=True) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py new file mode 100644 index 00000000..a2503299 --- /dev/null +++ b/mycli/packages/tabulate.py @@ -0,0 +1,1423 @@ +# -*- coding: utf-8 -*- + +"""Pretty-print tabular data.""" + +from __future__ import print_function +from __future__ import unicode_literals +from collections import namedtuple, Iterable +from platform import python_version_tuple +import re + + +if python_version_tuple()[0] < "3": + from itertools import izip_longest + from functools import partial + _none_type = type(None) + _bool_type = bool + _int_type = int + _long_type = long + _float_type = float + _text_type = unicode + _binary_type = str + + def _is_file(f): + return isinstance(f, file) + +else: + from itertools import zip_longest as izip_longest + from functools import reduce, partial + _none_type = type(None) + _bool_type = bool + _int_type = int + _long_type = int + _float_type = float + _text_type = str + _binary_type = bytes + basestring = str + + import io + + def _is_file(f): + return isinstance(f, io.IOBase) + +try: + import wcwidth # optional wide-character (CJK) support +except ImportError: + wcwidth = None + + +__all__ = ["tabulate", "tabulate_formats"] +__version__ = "0.8.0" + + +# minimum extra space in headers +MIN_PADDING = 2 + + +_DEFAULT_FLOATFMT = "g" +_DEFAULT_MISSINGVAL = "" + + +# if True, enable wide-character (CJK) support +WIDE_CHARS_MODE = wcwidth is not None + + +Line = namedtuple("Line", ["begin", "hline", "sep", "end"]) + + +DataRow = namedtuple("DataRow", ["begin", "sep", "end"]) + + +# A table structure is suppposed to be: +# +# --- lineabove --------- +# headerrow +# --- linebelowheader --- +# datarow +# --- linebewteenrows --- +# ... (more datarows) ... +# --- linebewteenrows --- +# last datarow +# --- linebelow --------- +# +# TableFormat's line* elements can be +# +# - either None, if the element is not used, +# - or a Line tuple, +# - or a function: [col_widths], [col_alignments] -> string. +# +# TableFormat's *row elements can be +# +# - either None, if the element is not used, +# - or a DataRow tuple, +# - or a function: [cell_values], [col_widths], [col_alignments] -> string. +# +# padding (an integer) is the amount of white space around data values. +# +# with_header_hide: +# +# - either None, to display all table elements unconditionally, +# - or a list of elements not to be displayed if the table has column +# headers. +# +TableFormat = namedtuple("TableFormat", ["lineabove", "linebelowheader", + "linebetweenrows", "linebelow", + "headerrow", "datarow", + "padding", "with_header_hide"]) + + +def _pipe_segment_with_colons(align, colwidth): + """Return a segment of a horizontal line with optional colons which + indicate column's alignment (as in `pipe` output format).""" + w = colwidth + if align in ["right", "decimal"]: + return ('-' * (w - 1)) + ":" + elif align == "center": + return ":" + ('-' * (w - 2)) + ":" + elif align == "left": + return ":" + ('-' * (w - 1)) + else: + return '-' * w + + +def _pipe_line_with_colons(colwidths, colaligns): + """Return a horizontal line with optional colons to indicate column's + alignment (as in `pipe` output format).""" + segments = [_pipe_segment_with_colons(a, w) for a, w in + zip(colaligns, colwidths)] + return "|" + "|".join(segments) + "|" + + +def _mediawiki_row_with_attrs(separator, cell_values, colwidths, colaligns): + alignment = {"left": '', + "right": 'align="right"| ', + "center": 'align="center"| ', + "decimal": 'align="right"| '} + # hard-coded padding _around_ align attribute and value together + # rather than padding parameter which affects only the value + values_with_attrs = [' ' + alignment.get(a, '') + c + ' ' + for c, a in zip(cell_values, colaligns)] + colsep = separator*2 + return (separator + colsep.join(values_with_attrs)).rstrip() + + +def _textile_row_with_attrs(cell_values, colwidths, colaligns): + cell_values[0] += ' ' + alignment = {"left": "<.", "right": ">.", "center": "=.", "decimal": ">."} + values = (alignment.get(a, '') + v for a, v in zip(colaligns, cell_values)) + return '|' + '|'.join(values) + '|' + + +def _html_begin_table_without_header(colwidths_ignore, colaligns_ignore): + # this table header will be suppressed if there is a header row + return "\n".join(["", ""]) + + +def _html_row_with_attrs(celltag, cell_values, colwidths, colaligns): + alignment = {"left": '', + "right": ' style="text-align: right;"', + "center": ' style="text-align: center;"', + "decimal": ' style="text-align: right;"'} + values_with_attrs = ["<{0}{1}>{2}".format( + celltag, alignment.get(a, ''), c) for c, a in + zip(cell_values, colaligns)] + rowhtml = "" + "".join(values_with_attrs).rstrip() + "" + if celltag == "th": # it's a header row, create a new table header + rowhtml = "\n".join(["
", + "", + rowhtml, + "", + ""]) + return rowhtml + + +def _moin_row_with_attrs(celltag, cell_values, colwidths, colaligns, + header=''): + alignment = {"left": '', + "right": '', + "center": '', + "decimal": ''} + values_with_attrs = ["{0}{1} {2} ".format(celltag, + alignment.get(a, ''), + header+c+header) + for c, a in zip(cell_values, colaligns)] + return "".join(values_with_attrs)+"||" + + +def _latex_line_begin_tabular(colwidths, colaligns, booktabs=False): + alignment = {"left": "l", "right": "r", "center": "c", "decimal": "r"} + tabular_columns_fmt = "".join([alignment.get(a, "l") for a in colaligns]) + return "\n".join(["\\begin{tabular}{" + tabular_columns_fmt + "}", + "\\toprule" if booktabs else "\hline"]) + + +LATEX_ESCAPE_RULES = {r"&": r"\&", r"%": r"\%", r"$": r"\$", r"#": r"\#", + r"_": r"\_", r"^": r"\^{}", r"{": r"\{", r"}": r"\}", + r"~": r"\textasciitilde{}", "\\": r"\textbackslash{}", + r"<": r"\ensuremath{<}", r">": r"\ensuremath{>}"} + + +def _latex_row(cell_values, colwidths, colaligns, escrules=LATEX_ESCAPE_RULES): + def escape_char(c): + return escrules.get(c, c) + escaped_values = ["".join(map(escape_char, cell)) for cell in cell_values] + rowfmt = DataRow("", "&", "\\\\") + return _build_simple_row(escaped_values, rowfmt) + + +def _rst_escape_first_column(rows, headers): + def escape_empty(val): + if isinstance(val, (_text_type, _binary_type)) and val.strip() is "": + return ".." + else: + return val + new_headers = list(headers) + new_rows = [] + if headers: + new_headers[0] = escape_empty(headers[0]) + for row in rows: + new_row = list(row) + if new_row: + new_row[0] = escape_empty(row[0]) + new_rows.append(new_row) + return new_rows, new_headers + + +_table_formats = {"simple": + TableFormat( + lineabove=Line("", "-", " ", ""), + linebelowheader=Line("", "-", " ", ""), + linebetweenrows=None, + linebelow=Line("", "-", " ", ""), + headerrow=DataRow("", " ", ""), + datarow=DataRow("", " ", ""), + padding=0, + with_header_hide=["lineabove", "linebelow"]), + "plain": + TableFormat( + lineabove=None, linebelowheader=None, + linebetweenrows=None, linebelow=None, + headerrow=DataRow("", " ", ""), + datarow=DataRow("", " ", ""), + padding=0, with_header_hide=None), + "grid": + TableFormat( + lineabove=Line("+", "-", "+", "+"), + linebelowheader=Line("+", "=", "+", "+"), + linebetweenrows=Line("+", "-", "+", "+"), + linebelow=Line("+", "-", "+", "+"), + headerrow=DataRow("|", "|", "|"), + datarow=DataRow("|", "|", "|"), + padding=1, with_header_hide=None), + "fancy_grid": + TableFormat( + lineabove=Line("╒", "═", "╤", "╕"), + linebelowheader=Line("╞", "═", "╪", "╡"), + linebetweenrows=Line("├", "─", "┼", "┤"), + linebelow=Line("╘", "═", "╧", "╛"), + headerrow=DataRow("│", "│", "│"), + datarow=DataRow("│", "│", "│"), + padding=1, with_header_hide=None), + "pipe": + TableFormat( + lineabove=_pipe_line_with_colons, + linebelowheader=_pipe_line_with_colons, + linebetweenrows=None, + linebelow=None, + headerrow=DataRow("|", "|", "|"), + datarow=DataRow("|", "|", "|"), + padding=1, + with_header_hide=["lineabove"]), + "orgtbl": + TableFormat( + lineabove=None, + linebelowheader=Line("|", "-", "+", "|"), + linebetweenrows=None, + linebelow=None, + headerrow=DataRow("|", "|", "|"), + datarow=DataRow("|", "|", "|"), + padding=1, with_header_hide=None), + "jira": + TableFormat( + lineabove=None, + linebelowheader=None, + linebetweenrows=None, + linebelow=None, + headerrow=DataRow("||", "||", "||"), + datarow=DataRow("|", "|", "|"), + padding=1, with_header_hide=None), + "psql": + TableFormat( + lineabove=Line("+", "-", "+", "+"), + linebelowheader=Line("|", "-", "+", "|"), + linebetweenrows=None, + linebelow=Line("+", "-", "+", "+"), + headerrow=DataRow("|", "|", "|"), + datarow=DataRow("|", "|", "|"), + padding=1, with_header_hide=None), + "rst": + TableFormat( + lineabove=Line("", "=", " ", ""), + linebelowheader=Line("", "=", " ", ""), + linebetweenrows=None, + linebelow=Line("", "=", " ", ""), + headerrow=DataRow("", " ", ""), + datarow=DataRow("", " ", ""), + padding=0, with_header_hide=None), + "mediawiki": + TableFormat(lineabove=Line( + "{| class=\"wikitable\" style=\"text-align: left;\"", + "", "", "\n|+ \n|-"), + linebelowheader=Line("|-", "", "", ""), + linebetweenrows=Line("|-", "", "", ""), + linebelow=Line("|}", "", "", ""), + headerrow=partial(_mediawiki_row_with_attrs, "!"), + datarow=partial(_mediawiki_row_with_attrs, "|"), + padding=0, with_header_hide=None), + "moinmoin": + TableFormat( + lineabove=None, + linebelowheader=None, + linebetweenrows=None, + linebelow=None, + headerrow=partial(_moin_row_with_attrs, "||", + header="'''"), + datarow=partial(_moin_row_with_attrs, "||"), + padding=1, with_header_hide=None), + "html": + TableFormat( + lineabove=_html_begin_table_without_header, + linebelowheader="", + linebetweenrows=None, + linebelow=Line("\n
", "", "", ""), + headerrow=partial(_html_row_with_attrs, "th"), + datarow=partial(_html_row_with_attrs, "td"), + padding=0, with_header_hide=["lineabove"]), + "latex": + TableFormat( + lineabove=_latex_line_begin_tabular, + linebelowheader=Line("\\hline", "", "", ""), + linebetweenrows=None, + linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), + headerrow=_latex_row, + datarow=_latex_row, + padding=1, with_header_hide=None), + "latex_raw": + TableFormat( + lineabove=_latex_line_begin_tabular, + linebelowheader=Line("\\hline", "", "", ""), + linebetweenrows=None, + linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), + headerrow=partial(_latex_row, escrules={}), + datarow=partial(_latex_row, escrules={}), + padding=1, with_header_hide=None), + "latex_booktabs": + TableFormat( + lineabove=partial(_latex_line_begin_tabular, + booktabs=True), + linebelowheader=Line("\\midrule", "", "", ""), + linebetweenrows=None, + linebelow=Line("\\bottomrule\n\\end{tabular}", "", "", + ""), + headerrow=_latex_row, + datarow=_latex_row, + padding=1, with_header_hide=None), + "textile": + TableFormat( + lineabove=None, linebelowheader=None, + linebetweenrows=None, linebelow=None, + headerrow=DataRow("|_. ", "|_.", "|"), + datarow=_textile_row_with_attrs, + padding=1, with_header_hide=None)} + + +tabulate_formats = list(sorted(_table_formats.keys())) + + +# ANSI color codes +_invisible_codes = re.compile(r"\x1b\[\d+[;\d]*m|\x1b\[\d*\;\d*\;\d*m") +_invisible_codes_bytes = re.compile(b"\x1b\[\d+[;\d]*m|\x1b\[\d*\;\d*\;\d*m") + + +def _isconvertible(conv, string): + try: + n = conv(string) + return True + except (ValueError, TypeError): + return False + + +def _isnumber(string): + """ + >>> _isnumber("123.45") + True + >>> _isnumber("123") + True + >>> _isnumber("spam") + False + """ + return _isconvertible(float, string) + + +def _isint(string, inttype=int): + """ + >>> _isint("123") + True + >>> _isint("123.45") + False + """ + return type(string) is inttype or\ + (isinstance(string, _binary_type) or isinstance(string, _text_type))\ + and\ + _isconvertible(inttype, string) + + +def _isbool(string): + """ + >>> _isbool(True) + True + >>> _isbool("False") + True + >>> _isbool(1) + False + """ + return type(string) is _bool_type or\ + (isinstance(string, (_binary_type, _text_type)) and + string in ("True", "False")) + + +def _type(string, has_invisible=True, numparse=True): + """The least generic type (type(None), int, float, str, unicode). + + >>> _type(None) is type(None) + True + >>> _type("foo") is type("") + True + >>> _type("1") is type(1) + True + >>> _type('\x1b[31m42\x1b[0m') is type(42) + True + >>> _type('\x1b[31m42\x1b[0m') is type(42) + True + + """ + + if has_invisible and \ + (isinstance(string, _text_type) or isinstance(string, _binary_type)): + string = _strip_invisible(string) + + if string is None: + return _none_type + elif hasattr(string, "isoformat"): # datetime.datetime, date, and time + return _text_type + elif _isbool(string): + return _bool_type + elif _isint(string) and numparse: + return int + elif _isint(string, _long_type) and numparse: + return int + elif _isnumber(string) and numparse: + return float + elif isinstance(string, _binary_type): + return _binary_type + else: + return _text_type + + +def _afterpoint(string): + """Symbols after a decimal point, -1 if the string lacks the decimal point. + + >>> _afterpoint("123.45") + 2 + >>> _afterpoint("1001") + -1 + >>> _afterpoint("eggs") + -1 + >>> _afterpoint("123e45") + 2 + + """ + if _isnumber(string): + if _isint(string): + return -1 + else: + pos = string.rfind(".") + pos = string.lower().rfind("e") if pos < 0 else pos + if pos >= 0: + return len(string) - pos - 1 + else: + return -1 # no point + else: + return -1 # not a number + + +def _padleft(width, s): + """Flush right. + + >>> _padleft(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430' + True + + """ + fmt = "{0:>%ds}" % width + return fmt.format(s) + + +def _padright(width, s): + """Flush left. + + >>> _padright(6, '\u044f\u0439\u0446\u0430') == '\u044f\u0439\u0446\u0430 ' + True + + """ + fmt = "{0:<%ds}" % width + return fmt.format(s) + + +def _padboth(width, s): + """Center string. + + >>> _padboth(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430 ' + True + + """ + fmt = "{0:^%ds}" % width + return fmt.format(s) + + +def _strip_invisible(s): + "Remove invisible ANSI color codes." + if isinstance(s, _text_type): + return re.sub(_invisible_codes, "", s) + else: # a bytestring + return re.sub(_invisible_codes_bytes, "", s) + + +def _visible_width(s): + """Visible width of a printed string. ANSI color codes are removed. + + >>> _visible_width('\x1b[31mhello\x1b[0m'), _visible_width("world") + (5, 5) + + """ + # optional wide-character support + if wcwidth is not None and WIDE_CHARS_MODE: + len_fn = wcwidth.wcswidth + else: + len_fn = len + if isinstance(s, _text_type) or isinstance(s, _binary_type): + return len_fn(_strip_invisible(s)) + else: + return len_fn(_text_type(s)) + + +def _align_column(strings, alignment, minwidth=0, has_invisible=True): + """[string] -> [padded_string] + + >>> list(map(str,_align_column( + ... ["12.345", "-1234.5", "1.23", "1234.5", "1e+234", "1.0e234"], + ... "decimal"))) + [' 12.345 ', '-1234.5 ', ' 1.23 ', ' 1234.5 ', ' 1e+234 ', ' 1.0e234'] + + >>> list(map(str,_align_column(['123.4', '56.7890'], None))) + ['123.4', '56.7890'] + + """ + if alignment == "right": + strings = [s.strip() for s in strings] + padfn = _padleft + elif alignment == "center": + strings = [s.strip() for s in strings] + padfn = _padboth + elif alignment == "decimal": + if has_invisible: + decimals = [_afterpoint(_strip_invisible(s)) for s in strings] + else: + decimals = [_afterpoint(s) for s in strings] + maxdecimals = max(decimals) + strings = [s + (maxdecimals - decs) * " " + for s, decs in zip(strings, decimals)] + padfn = _padleft + elif not alignment: + return strings + else: + strings = [s.strip() for s in strings] + padfn = _padright + + enable_widechars = wcwidth is not None and WIDE_CHARS_MODE + if has_invisible: + width_fn = _visible_width + elif enable_widechars: # optional wide-character support if available + width_fn = wcwidth.wcswidth + else: + width_fn = len + + s_lens = list(map(len, strings)) + s_widths = list(map(width_fn, strings)) + maxwidth = max(max(s_widths), minwidth) + if not enable_widechars and not has_invisible: + padded_strings = [padfn(maxwidth, s) for s in strings] + else: + # enable wide-character width corrections + visible_widths = [maxwidth - (w - l) for w, l in zip(s_widths, s_lens)] + # wcswidth and _visible_width don't count invisible characters; + # padfn doesn't need to apply another correction + padded_strings = [padfn(w, s) for s, w in zip(strings, visible_widths)] + return padded_strings + + +def _more_generic(type1, type2): + types = {_none_type: 0, _bool_type: 1, int: 2, float: 3, _binary_type: 4, + _text_type: 5} + invtypes = {5: _text_type, 4: _binary_type, 3: float, 2: int, + 1: _bool_type, 0: _none_type} + moregeneric = max(types.get(type1, 5), types.get(type2, 5)) + return invtypes[moregeneric] + + +def _column_type(strings, has_invisible=True, numparse=True): + """The least generic type all column values are convertible to. + + >>> _column_type([True, False]) is _bool_type + True + >>> _column_type(["1", "2"]) is _int_type + True + >>> _column_type(["1", "2.3"]) is _float_type + True + >>> _column_type(["1", "2.3", "four"]) is _text_type + True + >>> _column_type(["four", '\u043f\u044f\u0442\u044c']) is _text_type + True + >>> _column_type([None, "brux"]) is _text_type + True + >>> _column_type([1, 2, None]) is _int_type + True + >>> import datetime as dt + >>> _column_type([dt.datetime(1991,2,19), dt.time(17,35)]) is _text_type + True + + """ + types = [_type(s, has_invisible, numparse) for s in strings] + return reduce(_more_generic, types, _bool_type) + + +def _format(val, valtype, floatfmt, missingval="", has_invisible=True): + """Format a value accoding to its type. + + Unicode is supported: + + >>> hrow = ['\u0431\u0443\u043a\u0432\u0430', + ... '\u0446\u0438\u0444\u0440\u0430'] + >>> tbl = [['\u0430\u0437', 2], ['\u0431\u0443\u043a\u0438', 4]] + >>> good_result = ('\\u0431\\u0443\\u043a\\u0432\\u0430 ' + ... '\\u0446\\u0438\\u0444\\u0440\\u0430\\n------- ' + ... '-------\\n\\u0430\\u0437 ' + ... '2\\n\\u0431\\u0443\\u043a\\u0438 4') + >>> tabulate(tbl, headers=hrow) == good_result + True + + """ + if val is None: + return missingval + + if valtype in [int, _text_type]: + return "{0}".format(val) + elif valtype is _binary_type: + try: + return _text_type(val, "ascii") + except TypeError: + return _text_type(val) + elif valtype is float: + is_a_colored_number = (has_invisible and + isinstance(val, (_text_type, _binary_type))) + if is_a_colored_number: + raw_val = _strip_invisible(val) + formatted_val = format(float(raw_val), floatfmt) + return val.replace(raw_val, formatted_val) + else: + return format(float(val), floatfmt) + else: + return "{0}".format(val) + + +def _align_header(header, alignment, width, visible_width): + "Pad string header to width chars given known visible_width of the header." + width += len(header) - visible_width + if alignment == "left": + return _padright(width, header) + elif alignment == "center": + return _padboth(width, header) + elif not alignment: + return "{0}".format(header) + else: + return _padleft(width, header) + + +def _prepend_row_index(rows, index): + """Add a left-most index column.""" + if index is None or index is False: + return rows + if len(index) != len(rows): + print('index=', index) + print('rows=', rows) + raise ValueError('index must be as long as the number of data rows') + rows = [[v]+list(row) for v, row in zip(index, rows)] + return rows + + +def _bool(val): + "A wrapper around standard bool() which doesn't throw on NumPy arrays" + try: + return bool(val) + except ValueError: # val is likely to be a numpy array with many elements + return False + + +def _normalize_tabular_data(tabular_data, headers, showindex="default"): + """Transform a supported data type to a list of lists, and a list of headers. + + Supported tabular data types: + + * list-of-lists or another iterable of iterables + + * list of named tuples (usually used with headers="keys") + + * list of dicts (usually used with headers="keys") + + * list of OrderedDicts (usually used with headers="keys") + + * 2D NumPy arrays + + * NumPy record arrays (usually used with headers="keys") + + * dict of iterables (usually used with headers="keys") + + * pandas.DataFrame (usually used with headers="keys") + + The first row can be used as headers if headers="firstrow", + column indices can be used as headers if headers="keys". + + If showindex="default", show row indices of the pandas.DataFrame. + If showindex="always", show row indices for all types of data. + If showindex="never", don't show row indices for all types of data. + If showindex is an iterable, show its values as row indices. + + """ + + try: + bool(headers) + is_headers2bool_broken = False + except ValueError: # numpy.ndarray, pandas.core.index.Index, ... + is_headers2bool_broken = True + headers = list(headers) + + index = None + if hasattr(tabular_data, "keys") and hasattr(tabular_data, "values"): + # dict-like and pandas.DataFrame? + if hasattr(tabular_data.values, "__call__"): + # likely a conventional dict + keys = tabular_data.keys() + # columns have to be transposed + rows = list(izip_longest(*tabular_data.values())) + elif hasattr(tabular_data, "index"): + # values is a property, has .index => it's likely a + # pandas.DataFrame (pandas 0.11.0) + keys = list(tabular_data) + if tabular_data.index.name is not None: + if isinstance(tabular_data.index.name, list): + keys[:0] = tabular_data.index.name + else: + keys[:0] = [tabular_data.index.name] + # values matrix doesn't need to be transposed + vals = tabular_data.values + # for DataFrames add an index per default + index = list(tabular_data.index) + rows = [list(row) for row in vals] + else: + raise ValueError( + "tabular data doesn't appear to be a dict or a DataFrame") + + if headers == "keys": + headers = list(map(_text_type, keys)) # headers should be strings + + else: # it's a usual an iterable of iterables, or a NumPy array + rows = list(tabular_data) + + if (headers == "keys" and not rows): + # an empty table (issue #81) + headers = [] + elif (headers == "keys" and + hasattr(tabular_data, "dtype") and + getattr(tabular_data.dtype, "names")): + # numpy record array + headers = tabular_data.dtype.names + elif (headers == "keys" + and len(rows) > 0 + and isinstance(rows[0], tuple) + and hasattr(rows[0], "_fields")): + # namedtuple + headers = list(map(_text_type, rows[0]._fields)) + elif (len(rows) > 0 + and isinstance(rows[0], dict)): + # dict or OrderedDict + uniq_keys = set() # implements hashed lookup + keys = [] # storage for set + if headers == "firstrow": + firstdict = rows[0] if len(rows) > 0 else {} + keys.extend(firstdict.keys()) + uniq_keys.update(keys) + rows = rows[1:] + for row in rows: + for k in row.keys(): + # Save unique items in input order + if k not in uniq_keys: + keys.append(k) + uniq_keys.add(k) + if headers == 'keys': + headers = keys + elif isinstance(headers, dict): + # a dict of headers for a list of dicts + headers = [headers.get(k, k) for k in keys] + headers = list(map(_text_type, headers)) + elif headers == "firstrow": + if len(rows) > 0: + headers = [firstdict.get(k, k) for k in keys] + headers = list(map(_text_type, headers)) + else: + headers = [] + elif headers: + raise ValueError( + 'headers for a list of dicts is not a dict or a keyword') + rows = [[row.get(k) for k in keys] for row in rows] + + elif (headers == "keys" + and hasattr(tabular_data, "description") + and hasattr(tabular_data, "fetchone") + and hasattr(tabular_data, "rowcount")): + # Python Database API cursor object (PEP 0249) + # print tabulate(cursor, headers='keys') + headers = [column[0] for column in tabular_data.description] + + elif headers == "keys" and len(rows) > 0: + # keys are column indices + headers = list(map(_text_type, range(len(rows[0])))) + + # take headers from the first row if necessary + if headers == "firstrow" and len(rows) > 0: + if index is not None: + headers = [index[0]] + list(rows[0]) + index = index[1:] + else: + headers = rows[0] + headers = list(map(_text_type, headers)) # headers should be strings + rows = rows[1:] + + headers = list(map(_text_type, headers)) + rows = list(map(list, rows)) + + # add or remove an index column + showindex_is_a_str = type(showindex) in [_text_type, _binary_type] + if showindex == "default" and index is not None: + rows = _prepend_row_index(rows, index) + elif isinstance(showindex, Iterable) and not showindex_is_a_str: + rows = _prepend_row_index(rows, list(showindex)) + elif (showindex == "always" or + (_bool(showindex) and not showindex_is_a_str)): + if index is None: + index = list(range(len(rows))) + rows = _prepend_row_index(rows, index) + elif (showindex == "never" or + (not _bool(showindex) and not showindex_is_a_str)): + pass + + # pad with empty headers for initial columns if necessary + if headers and len(rows) > 0: + nhs = len(headers) + ncols = len(rows[0]) + if nhs < ncols: + headers = [""]*(ncols - nhs) + headers + + return rows, headers + + +def tabulate(tabular_data, headers=(), tablefmt="simple", + floatfmt=_DEFAULT_FLOATFMT, numalign="decimal", stralign="left", + missingval=_DEFAULT_MISSINGVAL, showindex="default", + disable_numparse=False): + """Format a fixed width table for pretty printing. + + >>> print(tabulate([[1, 2.34], [-56, "8.999"], ["2", "10001"]])) + --- --------- + 1 2.34 + -56 8.999 + 2 10001 + --- --------- + + The first required argument (`tabular_data`) can be a + list-of-lists (or another iterable of iterables), a list of named + tuples, a dictionary of iterables, an iterable of dictionaries, + a two-dimensional NumPy array, NumPy record array, or a Pandas' + dataframe. + + + Table headers + ------------- + + To print nice column headers, supply the second argument (`headers`): + + - `headers` can be an explicit list of column headers + - if `headers="firstrow"`, then the first row of data is used + - if `headers="keys"`, then dictionary keys or column indices are used + + Otherwise a headerless table is produced. + + If the number of headers is less than the number of columns, they + are supposed to be names of the last columns. This is consistent + with the plain-text format of R and Pandas' dataframes. + + >>> print(tabulate([["sex","age"],["Alice","F",24],["Bob","M",19]], + ... headers="firstrow")) + sex age + ----- ----- ----- + Alice F 24 + Bob M 19 + + By default, pandas.DataFrame data have an additional column called + row index. To add a similar column to all other types of data, + use `showindex="always"` or `showindex=True`. To suppress row indices + for all types of data, pass `showindex="never" or `showindex=False`. + To add a custom row index column, pass `showindex=some_iterable`. + + >>> print(tabulate([["F",24],["M",19]], showindex="always")) + - - -- + 0 F 24 + 1 M 19 + - - -- + + + Column alignment + ---------------- + + `tabulate` tries to detect column types automatically, and aligns + the values properly. By default it aligns decimal points of the + numbers (or flushes integer numbers to the right), and flushes + everything else to the left. Possible column alignments + (`numalign`, `stralign`) are: "right", "center", "left", "decimal" + (only for `numalign`), and None (to disable alignment). + + + Table formats + ------------- + + `floatfmt` is a format specification used for columns which + contain numeric data with a decimal point. This can also be + a list or tuple of format strings, one per column. + + `None` values are replaced with a `missingval` string (like + `floatfmt`, this can also be a list of values for different + columns): + + >>> print(tabulate([["spam", 1, None], + ... ["eggs", 42, 3.14], + ... ["other", None, 2.7]], missingval="?")) + ----- -- ---- + spam 1 ? + eggs 42 3.14 + other ? 2.7 + ----- -- ---- + + Various plain-text table formats (`tablefmt`) are supported: + 'plain', 'simple', 'grid', 'pipe', 'orgtbl', 'rst', 'mediawiki', + 'latex', 'latex_raw' and 'latex_booktabs'. Variable `tabulate_formats` + contains the list of currently supported formats. + + "plain" format doesn't use any pseudographics to draw tables, + it separates columns with a double space: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "plain")) + strings numbers + spam 41.9999 + eggs 451 + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="plain")) + spam 41.9999 + eggs 451 + + "simple" format is like Pandoc simple_tables: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "simple")) + strings numbers + --------- --------- + spam 41.9999 + eggs 451 + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="simple")) + ---- -------- + spam 41.9999 + eggs 451 + ---- -------- + + "grid" is similar to tables produced by Emacs table.el package or + Pandoc grid_tables: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "grid")) + +-----------+-----------+ + | strings | numbers | + +===========+===========+ + | spam | 41.9999 | + +-----------+-----------+ + | eggs | 451 | + +-----------+-----------+ + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="grid")) + +------+----------+ + | spam | 41.9999 | + +------+----------+ + | eggs | 451 | + +------+----------+ + + "fancy_grid" draws a grid using box-drawing characters: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "fancy_grid")) + ╒═══════════╤═══════════╕ + │ strings │ numbers │ + ╞═══════════╪═══════════╡ + │ spam │ 41.9999 │ + ├───────────┼───────────┤ + │ eggs │ 451 │ + ╘═══════════╧═══════════╛ + + "pipe" is like tables in PHP Markdown Extra extension or Pandoc + pipe_tables: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "pipe")) + | strings | numbers | + |:----------|----------:| + | spam | 41.9999 | + | eggs | 451 | + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="pipe")) + |:-----|---------:| + | spam | 41.9999 | + | eggs | 451 | + + "orgtbl" is like tables in Emacs org-mode and orgtbl-mode. They + are slightly different from "pipe" format by not using colons to + define column alignment, and using a "+" sign to indicate line + intersections: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "orgtbl")) + | strings | numbers | + |-----------+-----------| + | spam | 41.9999 | + | eggs | 451 | + + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="orgtbl")) + | spam | 41.9999 | + | eggs | 451 | + + "rst" is like a simple table format from reStructuredText; please + note that reStructuredText accepts also "grid" tables: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "rst")) + ========= ========= + strings numbers + ========= ========= + spam 41.9999 + eggs 451 + ========= ========= + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="rst")) + ==== ======== + spam 41.9999 + eggs 451 + ==== ======== + + "mediawiki" produces a table markup used in Wikipedia and on other + MediaWiki-based sites: + + >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], + ... ["eggs", "451.0"]], headers="firstrow", + ... tablefmt="mediawiki")) + {| class="wikitable" style="text-align: left;" + |+ + |- + ! strings !! align="right"| numbers + |- + | spam || align="right"| 41.9999 + |- + | eggs || align="right"| 451 + |} + + "html" produces HTML markup: + + >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], + ... ["eggs", "451.0"]], headers="firstrow", + ... tablefmt="html")) + + + + + + + + +
strings numbers
spam 41.9999
eggs 451
+ + "latex" produces a tabular environment of LaTeX document markup: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="latex")) + \\begin{tabular}{lr} + \\hline + spam & 41.9999 \\\\ + eggs & 451 \\\\ + \\hline + \\end{tabular} + + "latex_raw" is similar to "latex", but doesn't escape special characters, + such as backslash and underscore, so LaTeX commands may embedded into + cells' values: + + >>> print(tabulate([["spam$_9$", 41.9999], ["\\\\emph{eggs}", "451.0"]], + ... tablefmt="latex_raw")) + \\begin{tabular}{lr} + \\hline + spam$_9$ & 41.9999 \\\\ + \\emph{eggs} & 451 \\\\ + \\hline + \\end{tabular} + + "latex_booktabs" produces a tabular environment of LaTeX document markup + using the booktabs.sty package: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="latex_booktabs")) + \\begin{tabular}{lr} + \\toprule + spam & 41.9999 \\\\ + eggs & 451 \\\\ + \\bottomrule + \end{tabular} + + Number parsing + -------------- + By default, anything which can be parsed as a number is a number. + This ensures numbers represented as strings are aligned properly. + This can lead to weird results for particular strings such as + specific git SHAs e.g. "42992e1" will be parsed into the number + 429920 and aligned as such. + + To completely disable number parsing (and alignment), use + `disable_numparse=True`. For more fine grained control, a list column + indices is used to disable number parsing only on those columns + e.g. `disable_numparse=[0, 2]` would disable number parsing only on the + first and third columns. + """ + if tabular_data is None: + tabular_data = [] + list_of_lists, headers = _normalize_tabular_data( + tabular_data, headers, showindex=showindex) + + # empty values in the first column of RST tables should be escaped + # (issue #82). "" should be escaped as "\\ " or ".." + if tablefmt == 'rst': + list_of_lists, headers = _rst_escape_first_column(list_of_lists, + headers) + + # optimization: look for ANSI control codes once, + # enable smart width functions only if a control code is found + plain_text = '\n'.join(['\t'.join(map(_text_type, headers))] + + ['\t'.join(map(_text_type, row)) + for row in list_of_lists]) + + has_invisible = re.search(_invisible_codes, plain_text) + enable_widechars = wcwidth is not None and WIDE_CHARS_MODE + if has_invisible: + width_fn = _visible_width + elif enable_widechars: # optional wide-character support if available + width_fn = wcwidth.wcswidth + else: + width_fn = len + + # format rows and columns, convert numeric values to strings + cols = list(izip_longest(*list_of_lists)) + numparses = _expand_numparse(disable_numparse, len(cols)) + coltypes = [_column_type(col, numparse=np) for col, np in + zip(cols, numparses)] + if isinstance(floatfmt, basestring): # old version + # just duplicate the string to use in each column + float_formats = len(cols) * [floatfmt] + else: # if floatfmt is list, tuple etc we have one per column + float_formats = list(floatfmt) + if len(float_formats) < len(cols): + float_formats.extend((len(cols)-len(float_formats)) * + [_DEFAULT_FLOATFMT]) + if isinstance(missingval, basestring): + missing_vals = len(cols) * [missingval] + else: + missing_vals = list(missingval) + if len(missing_vals) < len(cols): + missing_vals.extend((len(cols)-len(missing_vals)) * + [_DEFAULT_MISSINGVAL]) + cols = [[_format(v, ct, fl_fmt, miss_v, has_invisible) for v in c] + for c, ct, fl_fmt, miss_v in zip(cols, coltypes, float_formats, + missing_vals)] + + # align columns + aligns = [numalign if ct in [int, float] else stralign for ct in coltypes] + minwidths = [width_fn(h) + MIN_PADDING + for h in headers] if headers else [0]*len(cols) + cols = [_align_column(c, a, minw, has_invisible) + for c, a, minw in zip(cols, aligns, minwidths)] + + if headers: + # align headers and add headers + t_cols = cols or [['']] * len(headers) + t_aligns = aligns or [stralign] * len(headers) + minwidths = [max(minw, width_fn(c[0])) + for minw, c in zip(minwidths, t_cols)] + headers = [_align_header(h, a, minw, width_fn(h)) + for h, a, minw in zip(headers, t_aligns, minwidths)] + rows = list(zip(*cols)) + else: + minwidths = [width_fn(c[0]) for c in cols] + rows = list(zip(*cols)) + + if not isinstance(tablefmt, TableFormat): + tablefmt = _table_formats.get(tablefmt, _table_formats["simple"]) + + return _format_table(tablefmt, headers, rows, minwidths, aligns) + + +def _expand_numparse(disable_numparse, column_count): + """ + Return a list of bools of length `column_count` which indicates whether + number parsing should be used on each column. + If `disable_numparse` is a list of indices, each of those indices are + False, and everything else is True. + If `disable_numparse` is a bool, then the returned list is all the same. + """ + if isinstance(disable_numparse, Iterable): + numparses = [True] * column_count + for index in disable_numparse: + numparses[index] = False + return numparses + else: + return [not disable_numparse] * column_count + + +def _build_simple_row(padded_cells, rowfmt): + "Format row according to DataRow format without padding." + begin, sep, end = rowfmt + return (begin + sep.join(padded_cells) + end).rstrip() + + +def _build_row(padded_cells, colwidths, colaligns, rowfmt): + "Return a string which represents a row of data cells." + if not rowfmt: + return None + if hasattr(rowfmt, "__call__"): + return rowfmt(padded_cells, colwidths, colaligns) + else: + return _build_simple_row(padded_cells, rowfmt) + + +def _build_line(colwidths, colaligns, linefmt): + "Return a string which represents a horizontal line." + if not linefmt: + return None + if hasattr(linefmt, "__call__"): + return linefmt(colwidths, colaligns) + else: + begin, fill, sep, end = linefmt + cells = [fill*w for w in colwidths] + return _build_simple_row(cells, (begin, sep, end)) + + +def _pad_row(cells, padding): + if cells: + pad = " "*padding + padded_cells = [pad + cell + pad for cell in cells] + return padded_cells + else: + return cells + + +def _format_table(fmt, headers, rows, colwidths, colaligns): + """Produce a plain-text representation of the table.""" + lines = [] + hidden = fmt.with_header_hide if (headers and fmt.with_header_hide) else [] + pad = fmt.padding + headerrow = fmt.headerrow + + padded_widths = [(w + 2*pad) for w in colwidths] + padded_headers = _pad_row(headers, pad) + padded_rows = [_pad_row(row, pad) for row in rows] + + if fmt.lineabove and "lineabove" not in hidden: + lines.append(_build_line(padded_widths, colaligns, fmt.lineabove)) + + if padded_headers: + lines.append(_build_row(padded_headers, padded_widths, colaligns, + headerrow)) + if fmt.linebelowheader and "linebelowheader" not in hidden: + lines.append(_build_line(padded_widths, colaligns, + fmt.linebelowheader)) + + if padded_rows and fmt.linebetweenrows and "linebetweenrows" not in hidden: + # initial rows with a line below + for row in padded_rows[:-1]: + lines.append(_build_row(row, padded_widths, colaligns, + fmt.datarow)) + lines.append(_build_line(padded_widths, colaligns, + fmt.linebetweenrows)) + # the last row without a line below + lines.append(_build_row(padded_rows[-1], padded_widths, colaligns, + fmt.datarow)) + else: + for row in padded_rows: + lines.append(_build_row(row, padded_widths, colaligns, + fmt.datarow)) + + if fmt.linebelow and "linebelow" not in hidden: + lines.append(_build_line(padded_widths, colaligns, fmt.linebelow)) + + if headers or rows: + return "\n".join(lines) + else: # a completely empty table + return "" + + +def _main(): + """\ + Usage: tabulate [options] [FILE ...] + + Pretty-print tabular data. + See also https://bitbucket.org/astanin/python-tabulate + + FILE a filename of the file with tabular data; + if "-" or missing, read data from stdin. + + Options: + + -h, --help show this message + -1, --header use the first row of data as a table header + -o FILE, --output FILE print table to FILE (default: stdout) + -s REGEXP, --sep REGEXP use a custom column separator (default: whitespace) + -F FPFMT, --float FPFMT floating point number format (default: g) + -f FMT, --format FMT set output table format; supported formats: + plain, simple, grid, fancy_grid, pipe, orgtbl, + rst, mediawiki, html, latex, latex_raw, + latex_booktabs, tsv + (default: simple) + """ + import getopt + import sys + import textwrap + usage = textwrap.dedent(_main.__doc__) + try: + opts, args = getopt.getopt( + sys.argv[1:], "h1o:s:F:f:", + ["help", "header", "output", "sep=", "float=", "format="]) + except getopt.GetoptError as e: + print(e) + print(usage) + sys.exit(2) + headers = [] + floatfmt = _DEFAULT_FLOATFMT + tablefmt = "simple" + sep = r"\s+" + outfile = "-" + for opt, value in opts: + if opt in ["-1", "--header"]: + headers = "firstrow" + elif opt in ["-o", "--output"]: + outfile = value + elif opt in ["-F", "--float"]: + floatfmt = value + elif opt in ["-f", "--format"]: + if value not in tabulate_formats: + print("%s is not a supported table format" % value) + print(usage) + sys.exit(3) + tablefmt = value + elif opt in ["-s", "--sep"]: + sep = value + elif opt in ["-h", "--help"]: + print(usage) + sys.exit(0) + files = [sys.stdin] if not args else args + with (sys.stdout if outfile == "-" else open(outfile, "w")) as out: + for f in files: + if f == "-": + f = sys.stdin + if _is_file(f): + _pprint_file(f, headers=headers, tablefmt=tablefmt, + sep=sep, floatfmt=floatfmt, file=out) + else: + with open(f) as fobj: + _pprint_file(fobj, headers=headers, tablefmt=tablefmt, + sep=sep, floatfmt=floatfmt, file=out) + + +def _pprint_file(fobject, headers, tablefmt, sep, floatfmt, file): + rows = fobject.readlines() + table = [re.split(sep, r.rstrip()) for r in rows if r.strip()] + print(tabulate(table, headers, tablefmt, floatfmt=floatfmt), file=file) + + +if __name__ == "__main__": + _main() diff --git a/setup.py b/setup.py index 21e55508..417929c6 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,6 @@ 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.5', 'pycryptodome >= 3', - 'tabulate >= 0.7.6', 'terminaltables >= 3.0.0', ] From 4c9060569d23fe177324387733a2a1c17a9532e2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:13:30 -0500 Subject: [PATCH 59/76] Add preserve whitespace option to tabulate. --- mycli/output_formatter/tabulate_adapter.py | 2 ++ mycli/packages/tabulate.py | 10 +++++++--- tests/test_tabulate.py | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 tests/test_tabulate.py diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 14e6c770..c9b1acd2 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -1,6 +1,8 @@ from mycli.packages import tabulate from .preprocessors import bytes_to_string, align_decimals, quote_whitespaces +tabulate.PRESERVE_WHITESPACE = True + supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', 'html', 'latex', 'latex_booktabs', 'textile') diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index a2503299..292f40bb 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -53,6 +53,7 @@ def _is_file(f): # minimum extra space in headers MIN_PADDING = 2 +PRESERVE_WHITESPACE = False _DEFAULT_FLOATFMT = "g" _DEFAULT_MISSINGVAL = "" @@ -563,10 +564,12 @@ def _align_column(strings, alignment, minwidth=0, has_invisible=True): """ if alignment == "right": - strings = [s.strip() for s in strings] + if not PRESERVE_WHITESPACE: + strings = [s.strip() for s in strings] padfn = _padleft elif alignment == "center": - strings = [s.strip() for s in strings] + if not PRESERVE_WHITESPACE: + strings = [s.strip() for s in strings] padfn = _padboth elif alignment == "decimal": if has_invisible: @@ -580,7 +583,8 @@ def _align_column(strings, alignment, minwidth=0, has_invisible=True): elif not alignment: return strings else: - strings = [s.strip() for s in strings] + if not PRESERVE_WHITESPACE: + strings = [s.strip() for s in strings] padfn = _padright enable_widechars = wcwidth is not None and WIDE_CHARS_MODE diff --git a/tests/test_tabulate.py b/tests/test_tabulate.py new file mode 100644 index 00000000..ae7c25ce --- /dev/null +++ b/tests/test_tabulate.py @@ -0,0 +1,17 @@ +from textwrap import dedent + +from mycli.packages import tabulate + +tabulate.PRESERVE_WHITESPACE = True + + +def test_dont_strip_leading_whitespace(): + data = [[' abc']] + headers = ['xyz'] + tbl = tabulate.tabulate(data, headers, tablefmt='psql') + assert tbl == dedent(''' + +---------+ + | xyz | + |---------| + | abc | + +---------+ ''').strip() From 694919eae6b1a4489e61c8f8adfc390b05ec6f55 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:17:12 -0500 Subject: [PATCH 60/76] Remove extra html format. --- mycli/output_formatter/tabulate_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index c9b1acd2..fc6a7d8d 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -5,7 +5,7 @@ supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', - 'html', 'latex', 'latex_booktabs', 'textile') + 'latex', 'latex_booktabs', 'textile') preprocessors = (bytes_to_string, align_decimals, quote_whitespaces) From 85d0e026278b7f0f2b044b483412832beae205d8 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:29:49 -0500 Subject: [PATCH 61/76] Do not align columns for markup tables. --- mycli/output_formatter/tabulate_adapter.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index fc6a7d8d..7b4dfac3 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -3,14 +3,20 @@ tabulate.PRESERVE_WHITESPACE = True -supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', - 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', - 'latex', 'latex_booktabs', 'textile') +supported_markup_formats = ('mediawiki', 'html', 'latex', 'latex_booktabs', + 'textile', 'moinmoin', 'jira') +supported_table_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', + 'orgtbl', 'psql', 'rst') +supported_formats = supported_markup_formats + supported_table_formats preprocessors = (bytes_to_string, align_decimals, quote_whitespaces) def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate.tabulate(data, headers, tablefmt=table_format, - missingval=missing_value, disable_numparse=True) + kwargs = {'tablefmt': table_format, 'missingval': missing_value, + 'disable_numparse': True} + if table_format in supported_markup_formats: + kwargs.update(numalign=None, stralign=None) + + return tabulate.tabulate(data, headers, **kwargs) From ae8311c5424a9dc4632364d671e8615d38d44d67 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:30:51 -0500 Subject: [PATCH 62/76] Do not quote whitespaces for tabulate formats. --- mycli/output_formatter/tabulate_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 7b4dfac3..26ef40aa 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -9,7 +9,7 @@ 'orgtbl', 'psql', 'rst') supported_formats = supported_markup_formats + supported_table_formats -preprocessors = (bytes_to_string, align_decimals, quote_whitespaces) +preprocessors = (bytes_to_string, align_decimals) def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): From 359044c8e87ed0e8394eac2dacb3e0d53cc498c5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:47:42 -0500 Subject: [PATCH 63/76] Pep8radius changes. --- mycli/packages/tabulate.py | 41 +++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index 292f40bb..ab3796f6 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -180,9 +180,9 @@ def _moin_row_with_attrs(celltag, cell_values, colwidths, colaligns, "decimal": ''} values_with_attrs = ["{0}{1} {2} ".format(celltag, alignment.get(a, ''), - header+c+header) + header + c + header) for c, a in zip(cell_values, colaligns)] - return "".join(values_with_attrs)+"||" + return "".join(values_with_attrs) + "||" def _latex_line_begin_tabular(colwidths, colaligns, booktabs=False): @@ -684,7 +684,8 @@ def _format(val, valtype, floatfmt, missingval="", has_invisible=True): def _align_header(header, alignment, width, visible_width): - "Pad string header to width chars given known visible_width of the header." + """Pad string header to width chars given known visible_width of the + header.""" width += len(header) - visible_width if alignment == "left": return _padright(width, header) @@ -704,12 +705,13 @@ def _prepend_row_index(rows, index): print('index=', index) print('rows=', rows) raise ValueError('index must be as long as the number of data rows') - rows = [[v]+list(row) for v, row in zip(index, rows)] + rows = [[v] + list(row) for v, row in zip(index, rows)] return rows def _bool(val): - "A wrapper around standard bool() which doesn't throw on NumPy arrays" + """A wrapper around standard bool() which doesn't throw on NumPy + arrays.""" try: return bool(val) except ValueError: # val is likely to be a numpy array with many elements @@ -717,7 +719,8 @@ def _bool(val): def _normalize_tabular_data(tabular_data, headers, showindex="default"): - """Transform a supported data type to a list of lists, and a list of headers. + """Transform a supported data type to a list of lists, and a list of + headers. Supported tabular data types: @@ -878,7 +881,7 @@ def _normalize_tabular_data(tabular_data, headers, showindex="default"): nhs = len(headers) ncols = len(rows[0]) if nhs < ncols: - headers = [""]*(ncols - nhs) + headers + headers = [""] * (ncols - nhs) + headers return rows, headers @@ -1169,11 +1172,12 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", indices is used to disable number parsing only on those columns e.g. `disable_numparse=[0, 2]` would disable number parsing only on the first and third columns. + """ if tabular_data is None: tabular_data = [] list_of_lists, headers = _normalize_tabular_data( - tabular_data, headers, showindex=showindex) + tabular_data, headers, showindex=showindex) # empty values in the first column of RST tables should be escaped # (issue #82). "" should be escaped as "\\ " or ".." @@ -1207,14 +1211,14 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", else: # if floatfmt is list, tuple etc we have one per column float_formats = list(floatfmt) if len(float_formats) < len(cols): - float_formats.extend((len(cols)-len(float_formats)) * + float_formats.extend((len(cols) - len(float_formats)) * [_DEFAULT_FLOATFMT]) if isinstance(missingval, basestring): missing_vals = len(cols) * [missingval] else: missing_vals = list(missingval) if len(missing_vals) < len(cols): - missing_vals.extend((len(cols)-len(missing_vals)) * + missing_vals.extend((len(cols) - len(missing_vals)) * [_DEFAULT_MISSINGVAL]) cols = [[_format(v, ct, fl_fmt, miss_v, has_invisible) for v in c] for c, ct, fl_fmt, miss_v in zip(cols, coltypes, float_formats, @@ -1223,7 +1227,7 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", # align columns aligns = [numalign if ct in [int, float] else stralign for ct in coltypes] minwidths = [width_fn(h) + MIN_PADDING - for h in headers] if headers else [0]*len(cols) + for h in headers] if headers else [0] * len(cols) cols = [_align_column(c, a, minw, has_invisible) for c, a, minw in zip(cols, aligns, minwidths)] @@ -1247,12 +1251,13 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", def _expand_numparse(disable_numparse, column_count): - """ - Return a list of bools of length `column_count` which indicates whether + """Return a list of bools of length `column_count` which indicates whether number parsing should be used on each column. - If `disable_numparse` is a list of indices, each of those indices are - False, and everything else is True. - If `disable_numparse` is a bool, then the returned list is all the same. + + If `disable_numparse` is a list of indices, each of those indices + are False, and everything else is True. If `disable_numparse` is a + bool, then the returned list is all the same. + """ if isinstance(disable_numparse, Iterable): numparses = [True] * column_count @@ -1346,8 +1351,7 @@ def _format_table(fmt, headers, rows, colwidths, colaligns): def _main(): - """\ - Usage: tabulate [options] [FILE ...] + """\ Usage: tabulate [options] [FILE ...] Pretty-print tabular data. See also https://bitbucket.org/astanin/python-tabulate @@ -1367,6 +1371,7 @@ def _main(): rst, mediawiki, html, latex, latex_raw, latex_booktabs, tsv (default: simple) + """ import getopt import sys From 13991b193a2d764a7baa16937193f37a5bdc3c6e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 16:06:23 -0500 Subject: [PATCH 64/76] Add output format changes to changelog. --- changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.md b/changelog.md index eb36b6ba..5bda9ae7 100644 --- a/changelog.md +++ b/changelog.md @@ -5,6 +5,8 @@ Features: --------- * Add ability to specify alternative myclirc file. (Thanks: [Dick Marinus]). +* Add new display formats for pretty printing query results. (Thanks: [Amjith + Ramanujam], [Dick Marinus], [Thomas Roten]). Bug Fixes: ---------- From 33e5c26eba1f3261b66d603fcbcb66b8de8f76e2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 16:11:03 -0500 Subject: [PATCH 65/76] Add additional table formats to myclirc. --- mycli/myclirc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mycli/myclirc b/mycli/myclirc index 2f359927..febd7e59 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -30,7 +30,9 @@ log_level = INFO # Timing of sql statments and table rendering. timing = True -# Table format. Possible values: ascii, single, double, or github. +# Table format. Possible values: ascii, single, double, github, +# psql, plain, simple, grid, fancy_grid, pipe, orgtbl, rst, mediawiki, html, +# latex, latex_booktabs, textile, moinmoin, jira, expanded, tsv, csv. # Recommended: ascii table_format = ascii From f06451cb4c12baf30f32d8fd0be3da0042d2f23c Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 16:16:13 -0500 Subject: [PATCH 66/76] Don't import quote_whitespaces since it's not used. --- mycli/output_formatter/tabulate_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 26ef40aa..ee63b5b1 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -1,5 +1,5 @@ from mycli.packages import tabulate -from .preprocessors import bytes_to_string, align_decimals, quote_whitespaces +from .preprocessors import bytes_to_string, align_decimals tabulate.PRESERVE_WHITESPACE = True From 8f70b12ef9db79e2610eba5c8c45a0b92d07fc93 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 21:11:04 -0500 Subject: [PATCH 67/76] Fix outdated docstring. --- mycli/output_formatter/output_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 759fb879..9f9b6a8e 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -27,7 +27,7 @@ class OutputFormatter(object): _output_formats = {} def __init__(self, format_name=None): - """Register the supported output formats.""" + """Set the default *format_name*.""" self._format_name = format_name def set_format_name(self, format_name): From fd861d41e041b08da83dface8b6d8a0843f3b2fa Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 21:30:21 -0500 Subject: [PATCH 68/76] Make SQLCompleter options pass through refresher. --- mycli/completion_refresher.py | 14 ++++++++------ mycli/main.py | 11 +++++++---- tests/test_completion_refresher.py | 2 +- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/mycli/completion_refresher.py b/mycli/completion_refresher.py index 33afa009..d24e03a8 100644 --- a/mycli/completion_refresher.py +++ b/mycli/completion_refresher.py @@ -13,7 +13,7 @@ def __init__(self): self._completer_thread = None self._restart_refresh = threading.Event() - def refresh(self, executor, callbacks): + def refresh(self, executor, callbacks, completer_options={}): """ Creates a SQLCompleter object and populates it with the relevant completion suggestions in a background thread. @@ -23,14 +23,16 @@ def refresh(self, executor, callbacks): callbacks - A function or a list of functions to call after the thread has completed the refresh. The newly created completion object will be passed in as an argument to each callback. + completer_options - dict of options to pass to SQLCompleter. """ if self.is_refreshing(): self._restart_refresh.set() return [(None, None, None, 'Auto-completion refresh restarted.')] else: - self._completer_thread = threading.Thread(target=self._bg_refresh, - args=(executor, callbacks), - name='completion_refresh') + self._completer_thread = threading.Thread( + target=self._bg_refresh, + args=(executor, callbacks, completer_options), + name='completion_refresh') self._completer_thread.setDaemon(True) self._completer_thread.start() return [(None, None, None, @@ -39,8 +41,8 @@ def refresh(self, executor, callbacks): def is_refreshing(self): return self._completer_thread and self._completer_thread.is_alive() - def _bg_refresh(self, sqlexecute, callbacks): - completer = SQLCompleter(smart_completion=True) + def _bg_refresh(self, sqlexecute, callbacks, completer_options): + completer = SQLCompleter(**completer_options) # Create a new pgexecute method to popoulate the completions. e = sqlexecute diff --git a/mycli/main.py b/mycli/main.py index 64c58705..2a9b0436 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -146,8 +146,9 @@ def __init__(self, sqlexecute=None, prompt=None, # Initialize completer. self.smart_completion = c['main'].as_bool('smart_completion') - self.completer = SQLCompleter(self.smart_completion, - supported_formats=self.formatter.supported_formats) + self.completer = SQLCompleter( + self.smart_completion, + supported_formats=self.formatter.supported_formats()) self._completer_lock = threading.Lock() # Register custom special commands. @@ -665,8 +666,10 @@ def refresh_completions(self, reset=False): if reset: with self._completer_lock: self.completer.reset_completions() - self.completion_refresher.refresh(self.sqlexecute, - self._on_completions_refreshed) + self.completion_refresher.refresh( + self.sqlexecute, self._on_completions_refreshed, + {'smart_completion': self.smart_completion, + 'supported_formats': self.formatter.supported_formats()}) return [(None, None, None, 'Auto-completion refresh started in the background.')] diff --git a/tests/test_completion_refresher.py b/tests/test_completion_refresher.py index a50ad749..8851eae6 100644 --- a/tests/test_completion_refresher.py +++ b/tests/test_completion_refresher.py @@ -37,7 +37,7 @@ def test_refresh_called_once(refresher): assert len(actual) == 1 assert len(actual[0]) == 4 assert actual[0][3] == 'Auto-completion refresh started in the background.' - bg_refresh.assert_called_with(sqlexecute, callbacks) + bg_refresh.assert_called_with(sqlexecute, callbacks, {}) def test_refresh_called_twice(refresher): From 81c54962c530e1656e0d6575829a67d913daa958 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 21:33:02 -0500 Subject: [PATCH 69/76] Pep8radius fixes. --- mycli/completion_refresher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/completion_refresher.py b/mycli/completion_refresher.py index d24e03a8..2bbe32d0 100644 --- a/mycli/completion_refresher.py +++ b/mycli/completion_refresher.py @@ -14,8 +14,7 @@ def __init__(self): self._restart_refresh = threading.Event() def refresh(self, executor, callbacks, completer_options={}): - """ - Creates a SQLCompleter object and populates it with the relevant + """Creates a SQLCompleter object and populates it with the relevant completion suggestions in a background thread. executor - SQLExecute object, used to extract the credentials to connect @@ -24,6 +23,7 @@ def refresh(self, executor, callbacks, completer_options={}): has completed the refresh. The newly created completion object will be passed in as an argument to each callback. completer_options - dict of options to pass to SQLCompleter. + """ if self.is_refreshing(): self._restart_refresh.set() From 9a23584a03a4e29eaff755768e04cc2589ac90ad Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 13 Apr 2017 08:47:16 -0500 Subject: [PATCH 70/76] Fix fancy_grid table format. --- mycli/packages/tabulate.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index ab3796f6..1e67cea7 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -252,12 +252,12 @@ def escape_empty(val): padding=1, with_header_hide=None), "fancy_grid": TableFormat( - lineabove=Line("â•’", "═", "╤", "â••"), - linebelowheader=Line("â•ž", "═", "╪", "â•¡"), - linebetweenrows=Line("├", "─", "┼", "┤"), - linebelow=Line("╘", "═", "╧", "â•›"), - headerrow=DataRow("│", "│", "│"), - datarow=DataRow("│", "│", "│"), + lineabove=Line("╒", "═", "╤", "╕"), + linebelowheader=Line("╞", "═", "╪", "╡"), + linebetweenrows=Line("├", "─", "┼", "┤"), + linebelow=Line("╘", "═", "╧", "╛"), + headerrow=DataRow("│", "│", "│"), + datarow=DataRow("│", "│", "│"), padding=1, with_header_hide=None), "pipe": TableFormat( @@ -1032,13 +1032,13 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], ... ["strings", "numbers"], "fancy_grid")) - ╒═══════════╤═══════════╕ - │ strings │ numbers │ - ╞═══════════╪═══════════╡ - │ spam │ 41.9999 │ - ├───────────┼───────────┤ - │ eggs │ 451 │ - ╘═══════════╧═══════════╛ + ╒═══════════╤═══════════╕ + │ strings │ numbers │ + ╞═══════════╪═══════════╡ + │ spam │ 41.9999 │ + ├───────────┼───────────┤ + │ eggs │ 451 │ + ╘═══════════╧═══════════╛ "pipe" is like tables in PHP Markdown Extra extension or Pandoc pipe_tables: From 4b0de47ebe3e9c91cf6136d8ee28fb59fbf92ee2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 13 Apr 2017 13:40:53 -0500 Subject: [PATCH 71/76] Remove single terminaltable format since it's causing problems. --- mycli/myclirc | 2 +- mycli/output_formatter/terminaltables_adapter.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mycli/myclirc b/mycli/myclirc index febd7e59..01a11426 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -30,7 +30,7 @@ log_level = INFO # Timing of sql statments and table rendering. timing = True -# Table format. Possible values: ascii, single, double, github, +# Table format. Possible values: ascii, double, github, # psql, plain, simple, grid, fancy_grid, pipe, orgtbl, rst, mediawiki, html, # latex, latex_booktabs, textile, moinmoin, jira, expanded, tsv, csv. # Recommended: ascii diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py index ac580517..0c702abe 100644 --- a/mycli/output_formatter/terminaltables_adapter.py +++ b/mycli/output_formatter/terminaltables_adapter.py @@ -3,7 +3,7 @@ from .preprocessors import (bytes_to_string, align_decimals, override_missing_value) -supported_formats = ('ascii', 'single', 'double', 'github') +supported_formats = ('ascii', 'double', 'github') preprocessors = (bytes_to_string, override_missing_value, align_decimals) @@ -12,7 +12,6 @@ def terminaltables_adapter(data, headers, table_format=None, **_): table_format_handler = { 'ascii': terminaltables.AsciiTable, - 'single': terminaltables.SingleTable, 'double': terminaltables.DoubleTable, 'github': terminaltables.GithubFlavoredMarkdownTable, } From cc76dee6198b759fd0219bff9cafc383a450456b Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 13 Apr 2017 14:27:19 -0500 Subject: [PATCH 72/76] Use constant for missing value default. --- mycli/output_formatter/output_formatter.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 9f9b6a8e..3e53d826 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -16,6 +16,8 @@ terminaltables_adapter, preprocessors as terminaltables_preprocessors, supported_formats as terminaltables_formats) +MISSING_VALUE = '' + OutputFormatHandler = namedtuple( 'OutputFormatHandler', 'format_name preprocessors formatter formatter_args') @@ -77,22 +79,22 @@ def format_output(self, data, headers, format_name=None, **kwargs): OutputFormatter.register_new_formatter('expanded', expanded_table, (override_missing_value, convert_to_string), - {'missing_value': ''}) + {'missing_value': MISSING_VALUE}) for delimiter_format in delimiter_formats: OutputFormatter.register_new_formatter(delimiter_format, delimiter_adapter, delimiter_preprocessors, {'table_format': delimiter_format, - 'missing_value': ''}) + 'missing_value': MISSING_VALUE}) for tabulate_format in tabulate_formats: OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, tabulate_preprocessors, {'table_format': tabulate_format, - 'missing_value': ''}) + 'missing_value': MISSING_VALUE}) for terminaltables_format in terminaltables_formats: OutputFormatter.register_new_formatter( terminaltables_format, terminaltables_adapter, terminaltables_preprocessors, - {'table_format': terminaltables_format, 'missing_value': ''}) + {'table_format': terminaltables_format, 'missing_value': MISSING_VALUE}) From 39c4aec92645306a3c80ad11ec9213c918437284 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 13 Apr 2017 14:38:38 -0500 Subject: [PATCH 73/76] Make registering a format not require kwargs. --- mycli/output_formatter/output_formatter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 3e53d826..c87da5ed 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -49,8 +49,8 @@ def supported_formats(self): return tuple(self._output_formats.keys()) @classmethod - def register_new_formatter(cls, format_name, handler, preprocessors=None, - kwargs=None): + def register_new_formatter(cls, format_name, handler, preprocessors=(), + kwargs={}): """Register a new formatter to format the output.""" cls._output_formats[format_name] = OutputFormatHandler( format_name, preprocessors, handler, kwargs) From 9c70ce12098fc4ebf1567999cd29745cbd767d3a Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 15 Apr 2017 17:10:07 +0200 Subject: [PATCH 74/76] don't import as --- mycli/output_formatter/output_formatter.py | 39 ++++++++++------------ 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index c87da5ed..ca8fb4eb 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -6,15 +6,10 @@ from .expanded import expanded_table from .preprocessors import (override_missing_value, convert_to_string) -from .delimited_output_adapter import (delimiter_adapter, - supported_formats as delimiter_formats, - delimiter_preprocessors) -from .tabulate_adapter import (tabulate_adapter, - supported_formats as tabulate_formats, - preprocessors as tabulate_preprocessors) -from .terminaltables_adapter import ( - terminaltables_adapter, preprocessors as terminaltables_preprocessors, - supported_formats as terminaltables_formats) + +from . import delimited_output_adapter +from . import tabulate_adapter +from . import terminaltables_adapter MISSING_VALUE = '' @@ -81,20 +76,20 @@ def format_output(self, data, headers, format_name=None, **kwargs): convert_to_string), {'missing_value': MISSING_VALUE}) -for delimiter_format in delimiter_formats: - OutputFormatter.register_new_formatter(delimiter_format, delimiter_adapter, - delimiter_preprocessors, - {'table_format': delimiter_format, - 'missing_value': MISSING_VALUE}) +for delimiter_format in delimited_output_adapter.supported_formats: + OutputFormatter.register_new_formatter( + delimiter_format, delimited_output_adapter.delimiter_adapter, + delimited_output_adapter.delimiter_preprocessors, + {'table_format': delimiter_format, 'missing_value': MISSING_VALUE}) -for tabulate_format in tabulate_formats: - OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, - tabulate_preprocessors, - {'table_format': tabulate_format, - 'missing_value': MISSING_VALUE}) +for tabulate_format in tabulate_adapter.supported_formats: + OutputFormatter.register_new_formatter( + tabulate_format, tabulate_adapter.tabulate_adapter, + tabulate_adapter.preprocessors, + {'table_format': tabulate_format, 'missing_value': MISSING_VALUE}) -for terminaltables_format in terminaltables_formats: +for terminaltables_format in terminaltables_adapter.supported_formats: OutputFormatter.register_new_formatter( - terminaltables_format, terminaltables_adapter, - terminaltables_preprocessors, + terminaltables_format, terminaltables_adapter.terminaltables_adapter, + terminaltables_adapter.preprocessors, {'table_format': terminaltables_format, 'missing_value': MISSING_VALUE}) From a5f11e6e023c0bccdc2d93cf8e13bf8c7aced016 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 15 Apr 2017 18:33:38 +0200 Subject: [PATCH 75/76] rename all adapter functions to adapter --- mycli/output_formatter/delimited_output_adapter.py | 4 ++-- mycli/output_formatter/output_formatter.py | 8 ++++---- mycli/output_formatter/tabulate_adapter.py | 2 +- mycli/output_formatter/terminaltables_adapter.py | 2 +- tests/test_output_formatter.py | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index 8036a5ff..a01a2843 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -8,10 +8,10 @@ from .preprocessors import override_missing_value, bytes_to_string supported_formats = ('csv', 'tsv') -delimiter_preprocessors = (override_missing_value, bytes_to_string) +preprocessors = (override_missing_value, bytes_to_string) -def delimiter_adapter(data, headers, table_format='csv', **_): +def adapter(data, headers, table_format='csv', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" with contextlib.closing(StringIO()) as content: if table_format == 'csv': diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index ca8fb4eb..61e3c8d5 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -78,18 +78,18 @@ def format_output(self, data, headers, format_name=None, **kwargs): for delimiter_format in delimited_output_adapter.supported_formats: OutputFormatter.register_new_formatter( - delimiter_format, delimited_output_adapter.delimiter_adapter, - delimited_output_adapter.delimiter_preprocessors, + delimiter_format, delimited_output_adapter.adapter, + delimited_output_adapter.preprocessors, {'table_format': delimiter_format, 'missing_value': MISSING_VALUE}) for tabulate_format in tabulate_adapter.supported_formats: OutputFormatter.register_new_formatter( - tabulate_format, tabulate_adapter.tabulate_adapter, + tabulate_format, tabulate_adapter.adapter, tabulate_adapter.preprocessors, {'table_format': tabulate_format, 'missing_value': MISSING_VALUE}) for terminaltables_format in terminaltables_adapter.supported_formats: OutputFormatter.register_new_formatter( - terminaltables_format, terminaltables_adapter.terminaltables_adapter, + terminaltables_format, terminaltables_adapter.adapter, terminaltables_adapter.preprocessors, {'table_format': terminaltables_format, 'missing_value': MISSING_VALUE}) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index ee63b5b1..b89dcc0b 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -12,7 +12,7 @@ preprocessors = (bytes_to_string, align_decimals) -def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): +def adapter(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" kwargs = {'tablefmt': table_format, 'missingval': missing_value, 'disable_numparse': True} diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py index 0c702abe..a8f50f98 100644 --- a/mycli/output_formatter/terminaltables_adapter.py +++ b/mycli/output_formatter/terminaltables_adapter.py @@ -7,7 +7,7 @@ preprocessors = (bytes_to_string, override_missing_value, align_decimals) -def terminaltables_adapter(data, headers, table_format=None, **_): +def adapter(data, headers, table_format=None, **_): """Wrap terminaltables inside a standard function for OutputFormatter.""" table_format_handler = { diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index dcc26e62..a0e0f198 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -13,11 +13,11 @@ to_string) from mycli.output_formatter.output_formatter import OutputFormatter from mycli.output_formatter.delimited_output_adapter import ( - delimiter_adapter as csv_wrapper) + adapter as csv_wrapper) from mycli.output_formatter.tabulate_adapter import ( - tabulate_adapter as tabulate_wrapper) + adapter as tabulate_wrapper) from mycli.output_formatter.terminaltables_adapter import ( - terminaltables_adapter as terminal_tables_wrapper) + adapter as terminal_tables_wrapper) def test_to_string(): From 8fa96a2f6f49f190b58e1c0f68d5eba788cc14f7 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 15 Apr 2017 21:21:53 +0200 Subject: [PATCH 76/76] don't import as (also for tests/test_output_formatter.py) --- tests/test_output_formatter.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index a0e0f198..9844c191 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -12,12 +12,9 @@ override_missing_value, to_string) from mycli.output_formatter.output_formatter import OutputFormatter -from mycli.output_formatter.delimited_output_adapter import ( - adapter as csv_wrapper) -from mycli.output_formatter.tabulate_adapter import ( - adapter as tabulate_wrapper) -from mycli.output_formatter.terminaltables_adapter import ( - adapter as terminal_tables_wrapper) +from mycli.output_formatter import delimited_output_adapter +from mycli.output_formatter import tabulate_adapter +from mycli.output_formatter import terminaltables_adapter def test_to_string(): @@ -98,7 +95,7 @@ def test_tabulate_wrapper(): """Test the *output_formatter.tabulate_wrapper()* function.""" data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = tabulate_wrapper(data, headers, table_format='psql') + output = tabulate_adapter.adapter(data, headers, table_format='psql') assert output == dedent('''\ +-----------+----------+ | letters | number | @@ -113,7 +110,7 @@ def test_csv_wrapper(): # Test comma-delimited output. data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = csv_wrapper(data, headers) + output = delimited_output_adapter.adapter(data, headers) assert output == dedent('''\ letters,number\r\n\ abc,1\r\n\ @@ -122,7 +119,8 @@ def test_csv_wrapper(): # Test tab-delimited output. data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = csv_wrapper(data, headers, table_format='tsv') + output = delimited_output_adapter.adapter( + data, headers, table_format='tsv') assert output == dedent('''\ letters\tnumber\r\n\ abc\t1\r\n\ @@ -133,7 +131,8 @@ def test_terminal_tables_wrapper(): """Test the *output_formatter.terminal_tables_wrapper()* function.""" data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = terminal_tables_wrapper(data, headers, table_format='ascii') + output = terminaltables_adapter.adapter( + data, headers, table_format='ascii') assert output == dedent('''\ +---------+--------+ | letters | number |