From a695509be8be812b00d4b18769c92608abb42aaf Mon Sep 17 00:00:00 2001 From: Taras Kopets Date: Wed, 27 Dec 2017 18:34:56 +0200 Subject: [PATCH] Review default DB settings, use set encoding Since before and after SQL is run for all commands - review all of them. Improve usability and functionality of Oracle and MSSQL. Also, fix to take into consideration the encoding set in DB connection. --- SQLTools.py | 10 +- SQLTools.sublime-settings | 186 ++++++++++++++++++++------- SQLToolsAPI/Command.py | 56 +++++--- SQLToolsAPI/Connection.py | 50 +++++-- SQLToolsConnections.sublime-settings | 9 +- 5 files changed, 225 insertions(+), 86 deletions(-) diff --git a/SQLTools.py b/SQLTools.py index 152c35e..6080c0c 100644 --- a/SQLTools.py +++ b/SQLTools.py @@ -497,7 +497,7 @@ def run(): return Window().status_message(MESSAGE_RUNNING_CMD) - ST.conn.execute(getSelection(), createOutput(), stream=settings.get('use_streams', False)) + ST.conn.execute(getSelection(), createOutput()) class StExecuteAll(WindowCommand): @@ -509,7 +509,7 @@ def run(): Window().status_message(MESSAGE_RUNNING_CMD) allText = View().substr(sublime.Region(0, View().size())) - ST.conn.execute(allText, createOutput(), stream=settings.get('use_streams', False)) + ST.conn.execute(allText, createOutput()) class StFormat(TextCommand): @@ -543,8 +543,7 @@ def run(): def cb(index): if index < 0: return None - return ST.conn.execute(history.get(index), createOutput(), - stream=settings.get('use_streams', False)) + return ST.conn.execute(history.get(index), createOutput()) Window().show_quick_panel(history.all(), cb) @@ -583,8 +582,7 @@ def cb(index): alias, query = options[index] if mode == "run": - ST.conn.execute(query, createOutput(), - stream=settings.get('use_streams', False)) + ST.conn.execute(query, createOutput()) elif mode == "insert": insertContent(query) else: diff --git a/SQLTools.sublime-settings b/SQLTools.sublime-settings index ea0c0ee..5d5b830 100644 --- a/SQLTools.sublime-settings +++ b/SQLTools.sublime-settings @@ -74,7 +74,7 @@ "PGPASSWORD": "{password}" }, "queries": { - "exec": { + "execute": { "options": [] }, "desc": { @@ -109,52 +109,123 @@ }, "oracle": { "options": ["-S"], - "before": [ - "SET AUTO OFF", - "SET COLSEP '|'", - "SET FEED ON", - "SET FEEDBACK ON", - "SET HEADING ON", - "SET LINESIZE 32767", - "SET LONG 100", - "SET NULL @", - "SET PAGESIZE 0 EMBEDDED ON", - "SET SERVEROUTPUT ON", - "SET SQLBLANKLINES ON", - "SET SQLPROMPT ''", - "SET TAB OFF", - "SET TI ON", - "SET TIMI OFF", - "SET TRIMSPOOL OFF", - "SET UND '-'", - "SET VERIFY OFF ", - "SET WRAP OFF" - ], + "before": [], "after": [], + "env_optional": { + "NLS_LANG": "{nls_lang}" + }, "args": "{username}/{password}@\"(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST={host})(PORT={port})))(CONNECT_DATA=(SERVICE_NAME={service})))\"", "queries": { - "exec": { - "options": [] + "execute": { + "options": [], + "before": [ + // "SET TIMING ON", + "SET LINESIZE 32767", + "SET PAGESIZE 0", + "SET EMBEDDED ON", + "SET WRAP OFF", + "SET TAB OFF", + "SET TRIMOUT ON", + "SET TRIMSPOOL ON", + "SET NULL '@'", + "SET COLSEP '|'", + "SET FEEDBACK ON", + "SET SERVEROUT ON", + "SET SQLBLANKLINES ON" + ] }, "desc" : { - "query": "select concat(concat(concat(concat('|', owner), '.'), table_name), '|') as tbls from all_tables UNION ALL select concat(concat(concat(concat('|', owner), '.'), view_name), '|') as tbls from all_views;", - "options": [] + "query": "select owner || '.' || name as obj from (select owner, table_name as name from all_tables union all select owner, view_name as name from all_views) o where owner not in ('ANONYMOUS','APPQOSSYS','CTXSYS','DBSNMP','EXFSYS', 'LBACSYS', 'MDSYS','MGMT_VIEW','OLAPSYS','OWBSYS','ORDPLUGINS', 'ORDSYS','OUTLN', 'SI_INFORMTN_SCHEMA','SYS','SYSMAN','SYSTEM', 'TSMSYS','WK_TEST','WKSYS', 'WKPROXY','WMSYS','XDB','APEX_040000', 'APEX_PUBLIC_USER','DIP', 'FLOWS_30000','FLOWS_FILES','MDDATA', 'ORACLE_OCM','SPATIAL_CSW_ADMIN_USR', 'SPATIAL_WFS_ADMIN_USR', 'XS$NULL','PUBLIC');", + "options": [], + "before": [ + "SET LINESIZE 32767", + "SET PAGESIZE 0", + "SET EMBEDDED ON", + "SET WRAP OFF", + "SET HEADING OFF", + "SET FEEDBACK OFF", + "SET TRIMOUT ON" + ] }, "columns": { - "query": "SELECT concat(concat(concat(concat('|', c.table_name), '.'), c.column_name), '|') AS cols FROM all_tab_columns c INNER JOIN all_tables t ON c.owner = t.owner AND c.table_name = t.table_name UNION ALL SELECT concat(concat(concat(concat('|', c.table_name), '.'), c.column_name), '|') AS cols FROM all_tab_columns c INNER JOIN all_views t ON c.owner = t.owner AND c.table_name = t.view_name;", - "options": [] + "query": "select table_name || '.' || column_name as obj from (select c.table_name, c.column_name, t.owner from all_tab_columns c inner join all_tables t on c.owner = t.owner and c.table_name = t.table_name union all select c.table_name, c.column_name, t.owner from all_tab_columns c inner join all_views t on c.owner = t.owner and c.table_name = t.view_name) o where owner not in ('ANONYMOUS','APPQOSSYS','CTXSYS','DBSNMP','EXFSYS', 'LBACSYS', 'MDSYS','MGMT_VIEW','OLAPSYS','OWBSYS','ORDPLUGINS', 'ORDSYS','OUTLN', 'SI_INFORMTN_SCHEMA','SYS','SYSMAN','SYSTEM', 'TSMSYS','WK_TEST','WKSYS', 'WKPROXY','WMSYS','XDB','APEX_040000', 'APEX_PUBLIC_USER','DIP', 'FLOWS_30000','FLOWS_FILES','MDDATA', 'ORACLE_OCM','SPATIAL_CSW_ADMIN_USR', 'SPATIAL_WFS_ADMIN_USR', 'XS$NULL','PUBLIC');", + "options": [], + "before": [ + "SET LINESIZE 32767", + "SET PAGESIZE 0", + "SET EMBEDDED ON", + "SET WRAP OFF", + "SET HEADING OFF", + "SET FEEDBACK OFF", + "SET TRIMOUT ON" + ] + }, + "functions": { + "query": "select case when object_type = 'PACKAGE' then object_name||'.'||procedure_name else owner || '.' || object_name end || '()' as obj from all_procedures where object_type in ('FUNCTION','PROCEDURE','PACKAGE') and owner = sys_context('USERENV', 'CURRENT_SCHEMA');", + "options": [], + "before": [ + "SET LINESIZE 32767", + "SET PAGESIZE 0", + "SET EMBEDDED ON", + "SET WRAP OFF", + "SET HEADING OFF", + "SET FEEDBACK OFF", + "SET TRIMOUT ON" + ] }, "desc table": { "query": "desc {0};", - "options": [] + "options": [], + "before": [ + "SET LINESIZE 80", // altered for readability + "SET PAGESIZE 0", + "SET EMBEDDED ON", + "SET WRAP OFF", + "SET TAB OFF", + "SET TRIMOUT ON", + "SET TRIMSPOOL ON", + "SET NULL '@'", + "SET COLSEP '|'", + "SET FEEDBACK ON", + "SET SERVEROUT ON", + "SET SQLBLANKLINES ON" + ] }, "show records": { "query": "select * from {0} WHERE ROWNUM <= {1};", - "options": [] + "options": [], + "before": [ + "SET LINESIZE 32767", + "SET PAGESIZE 0", + "SET EMBEDDED ON", + "SET WRAP OFF", + "SET TAB OFF", + "SET TRIMOUT ON", + "SET TRIMSPOOL ON", + "SET NULL '@'", + "SET COLSEP '|'", + "SET FEEDBACK ON", + "SET SERVEROUT ON", + "SET SQLBLANKLINES ON" + ] }, "explain plan": { "query": "explain plan for {0};\nselect plan_table_output from table(dbms_xplan.display());", - "options": [] + "options": [], + "before": [ + "SET LINESIZE 32767", + "SET PAGESIZE 0", + "SET EMBEDDED ON", + "SET WRAP OFF", + "SET TAB OFF", + "SET TRIMOUT ON", + "SET TRIMSPOOL ON", + "SET NULL '@'", + "SET COLSEP '|'", + "SET FEEDBACK ON", + "SET SERVEROUT ON", + "SET SQLBLANKLINES ON" + ] } } }, @@ -165,7 +236,7 @@ "args": "-h{host} -P{port} -u\"{username}\" -D\"{database}\"", "args_optional": ["--login-path=\"{login-path}\"", "--defaults-extra-file=\"{defaults-extra-file}\"", "-p\"{password}\""], "queries": { - "exec": { + "execute": { "options": ["--table", "-f"] }, "desc" : { @@ -201,28 +272,45 @@ "mssql": { "options": [], "before": [], - "after": ["GO", "QUIT"], + "after": ["go", "quit"], "args": "-d \"{database}\"", "args_optional": ["-S \"{host},{port}\"", "-S \"{host}\\{instance}\"", "-U \"{username}\"", "-P \"{password}\""], "queries": { - "exec": { + "execute": { + "options": ["-k"] + }, + "show records": { + "query": "select top {1} * from {0};", "options": [] }, + "desc table": { + "query": "exec sp_help N'{0}';", + "options": ["-y30", "-Y30"] + }, + "desc function": { + "query": "exec sp_helptext N'{0}';", + "options": ["-h-1"] + }, "desc": { - "query": "set nocount on; select concat(table_schema, '.', table_name) from information_schema.tables order by table_name;", - "options": ["-h", "-1"] + "query": "set nocount on; select concat(table_schema, '.', table_name) as obj from information_schema.tables order by table_schema, table_name;", + "options": ["-h-1"] }, "columns": { - "query": "set nocount on; select concat(table_name, '.', column_name) from information_schema.columns order by table_name, ordinal_position;", - "options": ["-h", "-1"] - }, - "desc table": { - "query": "exec sp_help \"{0}\";", - "options": [] + "query": "set nocount on; select distinct concat(table_name, '.', column_name) as obj from information_schema.columns;", + "options": ["-h-1"] }, - "show records": { - "query": "select top {1} * from {0};", - "options": [] + "functions": { + "query": "set nocount on; select concat(routine_schema, '.', routine_name) as obj from information_schema.routines order by routine_schema, routine_name;", + "options": ["-h-1"] + }, + "explain plan": { + "query": "{0};", + "options": ["-k"], + "before": [ + "SET STATISTICS PROFILE ON", + "SET STATISTICS IO ON", + "SET STATISTICS TIME ON" + ] } } }, @@ -232,7 +320,7 @@ "after": [], "args": "-h {host} -p {port} -U \"{username}\" -w \"{password}\" -d \"{database}\"", "queries": { - "exec": { + "execute": { "options": [] }, "desc" : { @@ -263,7 +351,7 @@ "after": [], "args": "-S {host}:{port} -U\"{username}\" -P\"{password}\" -D{database}", "queries": { - "exec": { + "execute": { "options": [], "before": ["\\set semicolon_cmd=\"\\go -mpretty -l\""] }, @@ -295,7 +383,7 @@ "after": [], "args": "\"{database}\"", "queries": { - "exec": { + "execute": { "options": ["-column", "-header"] }, "desc" : { @@ -322,7 +410,7 @@ "after": [], "args": "-u \"{username}\" -p \"{password}\" \"{host}/{port}:{database}\"", "queries": { - "exec": { + "execute": { "options": [] }, "desc" : { diff --git a/SQLToolsAPI/Command.py b/SQLToolsAPI/Command.py index 8842f05..abed226 100644 --- a/SQLToolsAPI/Command.py +++ b/SQLToolsAPI/Command.py @@ -15,7 +15,6 @@ def __init__(self, args, env, callback, query=None, encoding='utf-8', if options is None: options = {} - self.stream = stream self.args = args self.env = env self.callback = callback @@ -24,13 +23,14 @@ def __init__(self, args, env, callback, query=None, encoding='utf-8', self.options = options self.timeout = timeout self.silenceErrors = silenceErrors + self.stream = stream self.process = None if 'show_query' not in self.options: self.options['show_query'] = False elif self.options['show_query'] not in ['top', 'bottom']: - self.options['show_query'] = 'top' if (isinstance(self.options['show_query'], bool) - and self.options['show_query']) else False + self.options['show_query'] = 'top' if (isinstance(self.options['show_query'], bool) and + self.options['show_query']) else False def run(self): if not self.query: @@ -65,13 +65,12 @@ def run(self): startupinfo=si) if self.stream: - self.process.stdin.write(self.query.encode()) + self.process.stdin.write(self.query.encode(self.encoding)) self.process.stdin.close() hasWritten = False for line in self.process.stdout: - self.callback(line.decode(self.encoding, - 'replace').replace('\r', '')) + self.callback(line.decode(self.encoding, 'replace').replace('\r', '')) hasWritten = True queryTimerEnd = time.time() @@ -90,7 +89,7 @@ def run(self): # regular mode is handled with more reliable Popen.communicate # which also terminates the process afterwards - results, errors = self.process.communicate(input=self.query.encode()) + results, errors = self.process.communicate(input=self.query.encode(self.encoding)) queryTimerEnd = time.time() @@ -104,7 +103,7 @@ def run(self): resultString += errors.decode(self.encoding, 'replace').replace('\r', '') - if self.process == None and resultString != '': + if self.process is None and resultString != '': resultString += '\n' if self.options['show_query']: @@ -128,11 +127,19 @@ def _formatShowQuery(query, queryTimeStart, queryTimeEnd): return resultString @staticmethod - def createAndRun(args, env, query, callback, options=None, timeout=15, silenceErrors=False, stream=False): + def createAndRun(args, env, callback, query=None, encoding='utf-8', + options=None, timeout=15, silenceErrors=False, stream=False): if options is None: options = {} - command = Command(args, env, callback, query, options=options, - timeout=timeout, silenceErrors=silenceErrors) + command = Command(args=args, + env=env, + callback=callback, + query=query, + encoding=encoding, + options=options, + timeout=timeout, + silenceErrors=silenceErrors, + stream=stream) command.run() @@ -142,9 +149,15 @@ def __init__(self, args, env, callback, query=None, encoding='utf-8', if options is None: options = {} - Command.__init__(self, args, env, callback, query=query, - encoding=encoding, options=options, - timeout=timeout, silenceErrors=silenceErrors, + Command.__init__(self, + args=args, + env=env, + callback=callback, + query=query, + encoding=encoding, + options=options, + timeout=timeout, + silenceErrors=silenceErrors, stream=stream) Thread.__init__(self) @@ -168,14 +181,21 @@ def stop(self): pass @staticmethod - def createAndRun(args, env, query, callback, options=None, - timeout=Command.timeout, silenceErrors=False, stream=False): + def createAndRun(args, env, callback, query=None, encoding='utf-8', + options=None, timeout=Command.timeout, silenceErrors=False, stream=False): # Don't allow empty dicts or lists as defaults in method signature, # cfr http://nedbatchelder.com/blog/200806/pylint.html if options is None: options = {} - command = ThreadCommand(args, env, callback, query, options=options, - timeout=timeout, silenceErrors=silenceErrors, stream=stream) + command = ThreadCommand(args=args, + env=env, + callback=callback, + query=query, + encoding=encoding, + options=options, + timeout=timeout, + silenceErrors=silenceErrors, + stream=stream) command.start() killTimeout = Timer(command.timeout, command.stop) killTimeout.start() diff --git a/SQLToolsAPI/Connection.py b/SQLToolsAPI/Connection.py index eb32e3e..0702488 100644 --- a/SQLToolsAPI/Connection.py +++ b/SQLToolsAPI/Connection.py @@ -51,12 +51,13 @@ def __init__(self, name, options, settings=None, commandClass='ThreadCommand'): self.database = options.get('database', None) self.username = options.get('username', None) self.password = options.get('password', None) - self.encoding = options.get('encoding', None) + self.encoding = options.get('encoding', 'utf-8') self.service = options.get('service', None) self.safe_limit = settings.get('safe_limit', None) self.show_query = settings.get('show_query', False) self.rowsLimit = settings.get('show_records', {}).get('limit', 50) + self.useStreams = settings.get('use_streams', False) self.cli = settings.get('cli')[options['type']] cli_path = shutil.which(self.cli) @@ -76,15 +77,20 @@ def runInternalNamedQueryCommand(self, queryName, callback): if not query: return + queryToRun = self.buildNamedQuery(queryName, query) args = self.buildArgs(queryName) env = self.buildEnv() def cb(result): callback(U.getResultAsList(result)) - self.Command.createAndRun(args, env, - query, cb, - silenceErrors=True) + self.Command.createAndRun(args=args, + env=env, + callback=cb, + query=queryToRun, + encoding=self.encoding, + silenceErrors=True, + stream=False) def getTables(self, callback): self.runInternalNamedQueryCommand('desc', callback) @@ -112,7 +118,14 @@ def runFormattedNamedQueryCommand(self, queryName, formatValues, callback): queryToRun = self.buildNamedQuery(queryName, query) args = self.buildArgs(queryName) env = self.buildEnv() - self.Command.createAndRun(args, env, queryToRun, callback, timeout=self.timeout) + self.Command.createAndRun(args=args, + env=env, + callback=callback, + query=queryToRun, + encoding=self.encoding, + timeout=self.timeout, + silenceErrors=False, + stream=False) def getTableRecords(self, tableName, callback): # in case we expect multiple values pack them into tuple @@ -139,11 +152,22 @@ def explainPlan(self, queries, callback): queryToRun = self.buildNamedQuery(queryName, strippedQueries) args = self.buildArgs(queryName) env = self.buildEnv() - self.Command.createAndRun(args, env, queryToRun, callback, timeout=self.timeout) + self.Command.createAndRun(args=args, + env=env, + callback=callback, + query=queryToRun, + encoding=self.encoding, + timeout=self.timeout, + silenceErrors=False, + stream=self.useStreams) - def execute(self, queries, callback, stream=False): + def execute(self, queries, callback, stream=None): queryName = 'execute' + # if not explicitly overriden, use the value from settings + if stream is None: + stream = self.useStreams + if isinstance(queries, str): queries = [queries] @@ -175,9 +199,14 @@ def execute(self, queries, callback, stream=False): Log("Query: " + queryToRun) - self.Command.createAndRun(args, env, queryToRun, callback, + self.Command.createAndRun(args=args, + env=env, + callback=callback, + query=queryToRun, + encoding=self.encoding, options={'show_query': self.show_query}, timeout=self.timeout, + silenceErrors=False, stream=stream) def getNamedQuery(self, queryName): @@ -211,14 +240,15 @@ def buildNamedQuery(self, queryName, queries): builtQueries.extend(beforeQuery) if queries is not None: builtQueries.extend(queries) - if afterCli is not None: - builtQueries.extend(afterCli) if afterQuery is not None: builtQueries.extend(afterQuery) + if afterCli is not None: + builtQueries.extend(afterCli) # remove empty list items builtQueries = list(filter(None, builtQueries)) + print('\n'.join(builtQueries)) return '\n'.join(builtQueries) def buildArgs(self, queryName=None): diff --git a/SQLToolsConnections.sublime-settings b/SQLToolsConnections.sublime-settings index 9594efa..f6c447e 100644 --- a/SQLToolsConnections.sublime-settings +++ b/SQLToolsConnections.sublime-settings @@ -28,18 +28,20 @@ "port" : 5432, "database": "dbname", "username": "anotheruser", - // for PostgreSQL "password" is optional (setup "pgpass.conf" file instead) + // password is optional (setup "pgpass.conf" file instead) "password": "password", "encoding": "utf-8" }, "Connection Oracle": { "type" : "oracle", "host" : "127.0.0.1", - "port" : 1522, + "port" : 1521, "database": "dbname", "username": "anotheruser", "password": "password", "service" : "servicename", + // nls_lang is optional + "nls_lang": "american_america.al32utf8", "encoding": "utf-8" }, "Connection MSSQL": { @@ -57,7 +59,8 @@ }, "Connection SQLite": { "type" : "sqlite", - "database": "d:/sqlite/sample_db/chinook.db", // note the forward slashes in path + // note the forward slashes in path + "database": "d:/sqlite/sample_db/chinook.db", "encoding": "utf-8" } */