fix Py3 compatibility and make Flake8 happier

ckanext/qa/bin/common.py

@@ -1,5 +1,8 @@
+# encoding: utf-8
+
 from ckan.plugins import toolkit
 
+
 def load_config(config_filepath):
     toolkit.load_config(config_filepath)
 
@@ -48,5 +51,5 @@ def get_resources(state='active', publisher_ref=None, resource_id=None, dataset_
         resources = resources.filter(model.Resource.id == resource_id)
         criteria.append('Resource:%s' % resource_id)
     resources = resources.all()
-    print ('%i resources (%s)' % (len(resources), ' '.join(criteria)))
+    print('%i resources (%s)' % (len(resources), ' '.join(criteria)))
     return resources

ckanext/qa/bin/migrate_task_status.py

@@ -59,7 +59,7 @@ def migrate(options):
         # time, so some timezone nonesense going on. Can't do much.
         archival = Archival.get_for_resource(res.id)
         if not archival:
-            print (add_stat('QA but no Archival data', res, stats))
+            print(add_stat('QA but no Archival data', res, stats))
             continue
         archival_date = archival.updated
         # the state of the resource was as it was archived on the date of
@@ -112,10 +112,10 @@ def migrate(options):
                 model.Session.add(qa)
             add_stat('Added to QA table', res, stats)
 
-    print ('Summary\n', stats.report())
+    print('Summary\n', stats.report())
     if options.write:
         model.repo.commit_and_remove()
-        print ('Written')
+        print('Written')
 
 
 def add_stat(outcome, res, stats, extra_info=None):

ckanext/qa/bin/running_stats.py

@@ -14,7 +14,7 @@
         package_stats.increment('deleted')
     else:
         package_stats.increment('not deleted')
-print package_stats.report()
+print(package_stats.report())
 > deleted: 30
 > not deleted: 70
 
@@ -26,14 +26,15 @@
         package_stats.add('deleted', package.name)
     else:
         package_stats.add('not deleted' package.name)
-print package_stats.report()
+print(package_stats.report())
 > deleted: 30 pollution-uk, flood-regions, river-quality, ...
 > not deleted: 70 spending-bristol, ...
 
 '''
 
 import copy
 import datetime
+import six
 
 
 class StatsCount(dict):
@@ -68,9 +69,9 @@ def report(self, indent=1, order_by_title=False, show_time_taken=True):
             report_dict[category] = self.report_value(category)
 
         if order_by_title:
-            items = sorted(report_dict.items())
+            items = sorted(six.iteritems(report_dict))
         else:
-            items = sorted(report_dict.items(),
+            items = sorted(six.iteritems(report_dict),
                            key=lambda x: -x[1][1])
 
         for category, value_tuple in items:

ckanext/qa/cli.py

@@ -1,6 +1,9 @@
+# encoding: utf-8
+
 import click
 import ckanext.qa.commands as commands
 
+
 def get_commands():
 
     @click.group()
@@ -50,4 +53,4 @@ def sniff(args):
         """Opens the file and determines its type by the contents"""
         commands.sniff(args)
 
-    return [qa]
+    return [qa]

ckanext/qa/commands.py

@@ -92,50 +92,50 @@ def update(args, queue):
     log.info('Completed queueing')
 
 def view(package_ref=None):
-    
+
     q = model.Session.query(model.TaskStatus).filter_by(task_type='qa')
-    print ('QA records - %i TaskStatus rows' % q.count())
-    print ('      across %i Resources' % q.distinct('entity_id').count())
+    print('QA records - %i TaskStatus rows' % q.count())
+    print('      across %i Resources' % q.distinct('entity_id').count())
 
     if package_ref:
         pkg = model.Package.get(package_ref)
-        print ('Package %s %s' % (pkg.name, pkg.id))
+        print('Package %s %s' % (pkg.name, pkg.id))
         for res in pkg.resources:
-            print ('Resource %s' % res.id)
+            print('Resource %s' % res.id)
             for row in q.filter_by(entity_id=res.id):
-                print ('* %s = %r error=%r' % (row.key, row.value,
+                print('* %s = %r error=%r' % (row.key, row.value,
                                                 row.error))
 
 def migrate():
     q_status = model.Session.query(model.TaskStatus) \
         .filter_by(task_type='qa') \
         .filter_by(key='status')
-    print ('* %s with "status" will be deleted e.g. %s' % (q_status.count(),
+    print('* %s with "status" will be deleted e.g. %s' % (q_status.count(),
                                                             q_status.first()))
     q_failures = model.Session.query(model.TaskStatus) \
         .filter_by(task_type='qa') \
         .filter_by(key='openness_score_failure_count')
-    print ('* %s with openness_score_failure_count to be deleted e.g.\n%s'\
+    print('* %s with openness_score_failure_count to be deleted e.g.\n%s'\
         % (q_failures.count(), q_failures.first()))
     q_score = model.Session.query(model.TaskStatus) \
         .filter_by(task_type='qa') \
         .filter_by(key='openness_score')
-    print ('* %s with openness_score to migrate e.g.\n%s' % \
+    print('* %s with openness_score to migrate e.g.\n%s' % \
         (q_score.count(), q_score.first()))
     q_reason = model.Session.query(model.TaskStatus) \
         .filter_by(task_type='qa') \
         .filter_by(key='openness_score_reason')
-    print ('* %s with openness_score_reason to migrate e.g.\n%s' % \
+    print('* %s with openness_score_reason to migrate e.g.\n%s' % \
         (q_reason.count(), q_reason.first()))
     input('Press Enter to continue')
 
     q_status.delete()
     model.Session.commit()
-    print ('..."status" deleted')
+    print('..."status" deleted')
 
     q_failures.delete()
     model.Session.commit()
-    print ('..."openness_score_failure_count" deleted')
+    print('..."openness_score_failure_count" deleted')
 
     for task_status in q_score:
         reason_task_status = q_reason \
@@ -154,36 +154,36 @@ def migrate():
             'is_broken': None,
             })
         model.Session.commit()
-    print ('..."openness_score" and "openness_score_reason" migrated')
+    print('..."openness_score" and "openness_score_reason" migrated')
     count = q_reason.count()
     q_reason.delete()
     model.Session.commit()
-    print ('... %i remaining "openness_score_reason" deleted' % count)
+    print('... %i remaining "openness_score_reason" deleted' % count)
 
     model.Session.flush()
     model.Session.remove()
-    print ('Migration succeeded')
+    print('Migration succeeded')
 
 def clean():
-    print ('Before:')
+    print('Before:')
     view()
 
     q = model.Session.query(model.TaskStatus).filter_by(task_type='qa')
     q.delete()
     model.Session.commit()
 
-    print ('After:')
+    print('After:')
     view()
 
 def sniff(args):
     from ckanext.qa.sniff_format import sniff_file_format
     if len(args) < 1:
-        print ('Not enough arguments', args)
+        print('Not enough arguments', args)
         sys.exit(1)
     for filepath in args[0:]:
         format_ = sniff_file_format(filepath)
         if format_:
-            print ('Detected as: %s - %s' % (format_['format'],
+            print('Detected as: %s - %s' % (format_['format'],
                                             filepath))
         else:
-            print ('ERROR: Could not recognise format of: %s' % filepath)
+            print('ERROR: Could not recognise format of: %s' % filepath)

ckanext/qa/controllers.py

@@ -6,8 +6,13 @@
 import json
 import mimetypes
 import posixpath
-import urllib
-import urlparse
+import six
+import six.moves.urllib.parse as urlparse
+# this move isn't covered in the 'six' module
+if six.PY2:
+    from urllib import splittype
+else:
+    from urllib.parse import splittype
 
 from ckan.lib.base import request, BaseController
 from ckan.lib.helpers import parse_rfc_2822_date
@@ -78,7 +83,7 @@ def _check_link(self, url):
         """
 
         # If a user enters "www.example.com" then we assume they meant "http://www.example.com"
-        scheme, path = urllib.splittype(url)
+        scheme, path = splittype(url)
         if not scheme:
             url = 'http://' + path
 
@@ -103,7 +108,7 @@ def _check_link(self, url):
             result['size'] = headers.get('content-length', '')
             result['last_modified'] = self._parse_and_format_date(headers.get('last-modified', ''))
         except LinkCheckerError as e:
-            result['url_errors'].append(str(e))
+            result['url_errors'].append(six.text_type(e))
         return result
 
     def _extract_file_format(self, url, headers):

ckanext/qa/lib.py

@@ -1,7 +1,10 @@
-import os
+# encoding: utf-8
+
 import json
-import re
 import logging
+import os
+import re
+import six
 
 from ckan.plugins.toolkit import config
 
@@ -25,7 +28,7 @@ def compat_enqueue(name, fn, queue, args=None):
         # Fallback to Celery
         import uuid
         from ckan.lib.celery_app import celery
-        celery.send_task(name, args=args + [queue], task_id=str(uuid.uuid4()))
+        celery.send_task(name, args=args + [queue], task_id=six.text_type(uuid.uuid4()))
 
 
 def resource_format_scores():
@@ -87,7 +90,7 @@ def munge_format_to_be_canonical(format_name):
 
 def create_qa_update_package_task(package, queue):
     compat_enqueue('qa.update_package', tasks.update_package,
-                   queue,  args=[package.id])
+                   queue, args=[package.id])
     log.debug('QA of package put into queue %s: %s',
               queue, package.name)
 

ckanext/qa/logic/action.py

@@ -30,7 +30,7 @@ def qa_resource_show(context, data_dict):
         'name': pkg.name,
         'title': pkg.title,
         'id': res.id
-        }
+    }
     return_dict['archival'] = archival.as_dict()
     return_dict.update(qa.as_dict())
     return return_dict

ckanext/qa/model.py

@@ -1,5 +1,8 @@
+# encoding: utf-8
+
 import uuid
 import datetime
+import six
 
 from sqlalchemy import Column
 from sqlalchemy import types
@@ -15,7 +18,7 @@
 
 
 def make_uuid():
-    return str(uuid.uuid4())
+    return six.text_type(uuid.uuid4())
 
 
 class QA(Base):
@@ -40,7 +43,7 @@ class QA(Base):
 
     def __repr__(self):
         summary = 'score=%s format=%s' % (self.openness_score, self.format)
-        details = str(self.openness_score_reason).encode('unicode_escape')
+        details = six.text_type(self.openness_score_reason).encode('unicode_escape')
         package = model.Package.get(self.package_id)
         package_name = package.name if package else '?%s?' % self.package_id
         return '<QA %s /dataset/%s/resource/%s %s>' % \

ckanext/qa/plugin.py

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 import logging
 
 import ckan.model as model
@@ -6,9 +8,7 @@
 from ckanext.archiver.interfaces import IPipe
 from ckanext.qa.logic import action, auth
 from ckanext.qa.model import QA, aggregate_qa_for_a_dataset
-import ckanext.qa.helpers as helpers
-import ckanext.qa.lib as lib
-import ckanext.qa.cli as cli
+from ckanext.qa import cli, helpers, lib
 from ckanext.report.interfaces import IReport
 
 
@@ -69,15 +69,15 @@ def get_actions(self):
         return {
             'qa_resource_show': action.qa_resource_show,
             'qa_package_openness_show': action.qa_package_openness_show,
-            }
+        }
 
     # IAuthFunctions
 
     def get_auth_functions(self):
         return {
             'qa_resource_show': auth.qa_resource_show,
             'qa_package_openness_show': auth.qa_package_openness_show,
-            }
+        }
 
     # ITemplateHelpers
 
@@ -87,7 +87,7 @@ def get_helpers(self):
             helpers.qa_openness_stars_resource_html,
             'qa_openness_stars_dataset_html':
             helpers.qa_openness_stars_dataset_html,
-            }
+        }
 
     # IPackageController
 
@@ -116,4 +116,4 @@ def after_show(self, context, pkg_dict):
                 res['qa'] = qa_dict
 
     def get_commands(self):
-        return cli.get_commands()
+        return cli.get_commands()