Upgrade from python 3.7 to 3.11

Dockerfile

@@ -1,5 +1,5 @@
-# Use official python base image, small and debian edition
-FROM amd64/python:3.7.3-slim
+# Use official python base image, small edition
+FROM docker.io/amd64/python:3.11-slim
 
 ARG purpose=dev
 

docker-compose.yml

@@ -25,21 +25,21 @@ services:
       - "redis"
 
   redis:
-    image: redis:alpine
+    image: docker.io/library/redis:alpine
 
   # wikireplica simulator
   mywiki:
-    image: mariadb:10.4-focal
+    image: docker.io/library/mariadb:10.11
     volumes:
       - ./docker-replica/replica.sql:/docker-entrypoint-initdb.d/replica.sql
     environment:
       MYSQL_USER: repl
       MYSQL_PASSWORD: repl
-      MYSQL_DATABASE: repl
+      MYSQL_DATABASE: mywiki_p
       MYSQL_RANDOM_ROOT_PASSWORD: 1
 
   db:
-    image: mariadb:10.1.48-bionic
+    image: docker.io/library/mariadb:10.11
     volumes:
       - ./schema.sql:/docker-entrypoint-initdb.d/schema.sql
     environment:

docker-replica/replica.sql

@@ -10,13 +10,8 @@
 /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
 /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
 
-CREATE DATABASE /*!32312 IF NOT EXISTS*/ `mywiki_p` /*!40100 DEFAULT CHARACTER SET utf8mb4 */;
-
 USE `mywiki_p`;
 
-DROP DATABASE IF EXISTS repl;
-GRANT SELECT ON mywiki_p.* TO 'repl'@'%';
-
 DROP TABLE IF EXISTS `actor`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;

quarry/web/api.py

@@ -60,7 +60,7 @@ def star_query() -> Union[Tuple[str, int], str]:
 
 
 @api_blueprint.route("/api/query/meta", methods=["POST"])
-def api_set_meta() -> Tuple[Union[str, Response], int]:
+def api_set_meta() -> tuple[str, int] | Response:
     if get_user() is None:
         return "Authentication required", 401
 
@@ -81,14 +81,13 @@ def api_set_meta() -> Tuple[Union[str, Response], int]:
         query.description = request.form["description"]
     g.conn.session.add(query)
     g.conn.session.commit()
-    return (
-        Response(json.dumps({"id": query.id}), mimetype="application/json"),
-        200,
+    return Response(
+        json.dumps({"id": query.id}), mimetype="application/json", status=200
     )
 
 
 @api_blueprint.route("/api/query/run", methods=["POST"])
-def api_run_query() -> Tuple[Union[str, Response], int]:
+def api_run_query() -> tuple[str, int] | Response:
     if get_user() is None:
         return "Authentication required", 401
     text = request.form["text"]
@@ -141,17 +140,15 @@ def api_run_query() -> Tuple[Union[str, Response], int]:
     g.conn.session.add(query_rev)
     g.conn.session.add(query_run)
     g.conn.session.commit()
-    return (
-        Response(
-            json.dumps({"qrun_id": query_run.id}),
-            mimetype="application/json",
-        ),
-        200,
+    return Response(
+        json.dumps({"qrun_id": query_run.id}),
+        mimetype="application/json",
+        status=200,
     )
 
 
 @api_blueprint.route("/api/query/stop", methods=["POST"])
-def api_stop_query() -> Tuple[Union[str, Response], int]:
+def api_stop_query() -> tuple[str, int] | Response:
     if get_user() is None:
         return "Authentication required", 401
 
@@ -164,17 +161,17 @@ def api_stop_query() -> Tuple[Union[str, Response], int]:
     query_run = (
         g.conn.session.query(QueryRun).filter(QueryRun.id == qrun_id).one()
     )
-    result_dictionary = ast.literal_eval(query_run.extra_info)
-    if "connection_id" in result_dictionary:
-        g.replica.connection = db_of_process
-        cur = g.replica.connection.cursor()
-        try:
-            cur.execute("KILL %s;", (result_dictionary["connection_id"]))
-            output = "job stopped"
-        except OperationalError:
-            output = "job not running"
-    else:
-        output = "job not running"
+    output = "job not running"
+    if query_run.extra_info:
+        result_dictionary = ast.literal_eval(query_run.extra_info)
+        if "connection_id" in result_dictionary:
+            g.replica.connection = db_of_process
+            cur = g.replica.connection.cursor()
+            try:
+                cur.execute("KILL %s;", (result_dictionary["connection_id"]))
+                output = "job stopped"
+            except OperationalError:
+                output = "job not running"
 
     # Stopping the job usually gets a stopped status. However some jobs stopped
     # before the stop button was pressed, and didn't update the DB to reflect
@@ -185,14 +182,13 @@ def api_stop_query() -> Tuple[Union[str, Response], int]:
     query_run.status = QueryRun.STATUS_STOPPED
     g.conn.session.add(query_run)
     g.conn.session.commit()
-    return (
-        Response(json.dumps({"stopped": output}), mimetype="application/json"),
-        200,
+    return Response(
+        json.dumps({"stopped": output}), mimetype="application/json", status=200
     )
 
 
 @api_blueprint.route("/api/preferences/get/<key>")
-def pref_get(key) -> Response:
+def pref_get(key) -> tuple[str, int] | Response:
     if get_user() is None:
         return "Authentication required", 401
 
@@ -209,16 +205,15 @@ def pref_get(key) -> Response:
 
 
 @api_blueprint.route("/api/preferences/set/<key>/<value>")
-def pref_set(key, value) -> Union[Tuple[str, int], Tuple[Response, int]]:
+def pref_set(key, value) -> tuple[str, int] | Response:
     if get_user() is None:
         return "Authentication required", 401
 
     get_preferences()[key] = None if value == "null" else value
-    return (
-        Response(
-            json.dumps({"key": key, "success": ""}), mimetype="application/json"
-        ),
-        201,
+    return Response(
+        json.dumps({"key": key, "success": ""}),
+        mimetype="application/json",
+        status=201,
     )
 
 

quarry/web/metrics.py

@@ -29,7 +29,7 @@ def collect(self):
             labels=["status"],
         )
 
-        for (status_id, query_count) in queries_per_status:
+        for status_id, query_count in queries_per_status:
             metric_family.add_metric(
                 [QueryRun.STATUS_MESSAGES[status_id]], query_count
             )

quarry/web/models/queryrun.py

@@ -1,3 +1,4 @@
+import datetime
 import json
 from sqlalchemy import (
     Column,
@@ -37,7 +38,7 @@ class QueryRun(Base):
     id = Column(Integer, primary_key=True)
     query_rev_id = Column(Integer, ForeignKey("query_revision.id"))
     status = Column(Integer)
-    timestamp = Column(DateTime)
+    timestamp = Column(DateTime, default=datetime.datetime.utcnow)
     task_id = Column(String)
     extra_info = Column(UnicodeText)
 

quarry/web/output.py

@@ -4,7 +4,7 @@
 import types
 
 from flask import Response, escape
-from werkzeug.contrib.iterio import IterI
+from .utils.iterio import IterI
 import xlsxwriter
 
 

quarry/web/redissession.py

@@ -36,7 +36,7 @@ def get_redis_expiration_time(self, app, session):
         return timedelta(days=1)
 
     def open_session(self, app, request):
-        sid = request.cookies.get(app.session_cookie_name)
+        sid = request.cookies.get(app.config["SESSION_COOKIE_NAME"])
         if not sid:
             sid = self.generate_sid()
             return self.session_class(sid=sid, new=True)
@@ -51,7 +51,9 @@ def save_session(self, app, session, response):
         if not session:
             self.redis.delete(self.prefix + session.sid)
             if session.modified:
-                response.delete_cookie(app.session_cookie_name, domain=domain)
+                response.delete_cookie(
+                    app.config["SESSION_COOKIE_NAME"], domain=domain
+                )
             return
         redis_exp = self.get_redis_expiration_time(app, session)
         cookie_exp = self.get_expiration_time(app, session)
@@ -62,7 +64,7 @@ def save_session(self, app, session, response):
             time=int(redis_exp.total_seconds()),
         )
         response.set_cookie(
-            app.session_cookie_name,
+            app.config["SESSION_COOKIE_NAME"],
             session.sid,
             expires=cookie_exp,
             httponly=True,

quarry/web/results.py

@@ -6,9 +6,7 @@
 from decimal import Decimal
 from typing import List
 
-INITIAL_SQL = """
-CREATE TABLE resultsets (id, headers, rowcount)
-"""
+INITIAL_SQL = "CREATE TABLE resultsets (id, headers, rowcount)"
 
 
 def get_unique_columns(raw_columns: List[str]) -> List[str]:
@@ -57,12 +55,15 @@ def start_resultset(self, columns, rowcount):
         unique_columns = get_unique_columns(columns)
         sanitized_columns = [self._quote_identifier(c) for c in unique_columns]
 
+        # Create table that will store the resultset
         table_name = self._get_current_resultset_table()
         sql = "CREATE TABLE %s (__id__ INTEGER PRIMARY KEY, %s)" % (
             table_name,
             ", ".join(sanitized_columns),
         )
         self.db.execute(sql)
+
+        # Add the new one to the resultset index table
         self.db.execute(
             "INSERT INTO resultsets (id, headers, rowcount) VALUES (?, ?, ?)",
             (self.resultset_id, json.dumps(unique_columns), rowcount),