Saving the complete beta appstream in the same collection

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
 
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.9"
+          python-version: "3.10"
 
       - uses: actions/cache@v2
         with:

Dockerfile

@@ -1,24 +1,26 @@
-FROM python:3.9 as builder
+FROM python:3.10 as builder
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-        build-essential libcairo2-dev libgirepository1.0-dev \
-        gir1.2-ostree-1.0 flatpak
+    build-essential libcairo2-dev libgirepository1.0-dev \
+    gir1.2-ostree-1.0 flatpak
 
 ADD requirements.txt /requirements.txt
 RUN python -m venv /venv && \
     /venv/bin/pip install -r requirements.txt \
     && rm -f /requirements.txt
 
-FROM python:3.9-slim
+FROM python:3.10-slim
 
 EXPOSE 8000
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-        libcairo2 gir1.2-ostree-1.0 flatpak && \
+    libcairo2 gir1.2-ostree-1.0 flatpak && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
-    flatpak --user remote-add flathub https://flathub.org/repo/flathub.flatpakrepo
+    flatpak --user remote-add flathub https://flathub.org/repo/flathub.flatpakrepo && \
+    flatpak --user remote-add flathub-beta https://flathub.org/beta-repo/flathub-beta.flatpakrepo
+
 
 COPY ./app /app
 COPY ./data /data

app/apps.py

@@ -6,9 +6,13 @@
 
 
 def load_appstream():
-    apps = utils.appstream2dict("repo")
+    apps_stable = utils.appstream2dict("repo")
+    apps_beta = utils.appstream2dict("beta-repo")
 
-    current_apps = {app[5:] for app in db.redis_conn.smembers("apps:index")}
+    apps = merge_apps(apps_beta, apps_stable)
+
+    current_apps = db.get_apps("stable")
+    current_apps_beta = db.get_apps("beta")
     current_categories = db.redis_conn.smembers("categories:index")
 
     db.initialize()
@@ -19,30 +23,32 @@ def load_appstream():
             redis_key = f"apps:{appid}"
 
             clean_html_re = re.compile("<.*?>")
-            search_description = re.sub(clean_html_re, "", apps[appid]["description"])
-
-            if search_keywords := apps[appid].get("keywords"):
-                search_keywords = " ".join(search_keywords)
-            else:
-                search_keywords = ""
-
-            fts = {
-                "id": appid,
-                "name": apps[appid]["name"],
-                "summary": apps[appid]["summary"],
-                "description": search_description,
-                "keywords": search_keywords,
-            }
+            if "stable" in apps[appid]:
+                search_description = re.sub(
+                    clean_html_re, "", apps[appid]["stable"]["description"]
+                )
+
+                if search_keywords := apps[appid]["stable"].get("keywords"):
+                    search_keywords = " ".join(search_keywords)
+                else:
+                    search_keywords = ""
+
+                fts = {
+                    "id": appid,
+                    "name": apps[appid]["stable"]["name"],
+                    "summary": apps[appid]["stable"]["summary"],
+                    "description": search_description,
+                    "keywords": search_keywords,
+                }
+                p.hset(f"fts:{appid}", mapping=fts)
+                if categories := apps[appid]["stable"].get("categories"):
+                    for category in categories:
+                        p.sadd("categories:index", category)
+                        p.sadd(f"categories:{category}", redis_key)
 
             p.set(f"apps:{appid}", json.dumps(apps[appid]))
-            p.hset(f"fts:{appid}", mapping=fts)
 
-            if categories := apps[appid].get("categories"):
-                for category in categories:
-                    p.sadd("categories:index", category)
-                    p.sadd(f"categories:{category}", redis_key)
-
-        for appid in current_apps - set(apps):
+        for appid in current_apps - set(apps_stable):
             p.delete(
                 f"apps:{appid}",
                 f"fts:{appid}",
@@ -51,51 +57,75 @@ def load_appstream():
             )
             db.redis_search.delete_document(f"fts:{appid}")
 
-        new_apps = set(apps) - current_apps
+        new_apps = set(apps_stable) - current_apps
         if not len(new_apps):
             new_apps = None
 
+        new_apps_beta = set(apps_beta) - current_apps_beta
+        if not len(new_apps_beta):
+            new_apps_beta = None
+
         p.delete("apps:index")
-        p.sadd("apps:index", *[f"apps:{appid}" for appid in apps])
+        p.sadd("apps:index", *[f"apps:{appid}" for appid in apps_stable])
+        p.delete("apps:index_beta")
+        p.sadd("apps:index_beta", *[f"apps:{appid}" for appid in apps_beta])
         p.execute()
 
-    return new_apps
+    return new_apps, new_apps_beta
+
+
+def merge_apps(apps_beta, apps_stable):
+    apps = {}
+    for appid in apps_stable:
+        apps[appid] = {"stable": apps_stable[appid]}
+
+    for appid in apps_beta:
+        if appid not in apps:
+            apps[appid] = {"beta": apps_beta[appid]}
+        else:
+            apps[appid] = {"stable": apps_stable[appid], "beta": apps_beta[appid]}
+
+    return apps
 
 
 def list_appstream(repo: str = "stable"):
-    apps = {app[5:] for app in db.redis_conn.smembers("apps:index")}
-    return sorted(apps)
+    return sorted(db.get_apps(repo))
+
 
+def get_recently_updated(limit: int = 100, repo: str = "stable"):
+    if repo == "stable":
+        zset = db.redis_conn.zrevrange("recently_updated_zset", 0, limit - 1)
+    if repo == "beta":
+        zset = db.redis_conn.zrevrange("recently_updated_beta_zset", 0, limit - 1)
 
-def get_recently_updated(limit: int = 100):
-    zset = db.redis_conn.zrevrange("recently_updated_zset", 0, limit - 1)
     return [appid for appid in zset if db.redis_conn.exists(f"apps:{appid}")]
 
 
-def get_category(category: str):
+def get_category(category: str, repo: str = "stable"):
     if index := db.redis_conn.smembers(f"categories:{category}"):
         json_appdata = db.redis_conn.mget(index)
         appdata = [json.loads(app) for app in json_appdata]
 
-        return [(app["id"]) for app in appdata]
+        return [(app[repo]["id"]) for app in appdata if repo in app]
     else:
         return []
 
 
-def search(query: str):
+def search(query: str, repo: str = "stable"):
     if results := db.search(query):
         appids = tuple(doc_id.replace("fts", "apps") for doc_id in results)
         apps = [json.loads(x) for x in db.redis_conn.mget(appids)]
 
         ret = []
         for app in apps:
-            entry = {
-                "id": app["id"],
-                "name": app["name"],
-                "summary": app["summary"],
-                "icon": app.get("icon"),
-            }
-            ret.append(entry)
+            if repo in app:
+                entry = {
+                    "id": app[repo]["id"],
+                    "name": app[repo]["name"],
+                    "summary": app[repo]["summary"],
+                    "icon": app[repo].get("icon"),
+                }
+                ret.append(entry)
 
         return ret
 

app/db.py

@@ -126,5 +126,23 @@ def get_json_key(key: str):
     return None
 
 
-def get_app_count():
-    return redis_conn.scard("apps:index")
+def get_app_count(repo: str = "stable") -> int:
+    if repo == "stable_and_beta":
+        return len(get_apps(repo))
+    if repo == "stable":
+        return redis_conn.scard("apps:index")
+    if repo == "beta":
+        return redis_conn.scard("apps:index_beta")
+
+
+def get_apps(repo: str = "stable"):
+    if repo == "stable_and_beta":
+        apps = {app[5:] for app in redis_conn.smembers("apps:index")}
+        apps_beta = {app[5:] for app in redis_conn.smembers("apps:index_beta")}
+        return apps | apps_beta
+    elif repo == "stable":
+        apps = {app[5:] for app in redis_conn.smembers("apps:index")}
+        return apps
+    elif repo == "beta":
+        apps = {app[5:] for app in redis_conn.smembers("apps:index_beta")}
+        return apps

app/feeds.py

@@ -4,17 +4,27 @@
 from . import db
 
 
-def generate_feed(key: str, title: str, description: str, link: str):
+def generate_feed(
+    key: str,
+    title: str,
+    description: str,
+    link: str,
+    repo: str,
+):
     feed = FeedGenerator()
     feed.title(title)
     feed.description(description)
     feed.link(href=link)
     feed.language("en")
 
-    appids = db.redis_conn.zrevrange(key, 0, 10, withscores=True)
-    apps = [(db.get_json_key(f"apps:{appid[0]}"), appid[1]) for appid in appids]
+    # get 15, as we might have some that don't exist, due to summary knowing about consale-applications
+    appids = db.redis_conn.zrevrange(key, 0, 15, withscores=True)
+    app_score_tuple = [
+        (db.get_json_key(f"apps:{appid[0]}"), appid[1]) for appid in appids
+    ]
+    apps = [(item[0][repo], item[1]) for item in app_score_tuple if item[0] is not None]
 
-    for app, timestamp in reversed(apps):
+    for app, timestamp in reversed(apps[:10]):
         if not app.get("name"):
             continue
 
@@ -29,9 +39,7 @@ def generate_feed(key: str, title: str, description: str, link: str):
         entry.pubDate(f"{entry_date} UTC")
 
         content = [
-            '<img src="https://dl.flathub.org/repo/appstream/x86_64/icons/128x128/{}.png">'.format(
-                app["id"]
-            ),
+            '<img src="{}">'.format(app["icon"]),
             f"<p>{app['summary']}</p>",
             f"<p>{app['description']}</p>",
             "<h3>Additional information:</h3>",
@@ -63,19 +71,21 @@ def generate_feed(key: str, title: str, description: str, link: str):
     return feed.rss_str()
 
 
-def get_recently_updated_apps_feed():
+def get_recently_updated_apps_feed(repo: str = "stable"):
     return generate_feed(
-        "recently_updated_zset",
+        "recently_updated_zset" if repo == "stable" else "recently_updated_beta_zset",
         "Flathub – recently updated applications",
         "Recently updated applications published on Flathub",
         "https://flathub.org/apps/collection/recently-updated",
+        repo,
     )
 
 
-def get_new_apps_feed():
+def get_new_apps_feed(repo: str = "stable"):
     return generate_feed(
-        "new_apps_zset",
+        "new_apps_zset" if repo == "stable" else "new_apps_beta_zset",
         "Flathub – recently added applications",
         "Applications recently published on Flathub",
         "https://flathub.org/apps/collection/new",
+        repo,
     )

app/main.py

@@ -22,7 +22,7 @@ def startup_event():
 
 @app.post("/update")
 def update():
-    new_apps = apps.load_appstream()
+    new_apps, new_apps_beta = apps.load_appstream()
     summary.update()
     picks.update()
     stats.update()
@@ -31,9 +31,18 @@ def update():
         new_apps_zset = {}
         for appid in new_apps:
             if metadata := db.get_json_key(f"summary:{appid}"):
-                new_apps_zset[appid] = metadata.get("timestamp", 0)
+                if "stable" in metadata:
+                    new_apps_zset[appid] = metadata["stable"].get("timestamp", 0)
         db.redis_conn.zadd("new_apps_zset", new_apps_zset)
 
+    if new_apps_beta:
+        new_apps_beta_zset = {}
+        for appid in new_apps_beta:
+            if metadata := db.get_json_key(f"summary:{appid}"):
+                if "beta" in metadata:
+                    new_apps_beta_zset[appid] = metadata["beta"].get("timestamp", 0)
+        db.redis_conn.zadd("new_apps_beta_zset", new_apps_beta_zset)
+
     get_recently_updated.cache_clear()
 
 
@@ -54,12 +63,15 @@ def get_category(category: schemas.Category):
 
 
 @app.get("/appstream")
-def list_appstream():
-    return apps.list_appstream()
+def list_appstream(type: schemas.AppstreamType = "stable"):
+    return apps.list_appstream(type)
 
 
 @app.get("/appstream/{appid}", status_code=200)
-def get_appstream(appid: str, response: Response):
+def get_appstream(
+    appid: str,
+    response: Response,
+):
     if value := db.get_json_key(f"apps:{appid}"):
         return value
 
@@ -75,8 +87,8 @@ def search(userquery: str):
 @app.get("/collection/recently-updated")
 @app.get("/collection/recently-updated/{limit}")
 @lru_cache()
-def get_recently_updated(limit: int = 100):
-    return apps.get_recently_updated(limit)
+def get_recently_updated(limit: int = 100, type: schemas.AppstreamType = "stable"):
+    return apps.get_recently_updated(limit, type)
 
 
 @app.get("/picks/{pick}")
@@ -98,15 +110,15 @@ def get_popular_days(days: int):
 
 
 @app.get("/feed/recently-updated")
-def get_recently_updated_apps_feed():
+def get_recently_updated_apps_feed(type: schemas.AppstreamType = "stable"):
     return Response(
-        content=feeds.get_recently_updated_apps_feed(), media_type="application/rss+xml"
+        content=feeds.get_recently_updated_apps_feed(type), media_type="application/rss+xml"
     )
 
 
 @app.get("/feed/new")
-def get_new_apps_feed():
-    return Response(content=feeds.get_new_apps_feed(), media_type="application/rss+xml")
+def get_new_apps_feed(type: schemas.AppstreamType = "stable"):
+    return Response(content=feeds.get_new_apps_feed(type), media_type="application/rss+xml")
 
 
 @app.get("/status", status_code=200)

app/schemas.py

@@ -14,3 +14,9 @@ class Category(str, Enum):
     Science = "Science"
     System = "System"
     Utility = "Utility"
+
+
+class AppstreamType(str, Enum):
+    Stable = "stable"
+    Beta = "beta"
+    Stable_And_Beta = "stable_and_beta"