Working version of /apps/ api paths, no react app yet

api_app/controllers/apps.py

@@ -3,8 +3,13 @@
 
 from litestar import Controller, get
 
-from api_app.models import AppDetail, AppOverview
-from config.config import IMAGES_DIR
+from api_app.models import AppDetail, AppsOverview
+
+from dbcon.queries import get_appstore_categories, query_recent_apps, get_single_app
+
+from config import get_logger
+
+logger = get_logger(__name__)
 
 """
 /apps/{store_id} a specific app
@@ -17,41 +22,68 @@ def get_string_date_from_days_ago(days: int) -> str:
     return mydate_str
 
 
-def get_most_recent_storm_dirs() -> tuple[str, list[str]]:
-    storm_dirs: list = []
-    i = 0
-    while len(storm_dirs) == 0 and i <= 5:
-        date_str = get_string_date_from_days_ago(i)
-        if date_str in os.listdir(IMAGES_DIR):
-            storm_dirs = os.listdir(f"{IMAGES_DIR}/{date_str}")
-            print(f"{date_str=} found {len(storm_dirs)} directories")
-        else:
-            print(f"{date_str=} found no directories")
-        i += 1
-    return date_str, storm_dirs
+def category_overview() -> dict:
+    cats = get_appstore_categories()
+    # Make app count strings
+    cats["android"] = cats["android"].apply(
+        lambda x: "{:,.0f}".format(x) if x else "N/A"
+    )
+    cats["ios"] = cats["ios"].apply(lambda x: "{:,.0f}".format(x) if x else "N/A")
+    category_dicts = cats.to_dict(orient="records")
+    return category_dicts
+
+
+def get_app_overview_dict() -> AppsOverview:
+    new_apps = query_recent_apps(days=7)
+    trending_apps = query_recent_apps(days=30)
+    trending_ios_apps = trending_apps[~trending_apps["store"].str.contains("oogl")]
+    trending_google_apps = trending_apps[trending_apps["store"].str.contains("oogl")]
+    new_ios_dicts = new_apps[~new_apps["store"].str.contains("oogl")].to_dict(
+        orient="records"
+    )
+    new_google_dicts = new_apps[new_apps["store"].str.contains("oogl")].to_dict(
+        orient="records"
+    )
+    trending_google_dicts = trending_google_apps.to_dict(orient="records")
+    trending_ios_dicts = trending_ios_apps.to_dict(orient="records")
+    trending_title = "Trending Apps this Month"
+    recent_title = "New Apps this Month"
+    trending_dicts: dict = {}
+    trending_dicts[trending_title] = {}
+    trending_dicts[recent_title] = {}
+    trending_dicts["new_google"] = new_google_dicts
+    trending_dicts["new_ios"] = new_ios_dicts
+    trending_dicts["trending_google"] = trending_google_dicts
+    trending_dicts["trending_ios"] = trending_ios_dicts
+    my_dict = AppsOverview(
+        new_google=new_google_dicts,
+        new_ios=new_ios_dicts,
+        trending_google=trending_google_dicts,
+        trending_ios=trending_ios_dicts,
+    )
+    return my_dict
 
 
 class AppController(Controller):
     path = "/api/apps"
 
     @get(path="/")
-    async def get_apps_overview(self) -> AppOverview:
+    async def get_apps_overview(self) -> AppsOverview:
         """
         Handles a GET request for a list of apps
 
         Args:
 
         Returns:
-            Storms: A dictionary representation of the list of apps.
+            A dictionary representation of the list of apps for homepasge
         """
-        date_str, storm_dirs = get_most_recent_storm_dirs()
+        logger.info("inside a request")
+        home_dict = get_app_overview_dict()
 
-        mydict = Storms([Storm(id=mystorm, date=date_str) for mystorm in storm_dirs])
-
-        return mydict
+        return home_dict
 
     @get(path="/{store_id:str}", cache=3600)
-    async def get_storm_image(self, store_id: str) -> AppDetail:
+    async def get_app_detail(self, store_id: str) -> AppDetail:
         """
         Handles a GET request for a specific app.
 
@@ -62,5 +94,7 @@ async def get_storm_image(self, store_id: str) -> AppDetail:
             json
         """
 
-        app_dict = {"store_id"}
+        app_df = get_single_app(store_id)
+        app_dict = app_df.to_dict(orient="records")[0]
+
         return app_dict

app.py

@@ -1,26 +1,35 @@
 from litestar import Litestar
-from litestar.logging import LoggingConfig
+
+# from litestar.logging import LoggingConfig
 from litestar.openapi import OpenAPIConfig, OpenAPIController
 
+from litestar.logging import LoggingConfig
+
 from api_app.controllers.apps import AppController
 
+from config import get_logger
+
+logger = get_logger(__name__)
+
 
 class MyOpenAPIController(OpenAPIController):
     path = "/api/docs"
 
 
+logging_config = LoggingConfig(
+    loggers={
+        "my_app": {
+            "level": "INFO",
+            "handlers": ["queue_listener"],
+        }
+    }
+)
+
 app = Litestar(
     route_handlers=[AppController],
     openapi_config=OpenAPIConfig(
-        title="HackerNews API", version="1.0.0", openapi_controller=MyOpenAPIController
-    ),
-    logging_config=LoggingConfig(
-        loggers={
-            "my_app": {
-                "level": "INFO",
-                "handlers": ["queue_listener"],
-            }
-        }
+        title="App Store API", version="0.0.1", openapi_controller=MyOpenAPIController
     ),
+    logging_config=logging_config,
     debug=True,
 )

dbcon/queries.py

@@ -10,15 +10,6 @@
 logger = get_logger(__name__)
 
 
-def get_dash_users() -> dict:
-    sel_query = """SELECT *
-                    FROM dash.users
-                    ;"""
-    df = pd.read_sql(sel_query, DBCON.engine)
-    users_dict = df.set_index("username").to_dict(orient="index")
-    return users_dict
-
-
 def get_app_categories() -> list[str]:
     sel_query = """SELECT DISTINCT category
                     FROM networks_with_app_metrics
@@ -30,136 +21,6 @@ def get_app_categories() -> list[str]:
     return category_list
 
 
-def query_networks_with_app_metrics() -> pd.DataFrame:
-    table_name = "networks_with_app_metrics"
-    sel_query = f"""SELECT
-                    *
-                    FROM 
-                    {table_name}
-                    ;
-                """
-    df = pd.read_sql(sel_query, DBCON.engine)
-    df = df.rename(
-        columns={
-            "publisher_urls": "publishers_count",
-            "total_publisher_urls": "publishers_total",
-        }
-    )
-    return df
-
-
-def query_networks_count(top_only: bool = False) -> pd.DataFrame:
-    if top_only:
-        table_name = "network_counts_top"
-    else:
-        table_name = "network_counts"
-    sel_query = f"""SELECT
-                    *
-                    FROM 
-                    {table_name}
-                    ;
-                """
-    df = pd.read_sql(sel_query, DBCON.engine)
-    return df
-
-
-def query_network_uniqueness(limit=100):
-    sel_query = f"""
-        SELECT
-        ad_domain_url,
-        count(DISTINCT publisher_id) AS publisher_count,
-        sum(is_unique) AS unique_count
-    FROM
-        publisher_url_developer_ids_uniques
-    GROUP BY
-        ad_domain_url
-    ORDER BY publisher_count DESC
-    LIMIT {limit}
-    ;
-    """
-    df = pd.read_sql(sel_query, DBCON.engine)
-    df["percent"] = df["unique_count"] / df["publisher_count"]
-    return df
-
-
-def get_app_txt_view(developer_url: str, direct_only=True) -> pd.DataFrame:
-    if direct_only:
-        direct_only_str = "AND av.relationship = 'DIRECT'"
-    else:
-        direct_only_str = ""
-    sel_query = f"""WITH cte1 AS (
-            SELECT
-                av.developer_domain_url,
-                av.ad_domain AS ad_domain_id,
-                av.ad_domain_url,
-                av.publisher_id
-            FROM
-                app_ads_view av
-            WHERE
-                av.developer_domain_url LIKE '{developer_url}'
-                {direct_only_str}
-                )
-            SELECT
-                c1.developer_domain_url AS my_domain_url,
-                av2.developer_domain_url AS their_domain_url,
-                CASE
-                  WHEN av2.developer_domain_url != c1.developer_domain_url
-                    THEN 'FAIL'
-                    ELSE 'PASS'
-                  END AS is_my_id,
-                av2.publisher_id,
-                av2.ad_domain_url,
-                av2.ad_domain AS ad_domain_id,
-                av2.relationship AS relationship,
-                av2.txt_entry_crawled_at
-            FROM
-                cte1 c1
-            LEFT JOIN app_ads_view av2 ON
-                av2.ad_domain_url = c1.ad_domain_url
-                AND av2.publisher_id = c1.publisher_id
-                    ;
-                """
-    df = pd.read_sql(sel_query, DBCON.engine)
-    return df
-
-
-def query_store_apps_overview(start_date: str):
-    logger.info("Query logging.store_apps_snapshot")
-    sel_query = f"""SELECT
-                        sas.*,
-                        s.name AS store_name,
-                        coalesce(cr.outcome, 'not_crawled') AS outcome
-                    FROM
-                    logging.store_apps_snapshot sas
-                    LEFT JOIN crawl_results cr
-                        ON cr.id = sas.crawl_result
-                    LEFT JOIN stores s
-                        ON s.id = sas.store
-                    where updated_at >= '{start_date}'
-                    ;
-                """
-    df = pd.read_sql(sel_query, con=DBCON.engine)
-    df = df.drop(["store", "crawl_result"], axis=1)
-    return df
-
-
-def query_pub_domains_overview(start_date: str):
-    logger.info("Query logging.pub_domains_snapshot")
-    sel_query = f"""SELECT
-                        ss.*,
-                        coalesce(cr.outcome, 'not_crawled') AS outcome
-                    FROM
-                    logging.snapshot_pub_domains ss
-                    LEFT JOIN crawl_results cr
-                        ON cr.id = ss.crawl_result
-                    where updated_at >= '{start_date}'
-                    ;
-                """
-    df = pd.read_sql(sel_query, con=DBCON.engine)
-    df = df.drop(["crawl_result"], axis=1)
-    return df
-
-
 def query_recent_apps(days: int = 7):
     logger.info("Query app_store for recent apps")
     limit = 10