diff --git a/api_app/controllers/apps.py b/api_app/controllers/apps.py index 6c8ffcd..a50d8d9 100644 --- a/api_app/controllers/apps.py +++ b/api_app/controllers/apps.py @@ -3,8 +3,13 @@ from litestar import Controller, get -from api_app.models import AppDetail, AppOverview -from config.config import IMAGES_DIR +from api_app.models import AppDetail, AppsOverview + +from dbcon.queries import get_appstore_categories, query_recent_apps, get_single_app + +from config import get_logger + +logger = get_logger(__name__) """ /apps/{store_id} a specific app @@ -17,41 +22,68 @@ def get_string_date_from_days_ago(days: int) -> str: return mydate_str -def get_most_recent_storm_dirs() -> tuple[str, list[str]]: - storm_dirs: list = [] - i = 0 - while len(storm_dirs) == 0 and i <= 5: - date_str = get_string_date_from_days_ago(i) - if date_str in os.listdir(IMAGES_DIR): - storm_dirs = os.listdir(f"{IMAGES_DIR}/{date_str}") - print(f"{date_str=} found {len(storm_dirs)} directories") - else: - print(f"{date_str=} found no directories") - i += 1 - return date_str, storm_dirs +def category_overview() -> dict: + cats = get_appstore_categories() + # Make app count strings + cats["android"] = cats["android"].apply( + lambda x: "{:,.0f}".format(x) if x else "N/A" + ) + cats["ios"] = cats["ios"].apply(lambda x: "{:,.0f}".format(x) if x else "N/A") + category_dicts = cats.to_dict(orient="records") + return category_dicts + + +def get_app_overview_dict() -> AppsOverview: + new_apps = query_recent_apps(days=7) + trending_apps = query_recent_apps(days=30) + trending_ios_apps = trending_apps[~trending_apps["store"].str.contains("oogl")] + trending_google_apps = trending_apps[trending_apps["store"].str.contains("oogl")] + new_ios_dicts = new_apps[~new_apps["store"].str.contains("oogl")].to_dict( + orient="records" + ) + new_google_dicts = new_apps[new_apps["store"].str.contains("oogl")].to_dict( + orient="records" + ) + trending_google_dicts = trending_google_apps.to_dict(orient="records") + trending_ios_dicts = trending_ios_apps.to_dict(orient="records") + trending_title = "Trending Apps this Month" + recent_title = "New Apps this Month" + trending_dicts: dict = {} + trending_dicts[trending_title] = {} + trending_dicts[recent_title] = {} + trending_dicts["new_google"] = new_google_dicts + trending_dicts["new_ios"] = new_ios_dicts + trending_dicts["trending_google"] = trending_google_dicts + trending_dicts["trending_ios"] = trending_ios_dicts + my_dict = AppsOverview( + new_google=new_google_dicts, + new_ios=new_ios_dicts, + trending_google=trending_google_dicts, + trending_ios=trending_ios_dicts, + ) + return my_dict class AppController(Controller): path = "/api/apps" @get(path="/") - async def get_apps_overview(self) -> AppOverview: + async def get_apps_overview(self) -> AppsOverview: """ Handles a GET request for a list of apps Args: Returns: - Storms: A dictionary representation of the list of apps. + A dictionary representation of the list of apps for homepasge """ - date_str, storm_dirs = get_most_recent_storm_dirs() + logger.info("inside a request") + home_dict = get_app_overview_dict() - mydict = Storms([Storm(id=mystorm, date=date_str) for mystorm in storm_dirs]) - - return mydict + return home_dict @get(path="/{store_id:str}", cache=3600) - async def get_storm_image(self, store_id: str) -> AppDetail: + async def get_app_detail(self, store_id: str) -> AppDetail: """ Handles a GET request for a specific app. @@ -62,5 +94,7 @@ async def get_storm_image(self, store_id: str) -> AppDetail: json """ - app_dict = {"store_id"} + app_df = get_single_app(store_id) + app_dict = app_df.to_dict(orient="records")[0] + return app_dict diff --git a/app.py b/app.py index ce02e20..de2c10f 100644 --- a/app.py +++ b/app.py @@ -1,26 +1,35 @@ from litestar import Litestar -from litestar.logging import LoggingConfig + +# from litestar.logging import LoggingConfig from litestar.openapi import OpenAPIConfig, OpenAPIController +from litestar.logging import LoggingConfig + from api_app.controllers.apps import AppController +from config import get_logger + +logger = get_logger(__name__) + class MyOpenAPIController(OpenAPIController): path = "/api/docs" +logging_config = LoggingConfig( + loggers={ + "my_app": { + "level": "INFO", + "handlers": ["queue_listener"], + } + } +) + app = Litestar( route_handlers=[AppController], openapi_config=OpenAPIConfig( - title="HackerNews API", version="1.0.0", openapi_controller=MyOpenAPIController - ), - logging_config=LoggingConfig( - loggers={ - "my_app": { - "level": "INFO", - "handlers": ["queue_listener"], - } - } + title="App Store API", version="0.0.1", openapi_controller=MyOpenAPIController ), + logging_config=logging_config, debug=True, ) diff --git a/dbcon/queries.py b/dbcon/queries.py index 9f0c88b..1072349 100644 --- a/dbcon/queries.py +++ b/dbcon/queries.py @@ -10,15 +10,6 @@ logger = get_logger(__name__) -def get_dash_users() -> dict: - sel_query = """SELECT * - FROM dash.users - ;""" - df = pd.read_sql(sel_query, DBCON.engine) - users_dict = df.set_index("username").to_dict(orient="index") - return users_dict - - def get_app_categories() -> list[str]: sel_query = """SELECT DISTINCT category FROM networks_with_app_metrics @@ -30,136 +21,6 @@ def get_app_categories() -> list[str]: return category_list -def query_networks_with_app_metrics() -> pd.DataFrame: - table_name = "networks_with_app_metrics" - sel_query = f"""SELECT - * - FROM - {table_name} - ; - """ - df = pd.read_sql(sel_query, DBCON.engine) - df = df.rename( - columns={ - "publisher_urls": "publishers_count", - "total_publisher_urls": "publishers_total", - } - ) - return df - - -def query_networks_count(top_only: bool = False) -> pd.DataFrame: - if top_only: - table_name = "network_counts_top" - else: - table_name = "network_counts" - sel_query = f"""SELECT - * - FROM - {table_name} - ; - """ - df = pd.read_sql(sel_query, DBCON.engine) - return df - - -def query_network_uniqueness(limit=100): - sel_query = f""" - SELECT - ad_domain_url, - count(DISTINCT publisher_id) AS publisher_count, - sum(is_unique) AS unique_count - FROM - publisher_url_developer_ids_uniques - GROUP BY - ad_domain_url - ORDER BY publisher_count DESC - LIMIT {limit} - ; - """ - df = pd.read_sql(sel_query, DBCON.engine) - df["percent"] = df["unique_count"] / df["publisher_count"] - return df - - -def get_app_txt_view(developer_url: str, direct_only=True) -> pd.DataFrame: - if direct_only: - direct_only_str = "AND av.relationship = 'DIRECT'" - else: - direct_only_str = "" - sel_query = f"""WITH cte1 AS ( - SELECT - av.developer_domain_url, - av.ad_domain AS ad_domain_id, - av.ad_domain_url, - av.publisher_id - FROM - app_ads_view av - WHERE - av.developer_domain_url LIKE '{developer_url}' - {direct_only_str} - ) - SELECT - c1.developer_domain_url AS my_domain_url, - av2.developer_domain_url AS their_domain_url, - CASE - WHEN av2.developer_domain_url != c1.developer_domain_url - THEN 'FAIL' - ELSE 'PASS' - END AS is_my_id, - av2.publisher_id, - av2.ad_domain_url, - av2.ad_domain AS ad_domain_id, - av2.relationship AS relationship, - av2.txt_entry_crawled_at - FROM - cte1 c1 - LEFT JOIN app_ads_view av2 ON - av2.ad_domain_url = c1.ad_domain_url - AND av2.publisher_id = c1.publisher_id - ; - """ - df = pd.read_sql(sel_query, DBCON.engine) - return df - - -def query_store_apps_overview(start_date: str): - logger.info("Query logging.store_apps_snapshot") - sel_query = f"""SELECT - sas.*, - s.name AS store_name, - coalesce(cr.outcome, 'not_crawled') AS outcome - FROM - logging.store_apps_snapshot sas - LEFT JOIN crawl_results cr - ON cr.id = sas.crawl_result - LEFT JOIN stores s - ON s.id = sas.store - where updated_at >= '{start_date}' - ; - """ - df = pd.read_sql(sel_query, con=DBCON.engine) - df = df.drop(["store", "crawl_result"], axis=1) - return df - - -def query_pub_domains_overview(start_date: str): - logger.info("Query logging.pub_domains_snapshot") - sel_query = f"""SELECT - ss.*, - coalesce(cr.outcome, 'not_crawled') AS outcome - FROM - logging.snapshot_pub_domains ss - LEFT JOIN crawl_results cr - ON cr.id = ss.crawl_result - where updated_at >= '{start_date}' - ; - """ - df = pd.read_sql(sel_query, con=DBCON.engine) - df = df.drop(["crawl_result"], axis=1) - return df - - def query_recent_apps(days: int = 7): logger.info("Query app_store for recent apps") limit = 10