From 69d0d4e6b64bacc5d095ca7e09612f644994a6a4 Mon Sep 17 00:00:00 2001 From: Jacob Woliver Date: Thu, 12 Dec 2024 16:47:04 -0500 Subject: [PATCH] add demo apps from company meeting --- extensions/vulnerabilities-by-content/app.py | 188 ++++++++ .../connect-extension.toml | 4 + .../vulnerabilities-by-content/manifest.json | 24 + .../requirements.txt | 7 + extensions/vulnerabilities-by-package/app.py | 417 ++++++++++++++++++ .../connect-extension.toml | 4 + .../vulnerabilities-by-package/manifest.json | 24 + .../requirements.txt | 7 + 8 files changed, 675 insertions(+) create mode 100644 extensions/vulnerabilities-by-content/app.py create mode 100644 extensions/vulnerabilities-by-content/connect-extension.toml create mode 100644 extensions/vulnerabilities-by-content/manifest.json create mode 100644 extensions/vulnerabilities-by-content/requirements.txt create mode 100644 extensions/vulnerabilities-by-package/app.py create mode 100644 extensions/vulnerabilities-by-package/connect-extension.toml create mode 100644 extensions/vulnerabilities-by-package/manifest.json create mode 100644 extensions/vulnerabilities-by-package/requirements.txt diff --git a/extensions/vulnerabilities-by-content/app.py b/extensions/vulnerabilities-by-content/app.py new file mode 100644 index 0000000..a5bbd6e --- /dev/null +++ b/extensions/vulnerabilities-by-content/app.py @@ -0,0 +1,188 @@ +import re +from datetime import datetime +from functools import cache +from typing import Callable +import requests +import json + +import pandas as pd +from htmltools import tags +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from posit.connect import Client +from shiny import reactive +from shiny.express import input, render, ui + +client = Client() +packages = reactive.value(pd.DataFrame([])) + +with ui.sidebar(): + ui.input_text("guid", "Content GUID") + +@render.text +def datagrid_label(): + ui.busy_indicators.options(spinner_type=None) + if num_packages() > 0: + return f"{num_packages()} packages for content '{content_title()}'" + else: + return "Please enter a content GUID to search for packages." + +def update_packages(app_packages): + data = prepare_packages_data(get_packages(app_packages)) + packages.set(data) + + +@render.data_frame +def app_grid(): + app_packages = packages_spec() + if not app_packages: + return None + update_packages(app_packages) + return render.DataGrid( + packages.get(), + width="100%", + height="100%", + selection_mode="rows", + styles=dict(style={"white-space": "nowrap"}), + ) + + +@reactive.calc +def num_packages(): + return len(packages.get()) + +@reactive.calc +def content_title(): + if input.guid() == "": + return + response = client.get( + f"v1/content/{input.guid()}", + ) + if response.status_code != 200: + raise Exception(f"Failed to search for {input.guid()}: {response.text}") + + data = response.json() + title = data.get("title") + if title is None: + raise Exception(f"Invalid search response from server: {response.text}") + + return title + + +@reactive.calc +def packages_spec(): + if input.guid() == "": + return + return get_app_packages(input.guid()) + +def get_packages(app_packages) -> pd.DataFrame: + package_strs = [] + for package in app_packages: + package_str = f'{package["name"]}=={package["version"]}' + package_strs.append(package_str) + + response = stream_packages_info(package_strs) + for line in response.iter_lines(): + if line: + try: + package_data = json.loads(line) + for package in app_packages: + if package_data.get('name') == package.get('name') and package_data.get('version') == package.get('version'): + vulns = package_data.get('vulns', []) + cves = [] + for vuln in vulns: + cves.append(vuln["id"]) + available_versions = package_data.get('available_versions', []) + package["cves"] = cves + package["available_versions"] = ", ".join(available_versions) + package["package"] = f"{package.get('name')} {package.get('version')}" + + # The newest version is always the first + # TODO we aren't properly accounting for dev versions and such, need to correctly parse versions + package["up_to_date"] = get_up_to_date(package.get('version'), available_versions[0]) + break + + except json.JSONDecodeError as e: + print(f"Error parsing JSON: {e}") + + print(f"Found {len(app_packages)} packages for content '{content_title()}'") + return pd.DataFrame(app_packages) + +# TODO don't hard code this +def stream_packages_info(names): + url = "http://ec2-54-234-188-15.compute-1.amazonaws.com/__api__/filter/packages" + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJwYWNrYWdlbWFuYWdlciIsImp0aSI6IjZkNDVjNWQ1LTVmNzAtNDgzMy1hYmEzLTRjMDRlYWI4ZjAxYiIsImlhdCI6MTczMzk2OTQxNiwiaXNzIjoicGFja2FnZW1hbmFnZXIiLCJzY29wZXMiOnsiZ2xvYmFsIjoiYWRtaW4ifX0.o6Xez3vB7Oe8_XbWd6IFkvLWT4SOGO3dpVSeRND3Wqw" + } + + data = { + "names": names, + "repo": "pypi", + "snapshot": "latest" + } + + return requests.post(url, json=data, headers=headers, stream=True) + +def get_app_packages(app_guid: str): + # get the specific version used + package_response = client.get( + f'v1/content/{app_guid}/packages' + ) + + if package_response.status_code != 200: + raise Exception(f'Failed to get packages for {app_guid}: {package_response.text}') + + package_data = package_response.json() + if package_data is None: + raise Exception(f"Invalid packages response from server: {package_response.text}") + + return package_data + + +def get_up_to_date(current_version, latest_version): + if current_version >= latest_version: + return "โœ…" + return "โŒ" + +def prepare_packages_data(df: pd.DataFrame) -> pd.DataFrame: + if len(df) == 0: + return df + + for i, row in df.iterrows(): + + df.at[i, "package_link"] = tags.a( + tags.span("โ†—", style="font-size: 1.5em"), + target="_blank", + href=f"http://ec2-54-234-188-15.compute-1.amazonaws.com/client/#/repos/pypi/packages/overview?search={row['name']}", + ) + + if row.cves: + vulns = [] + for cve in row.cves: + link = tags.a( + tags.span(cve, style="font-size: 1em"), + target="_blank", + href=f"https://osv.dev/vulnerability/{cve}" + ) + vulns.append(link) + + df.at[i, "cves"] = tags.span([ + item if i == 0 else [", ", item] + for i, item in enumerate(vulns) + ]) + + columns = { + "package_link": "Link", + "name": "Name", + "version": "Version", + "cves": "Known Vulnerabilities", + "up_to_date": "Up to Date", + "available_versions": "Available Versions at Snapshot", + } + df = df[columns.keys()].rename(columns=columns) + return df + + +@reactive.calc +def has_selection(): + return len(input.app_grid_selected_rows()) > 0 diff --git a/extensions/vulnerabilities-by-content/connect-extension.toml b/extensions/vulnerabilities-by-content/connect-extension.toml new file mode 100644 index 0000000..f6eb3ea --- /dev/null +++ b/extensions/vulnerabilities-by-content/connect-extension.toml @@ -0,0 +1,4 @@ +name = "connect-extension-vulnerabilities-by-content" +title = "Find Content Vulnerabilities by Content" +description = "Connect Extension: Find Content Vulnerabilities by Content" +access_type = "logged_in" diff --git a/extensions/vulnerabilities-by-content/manifest.json b/extensions/vulnerabilities-by-content/manifest.json new file mode 100644 index 0000000..0575d9d --- /dev/null +++ b/extensions/vulnerabilities-by-content/manifest.json @@ -0,0 +1,24 @@ +{ + "version": 1, + "locale": "en_US.UTF-8", + "metadata": { + "appmode": "python-shiny", + "entrypoint": "shiny.express.app:app_2e_py" + }, + "python": { + "version": "3.11.9", + "package_manager": { + "name": "pip", + "version": "24.2", + "package_file": "requirements.txt" + } + }, + "files": { + "requirements.txt": { + "checksum": "2ed393d51266e315d6e7b55ac26c1062" + }, + "app.py": { + "checksum": "61ddac9f526f0d55ab94e3b02eae4070" + } + } + } diff --git a/extensions/vulnerabilities-by-content/requirements.txt b/extensions/vulnerabilities-by-content/requirements.txt new file mode 100644 index 0000000..159a0d6 --- /dev/null +++ b/extensions/vulnerabilities-by-content/requirements.txt @@ -0,0 +1,7 @@ +htmltools +packaging +pandas +posit-sdk +shiny +mobsf==4.1.3 +requests \ No newline at end of file diff --git a/extensions/vulnerabilities-by-package/app.py b/extensions/vulnerabilities-by-package/app.py new file mode 100644 index 0000000..f00a294 --- /dev/null +++ b/extensions/vulnerabilities-by-package/app.py @@ -0,0 +1,417 @@ +import re +from datetime import datetime +from functools import cache +from typing import Callable +import requests +import json +from collections import defaultdict + + +import pandas as pd +from htmltools import tags +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from posit.connect import Client +from posit.connect.errors import ClientError +from shiny import reactive +from shiny.express import input, render, ui + + +# This is the max page size accepted by the API. +# Use this until we have pagination. +page_size = 500 + +client = Client() +apps = reactive.value(pd.DataFrame([])) + +def fetch_all_packages(): + page_number = 1 + all_packages = [] + + while True: + response = client.get( + "v1/packages", + params={ + "name": input.name(), + "page_size": page_size, + "page_number": page_number + }, + ) + if response.status_code != 200: + raise Exception(f"Failed to get all packages") + + data = response.json() + results = data.get("results") + if results is None: + raise Exception(f"Invalid packages response from server: {response.text}") + + if not results: + # paged all the way through + break + + all_packages.extend(results) + page_number += 1 + return all_packages + +with ui.sidebar(): + ui.input_text("name", "Package Name") + ui.input_text("min_version", "Minimum version (>=)") + ui.input_text("max_version", "Maximum version (<)") + ui.hr() + ui.input_action_button("lock_selected", "Lock Selected") + ui.input_action_button("delete_selected", "Delete Selected") + + +@render.text +def datagrid_label(): + ui.busy_indicators.options(spinner_type=None) + if has_valid_spec(): + return f"{num_apps()} items using {package_spec()}" + else: + return "Please enter a package name and optional versions." + + +def update_apps(): + spec = package_spec() + if not has_valid_spec(): + return + data = prepare_app_data(get_apps(spec)) + apps.set(data) + + +@render.data_frame +def app_grid(): + if not has_valid_spec(): + return None + update_apps() + return render.DataGrid( + apps.get(), + width="100%", + height="100%", + selection_mode="rows", + styles=dict(style={"white-space": "nowrap"}), + ) + + +@reactive.calc +def num_apps(): + return len(apps.get()) + +@reactive.calc +def package_spec(): + spec = input.name() + if spec == "": + return "" + if input.min_version() != "": + spec += ">=" + input.min_version() + if input.max_version() != "": + comma = "," if input.min_version() != "" else "" + spec += comma + "<" + input.max_version() + return f'package:"{spec}"' + +@reactive.calc +def package_range(): + spec = input.name() + if spec == "": + return "" + elif input.min_version() == "" and input.max_version() == "": + return f"{input.name()}>0.0,<=99.99" + if input.min_version() != "": + spec += ">=" + input.min_version() + if input.max_version() != "": + comma = "," if input.min_version() != "" else "" + spec += comma + "<" + input.max_version() + return spec + + +@reactive.calc +def has_valid_spec(): + spec = package_spec() + if not spec: + return False + + match = re.match(r'package:"[A-Za-z0-9_.-]+(.*)"', spec) + if not match: + return False + + versionSpec = match.group(1) + if not versionSpec: + return True + + try: + _ = SpecifierSet(versionSpec) + return True + except InvalidSpecifier: + return False + + +def get_apps(spec: str) -> pd.DataFrame: + page_number = 1 + all_apps = [] + + all_packages = fetch_all_packages() + vulns_map = fetch_all_ppm_packages() + + for package in all_packages: + # TODO hacky for the demo, only use python versions + if package["language"] != "python": + continue + + # TODO version parsing doesn't filter at all currently + + response = client.get( + "v1/content/"+package["app_guid"], + params={ + "include": "owner" + }, + ) + if response.status_code != 200: + raise Exception(f"Failed to search for {spec}: {response.text}") + + app = response.json() + if app is None: + raise Exception(f"Invalid search response from server: {response.text}") + + if not app: + # paged all the way through + break + + # flatten the included owner sub-object + owner = app["owner"] + app["owner_username"] = owner["username"] + app["owner_first_name"] = owner["first_name"] + app["owner_last_name"] = owner["last_name"] + + app["cves"] = vulns_map[package["version"]] + app["package_name"] = package["name"] + app["package"] = f'{package["name"]} {package["version"]}' + + all_apps.append(app) + page_number += 1 + + print(f"Found {len(all_apps)} apps matching '{spec}'") + return pd.DataFrame(all_apps) + +def get_package_info(app_packages, package_name: str): + for package in app_packages: + if package["name"] == package_name: + return package + +def fetch_all_ppm_packages() -> list: + + package_str = package_range() + names = [] + names.append(package_str) + + vulns_map = defaultdict(list) + + response = stream_packages_info(names) + for line in response.iter_lines(): + if line: + try: + package_data = json.loads(line) + vulns = package_data.get('vulns', []) + cves = [] + for vuln in vulns: + cves.append(vuln["id"]) + vulns_map[package_data["version"]] = cves + + except json.JSONDecodeError as e: + print(f"Error parsing JSON: {e}") + return vulns_map + +# TODO don't hard code this +def stream_packages_info(names): + url = "http://ec2-54-234-188-15.compute-1.amazonaws.com/__api__/filter/packages" + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJwYWNrYWdlbWFuYWdlciIsImp0aSI6IjZkNDVjNWQ1LTVmNzAtNDgzMy1hYmEzLTRjMDRlYWI4ZjAxYiIsImlhdCI6MTczMzk2OTQxNiwiaXNzIjoicGFja2FnZW1hbmFnZXIiLCJzY29wZXMiOnsiZ2xvYmFsIjoiYWRtaW4ifX0.o6Xez3vB7Oe8_XbWd6IFkvLWT4SOGO3dpVSeRND3Wqw" + } + + data = { + "names": names, + "repo": "pypi", # TODO add logic for R vs Python + "snapshot": "latest" + } + + return requests.post(url, json=data, headers=headers, stream=True) + + +def get_display_name(row): + if row.owner_first_name and row.owner_last_name: + name = row.owner_first_name + " " + row.owner_last_name + elif row.owner_first_name: + name = row.owner_first_name + elif row.owner_last_name: + name = row.owner_last_name + else: + return row.owner_username + return f"{row.owner_username} ({name})" + + +@cache +def get_owner_email(owner_guid: str) -> str: + user = client.users.get(owner_guid) + return user.email + + +def prepare_app_data(df: pd.DataFrame) -> pd.DataFrame: + if len(df) == 0: + return df + + df["title_link"] = None + df["owner"] = None + df["owner_email"] = None + df["email_link"] = None + df["created"] = pd.to_datetime(df["created_time"]).dt.date + for i, row in df.iterrows(): + df.at[i, "app_link"] = tags.a( + tags.span("โ†—", style="font-size: 1.5em"), + target="_blank", + href=row.dashboard_url, + ) + df.at[i, "package"] = tags.a( + tags.span(row.package, style="font-size: 1em"), + target="_blank", + href=f"http://ec2-54-234-188-15.compute-1.amazonaws.com/client/#/repos/pypi/packages/overview?search={row.package_name}", + ) + if row.cves: + vulns = [] + + for cve in row.cves: + link = tags.a( + tags.span(cve, style="font-size: 1em"), + target="_blank", + href=f"https://osv.dev/vulnerability/{cve}" + ) + vulns.append(link) + + if vulns: + df.at[i, "cves"] = tags.span([ + item if i == 0 else [", ", item] + for i, item in enumerate(vulns) + ]) + else: + df.at[i, "cves"] = "" + else: + df.at[i, "cves"] = "" + + title_len = 50 + title = df.at[i, "title"] or "" + df.at[i, "title"] = title[:title_len] + ( + "..." if len(title) > title_len else "" + ) + + display_name = get_display_name(row) + email = get_owner_email(row.owner_guid) + + df.at[i, "owner_display_name"] = display_name + df.at[i, "owner_email"] = email + if email: + df.at[i, "email_link"] = tags.a(email, href=f"mailto:{email}") + else: + df.at[i, "email_link"] = email + + df.at[i, "lock_icon"] = "๐Ÿ”’" if row.locked else "" + + if row.bundle_id: + bundle_url = f"{client.cfg.url}/v1/content/{row.guid}/bundles/{row.bundle_id}/download" + df.at[i, "bundle_link"] = tags.a( + tags.span("โค“", style="font-size: 1.5em"), + target="_blank", + href=bundle_url, + ) + + columns = { + "app_link": "App", + "id": "ID", + "title": "Title", + "package": "Package", + "cves": "Known Vulnerabilities", + "owner_display_name": "Owner", + "email_link": "Email", + "created": "Created", + "guid": "GUID", + "bundle_link": "Download", + "lock_icon": "Locked", + } + df = df[columns.keys()].rename(columns=columns) + return df + + +@reactive.calc +def has_selection(): + return len(input.app_grid_selected_rows()) > 0 + + +def each_selected_app(message: str, func: Callable[[str], bool]): + rows = input.app_grid_selected_rows() + if not rows: + return + + selected_guids = apps.get().loc[list(rows)]["GUID"].tolist() + print("Locking selected apps:", selected_guids) + + with ui.Progress(min=0, max=len(selected_guids)) as p: + p.set(message=message + "...", value=0) + success_count = 0 + + for i, guid in enumerate(selected_guids): + if func(guid): + success_count += 1 + p.set(value=i + 1) + + update_apps() + return success_count + + + +@reactive.effect +@reactive.event(input.lock_selected) +def lock_selected_apps(): + rows = input.app_grid_selected_rows() + if not rows: + return + + successes = each_selected_app("Locking applications", lock_app) + ui.notification_show( + f"Locked {successes} out of {len(rows)} applications", + type="message", + ) + + +def lock_app(guid): + try: + content = client.content.get(guid) + today = datetime.now().date().isoformat() + content.update( + locked=True, + locked_message=f"Locked on {today} because it contains a vulnerable version of '{input.name()}'", + ) + return True + except Exception as e: + print(f"Error locking app {guid}: {e}") + return False + + +@reactive.effect +@reactive.event(input.delete_selected) +def delete_selected_apps(): + rows = input.app_grid_selected_rows() + if not rows: + return + + successes = each_selected_app("Deleting applications", delete_app) + ui.notification_show( + f"Deleted {successes} out of {len(rows)} applications", + type="message", + ) + + +def delete_app(guid): + try: + content = client.content.get(guid) + content.delete() + return True + except Exception as e: + print(f"Error deleting app {guid}: {e}") + return False diff --git a/extensions/vulnerabilities-by-package/connect-extension.toml b/extensions/vulnerabilities-by-package/connect-extension.toml new file mode 100644 index 0000000..e5da7b9 --- /dev/null +++ b/extensions/vulnerabilities-by-package/connect-extension.toml @@ -0,0 +1,4 @@ +name = "connect-extension-vulnerabilities-by-package" +title = "Find Content Vulnerabilities by Package" +description = "Connect Extension: Find Content Vulnerabilities by Package" +access_type = "logged_in" diff --git a/extensions/vulnerabilities-by-package/manifest.json b/extensions/vulnerabilities-by-package/manifest.json new file mode 100644 index 0000000..0575d9d --- /dev/null +++ b/extensions/vulnerabilities-by-package/manifest.json @@ -0,0 +1,24 @@ +{ + "version": 1, + "locale": "en_US.UTF-8", + "metadata": { + "appmode": "python-shiny", + "entrypoint": "shiny.express.app:app_2e_py" + }, + "python": { + "version": "3.11.9", + "package_manager": { + "name": "pip", + "version": "24.2", + "package_file": "requirements.txt" + } + }, + "files": { + "requirements.txt": { + "checksum": "2ed393d51266e315d6e7b55ac26c1062" + }, + "app.py": { + "checksum": "61ddac9f526f0d55ab94e3b02eae4070" + } + } + } diff --git a/extensions/vulnerabilities-by-package/requirements.txt b/extensions/vulnerabilities-by-package/requirements.txt new file mode 100644 index 0000000..159a0d6 --- /dev/null +++ b/extensions/vulnerabilities-by-package/requirements.txt @@ -0,0 +1,7 @@ +htmltools +packaging +pandas +posit-sdk +shiny +mobsf==4.1.3 +requests \ No newline at end of file