diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index e099dfe7b..51909a607 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -41,11 +41,11 @@ jobs:
# Maps tcp port 5432 on service container to the host
- 5432:5432
steps:
- - uses: actions/checkout@v2
- - name: Set up Python 3.8
- uses: actions/setup-python@v2
+ - uses: actions/checkout@v3
+ - name: Set up Python 3.10
+ uses: actions/setup-python@v4
with:
- python-version: 3.8
+ python-version: 3.10.6
- name: Setup virtual environment
run: |
python -m pip install --upgrade pip
diff --git a/.gitignore b/.gitignore
index db91d77c4..c95ebb1d4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,9 +30,10 @@ kaybee/pkged.go
kaybeeconf.yaml
prospector/.env
prospector/workspace.code-workspace
-prospector/.env
prospector/disabled_tests/skip_test-commits.db
prospector/disabled_tests/skip_test-vulnerabilities.db
+prospector/results
+prospector/*.py
prospector/.vscode/launch.json
prospector/.vscode/settings.json
prospector/install_fastext.sh
@@ -45,7 +46,8 @@ prospector/client/cli/cov_html/*
prospector/client/web/node-app/node_modules
prospector/.coverage.*
prospector/.coverage
-**/cov_html/*
+**/cov_html
+prospector/cov_html
.coverage
prospector/prospector.code-workspace
prospector/requests-cache.sqlite
diff --git a/prospector/.flake8 b/prospector/.flake8
index 17f4f1e64..ef01cf82b 100644
--- a/prospector/.flake8
+++ b/prospector/.flake8
@@ -1,5 +1,5 @@
[flake8]
-ignore = E203, E501, W503,F401,F403
+ignore = E203, E501, W503,F401,F403,W605
exclude =
# No need to traverse our git directory
.git,
diff --git a/prospector/Makefile b/prospector/Makefile
index 2540f0a59..164b9ce90 100644
--- a/prospector/Makefile
+++ b/prospector/Makefile
@@ -13,7 +13,7 @@ test:
setup: requirements.txt
@echo "$(PROGRESS) Installing requirements"
- pip install -r requirements.txt
+ @pip install -r requirements.txt
@echo "$(DONE) Installed requirements"
@echo "$(PROGRESS) Installing pre-commit and other modules"
@pre-commit install
@@ -26,7 +26,7 @@ dev-setup: setup requirements-dev.txt
@mkdir -p $(CVE_DATA_PATH)
@echo "$(DONE) Created directory $(CVE_DATA_PATH)"
@echo "$(PROGRESS) Installing development requirements"
- pip install -r requirements-dev.txt
+ @pip install -r requirements-dev.txt
@echo "$(DONE) Installed development requirements"
docker-setup:
@@ -56,7 +56,11 @@ select-run:
python client/cli/main.py $(cve) --repository $(repository) --use-nvd
clean:
- rm prospector-report.html
- rm -f all.log* error.log*
- rm -rf $(GIT_CACHE)/*
- rm -rf __pycache__
\ No newline at end of file
+ @rm -f prospector.log
+ @rm -rf $(GIT_CACHE)/*
+ @rm -rf __pycache__
+ @rm -rf */__pycache__
+ @rm -rf */*/__pycache__
+ @rm -rf *report.html
+ @rm -rf *.json
+ @rm -rf requests-cache.sqlite
\ No newline at end of file
diff --git a/prospector/api/__init__.py b/prospector/api/__init__.py
index 7b553be39..e69de29bb 100644
--- a/prospector/api/__init__.py
+++ b/prospector/api/__init__.py
@@ -1,9 +0,0 @@
-import os
-
-DB_CONNECT_STRING = "postgresql://{}:{}@{}:{}/{}".format(
- os.environ["POSTGRES_USER"],
- os.environ["POSTGRES_PASSWORD"],
- os.environ["POSTGRES_HOST"],
- os.environ["POSTGRES_PORT"],
- os.environ["POSTGRES_DBNAME"],
-).lower()
diff --git a/prospector/api/api_test.py b/prospector/api/api_test.py
index f76bbfbc4..03957bc75 100644
--- a/prospector/api/api_test.py
+++ b/prospector/api/api_test.py
@@ -1,5 +1,4 @@
from fastapi.testclient import TestClient
-import pytest
from api.main import app
from datamodel.commit import Commit
@@ -22,13 +21,13 @@ def test_status():
def test_post_preprocessed_commits():
commit_1 = Commit(
repository="https://github.com/apache/dubbo", commit_id="yyy"
- ).__dict__
+ ).as_dict()
commit_2 = Commit(
repository="https://github.com/apache/dubbo", commit_id="zzz"
- ).__dict__
+ ).as_dict()
commit_3 = Commit(
repository="https://github.com/apache/struts", commit_id="bbb"
- ).__dict__
+ ).as_dict()
commits = [commit_1, commit_2, commit_3]
response = client.post("/commits/", json=commits)
assert response.status_code == 200
@@ -43,7 +42,7 @@ def test_get_specific_commit():
assert response.json()[0]["commit_id"] == commit_id
-@pytest.mark.skip(reason="will raise exception")
+# @pytest.mark.skip(reason="will raise exception")
def test_get_commits_by_repository():
repository = "https://github.com/apache/dubbo"
response = client.get("/commits/" + repository)
diff --git a/prospector/api/main.py b/prospector/api/main.py
index 641b57772..87551536d 100644
--- a/prospector/api/main.py
+++ b/prospector/api/main.py
@@ -1,23 +1,12 @@
-# import os
-
import uvicorn
from fastapi import FastAPI
-# from fastapi import Depends
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import HTMLResponse, RedirectResponse
# from .dependencies import oauth2_scheme
from api.routers import jobs, nvd, preprocessed, users
-# from commitdb.postgres import PostgresCommitDB
-
-# from pprint import pprint
-
-
-# db = PostgresCommitDB()
-# db.connect(DB_CONNECT_STRING)
-
api_metadata = [
{"name": "data", "description": "Operations with data used to train ML models."},
{
@@ -72,4 +61,9 @@ async def get_status():
if __name__ == "__main__":
- uvicorn.run(app, host="0.0.0.0", port=80)
+
+ uvicorn.run(
+ app,
+ host="0.0.0.0",
+ port=80,
+ )
diff --git a/prospector/api/routers/jobs.py b/prospector/api/routers/jobs.py
index eb8df622a..4ef0a20c8 100644
--- a/prospector/api/routers/jobs.py
+++ b/prospector/api/routers/jobs.py
@@ -5,11 +5,10 @@
from rq import Connection, Queue
from rq.job import Job
-import log.util
+from log.logger import logger
from api.routers.nvd_feed_update import main
from git.git import do_clone
-_logger = log.util.init_local_logger()
redis_url = os.environ["REDIS_URL"]
@@ -57,7 +56,7 @@ async def get_job(job_id):
queue = Queue()
job = queue.fetch_job(job_id)
if job:
- _logger.info("job {} result: {}".format(job.get_id(), job.result))
+ logger.info("job {} result: {}".format(job.get_id(), job.result))
response_object = {
"job_data": {
"job_id": job.get_id(),
diff --git a/prospector/api/routers/nvd.py b/prospector/api/routers/nvd.py
index 7a1d59345..7d2142ed6 100644
--- a/prospector/api/routers/nvd.py
+++ b/prospector/api/routers/nvd.py
@@ -6,9 +6,7 @@
from fastapi import APIRouter, HTTPException
from fastapi.responses import JSONResponse
-import log.util
-
-_logger = log.util.init_local_logger()
+from log.logger import logger
router = APIRouter(
@@ -25,19 +23,19 @@
@router.get("/vulnerabilities/by-year/{year}")
async def get_vuln_list_by_year(year: str):
- _logger.debug("Requested list of vulnerabilities for " + year)
+ logger.debug("Requested list of vulnerabilities for " + year)
if len(year) != 4 or not year.isdigit():
return JSONResponse([])
data_dir = os.path.join(DATA_PATH, year)
if not os.path.isdir(data_dir):
- _logger.info("No data found for year " + year)
+ logger.info("No data found for year " + year)
raise HTTPException(
status_code=404, detail="No vulnerabilities found for " + year
)
- _logger.debug("Serving data for year " + year)
+ logger.debug("Serving data for year " + year)
vuln_ids = [vid.rstrip(".json") for vid in os.listdir(data_dir)]
results = {"count": len(vuln_ids), "data": vuln_ids}
return JSONResponse(results)
@@ -45,17 +43,17 @@ async def get_vuln_list_by_year(year: str):
@router.get("/vulnerabilities/{vuln_id}")
async def get_vuln_data(vuln_id):
- _logger.debug("Requested data for vulnerability " + vuln_id)
+ logger.debug("Requested data for vulnerability " + vuln_id)
year = vuln_id.split("-")[1]
json_file = os.path.join(DATA_PATH, year, vuln_id.upper() + ".json")
if not os.path.isfile(json_file):
- _logger.info("No file found: " + json_file)
+ logger.info("No file found: " + json_file)
raise HTTPException(
status_code=404, detail=json_file
) # detail="Vulnerability data not found")
- _logger.debug("Serving file: " + json_file)
+ logger.debug("Serving file: " + json_file)
with open(json_file) as f:
data = json.loads(f.read())
@@ -64,7 +62,7 @@ async def get_vuln_data(vuln_id):
@router.get("/status")
async def status():
- _logger.debug("Serving status page")
+ logger.debug("Serving status page")
out = dict()
metadata_file = os.path.join(DATA_PATH, "metadata.json")
if os.path.isfile(metadata_file):
diff --git a/prospector/api/routers/nvd_feed_update.py b/prospector/api/routers/nvd_feed_update.py
index 478ca7b84..ec0292bd6 100644
--- a/prospector/api/routers/nvd_feed_update.py
+++ b/prospector/api/routers/nvd_feed_update.py
@@ -24,9 +24,10 @@
import requests
from tqdm import tqdm
-import log.util
+from log.logger import logger
-_logger = log.util.init_local_logger()
+
+NVD_API_KEY = os.getenv("NVD_API_KEY", "")
# note: The NVD has not data older than 2002
START_FROM_YEAR = os.getenv("CVE_DATA_AS_OF_YEAR", "2002")
@@ -41,22 +42,20 @@ def do_update(quiet=False):
with open(os.path.join(DATA_PATH, "metadata.json"), "r") as f:
last_fetch_metadata = json.load(f)
if not quiet:
- _logger.info("last fetch: " + last_fetch_metadata["sha256"])
+ logger.info("last fetch: " + last_fetch_metadata["sha256"])
except Exception:
last_fetch_metadata["sha256"] = ""
- _logger.info(
+ logger.info(
"Could not read metadata about previous fetches"
" (this might be the first time we fetch data).",
exc_info=True,
)
# read metadata of new data from the NVD site
- url = "https://nvd.nist.gov/feeds/json/cve/{}/nvdcve-{}-modified.meta".format(
- FEED_SCHEMA_VERSION, FEED_SCHEMA_VERSION
- )
- r = requests.get(url)
+ url = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.meta"
+ r = requests.get(url, params={"apiKey": NVD_API_KEY})
if r.status_code != 200:
- _logger.error(
+ logger.error(
"Received status code {} when contacting {}.".format(r.status_code, url)
)
return False
@@ -67,12 +66,12 @@ def do_update(quiet=False):
d_split = d.split(":", 1)
metadata_dict[d_split[0]] = d_split[1].strip()
if not quiet:
- _logger.info("current: " + metadata_dict["sha256"])
+ logger.info("current: " + metadata_dict["sha256"])
# check if the new data is actually new
if last_fetch_metadata["sha256"] == metadata_dict["sha256"]:
if not quiet:
- _logger.info("We already have this update, no new data to fetch.")
+ logger.info("We already have this update, no new data to fetch.")
return False
do_fetch("modified")
@@ -86,30 +85,28 @@ def do_fetch_full(start_from_year=START_FROM_YEAR, quiet=False):
y for y in range(int(start_from_year), int(time.strftime("%Y")) + 1)
]
if not quiet:
- _logger.info("Fetching feeds: " + str(years_to_fetch))
+ logger.info("Fetching feeds: " + str(years_to_fetch))
for y in years_to_fetch:
if not do_fetch(y):
- _logger.error("Could not fetch data for year " + str(y))
+ logger.error("Could not fetch data for year " + str(y))
def do_fetch(what, quiet=True):
"""
the 'what' parameter can be a year or 'recent' or 'modified'
"""
- url = "https://nvd.nist.gov/feeds/json/cve/{}/nvdcve-{}-{}.json.zip".format(
- FEED_SCHEMA_VERSION, FEED_SCHEMA_VERSION, what
- )
- r = requests.get(url)
+ url = f"https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{what}.json.zip"
+ r = requests.get(url, params={"apiKey": NVD_API_KEY})
if r.status_code != 200:
- _logger.error(
+ logger.error(
"Received status code {} when contacting {}.".format(r.status_code, url)
)
return False
with closing(r), zipfile.ZipFile(io.BytesIO(r.content)) as archive:
for f in archive.infolist():
- _logger.info(f.filename)
+ logger.info(f.filename)
data = json.loads(archive.read(f).decode())
if not quiet:
@@ -135,17 +132,17 @@ def need_full(quiet=False):
if os.path.exists(DATA_PATH) and os.path.isdir(DATA_PATH):
if not os.listdir(DATA_PATH):
if not quiet:
- _logger.info("Data folder {} is empty".format(DATA_PATH))
+ logger.info("Data folder {} is empty".format(DATA_PATH))
return True
# Directory exists and is not empty
if not quiet:
- _logger.info("Data folder found at " + DATA_PATH)
+ logger.info("Data folder found at " + DATA_PATH)
return False
# Directory doesn't exist
if not quiet:
- _logger.info("Data folder {} does not exist".format(DATA_PATH))
+ logger.info("Data folder {} does not exist".format(DATA_PATH))
return True
@@ -162,5 +159,5 @@ def main(force, quiet):
do_update(quiet=quiet)
-if __name__ == "__main__":
- plac.call(main)
+# if __name__ == "__main__":
+# plac.call(main)
diff --git a/prospector/api/routers/preprocessed.py b/prospector/api/routers/preprocessed.py
index 53bb0739b..9b09f09ef 100644
--- a/prospector/api/routers/preprocessed.py
+++ b/prospector/api/routers/preprocessed.py
@@ -1,12 +1,9 @@
-import stat
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
from fastapi import APIRouter
from fastapi.responses import JSONResponse
-from api import DB_CONNECT_STRING
from commitdb.postgres import PostgresCommitDB
-from datamodel.commit import Commit
router = APIRouter(
prefix="/commits",
@@ -22,22 +19,21 @@ async def get_commits(
commit_id: Optional[str] = None,
):
db = PostgresCommitDB()
- db.connect(DB_CONNECT_STRING)
- # use case: if a particular commit is queried, details should be returned
+ db.connect()
data = db.lookup(repository_url, commit_id)
- if not len(data):
+ if len(data) == 0:
return JSONResponse(status_code=404, content={"message": "Not found"})
- return JSONResponse([d.dict() for d in data])
+ return JSONResponse(data)
# -----------------------------------------------------------------------------
@router.post("/")
-async def upload_preprocessed_commit(payload: List[Commit]):
+async def upload_preprocessed_commit(payload: List[Dict[str, Any]]):
db = PostgresCommitDB()
- db.connect(DB_CONNECT_STRING)
+ db.connect()
for commit in payload:
db.save(commit)
diff --git a/prospector/api/routers/users.py b/prospector/api/routers/users.py
index f6fdb70be..fc60d6997 100644
--- a/prospector/api/routers/users.py
+++ b/prospector/api/routers/users.py
@@ -1,6 +1,9 @@
from fastapi import APIRouter, Depends, HTTPException
from fastapi.security import OAuth2PasswordRequestForm
+# from http import HTTPStatus
+
+
from ..dependencies import (
User,
UserInDB,
diff --git a/prospector/client/cli/console.py b/prospector/client/cli/console.py
index ce112e07c..83efaaa36 100644
--- a/prospector/client/cli/console.py
+++ b/prospector/client/cli/console.py
@@ -18,14 +18,15 @@ def __init__(self, message: str):
self.status: MessageStatus = MessageStatus.OK
def __enter__(self):
- print(f"{Fore.LIGHTWHITE_EX}{self._message}{Style.RESET_ALL} ...")
+ print(f"{Fore.LIGHTWHITE_EX}{self._message}{Style.RESET_ALL}", end=" ")
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_val is not None:
self.status = MessageStatus.ERROR
print(
- f"{ConsoleWriter.indent}[{self.status.value}{self.status.name}{Style.RESET_ALL}]"
+ f"{ConsoleWriter.indent}[{self.status.value}{self.status.name}{Style.RESET_ALL}]",
+ end="\n",
)
if exc_val is not None:
raise exc_val
@@ -34,6 +35,6 @@ def set_status(self, status: MessageStatus):
self.status = status
def print(self, note: str, status: Optional[MessageStatus] = None):
- print(f"{ConsoleWriter.indent}{Fore.WHITE}{note}")
+ print(f"{ConsoleWriter.indent}{Fore.WHITE}{note}", end="\n")
if isinstance(status, MessageStatus):
self.set_status(status)
diff --git a/prospector/client/cli/console_report.py b/prospector/client/cli/console_report.py
deleted file mode 100644
index b426dabd4..000000000
--- a/prospector/client/cli/console_report.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from datamodel.advisory import AdvisoryRecord
-from datamodel.commit import Commit
-
-
-def report_on_console(
- results: "list[Commit]", advisory_record: AdvisoryRecord, verbose=False
-):
- def format_annotations(commit: Commit) -> str:
- out = ""
- if verbose:
- for tag in commit.annotations:
- out += " - [{}] {}".format(tag, commit.annotations[tag])
- else:
- out = ",".join(commit.annotations.keys())
-
- return out
-
- print("-" * 80)
- print("Rule filtered results")
- print("-" * 80)
- count = 0
- for commit in results:
- count += 1
- print(
- f"\n----------\n{commit.repository}/commit/{commit.commit_id}\n"
- + "\n".join(commit.changed_files)
- + f"{commit.message}\n{format_annotations(commit)}"
- )
-
- print(f"Found {count} candidates\nAdvisory record\n{advisory_record}")
diff --git a/prospector/client/cli/html_report.py b/prospector/client/cli/html_report.py
deleted file mode 100644
index 526f943f5..000000000
--- a/prospector/client/cli/html_report.py
+++ /dev/null
@@ -1,42 +0,0 @@
-import os
-from typing import List
-
-import jinja2
-
-import log.util
-from datamodel.advisory import AdvisoryRecord
-from datamodel.commit import Commit
-from stats.execution import execution_statistics
-
-_logger = log.util.init_local_logger()
-
-
-def report_as_html(
- results: List[Commit],
- advisory_record: AdvisoryRecord,
- filename: str = "prospector-report.html",
- statistics=None,
-):
- annotations_count = {}
- annotation: Commit
- for commit in results:
- for annotation in commit.annotations.keys():
- annotations_count[annotation] = annotations_count.get(annotation, 0) + 1
-
- _logger.info("Writing results to " + filename)
- environment = jinja2.Environment(
- loader=jinja2.FileSystemLoader(os.path.join("client", "cli", "templates")),
- autoescape=jinja2.select_autoescape(),
- )
- template = environment.get_template("results.html")
- with open(filename, "w", encoding="utf8") as html_file:
- for content in template.generate(
- candidates=results,
- present_annotations=annotations_count,
- advisory_record=advisory_record,
- execution_statistics=(
- execution_statistics if statistics is None else statistics
- ).as_html_ul(),
- ):
- html_file.write(content)
- return filename
diff --git a/prospector/client/cli/json_report.py b/prospector/client/cli/json_report.py
deleted file mode 100644
index 6ad9759a6..000000000
--- a/prospector/client/cli/json_report.py
+++ /dev/null
@@ -1,30 +0,0 @@
-import json
-
-import log.util
-from datamodel.advisory import AdvisoryRecord
-from datamodel.commit import Commit
-
-_logger = log.util.init_local_logger()
-
-
-class SetEncoder(json.JSONEncoder):
- def default(self, obj):
- if isinstance(obj, set):
- return list(obj)
- return json.JSONEncoder.default(self, obj)
-
-
-def report_as_json(
- results: "list[Commit]",
- advisory_record: AdvisoryRecord,
- filename: str = "prospector-report.json",
-):
- # Need to convert the Sets to Lists for JSON serialization
- data = {
- "advisory_record": advisory_record.dict(),
- "commits": [r.dict() for r in results],
- }
- _logger.info("Writing results to " + filename)
- with open(filename, "w", encoding="utf8") as json_file:
- json.dump(data, json_file, ensure_ascii=True, indent=4, cls=SetEncoder)
- return filename
diff --git a/prospector/client/cli/main.py b/prospector/client/cli/main.py
index 83fbfe85c..f5edae948 100644
--- a/prospector/client/cli/main.py
+++ b/prospector/client/cli/main.py
@@ -1,6 +1,4 @@
#!/usr/bin/python3
-
-# from advisory_processor.advisory_processor import AdvisoryProcessor
import argparse
import configparser
import logging
@@ -9,30 +7,33 @@
import sys
from pathlib import Path
from typing import Any, Dict
+from dotenv import load_dotenv
+
path_root = os.getcwd()
if path_root not in sys.path:
sys.path.append(path_root)
-import log # noqa: E402
+# Loading .env file before doint anything else
+load_dotenv()
+
+# Load logger before doing anything else
+from log.logger import logger, get_level, pretty_log # noqa: E402
from client.cli.console import ConsoleWriter, MessageStatus # noqa: E402
-from client.cli.console_report import report_on_console # noqa: E402
-from client.cli.html_report import report_as_html # noqa: E402
-from client.cli.json_report import report_as_json # noqa: E402
+from client.cli.report import as_json, as_html, report_on_console # noqa: E402
from client.cli.prospector_client import ( # noqa: E402
MAX_CANDIDATES, # noqa: E402
TIME_LIMIT_AFTER, # noqa: E402
TIME_LIMIT_BEFORE, # noqa: E402
+ DEFAULT_BACKEND, # noqa: E402
prospector, # noqa: E402
)
from git.git import GIT_CACHE # noqa: E402
from stats.execution import execution_statistics # noqa: E402
from util.http import ping_backend # noqa: E402
-_logger = log.util.init_local_logger()
-DEFAULT_BACKEND = "http://localhost:8000"
# VERSION = '0.1.0'
# SCRIPT_PATH=os.path.dirname(os.path.realpath(__file__))
# print(SCRIPT_PATH)
@@ -49,6 +50,19 @@ def parseArguments(args):
parser.add_argument("--repository", default="", type=str, help="Git repository")
+ parser.add_argument(
+ "--find-twin",
+ default="",
+ type=str,
+ help="Lookup for a twin of the specified commit",
+ )
+
+ parser.add_argument(
+ "--preprocess-only",
+ action="store_true",
+ help="Only preprocess the commits for the specified repository",
+ )
+
parser.add_argument(
"--pub-date", default="", help="Publication date of the advisory"
)
@@ -85,7 +99,7 @@ def parseArguments(args):
parser.add_argument(
"--filter-extensions",
- default="java",
+ default="",
type=str,
help="Filter out commits that do not modify at least one file with this extension",
)
@@ -128,7 +142,7 @@ def parseArguments(args):
parser.add_argument(
"--report",
default="html",
- choices=["html", "json", "console"],
+ choices=["html", "json", "console", "allfiles"],
type=str,
help="Format of the report (options: console, json, html)",
)
@@ -154,7 +168,7 @@ def parseArguments(args):
help="Set the logging level",
)
- return parser.parse_args(args[1:])
+ return parser.parse_args()
def getConfiguration(customConfigFile=None):
@@ -181,7 +195,7 @@ def getConfiguration(customConfigFile=None):
else:
return None
- _logger.info("Loading configuration from " + configFile)
+ logger.info("Loading configuration from " + configFile)
config.read(configFile)
return parse_config(config)
@@ -200,17 +214,15 @@ def parse_config(configuration: configparser.ConfigParser) -> Dict[str, Any]:
def main(argv): # noqa: C901
with ConsoleWriter("Initialization") as console:
- args = parseArguments(argv) # print(args)
+ args = parseArguments(argv)
if args.log_level:
- log.config.level = getattr(logging, args.log_level)
+ logger.setLevel(args.log_level)
- _logger.info(
- f"global log level is set to {logging.getLevelName(log.config.level)}"
- )
+ logger.info(f"global log level is set to {get_level(string=True)}")
if args.vulnerability_id is None:
- _logger.error("No vulnerability id was specified. Cannot proceed.")
+ logger.error("No vulnerability id was specified. Cannot proceed.")
console.print(
"No vulnerability id was specified. Cannot proceed.",
status=MessageStatus.ERROR,
@@ -220,10 +232,11 @@ def main(argv): # noqa: C901
configuration = getConfiguration(args.conf)
if configuration is None:
- _logger.error("Invalid configuration, exiting.")
+ logger.error("Invalid configuration, exiting.")
return False
report = args.report or configuration.get("report")
+ report_filename = args.report_filename or configuration.get("report_filename")
nvd_rest_endpoint = configuration.get("nvd_rest_endpoint", "") # default ???
@@ -232,7 +245,7 @@ def main(argv): # noqa: C901
use_backend = args.use_backend
if args.ping:
- return ping_backend(backend, log.config.level < logging.INFO)
+ return ping_backend(backend, get_level() < logging.INFO)
vulnerability_id = args.vulnerability_id
repository_url = args.repository
@@ -274,12 +287,12 @@ def main(argv): # noqa: C901
git_cache = configuration.get("git_cache", git_cache)
- _logger.debug("Using the following configuration:")
- _logger.pretty_log(configuration)
+ logger.debug("Using the following configuration:")
+ pretty_log(logger, configuration)
- _logger.debug("Vulnerability ID: " + vulnerability_id)
- _logger.debug("time-limit before: " + str(time_limit_before))
- _logger.debug("time-limit after: " + str(time_limit_after))
+ logger.debug("Vulnerability ID: " + vulnerability_id)
+ logger.debug("time-limit before: " + str(time_limit_before))
+ logger.debug("time-limit after: " + str(time_limit_after))
active_rules = ["ALL"]
@@ -305,27 +318,29 @@ def main(argv): # noqa: C901
rules=active_rules,
)
+ if args.preprocess_only:
+ return True
+
with ConsoleWriter("Generating report") as console:
report_file = None
if report == "console":
- report_on_console(results, advisory_record, log.config.level < logging.INFO)
+ report_on_console(results, advisory_record, get_level() < logging.INFO)
elif report == "json":
- report_file = report_as_json(
- results, advisory_record, args.report_filename + ".json"
- )
+ report_file = as_json(results, advisory_record, report_filename)
elif report == "html":
- report_file = report_as_html(
- results, advisory_record, args.report_filename + ".html"
- )
+ report_file = as_html(results, advisory_record, report_filename)
+ elif report == "allfiles":
+ as_json(results, advisory_record, report_filename)
+ as_html(results, advisory_record, report_filename)
else:
- _logger.warning("Invalid report type specified, using 'console'")
+ logger.warning("Invalid report type specified, using 'console'")
console.set_status(MessageStatus.WARNING)
console.print(
f"{report} is not a valid report type, 'console' will be used instead",
)
- report_on_console(results, advisory_record, log.config.level < logging.INFO)
+ report_on_console(results, advisory_record, get_level() < logging.INFO)
- _logger.info("\n" + execution_statistics.generate_console_tree())
+ logger.info("\n" + execution_statistics.generate_console_tree())
execution_time = execution_statistics["core"]["execution time"][0]
console.print(f"Execution time: {execution_time:.4f} sec")
if report_file:
@@ -334,7 +349,7 @@ def main(argv): # noqa: C901
def signal_handler(signal, frame):
- _logger.info("Exited with keyboard interrupt")
+ logger.info("Exited with keyboard interrupt")
sys.exit(0)
diff --git a/prospector/client/cli/prospector_client.py b/prospector/client/cli/prospector_client.py
index f76012c43..076727cbb 100644
--- a/prospector/client/cli/prospector_client.py
+++ b/prospector/client/cli/prospector_client.py
@@ -1,22 +1,20 @@
+from datetime import datetime
import logging
import sys
-from datetime import datetime
-from typing import List, Set, Tuple
+from typing import Dict, List, Set, Tuple
import requests
from tqdm import tqdm
-
-import log
from client.cli.console import ConsoleWriter, MessageStatus
from datamodel.advisory import AdvisoryRecord, build_advisory_record
from datamodel.commit import Commit, apply_ranking, make_from_raw_commit
from filtering.filter import filter_commits
from git.git import GIT_CACHE, Git
+from git.raw_commit import RawCommit
from git.version_to_tag import get_tag_for_version
-from log.util import init_local_logger
-from rules import apply_rules
+from log.logger import logger, pretty_log, get_level
+from rules.rules import apply_rules
-# from util.profile import profile
from stats.execution import (
Counter,
ExecutionTimer,
@@ -24,14 +22,14 @@
measure_execution_time,
)
-_logger = init_local_logger()
-
SECS_PER_DAY = 86400
TIME_LIMIT_BEFORE = 3 * 365 * SECS_PER_DAY
TIME_LIMIT_AFTER = 180 * SECS_PER_DAY
-MAX_CANDIDATES = 1000
+MAX_CANDIDATES = 2000
+DEFAULT_BACKEND = "http://localhost:8000"
+
core_statistics = execution_statistics.sub_collection("core")
@@ -53,17 +51,17 @@ def prospector( # noqa: C901
use_nvd: bool = True,
nvd_rest_endpoint: str = "",
fetch_references: bool = False,
- backend_address: str = "",
+ backend_address: str = DEFAULT_BACKEND,
use_backend: str = "always",
git_cache: str = GIT_CACHE,
limit_candidates: int = MAX_CANDIDATES,
rules: List[str] = ["ALL"],
) -> Tuple[List[Commit], AdvisoryRecord]:
- _logger.debug("begin main commit and CVE processing")
+ logger.debug("begin main commit and CVE processing")
# construct an advisory record
- with ConsoleWriter("Processing advisory"):
+ with ConsoleWriter("Processing advisory") as _:
advisory_record = build_advisory_record(
vulnerability_id,
repository_url,
@@ -77,180 +75,155 @@ def prospector( # noqa: C901
filter_extensions,
)
- with ConsoleWriter("Obtaining initial set of candidates") as writer:
-
- # obtain a repository object
- repository = Git(repository_url, git_cache)
-
- # retrieve of commit candidates
- candidates = get_candidates(
- advisory_record,
- repository,
- tag_interval,
- version_interval,
- time_limit_before,
- time_limit_after,
- filter_extensions[0],
- )
- _logger.debug(f"Collected {len(candidates)} candidates")
-
- if len(candidates) > limit_candidates:
- _logger.error(
- "Number of candidates exceeds %d, aborting." % limit_candidates
- )
- _logger.error(
- "Possible cause: the backend might be unreachable or otherwise unable to provide details about the advisory."
- )
- writer.print(
- f"Found {len(candidates)} candidates, too many to proceed.",
- status=MessageStatus.ERROR,
- )
- writer.print("Please try running the tool again.")
- sys.exit(-1)
-
- writer.print(f"Found {len(candidates)} candidates")
+ # obtain a repository object
+ repository = Git(repository_url, git_cache)
+
+ # retrieve of commit candidates
+ candidates = get_candidates(
+ advisory_record,
+ repository,
+ tag_interval,
+ version_interval,
+ time_limit_before,
+ time_limit_after,
+ limit_candidates,
+ )
- # -------------------------------------------------------------------------
- # commit preprocessing
- # -------------------------------------------------------------------------
with ExecutionTimer(
- core_statistics.sub_collection(name="commit preprocessing")
+ core_statistics.sub_collection("commit preprocessing")
) as timer:
with ConsoleWriter("Preprocessing commits") as writer:
try:
if use_backend != "never":
missing, preprocessed_commits = retrieve_preprocessed_commits(
- repository_url, backend_address, candidates
+ repository_url,
+ backend_address,
+ candidates,
)
except requests.exceptions.ConnectionError:
- print("Backend not reachable", end="")
- _logger.error(
+ logger.error(
"Backend not reachable",
- exc_info=log.config.level < logging.WARNING,
+ exc_info=get_level() < logging.WARNING,
)
if use_backend == "always":
- print(": aborting")
+ print("Backend not reachable: aborting")
sys.exit(0)
- print(": continuing without backend")
- finally:
- # If missing is not initialized and we are here, we initialize it
- if "missing" not in locals():
- missing = candidates
- preprocessed_commits = []
+ print("Backend not reachable: continuing")
+
+ if "missing" not in locals():
+ missing = list(candidates.values())
+ preprocessed_commits: List[Commit] = list()
pbar = tqdm(missing, desc="Preprocessing commits", unit="commit")
with Counter(
- timer.collection.sub_collection(name="commit preprocessing")
+ timer.collection.sub_collection("commit preprocessing")
) as counter:
counter.initialize("preprocessed commits", unit="commit")
- for commit_id in pbar:
+ # Now pbar has Raw commits inside so we can skip the "get_commit" call
+ for raw_commit in pbar:
counter.increment("preprocessed commits")
- preprocessed_commits.append(
- make_from_raw_commit(repository.get_commit(commit_id))
- )
+ # TODO: here we need to check twins with the commit not already in the backend and update everything
+ preprocessed_commits.append(make_from_raw_commit(raw_commit))
+
+ # Cleanup candidates to save memory
+ del candidates
- _logger.pretty_log(advisory_record)
- _logger.debug(f"preprocessed {len(preprocessed_commits)} commits")
- payload = [c.__dict__ for c in preprocessed_commits]
-
- # -------------------------------------------------------------------------
- # save preprocessed commits to backend
- # -------------------------------------------------------------------------
-
- if (
- len(payload) > 0 and use_backend != "never"
- ): # and len(missing) > 0: # len(missing) is useless
- with ExecutionTimer(
- core_statistics.sub_collection(name="save preprocessed commits to backend")
- ):
- save_preprocessed_commits(backend_address, payload)
+ pretty_log(logger, advisory_record)
+ logger.debug(
+ f"preprocessed {len(preprocessed_commits)} commits are only composed of test files"
+ )
+ payload = [c.to_dict() for c in preprocessed_commits]
+
+ if len(payload) > 0 and use_backend != "never":
+ save_preprocessed_commits(backend_address, payload)
else:
- _logger.warning("No preprocessed commits to send to backend.")
+ logger.warning("Preprocessed commits are not being sent to backend")
- # -------------------------------------------------------------------------
# filter commits
- # -------------------------------------------------------------------------
+ preprocessed_commits = filter(preprocessed_commits)
+
+ # apply rules and rank candidates
+ ranked_candidates = evaluate_commits(preprocessed_commits, advisory_record, rules)
+
+ return ranked_candidates, advisory_record
+
+
+def filter(commits: List[Commit]) -> List[Commit]:
with ConsoleWriter("Candidate filtering") as console:
- candidate_count = len(preprocessed_commits)
- console.print(f"Filtering {candidate_count} candidates")
-
- preprocessed_commits, rejected = filter_commits(preprocessed_commits)
- if len(rejected) > 0:
- console.print(f"Dropped {len(rejected)} candidates")
- # Maybe print reasons for rejection? PUT THEM IN A FILE?
- # console.print(f"{rejected}")
-
- # -------------------------------------------------------------------------
- # analyze candidates by applying rules and rank them
- # -------------------------------------------------------------------------
- with ExecutionTimer(
- core_statistics.sub_collection(name="analyze candidates")
- ) as timer:
+ commits, rejected = filter_commits(commits)
+ if rejected > 0:
+ console.print(f"Dropped {rejected} candidates")
+ return commits
+
+
+def evaluate_commits(commits: List[Commit], advisory: AdvisoryRecord, rules: List[str]):
+ with ExecutionTimer(core_statistics.sub_collection("candidates analysis")):
with ConsoleWriter("Applying rules"):
- annotated_candidates = apply_rules(
- preprocessed_commits, advisory_record, rules=rules
- )
+ ranked_commits = apply_ranking(apply_rules(commits, advisory, rules=rules))
- annotated_candidates = apply_ranking(annotated_candidates)
+ return ranked_commits
- return annotated_candidates, advisory_record
+def retrieve_preprocessed_commits(
+ repository_url: str, backend_address: str, candidates: Dict[str, RawCommit]
+) -> Tuple[List[RawCommit], List[Commit]]:
+ retrieved_commits: List[dict] = list()
+ missing: List[RawCommit] = list()
-def retrieve_preprocessed_commits(repository_url, backend_address, candidates):
- retrieved_commits = dict()
- missing = []
+ responses = list()
+ for i in range(0, len(candidates), 500):
+ args = list(candidates.keys())[i : i + 500]
+ r = requests.get(
+ f"{backend_address}/commits/{repository_url}?commit_id={','.join(args)}"
+ )
+ if r.status_code != 200:
+ logger.info("One or more commits are not in the backend")
+ break # return list(candidates.values()), list()
+ responses.append(r.json())
- # This will raise exception if backend is not reachable
- r = requests.get(
- f"{backend_address}/commits/{repository_url}?commit_id={','.join(candidates)}"
- )
+ retrieved_commits = [commit for response in responses for commit in response]
- _logger.debug(f"The backend returned status {r.status_code}")
- if r.status_code != 200:
- _logger.info("Preprocessed commits not found in the backend")
- missing = candidates
- else:
- retrieved_commits = r.json()
- _logger.info(f"Found {len(retrieved_commits)} preprocessed commits")
- if len(retrieved_commits) != len(candidates):
- missing = list(
- set(candidates).difference(rc["commit_id"] for rc in retrieved_commits)
+ logger.info(f"Found {len(retrieved_commits)} preprocessed commits")
+
+ if len(retrieved_commits) != len(candidates):
+ missing = [
+ candidates[c]
+ for c in set(candidates.keys()).difference(
+ rc["commit_id"] for rc in retrieved_commits
)
- _logger.error(f"Missing {len(missing)} commits")
+ ]
- preprocessed_commits: "list[Commit]" = []
- for idx, commit in enumerate(retrieved_commits):
- if len(retrieved_commits) + len(missing) == len(
- candidates
- ): # Parsing like this is possible because the backend handles the major work
- preprocessed_commits.append(Commit.parse_obj(commit))
- else:
- missing.append(candidates[idx])
- return missing, preprocessed_commits
+ logger.error(f"Missing {len(missing)} commits")
+
+ return missing, [Commit.parse_obj(rc) for rc in retrieved_commits]
def save_preprocessed_commits(backend_address, payload):
- with ConsoleWriter("Saving preprocessed commits to backend") as writer:
- _logger.debug("Sending preprocessing commits to backend...")
- try:
- r = requests.post(backend_address + "/commits/", json=payload)
- _logger.debug(
- "Saving to backend completed (status code: %d)" % r.status_code
- )
- except requests.exceptions.ConnectionError:
- _logger.error(
- "Could not reach backend, is it running?"
- "The result of commit pre-processing will not be saved."
- "Continuing anyway.....",
- exc_info=log.config.level < logging.WARNING,
- )
- writer.print(
- "Could not save preprocessed commits to backend",
- status=MessageStatus.WARNING,
- )
+ with ExecutionTimer(core_statistics.sub_collection(name="save commits to backend")):
+ with ConsoleWriter("Saving preprocessed commits to backend") as writer:
+ logger.debug("Sending preprocessing commits to backend...")
+ try:
+ r = requests.post(
+ backend_address + "/commits/",
+ json=payload,
+ headers={"Content-type": "application/json"},
+ )
+ logger.debug(
+ f"Saving to backend completed (status code: {r.status_code})"
+ )
+ except requests.exceptions.ConnectionError:
+ logger.error(
+ "Could not reach backend, is it running?"
+ "The result of commit pre-processing will not be saved."
+ "Continuing anyway.....",
+ exc_info=get_level() < logging.WARNING,
+ )
+ writer.print(
+ "Could not save preprocessed commits to backend",
+ status=MessageStatus.WARNING,
+ )
-# TODO: Cleanup many parameters should be recovered from the advisory record object
def get_candidates(
advisory_record: AdvisoryRecord,
repository: Git,
@@ -258,32 +231,30 @@ def get_candidates(
version_interval: str,
time_limit_before: int,
time_limit_after: int,
- filter_extensions: str,
-) -> List[str]:
+ limit_candidates: int,
+):
with ExecutionTimer(
core_statistics.sub_collection(name="retrieval of commit candidates")
):
- with ConsoleWriter("Git repository cloning"):
- _logger.info(
- f"Downloading repository {repository._url} in {repository._path}"
- )
+ with ConsoleWriter("Git repository cloning") as _:
+ logger.info(f"Downloading repository {repository.url} in {repository.path}")
repository.clone()
tags = repository.get_tags()
- _logger.debug(f"Found tags: {tags}")
- _logger.info(f"Done retrieving {repository._url}")
+ logger.debug(f"Found tags: {tags}")
+ logger.info(f"Done retrieving {repository.url}")
- with ConsoleWriter("Candidate commit retrieval"):
+ with ConsoleWriter("Candidate commit retrieval") as writer:
prev_tag = None
- following_tag = None
+ next_tag = None
if tag_interval != "":
- prev_tag, following_tag = tag_interval.split(":")
+ prev_tag, next_tag = tag_interval.split(":")
elif version_interval != "":
vuln_version, fixed_version = version_interval.split(":")
prev_tag = get_tag_for_version(tags, vuln_version)[0]
- following_tag = get_tag_for_version(tags, fixed_version)[0]
+ next_tag = get_tag_for_version(tags, fixed_version)[0]
since = None
until = None
@@ -291,15 +262,27 @@ def get_candidates(
since = advisory_record.published_timestamp - time_limit_before
until = advisory_record.published_timestamp + time_limit_after
# Here i need to strip the github tags of useless stuff
- candidates = repository.get_commits(
+ # This is now a list of raw commits
+ # TODO: get_commits replaced for now
+ candidates = repository.create_commits(
since=since,
until=until,
- ancestors_of=following_tag,
+ ancestors_of=next_tag,
exclude_ancestors_of=prev_tag,
- filter_files=filter_extensions,
)
core_statistics.record("candidates", len(candidates), unit="commits")
- _logger.info("Found %d candidates" % len(candidates))
+ logger.info("Found %d candidates" % len(candidates))
+ writer.print(f"Found {len(candidates)} candidates")
+
+ if len(candidates) > limit_candidates:
+ logger.error(f"Number of candidates exceeds {limit_candidates}, aborting.")
+
+ writer.print(
+ f"Found {len(candidates)} candidates, too many to proceed.",
+ status=MessageStatus.ERROR,
+ )
+ writer.print("Please try running the tool again.")
+ sys.exit(-1)
return candidates
diff --git a/prospector/client/cli/prospector_client_test.py b/prospector/client/cli/prospector_client_test.py
index 2c4c4b0b8..3f59d8742 100644
--- a/prospector/client/cli/prospector_client_test.py
+++ b/prospector/client/cli/prospector_client_test.py
@@ -1,7 +1,7 @@
+import subprocess
+import os
import pytest
-from api import DB_CONNECT_STRING
-from client.cli.prospector_client import build_advisory_record
from commitdb.postgres import PostgresCommitDB
from stats.execution import execution_statistics
@@ -22,25 +22,25 @@
@pytest.fixture
def setupdb():
db = PostgresCommitDB()
- db.connect(DB_CONNECT_STRING)
+ db.connect()
db.reset()
return db
-def test_main_runonce(setupdb):
- db = setupdb
- db.connect(DB_CONNECT_STRING)
+@pytest.mark.skip(reason="not implemented yet")
+def test_main_runonce(setupdb: PostgresCommitDB):
args = [
- "PROGRAM_NAME",
+ "python",
+ "main.py",
"CVE-2019-11278",
"--repository",
"https://github.com/cloudfoundry/uaa",
"--tag-interval=v74.0.0:v74.1.0",
"--use-backend=optional",
]
- execution_statistics.drop_all()
- main(args)
- db.reset()
+ subprocess.run(args)
+
+ setupdb.reset()
# def test_main_runtwice(setupdb):
diff --git a/prospector/client/cli/report.py b/prospector/client/cli/report.py
new file mode 100644
index 000000000..e40ef94a0
--- /dev/null
+++ b/prospector/client/cli/report.py
@@ -0,0 +1,103 @@
+import json
+import os
+import jinja2
+from typing import List
+from log.logger import logger
+from datamodel.advisory import AdvisoryRecord
+from datamodel.commit import Commit
+from pathlib import Path
+from stats.execution import execution_statistics
+
+
+# Handles Set setialization
+class SetEncoder(json.JSONEncoder):
+ def default(self, obj):
+ if isinstance(obj, set):
+ return list(obj)
+ return json.JSONEncoder.default(self, obj)
+
+
+def as_json(
+ results: List[Commit],
+ advisory_record: AdvisoryRecord,
+ filename: str = "prospector-report.json",
+):
+ fn = filename if filename.endswith(".json") else f"{filename}.json"
+
+ data = {
+ "advisory_record": advisory_record.dict(),
+ "commits": [r.as_dict(no_hash=True, no_rules=False) for r in results],
+ }
+ logger.info(f"Writing results to {fn}")
+ file = Path(fn)
+ file.parent.mkdir(parents=True, exist_ok=True)
+ with open(fn, "w", encoding="utf8") as json_file:
+ json.dump(data, json_file, ensure_ascii=True, indent=4, cls=SetEncoder)
+ return fn
+
+
+def as_html(
+ results: List[Commit],
+ advisory_record: AdvisoryRecord,
+ filename: str = "prospector-report.html",
+ statistics=None,
+):
+ fn = filename if filename.endswith(".html") else f"{filename}.html"
+
+ annotations_count = {}
+ # annotation: Commit
+ # Match number per rules
+ for commit in results:
+ for rule in commit.matched_rules:
+ id = rule.get("id")
+ annotations_count[id] = annotations_count.get(id, 0) + 1
+ # for annotation in commit.annotations.keys():
+ # annotations_count[annotation] = annotations_count.get(annotation, 0) + 1
+
+ logger.info(f"Writing results to {fn}")
+ environment = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(os.path.join("client", "cli", "templates")),
+ autoescape=jinja2.select_autoescape(),
+ )
+ template = environment.get_template("results.html")
+ file = Path(fn)
+ file.parent.mkdir(parents=True, exist_ok=True)
+ with open(fn, "w", encoding="utf8") as html_file:
+ for content in template.generate(
+ candidates=results,
+ present_annotations=annotations_count,
+ advisory_record=advisory_record,
+ execution_statistics=(
+ execution_statistics if statistics is None else statistics
+ ).as_html_ul(),
+ ):
+ html_file.write(content)
+ return fn
+
+
+def report_on_console(
+ results: List[Commit], advisory_record: AdvisoryRecord, verbose=False
+):
+ def format_annotations(commit: Commit) -> str:
+ out = ""
+ if verbose:
+ for tag in commit.annotations:
+ out += " - [{}] {}".format(tag, commit.annotations[tag])
+ else:
+ out = ",".join(commit.annotations.keys())
+
+ return out
+
+ print("-" * 80)
+ print("Rule filtered results")
+ print("-" * 80)
+ count = 0
+ for commit in results:
+ count += 1
+ print(
+ f"\n----------\n{commit.repository}/commit/{commit.commit_id}\n"
+ + "\n".join(commit.changed_files)
+ + f"{commit.message}\n{format_annotations(commit)}"
+ )
+
+ print(f"Found {count} candidates\nAdvisory record\n{advisory_record}")
diff --git a/prospector/client/cli/html_report_test.py b/prospector/client/cli/report_test.py
similarity index 81%
rename from prospector/client/cli/html_report_test.py
rename to prospector/client/cli/report_test.py
index 6a8151ad8..a8d35926b 100644
--- a/prospector/client/cli/html_report_test.py
+++ b/prospector/client/cli/report_test.py
@@ -2,7 +2,7 @@
import os.path
from random import randint
-from client.cli.html_report import report_as_html
+from client.cli.report import as_html, as_json
from datamodel.advisory import AdvisoryRecord
from datamodel.commit import Commit
from util.sample_data_generation import ( # random_list_of_url,
@@ -11,7 +11,6 @@
random_dict_of_strs,
random_list_of_cve,
random_dict_of_github_issue_ids,
- random_list_of_hunks,
random_dict_of_jira_refs,
random_list_of_path,
random_list_of_strs,
@@ -29,7 +28,7 @@ def test_report_generation():
repository=random_url(4),
message=" ".join(random_list_of_strs(100)),
timestamp=randint(0, 100000),
- hunks=random_list_of_hunks(1000, 42),
+ hunks=randint(1, 50),
diff=random_list_of_strs(200),
changed_files=random_list_of_path(4, 42),
message_reference_content=random_list_of_strs(42),
@@ -59,10 +58,14 @@ def test_report_generation():
keywords=tuple(random_list_of_strs(42)),
)
- filename = "test_report.html"
- if os.path.isfile(filename):
- os.remove(filename)
- generated_report = report_as_html(
- candidates, advisory, filename, statistics=sample_statistics()
+ if os.path.isfile("test_report.html"):
+ os.remove("test_report.html")
+ if os.path.isfile("test_report.json"):
+ os.remove("test_report.json")
+ html = as_html(
+ candidates, advisory, "test_report.html", statistics=sample_statistics()
)
- assert os.path.isfile(generated_report)
+ json = as_json(candidates, advisory, "test_report.json")
+
+ assert os.path.isfile(html)
+ assert os.path.isfile(json)
diff --git a/prospector/client/cli/templates/card/annotations_block.html b/prospector/client/cli/templates/card/annotations_block.html
deleted file mode 100644
index 36d64b9c4..000000000
--- a/prospector/client/cli/templates/card/annotations_block.html
+++ /dev/null
@@ -1,15 +0,0 @@
-{% extends "titled_block.html" %}
-{% set title = "Annotations" %}
-{% set icon = "fas fa-bullhorn" %}
-{% block body %}
-
- {% for annotation, comment in annotated_commit.annotations.items() | sort %}
-
- {{ comment }}
-
- {{ annotation }}
-
-
- {% endfor %}
-
-{% endblock %}
diff --git a/prospector/client/cli/templates/card/commit_header.html b/prospector/client/cli/templates/card/commit_header.html
index f5874217b..5a6bae0ea 100644
--- a/prospector/client/cli/templates/card/commit_header.html
+++ b/prospector/client/cli/templates/card/commit_header.html
@@ -12,8 +12,8 @@
{{ annotation }}
+ {% for rule in annotated_commit.matched_rules %}
+ {{ rule.id }}
{% endfor %}
diff --git a/prospector/client/cli/templates/card/commit_title_block.html b/prospector/client/cli/templates/card/commit_title_block.html
index b37d9ef30..37f0b7a3a 100644
--- a/prospector/client/cli/templates/card/commit_title_block.html
+++ b/prospector/client/cli/templates/card/commit_title_block.html
@@ -2,10 +2,11 @@
- Open Commit (assume GitHub-like API)
+ {{
+ annotated_commit.repository }}/commit/{{ annotated_commit.commit_id }}
{% else %}
Open Repository (unknown
API)
diff --git a/prospector/client/cli/templates/card/matched_rules_block.html b/prospector/client/cli/templates/card/matched_rules_block.html
new file mode 100644
index 000000000..d783729dd
--- /dev/null
+++ b/prospector/client/cli/templates/card/matched_rules_block.html
@@ -0,0 +1,16 @@
+{% extends "titled_block.html" %}
+{% set title = "Matched rules" %}
+{% set icon = "fas fa-bullhorn" %}
+{% block body %}
+
+ {% for rule in annotated_commit.matched_rules %}
+
+ {{ rule.message }}
+
+ {{ rule.id }}
+
+
+ {% endfor %}
+
+{% endblock %}
\ No newline at end of file
diff --git a/prospector/client/cli/templates/card/mentioned_cves_block.html b/prospector/client/cli/templates/card/mentioned_cves_block.html
index f427b045f..31dd16e27 100644
--- a/prospector/client/cli/templates/card/mentioned_cves_block.html
+++ b/prospector/client/cli/templates/card/mentioned_cves_block.html
@@ -1,12 +1,12 @@
{% extends "titled_block.html" %}
-{% set title = "Other CVEs mentioned in message" %}
+{% set title = "Commit twins links" %}
{% set icon = "fas fa-shield-alt" %}
{% block body %}
- {% for cve in annotated_commit.other_CVE_in_message %}
- {{
- cve }}
+ {% for id in annotated_commit.twins %}
+
+ {{ annotated_commit.repository }}/commit/{{ id }}
{% endfor %}
-{% endblock %}
+{% endblock %}
\ No newline at end of file
diff --git a/prospector/client/cli/templates/card/pages_linked_from_advisories_block.html b/prospector/client/cli/templates/card/pages_linked_from_advisories_block.html
index 25f18055f..33c05d4be 100644
--- a/prospector/client/cli/templates/card/pages_linked_from_advisories_block.html
+++ b/prospector/client/cli/templates/card/pages_linked_from_advisories_block.html
@@ -7,4 +7,4 @@
{{ page }}
{% endfor %}
-{% endblock %}
+{% endblock %}
\ No newline at end of file
diff --git a/prospector/client/cli/templates/card/relevant_paths_block.html b/prospector/client/cli/templates/card/relevant_paths_block.html
deleted file mode 100644
index 0c0399c2f..000000000
--- a/prospector/client/cli/templates/card/relevant_paths_block.html
+++ /dev/null
@@ -1,10 +0,0 @@
-{% extends "titled_block.html" %}
-{% set title = "Path relevant for changes" %}
-{% set icon = "fas fa-file-signature" %}
-{% block body %}
-
- {% for path in annotated_commit.changes_relevant_path %}
- {{ path }}
- {% endfor %}
-
-{% endblock %}
diff --git a/prospector/client/cli/templates/filtering_scripts.html b/prospector/client/cli/templates/filtering_scripts.html
index c1f17f2ee..c0ee8c0fa 100644
--- a/prospector/client/cli/templates/filtering_scripts.html
+++ b/prospector/client/cli/templates/filtering_scripts.html
@@ -27,11 +27,11 @@
for (let b of buttons) {
if (b.id == "relevancefilter") {
- b.addEventListener('click', function () { showFromRelevance(15); })
+ b.addEventListener('click', function () { showFromRelevance(20); })
} else if (b.id == "relevancefilter2") {
- b.addEventListener('click', function () { showFromRelevance(10); })
+ b.addEventListener('click', function () { showFromRelevance(15); })
} else if (b.id == "relevancefilter3") {
- b.addEventListener('click', function () { showFromRelevance(5); })
+ b.addEventListener('click', function () { showFromRelevance(10); })
} else if (b.id == "relevancefilter4") {
b.addEventListener('click', function () { showFromRelevance(0); })
}
diff --git a/prospector/client/cli/templates/results.html b/prospector/client/cli/templates/results.html
index 2e79a3033..e23556bb9 100644
--- a/prospector/client/cli/templates/results.html
+++ b/prospector/client/cli/templates/results.html
@@ -24,11 +24,11 @@ Prospector Report
{% for annotated_commit in candidates %}
{% if annotated_commit.relevance > 10 %}
{% else %}
{% endif %}
@@ -43,12 +43,11 @@
Prospector Report
aria-labelledby="candidateheader-{{ loop.index }}" data-parent="#accordion">
{% include "card/commit_title_block.html" %}
- {% include "card/annotations_block.html" %}
+ {% include "card/matched_rules_block.html" %}
{% include "card/message_block.html" %}
- {% include "card/relevant_paths_block.html" %}
{% include "card/changed_paths_block.html" %}
{% include "card/mentioned_cves_block.html" %}
- {% include "card/pages_linked_from_advisories_block.html" %}
+