diff --git a/importer/README.md b/importer/README.md index c4c65a550..333ccdf99 100644 --- a/importer/README.md +++ b/importer/README.md @@ -34,7 +34,7 @@ cargo run -p trustify-trustd -- importer csaf data/csaf ## Importing SBOMs ```bash -cargo run -p trustify-trustd -- importer sbom https://access.redhat.com/security/data/sbom/beta/ +cargo run -p trustify-trustd -- importer sbom https://access.redhat.com/security/data/sbom/beta/ --key https://access.redhat.com/security/data/97f5eac4.txt#77E79ABE93673533ED09EBE2DCE3823597F5EAC4 ``` Or, using a locally cached version: diff --git a/importer/src/csaf/mod.rs b/importer/src/csaf/mod.rs index 35234f219..8bff1e341 100644 --- a/importer/src/csaf/mod.rs +++ b/importer/src/csaf/mod.rs @@ -1,24 +1,24 @@ use crate::progress::init_log_and_progress; -use csaf::Csaf; use csaf_walker::{ retrieve::RetrievingVisitor, source::{DispatchSource, FileSource, HttpSource}, - validation::{ValidatedAdvisory, ValidationError, ValidationVisitor}, + validation::ValidationVisitor, visitors::filter::{FilterConfig, FilteringVisitor}, walker::Walker, }; -use sha2::{Digest, Sha256}; +use parking_lot::Mutex; use std::collections::HashSet; use std::process::ExitCode; -use std::time::SystemTime; -use time::{Date, Month, UtcOffset}; -use trustify_common::config::Database; -use trustify_common::db; +use std::sync::Arc; +use trustify_common::{config::Database, db}; use trustify_graph::graph::Graph; -use trustify_ingestors as ingestors; +use trustify_module_importer::server::{ + common::validation, + csaf::storage, + report::{Report, ReportBuilder, ScannerError, SplitScannerError}, +}; use url::Url; -use walker_common::utils::hex::Hex; -use walker_common::{fetcher::Fetcher, validate::ValidationOptions}; +use walker_common::{fetcher::Fetcher, progress::Progress}; /// Import from a CSAF source #[derive(clap::Args, Debug)] @@ -49,20 +49,27 @@ impl ImportCsafCommand { pub async fn run(self) -> anyhow::Result { let progress = init_log_and_progress()?; + log::info!("Ingesting CSAF"); + + let (report, result) = self.run_once(progress).await.split()?; + + log::info!("Import report: {report:#?}"); + + result.map(|()| ExitCode::SUCCESS) + } + + pub async fn run_once(self, progress: Progress) -> Result { + let report = Arc::new(Mutex::new(ReportBuilder::new())); + let db = db::Database::with_external_config(&self.database, false).await?; let system = Graph::new(db); - // because we still have GPG v3 signatures - let options = ValidationOptions::new().validation_date(SystemTime::from( - Date::from_calendar_date(2007, Month::January, 1)? - .midnight() - .assume_offset(UtcOffset::UTC), - )); - let source: DispatchSource = match Url::parse(&self.source) { Ok(mut url) => { if !self.full_source_url { - url = url.join("/.well-known/csaf/provider-metadata.json")?; + url = url + .join("/.well-known/csaf/provider-metadata.json") + .map_err(|err| ScannerError::Critical(err.into()))?; } log::info!("Provider metadata: {url}"); HttpSource::new( @@ -75,31 +82,18 @@ impl ImportCsafCommand { Err(_) => FileSource::new(&self.source, None)?.into(), }; + // storage (called by validator) + + let visitor = storage::StorageVisitor { + system, + report: report.clone(), + }; + // validate (called by retriever) - let visitor = - ValidationVisitor::new(move |doc: Result| { - let system = system.clone(); - async move { - let doc = match doc { - Ok(doc) => doc, - Err(err) => { - log::warn!("Ignore error: {err}"); - return Ok::<(), anyhow::Error>(()); - } - }; - - let url = doc.url.clone(); - log::debug!("processing: {url}"); - - if let Err(err) = process(&system, doc).await { - log::warn!("Failed to process {url}: {err}"); - } - - Ok(()) - } - }) - .with_options(options); + // because we still have GPG v3 signatures + let options = validation::options(true)?; + let visitor = ValidationVisitor::new(visitor).with_options(options); // retrieve (called by filter) @@ -122,24 +116,19 @@ impl ImportCsafCommand { }); } - walker.walk_parallel(self.workers, visitor).await?; - - Ok(ExitCode::SUCCESS) - } -} - -/// Process a single, validated advisory -async fn process(system: &Graph, doc: ValidatedAdvisory) -> anyhow::Result<()> { - let csaf = serde_json::from_slice::(&doc.data)?; - - let sha256 = match doc.sha256.clone() { - Some(sha) => sha.expected.clone(), - None => { - let digest = Sha256::digest(&doc.data); - Hex(&digest).to_lower() + walker + .walk_parallel(self.workers, visitor) + .await // if the walker fails, we record the outcome as part of the report, but skip any + // further processing, like storing the marker + .map_err(|err| ScannerError::Normal { + err: err.into(), + report: report.lock().clone().build(), + })?; + + Ok(match Arc::try_unwrap(report) { + Ok(report) => report.into_inner(), + Err(report) => report.lock().clone(), } - }; - - ingestors::advisory::csaf::ingest(system, csaf, &sha256, doc.url.as_str()).await?; - Ok(()) + .build()) + } } diff --git a/importer/src/sbom/mod.rs b/importer/src/sbom/mod.rs index 87623d27b..2348b48aa 100644 --- a/importer/src/sbom/mod.rs +++ b/importer/src/sbom/mod.rs @@ -8,16 +8,15 @@ use sbom_walker::{ }; use std::process::ExitCode; use std::sync::Arc; -use std::time::SystemTime; -use time::{Date, Month, UtcOffset}; use trustify_common::{config::Database, db}; use trustify_graph::graph::Graph; use trustify_module_importer::server::{ + common::validation, report::{Report, ReportBuilder, ScannerError, SplitScannerError}, sbom::storage, }; use url::Url; -use walker_common::{fetcher::Fetcher, progress::Progress, validate::ValidationOptions}; +use walker_common::{fetcher::Fetcher, progress::Progress}; /// Import SBOMs #[derive(clap::Args, Debug)] @@ -69,9 +68,9 @@ impl ImportSbomCommand { Err(_) => FileSource::new(&self.source, None)?.into(), }; - // process (called by validator) + // storage (called by validator) - let process = storage::StorageVisitor { + let storage = storage::StorageVisitor { system, report: report.clone(), }; @@ -79,14 +78,8 @@ impl ImportSbomCommand { // validate (called by retriever) // because we still have GPG v3 signatures - let options = ValidationOptions::new().validation_date(SystemTime::from( - Date::from_calendar_date(2007, Month::January, 1) - .map_err(|err| ScannerError::Critical(err.into()))? - .midnight() - .assume_offset(UtcOffset::UTC), - )); - - let validation = ValidationVisitor::new(process).with_options(options); + let options = validation::options(true)?; + let validation = ValidationVisitor::new(storage).with_options(options); // retriever (called by filter) diff --git a/ingestors/Cargo.toml b/ingestors/Cargo.toml index c3a68fb34..dc8b75652 100644 --- a/ingestors/Cargo.toml +++ b/ingestors/Cargo.toml @@ -9,20 +9,21 @@ trustify-common = { path = "../common"} trustify-graph = { path = "../graph"} trustify-entity = { path = "../entity" } -serde = { version = "1.0.183", features = ["derive"] } -serde_json = "1.0.114" -chrono = { version = "0.4.35", features = ["serde"] } -tokio = { version = "1.30.0", features = ["full"] } -thiserror = "1.0.58" anyhow = "1.0.72" +chrono = { version = "0.4.35", features = ["serde"] } +csaf = "0.5.0" +env_logger = "0.11.0" +hex = "0.4.3" +humantime = "2" log = "0.4.19" -env_logger = "0.10.0" reqwest = "0.11" ring = "0.17.8" -hex = "0.4.3" -csaf = "0.5.0" -sha2 = "0.10.8" sea-orm = "*" +serde = { version = "1.0.183", features = ["derive"] } +serde_json = "1.0.114" +sha2 = "0.10.8" +thiserror = "1.0.58" +tokio = { version = "1.30.0", features = ["full"] } [dev-dependencies] test-log = { version = "0.2.15", features = ["env_logger", "trace"] } diff --git a/ingestors/src/advisory/csaf.rs b/ingestors/src/advisory/csaf.rs index 659ca4b1a..95262c11e 100644 --- a/ingestors/src/advisory/csaf.rs +++ b/ingestors/src/advisory/csaf.rs @@ -1,4 +1,5 @@ use csaf::Csaf; +use std::time::Instant; use trustify_common::db::Transactional; use trustify_graph::graph::Graph; @@ -9,15 +10,25 @@ pub async fn ingest( sha256: &str, location: &str, ) -> anyhow::Result { - let identifier = &csaf.document.tracking.id; + let identifier = csaf.document.tracking.id.clone(); - log::info!("Ingesting: {} from {}", identifier, location); + log::debug!("Ingesting: {} from {}", identifier, location); + + let start = Instant::now(); let advisory = system - .ingest_advisory(identifier, location, sha256, Transactional::None) + .ingest_advisory(&identifier, location, sha256, Transactional::None) .await?; advisory.ingest_csaf(csaf).await?; + let duration = Instant::now() - start; + log::info!( + "Ingested: {} from {}: took {}", + identifier, + location, + humantime::Duration::from(duration), + ); + Ok(advisory.advisory.id) } diff --git a/modules/importer/Cargo.toml b/modules/importer/Cargo.toml index eb67c4adf..17fa65a71 100644 --- a/modules/importer/Cargo.toml +++ b/modules/importer/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" trustify-common = { path = "../../common" } trustify-entity = { path = "../../entity" } trustify-graph = { path = "../../graph" } +trustify-ingestors = { path = "../../ingestors" } actix-web = "4" anyhow = "1.0.72" diff --git a/modules/importer/README.md b/modules/importer/README.md index 7b33e2df6..6aaed75e2 100644 --- a/modules/importer/README.md +++ b/modules/importer/README.md @@ -1,10 +1,16 @@ +Create a new CSAF importer: + +```bash +http POST localhost:8080/api/v1/importer/redhat-csaf csaf[source]=https://redhat.com/.well-known/csaf/provider-metadata.json csaf[disabled]:=false csaf[onlyPatterns][]="^cve-2023-" csaf[period]=30s csaf[v3Signatures]:=true +``` + Create a new SBOM importer: ```bash -http POST localhost:8080/api/v1/importer/test sbom[source]=https://access.redhat.com/security/data/sbom/beta/ sbom[keys][]=https://access.redhat.com/security/data/97f5eac4.txt#77E79ABE93673533ED09EBE2DCE3823597F5EAC4 sbom[disabled]:=false sbom[onlyPatterns][]=quarkus sbom[period]=30s +http POST localhost:8080/api/v1/importer/redhat-sbom sbom[source]=https://access.redhat.com/security/data/sbom/beta/ sbom[keys][]=https://access.redhat.com/security/data/97f5eac4.txt#77E79ABE93673533ED09EBE2DCE3823597F5EAC4 sbom[disabled]:=false sbom[onlyPatterns][]=quarkus sbom[period]=30s sbom[v3Signatures]:=true ``` -Get all importer: +Get all importers: ```bash http GET localhost:8080/api/v1/importer @@ -13,11 +19,13 @@ http GET localhost:8080/api/v1/importer Get a specific importer: ```bash -http GET localhost:8080/api/v1/importer/test +http GET localhost:8080/api/v1/importer/redhat-csaf +http GET localhost:8080/api/v1/importer/redhat-sbom ``` Get reports: ```bash -http GET localhost:8080/api/v1/importer/test/report +http GET localhost:8080/api/v1/importer/redhat-csaf/report +http GET localhost:8080/api/v1/importer/redhat-sbom/report ``` diff --git a/modules/importer/src/model/csaf.rs b/modules/importer/src/model/csaf.rs new file mode 100644 index 000000000..84b3c1079 --- /dev/null +++ b/modules/importer/src/model/csaf.rs @@ -0,0 +1,29 @@ +use super::*; + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CsafImporter { + #[serde(flatten)] + pub common: CommonImporter, + + pub source: String, + + #[serde(default)] + pub v3_signatures: bool, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub only_patterns: Vec, +} + +impl Deref for CsafImporter { + type Target = CommonImporter; + + fn deref(&self) -> &Self::Target { + &self.common + } +} + +impl DerefMut for CsafImporter { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.common + } +} diff --git a/modules/importer/src/model.rs b/modules/importer/src/model/mod.rs similarity index 85% rename from modules/importer/src/model.rs rename to modules/importer/src/model/mod.rs index 45e56d103..ba2ac1ce8 100644 --- a/modules/importer/src/model.rs +++ b/modules/importer/src/model/mod.rs @@ -1,9 +1,17 @@ +mod csaf; +mod sbom; + +pub use csaf::*; +pub use sbom::*; + use std::ops::{Deref, DerefMut}; use std::time::Duration; use time::OffsetDateTime; use trustify_common::model::Revisioned; -use trustify_entity::importer::Model; -use trustify_entity::{importer, importer_report}; +use trustify_entity::{ + importer::{self, Model}, + importer_report, +}; use url::Url; use utoipa::ToSchema; @@ -70,6 +78,7 @@ pub struct ImporterData { #[serde(rename_all = "camelCase")] pub enum ImporterConfiguration { Sbom(SbomImporter), + Csaf(CsafImporter), } impl Deref for ImporterConfiguration { @@ -78,6 +87,7 @@ impl Deref for ImporterConfiguration { fn deref(&self) -> &Self::Target { match self { Self::Sbom(importer) => &importer.common, + Self::Csaf(importer) => &importer.common, } } } @@ -92,37 +102,6 @@ pub struct CommonImporter { pub period: Duration, } -#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, ToSchema)] -#[serde(rename_all = "camelCase")] -pub struct SbomImporter { - #[serde(flatten)] - pub common: CommonImporter, - - pub source: String, - - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub keys: Vec, - - #[serde(default)] - pub v3_signatures: bool, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub only_patterns: Vec, -} - -impl Deref for SbomImporter { - type Target = CommonImporter; - - fn deref(&self) -> &Self::Target { - &self.common - } -} - -impl DerefMut for SbomImporter { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.common - } -} - impl TryFrom for Importer { type Error = serde_json::Error; diff --git a/modules/importer/src/model/sbom.rs b/modules/importer/src/model/sbom.rs new file mode 100644 index 000000000..f742c1b93 --- /dev/null +++ b/modules/importer/src/model/sbom.rs @@ -0,0 +1,32 @@ +use super::*; + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct SbomImporter { + #[serde(flatten)] + pub common: CommonImporter, + + pub source: String, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub keys: Vec, + + #[serde(default)] + pub v3_signatures: bool, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub only_patterns: Vec, +} + +impl Deref for SbomImporter { + type Target = CommonImporter; + + fn deref(&self) -> &Self::Target { + &self.common + } +} + +impl DerefMut for SbomImporter { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.common + } +} diff --git a/modules/importer/src/server/common/filter.rs b/modules/importer/src/server/common/filter.rs new file mode 100644 index 000000000..a6b21e67b --- /dev/null +++ b/modules/importer/src/server/common/filter.rs @@ -0,0 +1,109 @@ +use crate::server::report::ScannerError; +use async_trait::async_trait; +use regex::Regex; +use std::str::FromStr; +use walker_common::utils::url::Urlify; + +mod csaf { + pub use csaf_walker::discover::{DiscoveredAdvisory, DiscoveredContext, DiscoveredVisitor}; +} +mod sbom { + pub use sbom_walker::discover::{DiscoveredContext, DiscoveredSbom, DiscoveredVisitor}; +} + +pub struct Filter { + pub only_patterns: Vec, + pub next: T, +} + +impl Filter { + pub fn from_config(next: T, only_patterns: Vec) -> Result { + Ok(Self { + only_patterns: only_patterns + .into_iter() + .map(|r| Regex::from_str(&r)) + .collect::>() + .map_err(|err| ScannerError::Critical(err.into()))?, + next, + }) + } + + /// check if the document should be skipped + /// + /// return `true` of the document should be skipped, `false` otherwise + fn skip(&self, document: &impl Urlify) -> bool { + if self.only_patterns.is_empty() { + false + } else { + let url = document.url(); + let name = if let Some(name) = url.path_segments().into_iter().flatten().last() { + name + } else { + url.path() + }; + + let found = self + .only_patterns + .iter() + .any(|pattern| pattern.is_match(name)); + + !found + } + } +} + +#[async_trait(?Send)] +impl csaf::DiscoveredVisitor for Filter +where + T: csaf::DiscoveredVisitor, +{ + type Error = T::Error; + type Context = T::Context; + + async fn visit_context( + &self, + context: &csaf::DiscoveredContext, + ) -> Result { + self.next.visit_context(context).await + } + + async fn visit_advisory( + &self, + context: &Self::Context, + document: csaf::DiscoveredAdvisory, + ) -> Result<(), Self::Error> { + if !self.skip(&document) { + self.next.visit_advisory(context, document).await + } else { + Ok(()) + } + } +} + +#[async_trait(?Send)] +impl sbom::DiscoveredVisitor for Filter +where + T: sbom::DiscoveredVisitor, +{ + type Error = T::Error; + type Context = T::Context; + + async fn visit_context( + &self, + context: &sbom::DiscoveredContext, + ) -> Result { + self.next.visit_context(context).await + } + + async fn visit_sbom( + &self, + context: &Self::Context, + document: sbom::DiscoveredSbom, + ) -> Result<(), Self::Error> { + if !self.skip(&document) { + self.next.visit_sbom(context, document).await + } else { + Ok(()) + } + } +} diff --git a/modules/importer/src/server/common/mod.rs b/modules/importer/src/server/common/mod.rs new file mode 100644 index 000000000..86e6f914e --- /dev/null +++ b/modules/importer/src/server/common/mod.rs @@ -0,0 +1,2 @@ +pub mod filter; +pub mod validation; diff --git a/modules/importer/src/server/common/validation.rs b/modules/importer/src/server/common/validation.rs new file mode 100644 index 000000000..3c2723f76 --- /dev/null +++ b/modules/importer/src/server/common/validation.rs @@ -0,0 +1,19 @@ +use crate::server::report::ScannerError; +use std::time::SystemTime; +use time::{Date, Month, UtcOffset}; +use walker_common::validate::ValidationOptions; + +pub fn options(v3_signatures: bool) -> Result { + let mut options = ValidationOptions::new(); + + if v3_signatures { + options = options.validation_date(SystemTime::from( + Date::from_calendar_date(2007, Month::January, 1) + .map_err(|err| ScannerError::Critical(err.into()))? + .midnight() + .assume_offset(UtcOffset::UTC), + )); + } + + Ok(options) +} diff --git a/modules/importer/src/server/csaf/mod.rs b/modules/importer/src/server/csaf/mod.rs new file mode 100644 index 000000000..341840d87 --- /dev/null +++ b/modules/importer/src/server/csaf/mod.rs @@ -0,0 +1,86 @@ +use crate::{ + model::CsafImporter, + server::{ + common::{filter::Filter, validation}, + csaf::report::CsafReportVisitor, + report::{Report, ReportBuilder, ReportVisitor, ScannerError}, + }, +}; +use csaf_walker::{ + retrieve::RetrievingVisitor, + source::{DispatchSource, FileSource, HttpOptions, HttpSource}, + validation::ValidationVisitor, + walker::Walker, +}; +use parking_lot::Mutex; +use std::sync::Arc; +use std::time::SystemTime; +use trustify_graph::graph::Graph; +use url::Url; +use walker_common::fetcher::Fetcher; + +mod report; +pub mod storage; + +impl super::Server { + pub async fn run_once_csaf( + &self, + importer: CsafImporter, + last_run: Option, + ) -> Result { + let report = Arc::new(Mutex::new(ReportBuilder::new())); + + let source: DispatchSource = match Url::parse(&importer.source) { + Ok(url) => HttpSource::new( + url, + Fetcher::new(Default::default()).await?, + HttpOptions::new().since(last_run), + ) + .into(), + Err(_) => FileSource::new(&importer.source, None)?.into(), + }; + + // storage (called by validator) + + let storage = storage::StorageVisitor { + system: Graph::new(self.db.clone()), + report: report.clone(), + }; + + // wrap storage with report + + let storage = CsafReportVisitor(ReportVisitor::new(report.clone(), storage)); + + // validate (called by retriever) + + let options = validation::options(importer.v3_signatures)?; + let validation = ValidationVisitor::new(storage).with_options(options); + + // retriever (called by filter) + + let visitor = RetrievingVisitor::new(source.clone(), validation); + + // filter + + let filter = Filter::from_config(visitor, importer.only_patterns)?; + + // walker + + // FIXME: track progress + Walker::new(source) + .walk(filter) + .await + // if the walker fails, we record the outcome as part of the report, but skip any + // further processing, like storing the marker + .map_err(|err| ScannerError::Normal { + err: err.into(), + report: report.lock().clone().build(), + })?; + + Ok(match Arc::try_unwrap(report) { + Ok(report) => report.into_inner(), + Err(report) => report.lock().clone(), + } + .build()) + } +} diff --git a/modules/importer/src/server/csaf/report.rs b/modules/importer/src/server/csaf/report.rs new file mode 100644 index 000000000..c00cf7de0 --- /dev/null +++ b/modules/importer/src/server/csaf/report.rs @@ -0,0 +1,95 @@ +use crate::server::{ + csaf::storage::{StorageError, StorageVisitor}, + report::{Phase, ReportVisitor, Severity}, +}; +use async_trait::async_trait; +use csaf_walker::{ + retrieve::RetrievalError, + validation::{ValidatedAdvisory, ValidatedVisitor, ValidationContext, ValidationError}, +}; +use walker_common::utils::url::Urlify; + +pub struct CsafReportVisitor(pub ReportVisitor); + +#[async_trait(?Send)] +impl ValidatedVisitor for CsafReportVisitor { + type Error = ::Error; + type Context = ::Context; + + async fn visit_context( + &self, + context: &ValidationContext, + ) -> Result { + self.0.next.visit_context(context).await + } + + async fn visit_advisory( + &self, + context: &Self::Context, + result: Result, + ) -> Result<(), Self::Error> { + let file = result.url().to_string(); + + self.0.report.lock().tick(); + + let result = self.0.next.visit_advisory(context, result).await; + + if let Err(err) = &result { + match err { + StorageError::Validation(ValidationError::Retrieval( + RetrievalError::InvalidResponse { code, .. }, + )) => { + self.0.report.lock().add_error( + Phase::Retrieval, + file, + Severity::Error, + format!("retrieval of document failed: {code}"), + ); + + if code.is_client_error() { + // If it's a client error, there's no need to re-try. We simply claim + // success after we logged it. + return Ok(()); + } + } + StorageError::Validation(ValidationError::DigestMismatch { + expected, + actual, + .. + }) => { + self.0.report.lock().add_error( + Phase::Validation, + file, + Severity::Error, + format!("digest mismatch - expected: {expected}, actual: {actual}"), + ); + + // If there's a digest error, we can't do much other than ignoring the + // current file. Once it gets updated, we can reprocess it. + return Ok(()); + } + StorageError::Validation(ValidationError::Signature { error, .. }) => { + self.0.report.lock().add_error( + Phase::Validation, + file, + Severity::Error, + format!("unable to verify signature: {error}"), + ); + + // If there's a signature error, we can't do much other than ignoring the + // current file. Once it gets updated, we can reprocess it. + } + StorageError::Storage(err) => { + self.0.report.lock().add_error( + Phase::Upload, + file, + Severity::Error, + format!("upload failed: {err}"), + ); + } + } + } + + result + } +} diff --git a/modules/importer/src/server/csaf/storage.rs b/modules/importer/src/server/csaf/storage.rs new file mode 100644 index 000000000..add719f0f --- /dev/null +++ b/modules/importer/src/server/csaf/storage.rs @@ -0,0 +1,65 @@ +use crate::server::report::ReportBuilder; +use async_trait::async_trait; +use csaf::Csaf; +use csaf_walker::{ + retrieve::RetrievedAdvisory, + validation::{ValidatedAdvisory, ValidatedVisitor, ValidationContext, ValidationError}, +}; +use parking_lot::Mutex; +use sha2::{Digest, Sha256}; +use std::sync::Arc; +use trustify_graph::graph::Graph; +use walker_common::utils::hex::Hex; + +#[derive(Debug, thiserror::Error)] +pub enum StorageError { + #[error(transparent)] + Validation(#[from] ValidationError), + #[error(transparent)] + Storage(anyhow::Error), +} + +pub struct StorageVisitor { + pub system: Graph, + /// the report to report our messages to + pub report: Arc>, +} + +#[async_trait(? Send)] +impl ValidatedVisitor for StorageVisitor { + type Error = StorageError; + type Context = (); + + async fn visit_context(&self, _: &ValidationContext) -> Result { + Ok(()) + } + + async fn visit_advisory( + &self, + _context: &Self::Context, + result: Result, + ) -> Result<(), Self::Error> { + self.store(&result?.retrieved).await?; + Ok(()) + } +} + +impl StorageVisitor { + async fn store(&self, doc: &RetrievedAdvisory) -> Result<(), StorageError> { + let csaf = serde_json::from_slice::(&doc.data) + .map_err(|err| StorageError::Storage(err.into()))?; + + let sha256 = match doc.sha256.clone() { + Some(sha) => sha.expected, + None => { + let digest = Sha256::digest(&doc.data); + Hex(&digest).to_lower() + } + }; + + trustify_ingestors::advisory::csaf::ingest(&self.system, csaf, &sha256, doc.url.as_str()) + .await + .map_err(StorageError::Storage)?; + Ok(()) + } +} diff --git a/modules/importer/src/server/mod.rs b/modules/importer/src/server/mod.rs index d79609303..8574218d7 100644 --- a/modules/importer/src/server/mod.rs +++ b/modules/importer/src/server/mod.rs @@ -1,3 +1,5 @@ +pub mod common; +pub mod csaf; pub mod report; pub mod sbom; @@ -36,7 +38,7 @@ impl Server { continue; } - log::info!(" {}: {:?}", importer.name, importer.data.configuration); + log::debug!(" {}: {:?}", importer.name, importer.data.configuration); service.update_start(&importer.name, None).await?; @@ -78,6 +80,7 @@ impl Server { match configuration { ImporterConfiguration::Sbom(sbom) => self.run_once_sbom(sbom, last_run).await, + ImporterConfiguration::Csaf(csaf) => self.run_once_csaf(csaf, last_run).await, } } } diff --git a/modules/importer/src/server/report.rs b/modules/importer/src/server/report.rs index 12600cd3e..767910996 100644 --- a/modules/importer/src/server/report.rs +++ b/modules/importer/src/server/report.rs @@ -1,4 +1,3 @@ -use crate::server::sbom::storage; use parking_lot::Mutex; use std::collections::BTreeMap; use std::sync::Arc; @@ -93,13 +92,13 @@ impl Default for ReportBuilder { } } -pub struct ReportVisitor { +pub struct ReportVisitor { pub report: Arc>, - pub next: storage::StorageVisitor, + pub next: V, } -impl ReportVisitor { - pub fn new(report: Arc>, next: storage::StorageVisitor) -> Self { +impl ReportVisitor { + pub fn new(report: Arc>, next: V) -> Self { Self { report, next } } } @@ -133,10 +132,3 @@ impl SplitScannerError for Result { } } } - -/// Handle the report -pub async fn handle_report(report: Report) -> anyhow::Result<()> { - // FIXME: this is a very simplistic version of handling the error - log::info!("Import report: {report:#?}"); - Ok(()) -} diff --git a/modules/importer/src/server/sbom/filter.rs b/modules/importer/src/server/sbom/filter.rs deleted file mode 100644 index 57ad19cf2..000000000 --- a/modules/importer/src/server/sbom/filter.rs +++ /dev/null @@ -1,50 +0,0 @@ -use async_trait::async_trait; -use regex::Regex; -use sbom_walker::discover::{DiscoveredContext, DiscoveredSbom, DiscoveredVisitor}; - -pub struct Filter { - pub only_patterns: Vec, - pub next: T, -} - -#[async_trait(?Send)] -impl DiscoveredVisitor for Filter -where - T: DiscoveredVisitor, -{ - type Error = T::Error; - type Context = T::Context; - - async fn visit_context( - &self, - context: &DiscoveredContext, - ) -> Result { - self.next.visit_context(context).await - } - - async fn visit_sbom( - &self, - context: &Self::Context, - sbom: DiscoveredSbom, - ) -> Result<(), Self::Error> { - if !self.only_patterns.is_empty() { - let name = if let Some(name) = sbom.url.path_segments().into_iter().flatten().last() { - name - } else { - sbom.url.path() - }; - - let found = self - .only_patterns - .iter() - .any(|pattern| pattern.is_match(name)); - - if !found { - // do not pass to the next, return now - return Ok(()); - } - } - - self.next.visit_sbom(context, sbom).await - } -} diff --git a/modules/importer/src/server/sbom/mod.rs b/modules/importer/src/server/sbom/mod.rs index 3bea303a0..2948a7f25 100644 --- a/modules/importer/src/server/sbom/mod.rs +++ b/modules/importer/src/server/sbom/mod.rs @@ -1,36 +1,38 @@ -use crate::model::SbomImporter; -use crate::server::report::{Report, ReportBuilder, ReportVisitor, ScannerError}; -use crate::server::sbom::filter::Filter; -use crate::server::sbom::report::SbomReportVisitor; +use crate::{ + model::SbomImporter, + server::report::{Report, ReportBuilder, ReportVisitor, ScannerError}, + server::sbom::report::SbomReportVisitor, +}; + +use crate::server::common::filter::Filter; +use crate::server::common::validation; use parking_lot::Mutex; -use regex::Regex; -use sbom_walker::retrieve::RetrievingVisitor; -use sbom_walker::source::{DispatchSource, FileSource, HttpOptions, HttpSource}; -use sbom_walker::validation::ValidationVisitor; -use sbom_walker::walker::Walker; -use std::str::FromStr; +use sbom_walker::{ + retrieve::RetrievingVisitor, + source::{DispatchSource, FileSource, HttpOptions, HttpSource}, + validation::ValidationVisitor, + walker::Walker, +}; use std::sync::Arc; use std::time::SystemTime; -use time::{Date, Month, UtcOffset}; use trustify_graph::graph::Graph; use url::Url; -use walker_common::{fetcher::Fetcher, validate::ValidationOptions}; +use walker_common::fetcher::Fetcher; -mod filter; -pub mod report; +mod report; pub mod storage; impl super::Server { pub async fn run_once_sbom( &self, - sbom: SbomImporter, + importer: SbomImporter, last_run: Option, ) -> Result { let report = Arc::new(Mutex::new(ReportBuilder::new())); - let source: DispatchSource = match Url::parse(&sbom.source) { + let source: DispatchSource = match Url::parse(&importer.source) { Ok(url) => { - let keys = sbom + let keys = importer .keys .into_iter() .map(|key| key.into()) @@ -42,7 +44,7 @@ impl super::Server { ) .into() } - Err(_) => FileSource::new(&sbom.source, None)?.into(), + Err(_) => FileSource::new(&importer.source, None)?.into(), }; // storage (called by validator) @@ -58,14 +60,8 @@ impl super::Server { // validate (called by retriever) - // because we still have GPG v3 signatures - let options = ValidationOptions::new().validation_date(SystemTime::from( - Date::from_calendar_date(2007, Month::January, 1) - .map_err(|err| ScannerError::Critical(err.into()))? - .midnight() - .assume_offset(UtcOffset::UTC), - )); - + // because we might still have GPG v3 signatures + let options = validation::options(importer.v3_signatures)?; let validation = ValidationVisitor::new(storage).with_options(options); // retriever (called by filter) @@ -74,15 +70,7 @@ impl super::Server { // filter - let filter = Filter { - only_patterns: sbom - .only_patterns - .into_iter() - .map(|r| Regex::from_str(&r)) - .collect::>() - .map_err(|err| ScannerError::Critical(err.into()))?, - next: visitor, - }; + let filter = Filter::from_config(visitor, importer.only_patterns)?; // walker diff --git a/modules/importer/src/server/sbom/report.rs b/modules/importer/src/server/sbom/report.rs index 47392e923..4095a1640 100644 --- a/modules/importer/src/server/sbom/report.rs +++ b/modules/importer/src/server/sbom/report.rs @@ -9,7 +9,7 @@ use sbom_walker::{ }; use walker_common::utils::url::Urlify; -pub struct SbomReportVisitor(pub ReportVisitor); +pub struct SbomReportVisitor(pub ReportVisitor); #[async_trait(?Send)] impl ValidatedVisitor for SbomReportVisitor {