diff --git a/Cargo.toml b/Cargo.toml index 58054e062..75d680835 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,5 +10,6 @@ argh = "0.1.12" serde = { version = "1.0.196", features = ["derive"] } serde_yaml = "0.9" serde_json = "1" +pinyin = "0.10.0" engine = { git = "https://github.com/emo-crab/observer_ward.git" } #engine = { path = "../observer_ward/engine" } \ No newline at end of file diff --git a/src/cli.rs b/src/cli.rs index b9cb4a4e8..95b9c4383 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,3 +1,4 @@ +use std::path::PathBuf; use argh::FromArgs; #[derive(Debug, Clone, FromArgs)] @@ -15,6 +16,9 @@ pub struct HelperConfig { /// convert service fingerprint yaml #[argh(switch)] pub service: bool, + /// convert v3 yaml to v4 yaml + #[argh(option)] + pub v3_to_v4: Option, } impl Default for HelperConfig { diff --git a/src/lib.rs b/src/lib.rs index a49bf1273..64280d20e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,12 +2,15 @@ pub mod cli; mod error; pub mod nmap; mod service; +mod v3; +pub use v3::{WebFingerPrint, V3WebFingerPrint}; use crate::error::{new_io_error, Result}; pub use crate::service::match_line::MatchLine; pub use crate::service::probe::{Probe, ZeroDuration}; use engine::request::PortRange; use std::str::{FromStr, Lines}; +use pinyin::ToPinyin; // 转下划线风格 pub fn to_kebab_case(input: &str) -> String { @@ -163,6 +166,32 @@ impl<'buffer> FingerPrintParse<'buffer> { } } +pub fn hans_to_pinyin(hans: &str) -> String { + let mut pinyin_str = String::new(); + let mut is_han = false; + let mut is_letter = false; + for c in hans.chars() { + if let Some(py) = c.to_pinyin() { + if is_han || is_letter { + pinyin_str.push('-'); + } + pinyin_str.push_str(py.plain()); + is_han = true; + is_letter = false; + } else { + if is_han { + pinyin_str.push('-'); + is_han = false; + } + is_letter = true; + pinyin_str.push(c); + } + } + pinyin_str = pinyin_str.replace(" ", "-"); + pinyin_str = pinyin_str.replacen("--", "-", 10); + pinyin_str +} + #[cfg(test)] mod tests { #[test] diff --git a/src/main.rs b/src/main.rs index c4d9da2d6..29c837153 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,11 +5,11 @@ use engine::request::{HttpRaw, Requests}; use engine::template::Template; use helper::cli::HelperConfig; use helper::nmap::nmap; -use helper::to_kebab_case; +use helper::{to_kebab_case, V3WebFingerPrint}; use std::collections::BTreeMap; use std::env; use std::fs::{File, OpenOptions}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::str::FromStr; const UNKNOWN_VENDOR: &str = "00_unknown"; @@ -138,7 +138,7 @@ fn sync_nuclei() { .join(sub_tag) .join(yaml_path.file_name().unwrap().to_string_lossy().to_string()), ) - .unwrap(); + .unwrap(); break; } } @@ -153,7 +153,7 @@ fn sync_nuclei() { .join(product) .join(yaml_path.file_name().unwrap().to_string_lossy().to_string()), ) - .unwrap(); + .unwrap(); } continue; } @@ -520,6 +520,29 @@ fn cse_to_template(one_cse: CSE, vpf: VPF) -> Template { return t; } +fn v3_to_v4(v3_path: PathBuf) { + let v3_yaml_list = find_yaml_file(&v3_path, false); + let current_fingerprint_dir = env::current_dir().unwrap().join("web-fingerprint"); + let all_product: Vec = find_yaml_file(¤t_fingerprint_dir, true).into_iter() + .map(|p| p.file_name().unwrap().to_string_lossy().trim_end_matches(".yaml").to_string()) + .collect(); + for v3_path in v3_yaml_list { + let v3_file = File::open(&v3_path).unwrap(); + let v3_finger: V3WebFingerPrint = serde_yaml::from_reader(v3_file).unwrap(); + let template: Template = v3_finger.into(); + if !all_product.contains(&template.info.name) { + if let Some((_x, y)) = template.info.name.split_once("-") { + if all_product.contains(&y.to_string()) { + continue; + } + } + let v4_path = current_fingerprint_dir.join("00_unknown").join(format!("{}.yaml", template.info.name)); + let v4_file = File::create(&v4_path).unwrap(); + serde_yaml::to_writer(v4_file, &template).unwrap(); + } + } +} + fn main() { let config = HelperConfig::default(); if config.convert { @@ -538,6 +561,9 @@ fn main() { if config.service { nmap(); } + if let Some(v3_path) = config.v3_to_v4 { + v3_to_v4(v3_path); + } } #[cfg(test)] diff --git a/src/v3.rs b/src/v3.rs new file mode 100644 index 000000000..d54c0ff22 --- /dev/null +++ b/src/v3.rs @@ -0,0 +1,133 @@ +use std::collections::{BTreeMap, HashSet}; +use engine::info::{Info, Severity, VPF}; +use engine::matchers::{Condition, Favicon, Matcher, MatcherType, Part, Word}; +use engine::request::Requests; +use engine::template::Template; +use serde::{Deserialize, Serialize}; +use crate::hans_to_pinyin; +// 旧版指纹，数据结构 + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct WebFingerPrintRequest { + /// 请求路径 + pub path: String, + /// 请求方法 + pub request_method: String, + /// 请求头 + pub request_headers: BTreeMap, + /// 请求数据，base64编码后的 + pub request_data: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct WebFingerPrintMatch { + /// 匹配状态码 + pub status_code: u16, + /// 匹配favicon的hash列表 + #[serde(default)] + pub favicon_hash: Vec, + /// 匹配的请求头 + pub headers: BTreeMap, + /// 匹配的关键词列表 + pub keyword: Vec, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct V3WebFingerPrint { + /// 组件名称 + #[serde(default)] + pub name: String, + /// 权重 + #[serde(default)] + pub priority: u32, + pub fingerprint: Vec, +} + +/// 单个指纹结构 +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct WebFingerPrint { + /// 指纹的自定义请求 + #[serde(flatten)] + pub fingerprint: WebFingerPrintRequest, + /// 匹配部分 + #[serde(flatten)] + pub match_rules: WebFingerPrintMatch, +} + +impl Into