Skip to content

Commit

Permalink
[Agent] fix http test and extend http req resp rewrite
Browse files Browse the repository at this point in the history
  • Loading branch information
@qq906907952
qq906907952 committed Aug 31, 2023
1 parent 6c9df59 commit 5d8494d
Show file tree
Hide file tree
Showing 12 changed files with 146 additions and 192 deletions.
4 changes: 2 additions & 2 deletions agent/resources/test/flow_generator/http/h2c_ascii.result
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Grpc, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: Some(1), version: "2", trace_id: "", span_id: "", method: "POST", path: "/hipstershop.CartService/GetCart", host: "cartservice:7070", user_agent: Some("grpc-go/1.22.0"), referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: Some(43), resp_content_length: None, status_code: None, status: Ok, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Grpc, is_tls: false, msg_type: Response, raw_data_type: RawProtocol, stream_id: Some(1), version: "2", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: Some(21), status_code: Some(200), status: Ok, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Grpc, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: Some(1), version: "2", trace_id: "", span_id: "", method: "POST", path: "/hipstershop.CartService/GetCart", host: "cartservice:7070", user_agent: Some("grpc-go/1.22.0"), referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: Some(43), resp_content_length: None, status_code: None, status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Grpc, is_tls: false, msg_type: Response, raw_data_type: RawProtocol, stream_id: Some(1), version: "2", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: Some(21), status_code: Some(200), status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: true
4 changes: 2 additions & 2 deletions agent/resources/test/flow_generator/http/httpv1.result
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "", span_id: "", method: "POST", path: "/query?1590632942", host: "rq.cct.cloud.duba.net", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: Some(85), resp_content_length: None, status_code: None, status: Ok, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Response, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: Some(54), status_code: Some(200), status: Ok, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "", span_id: "", method: "POST", path: "/query?1590632942", host: "rq.cct.cloud.duba.net", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: Some(85), resp_content_length: None, status_code: None, status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Response, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: Some(54), status_code: Some(200), status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: true
2 changes: 1 addition & 1 deletion agent/resources/test/flow_generator/http/httpv2-stream-id.result
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http2, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: Some(1392369), version: "2", trace_id: "", span_id: "", method: "POST", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: Some(0), resp_content_length: None, status_code: None, status: Ok, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http2, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: Some(1392369), version: "2", trace_id: "", span_id: "", method: "POST", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: Some(0), resp_content_length: None, status_code: None, status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: true
4 changes: 2 additions & 2 deletions agent/resources/test/flow_generator/http/istio-tcp-frag.result
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "", span_id: "", method: "GET", path: "/productpage", host: "productpage:9080", user_agent: Some("curl/7.81.0-DEV"), referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: None, status: Ok, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Response, raw_data_type: RawProtocol, stream_id: None, version: "1.0", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: Some(200), status: Ok, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "", span_id: "", method: "GET", path: "/productpage", host: "productpage:9080", user_agent: Some("curl/7.81.0-DEV"), referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: None, status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Response, raw_data_type: RawProtocol, stream_id: None, version: "1.0", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: Some(200), status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: true
18 changes: 9 additions & 9 deletions agent/resources/test/flow_generator/http/sw8.result
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "3912196de0cf41f4bab8a8a8108fc3a8.63.16294441329780027", span_id: "3912196de0cf41f4bab8a8a8108fc3a8.63.16294441329780026-4", method: "POST", path: "/createOrder", host: "10.100.18.175:20880", user_agent: Some("Apache-HttpClient/4.5.10 (Java/1.8.0_212)"), referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: Some(351), resp_content_length: None, status_code: None, status: Ok, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Response, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: Some(200), status: Ok, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Other, raw_data_type: RawProtocol, stream_id: None, version: "", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: None, status: Ok, attributes: [] } is_http: false
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "3912196de0cf41f4bab8a8a8108fc3a8.65.16294441341700021", span_id: "3912196de0cf41f4bab8a8a8108fc3a8.65.16294441341700020-3", method: "POST", path: "/createOrder", host: "10.100.18.175:20880", user_agent: Some("Apache-HttpClient/4.5.10 (Java/1.8.0_212)"), referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: Some(247), resp_content_length: None, status_code: None, status: Ok, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Response, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: Some(200), status: Ok, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Other, raw_data_type: RawProtocol, stream_id: None, version: "", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: None, status: Ok, attributes: [] } is_http: false
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "3912196de0cf41f4bab8a8a8108fc3a8.56.16294441349520027", span_id: "3912196de0cf41f4bab8a8a8108fc3a8.56.16294441349520026-4", method: "POST", path: "/createOrder", host: "10.100.18.175:20880", user_agent: Some("Apache-HttpClient/4.5.10 (Java/1.8.0_212)"), referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: Some(350), resp_content_length: None, status_code: None, status: Ok, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Response, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: Some(200), status: Ok, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Other, raw_data_type: RawProtocol, stream_id: None, version: "", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: None, status: Ok, attributes: [] } is_http: false
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "3912196de0cf41f4bab8a8a8108fc3a8.63.16294441329780027", span_id: "3912196de0cf41f4bab8a8a8108fc3a8.63.16294441329780026-4", method: "POST", path: "/createOrder", host: "10.100.18.175:20880", user_agent: Some("Apache-HttpClient/4.5.10 (Java/1.8.0_212)"), referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: Some(351), resp_content_length: None, status_code: None, status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Response, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: Some(200), status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Other, raw_data_type: RawProtocol, stream_id: None, version: "", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: None, status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: false
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "3912196de0cf41f4bab8a8a8108fc3a8.65.16294441341700021", span_id: "3912196de0cf41f4bab8a8a8108fc3a8.65.16294441341700020-3", method: "POST", path: "/createOrder", host: "10.100.18.175:20880", user_agent: Some("Apache-HttpClient/4.5.10 (Java/1.8.0_212)"), referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: Some(247), resp_content_length: None, status_code: None, status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Response, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: Some(200), status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Other, raw_data_type: RawProtocol, stream_id: None, version: "", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: None, status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: false
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Request, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "3912196de0cf41f4bab8a8a8108fc3a8.56.16294441349520027", span_id: "3912196de0cf41f4bab8a8a8108fc3a8.56.16294441349520026-4", method: "POST", path: "/createOrder", host: "10.100.18.175:20880", user_agent: Some("Apache-HttpClient/4.5.10 (Java/1.8.0_212)"), referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: Some(350), resp_content_length: None, status_code: None, status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Response, raw_data_type: RawProtocol, stream_id: None, version: "1.1", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: Some(200), status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: true
HttpInfo { is_req_end: false, is_resp_end: false, rrt: 0, proto: Http1, is_tls: false, msg_type: Other, raw_data_type: RawProtocol, stream_id: None, version: "", trace_id: "", span_id: "", method: "", path: "", host: "", user_agent: None, referer: None, client_ip: "", x_request_id_0: "", x_request_id_1: "", req_content_length: None, resp_content_length: None, status_code: None, status: Ok, custom_endpoint: None, custom_result: None, custom_exception: None, attributes: [] } is_http: false
Binary file modified agent/resources/test/plugins/wasm_test.wasm.gz
View file
Binary file not shown.
77 changes: 68 additions & 9 deletions agent/src/flow_generator/protocol_logs/http.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ use super::{decode_new_rpc_trace_context_with_type, LogMessageType};

use crate::common::flow::L7PerfStats;
use crate::common::l7_protocol_log::L7ParseResult;
use crate::plugin::CustomInfo;
use crate::{
common::{
ebpf::EbpfType,
Expand Down Expand Up @@ -99,12 +100,68 @@ pub struct HttpInfo {
#[serde(rename = "response_code", skip_serializing_if = "Option::is_none")]
pub status_code: Option<i32>,
#[serde(rename = "response_status")]
status: L7ResponseStatus,
pub status: L7ResponseStatus,

// set by wasm plugin
custom_endpoint: Option<String>,
custom_result: Option<String>,
custom_exception: Option<String>,

#[serde(skip)]
attributes: Vec<KeyVal>,
}

impl HttpInfo {
pub fn merge_custom_to_http1(&mut self, custom: CustomInfo) {
// req rewrite
if !custom.req.domain.is_empty() {
self.host = custom.req.domain;
}

if !custom.req.req_type.is_empty() {
self.method = custom.req.req_type;
}

if !custom.req.resource.is_empty() {
self.path = custom.req.resource;
}

if !custom.req.endpoint.is_empty() {
self.custom_endpoint = Some(custom.req.endpoint)
}

//req write
if custom.resp.code.is_some() {
self.status_code = custom.resp.code;
}

if custom.resp.status != self.status {
self.status = custom.resp.status;
}

if !custom.resp.result.is_empty() {
self.custom_result = Some(custom.resp.result)
}

if !custom.resp.exception.is_empty() {
self.custom_exception = Some(custom.resp.exception)
}

//trace info rewrite
if custom.trace.trace_id.is_some() {
self.trace_id = custom.trace.trace_id.unwrap();
}
if custom.trace.span_id.is_some() {
self.span_id = custom.trace.span_id.unwrap();
}

// extend attribute
if !custom.attributes.is_empty() {
self.attributes.extend(custom.attributes);
}
}
}

impl L7ProtocolInfoInterface for HttpInfo {
fn session_id(&self) -> Option<u32> {
self.stream_id
Expand Down Expand Up @@ -309,7 +366,12 @@ impl From<HttpInfo> for L7ProtocolSendLog {
f.path,
)
} else {
(f.method, f.path.clone(), f.host, String::new())
(
f.method,
f.path.clone(),
f.host,
f.custom_endpoint.unwrap_or_default(),
)
};

L7ProtocolSendLog {
Expand All @@ -325,7 +387,8 @@ impl From<HttpInfo> for L7ProtocolSendLog {
resp: L7Response {
status: f.status,
code: f.status_code,
..Default::default()
exception: f.custom_exception.unwrap_or_default(),
result: f.custom_result.unwrap_or_default(),
},
trace_info: Some(TraceInfo {
trace_id: Some(f.trace_id),
Expand Down Expand Up @@ -999,12 +1062,8 @@ impl HttpLog {
PacketDirection::ClientToServer => vm.on_http_req(payload, param, info),
PacketDirection::ServerToClient => vm.on_http_resp(payload, param, info),
}
.map(|(trace, kv)| {
if let Some(trace) = trace {
trace.trace_id.map(|s| info.trace_id = s);
trace.span_id.map(|s| info.span_id = s);
}
info.attributes.extend(kv);
.map(|custom| {
info.merge_custom_to_http1(custom);
});
}
}
Expand Down
Loading

0 comments on commit 5d8494d

Please sign in to comment.