mirror of
https://github.com/storytold/craft-anything.git
synced 2026-11-21 15:00:49 +01:00
Count JSON responses, not all requests, for body outlines (#1845)
`web_page_show` read bodies for the first 200 requests of any kind, so a page that loads its data after a few hundred scripts, styles and telemetry calls got no JSON outline at all, and nothing said why. The cap now counts JSON responses, the network result counts them, and when some got no outline the evidence says how many and why: past the cap, after the window's deadline, or a body that couldn't be read as JSON (part of #1811).
This commit is contained in:
1 parent
e52b21539f
commit
a3e5d04df6
2 files changed
+101
-9
No files matched your search
@@ -315,6 +315,8 @@ const SCRIPT_RECORD_CAP: usize = 50_000;
|
||||
const SCRIPT_META_BUDGET: usize = 8 << 20;
|
||||
const SOURCE_TEXT_BUDGET: usize = 32 << 20;
|
||||
const NETWORK_RECORD_CAP: usize = 20_000;
|
||||
/// JSON responses read for an outline; counts JSON responses, not all requests (#1811).
|
||||
const JSON_OUTLINE_CAP: usize = 200;
|
||||
|
||||
/// Enable the needed domains, observe for `watch_ms`, and build a data-minimised capture.
|
||||
pub(crate) fn capture_page(
|
||||
@@ -567,17 +569,24 @@ pub(crate) fn collect_network(
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
// Optional JSON body outlines for JSON responses.
|
||||
// Optional JSON body outlines for JSON responses. The cap counts JSON responses, not every
|
||||
// request: a page that loads its data after 230 scripts, styles and telemetry calls still got
|
||||
// outlines for what mattered (#1811).
|
||||
let mut json = None;
|
||||
if want_body_outlines {
|
||||
let ids: Vec<String> = order.iter().take(200).cloned().collect();
|
||||
for id in ids {
|
||||
let json_ids: Vec<String> = order
|
||||
.iter()
|
||||
.filter(|id| reqs.get(*id).and_then(|e| e.get("mime_type")).and_then(Value::as_str).is_some_and(|m| m.contains("json")))
|
||||
.cloned()
|
||||
.collect();
|
||||
let mut outlined = 0u64;
|
||||
let mut tried = 0u64;
|
||||
let mut ids = json_ids.iter().take(JSON_OUTLINE_CAP);
|
||||
for id in ids.by_ref() {
|
||||
if dl.expired() {
|
||||
break;
|
||||
}
|
||||
let is_json = reqs.get(&id).and_then(|e| e.get("mime_type")).and_then(Value::as_str).is_some_and(|m| m.contains("json"));
|
||||
if !is_json {
|
||||
continue;
|
||||
}
|
||||
tried += 1;
|
||||
if let Ok(body) = client.call("Network.getResponseBody", json!({"requestId": id}), None, dl)
|
||||
&& let Some(text) = body.get("body").and_then(Value::as_str)
|
||||
{
|
||||
@@ -588,12 +597,26 @@ pub(crate) fn collect_network(
|
||||
};
|
||||
if let Some(t) = decoded
|
||||
&& let Ok(v) = serde_json::from_str::<Value>(&t)
|
||||
&& let Some(entry) = reqs.get_mut(&id)
|
||||
&& let Some(entry) = reqs.get_mut(id)
|
||||
{
|
||||
entry["json_body_outline"] = web_common::json_outline(&v);
|
||||
outlined += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Why each JSON response got no outline, so a missing outline isn't read as a response with
|
||||
// no JSON body: it was past the cap, the deadline ended the reading, or its body couldn't
|
||||
// be read as JSON (a redirect, a body Chrome no longer holds, text that isn't JSON).
|
||||
let total = json_ids.len() as u64;
|
||||
let past_cap = total.saturating_sub(JSON_OUTLINE_CAP as u64);
|
||||
let after_deadline = total.saturating_sub(past_cap).saturating_sub(tried);
|
||||
json = Some(json!({
|
||||
"json_responses": total,
|
||||
"json_outlines": outlined,
|
||||
"json_outlines_past_cap": past_cap,
|
||||
"json_outlines_after_deadline": after_deadline,
|
||||
"json_outlines_unreadable": tried.saturating_sub(outlined),
|
||||
}));
|
||||
}
|
||||
if content.response_body {
|
||||
let ids: Vec<String> = order.iter().take(200).cloned().collect();
|
||||
@@ -619,7 +642,45 @@ pub(crate) fn collect_network(
|
||||
}
|
||||
}
|
||||
let items: Vec<Value> = order.iter().filter_map(|id| reqs.get(id).cloned()).collect();
|
||||
json!({"request_count": items.len(), "requests": items})
|
||||
let mut out = json!({"request_count": items.len(), "requests": items});
|
||||
if let Some(json) = json {
|
||||
out["json_outlines"] = json;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// How many JSON responses in the window got a body outline. When some didn't, a limitation says
|
||||
/// so, so an absent outline is never read as "this response had no JSON body" (#1811).
|
||||
fn json_outline_limitation(result: &Value) -> Option<Limitation> {
|
||||
let counts = result.pointer("/network/json_outlines")?;
|
||||
let total = counts.get("json_responses")?.as_u64()?;
|
||||
let count = |key: &str| counts.get(key).and_then(Value::as_u64).unwrap_or(0);
|
||||
let outlined = count("json_outlines");
|
||||
if total <= outlined {
|
||||
return None;
|
||||
}
|
||||
let mut why = Vec::new();
|
||||
for (key, words) in [
|
||||
("json_outlines_past_cap", format!("past the first {JSON_OUTLINE_CAP} JSON responses")),
|
||||
("json_outlines_after_deadline", "after the window's deadline".to_string()),
|
||||
("json_outlines_unreadable", "whose body couldn't be read as JSON".to_string()),
|
||||
] {
|
||||
let n = count(key);
|
||||
if n > 0 {
|
||||
why.push(format!("{n} {words}"));
|
||||
}
|
||||
}
|
||||
let rest = match why.split_last() {
|
||||
None => String::new(),
|
||||
Some((last, [])) => last.clone(),
|
||||
Some((last, head)) => format!("{} and {last}", head.join(", ")),
|
||||
};
|
||||
Some(Limitation::new(
|
||||
"json_outlines_partial",
|
||||
format!(
|
||||
"{outlined} of {total} JSON responses got a body outline; the rest were {rest}, so a missing outline doesn't mean the response had no JSON body."
|
||||
),
|
||||
))
|
||||
}
|
||||
|
||||
fn collect_websockets(events: &[cdp::CdpEvent], secrets: &mut Secrets) -> Value {
|
||||
@@ -835,6 +896,7 @@ fn passive_evidence(
|
||||
declared_secrets: usize,
|
||||
redactions: u64,
|
||||
) -> Result<craft_anything_evidence::Evidence, ToolError> {
|
||||
let outlines_partial = json_outline_limitation(&result);
|
||||
let mut b = EvidenceBuilder::new(op, Subject { kind: "website".into(), sha256: None, name: Some(origin.to_string()) }, browser_env(clean_room, browser))
|
||||
.params(json!({"endpoint": browser.endpoint.origin, "sensitive_value_declarations": declared_secrets}))
|
||||
.result(result)
|
||||
@@ -851,6 +913,9 @@ fn passive_evidence(
|
||||
))
|
||||
.coverage(Coverage { status: CoverageStatus::Partial, examined: None, total: None, next_offset: None })
|
||||
.location(Location::new("url", origin.to_string()));
|
||||
if let Some(partial) = outlines_partial {
|
||||
b = b.limitation(partial);
|
||||
}
|
||||
if dropped > 0 {
|
||||
b = b.limitation(Limitation::new("events_dropped", format!("{dropped} event(s) were dropped against the capture budget")));
|
||||
}
|
||||
|
||||
@@ -995,3 +995,30 @@ fn same_origin_loopback_source_maps_are_fetched() {
|
||||
assert!(other_seen.lock().unwrap().is_empty());
|
||||
assert_eq!(seen.lock().unwrap().len(), 2);
|
||||
}
|
||||
|
||||
/// #1811: a JSON response with no outline is said to be unread, not taken as a response with no
|
||||
/// body; when every JSON response got one there is nothing to say.
|
||||
#[test]
|
||||
fn a_missing_json_outline_is_said_in_the_evidence() {
|
||||
// Each reason a JSON response got no outline is named with its own count, so a response whose
|
||||
// body couldn't be read isn't said to be past the cap.
|
||||
let partial = json!({"network": {"json_outlines": {
|
||||
"json_responses": 60, "json_outlines": 39,
|
||||
"json_outlines_past_cap": 0, "json_outlines_after_deadline": 0, "json_outlines_unreadable": 21,
|
||||
}}});
|
||||
let limitation = super::json_outline_limitation(&partial).expect("39 of 60 went unread");
|
||||
assert_eq!(limitation.code, "json_outlines_partial");
|
||||
assert!(limitation.message.contains("39 of 60"), "{}", limitation.message);
|
||||
assert!(limitation.message.contains("21 whose body couldn't be read as JSON"), "{}", limitation.message);
|
||||
assert!(!limitation.message.contains("past the first"), "{}", limitation.message);
|
||||
let mixed = json!({"network": {"json_outlines": {
|
||||
"json_responses": 250, "json_outlines": 150,
|
||||
"json_outlines_past_cap": 50, "json_outlines_after_deadline": 0, "json_outlines_unreadable": 50,
|
||||
}}});
|
||||
let limitation = super::json_outline_limitation(&mixed).expect("150 of 250");
|
||||
assert!(limitation.message.contains("past the first 200 JSON responses") && limitation.message.contains("and 50 whose"), "{}", limitation.message);
|
||||
let all = json!({"network": {"json_outlines": {"json_responses": 39, "json_outlines": 39}}});
|
||||
assert!(super::json_outline_limitation(&all).is_none());
|
||||
// Outlines weren't asked for at all: nothing to report.
|
||||
assert!(super::json_outline_limitation(&json!({"network": {"request_count": 3}})).is_none());
|
||||
}
|
||||
Reference in new issue
Block a user