feat(fetch): count bytes on the wire as well as decoded

fetch-stats reported only `bytes`, len(response.content), and that read as
the download volume. ESPN gzips every scoreboard, so it overstated real
traffic about 14x: a college football Saturday is 865 KB decoded, 63 KB on
the wire. Every counter set now carries `wire_bytes`, read from urllib3's
count of raw bytes taken off the socket (decoded size when there is no
urllib3 response behind it).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chuck
2026-10-04 17:19:43 -04:00
co-authored by Claude Opus 5.5
parent a74b5a2f0f
commit b210fb6083
3 changed files with 78 additions and 1 deletions
+36
View File
@@ -576,6 +576,42 @@ class TestCounters:
assert snap["hosts"]["site.api.espn.com"]["requests"] == 1
assert snap["totals"]["bytes"] == 3 * len(b'{"ok": 1}')
def test_wire_bytes_are_the_compressed_size(self, service):
# Built the way requests builds a real response: a urllib3
# HTTPResponse carrying a gzip body, decoded when .content is read.
import gzip
import io
from requests.adapters import HTTPAdapter
from urllib3.response import HTTPResponse
decoded = json.dumps({"events": [{"id": str(i), "name": "x" * 200}
for i in range(50)]}).encode()
wire = gzip.compress(decoded)
def handler(url, kwargs):
raw = HTTPResponse(body=io.BytesIO(wire), status=200,
headers={"Content-Encoding": "gzip",
"Content-Type": "application/json"},
preload_content=False, decode_content=True)
request = requests.Request("GET", url).prepare()
response = HTTPAdapter().build_response(request, raw)
response.content # what Session.get does for a non-streamed call
return response
response = service.get(FakeSession(handler), "https://site.api.espn.com/x")
assert response.content == decoded
totals = _counters(service)
assert totals["bytes"] == len(decoded)
assert totals["wire_bytes"] == len(wire) < len(decoded)
def test_wire_bytes_fall_back_to_the_decoded_size(self, service):
# No urllib3 response behind it (a test double, another adapter):
# count what is known rather than nothing.
service.get(FakeSession(), "https://api.test/x")
totals = _counters(service)
assert totals["wire_bytes"] == totals["bytes"] == len(b'{"ok": 1}')
def test_errors_and_http_errors(self, service):
def handler(url, kwargs):
if url.endswith("/down"):