feat(fetch): count bytes on the wire as well as decoded (#760)

fetch-stats now reports wire_bytes (urllib3's raw socket byte count) beside the decoded bytes in every counter set. ESPN gzips its scoreboards, so the decoded count overstated real traffic ~10-14x: ledpi measured football at 33.2 MB/h decoded vs 3.15 MB/h on the wire.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chuck
2026-10-04 18:56:32 -04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent caec9f5bf5
commit ee7c3389f9
3 changed files with 78 additions and 1 deletions
+36
View File
@@ -576,6 +576,42 @@ class TestCounters:
assert snap["hosts"]["site.api.espn.com"]["requests"] == 1
assert snap["totals"]["bytes"] == 3 * len(b'{"ok": 1}')
def test_wire_bytes_are_the_compressed_size(self, service):
# Built the way requests builds a real response: a urllib3
# HTTPResponse carrying a gzip body, decoded when .content is read.
import gzip
import io
from requests.adapters import HTTPAdapter
from urllib3.response import HTTPResponse
decoded = json.dumps({"events": [{"id": str(i), "name": "x" * 200}
for i in range(50)]}).encode()
wire = gzip.compress(decoded)
def handler(url, kwargs):
raw = HTTPResponse(body=io.BytesIO(wire), status=200,
headers={"Content-Encoding": "gzip",
"Content-Type": "application/json"},
preload_content=False, decode_content=True)
request = requests.Request("GET", url).prepare()
response = HTTPAdapter().build_response(request, raw)
response.content # what Session.get does for a non-streamed call
return response
response = service.get(FakeSession(handler), "https://site.api.espn.com/x")
assert response.content == decoded
totals = _counters(service)
assert totals["bytes"] == len(decoded)
assert totals["wire_bytes"] == len(wire) < len(decoded)
def test_wire_bytes_fall_back_to_the_decoded_size(self, service):
# No urllib3 response behind it (a test double, another adapter):
# count what is known rather than nothing.
service.get(FakeSession(), "https://api.test/x")
totals = _counters(service)
assert totals["wire_bytes"] == totals["bytes"] == len(b'{"ok": 1}')
def test_errors_and_http_errors(self, service):
def handler(url, kwargs):
if url.endswith("/down"):