1"""Chapter 2 practice: a back-of-envelope estimator (Level 3 Build) and the
2bits-vs-bytes bug (Level 4 Debug). Run: python3 ch02_estimator.py"""
3
4SECONDS_PER_DAY = 86_400
5
6
7def human_bytes(n):
8 """1,460,000,000,000,000 -> '1.46 PB' (decimal units, as used in estimates)."""
9 for unit in ["B", "KB", "MB", "GB", "TB", "PB", "EB"]:
10 if abs(n) < 1000:
11 return f"{n:.3g} {unit}"
12 n /= 1000
13 return f"{n:.3g} ZB"
14
15
16def estimate(dau, reads_per_user, writes_per_user, write_size_bytes,
17 read_size_bytes, peak_factor=3, years=1, replicas=1, cache_fraction=0.2):
18 reads_per_day = dau * reads_per_user
19 writes_per_day = dau * writes_per_user
20 read_rps = reads_per_day / SECONDS_PER_DAY
21 write_rps = writes_per_day / SECONDS_PER_DAY
22 storage = writes_per_day * write_size_bytes * 365 * years * replicas
23 egress_bytes_per_s = reads_per_day * read_size_bytes / SECONDS_PER_DAY
24 return {
25 "read_rps": read_rps,
26 "write_rps": write_rps,
27 "peak_read_rps": read_rps * peak_factor,
28 "read_write_ratio": reads_per_day / writes_per_day if writes_per_day else float("inf"),
29 "storage_bytes": storage,
30 "egress_gbps": egress_bytes_per_s * 8 / 1e9, # bytes -> bits, then giga
31 "cache_bytes": reads_per_day * read_size_bytes * cache_fraction,
32 }
33
34
35def servers_needed(peak_rps, rps_per_server, target_utilisation=0.6, spare=1):
36 """Servers to carry the peak at the target utilisation, plus spares for failures."""
37 import math
38 return math.ceil(peak_rps / (rps_per_server * target_utilisation)) + spare
39
40
41# ---- Level 4: the buggy version ------------------------------------------------
42
43def buggy_egress_gbps(reads_per_day, read_size_bytes):
44 bytes_per_s = reads_per_day * read_size_bytes / SECONDS_PER_DAY
45 return bytes_per_s / 1e9 # BUG: this is gigaBYTES per second, not gigaBITS
46
47
48def fixed_egress_gbps(reads_per_day, read_size_bytes):
49 bytes_per_s = reads_per_day * read_size_bytes / SECONDS_PER_DAY
50 return bytes_per_s * 8 / 1e9
51
52
53def close(a, b, tol=0.01):
54 return abs(a - b) <= tol * abs(b)
55
56
57def run_checks():
58 # The chapter's running example: a photo-sharing app.
59 # 10M daily users, 50 photo views each, 0.2 uploads each (2M uploads a day),
60 # 2 MB per uploaded photo, 200 KB per photo served.
61 e = estimate(dau=10_000_000, reads_per_user=50, writes_per_user=0.2,
62 write_size_bytes=2_000_000, read_size_bytes=200_000)
63
64 assert close(e["read_rps"], 5787.04) # about 5,800 reads a second
65 assert close(e["write_rps"], 23.15) # about 23 uploads a second
66 assert close(e["peak_read_rps"], 17361.1) # about 17,400 at 3x peak
67 assert close(e["read_write_ratio"], 250)
68 assert close(e["storage_bytes"], 1.46e15) # 1.46 PB a year, one copy
69 assert close(e["egress_gbps"], 9.26) # about 9.3 Gbps out
70 assert close(e["cache_bytes"], 2e13) # 20% of a day's image bytes
71
72 with3 = estimate(10_000_000, 50, 0.2, 2_000_000, 200_000, replicas=3)
73 assert close(with3["storage_bytes"], 4.38e15)
74
75 assert human_bytes(1.46e15) == "1.46 PB"
76 assert human_bytes(4e12) == "4 TB"
77
78 # 17,400 peak reads, 1,000 per server, run at 60%, keep 1 spare.
79 assert servers_needed(17_361, 1_000) == 30
80
81 # Level 4: the bug is exactly a factor of 8.
82 bad = buggy_egress_gbps(500_000_000, 200_000)
83 good = fixed_egress_gbps(500_000_000, 200_000)
84 assert close(good / bad, 8)
85 assert close(good, 9.26)
86
87 print("All checks pass.")
88 print(f"Photo app: {e['read_rps']:,.0f} reads/s, {e['write_rps']:.0f} writes/s, "
89 f"peak {e['peak_read_rps']:,.0f}, storage {human_bytes(e['storage_bytes'])}/yr, "
90 f"egress {e['egress_gbps']:.2f} Gbps. Buggy egress: {bad:.2f} (8x too small).")
91
92
93if __name__ == "__main__":
94 run_checks()