#!/usr/bin/env python3 """Avro is the smallest -- and unreadable without its writer's schema. DDIA 2e Ch. 5. Avro stores only values in schema order: no field tags, no field names. That makes it the most compact of the binary encodings, but it also means the bytes are just an undifferentiated stream of values. Decode them with the WRONG schema and you do not get an error you can localize -- you get silent garbage, or a hard failure, because there is nothing in the data that says where one field ends and the next begins. Protocol Buffers, which tags every field with a small integer, can still recover the fields it knows even from a partial or reordered schema. Run it: uv run --python 3.12 --with fastavro --with grpcio-tools --with protobuf \ python3 avro_needs_schema.py (Protobuf's .proto is compiled at runtime via grpc_tools, so no separate protoc step is needed. Deterministic -> the bytes and outcomes reproduce exactly.) """ import io import os import sys import tempfile # One record: a person. Same values across Avro and Protobuf below. RECORD = {"name": "Zhang Wei", "birthYear": 1984, "city": "Shanghai"} # --------------------------------------------------------------------------- # (a) + (b): Avro -- smallest, and undecodable without the writer's schema. # --------------------------------------------------------------------------- # The schema the writer used. Field ORDER is part of the contract: Avro writes # the values back-to-back in exactly this order, with no names and no tags. WRITER_SCHEMA = { "type": "record", "name": "Person", "fields": [ {"name": "name", "type": "string"}, {"name": "birthYear", "type": "long"}, {"name": "city", "type": "string"}, ], } # A DIFFERENT schema: same field names and types, but city and birthYear are # swapped. To Avro this is just "string, string, long" vs the writer's # "string, long, string" -- a different byte layout entirely. WRONG_SCHEMA = { "type": "record", "name": "Person", "fields": [ {"name": "name", "type": "string"}, {"name": "city", "type": "string"}, {"name": "birthYear", "type": "long"}, ], } def avro_encode(): import fastavro buf = io.BytesIO() fastavro.schemaless_writer(buf, fastavro.parse_schema(WRITER_SCHEMA), RECORD) return buf.getvalue() def avro_decode(avro_bytes, schema): import fastavro buf = io.BytesIO(avro_bytes) return fastavro.schemaless_reader(buf, fastavro.parse_schema(schema)) # --------------------------------------------------------------------------- # (c): Protobuf -- tags make each field self-locating, so a reader that knows # only some of the tags still recovers the fields it knows. # --------------------------------------------------------------------------- def protobuf_encode(): """Full writer: name=1, birth_year=2, city=3.""" work = tempfile.mkdtemp() proto = os.path.join(work, "person_full.proto") with open(proto, "w") as f: f.write('syntax = "proto3";\n' 'package full;\n' 'message Person {\n' ' string name = 1;\n' ' int64 birth_year = 2;\n' ' string city = 3;\n' '}\n') from grpc_tools import protoc protoc.main(["protoc", f"-I{work}", f"--python_out={work}", proto]) sys.path.insert(0, work) import person_full_pb2 p = person_full_pb2.Person(name=RECORD["name"], birth_year=RECORD["birthYear"], city=RECORD["city"]) return p.SerializeToString() def protobuf_decode_partial(pb_bytes): """Partial reader: knows tags 1 and 3 only. Field 2 (birth_year) is unknown, and field 3 has been RENAMED (city -> town) -- but same tag 3, so it still reads correctly. The name is only in the schema, never on the wire. proto3 preserves unknown fields, so re-serializing the partial parse still carries tag 2 -- we prove it by re-reading the result with the FULL schema.""" work = tempfile.mkdtemp() proto = os.path.join(work, "person_partial.proto") with open(proto, "w") as f: f.write('syntax = "proto3";\n' 'package partial;\n' 'message Person {\n' ' string name = 1;\n' ' string town = 3;\n' # tag 2 (birth_year) omitted; 3 renamed '}\n') from grpc_tools import protoc protoc.main(["protoc", f"-I{work}", f"--python_out={work}", proto]) sys.path.insert(0, work) import person_partial_pb2 p = person_partial_pb2.Person() p.ParseFromString(pb_bytes) # unknown tag 2 is tolerated, kept as unknown recovered = {"name": p.name, "town": p.town} # Re-serialize: the unknown tag 2 survives the round-trip. Prove it by # decoding the re-serialized bytes with the full schema. import person_full_pb2 full = person_full_pb2.Person() full.ParseFromString(p.SerializeToString()) return recovered, full.birth_year def hexdump(b): return " ".join(f"{x:02x}" for x in b) def main(): # -- (a) Avro is the smallest, and the bytes are just values -------------- av = avro_encode() print("(a) Avro encoding of {name, birthYear, city} -- values only, in " "schema order:") print(f" {len(av)} bytes: {hexdump(av)}") print(" no field names, no tags -- just a length + \"Zhang Wei\", a " "varint 1984, a length + \"Shanghai\".") print() # -- (b) Decode with the RIGHT schema, then the WRONG schema -------------- right = avro_decode(av, WRITER_SCHEMA) print("(b) Decode those same bytes with the WRITER's schema " "(name, birthYear, city):") print(f" -> {right} (correct)") print() print(" Now decode the SAME bytes with a different schema -- city and " "birthYear swapped") print(" (to Avro: \"string, string, long\" instead of \"string, long, " "string\"):") try: wrong = avro_decode(av, WRONG_SCHEMA) print(f" -> {wrong}") print(" Garbage: the reader took the 1-byte varint 1984 as a string " "length and mis-sliced the rest.") except Exception as e: print(f" -> RAISED {type(e).__name__}: {e}") print(" The bytes carry no field boundaries, so a wrong schema " "cannot even be applied. Either way:") print(" without the exact writer's schema, the data is unreadable.") print() # -- (c) Protobuf: tags let a partial/renamed reader recover fields ------- pb = protobuf_encode() print(f"(c) Protobuf encoding of the same record: {len(pb)} bytes: " f"{hexdump(pb)}") print(" Each field is a (tag, value) pair. Decode with a PARTIAL schema " "that knows only") print(" tag 1 and tag 3 -- and has renamed tag 3's field city -> town, " "and dropped tag 2:") recovered, preserved_birth_year = protobuf_decode_partial(pb) print(f" -> {recovered}") print(f" tag 2 (birthYear) was unknown, but preserved: re-serialize " f"and a full reader still gets {preserved_birth_year}.") print(" The tags made name and city self-locating despite the rename " "and the missing field.") print() # -- (d) The consequence -------------------------------------------------- print("(d) So Avro must SHIP the writer's schema alongside the data:") print(" - an Avro object-container file writes the schema once, in its " "header;") print(" - a database or stream stores a schema version/ID and looks it " "up in a registry.") print(" The bytes alone are undecodable -- that is the price of being " "the smallest.") if __name__ == "__main__": main()