Use upb for protobufs

.gitignore

@@ -9,6 +9,10 @@ pg_query.lib
 examples/*
 !examples/*.c
 
+benchmark/*
+!benchmark/*.c
+!benchmark/*.md
+
 test/*
 !test/*.c
 !test/*.sql
@@ -21,3 +25,5 @@ tmp/*
 !tmp/.gitkeep
 
 .vs/*
+
+vendor/upb/.upb-update-tmp/

Makefile

@@ -11,6 +11,7 @@ PG_VERSION = 18.4
 PG_VERSION_MAJOR = $(call word-dot,$(PG_VERSION),1)
 PG_VERSION_NUM = 180004
 PROTOC_VERSION = 25.1
+UPB_PROTOC_VERSION := $(patsubst v%,%,$(shell head -n1 vendor/upb/VERSION 2>/dev/null))
 
 VERSION = 18.0.0
 VERSION_MAJOR = $(call word-dot,$(VERSION),1)
@@ -31,10 +32,14 @@ else
 	SOFLAG = -soname
 endif
 
-SRC_FILES := $(wildcard src/*.c src/postgres/*.c) vendor/protobuf-c/protobuf-c.c vendor/xxhash/xxhash.c protobuf/pg_query.pb-c.c
+UPB_DIR := vendor/upb
+UPB_INCLUDES := -I./$(UPB_DIR) -I./$(UPB_DIR)/third_party/utf8_range
+UPB_SRC_FILES := $(UPB_DIR)/upb.c $(UPB_DIR)/third_party/utf8_range/utf8_range.c protobuf/pg_query.upb_minitable.c protobuf/pg_query.upb.c protobuf/pg_query.enum_names.c
+
+SRC_FILES := $(wildcard src/*.c src/postgres/*.c) vendor/xxhash/xxhash.c $(UPB_SRC_FILES)
 OBJ_FILES := $(SRC_FILES:.c=.o)
 
-override CFLAGS += -g -I. -I./vendor -I./src/include -I./src/postgres/include -Wall -Wno-unused-function -Wno-unused-value -Wno-unused-variable -fno-strict-aliasing -fwrapv -fPIC
+override CFLAGS += -g -I. -I./vendor $(UPB_INCLUDES) -I./src/include -I./src/postgres/include -Wall -Wno-unused-function -Wno-unused-value -Wno-unused-variable -fno-strict-aliasing -fwrapv -fPIC
 
 ifeq ($(OS),Windows_NT)
 override CFLAGS += -I./src/postgres/include/port/win32
@@ -43,7 +48,7 @@ endif
 
 override PG_CONFIGURE_FLAGS += -q --without-readline --without-zlib --without-icu
 
-override TEST_CFLAGS += -g -I. -I./vendor -Wall
+override TEST_CFLAGS += -g -I. -I./vendor $(UPB_INCLUDES) -Wall
 override TEST_LDFLAGS += -pthread
 
 CFLAGS_OPT_LEVEL = -O3
@@ -109,11 +114,11 @@ build: $(ARLIB)
 build_shared: $(SOLIB)
 
 clean:
-	-@ $(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) $(EXAMPLES) $(TESTS)
-	-@ $(RM) -rf {test,examples}/*.dSYM
+	-@ $(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) $(EXAMPLES) $(TESTS) $(BENCHMARKS)
+	-@ $(RM) -rf {test,examples,benchmark}/*.dSYM
 	-@ $(RM) -r $(PGDIR) $(PGDIRBZ2) $(PGDIRZIP)
 
-.PHONY: all clean build build_shared extract_source examples test install
+.PHONY: all clean build build_shared extract_source examples test benchmark install
 
 $(PGDIR):
 	curl -o $(PGDIRBZ2) https://ftp.postgresql.org/pub/source/v$(PG_VERSION)/postgresql-$(PG_VERSION).tar.bz2
@@ -194,14 +199,20 @@ $(ARLIB): $(OBJ_FILES) Makefile
 $(SOLIB): $(OBJ_FILES) Makefile
 	@$(CC) $(CFLAGS) -shared -Wl,$(SOFLAG),$(SONAME) $(LDFLAGS) -o $@ $(OBJ_FILES) $(LIBS)
 
-protobuf/pg_query.pb-c.c protobuf/pg_query.pb-c.h: protobuf/pg_query.proto
-ifneq ($(shell which protoc-gen-c), )
-	protoc --c_out=. protobuf/pg_query.proto
+# upb-generated message code + minitables (regenerated only when the upb protoc
+# plugins are installed; otherwise the committed files are used as-is). Must be
+# generated with a protoc/protoc-gen-upb matching vendor/upb/VERSION.
+protobuf/pg_query.upb.h protobuf/pg_query.upb.c protobuf/pg_query.upb_minitable.h protobuf/pg_query.upb_minitable.c: protobuf/pg_query.proto
+ifneq ($(shell which protoc-gen-upb), )
+ifneq ($(shell protoc --version 2>/dev/null | cut -f2 -d" "), $(UPB_PROTOC_VERSION))
+	$(error ERROR - upb codegen needs protoc $(UPB_PROTOC_VERSION) to match vendor/upb/VERSION (found "$(shell protoc --version 2>/dev/null | cut -f2 -d' ')"); run 'make -C vendor/upb update TAG=...' to change the pinned version)
+endif
+	protoc --upb_out=. --upb_minitable_out=. protobuf/pg_query.proto
 else
-	@echo 'Warning: protoc-gen-c not found, skipping protocol buffer regeneration'
+	@echo 'Warning: protoc-gen-upb not found, skipping upb regeneration'
 endif
 
-src/pg_query_protobuf.c src/pg_query_scan.c: protobuf/pg_query.pb-c.h
+src/pg_query_scan.c: protobuf/pg_query.upb.h
 
 # Only used when USE_PROTOBUF_CPP is used (experimental for testing only)
 src/pg_query_outfuncs_protobuf_cpp.cc: protobuf/pg_query.pb.cc
@@ -211,6 +222,18 @@ ifneq ($(shell protoc --version 2>/dev/null | cut -f2 -d" "), $(PROTOC_VERSION))
 endif
 	protoc --cpp_out=. protobuf/pg_query.proto
 
+BENCHMARKS = benchmark/bench_protobuf benchmark/microbench_protobuf
+BENCHMARK_CFLAGS = $(TEST_CFLAGS) -O2 -I./src -I./src/include -I./src/postgres/include
+benchmark: $(BENCHMARKS)
+	benchmark/bench_protobuf
+	benchmark/microbench_protobuf
+
+benchmark/bench_protobuf: benchmark/bench_protobuf.c $(ARLIB)
+	$(CC) $(BENCHMARK_CFLAGS) -o $@ benchmark/bench_protobuf.c $(ARLIB) $(TEST_LDFLAGS)
+
+benchmark/microbench_protobuf: benchmark/microbench_protobuf.c $(ARLIB)
+	$(CC) $(BENCHMARK_CFLAGS) -o $@ benchmark/microbench_protobuf.c $(ARLIB) $(TEST_LDFLAGS)
+
 EXAMPLES = examples/simple examples/scan examples/normalize examples/simple_error examples/normalize_error examples/simple_plpgsql
 examples: $(EXAMPLES)
 	examples/simple

Makefile.msvc

@@ -3,9 +3,9 @@
 TARGET = pg_query
 ARLIB = $(TARGET).lib
 
-SRC_FILES = src/*.c src/postgres/*.c vendor/protobuf-c/protobuf-c.c vendor/xxhash/xxhash.c protobuf/pg_query.pb-c.c
+SRC_FILES = src/*.c src/postgres/*.c vendor/xxhash/xxhash.c vendor/upb/upb.c vendor/upb/third_party/utf8_range/utf8_range.c protobuf/pg_query.upb.c protobuf/pg_query.upb_minitable.c protobuf/pg_query.enum_names.c
 
-CFLAGS = -I. -I./vendor -I./src/postgres/include -I./src/include -I./src/postgres/include/port/win32 -I./src/postgres/include/port/win32_msvc
+CFLAGS = -I. -I./vendor -I./vendor/upb -I./vendor/upb/third_party/utf8_range -I./src/postgres/include -I./src/include -I./src/postgres/include/port/win32 -I./src/postgres/include/port/win32_msvc
 
 RM = del
 
@@ -14,14 +14,14 @@ all: examples test build
 build: $(ARLIB)
 
 clean:
-	$(RM) *.obj
-	$(RM) pg_query.lib
+	-$(RM) *.obj
+	-$(RM) pg_query.lib
 
 .PHONY: all clean build build_shared extract_source examples test install
 
 $(ARLIB): clean $(SRC_FILES)
 	$(CC) $(CFLAGS) /c $(SRC_FILES)
-    lib /OUT:pg_query.lib *.obj
+	lib /OUT:pg_query.lib *.obj
 
 EXAMPLES = examples/simple examples/scan examples/normalize examples/simple_error examples/normalize_error examples/simple_plpgsql
 examples: $(EXAMPLES)

benchmark/README.md

@@ -0,0 +1,51 @@
+# Protobuf encode/decode benchmarks
+
+Microbenchmarks for the protobuf serialization layer
+(`pg_query_nodes_to_protobuf` / `pg_query_protobuf_to_nodes`, exercised by
+`pg_query_parse_protobuf` / `pg_query_deparse_protobuf`).
+
+## Running
+
+```sh
+make benchmark            # builds libpg_query.a and runs both harnesses
+./benchmark/bench_protobuf      [iters] [label]   # default iters: 20000
+./benchmark/microbench_protobuf [iters] [label]   # default iters: 50000
+```
+
+- **`bench_protobuf`** — end-to-end via the public API: `pg_query_parse_protobuf`
+  (parse + encode) and `pg_query_deparse_protobuf` (decode + deparse). Reports
+  ns/op, wire bytes, and peak RSS.
+- **`microbench_protobuf`** — isolates *only* the protobuf layer by calling the
+  internal encode/decode on pre-parsed trees / pre-encoded bytes, with a scratch
+  memory context reset after each op (so per-node `palloc` churn and arena
+  lifecycle are both counted, while the parse trees persist). Removes the shared
+  parse/deparse cost from the measurement.
+
+Both use the same small mixed-complexity query corpus. Point them at your own
+corpus by editing the `queries[]` array.
+
+## A/B comparing two implementations (e.g. before/after a change)
+
+The benchmarks measure whatever `libpg_query.a` they link against. To compare a
+change against a baseline ref, build the baseline in an isolated worktree and
+run the same harness against each archive:
+
+```sh
+# baseline (e.g. the commit before the change)
+git worktree add /tmp/libpg_base <ref>
+make -C /tmp/libpg_base build
+
+# build the harness against each archive (microbench needs the internal headers)
+cc -O2 -w -I. -Isrc -Isrc/include -Isrc/postgres/include -Ivendor \
+   benchmark/microbench_protobuf.c libpg_query.a -pthread -o /tmp/micro_new
+cc -O2 -w -I/tmp/libpg_base -I/tmp/libpg_base/src -I/tmp/libpg_base/src/include \
+   -I/tmp/libpg_base/src/postgres/include -I/tmp/libpg_base/vendor \
+   benchmark/microbench_protobuf.c /tmp/libpg_base/libpg_query.a -pthread -o /tmp/micro_base
+
+/tmp/micro_base 50000 baseline
+/tmp/micro_new  50000 new
+git worktree remove --force /tmp/libpg_base
+```
+
+Because the parser and deparse walk are identical across builds, the per-op
+*difference* isolates the protobuf-layer change.

benchmark/bench_protobuf.c

@@ -0,0 +1,101 @@
+/*
+ * A/B benchmark for the protobuf encode/decode layer.
+ * Identical source compiled against the old (protobuf-c) and new (upb)
+ * libpg_query.a. Only the protobuf implementation differs between the two
+ * archives; parser and deparse walk are shared, so the per-op delta isolates
+ * the encode/decode cost.
+ */
+#include <pg_query.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <sys/resource.h>
+
+static const char *queries[] = {
+	"SELECT 1",
+	"SELECT id, name FROM users WHERE active = true ORDER BY created_at DESC LIMIT 10",
+	"SELECT u.id, count(*) FROM users u JOIN orders o ON o.user_id = u.id "
+	  "WHERE o.total > 100 GROUP BY u.id HAVING count(*) > 5",
+	"INSERT INTO t (a,b,c,d,e) VALUES (1,2,3,4,5),(6,7,8,9,10),(11,12,13,14,15),"
+	  "(16,17,18,19,20),(21,22,23,24,25),(26,27,28,29,30)",
+	"WITH RECURSIVE cte AS (SELECT 1 AS n UNION ALL SELECT n+1 FROM cte WHERE n < 100) "
+	  "SELECT sum(n) OVER (ORDER BY n ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) FROM cte",
+	"UPDATE accounts SET balance = balance - 50 WHERE id = 7 AND balance >= 50 RETURNING id, balance",
+	"SELECT CASE WHEN a > b THEN 'x' WHEN a < b THEN 'y' ELSE 'z' END, "
+	  "coalesce(nullif(c, 0), d, 1), array_agg(e ORDER BY f) FROM tbl GROUP BY 1",
+	"SELECT * FROM a LEFT JOIN b USING (k) LEFT JOIN c ON c.k = b.k "
+	  "FULL JOIN d ON d.k = c.k WHERE a.x IN (SELECT x FROM e WHERE y = $1)",
+	"CREATE TABLE foo (id serial PRIMARY KEY, name text NOT NULL, "
+	  "data jsonb DEFAULT '{}', created timestamptz DEFAULT now(), CHECK (length(name) > 0))",
+	"SELECT to_char(d, 'YYYY-MM-DD'), extract(epoch FROM d), d + interval '1 day' "
+	  "FROM (SELECT generate_series('2020-01-01'::date, '2020-12-31'::date, '1 day') AS d) s",
+};
+static const int NQ = sizeof(queries) / sizeof(queries[0]);
+
+static double now_sec(void) {
+	struct timespec ts;
+	clock_gettime(CLOCK_MONOTONIC, &ts);
+	return ts.tv_sec + ts.tv_nsec / 1e9;
+}
+
+int main(int argc, char **argv) {
+	int iters = (argc > 1) ? atoi(argv[1]) : 20000;
+	const char *label = (argc > 2) ? argv[2] : "?";
+
+	/* Pre-encode each query once and keep the bytes for the decode benchmark. */
+	PgQueryProtobuf bufs[64];
+	size_t total_bytes = 0;
+	for (int i = 0; i < NQ; i++) {
+		PgQueryProtobufParseResult r = pg_query_parse_protobuf(queries[i]);
+		if (r.error) { fprintf(stderr, "parse error: %s\n", r.error->message); return 1; }
+		bufs[i].len = r.parse_tree.len;
+		bufs[i].data = malloc(r.parse_tree.len);
+		memcpy(bufs[i].data, r.parse_tree.data, r.parse_tree.len);
+		total_bytes += r.parse_tree.len;
+		pg_query_free_protobuf_parse_result(r);
+	}
+
+	/* Warmup */
+	for (int w = 0; w < 200; w++)
+		for (int i = 0; i < NQ; i++) {
+			PgQueryProtobufParseResult r = pg_query_parse_protobuf(queries[i]);
+			pg_query_free_protobuf_parse_result(r);
+			PgQueryDeparseResult d = pg_query_deparse_protobuf(bufs[i]);
+			pg_query_free_deparse_result(d);
+		}
+
+	/* ENCODE: parse_protobuf (parse + protobuf encode) */
+	double t0 = now_sec();
+	volatile size_t sink = 0;
+	for (int it = 0; it < iters; it++)
+		for (int i = 0; i < NQ; i++) {
+			PgQueryProtobufParseResult r = pg_query_parse_protobuf(queries[i]);
+			sink += r.parse_tree.len;
+			pg_query_free_protobuf_parse_result(r);
+		}
+	double t_enc = now_sec() - t0;
+
+	/* DECODE: deparse_protobuf (protobuf decode + deparse walk) */
+	t0 = now_sec();
+	for (int it = 0; it < iters; it++)
+		for (int i = 0; i < NQ; i++) {
+			PgQueryDeparseResult d = pg_query_deparse_protobuf(bufs[i]);
+			sink += d.query ? strlen(d.query) : 0;
+			pg_query_free_deparse_result(d);
+		}
+	double t_dec = now_sec() - t0;
+
+	long ops = (long) iters * NQ;
+	struct rusage ru;
+	getrusage(RUSAGE_SELF, &ru);
+
+	printf("[%s] iters=%d queries=%d ops=%ld\n", label, iters, NQ, ops);
+	printf("  encode (parse_protobuf): %8.3f s   %8.1f ns/op   %9.0f ops/s\n",
+		t_enc, t_enc * 1e9 / ops, ops / t_enc);
+	printf("  decode (deparse_protobuf): %6.3f s   %8.1f ns/op   %9.0f ops/s\n",
+		t_dec, t_dec * 1e9 / ops, ops / t_dec);
+	printf("  protobuf bytes/corpus pass: %zu (per-op avg %.1f)\n", total_bytes, (double) total_bytes / NQ);
+	printf("  maxrss: %.1f MB    (sink=%zu)\n", ru.ru_maxrss / (1024.0 * 1024.0), sink);
+	return 0;
+}