From d1cc9d390835dbe11afcc33e8d05da6e7dbe3275 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:30:58 +0900 Subject: [PATCH 1/7] Add registry-based v1.8.5 vs v2.0.0 benchmark comparison workflow (#1) * feat: add benchmark comparison scaffolding Co-authored-by: sator-imaging <16752340+sator-imaging@users.noreply.github.com> * chore: add benchmark module lockfiles Co-authored-by: sator-imaging <16752340+sator-imaging@users.noreply.github.com> * fix: set minimal github token permissions for benchmark workflow Co-authored-by: sator-imaging <16752340+sator-imaging@users.noreply.github.com> * ci * ci 2 * ci 3 * ci 4 * ci 5 * ci 6 * ci 7 * ci 8 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: sator-imaging <16752340+sator-imaging@users.noreply.github.com> --- .github/workflows/benchmark-comparison.yaml | 69 +++++++++++++++++++++ _benchmark/comparison/v1/benchmark_test.go | 39 ++++++++++++ _benchmark/comparison/v1/go.mod | 5 ++ _benchmark/comparison/v1/go.sum | 2 + _benchmark/comparison/v2/benchmark_test.go | 38 ++++++++++++ _benchmark/comparison/v2/go.mod | 5 ++ _benchmark/comparison/v2/go.sum | 2 + 7 files changed, 160 insertions(+) create mode 100644 .github/workflows/benchmark-comparison.yaml create mode 100644 _benchmark/comparison/v1/benchmark_test.go create mode 100644 _benchmark/comparison/v1/go.mod create mode 100644 _benchmark/comparison/v1/go.sum create mode 100644 _benchmark/comparison/v2/benchmark_test.go create mode 100644 _benchmark/comparison/v2/go.mod create mode 100644 _benchmark/comparison/v2/go.sum diff --git a/.github/workflows/benchmark-comparison.yaml b/.github/workflows/benchmark-comparison.yaml new file mode 100644 index 0000000..aba759a --- /dev/null +++ b/.github/workflows/benchmark-comparison.yaml @@ -0,0 +1,69 @@ +name: benchmark-comparison + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + id: setup-go + with: + go-version: 1.27.x + check-latest: true + + - name: Benchmark + run: | + cd _benchmark/go + + go test -bench . -benchmem -run '^$' | tee benchmark.out + { + echo '```text' + cat benchmark.out + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + compare: + # Don't use matrix due to the runner spec cannot be chosen. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + id: setup-go + with: + go-version: 1.27.x + check-latest: true + + - name: Comparison + run: | + run_benchmark() { + local version="$1" + local package="$2" + + pushd "_benchmark/comparison/$version" + go get "$package@latest" + + go test -bench . -benchmem -run '^$' -count 10 | tee benchmark.out + { + echo "## $version" + echo + echo '```text' + cat benchmark.out + echo '```' + echo + } >> "$GITHUB_STEP_SUMMARY" + + popd + } + + echo "Go version: ${{ steps.setup-go.outputs.go-version }}" \ + >> "$GITHUB_STEP_SUMMARY" + + run_benchmark v2 "github.com/yuin/goldmark/v2" + run_benchmark v1 "github.com/yuin/goldmark" diff --git a/_benchmark/comparison/v1/benchmark_test.go b/_benchmark/comparison/v1/benchmark_test.go new file mode 100644 index 0000000..bfe9d2a --- /dev/null +++ b/_benchmark/comparison/v1/benchmark_test.go @@ -0,0 +1,39 @@ +package benchmark + +import ( + "bytes" + "os" + "testing" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/renderer/html" +) + +func BenchmarkGoldmarkV1(b *testing.B) { + markdown := goldmark.New( + goldmark.WithRendererOptions(html.WithXHTML(), html.WithUnsafe()), + ) + doBenchmark(b, func(source []byte) ([]byte, error) { + var out bytes.Buffer + err := markdown.Convert(source, &out) + return out.Bytes(), err + }) +} + +func doBenchmark(b *testing.B, render func(src []byte) ([]byte, error)) { + b.StopTimer() + source, err := os.ReadFile("../../go/_data.md") + if err != nil { + b.Fatal(err) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + out, err := render(source) + if err != nil { + b.Fatal(err) + } + if len(out) < 100 { + b.Fatal("No result") + } + } +} diff --git a/_benchmark/comparison/v1/go.mod b/_benchmark/comparison/v1/go.mod new file mode 100644 index 0000000..2138852 --- /dev/null +++ b/_benchmark/comparison/v1/go.mod @@ -0,0 +1,5 @@ +module benchmark-comparison-v1 + +go 1.27 + +require github.com/yuin/goldmark v1.8.5 diff --git a/_benchmark/comparison/v1/go.sum b/_benchmark/comparison/v1/go.sum new file mode 100644 index 0000000..98fbe0d --- /dev/null +++ b/_benchmark/comparison/v1/go.sum @@ -0,0 +1,2 @@ +github.com/yuin/goldmark v1.8.5 h1:r6N5afV5qj/5S4UTch8agZHJ8UxNCMwX7WjkkJam2NA= +github.com/yuin/goldmark v1.8.5/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= diff --git a/_benchmark/comparison/v2/benchmark_test.go b/_benchmark/comparison/v2/benchmark_test.go new file mode 100644 index 0000000..a56a997 --- /dev/null +++ b/_benchmark/comparison/v2/benchmark_test.go @@ -0,0 +1,38 @@ +package benchmark + +import ( + "bytes" + "os" + "testing" + + "github.com/yuin/goldmark/v2/parser" + "github.com/yuin/goldmark/v2/renderer/html" +) + +func BenchmarkGoldmarkV2(b *testing.B) { + p := parser.New() + r := html.New(html.WithXHTML(), html.WithUnsafe()) + doBenchmark(b, func(source []byte) ([]byte, error) { + var out bytes.Buffer + err := r.Render(&out, source, p.Parse(source)) + return out.Bytes(), err + }) +} + +func doBenchmark(b *testing.B, render func(src []byte) ([]byte, error)) { + b.StopTimer() + source, err := os.ReadFile("../../go/_data.md") + if err != nil { + b.Fatal(err) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + out, err := render(source) + if err != nil { + b.Fatal(err) + } + if len(out) < 100 { + b.Fatal("No result") + } + } +} diff --git a/_benchmark/comparison/v2/go.mod b/_benchmark/comparison/v2/go.mod new file mode 100644 index 0000000..1d36532 --- /dev/null +++ b/_benchmark/comparison/v2/go.mod @@ -0,0 +1,5 @@ +module benchmark-comparison-v2 + +go 1.27 + +require github.com/yuin/goldmark/v2 v2.0.0 diff --git a/_benchmark/comparison/v2/go.sum b/_benchmark/comparison/v2/go.sum new file mode 100644 index 0000000..9ab8b98 --- /dev/null +++ b/_benchmark/comparison/v2/go.sum @@ -0,0 +1,2 @@ +github.com/yuin/goldmark/v2 v2.0.0 h1:P6JP1Px30eqo339He8dDFE8D0BSM21eQcbozLWH+E6g= +github.com/yuin/goldmark/v2 v2.0.0/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ= From 7a79359c799acc1d9b756a34fbdba849c003f96e Mon Sep 17 00:00:00 2001 From: sator-imaging <16752340+sator-imaging@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:57:48 +0000 Subject: [PATCH 2/7] reorder step summary --- .github/workflows/benchmark-comparison.yaml | 42 ++++++++++----------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/benchmark-comparison.yaml b/.github/workflows/benchmark-comparison.yaml index aba759a..f5bb713 100644 --- a/.github/workflows/benchmark-comparison.yaml +++ b/.github/workflows/benchmark-comparison.yaml @@ -8,27 +8,6 @@ permissions: jobs: - benchmark: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-go@v7 - id: setup-go - with: - go-version: 1.27.x - check-latest: true - - - name: Benchmark - run: | - cd _benchmark/go - - go test -bench . -benchmem -run '^$' | tee benchmark.out - { - echo '```text' - cat benchmark.out - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - compare: # Don't use matrix due to the runner spec cannot be chosen. runs-on: ubuntu-latest @@ -67,3 +46,24 @@ jobs: run_benchmark v2 "github.com/yuin/goldmark/v2" run_benchmark v1 "github.com/yuin/goldmark" + + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + id: setup-go + with: + go-version: 1.27.x + check-latest: true + + - name: Benchmark + run: | + cd _benchmark/go + + go test -bench . -benchmem -run '^$' | tee benchmark.out + { + echo '```text' + cat benchmark.out + echo '```' + } >> "$GITHUB_STEP_SUMMARY" From cbd6e936fe5f901467ce897efd13b5abfa92a02e Mon Sep 17 00:00:00 2001 From: sator-imaging <16752340+sator-imaging@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:48:21 +0900 Subject: [PATCH 3/7] feat: add stats (#2) * add stats * fix * upd * done * done 2 --- .github/workflows/benchmark-comparison.yaml | 94 ++++++++++++++++++++- 1 file changed, 91 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark-comparison.yaml b/.github/workflows/benchmark-comparison.yaml index f5bb713..7d185e8 100644 --- a/.github/workflows/benchmark-comparison.yaml +++ b/.github/workflows/benchmark-comparison.yaml @@ -27,22 +27,110 @@ jobs: pushd "_benchmark/comparison/$version" go get "$package@latest" - go test -bench . -benchmem -run '^$' -count 10 | tee benchmark.out + { echo "## $version" echo + } >> "$GITHUB_STEP_SUMMARY" + + awk ' + function comma(n, s, sign, result) { + sign = "" + if (n < 0) { + sign = "-" + n = -n + } + + s = sprintf("%.0f", n) + result = "" + + while (length(s) > 3) { + result = "," substr(s, length(s) - 2, 3) result + s = substr(s, 1, length(s) - 3) + } + + return sign s result + } + + # ns/op + /^Benchmark/ { + ns[++n] = $3 + bytes[n] = $5 + } + + END { + # Sort ns/op + for (i = 1; i <= n; i++) + for (j = i + 1; j <= n; j++) + if (ns[i] > ns[j]) { + tmp = ns[i] + ns[i] = ns[j] + ns[j] = tmp + } + + # Sort B/op + for (i = 1; i <= n; i++) + for (j = i + 1; j <= n; j++) + if (bytes[i] > bytes[j]) { + tmp = bytes[i] + bytes[i] = bytes[j] + bytes[j] = tmp + } + + # ns/op statistics + ns_sum = 0 + for (i = 1; i <= n; i++) + ns_sum += ns[i] + + if (n % 2) + ns_median = ns[(n + 1) / 2] + else + ns_median = (ns[n / 2] + ns[n / 2 + 1]) / 2 + + # B/op statistics + bytes_sum = 0 + for (i = 1; i <= n; i++) + bytes_sum += bytes[i] + + if (n % 2) + bytes_median = bytes[(n + 1) / 2] + else + bytes_median = (bytes[n / 2] + bytes[n / 2 + 1]) / 2 + + printf "ns/op\n" + printf "* Min: %s\n", comma(ns[1]) + printf "* Max: %s\n", comma(ns[n]) + printf "* Median: %s\n", comma(ns_median) + printf "* Average: %s\n", comma(ns_sum / n) + printf "\n" + printf "B/op\n" + printf "* Min: %s\n", comma(bytes[1]) + printf "* Max: %s\n", comma(bytes[n]) + printf "* Median: %s\n", comma(bytes_median) + printf "* Average: %s\n", comma(bytes_sum / n) + printf "\n" + } + ' benchmark.out >> "$GITHUB_STEP_SUMMARY" + + { + echo '
' + echo echo '```text' cat benchmark.out echo '```' echo + echo '
' + echo } >> "$GITHUB_STEP_SUMMARY" popd } - echo "Go version: ${{ steps.setup-go.outputs.go-version }}" \ - >> "$GITHUB_STEP_SUMMARY" + { + echo "Go version: ${{ steps.setup-go.outputs.go-version }}" + echo + } >> "$GITHUB_STEP_SUMMARY" run_benchmark v2 "github.com/yuin/goldmark/v2" run_benchmark v1 "github.com/yuin/goldmark" From 3a4a0f8003507a1a84645e3ad8c9c474fe3a66a6 Mon Sep 17 00:00:00 2001 From: sator-imaging <16752340+sator-imaging@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:40:54 +0900 Subject: [PATCH 4/7] feat: table (#3) * feat: table * lol --- .github/workflows/benchmark-comparison.yaml | 25 ++++++++++++--------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/benchmark-comparison.yaml b/.github/workflows/benchmark-comparison.yaml index 7d185e8..e069113 100644 --- a/.github/workflows/benchmark-comparison.yaml +++ b/.github/workflows/benchmark-comparison.yaml @@ -98,17 +98,20 @@ jobs: else bytes_median = (bytes[n / 2] + bytes[n / 2 + 1]) / 2 - printf "ns/op\n" - printf "* Min: %s\n", comma(ns[1]) - printf "* Max: %s\n", comma(ns[n]) - printf "* Median: %s\n", comma(ns_median) - printf "* Average: %s\n", comma(ns_sum / n) - printf "\n" - printf "B/op\n" - printf "* Min: %s\n", comma(bytes[1]) - printf "* Max: %s\n", comma(bytes[n]) - printf "* Median: %s\n", comma(bytes_median) - printf "* Average: %s\n", comma(bytes_sum / n) + printf "| | Median | Average | Min | Max |\n" + printf "|--:|-------:|--------:|----:|----:|\n" + printf "| ns/op" + printf " | %s", comma(ns_median) + printf " | %s", comma(ns_sum / n) + printf " | %s", comma(ns[1]) + printf " | %s", comma(ns[n]) + printf " |\n" + printf "| B/op" + printf " | %s", comma(bytes_median) + printf " | %s", comma(bytes_sum / n) + printf " | %s", comma(bytes[1]) + printf " | %s", comma(bytes[n]) + printf " |\n" printf "\n" } ' benchmark.out >> "$GITHUB_STEP_SUMMARY" From 9fab8f166c5aa24fd9b0098ba6ce4e9a434dca37 Mon Sep 17 00:00:00 2001 From: sator-imaging <16752340+sator-imaging@users.noreply.github.com> Date: Tue, 1 Sep 2026 10:51:04 +0900 Subject: [PATCH 5/7] add allocs columns (#4) * add allocs columns * fix --- .github/workflows/benchmark-comparison.yaml | 65 ++++++++++++++++----- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/.github/workflows/benchmark-comparison.yaml b/.github/workflows/benchmark-comparison.yaml index e069113..f3dcd5d 100644 --- a/.github/workflows/benchmark-comparison.yaml +++ b/.github/workflows/benchmark-comparison.yaml @@ -9,8 +9,12 @@ permissions: jobs: compare: - # Don't use matrix due to the runner spec cannot be chosen. - runs-on: ubuntu-latest + strategy: + matrix: + runner: + - ubuntu-latest + - ubuntu-slim + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v7 - uses: actions/setup-go@v7 @@ -30,7 +34,7 @@ jobs: go test -bench . -benchmem -run '^$' -count 10 | tee benchmark.out { - echo "## $version" + echo "## $version (${{ matrix.runner }})" echo } >> "$GITHUB_STEP_SUMMARY" @@ -55,12 +59,16 @@ jobs: # ns/op /^Benchmark/ { - ns[++n] = $3 + ++n + ns[n] = $3 bytes[n] = $5 + allocs[n] = $7 } END { - # Sort ns/op + # + # Sort ns/op + # for (i = 1; i <= n; i++) for (j = i + 1; j <= n; j++) if (ns[i] > ns[j]) { @@ -69,15 +77,6 @@ jobs: ns[j] = tmp } - # Sort B/op - for (i = 1; i <= n; i++) - for (j = i + 1; j <= n; j++) - if (bytes[i] > bytes[j]) { - tmp = bytes[i] - bytes[i] = bytes[j] - bytes[j] = tmp - } - # ns/op statistics ns_sum = 0 for (i = 1; i <= n; i++) @@ -88,6 +87,17 @@ jobs: else ns_median = (ns[n / 2] + ns[n / 2 + 1]) / 2 + # + # Sort B/op + # + for (i = 1; i <= n; i++) + for (j = i + 1; j <= n; j++) + if (bytes[i] > bytes[j]) { + tmp = bytes[i] + bytes[i] = bytes[j] + bytes[j] = tmp + } + # B/op statistics bytes_sum = 0 for (i = 1; i <= n; i++) @@ -98,6 +108,27 @@ jobs: else bytes_median = (bytes[n / 2] + bytes[n / 2 + 1]) / 2 + # + # Sort allocs/op + # + for (i = 1; i <= n; i++) + for (j = i + 1; j <= n; j++) + if (allocs[i] > allocs[j]) { + tmp = allocs[i] + allocs[i] = allocs[j] + allocs[j] = tmp + } + + # allocs/op statistics + allocs_sum = 0 + for (i = 1; i <= n; i++) + allocs_sum += allocs[i] + + if (n % 2) + allocs_median = allocs[(n + 1) / 2] + else + allocs_median = (allocs[n / 2] + allocs[n / 2 + 1]) / 2 + printf "| | Median | Average | Min | Max |\n" printf "|--:|-------:|--------:|----:|----:|\n" printf "| ns/op" @@ -112,6 +143,12 @@ jobs: printf " | %s", comma(bytes[1]) printf " | %s", comma(bytes[n]) printf " |\n" + printf "| allocs/op" + printf " | %s", comma(allocs_median) + printf " | %s", comma(allocs_sum / n) + printf " | %s", comma(allocs[1]) + printf " | %s", comma(allocs[n]) + printf " |\n" printf "\n" } ' benchmark.out >> "$GITHUB_STEP_SUMMARY" From bbcfc6bc8200a89938acac3ba6e79a38515a4648 Mon Sep 17 00:00:00 2001 From: sator-imaging <16752340+sator-imaging@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:36:59 +0900 Subject: [PATCH 6/7] bump: 2.0.1 vs 1.8.6 (#5) --- _benchmark/comparison/v1/go.mod | 2 +- _benchmark/comparison/v2/go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/_benchmark/comparison/v1/go.mod b/_benchmark/comparison/v1/go.mod index 2138852..d7c3e9b 100644 --- a/_benchmark/comparison/v1/go.mod +++ b/_benchmark/comparison/v1/go.mod @@ -2,4 +2,4 @@ module benchmark-comparison-v1 go 1.27 -require github.com/yuin/goldmark v1.8.5 +require github.com/yuin/goldmark v1.8.6 diff --git a/_benchmark/comparison/v2/go.mod b/_benchmark/comparison/v2/go.mod index 1d36532..6338ad3 100644 --- a/_benchmark/comparison/v2/go.mod +++ b/_benchmark/comparison/v2/go.mod @@ -2,4 +2,4 @@ module benchmark-comparison-v2 go 1.27 -require github.com/yuin/goldmark/v2 v2.0.0 +require github.com/yuin/goldmark/v2 v2.0.1 From 8b006d035be2cb010e95179632c484944b59740c Mon Sep 17 00:00:00 2001 From: sator-imaging <16752340+sator-imaging@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:28:27 +0900 Subject: [PATCH 7/7] add version upgrading log (#6) * chore: clarify what actually happening * stderr? * done --- .github/workflows/benchmark-comparison.yaml | 9 +++++++-- _benchmark/comparison/v1/go.mod | 2 +- _benchmark/comparison/v2/go.mod | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark-comparison.yaml b/.github/workflows/benchmark-comparison.yaml index f3dcd5d..1f172f3 100644 --- a/.github/workflows/benchmark-comparison.yaml +++ b/.github/workflows/benchmark-comparison.yaml @@ -30,8 +30,13 @@ jobs: local package="$2" pushd "_benchmark/comparison/$version" - go get "$package@latest" - go test -bench . -benchmem -run '^$' -count 10 | tee benchmark.out + + { + echo '$' go get "$package@latest" + go get "$package@latest" 2>&1 + echo + go test -bench . -benchmem -run '^$' -count 10 + } | tee benchmark.out { echo "## $version (${{ matrix.runner }})" diff --git a/_benchmark/comparison/v1/go.mod b/_benchmark/comparison/v1/go.mod index d7c3e9b..2138852 100644 --- a/_benchmark/comparison/v1/go.mod +++ b/_benchmark/comparison/v1/go.mod @@ -2,4 +2,4 @@ module benchmark-comparison-v1 go 1.27 -require github.com/yuin/goldmark v1.8.6 +require github.com/yuin/goldmark v1.8.5 diff --git a/_benchmark/comparison/v2/go.mod b/_benchmark/comparison/v2/go.mod index 6338ad3..1d36532 100644 --- a/_benchmark/comparison/v2/go.mod +++ b/_benchmark/comparison/v2/go.mod @@ -2,4 +2,4 @@ module benchmark-comparison-v2 go 1.27 -require github.com/yuin/goldmark/v2 v2.0.1 +require github.com/yuin/goldmark/v2 v2.0.0