From 4646c3cd0a7a6da52b43a266000cc826a4453716 Mon Sep 17 00:00:00 2001 From: laiso Date: Sat, 21 Mar 2026 10:51:57 +0700 Subject: [PATCH 1/3] Rewrite Python implementation in Rust with PyPI binary distribution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Why: ISSUE.md — wget dependency removal, 5x faster conversion pipeline, single-binary distribution via PyPI wheel. - Replace Python site2skill/ with Rust src/ (reqwest/tokio crawler, scraper/htmd converter, zip packager) - Add python/ wrapper for PyPI binary distribution (sqlite-scanner pattern) - Fix path traversal vulnerability in skill structure generation - Fix url_to_file_path extension bug, unify URL scope filtering - Cache all regex with lazy_static, eliminate memory accumulation in crawler - Add 34 regression tests (15 → 49 unit + 2 doc tests) - Add benchmark tooling (scripts/generate_bench_site.py, run_benchmark.py) Co-Authored-By: Claude Opus 4.6 (1M context) --- .cargo/config.toml | 23 + .github/workflows/test.yml | 19 +- .gitignore | 12 + Cargo.lock | 2410 +++++++++++++++++ Cargo.toml | 62 + ISSUE.md | 35 + PR.md | 92 + README.md | 26 +- pyproject.toml | 22 +- python/CHANGELOG.md | 7 + python/LICENSE | 21 + python/README.md | 108 + python/pyproject.toml | 38 + python/site2skill/__init__.py | 45 + python/site2skill/__main__.py | 6 + scripts/bench-run.sh | 202 ++ scripts/build-wheel.sh | 62 + scripts/build_wheels.py | 119 + scripts/generate_bench_site.py | 308 +++ scripts/run_benchmark.py | 533 ++++ scripts/validate_benchmark.py | 258 ++ site2skill/__init__.py | 3 - site2skill/convert_to_markdown.py | 113 - site2skill/fetch_site.py | 200 -- site2skill/generate_skill_structure.py | 164 -- site2skill/main.py | 181 -- site2skill/normalize_markdown.py | 81 - site2skill/package_skill.py | 45 - site2skill/templates/scripts_README.md | 31 - site2skill/url_filter.py | 72 - site2skill/utils.py | 70 - site2skill/validate_skill.py | 165 -- src/convert/html.rs | 220 ++ src/convert/markdown.rs | 123 + src/convert/mod.rs | 7 + src/fetch/crawler.rs | 382 +++ src/fetch/mod.rs | 13 + src/fetch/robots.rs | 171 ++ src/lib.rs | 16 + src/main.rs | 281 ++ src/normalize.rs | 150 + src/skill/mod.rs | 7 + src/skill/package.rs | 124 + src/skill/structure.rs | 250 ++ src/url_filter.rs | 157 ++ src/utils.rs | 88 + src/validate.rs | 290 ++ templates/scripts_README.md | 27 + .../templates => templates}/search_docs.py | 38 +- test_agentskills_integration.py | 37 - test_fetch_site.py | 67 - test_filename_conversion.py | 176 -- test_integration.py | 215 -- test_site2skill.py | 71 - test_url_filter.py | 138 - 55 files changed, 6693 insertions(+), 1888 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 ISSUE.md create mode 100644 PR.md create mode 100644 python/CHANGELOG.md create mode 100644 python/LICENSE create mode 100644 python/README.md create mode 100644 python/pyproject.toml create mode 100644 python/site2skill/__init__.py create mode 100644 python/site2skill/__main__.py create mode 100755 scripts/bench-run.sh create mode 100755 scripts/build-wheel.sh create mode 100644 scripts/build_wheels.py create mode 100755 scripts/generate_bench_site.py create mode 100755 scripts/run_benchmark.py create mode 100755 scripts/validate_benchmark.py delete mode 100644 site2skill/__init__.py delete mode 100644 site2skill/convert_to_markdown.py delete mode 100644 site2skill/fetch_site.py delete mode 100644 site2skill/generate_skill_structure.py delete mode 100644 site2skill/main.py delete mode 100644 site2skill/normalize_markdown.py delete mode 100644 site2skill/package_skill.py delete mode 100644 site2skill/templates/scripts_README.md delete mode 100644 site2skill/url_filter.py delete mode 100644 site2skill/utils.py delete mode 100644 site2skill/validate_skill.py create mode 100644 src/convert/html.rs create mode 100644 src/convert/markdown.rs create mode 100644 src/convert/mod.rs create mode 100644 src/fetch/crawler.rs create mode 100644 src/fetch/mod.rs create mode 100644 src/fetch/robots.rs create mode 100644 src/lib.rs create mode 100644 src/main.rs create mode 100644 src/normalize.rs create mode 100644 src/skill/mod.rs create mode 100644 src/skill/package.rs create mode 100644 src/skill/structure.rs create mode 100644 src/url_filter.rs create mode 100644 src/utils.rs create mode 100644 src/validate.rs create mode 100644 templates/scripts_README.md rename {site2skill/templates => templates}/search_docs.py (97%) delete mode 100644 test_agentskills_integration.py delete mode 100644 test_fetch_site.py delete mode 100644 test_filename_conversion.py delete mode 100644 test_integration.py delete mode 100644 test_site2skill.py delete mode 100644 test_url_filter.py diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..641b0a3 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,23 @@ +# Memory-optimized build configuration +# For environments with limited RAM + +[build] +# Use single job to reduce memory usage +jobs = 1 + +# Disable incremental compilation to save memory +incremental = false + +# Target the native architecture (auto-detected) +# target = "aarch64-apple-darwin" + +[profile.release] +# Optimize for size to reduce memory during compilation +opt-level = "z" +lto = false +codegen-units = 1 + +[profile.dev] +# Debug builds also optimized for memory +opt-level = 0 +debug = false diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cb55e92..c3bbba4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,16 +13,17 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install uv - uses: astral-sh/setup-uv@v5 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable - - name: Set up Python - uses: actions/setup-python@v5 + - name: Cache cargo + uses: actions/cache@v4 with: - python-version-file: ".python-version" - - - name: Install dependencies - run: uv sync --all-extras --dev + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Run tests - run: uv run pytest + run: cargo test diff --git a/.gitignore b/.gitignore index ca8d8ea..a554be1 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,15 @@ venv/ # Pytest .pytest_cache/ + +# Rust +target/ + +# Benchmark artifacts +bench-site-*/ +bench-results/ + +# Misc +wget-log +*.skill +python/site2skill/bin/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..150dceb --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2410 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "cssparser" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3df4f93e5fbbe73ec01ec8d3f68bba73107993a5b1e7519273c32db9b0d5be" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.11.3", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_more" +version = "0.99.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12a0bb14ac04a9fcf170d0bbbef949b44cc492f4452bd20c095636956f653642" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "htmd" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad1642def6e8e4dc182941f35454f7d2af917787f91f3f5133300030b41006d0" +dependencies = [ + "html5ever 0.27.0", + "markup5ever_rcdom", +] + +[[package]] +name = "html5ever" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" +dependencies = [ + "log", + "mac", + "markup5ever 0.11.0", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "html5ever" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13771afe0e6e846f1e67d038d4cb29998a6779f93c809212e4e9c32efd244d4" +dependencies = [ + "log", + "mac", + "markup5ever 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", + "web-time", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" +dependencies = [ + "log", + "phf 0.10.1", + "phf_codegen 0.10.0", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "markup5ever" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16ce3abbeba692c8b8441d036ef91aea6df8da2c6b6e21c7e14d3c18e526be45" +dependencies = [ + "log", + "phf 0.11.3", + "phf_codegen 0.11.3", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "markup5ever_rcdom" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edaa21ab3701bfee5099ade5f7e1f84553fd19228cf332f13cd6e964bf59be18" +dependencies = [ + "html5ever 0.27.0", + "markup5ever 0.12.1", + "tendril", + "xml5ever", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" +dependencies = [ + "phf_shared 0.10.0", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_codegen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_generator" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +dependencies = [ + "phf_shared 0.10.0", + "rand", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher 0.3.11", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher 1.0.2", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-rustls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scraper" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585480e3719b311b78a573db1c9d9c4c1f8010c2dee4cc59c2efe58ea4dbc3e1" +dependencies = [ + "ahash", + "cssparser", + "ego-tree", + "getopts", + "html5ever 0.26.0", + "once_cell", + "selectors", + "tendril", +] + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "selectors" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06" +dependencies = [ + "bitflags 2.11.0", + "cssparser", + "derive_more", + "fxhash", + "log", + "new_debug_unreachable", + "phf 0.10.1", + "phf_codegen 0.10.0", + "precomputed-hash", + "servo_arc", + "smallvec", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "servo_arc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d036d71a959e00c77a63538b90a6c2390969f9772b096ea837205c6bd0491a44" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "site2skill" +version = "0.2.0" +dependencies = [ + "chrono", + "clap", + "futures", + "htmd", + "indicatif", + "lazy_static", + "regex", + "reqwest", + "scraper", + "serde", + "serde_yaml", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", + "url", + "walkdir", + "zip", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "string_cache" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared 0.11.3", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", + "proc-macro2", + "quote", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.3", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.117", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "xml5ever" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bbb26405d8e919bc1547a5aa9abc95cbfa438f04844f5fdd9dc7596b748bf69" +dependencies = [ + "log", + "mac", + "markup5ever 0.12.1", +] + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "byteorder", + "crc32fast", + "crossbeam-utils", + "flate2", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..fde7521 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,62 @@ +[package] +name = "site2skill" +version = "0.2.0" +edition = "2021" +authors = ["laiso "] +description = "Turn any documentation website into a Claude Agent Skill" +license = "MIT" +readme = "README.md" +repository = "https://github.com/laiso/site2skill" +keywords = ["claude", "agent", "skill", "documentation", "cli"] + +[dependencies] +# CLI +clap = { version = "4.4", features = ["derive"] } + +# Async runtime +tokio = { version = "1.35", features = ["full"] } + +# HTTP client +reqwest = { version = "0.11", features = ["rustls-tls"], default-features = false } + +# HTML parsing +scraper = "0.18" +htmd = "0.1" + +# Serialization +serde = { version = "1", features = ["derive"] } +serde_yaml = "0.9" + +# URL handling +url = "2.5" + +# ZIP creation +zip = { version = "0.6", default-features = false, features = ["deflate"] } + +# Async utilities +futures = "0.3" + +# Logging +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +# Progress bars +indicatif = "0.17" + +# Error handling +thiserror = "1" + +# Regex +regex = "1" +lazy_static = "1" + +# Date/time +chrono = { version = "0.4", features = ["serde"] } + +# Directory walking +walkdir = "2" + +[profile.release] +opt-level = "z" +lto = true +codegen-units = 1 diff --git a/ISSUE.md b/ISSUE.md new file mode 100644 index 0000000..c4b8ffb --- /dev/null +++ b/ISSUE.md @@ -0,0 +1,35 @@ +# Why: Python to Rust Rewrite + +## Motivation + +Rewrite the Python implementation of site2skill in Rust. + +### 1. Performance + +The Python version takes over 15 minutes to process large documentation sites (500+ pages). HTTP fetching uses the external `wget` command for sequential downloads, and HTML-to-Markdown conversion processes files one at a time with no parallelism. + +Rust's async/await with tokio enables semaphore-controlled concurrent HTTP fetching and native-code conversion for faster processing. + +**Measured results (100 pages, conversion pipeline only with `--skip-fetch`):** + +| | Python | Rust | Ratio | +|---|--------|------|-------| +| Wall time | 0.77s | 0.15s | **5.1x** | +| Peak memory | 34MB | 11MB | 3.1x | + +### 2. Simpler distribution + +The Python version requires Python 3.10+ and wget, which needs separate installation on macOS/Windows. + +The Rust version ships as a single binary with no runtime dependencies. PyPI distribution uses the [sqlite-scanner pattern](https://simonwillison.net/2026/Feb/4/distributing-go-binaries/) to bundle the binary inside a wheel, so `pip install site2skill` is all that's needed. + +### 3. Remove wget dependency + +The Python version shells out to `wget` for HTTP crawling, offering limited control over robots.txt handling and crawl scope. The Rust version uses reqwest with a custom crawler that handles robots.txt compliance, depth control, concurrency limits, and URL scope filtering entirely in code. + +## Scope + +- Delete the Python implementation (`site2skill/` directory, `test_*.py`) +- Reimplement equivalent functionality in Rust (`src/`) +- Add a Python wrapper for PyPI distribution (`python/`) +- Maintain CLI interface compatibility diff --git a/PR.md b/PR.md new file mode 100644 index 0000000..65d14b6 --- /dev/null +++ b/PR.md @@ -0,0 +1,92 @@ +# Pull Request: Rust Rewrite & PyPI Binary Distribution + +Background: [ISSUE.md](./ISSUE.md) + +## How + +### Rust implementation (`src/`) + +Reimplemented the 6-step pipeline from Python in Rust. + +- **Fetch** — async crawler with reqwest + tokio. Semaphore-based concurrency control, robots.txt compliance +- **Convert** — HTML parsing with scraper, Markdown conversion with htmd. Removes nav/sidebar/footer via regex +- **Normalize** — reads source_url from frontmatter, resolves relative links to absolute URLs +- **Generate** — creates SKILL.md + references/ directory structure. Bundles search_docs.py template +- **Validate** — checks SKILL.md frontmatter, references/ existence, 8MB size limit +- **Package** — generates .skill file as ZIP archive + +### PyPI distribution (`python/`) + +Bundles the Rust binary inside a wheel using the [sqlite-scanner pattern](https://simonwillison.net/2026/Feb/4/distributing-go-binaries/). + +``` +python/site2skill/ +├── __init__.py # thin wrapper that locates and execs the binary +├── __main__.py # python -m site2skill support +└── bin/site2skill # compiled binary +``` + +`scripts/build-wheel.sh` handles platform detection, build, and wheel renaming. + +### Code quality fixes + +| Issue | Fix | +|-------|-----| +| `url_to_file_path` produces `api..html` for extensionless URLs | Append `.html` to full filename | +| `generate_skill_structure` path traversal check bypassed | Component-level `..` detection + post-mkdir canonicalize | +| `clean_html` strips only class/id attributes, leaving elements | Remove entire nav/header/footer/aside/sidebar elements via regex | +| Duplicate scope logic in `is_url_in_scope` and `url_filter::is_url_allowed` | Unified to `is_url_allowed` | +| `Regex::new()` compiled on every call across all modules | Cached with `lazy_static!` | +| Crawler accumulates all page content in `Vec` | Replaced with counter | +| Compiler warnings | All resolved (unused variables, unused types) | + +### Tests + +49 unit tests + 2 doc tests. Expanded from 15 to 49. + +| Module | +Tests | Coverage | +|--------|--------|----------| +| `convert::html` | +8 | multiline script/style removal, nav/sidebar/footer removal, content extraction | +| `fetch::crawler` | +9 | `url_to_file_path` 6 edge cases, link extraction 3 patterns | +| `normalize` | +4 | missing frontmatter, no source_url, anchor/mailto preservation | +| `skill::structure` | +3 | structure generation, subdirectory preservation, path traversal detection | +| `skill::package` | +2 | ZIP creation with content verification, nonexistent directory | +| `validate` | +6 | valid skill, missing SKILL.md, missing references, missing frontmatter, legacy docs | + +## Files Changed + +### Added +- `src/` — Rust implementation +- `python/` — PyPI distribution wrapper +- `scripts/build-wheel.sh` — wheel build script +- `.cargo/config.toml` — build optimization settings +- `Cargo.toml`, `Cargo.lock` +- `templates/` — bundled templates + +### Removed +- `site2skill/` — Python implementation +- `test_*.py` — Python tests + +### Modified +- `pyproject.toml` — changed to hatchling-based binary distribution +- `ISSUE.md` — rewritten as motivation document + +## Benchmark + +Measured the conversion pipeline (Convert, Normalize, Generate, Validate) against 100 pre-downloaded HTML pages using `--skip-fetch`. Crawling (Fetch) excluded because the architectures differ (Python uses external `wget`, Rust uses a built-in async crawler). + +| | Python | Rust | Ratio | +|---|--------|------|-------| +| Wall time | 0.77s | 0.15s | **5.1x** | +| CPU (user) | 0.64s | 0.09s | 7.1x | +| Peak memory (RSS) | 34MB | 11MB | 3.1x | +| Output files | 101 | 101 | equal | + +Environment: macOS ARM64 (Apple Silicon), static HTML served from localhost, generated with `scripts/generate_bench_site.py --pages 100` + +## TODO + +- [ ] Linux build verification +- [ ] GitHub Actions CI +- [ ] Multi-platform wheels (cross-compilation) +- [ ] Publish to PyPI diff --git a/README.md b/README.md index b304d8a..34bd76f 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,6 @@ Agent Skills are dynamically loaded knowledge modules that Claude uses on demand ## Installation -**Requirements:** -* **Python 3.10+** -* **wget**: Must be installed and available in your PATH. - * macOS: `brew install wget` - * Linux: `apt install wget` - * Windows: Use WSL, or install via `choco install wget` / `scoop install wget` - ### Install from PyPI ```bash @@ -32,11 +25,14 @@ uv tool install site2skill uvx site2skill ``` -### Install from GitHub (Latest) +### Build from Source ```bash -pip install git+https://github.com/laiso/site2skill.git -uvx --from git+https://github.com/laiso/site2skill site2skill +# Requires Rust toolchain +git clone https://github.com/laiso/site2skill.git +cd site2skill +cargo build --release +./target/release/site2skill --help ``` ## Usage @@ -71,9 +67,9 @@ Options: ## How it works -1. **Fetch**: Downloads the documentation site recursively using `wget`. -2. **Convert**: Converts HTML pages to Markdown using `beautifulsoup4` and `markdownify`. -3. **Normalize**: Cleans up links and formatting. +1. **Fetch**: Crawls the documentation site with a built-in async HTTP crawler (robots.txt compliant, concurrent). +2. **Convert**: Converts HTML pages to Markdown using scraper and htmd. +3. **Normalize**: Resolves relative links to absolute URLs. 4. **Validate**: Checks the skill structure and size limits. 5. **Package**: Generates `SKILL.md` and zips everything into a `.skill` file. @@ -89,14 +85,14 @@ The tool generates a skill directory in `.claude/skills//` containin └── search_docs.py # Search tool for documentation ``` -Additionally, a `.skill` file (ZIP archive) is created in the current directory. +Additionally, a `.skill` file (ZIP archive) is created in the current directory when targeting `claude-desktop`. Legacy note: older skills may use `docs/` instead of `references/`. The search tool and validator now support both, preferring `references/` when present. ### Search Tool -Each generated skill includes a search script: +Each generated skill includes a Python search script (requires Python 3 at runtime): ```bash python scripts/search_docs.py "" diff --git a/pyproject.toml b/pyproject.toml index bc2bf39..cfc17e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,15 +4,15 @@ build-backend = "hatchling.build" [project] name = "site2skill" -version = "0.1.1" -description = "Turn any website into a Claude Skill" +version = "0.2.0" +description = "Turn any documentation website into a Claude Agent Skill" readme = "README.md" license = { text = "MIT" } -requires-python = ">=3.10" +requires-python = ">=3.9" authors = [ { name = "laiso", email = "laiso@users.noreply.github.com" } ] -keywords = ["claude", "agent", "skill", "documentation", "cli"] +keywords = ["claude", "agent", "skill", "documentation", "cli", "rust"] classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", @@ -20,15 +20,11 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", -] -dependencies = [ - "beautifulsoup4>=4.12.0", - "markdownify>=0.11.0", - "pyyaml>=6.0", + "Programming Language :: Rust", ] [project.scripts] -site2skill = "site2skill.main:main" +site2skill = "site2skill:main" [tool.hatch.build.targets.wheel] packages = ["site2skill"] @@ -40,9 +36,3 @@ include = [ "CHANGELOG.md", "LICENSE", ] - -[dependency-groups] -dev = [ - "pytest>=9.0.2", - "skills-ref; python_version >= '3.11'", -] diff --git a/python/CHANGELOG.md b/python/CHANGELOG.md new file mode 100644 index 0000000..7af819f --- /dev/null +++ b/python/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +## 0.1.1 - 2026-03-07 + +- Restrict recursive crawling to the starting URL scope instead of the whole domain. +- Reject localization-only query variants such as `hl`, `lang`, and `locale` during fetch. +- Add fetch-site tests that verify `wget` receives the crawl-scope filters. diff --git a/python/LICENSE b/python/LICENSE new file mode 100644 index 0000000..2a4c2aa --- /dev/null +++ b/python/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 laiso + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..b304d8a --- /dev/null +++ b/python/README.md @@ -0,0 +1,108 @@ +# site2skill + +**Turn any documentation website into a Claude Agent Skill.** + +`site2skill` is a tool that scrapes a documentation website, converts it to Markdown, and packages it as a Claude [Agent Skill](https://www.anthropic.com/news/skills) (ZIP format) with a proper `SKILL.md` entry point. + +Agent Skills are dynamically loaded knowledge modules that Claude uses on demand. They work across Claude Code, Claude apps, and the API. + +## Installation + +**Requirements:** +* **Python 3.10+** +* **wget**: Must be installed and available in your PATH. + * macOS: `brew install wget` + * Linux: `apt install wget` + * Windows: Use WSL, or install via `choco install wget` / `scoop install wget` + +### Install from PyPI + +```bash +# Standard installation with pip +pip install site2skill + +# Or with uv +uv tool install site2skill +``` + +### Run without Installation + +```bash +# Run directly with uvx +uvx site2skill +``` + +### Install from GitHub (Latest) + +```bash +pip install git+https://github.com/laiso/site2skill.git +uvx --from git+https://github.com/laiso/site2skill site2skill +``` + +## Usage + +```bash +# Basic usage +site2skill + +# Example: Create a skill for PAY.JP +site2skill https://docs.pay.jp/v1/ payjp + +# Example: Create a skill for uv documentation +site2skill https://docs.astral.sh/uv/ uv-docs + +# Target specific agent (sets default output directory) +site2skill --target claude-desktop +``` + +## CLI Options + +``` +site2skill [options] + +Options: + --target Target agent (claude|claude-desktop|cursor|gemini|codex). Sets default output directory + --output, -o Base output directory for skill structure (overrides target default) + --skill-output Output directory for .skill file (default: .) + --temp-dir Temporary directory for processing (default: build) + --skip-fetch Skip the download step (use existing files in temp dir) + --clean Clean up temporary directory after completion +``` + +## How it works + +1. **Fetch**: Downloads the documentation site recursively using `wget`. +2. **Convert**: Converts HTML pages to Markdown using `beautifulsoup4` and `markdownify`. +3. **Normalize**: Cleans up links and formatting. +4. **Validate**: Checks the skill structure and size limits. +5. **Package**: Generates `SKILL.md` and zips everything into a `.skill` file. + +## Output + +The tool generates a skill directory in `.claude/skills//` containing: + +``` +/ +├── SKILL.md # Entry point with usage instructions +├── references/ # Markdown documentation files (preferred) +└── scripts/ + └── search_docs.py # Search tool for documentation +``` + +Additionally, a `.skill` file (ZIP archive) is created in the current directory. + +Legacy note: older skills may use `docs/` instead of `references/`. The search tool and validator +now support both, preferring `references/` when present. + +### Search Tool + +Each generated skill includes a search script: + +```bash +python scripts/search_docs.py "" +python scripts/search_docs.py "" --json --max-results 5 +``` + +## License + +MIT diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..cfc17e2 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,38 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "site2skill" +version = "0.2.0" +description = "Turn any documentation website into a Claude Agent Skill" +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.9" +authors = [ + { name = "laiso", email = "laiso@users.noreply.github.com" } +] +keywords = ["claude", "agent", "skill", "documentation", "cli", "rust"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Rust", +] + +[project.scripts] +site2skill = "site2skill:main" + +[tool.hatch.build.targets.wheel] +packages = ["site2skill"] + +[tool.hatch.build.targets.sdist] +include = [ + "site2skill", + "README.md", + "CHANGELOG.md", + "LICENSE", +] diff --git a/python/site2skill/__init__.py b/python/site2skill/__init__.py new file mode 100644 index 0000000..ecc85fe --- /dev/null +++ b/python/site2skill/__init__.py @@ -0,0 +1,45 @@ +"""site2skill - Turn any documentation website into a Claude Agent Skill.""" + +import os +import stat +import subprocess +import sys +from pathlib import Path + + +def get_binary_path() -> Path: + """Return the path to the bundled binary.""" + package_dir = Path(__file__).parent + binary = package_dir / "bin" / "site2skill" + + # Ensure binary is executable on Unix + if sys.platform != "win32": + if binary.exists(): + current_mode = os.stat(binary).st_mode + if not (current_mode & stat.S_IXUSR): + os.chmod( + binary, current_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + ) + + return binary + + +def main(): + """Execute the bundled binary.""" + binary = get_binary_path() + + if not binary.exists(): + print(f"Error: Binary not found at {binary}", file=sys.stderr) + sys.exit(1) + + if sys.platform == "win32": + # On Windows, use subprocess to properly handle signals + result = subprocess.run([str(binary)] + sys.argv[1:]) + sys.exit(result.returncode) + else: + # On Unix, exec replaces the process + os.execvp(str(binary), [str(binary)] + sys.argv[1:]) + + +if __name__ == "__main__": + main() diff --git a/python/site2skill/__main__.py b/python/site2skill/__main__.py new file mode 100644 index 0000000..0ed73ff --- /dev/null +++ b/python/site2skill/__main__.py @@ -0,0 +1,6 @@ +"""Entry point for running site2skill as a module.""" + +from site2skill import main + +if __name__ == "__main__": + main() diff --git a/scripts/bench-run.sh b/scripts/bench-run.sh new file mode 100755 index 0000000..5ef30df --- /dev/null +++ b/scripts/bench-run.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# +# Quick benchmark runner for site2skill +# Usage: ./scripts/bench-run.sh [python|rust|both] [size] +# + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +RESULTS_DIR="$PROJECT_ROOT/bench-results" +SITE_BASE="$PROJECT_ROOT/bench-site" + +# Default values +RUN_MODE="${1:-both}" +SIZE="${2:-100}" +PORT="${PORT:-8888}" + +echo "========================================" +echo "site2skill Benchmark Runner" +echo "========================================" +echo "Mode: $RUN_MODE" +echo "Size: $SIZE pages" +echo "Port: $PORT" +echo "" + +# Create results directory +mkdir -p "$RESULTS_DIR" + +# Generate test site if needed +generate_site() { + local size=$1 + local site_path="$SITE_BASE-$size" + + if [ ! -d "$site_path" ]; then + echo "Generating $size-page test site..." + python3 "$SCRIPT_DIR/generate_bench_site.py" --pages "$size" --output "$site_path" + fi +} + +# Start HTTP server +start_server() { + local site_path=$1 + echo "Starting HTTP server on port $PORT..." + python3 -m http.server "$PORT" --directory "$site_path" & + SERVER_PID=$! + sleep 1 + + # Check if server started + if ! kill -0 $SERVER_PID 2>/dev/null; then + echo "Failed to start server" + exit 1 + fi + echo "Server started (PID: $SERVER_PID)" +} + +# Stop HTTP server +stop_server() { + if [ -n "$SERVER_PID" ]; then + echo "Stopping server (PID: $SERVER_PID)..." + kill $SERVER_PID 2>/dev/null || true + wait $SERVER_PID 2>/dev/null || true + echo "Server stopped" + fi +} + +trap stop_server EXIT + +# Run Python benchmark +run_python() { + local size=$1 + local site_path="$SITE_BASE-$size" + local output_dir="$RESULTS_DIR/bench-py-$size" + local temp_dir="/tmp/bench-py-$size" + local time_file="$RESULTS_DIR/python-$size.time.txt" + + echo "" + echo "=== Python: $size pages ===" + + generate_site $size + start_server "$site_path" + + # Clean output + rm -rf "$output_dir" "$temp_dir" + mkdir -p "$output_dir" + + # Run with timing + if command -v /usr/bin/time &> /dev/null; then + /usr/bin/time -l site2skill "http://localhost:$PORT/" "$output_dir/test-skill" \ + --temp-dir "$temp_dir" --clean \ + 2> "$time_file" + else + time site2skill "http://localhost:$PORT/" "$output_dir/test-skill" \ + --temp-dir "$temp_dir" --clean \ + 2>&1 | tee "$time_file" + fi + + # Count results + local ref_dir="$output_dir/test-skill/references" + if [ -d "$ref_dir" ]; then + local file_count=$(find "$ref_dir" -name '*.md' | wc -l | tr -d ' ') + local total_size=$(du -sh "$ref_dir" | cut -f1) + echo "Output: $file_count files, $total_size" + fi + + stop_server +} + +# Run Rust benchmark +run_rust() { + local size=$1 + local site_path="$SITE_BASE-$size" + local output_dir="$RESULTS_DIR/bench-rs-$size" + local temp_dir="/tmp/bench-rs-$size" + local time_file="$RESULTS_DIR/rust-$size.time.txt" + + echo "" + echo "=== Rust: $size pages ===" + + # Build if needed + if [ ! -f "$PROJECT_ROOT/target/release/site2skill" ]; then + echo "Building Rust binary..." + cargo build --release + fi + + generate_site $size + start_server "$site_path" + + # Clean output + rm -rf "$output_dir" "$temp_dir" + mkdir -p "$output_dir" + + # Run with timing + if command -v /usr/bin/time &> /dev/null; then + /usr/bin/time -l "$PROJECT_ROOT/target/release/site2skill" "http://localhost:$PORT/" "$output_dir/test-skill" \ + --temp-dir "$temp_dir" --clean \ + 2> "$time_file" + else + time "$PROJECT_ROOT/target/release/site2skill" "http://localhost:$PORT/" "$output_dir/test-skill" \ + --temp-dir "$temp_dir" --clean \ + 2>&1 | tee "$time_file" + fi + + # Count results + local ref_dir="$output_dir/test-skill/references" + if [ -d "$ref_dir" ]; then + local file_count=$(find "$ref_dir" -name '*.md' | wc -l | tr -d ' ') + local total_size=$(du -sh "$ref_dir" | cut -f1) + echo "Output: $file_count files, $total_size" + fi + + stop_server +} + +# Compare results +compare_results() { + local size=$1 + + echo "" + echo "=== Comparison: $size pages ===" + + local py_time_file="$RESULTS_DIR/python-$size.time.txt" + local rs_time_file="$RESULTS_DIR/rust-$size.time.txt" + + if [ -f "$py_time_file" ] && [ -f "$rs_time_file" ]; then + echo "Python time file: $py_time_file" + echo "Rust time file: $rs_time_file" + + # Extract max RSS if available (macOS format) + local py_rss=$(grep "maximum resident set size" "$py_time_file" | awk '{print $NF}' || echo "N/A") + local rs_rss=$(grep "maximum resident set size" "$rs_time_file" | awk '{print $NF}' || echo "N/A") + + echo "Python Max RSS: $py_rss" + echo "Rust Max RSS: $rs_rss" + fi +} + +# Main execution +case "$RUN_MODE" in + python) + run_python "$SIZE" + ;; + rust) + run_rust "$SIZE" + ;; + both) + run_python "$SIZE" + run_rust "$SIZE" + compare_results "$SIZE" + ;; + *) + echo "Unknown mode: $RUN_MODE" + echo "Usage: $0 [python|rust|both] [size]" + exit 1 + ;; +esac + +echo "" +echo "========================================" +echo "Benchmark complete!" +echo "Results saved to: $RESULTS_DIR" +echo "========================================" diff --git a/scripts/build-wheel.sh b/scripts/build-wheel.sh new file mode 100755 index 0000000..cbb9edc --- /dev/null +++ b/scripts/build-wheel.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Build platform-specific wheel for site2skill + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +cd "$PROJECT_DIR" + +# Determine platform +if [[ "$(uname -s)" == "Darwin" ]]; then + ARCH=$(uname -m) + if [[ "$ARCH" == "arm64" ]]; then + PLATFORM_TAG="macosx_11_0_arm64" + RUST_TARGET="aarch64-apple-darwin" + else + PLATFORM_TAG="macosx_10_9_x86_64" + RUST_TARGET="x86_64-apple-darwin" + fi +elif [[ "$(uname -s)" == "Linux" ]]; then + ARCH=$(uname -m) + if ldd --version 2>&1 | grep -q musl; then + PLATFORM_TAG="musllinux_1_2_${ARCH}" + else + PLATFORM_TAG="manylinux_2_17_${ARCH}" + fi + RUST_TARGET="${ARCH}-unknown-linux-gnu" +else + echo "Unsupported platform" + exit 1 +fi + +echo "Building for platform: $PLATFORM_TAG" +echo "Rust target: $RUST_TARGET" + +# Build Rust binary +echo "Building Rust binary..." +cargo build --release --target "$RUST_TARGET" + +# Copy binary to Python package +mkdir -p python/site2skill/bin +cp "target/${RUST_TARGET}/release/site2skill" python/site2skill/bin/ +chmod +x python/site2skill/bin/site2skill + +# Build wheel +cd python +python -m build --wheel + +# Rename wheel with correct platform tag +cd dist +for wheel in site2skill-*-py3-none-any.whl; do + if [ -f "$wheel" ]; then + # Extract version from wheel name + VERSION=$(echo "$wheel" | sed 's/site2skill-\(.*\)-py3-none-any.whl/\1/') + NEW_NAME="site2skill-${VERSION}-py3-none-${PLATFORM_TAG}.whl" + mv "$wheel" "$NEW_NAME" + echo "Renamed: $wheel -> $NEW_NAME" + fi +done + +echo "Build complete!" +ls -lh *.whl diff --git a/scripts/build_wheels.py b/scripts/build_wheels.py new file mode 100644 index 0000000..4f68336 --- /dev/null +++ b/scripts/build_wheels.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Build script to compile Rust binary and create Python wheels.""" + +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +def get_platform_tag(): + """Get the platform tag for the current system.""" + import sysconfig + + # Get platform tag from sysconfig + platform = sysconfig.get_platform() + if sys.platform == "darwin": + # macOS + arch = platform.split("-")[-1] # arm64 or x86_64 + # Get macOS version + import platform as plat + mac_version = plat.mac_ver()[0] + major_version = int(mac_version.split(".")[0]) + if major_version >= 11: + return f"macosx_{major_version}_0_{arch}" + else: + return f"macosx_10_9_{arch}" + elif sys.platform == "linux": + # Linux + arch = platform.split("-")[-1] + # Check for musl vs glibc + try: + ldd_output = subprocess.check_output(["ldd", "--version"], text=True) + if "musl" in ldd_output: + return f"musllinux_1_2_{arch}" + else: + return f"manylinux_2_17_{arch}" + except Exception: + return f"manylinux_2_17_{arch}" + elif sys.platform == "win32": + arch = "win_amd64" if platform.endswith("AMD64") else "win_arm64" + return arch + else: + raise RuntimeError(f"Unsupported platform: {sys.platform}") + + +def build_rust_binary(target_dir: Path): + """Build the Rust binary in release mode.""" + print("Building Rust binary...") + + # Determine target triple + if sys.platform == "darwin": + import platform as plat + arch = plat.machine() + if arch == "arm64": + target = "aarch64-apple-darwin" + else: + target = "x86_64-apple-darwin" + elif sys.platform == "linux": + import platform as plat + arch = plat.machine() + if arch == "aarch64": + target = "aarch64-unknown-linux-gnu" + else: + target = "x86_64-unknown-linux-gnu" + elif sys.platform == "win32": + target = "x86_64-pc-windows-msvc" + else: + raise RuntimeError(f"Unsupported platform: {sys.platform}") + + # Build command + cmd = [ + "cargo", + "build", + "--release", + "--target", + target, + "--manifest-path", + "Cargo.toml", + ] + + # Set environment variables for memory-efficient build + env = os.environ.copy() + env["CARGO_INCREMENTAL"] = "0" + env["CARGO_BUILD_JOBS"] = "1" + + result = subprocess.run(cmd, env=env, cwd=Path(__file__).parent.parent) + + if result.returncode != 0: + print("Error: Rust build failed", file=sys.stderr) + sys.exit(1) + + # Copy binary to target directory + if sys.platform == "win32": + binary_name = "site2skill.exe" + else: + binary_name = "site2skill" + + target_path = Path(__file__).parent.parent / "target" / target / "release" / binary_name + dest_path = target_dir / "bin" / binary_name + + dest_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(target_path, dest_path) + print(f"Copied binary to {dest_path}") + + +def main(): + """Main build function.""" + # Get the python package directory + package_dir = Path(__file__).parent / "python" / "site2skill" + + # Build Rust binary + build_rust_binary(package_dir) + + print("Build complete!") + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_bench_site.py b/scripts/generate_bench_site.py new file mode 100755 index 0000000..9469424 --- /dev/null +++ b/scripts/generate_bench_site.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +""" +Generate a static documentation website for benchmarking site2skill. + +Creates a realistic documentation site with: +- Main index page +- Multiple documentation pages with headings, paragraphs, code blocks, and links +- Inter-page linking for crawler testing +- Consistent structure across all pages +""" + +import argparse +import os +import random +from pathlib import Path + + +def generate_page_content(page_num: int, total_pages: int, all_page_nums: list[int]) -> str: + """Generate realistic documentation content for a page.""" + + topics = [ + "Getting Started", + "Installation", + "Configuration", + "API Reference", + "Usage Guide", + "Best Practices", + "Troubleshooting", + "Advanced Topics", + "Examples", + "FAQ", + ] + + topic = topics[page_num % len(topics)] + + # Generate related links (link to nearby pages for realistic structure) + related_pages = [] + for i in range(min(3, total_pages)): + related_idx = (page_num + i + 1) % total_pages + if related_idx != page_num: + related_pages.append(related_pages) + + # Build related links HTML + related_links = "" + for idx in random.sample(all_page_nums, min(3, len(all_page_nums))): + if idx != page_num: + related_links += f'
  • Related: Page {idx}
    • \n' + + content = f''' + + + + + {topic} - Documentation Page {page_num} + + + +
    + + +

    {topic}

    +

    This is documentation page {page_num} of {total_pages}. + This page covers {topic.lower()} for the project.

    + +

    Overview

    +

    Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor + incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis + nostrud exercitation ullamco laboris.

    + +

    Key Points

    +
      +
    • First important point about {topic.lower()}
      • +
    • Second consideration for implementation
      • +
    • Third best practice to follow
      • +
    • Additional notes and recommendations
      • +
    + +

    Installation

    +

    To get started, install the package using one of the following methods:

    + + 
    # Using pip
+pip install example-package
+
+# Using uv
+uv pip install example-package
+
+# From source
+git clone https://github.com/example/repo.git
+cd repo
+pip install -e .
    + +

    Usage

    +

    Here's a basic example of how to use the library:

    + + 
    import example_package
+
+# Initialize the client
+client = example_package.Client(api_key="your-key")
+
+# Make a request
+result = client.process(data)
+print(result)
    + +

    Configuration

    +

    The following configuration options are available:

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    OptionTypeDefaultDescription
    timeoutint30Request timeout in seconds
    retriesint3Number of retry attempts
    debugboolfalseEnable debug logging
    + +

    Advanced Example

    +

    For more complex use cases, you can use the advanced API:

    + + 
    from example_package import AdvancedClient, Config
+
+config = Config(
+    timeout=60,
+    retries=5,
+    debug=True
+)
+
+client = AdvancedClient(config=config)
+
+async def process_data():
+    async with client.session() as session:
+        result = await session.fetch(url)
+        return result.transform()
    + +

    Troubleshooting

    +

    Common issues and solutions:

    + +

    Connection Errors

    +

    If you encounter connection errors, check your network settings and firewall + configuration. Ensure that the API endpoint is accessible.

    + +

    Authentication Failures

    +

    Verify that your API key is correct and has not expired. You can regenerate + your API key from the dashboard.

    + +

    See Also

    + +
    + + +''' + return content + + +def generate_index_page(total_pages: int) -> str: + """Generate the main index page.""" + + page_links = "" + for i in range(1, total_pages + 1): + page_links += f'
  • Documentation Page {i}
    • \n' + + content = f''' + + + + + Documentation Site + + + +
    +

    Documentation Site

    +

    Welcome to the benchmark documentation site. This site contains {total_pages} pages + of sample documentation for testing the site2skill tool.

    + +

    Navigation

    + + +

    About This Site

    +

    This site was generated for benchmarking purposes. Each page contains typical + documentation elements including:

    +
      +
    • Headings (h1, h2, h3)
      • +
    • Paragraphs with text content
      • +
    • Code blocks (inline and block)
      • +
    • Tables
      • +
    • Lists (ordered and unordered)
      • +
    • Internal and external links
      • +
    + +

    Usage

    +

    Use this site to test the performance of HTML to Markdown conversion tools. + The site is designed to be served locally for consistent benchmarking.

    + + 
    # Serve this site locally
+python -m http.server 8888 --directory bench-site
+
+# Test with site2skill
+site2skill http://localhost:8888/ test-skill
    +
    + + +''' + return content + + +def generate_site(num_pages: int, output_dir: str) -> None: + """Generate the complete documentation site.""" + + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + # Generate index page + index_content = generate_index_page(num_pages) + (output_path / "index.html").write_text(index_content) + + # Generate all documentation pages + all_page_nums = list(range(1, num_pages + 1)) + for page_num in all_page_nums: + page_content = generate_page_content(page_num, num_pages, all_page_nums) + (output_path / f"page{page_num}.html").write_text(page_content) + + print(f"Generated {num_pages + 1} files in {output_dir}/") + print(f" - 1 index page") + print(f" - {num_pages} documentation pages") + + +def main(): + parser = argparse.ArgumentParser( + description="Generate a static documentation website for benchmarking site2skill" + ) + parser.add_argument( + "--pages", + type=int, + default=100, + help="Number of documentation pages to generate (default: 100)" + ) + parser.add_argument( + "--output", + type=str, + default="bench-site", + help="Output directory for the generated site (default: bench-site)" + ) + + args = parser.parse_args() + + generate_site(args.pages, args.output) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_benchmark.py b/scripts/run_benchmark.py new file mode 100755 index 0000000..ebcc371 --- /dev/null +++ b/scripts/run_benchmark.py @@ -0,0 +1,533 @@ +#!/usr/bin/env python3 +""" +Benchmark runner for site2skill - Python vs Rust performance comparison. + +This script automates the benchmarking process: +1. Generates test sites of different sizes +2. Runs Python and Rust versions against them +3. Collects timing and memory metrics +4. Compares results and generates a report +""" + +import argparse +import os +import re +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + + +@dataclass +class BenchmarkResult: + """Results from a single benchmark run.""" + version: str # "python" or "rust" + pages: int + real_time: float # seconds + user_time: float # seconds + sys_time: float # seconds + max_rss_kb: Optional[int] # KB, None if not available + output_files: int + output_size_bytes: int + success: bool + error: Optional[str] = None + + +def parse_time_output(time_output: str) -> dict: + """Parse the output of /usr/bin/time -l or time command.""" + result = {} + + # macOS /usr/bin/time -l format + rss_match = re.search(r'maximum resident set size\s*=\s*(\d+)', time_output) + if rss_match: + result['max_rss_kb'] = int(rss_match.group(1)) // 1024 # Convert to KB + else: + result['max_rss_kb'] = None + + # Elapsed time (real) + elapsed_match = re.search(r'(\d+):(\d+\.\d+) elapsed', time_output) + if elapsed_match: + result['real_time'] = int(elapsed_match.group(1)) * 60 + float(elapsed_match.group(2)) + else: + # Try alternative format + elapsed_match = re.search(r'(\d+\.\d+) real', time_output) + if elapsed_match: + result['real_time'] = float(elapsed_match.group(1)) + else: + result['real_time'] = 0.0 + + # CPU time + user_match = re.search(r'(\d+\.\d+) user', time_output) + sys_match = re.search(r'(\d+\.\d+) sys', time_output) + result['user_time'] = float(user_match.group(1)) if user_match else 0.0 + result['sys_time'] = float(sys_match.group(1)) if sys_match else 0.0 + + return result + + +def run_benchmark( + version: str, + pages: int, + site_dir: Path, + output_dir: Path, + temp_dir: Path, + executable: str, + server_port: int = 8888, + wait: bool = False, +) -> BenchmarkResult: + """Run a single benchmark.""" + + print(f"\n{'='*60}") + print(f"Benchmark: {version.upper()} - {pages} pages") + print(f"{'='*60}") + + # Clean output directory + if output_dir.exists(): + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True) + + # Clean temp directory + if temp_dir.exists(): + shutil.rmtree(temp_dir) + temp_dir.mkdir(parents=True) + + # Build command + url = f"http://localhost:{server_port}/" + cmd = [ + executable, + url, + str(output_dir / "test-skill"), + "--temp-dir", str(temp_dir), + "--clean", + ] + + if wait: + # Add delay for fair comparison + if version == "rust": + cmd.extend(["--delay-ms", "100"]) + + print(f"Command: {' '.join(cmd)}") + + # Run with timing + start_time = time.time() + + try: + # Use /usr/bin/time for detailed metrics on macOS + time_cmd = ["/usr/bin/time", "-l"] + cmd + result = subprocess.run( + time_cmd, + capture_output=True, + text=True, + timeout=600, # 10 minute timeout + ) + + stderr_output = result.stderr + stdout_output = result.stdout + + except subprocess.TimeoutExpired: + return BenchmarkResult( + version=version, + pages=pages, + real_time=0, + user_time=0, + sys_time=0, + max_rss_kb=None, + output_files=0, + output_size_bytes=0, + success=False, + error="Timeout after 10 minutes" + ) + except Exception as e: + return BenchmarkResult( + version=version, + pages=pages, + real_time=0, + user_time=0, + sys_time=0, + max_rss_kb=None, + output_files=0, + output_size_bytes=0, + success=False, + error=str(e) + ) + + real_time = time.time() - start_time + + # Parse time output + time_metrics = parse_time_output(stderr_output) + + # Count output files + references_dir = output_dir / "test-skill" / "references" + if references_dir.exists(): + output_files = len(list(references_dir.glob("*.md"))) + else: + # Try legacy docs/ directory + docs_dir = output_dir / "test-skill" / "docs" + if docs_dir.exists(): + output_files = len(list(docs_dir.glob("*.md"))) + else: + output_files = 0 + + # Calculate output size + output_size = 0 + if references_dir.exists(): + for f in references_dir.glob("*.md"): + output_size += f.stat().st_size + elif docs_dir.exists(): + for f in docs_dir.glob("*.md"): + output_size += f.stat().st_size + + success = result.returncode == 0 + + return BenchmarkResult( + version=version, + pages=pages, + real_time=real_time, + user_time=time_metrics.get('user_time', 0), + sys_time=time_metrics.get('sys_time', 0), + max_rss_kb=time_metrics.get('max_rss_kb'), + output_files=output_files, + output_size_bytes=output_size, + success=success, + error=None if success else f"Exit code: {result.returncode}" + ) + + +def start_http_server(site_dir: Path, port: int = 8888) -> subprocess.Popen: + """Start a local HTTP server for the test site.""" + print(f"Starting HTTP server on port {port}...") + + server = subprocess.Popen( + [sys.executable, "-m", "http.server", str(port), "--directory", str(site_dir)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + # Wait for server to start + time.sleep(1) + + # Check if server is running + if server.poll() is not None: + raise RuntimeError("Failed to start HTTP server") + + return server + + +def run_benchmarks( + python_sizes: list[int], + rust_sizes: list[int], + site_base: Path, + results_dir: Path, + server_port: int = 8888, + wait: bool = False, +) -> list[BenchmarkResult]: + """Run all benchmarks.""" + + all_results = [] + + # Start HTTP server + site_dir = site_base / "bench-site" + server = start_http_server(site_dir, server_port) + + try: + # Run Python benchmarks + for size in python_sizes: + # Generate site for this size if needed + site_path = site_base / f"bench-site-{size}" + if not site_path.exists(): + print(f"\nGenerating {size}-page test site...") + subprocess.run( + [sys.executable, str(site_base / "generate_bench_site.py"), + "--pages", str(size), "--output", str(site_path)], + check=True, + ) + + result = run_benchmark( + version="python", + pages=size, + site_dir=site_path, + output_dir=results_dir / f"bench-py-{size}", + temp_dir=Path(f"/tmp/bench-py-{size}"), + executable="site2skill", + server_port=server_port, + wait=wait, + ) + all_results.append(result) + + # Save individual result + save_result(result, results_dir / f"python-{size}.txt") + + # Run Rust benchmarks + for size in rust_sizes: + # Generate site for this size if needed + site_path = site_base / f"bench-site-{size}" + if not site_path.exists(): + print(f"\nGenerating {size}-page test site...") + subprocess.run( + [sys.executable, str(site_base / "generate_bench_site.py"), + "--pages", str(size), "--output", str(site_path)], + check=True, + ) + + result = run_benchmark( + version="rust", + pages=size, + site_dir=site_path, + output_dir=results_dir / f"bench-rs-{size}", + temp_dir=Path(f"/tmp/bench-rs-{size}"), + executable="./target/release/site2skill", + server_port=server_port, + wait=wait, + ) + all_results.append(result) + + # Save individual result + save_result(result, results_dir / f"rust-{size}.txt") + + finally: + # Stop HTTP server + server.terminate() + server.wait() + print("\nHTTP server stopped") + + return all_results + + +def save_result(result: BenchmarkResult, filepath: Path) -> None: + """Save benchmark result to file.""" + filepath.parent.mkdir(parents=True, exist_ok=True) + + with open(filepath, 'w') as f: + f.write(f"Version: {result.version}\n") + f.write(f"Pages: {result.pages}\n") + f.write(f"Real Time: {result.real_time:.2f}s\n") + f.write(f"User Time: {result.user_time:.2f}s\n") + f.write(f"Sys Time: {result.sys_time:.2f}s\n") + f.write(f"Max RSS: {result.max_rss_kb} KB\n" if result.max_rss_kb else "Max RSS: N/A\n") + f.write(f"Output Files: {result.output_files}\n") + f.write(f"Output Size: {result.output_size_bytes / 1024:.1f} KB\n") + f.write(f"Success: {result.success}\n") + if result.error: + f.write(f"Error: {result.error}\n") + + +def generate_report(results: list[BenchmarkResult], results_dir: Path) -> None: + """Generate a comparison report.""" + + report_path = results_dir / "benchmark_report.md" + + # Group results by size + by_size = {} + for r in results: + if r.pages not in by_size: + by_size[r.pages] = {} + by_size[r.pages][r.version] = r + + with open(report_path, 'w') as f: + f.write("# Benchmark Results: Python vs Rust\n\n") + f.write(f"Generated: {time.strftime('%Y-%m-%d %H:%M:%S')}\n\n") + + # Time comparison table + f.write("## Execution Time Comparison\n\n") + f.write("| Pages | Python (s) | Rust (s) | Speedup |\n") + f.write("|-------|------------|----------|---------|\n") + + for size in sorted(by_size.keys()): + py = by_size[size].get("python") + rs = by_size[size].get("rust") + + if py and rs: + speedup = py.real_time / rs.real_time if rs.real_time > 0 else 0 + f.write(f"| {size} | {py.real_time:.2f} | {rs.real_time:.2f} | {speedup:.2f}x |\n") + elif py: + f.write(f"| {size} | {py.real_time:.2f} | - | - |\n") + elif rs: + f.write(f"| {size} | - | {rs.real_time:.2f} | - |\n") + + # Memory comparison table + f.write("\n## Memory Usage Comparison\n\n") + f.write("| Pages | Python (KB) | Rust (KB) | Ratio |\n") + f.write("|-------|-------------|-----------|-------|\n") + + for size in sorted(by_size.keys()): + py = by_size[size].get("python") + rs = by_size[size].get("rust") + + if py and rs and py.max_rss_kb and rs.max_rss_kb: + ratio = py.max_rss_kb / rs.max_rss_kb if rs.max_rss_kb > 0 else 0 + f.write(f"| {size} | {py.max_rss_kb} | {rs.max_rss_kb} | {ratio:.2f}x |\n") + elif py and py.max_rss_kb: + f.write(f"| {size} | {py.max_rss_kb} | - | - |\n") + elif rs and rs.max_rss_kb: + f.write(f"| {size} | - | {rs.max_rss_kb} | - |\n") + + # Output validation + f.write("\n## Output Validation\n\n") + f.write("| Pages | Python Files | Rust Files | Python Size | Rust Size |\n") + f.write("|-------|--------------|------------|-------------|-----------|\n") + + for size in sorted(by_size.keys()): + py = by_size[size].get("python") + rs = by_size[size].get("rust") + + py_size = f"{py.output_size_bytes / 1024:.1f} KB" if py else "-" + rs_size = f"{rs.output_size_bytes / 1024:.1f} KB" if rs else "-" + + f.write(f"| {size} | {py.output_files if py else '-'} | {rs.output_files if rs else '-'} | {py_size} | {rs_size} |\n") + + # Summary + f.write("\n## Summary\n\n") + + rust_faster = [] + for size, versions in by_size.items(): + if "python" in versions and "rust" in versions: + py = versions["python"] + rs = versions["rust"] + if py.real_time > 0 and rs.real_time > 0: + speedup = py.real_time / rs.real_time + rust_faster.append((size, speedup)) + + if rust_faster: + avg_speedup = sum(s for _, s in rust_faster) / len(rust_faster) + f.write(f"Rust is **{avg_speedup:.2f}x** faster on average.\n\n") + + for size, speedup in rust_faster: + f.write(f"- {size} pages: **{speedup:.2f}x** speedup\n") + + print(f"\nReport saved to: {report_path}") + + +def main(): + parser = argparse.ArgumentParser( + description="Benchmark runner for site2skill Python vs Rust" + ) + parser.add_argument( + "--python-sizes", + type=int, + nargs="+", + default=[10, 100, 500], + help="Page counts for Python benchmarks (default: 10 100 500)" + ) + parser.add_argument( + "--rust-sizes", + type=int, + nargs="+", + default=[10, 100, 500], + help="Page counts for Rust benchmarks (default: 10 100 500)" + ) + parser.add_argument( + "--site-base", + type=str, + default="bench-site", + help="Base directory for test sites (default: bench-site)" + ) + parser.add_argument( + "--results-dir", + type=str, + default="bench-results", + help="Directory for benchmark results (default: bench-results)" + ) + parser.add_argument( + "--server-port", + type=int, + default=8888, + help="HTTP server port (default: 8888)" + ) + parser.add_argument( + "--wait", + action="store_true", + help="Add delay between requests for fair comparison" + ) + parser.add_argument( + "--rust-only", + action="store_true", + help="Run only Rust benchmarks" + ) + parser.add_argument( + "--python-only", + action="store_true", + help="Run only Python benchmarks" + ) + + args = parser.parse_args() + + site_base = Path(args.site_base) + results_dir = Path(args.results_dir) + + # Check prerequisites + if not args.python_only: + rust_binary = Path("target/release/site2skill") + if not rust_binary.exists(): + print("Rust binary not found. Building...") + subprocess.run(["cargo", "build", "--release"], check=True) + + # Run benchmarks + results = [] + + if not args.rust_only: + print("\n" + "="*60) + print("Running Python Benchmarks") + print("="*60) + for size in args.python_sizes: + site_path = site_base / f"bench-site-{size}" + if not site_path.exists(): + print(f"\nGenerating {size}-page test site...") + subprocess.run( + [sys.executable, "scripts/generate_bench_site.py", + "--pages", str(size), "--output", str(site_path)], + check=True, + ) + + result = run_benchmark( + version="python", + pages=size, + site_dir=site_path, + output_dir=results_dir / f"bench-py-{size}", + temp_dir=Path(f"/tmp/bench-py-{size}"), + executable="site2skill", + server_port=args.server_port, + wait=args.wait, + ) + results.append(result) + save_result(result, results_dir / f"python-{size}.txt") + + if not args.python_only: + print("\n" + "="*60) + print("Running Rust Benchmarks") + print("="*60) + for size in args.rust_sizes: + site_path = site_base / f"bench-site-{size}" + if not site_path.exists(): + print(f"\nGenerating {size}-page test site...") + subprocess.run( + [sys.executable, "scripts/generate_bench_site.py", + "--pages", str(size), "--output", str(site_path)], + check=True, + ) + + result = run_benchmark( + version="rust", + pages=size, + site_dir=site_path, + output_dir=results_dir / f"bench-rs-{size}", + temp_dir=Path(f"/tmp/bench-rs-{size}"), + executable="./target/release/site2skill", + server_port=args.server_port, + wait=args.wait, + ) + results.append(result) + save_result(result, results_dir / f"rust-{size}.txt") + + # Generate report + generate_report(results, results_dir) + + print("\n" + "="*60) + print("Benchmark Complete!") + print("="*60) + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_benchmark.py b/scripts/validate_benchmark.py new file mode 100755 index 0000000..2cab959 --- /dev/null +++ b/scripts/validate_benchmark.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +""" +Validate and compare benchmark outputs between Python and Rust versions. + +This script: +1. Compares file counts between Python and Rust outputs +2. Checks structural similarity of generated markdown +3. Reports any significant differences +""" + +import argparse +import difflib +import os +from pathlib import Path + + +def count_md_files(directory: Path) -> int: + """Count markdown files in references or docs directory.""" + references_dir = directory / "references" + docs_dir = directory / "docs" + + if references_dir.exists(): + return len(list(references_dir.glob("*.md"))) + elif docs_dir.exists(): + return len(list(docs_dir.glob("*.md"))) + else: + return 0 + + +def get_md_files(directory: Path) -> list[Path]: + """Get list of markdown files.""" + references_dir = directory / "references" + docs_dir = directory / "docs" + + if references_dir.exists(): + return sorted(references_dir.glob("*.md")) + elif docs_dir.exists(): + return sorted(docs_dir.glob("*.md")) + else: + return [] + + +def read_file_head(filepath: Path, lines: int = 20) -> list[str]: + """Read first N lines of a file.""" + try: + with open(filepath, 'r', encoding='utf-8') as f: + return [next(f) for _ in range(lines)] + except (StopIteration, FileNotFoundError): + return [] + + +def normalize_content(lines: list[str]) -> str: + """Normalize content for comparison (remove timestamps, paths, etc.).""" + normalized = [] + for line in lines: + # Remove or normalize variable content + # Keep structure but ignore minor differences + normalized.append(line) + return ''.join(normalized) + + +def compare_files(file1: Path, file2: Path, name1: str, name2: str) -> dict: + """Compare two markdown files.""" + lines1 = read_file_head(file1) + lines2 = read_file_head(file2) + + content1 = normalize_content(lines1) + content2 = normalize_content(lines2) + + # Calculate similarity + similarity = difflib.SequenceMatcher(None, content1, content2).ratio() + + # Generate diff if significantly different + diff = [] + if similarity < 0.9: + diff = list(difflib.unified_diff( + lines1, + lines2, + fromfile=f"{name1}/{file1.name}", + tofile=f"{name2}/{file2.name}", + n=5 + )) + + return { + 'file1': str(file1), + 'file2': str(file2), + 'similarity': similarity, + 'diff': diff, + } + + +def validate_benchmark( + python_dir: Path, + rust_dir: Path, + size: int, + verbose: bool = False, +) -> dict: + """Validate benchmark outputs.""" + + result = { + 'size': size, + 'python_files': 0, + 'rust_files': 0, + 'file_match': False, + 'comparisons': [], + 'avg_similarity': 0.0, + } + + # Count files + result['python_files'] = count_md_files(python_dir) + result['rust_files'] = count_md_files(rust_dir) + result['file_match'] = result['python_files'] == result['rust_files'] + + # Get file lists + py_files = get_md_files(python_dir) + rs_files = get_md_files(rust_dir) + + if not py_files or not rs_files: + return result + + # Compare files (match by filename when possible) + py_by_name = {f.name: f for f in py_files} + rs_by_name = {f.name: f for f in rs_files} + + common_names = set(py_by_name.keys()) & set(rs_by_name.keys()) + + if not common_names: + # No common filenames, compare by position + min_len = min(len(py_files), len(rs_files)) + comparisons = min(5, min_len) # Compare up to 5 files + + for i in range(comparisons): + comparison = compare_files( + py_files[i], rs_files[i], + "Python", "Rust" + ) + result['comparisons'].append(comparison) + else: + # Compare files with matching names + comparisons = min(5, len(common_names)) + for name in list(common_names)[:comparisons]: + comparison = compare_files( + py_by_name[name], rs_by_name[name], + "Python", "Rust" + ) + result['comparisons'].append(comparison) + + # Calculate average similarity + if result['comparisons']: + result['avg_similarity'] = sum( + c['similarity'] for c in result['comparisons'] + ) / len(result['comparisons']) + + return result + + +def print_report(results: list[dict], verbose: bool = False) -> None: + """Print validation report.""" + + print("\n" + "="*60) + print("Benchmark Output Validation Report") + print("="*60) + + for result in results: + print(f"\n### {result['size']} pages") + print(f" Python files: {result['python_files']}") + print(f" Rust files: {result['rust_files']}") + print(f" File count match: {'✓' if result['file_match'] else '✗'}") + + if result['comparisons']: + print(f" Average similarity: {result['avg_similarity']*100:.1f}%") + + if verbose: + for comp in result['comparisons']: + print(f"\n File: {comp['file1']}") + print(f" Similarity: {comp['similarity']*100:.1f}%") + + if comp['diff']: + print(" Differences:") + for line in comp['diff'][:10]: + print(f" {line.rstrip()}") + if len(comp['diff']) > 10: + print(f" ... and {len(comp['diff']) - 10} more lines") + + # Summary + print("\n" + "="*60) + print("Summary") + print("="*60) + + all_match = all(r['file_match'] for r in results) + avg_sim = sum(r['avg_similarity'] for r in results if r['comparisons']) + avg_sim /= len([r for r in results if r['comparisons']]) if any(r['comparisons'] for r in results) else 0 + + print(f"All file counts match: {'✓' if all_match else '✗'}") + print(f"Average content similarity: {avg_sim*100:.1f}%") + + if avg_sim > 0.8: + print("\n✓ Outputs are structurally similar") + else: + print("\n⚠ Outputs show significant differences (expected due to different converters)") + + +def main(): + parser = argparse.ArgumentParser( + description="Validate benchmark outputs between Python and Rust versions" + ) + parser.add_argument( + "--results-dir", + type=str, + default="bench-results", + help="Directory containing benchmark results (default: bench-results)" + ) + parser.add_argument( + "--sizes", + type=int, + nargs="+", + default=[10, 100, 500], + help="Page sizes to validate (default: 10 100 500)" + ) + parser.add_argument( + "--verbose", "-v", + action="store_true", + help="Show detailed diff output" + ) + + args = parser.parse_args() + + results_dir = Path(args.results_dir) + results = [] + + for size in args.sizes: + python_dir = results_dir / f"bench-py-{size}" / "test-skill" + rust_dir = results_dir / f"bench-rs-{size}" / "test-skill" + + if not python_dir.exists() and not rust_dir.exists(): + print(f"Skipping {size} pages: No benchmark data found") + continue + + result = validate_benchmark(python_dir, rust_dir, size, args.verbose) + results.append(result) + + print_report(results, args.verbose) + + # Save report + report_path = results_dir / "validation_report.txt" + with open(report_path, 'w') as f: + for result in results: + f.write(f"Size: {result['size']} pages\n") + f.write(f" Python files: {result['python_files']}\n") + f.write(f" Rust files: {result['rust_files']}\n") + f.write(f" Match: {result['file_match']}\n") + f.write(f" Similarity: {result['avg_similarity']*100:.1f}%\n\n") + + print(f"\nValidation report saved to: {report_path}") + + +if __name__ == "__main__": + main() diff --git a/site2skill/__init__.py b/site2skill/__init__.py deleted file mode 100644 index 495d297..0000000 --- a/site2skill/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""site2skill - Turn any website into a Claude Skill""" - -__version__ = "0.1.1" diff --git a/site2skill/convert_to_markdown.py b/site2skill/convert_to_markdown.py deleted file mode 100644 index 23023d9..0000000 --- a/site2skill/convert_to_markdown.py +++ /dev/null @@ -1,113 +0,0 @@ -import os -import re -import argparse -import logging -from typing import Optional -from bs4 import BeautifulSoup -from markdownify import MarkdownConverter - -# Configure logging -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - -def clean_html(soup: BeautifulSoup) -> BeautifulSoup: - """Remove unwanted tags and noise from the HTML.""" - # Remove non-content tags - for tag in soup(["script", "style", "meta", "link", "noscript", "iframe", "svg"]): - tag.decompose() - - # Remove navigation, header, footer, sidebar if they exist - # Common selectors for documentation sites - selectors = [ - ".sidebar", "header", "footer", ".nav", ".menu", "#sidebar", - ".navigation", ".toc", "#toc", ".footer", "#footer" - ] - - for selector in selectors: - for tag in soup.select(selector): - tag.decompose() - - return soup - -def post_process_markdown(md_content: str) -> str: - """Clean up the generated Markdown.""" - # Remove multiple consecutive blank lines - md_content = re.sub(r'\n{3,}', '\n\n', md_content) - - # Remove trailing whitespace - md_content = "\n".join([line.rstrip() for line in md_content.splitlines()]) - - return md_content - -def convert_html_to_md(html_path: str, output_path: Optional[str] = None, source_url: Optional[str] = None, fetched_at: Optional[str] = None) -> None: - """Convert a single HTML file to Markdown with Frontmatter.""" - try: - with open(html_path, 'r', encoding='utf-8') as f: - html_content = f.read() - - soup = BeautifulSoup(html_content, 'html.parser') - - # Extract title - title = "Untitled" - if soup.title: - title = soup.title.string.strip() - elif soup.h1: - title = soup.h1.get_text().strip() - - # Extract main content - # Try to find the most relevant content container - main_content = soup.find('main') - if not main_content: - main_content = soup.find('article') - if not main_content: - main_content = soup.find('div', class_='content') - if not main_content: - main_content = soup.body - - if not main_content: - logger.warning(f"No main content found in {html_path}") - return - - clean_html(main_content) - - # Convert to Markdown - # heading_style="atx" uses # for headers - md_body = MarkdownConverter(heading_style="atx").convert_soup(main_content) - md_body = post_process_markdown(md_body) - - # Create Frontmatter - # Escape quotes in title for YAML - escaped_title = title.replace('"', '\\"') - - frontmatter = "---\n" - frontmatter += f'title: "{escaped_title}"\n' - if source_url: - frontmatter += f'source_url: "{source_url}"\n' - if fetched_at: - frontmatter += f'fetched_at: "{fetched_at}"\n' - frontmatter += "---\n\n" - - final_md = frontmatter + md_body - - if output_path: - # Ensure output directory exists - os.makedirs(os.path.dirname(output_path), exist_ok=True) - with open(output_path, 'w', encoding='utf-8') as f: - f.write(final_md) - logger.info(f"Converted: {html_path} -> {output_path}") - else: - print(final_md) - - except Exception as e: - logger.error(f"Error converting {html_path}: {e}") - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Convert HTML to Markdown with Metadata.") - parser.add_argument("input_file", help="Path to input HTML file") - parser.add_argument("--output", "-o", help="Path to output Markdown file") - parser.add_argument("--url", help="Source URL of the page") - parser.add_argument("--fetched-at", help="Timestamp of fetch (ISO8601)") - - args = parser.parse_args() - - convert_html_to_md(args.input_file, args.output, args.url, args.fetched_at) diff --git a/site2skill/fetch_site.py b/site2skill/fetch_site.py deleted file mode 100644 index 6e59847..0000000 --- a/site2skill/fetch_site.py +++ /dev/null @@ -1,200 +0,0 @@ -import argparse -import subprocess -import sys -import os -import shutil -import logging -import re -from urllib.parse import urlparse -from dataclasses import dataclass - -# Configure logging -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - -LOCALIZED_QUERY_KEYS = ("hl", "lang", "locale") - - -@dataclass(frozen=True) -class CrawlConstraints: - domain: str - include_directory: str | None - path_description: str - accept_regex: str - reject_regex: str - - -def build_crawl_constraints(url: str) -> CrawlConstraints: - """Build wget-compatible crawl constraints from the starting URL.""" - parsed_url = urlparse(url) - domain = parsed_url.netloc - path = parsed_url.path or "/" - has_trailing_slash = path.endswith("/") - - if has_trailing_slash: - include_directory = path - path_description = path - allowed_prefix = re.escape(f"{parsed_url.scheme}://{domain}{path}") - accept_regex = rf"^{allowed_prefix}.*$" - else: - include_directory = os.path.dirname(path) or "/" - path_description = f"{path} and descendants" - exact_url = re.escape(f"{parsed_url.scheme}://{domain}{path}") - descendant_prefix = re.escape(f"{parsed_url.scheme}://{domain}{path}/") - accept_regex = rf"^({exact_url}([?#].*)?|{descendant_prefix}.*)$" - - query_keys = "|".join(re.escape(key) for key in LOCALIZED_QUERY_KEYS) - reject_regex = rf"[?&]({query_keys})=" - - return CrawlConstraints( - domain=domain, - include_directory=include_directory, - path_description=path_description, - accept_regex=accept_regex, - reject_regex=reject_regex, - ) - - -def check_wget_installed() -> bool: - """Check if wget is installed and available in the PATH.""" - return shutil.which("wget") is not None - - -def fetch_site(url: str, output_dir: str) -> None: - """ - Fetch a website using wget with robust settings. - - Args: - url: The URL to fetch. - output_dir: The directory to save the fetched content. - """ - # Validate URL scheme - parsed_url = urlparse(url) - if parsed_url.scheme not in ('http', 'https'): - logger.error(f"Invalid URL scheme: {parsed_url.scheme}. Only 'http' and 'https' are supported.") - sys.exit(1) - - if not parsed_url.netloc: - logger.error(f"Invalid URL: {url}. Domain is missing.") - sys.exit(1) - - # Check for wget - if not check_wget_installed(): - logger.error("wget is not installed. Please install wget to use this tool.") - sys.exit(1) - - constraints = build_crawl_constraints(url) - - # Define temporary crawl directory - crawl_dir = os.path.join(output_dir, "crawl") - - # Create output directory if it doesn't exist - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - # Clean/Create crawl directory - if os.path.exists(crawl_dir): - shutil.rmtree(crawl_dir) - os.makedirs(crawl_dir) - - logger.info(f"Fetching {url} to {crawl_dir}...") - logger.info(f"Domain restricted to: {constraints.domain}") - logger.info(f"Path restricted to: {constraints.path_description}") - logger.info(f"Rejected query keys: {', '.join(LOCALIZED_QUERY_KEYS)}") - - # Construct wget command - # --recursive: download recursively - # --level=5: max recursion depth - # --no-parent: don't go up - # --domains: restrict to specific domain - # --adjust-extension: save as .html - # --convert-links: make links local - # --accept: only html files - # --user-agent: custom UA - # --execute robots=on: respect robots.txt - # --wait=1: be polite - - cmd = [ - "wget", - "--recursive", - "--level=5", - "--no-parent", - f"--domains={constraints.domain}", - "--adjust-extension", - "--convert-links", - # Use reject instead of accept to allow extensionless URLs (which are often HTML) - "--reject=css,js,png,jpg,jpeg,gif,svg,ico,woff,woff2,ttf,eot,zip,tar,gz,pdf,xml,json,txt", - f"--accept-regex={constraints.accept_regex}", - f"--reject-regex={constraints.reject_regex}", - "--user-agent=site2skill/0.1 (+https://github.com/laiso/site2skill)", - "--execute", "robots=on", - "--wait=1", - "--random-wait", - "-P", crawl_dir, - "--", - url - ] - - if constraints.include_directory: - cmd.insert(5, f"--include-directories={constraints.include_directory}") - - # Run wget with progress tracking - try: - import re as regex - process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1 - ) - - import time - downloaded_urls = set() - current_url = "" - start_time = time.time() - - for line in process.stdout: - # Match URL being fetched - url_match = regex.search(r'--\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}--\s+(\S+)', line) - if url_match: - current_url = url_match.group(1) - - # Match successful save - if "saved" in line.lower() or "Saving to:" in line: - downloaded_urls.add(current_url) - count = len(downloaded_urls) - elapsed = time.time() - start_time - rate = count / elapsed if elapsed > 0 else 0 - mins, secs = divmod(int(elapsed), 60) - short_url = current_url[-40:] if len(current_url) > 40 else current_url - print(f"\r[{count} pages | {mins}m{secs:02d}s | {rate:.1f}/s] {short_url:<40}", end="", flush=True) - - process.wait() - print() # New line after progress - - elapsed = time.time() - start_time - mins, secs = divmod(int(elapsed), 60) - logger.info(f"Download complete. {len(downloaded_urls)} pages in {mins}m{secs:02d}s.") - - if process.returncode == 4: - logger.warning("Wget returned exit code 4 (Network Failure). Some files may not have been downloaded. Continuing...") - elif process.returncode == 6: - logger.warning("Wget returned exit code 6 (Username/Password Authentication Failure). Continuing...") - elif process.returncode == 8: - logger.warning("Wget returned exit code 8 (Server Error). Some links returned 404/403. Continuing...") - elif process.returncode != 0: - logger.warning(f"Wget returned exit code {process.returncode}. Download may be incomplete...") - - except Exception as e: - logger.error(f"An error occurred while running wget: {e}") - sys.exit(1) - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Fetch a website for Skill creation.") - parser.add_argument("url", help="URL of the documentation site") - parser.add_argument("--output", "-o", default="temp_docs", help="Output directory") - - args = parser.parse_args() - - fetch_site(args.url, args.output) diff --git a/site2skill/generate_skill_structure.py b/site2skill/generate_skill_structure.py deleted file mode 100644 index 8edb182..0000000 --- a/site2skill/generate_skill_structure.py +++ /dev/null @@ -1,164 +0,0 @@ -import os -import shutil -import argparse -import logging -import sys -from typing import Optional - -if sys.version_info >= (3, 9): - from importlib.resources import files as importlib_files -else: - from importlib_resources import files as importlib_files - -# Configure logging -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - - -def generate_skill_structure( - skill_name: str, - source_dir: Optional[str], - output_base: str = ".claude/skills", - target_agent: Optional[str] = None, -) -> None: - """ - Generate the Skill structure following SKILL.md + references/ pattern. - Structure: - / - SKILL.md # Entry point, usage instructions - references/ # Documentation files (preserves directory structure) - scripts/ # (Optional) Executable code - """ - skill_dir = os.path.join(output_base, skill_name) - - # Define subdirectories - references_dir = os.path.join(skill_dir, "references") - scripts_dir = os.path.join(skill_dir, "scripts") - - # Create directories - if os.path.exists(skill_dir): - logger.warning(f"Skill directory {skill_dir} already exists.") - else: - os.makedirs(skill_dir) - - os.makedirs(references_dir, exist_ok=True) - os.makedirs(scripts_dir, exist_ok=True) - - # Create SKILL.md - skill_md_path = os.path.join(skill_dir, "SKILL.md") - if not os.path.exists(skill_md_path): - frontmatter_lines = [ - "---", - f"name: {skill_name}", - f"description: {skill_name.upper()} documentation assistant", - ] - if target_agent: - frontmatter_lines.append("metadata:") - frontmatter_lines.append(f" target_agent: {target_agent}") - frontmatter_lines.append("---") - frontmatter = "\n".join(frontmatter_lines) - with open(skill_md_path, "w", encoding="utf-8") as f: - f.write(f"""{frontmatter} - -# {skill_name.upper()} Skill - -This skill provides access to {skill_name.upper()} documentation. - -## Documentation - -All documentation files are in the `references/` directory as Markdown files. -For legacy skills, documentation may live in `docs/`. - -## Search Tool - -```bash -# Run the search script (use python or python3) -python scripts/search_docs.py "" -``` - -Options: -- `--json` - Output as JSON -- `--max-results N` - Limit results (default: 10) - -## Usage - -1. Search or read files in `references/` for relevant information (fallback to `docs/` for legacy) -2. Each file has frontmatter with `source_url` and `fetched_at` -3. Always cite the source URL in responses -4. Note the fetch date - documentation may have changed - -## Response Format - -``` -[Answer based on documentation] - -**Source:** [source_url] -**Fetched:** [fetched_at] -``` -""") - logger.info(f"Created {skill_md_path}") - - # Copy scripts using importlib.resources - dest_search_script = os.path.join(scripts_dir, "search_docs.py") - dest_readme = os.path.join(scripts_dir, "README.md") - - try: - templates = importlib_files("site2skill").joinpath("templates") - - search_script_resource = templates.joinpath("search_docs.py") - with open(dest_search_script, "w", encoding="utf-8") as f: - f.write(search_script_resource.read_text(encoding="utf-8")) - os.chmod(dest_search_script, 0o755) - logger.info("Installed search_docs.py") - - readme_resource = templates.joinpath("scripts_README.md") - with open(dest_readme, "w", encoding="utf-8") as f: - f.write(readme_resource.read_text(encoding="utf-8")) - logger.info("Installed scripts/README.md") - except Exception as e: - logger.warning(f"Failed to copy templates: {e}") - - # Copy Markdown files (preserve directory structure) - if source_dir and os.path.exists(source_dir): - logger.info(f"Copying files from {source_dir}...") - file_count = 0 - - for root, _, files in os.walk(source_dir): - for file in files: - if file.endswith(".md"): - src_path = os.path.join(root, file) - - # Preserve directory structure relative to source_dir - rel_path = os.path.relpath(src_path, source_dir) - dst_path = os.path.join(references_dir, rel_path) - - # Security check: Ensure dst_path is strictly within references_dir - abs_dst_path = os.path.abspath(dst_path) - abs_references_dir = os.path.abspath(references_dir) - - if os.path.commonpath([abs_dst_path, abs_references_dir]) != abs_references_dir: - logger.warning(f"Skipping potential path traversal file: {file}") - continue - - # Create parent directories if needed - parent_dir = os.path.dirname(dst_path) - if parent_dir: - os.makedirs(parent_dir, exist_ok=True) - - shutil.copy2(src_path, dst_path) - file_count += 1 - - logger.info(f"Copied {file_count} files to references/") - else: - logger.warning(f"Source directory {source_dir} not found or empty.") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Generate Skill Structure.") - parser.add_argument("skill_name", help="Name of the skill (e.g., payjp)") - parser.add_argument("--source", "-s", help="Source directory containing Markdown files") - parser.add_argument("--output", "-o", default=".claude/skills", help="Base output directory") - - args = parser.parse_args() - - generate_skill_structure(args.skill_name, args.source, args.output) diff --git a/site2skill/main.py b/site2skill/main.py deleted file mode 100644 index 2f8ffe1..0000000 --- a/site2skill/main.py +++ /dev/null @@ -1,181 +0,0 @@ -import argparse -import os -import shutil -import glob -import datetime -import re -import logging -from urllib.parse import urlparse - -# Configure logging -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - -# Import our modules -try: - from .fetch_site import fetch_site - from .convert_to_markdown import convert_html_to_md - from .normalize_markdown import normalize_markdown - from .generate_skill_structure import generate_skill_structure - from .validate_skill import validate_skill - from .package_skill import package_skill - from .utils import sanitize_path, html_to_md_path -except ImportError as e: - logger.error(f"Could not import pipeline modules: {e}") - logger.error("Make sure you have installed dependencies: pip install beautifulsoup4 markdownify pyyaml") - exit(1) - -def main(): - parser = argparse.ArgumentParser(description="Web Docs to Claude Code Skill Pipeline") - parser.add_argument("url", help="URL of the documentation site") - parser.add_argument("skill_name", help="Name of the skill (e.g., payjp)") - parser.add_argument( - "--target", - choices=["claude", "claude-desktop", "cursor", "gemini", "codex"], - default="claude", - help="Target agent (sets default output directory)", - ) - parser.add_argument( - "--output", - "-o", - default=None, - help="Base output directory for skill structure (overrides target default)", - ) - parser.add_argument("--skill-output", default=".", help="Output directory for .skill file") - parser.add_argument("--temp-dir", default="build", help="Temporary directory for processing") - - parser.add_argument("--skip-fetch", action="store_true", help="Skip the download step (use existing files in temp dir)") - parser.add_argument("--clean", action="store_true", help="Clean up temporary directory after completion") - - args = parser.parse_args() - - try: - # 1. Setup Directories - output_base = args.output - if output_base is None: - target_output_map = { - "claude": ".claude/skills", - "claude-desktop": ".claude/skills", - "cursor": ".cursor/skills", - "gemini": ".gemini/skills", - "codex": ".codex/skills", - } - output_base = target_output_map[args.target] - - temp_download_dir = os.path.join(args.temp_dir, "download") - temp_md_dir = os.path.join(args.temp_dir, "markdown") - - if not args.skip_fetch: - if os.path.exists(args.temp_dir): - shutil.rmtree(args.temp_dir) - os.makedirs(temp_download_dir) - - os.makedirs(temp_md_dir, exist_ok=True) - - # Timestamp for fetched_at - fetched_at = datetime.datetime.now(datetime.timezone.utc).isoformat() - - if not args.skip_fetch: - logger.info(f"=== Step 1: Fetching {args.url} ===") - fetch_site(args.url, temp_download_dir) - else: - logger.info(f"=== Step 1: Skipped Fetching (Using {temp_download_dir}) ===") - - # fetch_site creates a 'crawl' subdirectory - crawl_dir = os.path.join(temp_download_dir, "crawl") - - logger.info(f"=== Step 2: Converting HTML to Markdown ===") - # Find all HTML files in the crawl directory - html_files = glob.glob(os.path.join(crawl_dir, "**/*.html"), recursive=True) - logger.info(f"Found {len(html_files)} HTML files.") - - for html_file in html_files: - # Calculate source_url - # wget creates directory structure: crawl_dir/domain/path/to/file.html - # We need to reconstruct the URL. - - # Security check: Ensure html_file is strictly within crawl_dir - abs_html_file = os.path.abspath(html_file) - abs_crawl_dir = os.path.abspath(crawl_dir) - - if os.path.commonpath([abs_html_file, abs_crawl_dir]) != abs_crawl_dir: - logger.warning(f"Skipping potential path traversal file: {html_file}") - continue - - # Rel path from crawl_dir - rel_path = os.path.relpath(html_file, crawl_dir) - # rel_path is like "docs.pay.jp/v1/cardtoken.html" - # We assume https for now, or we could parse args.url to get scheme - parsed_input_url = urlparse(args.url) - scheme = parsed_input_url.scheme if parsed_input_url.scheme else "https" - - # Construct URL - # Note: This assumes wget preserved the domain directory. - # If wget was run with -nH (no host directories), this might be different. - # But fetch_site.py uses standard wget -r, which usually creates host dir. - # Remove .html extension from source_url (PAY.JP site doesn't use .html in URLs) - rel_path_for_url = rel_path[:-5] if rel_path.endswith('.html') else rel_path - source_url = f"{scheme}://{rel_path_for_url}" - - # Determine output filename (preserve directory structure) - # rel_path is like "docs.pay.jp/v1/cardtoken.html" or "docs.pay.jp/a/b/index.html" - # We want to preserve the structure and replace .html with .md - md_rel_path = html_to_md_path(rel_path) - - # Sanitize path components to avoid invalid characters in zip - md_rel_path = sanitize_path(md_rel_path) - md_path = os.path.join(temp_md_dir, md_rel_path) - - if os.path.exists(md_path): - logger.warning(f"Name collision for {md_rel_path}. Overwriting.") - - convert_html_to_md(html_file, md_path, source_url=source_url, fetched_at=fetched_at) - - logger.info(f"=== Step 3: Normalizing Markdown ===") - md_files = glob.glob(os.path.join(temp_md_dir, "**/*.md"), recursive=True) - for md_file in md_files: - # Normalize in place - normalize_markdown(md_file, md_file) - - logger.info(f"=== Step 4: Generating Skill Structure ===") - generate_skill_structure( - args.skill_name, - temp_md_dir, - output_base, - target_agent=args.target, - ) - - skill_dir = os.path.join(output_base, args.skill_name) - - logger.info(f"=== Step 5: Validating Skill ===") - if not validate_skill(skill_dir): - logger.error("Validation failed. Please check errors.") - # We don't exit here, we might still want to package or debug - - # Note: check_skill_size is now called inside validate_skill - - skill_file = None - if args.target == "claude-desktop": - logger.info(f"=== Step 6: Packaging Skill ===") - skill_file = package_skill(skill_dir, args.skill_output) - else: - logger.info("=== Step 6: Packaging Skill (skipped for non-claude-desktop targets) ===") - - logger.info(f"=== Done! ===") - logger.info(f"Skill directory: {skill_dir}") - if skill_file: - logger.info(f"Skill package: {skill_file}") - - # Cleanup - if args.clean: - shutil.rmtree(args.temp_dir) - logger.info(f"Temporary files removed from {args.temp_dir}") - else: - logger.info(f"Temporary files kept in {args.temp_dir}") - - except Exception as e: - logger.error(f"An unexpected error occurred: {e}") - exit(1) - -if __name__ == "__main__": - main() diff --git a/site2skill/normalize_markdown.py b/site2skill/normalize_markdown.py deleted file mode 100644 index 10c4ddb..0000000 --- a/site2skill/normalize_markdown.py +++ /dev/null @@ -1,81 +0,0 @@ -import re -import argparse -import os -import yaml -from urllib.parse import urljoin - -def extract_frontmatter(content): - """Extract YAML frontmatter from markdown content.""" - match = re.match(r'^---\n(.*?)\n---\n', content, re.DOTALL) - if match: - try: - return yaml.safe_load(match.group(1)) - except yaml.YAMLError: - return None - return None - -def normalize_links(md_content, source_url=None): - """ - Convert relative links to absolute URLs based on source_url. - Matches: [text](path) - """ - if not source_url: - return md_content - - # Regex to capture links - # \[([^\]]*)\]\(([^)]+)\) - pattern = re.compile(r'\[([^\]]*)\]\(([^)]+)\)') - - def callback(match): - text = match.group(1) - url = match.group(2) - - # Skip if already absolute - if url.startswith("http:") or url.startswith("https:") or url.startswith("mailto:"): - return match.group(0) - - # Skip anchors only - if url.startswith("#"): - return match.group(0) - - # Resolve absolute URL - # urljoin handles relative paths correctly - absolute_url = urljoin(source_url, url) - - return f"[{text}]({absolute_url})" - - return pattern.sub(callback, md_content) - -def normalize_markdown(input_path, output_path=None): - try: - with open(input_path, 'r', encoding='utf-8') as f: - content = f.read() - - # Extract source_url from frontmatter - frontmatter = extract_frontmatter(content) - source_url = frontmatter.get('source_url') if frontmatter else None - - if source_url: - normalized = normalize_links(content, source_url) - else: - print(f"Warning: No source_url found in {input_path}, skipping link normalization.") - normalized = content - - if output_path: - with open(output_path, 'w', encoding='utf-8') as f: - f.write(normalized) - print(f"Normalized: {input_path}") - else: - print(normalized) - - except Exception as e: - print(f"Error normalizing {input_path}: {e}") - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Normalize Markdown links to absolute URLs.") - parser.add_argument("input_file", help="Path to input Markdown file") - parser.add_argument("--output", "-o", help="Path to output Markdown file") - - args = parser.parse_args() - - normalize_markdown(args.input_file, args.output) diff --git a/site2skill/package_skill.py b/site2skill/package_skill.py deleted file mode 100644 index ad4686e..0000000 --- a/site2skill/package_skill.py +++ /dev/null @@ -1,45 +0,0 @@ -import shutil -import os -import argparse -import sys - -def package_skill(skill_dir, output_dir=None): - """ - Packages a skill directory into a .skill file (zip). - """ - if not os.path.isdir(skill_dir): - print(f"Error: Directory not found: {skill_dir}") - return False - - skill_name = os.path.basename(os.path.normpath(skill_dir)) - if output_dir is None: - output_dir = os.path.dirname(os.path.normpath(skill_dir)) - - output_filename = os.path.join(output_dir, f"{skill_name}") # shutil.make_archive adds extension - - print(f"Packaging {skill_dir} to {output_filename}.zip...") - - try: - # Create zip - archive_path = shutil.make_archive(output_filename, 'zip', skill_dir) - - # Rename .zip to .skill - final_path = output_filename + ".skill" - if os.path.exists(final_path): - os.remove(final_path) - - os.rename(archive_path, final_path) - print(f"Successfully created: {final_path}") - return final_path - except Exception as e: - print(f"Error packaging skill: {e}") - return None - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Package a Skill directory into a .skill file.") - parser.add_argument("skill_dir", help="Path to the skill directory") - parser.add_argument("--output", "-o", help="Output directory", default=".") - args = parser.parse_args() - - if not package_skill(args.skill_dir, args.output): - sys.exit(1) diff --git a/site2skill/templates/scripts_README.md b/site2skill/templates/scripts_README.md deleted file mode 100644 index 73ab26b..0000000 --- a/site2skill/templates/scripts_README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Skill Scripts - -This directory contains helper tools for working with this skill. - -## search_docs.py - -Full-text search across all documentation files (prefers references/, falls back to docs/). - -**Usage:** -```bash -# Use python or python3 -python search_docs.py "" [options] -``` - -**Options:** -- `--category {api,guides,reference}` - Filter by category -- `--max-results N` - Limit number of results (default: 10) -- `--json` - Output as JSON -- `--skill-dir PATH` - Specify skill directory (default: current) - -**Examples:** -```bash -# Basic search -python search_docs.py "subscription" - -# Search only API documentation -python search_docs.py --category api "charge" - -# Get top 5 results as JSON -python search_docs.py --max-results 5 --json "refund" -``` diff --git a/site2skill/url_filter.py b/site2skill/url_filter.py deleted file mode 100644 index 3b525e2..0000000 --- a/site2skill/url_filter.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -URL filtering for crawl scope control. - -Restricts crawl boundaries based on the starting URL and filters out -URLs with localization-only query parameters to prevent over-crawling -of duplicate content in different languages. -""" -from urllib.parse import urlparse, parse_qs - -DEFAULT_EXCLUDED_QUERY_KEYS = frozenset({"hl", "lang", "locale"}) - - -def is_url_allowed( - start_url: str, - candidate_url: str, - excluded_query_keys: set[str] | None = None, -) -> bool: - """ - Determine whether a candidate URL is within the allowed crawl scope. - - The scope is defined by two rules: - - 1. **Path scope** – the candidate must share the same scheme and host as - the starting URL and its path must be equal to or a descendant of the - starting URL's path. - 2. **Query-key filtering** – if *every* query parameter of the candidate - URL is in the exclusion list (e.g. localization parameters such as - ``hl``, ``lang``, ``locale``), the URL is rejected because it is - likely a duplicate of the same page in a different language. URLs - that carry at least one non-excluded query key are allowed. - - Args: - start_url: The URL that was originally given to the crawler. - candidate_url: The URL being evaluated for crawling. - excluded_query_keys: Query-parameter keys to treat as - localization-only. Defaults to - :data:`DEFAULT_EXCLUDED_QUERY_KEYS`. - - Returns: - ``True`` if the URL may be crawled, ``False`` otherwise. - """ - if excluded_query_keys is None: - excluded_query_keys = DEFAULT_EXCLUDED_QUERY_KEYS - - start = urlparse(start_url) - candidate = urlparse(candidate_url) - - # --- scheme & host must match ------------------------------------------ - if start.scheme != candidate.scheme: - return False - if start.netloc != candidate.netloc: - return False - - # --- path scope: candidate must be under the starting path ------------- - # Normalise so that "/a/b" is treated the same as "/a/b/" - start_path = start.path.rstrip("/") + "/" - candidate_path = candidate.path.rstrip("/") + "/" - - if not candidate_path.startswith(start_path): - # Also allow the exact starting path itself (without trailing slash) - if candidate.path.rstrip("/") != start.path.rstrip("/"): - return False - - # --- query-key filtering ----------------------------------------------- - query_params = parse_qs(candidate.query, keep_blank_values=True) - - if query_params: - # If every key belongs to the excluded set, reject the URL. - if all(key in excluded_query_keys for key in query_params): - return False - - return True diff --git a/site2skill/utils.py b/site2skill/utils.py deleted file mode 100644 index 0a1e743..0000000 --- a/site2skill/utils.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -Utility functions for site2skill. -""" -import os -import re - - -def sanitize_path(path: str) -> str: - """ - Sanitize a file path by replacing invalid characters with underscores. - - This function sanitizes each path component separately to avoid issues - with invalid characters in zip files or file systems. - - Args: - path: The file path to sanitize (can be relative or absolute) - - Returns: - The sanitized path with safe characters only. If all parts are empty - after sanitization, returns "file.md" as a safe default. - - Examples: - >>> sanitize_path("references.example.com/api/index.md") - 'references.example.com/api/index.md' - >>> sanitize_path("references@example.com/api#v1/index.md") - 'references_example.com/api_v1/index.md' - >>> sanitize_path("") - 'file.md' - """ - # Split path into components - path_parts = path.split(os.sep) - - # Sanitize each component - sanitized_parts = [] - for part in path_parts: - if part: # Skip empty parts - # Replace non-alphanumeric characters (except ._-) with _ - sanitized_part = re.sub(r'[^a-zA-Z0-9._-]', '_', part) - sanitized_parts.append(sanitized_part) - - # If all parts were sanitized away, use a default - if not sanitized_parts: - return "file.md" - - # Rejoin with path separator - return os.path.join(*sanitized_parts) - - -def html_to_md_path(html_path: str) -> str: - """ - Convert an HTML file path to a corresponding markdown file path. - - Args: - html_path: Path to HTML file (e.g., "references/page.html") - - Returns: - Path to markdown file (e.g., "references/page.md") - - Examples: - >>> html_to_md_path("references/index.html") - 'references/index.md' - >>> html_to_md_path("page.html") - 'page.md' - >>> html_to_md_path("references/page") - 'references/page.md' - """ - if html_path.endswith('.html'): - return html_path[:-5] + '.md' - else: - return html_path + '.md' diff --git a/site2skill/validate_skill.py b/site2skill/validate_skill.py deleted file mode 100644 index b419bcf..0000000 --- a/site2skill/validate_skill.py +++ /dev/null @@ -1,165 +0,0 @@ -import os -import re -import sys -import argparse -import logging -from typing import List, Tuple - -# Configure logging -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - - -def _get_docs_dir(skill_dir: str) -> str | None: - references_dir = os.path.join(skill_dir, "references") - if os.path.isdir(references_dir): - return references_dir - docs_dir = os.path.join(skill_dir, "docs") - if os.path.isdir(docs_dir): - return docs_dir - return None - - -def check_skill_size(skill_dir: str) -> None: - """ - Checks the total size of the skill directory (references/ preferred, fallback to docs/). - Warns if it exceeds 8MB. - Lists top 10 largest files. - """ - content_dir = _get_docs_dir(skill_dir) - if not content_dir: - return - - total_size = 0 - file_sizes: List[Tuple[int, str]] = [] - - for root, _, files in os.walk(content_dir): - for f in files: - fp = os.path.join(root, f) - try: - size = os.path.getsize(fp) - total_size += size - file_sizes.append((size, fp)) - except OSError: - pass - - # Sort by size descending - file_sizes.sort(key=lambda x: x[0], reverse=True) - - total_size_mb = total_size / (1024 * 1024) - logger.info("\n--- Skill Size Analysis ---") - logger.info(f"Total Uncompressed Size: {total_size_mb:.2f} MB") - - if total_size > 8 * 1024 * 1024: - logger.warning("Skill uncompressed size exceeds Claude's 8MB limit.") - logger.warning("The skill may fail to load in Claude.") - else: - logger.info("Size is within limits (< 8MB).") - - logger.info("\nTop 10 Largest Files:") - for size, fp in file_sizes[:10]: - rel_path = os.path.relpath(fp, skill_dir) - logger.info(f" {size / 1024:.1f} KB - {rel_path}") - logger.info("---------------------------\n") - - -def validate_skill(skill_dir: str) -> bool: - """ - Validates a skill directory structure and metadata. - Checks for SKILL.md + references/ structure (fallback to docs/). - """ - logger.info(f"Validating skill in: {skill_dir}") - - errors = [] - warnings = [] - - # 1. Check directory existence - if not os.path.isdir(skill_dir): - logger.error(f"Directory not found: {skill_dir}") - return False - - # 2. Check SKILL.md - skill_md_path = os.path.join(skill_dir, "SKILL.md") - if not os.path.exists(skill_md_path): - errors.append("SKILL.md not found.") - else: - logger.info("Found SKILL.md") - # Validate frontmatter - try: - with open(skill_md_path, 'r', encoding='utf-8') as f: - content = f.read() - # Check for YAML frontmatter - if content.startswith('---\n'): - frontmatter_match = re.match(r'^---\n(.*?)\n---', content, re.DOTALL) - if frontmatter_match: - frontmatter = frontmatter_match.group(1) - required_fields = ['name', 'description'] - for field in required_fields: - if f'{field}:' not in frontmatter: - warnings.append(f"SKILL.md frontmatter missing '{field}' field") - logger.info(" YAML frontmatter present") - else: - warnings.append("SKILL.md has incomplete frontmatter") - else: - warnings.append("SKILL.md missing YAML frontmatter") - except Exception as e: - warnings.append(f"Could not validate SKILL.md: {e}") - - # 3. Check references/ directory (fallback to docs/) - references_dir = os.path.join(skill_dir, "references") - docs_dir = os.path.join(skill_dir, "docs") - if os.path.isdir(references_dir): - logger.info("Found references/") - content_dir = references_dir - elif os.path.isdir(docs_dir): - warnings.append("references/ not found, using legacy docs/ directory") - logger.info("Found docs/ (legacy)") - content_dir = docs_dir - else: - errors.append("references/ directory not found (and no legacy docs/).") - content_dir = None - - if content_dir: - # Count markdown files - md_files = [] - for root, _, files in os.walk(content_dir): - for file in files: - if file.endswith('.md'): - md_files.append(os.path.join(root, file)) - - if len(md_files) == 0: - warnings.append(f"{os.path.basename(content_dir)}/ directory is empty (no .md files)") - else: - logger.info(f" {len(md_files)} markdown files") - - # 4. Check optional directories - scripts_dir = os.path.join(skill_dir, "scripts") - if os.path.isdir(scripts_dir): - logger.info("Found scripts/ (optional)") - - # 5. Check skill size - check_skill_size(skill_dir) - - # 6. Report results - if errors: - logger.error("VALIDATION FAILED:") - for error in errors: - logger.error(f" - {error}") - return False - - if warnings: - logger.warning("Warnings:") - for warning in warnings: - logger.warning(f" - {warning}") - - logger.info("Validation passed!") - return True - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Validate a Skill directory.") - parser.add_argument("skill_dir", help="Path to the skill directory") - args = parser.parse_args() - - if not validate_skill(args.skill_dir): - sys.exit(1) diff --git a/src/convert/html.rs b/src/convert/html.rs new file mode 100644 index 0000000..6d37b8c --- /dev/null +++ b/src/convert/html.rs @@ -0,0 +1,220 @@ +//! HTML parsing and content extraction + +use lazy_static::lazy_static; +use regex::Regex; +use scraper::{Html, Selector}; + +lazy_static! { + /// Regex patterns for paired tags (script, style, etc.) - matches open+content+close + static ref PAIRED_TAG_REGEXES: Vec = { + ["script", "style", "noscript", "iframe", "svg"] + .iter() + .map(|tag| Regex::new(&format!(r"(?si)<{tag}[^>]*>.*?")).unwrap()) + .collect() + }; + + /// Regex patterns for self-closing/void tags (meta, link) + static ref VOID_TAG_REGEXES: Vec = { + ["meta", "link"] + .iter() + .map(|tag| Regex::new(&format!(r"(?si)<{tag}[^>]*/?>")).unwrap()) + .collect() + }; + + /// Regex patterns for nav/sidebar/footer elements (tag with class/id + all content + closing tag) + static ref NAV_ELEMENT_REGEXES: Vec = vec![ + Regex::new(r"(?si)]*>.*?").unwrap(), + Regex::new(r"(?si)]*>.*?").unwrap(), + Regex::new(r"(?si)]*>.*?").unwrap(), + Regex::new(r"(?si)]*>.*?").unwrap(), + Regex::new(r#"(?si)]*class="[^"]*sidebar[^"]*"[^>]*>.*?"#).unwrap(), + Regex::new(r#"(?si)]*id="[^"]*sidebar[^"]*"[^>]*>.*?"#).unwrap(), + Regex::new(r#"(?si)]*class="[^"]*navigation[^"]*"[^>]*>.*?"#).unwrap(), + Regex::new(r#"(?si)]*id="[^"]*toc[^"]*"[^>]*>.*?"#).unwrap(), + Regex::new(r#"(?si)]*id="[^"]*footer[^"]*"[^>]*>.*?"#).unwrap(), + ]; + + static ref MAIN_SELECTOR: Selector = Selector::parse("main").unwrap(); + static ref ARTICLE_SELECTOR: Selector = Selector::parse("article").unwrap(); + static ref CONTENT_SELECTOR: Selector = Selector::parse(".content").unwrap(); + static ref BODY_SELECTOR: Selector = Selector::parse("body").unwrap(); + static ref TITLE_SELECTOR: Selector = Selector::parse("title").unwrap(); + static ref H1_SELECTOR: Selector = Selector::parse("h1").unwrap(); +} + +/// Extract the title from HTML +pub fn extract_title(html: &str) -> String { + let document = Html::parse_document(html); + + // Try tag first + if let Some(title_elem) = document.select(&TITLE_SELECTOR).next() { + let title = title_elem.text().collect::<String>().trim().to_string(); + if !title.is_empty() { + return title; + } + } + + // Try <h1> tag + if let Some(h1_elem) = document.select(&H1_SELECTOR).next() { + let title = h1_elem.text().collect::<String>().trim().to_string(); + if !title.is_empty() { + return title; + } + } + + "Untitled".to_string() +} + +/// Extract the main content from HTML +pub fn extract_content(html: &str) -> Option<String> { + let document = Html::parse_document(html); + + // Try to find the most relevant content container + // Order: <main> > <article> > .content > <body> + + if let Some(elem) = document.select(&MAIN_SELECTOR).next() { + return Some(elem.inner_html()); + } + + if let Some(elem) = document.select(&ARTICLE_SELECTOR).next() { + return Some(elem.inner_html()); + } + + if let Some(elem) = document.select(&CONTENT_SELECTOR).next() { + return Some(elem.inner_html()); + } + + if let Some(elem) = document.select(&BODY_SELECTOR).next() { + return Some(elem.inner_html()); + } + + Some(document.html()) +} + +/// Clean HTML by removing unwanted tags and noise +/// +/// Removes: script, style, noscript, iframe, svg, meta, link tags, +/// and navigation elements (nav, header, footer, aside, sidebar divs). +pub fn clean_html(html: &str) -> String { + let mut result = html.to_string(); + + // Remove paired tags (script, style, etc.) + for re in PAIRED_TAG_REGEXES.iter() { + result = re.replace_all(&result, "").to_string(); + } + + // Remove void/self-closing tags (meta, link) + for re in VOID_TAG_REGEXES.iter() { + result = re.replace_all(&result, "").to_string(); + } + + // Remove navigation/sidebar/footer elements + for re in NAV_ELEMENT_REGEXES.iter() { + result = re.replace_all(&result, "").to_string(); + } + + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_title() { + let html = r#"<!DOCTYPE html><html><head><title>Test Page

    Other

    "#; + assert_eq!(extract_title(html), "Test Page"); + } + + #[test] + fn test_extract_title_from_h1() { + let html = r#"

    Page Title

    "#; + assert_eq!(extract_title(html), "Page Title"); + } + + #[test] + fn test_extract_title_untitled() { + let html = r#"

    No title

    "#; + assert_eq!(extract_title(html), "Untitled"); + } + + #[test] + fn test_clean_html() { + let html = r#"

    Hello

    "#; + let cleaned = clean_html(html); + assert!(!cleaned.contains("

    Content

    "; + let cleaned = clean_html(html); + assert!(!cleaned.contains("Content

    ")); + } + + #[test] + fn test_clean_html_removes_style() { + let html = "

    Visible

    "; + let cleaned = clean_html(html); + assert!(!cleaned.contains("

    Main content

    "#; + let cleaned = clean_html(html); + assert!(!cleaned.contains("