diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3638b1f --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +main.go +gocdx +.vscode/ \ No newline at end of file diff --git a/generate.go b/generate.go new file mode 100644 index 0000000..f15f96c --- /dev/null +++ b/generate.go @@ -0,0 +1,280 @@ +package gocdx + +import ( + "bufio" + "context" + "crypto/sha1" + "encoding/base32" + "io" + "runtime" + "strconv" + "strings" + "time" + + "github.com/internetarchive/gocdx/pkg/surt" + warc "github.com/internetarchive/gowarc" + "golang.org/x/sync/errgroup" +) + +type overloadedWARCRecord struct { + *warc.Record + + compByteOffset int64 + compByteLength int64 + warcFileName string +} + +// contains helper +func hasField(fields []string, want string) bool { + for _, f := range fields { + if f == want { + return true + } + } + return false +} + +// Generate reads a WARC file from the provided reader and returns a slice of Record +// generated from the given WARC response records. Reading from the WARC reader is +// strictly single-threaded; record processing is concurrent. +func Generate(warcFile io.Reader, header string) ([]*Record, error) { + warcReader, err := warc.NewReader(warcFile) + if err != nil { + return nil, err + } + defer warcReader.Close() + + // Which fields are requested? + headerFields := strings.Fields(header) + needStatus := hasField(headerFields, "s") // requires parsing HTTP status line + needDigest := hasField(headerFields, "k") // may require reading full record content + + // ---- Stage 1: single-threaded collection (the only critical section) ---- + var ( + warcFileName string + warcRecords []*overloadedWARCRecord // only "response" records are collected + ) + + for { + rec, err := warcReader.ReadRecord() + if err == io.EOF { + break + } + if err != nil { + // Best effort: close any content we already collected + for _, r := range warcRecords { + if r.Record != nil && r.Record.Content != nil { + _ = r.Record.Content.Close() + } + } + return nil, err + } + + switch rec.Header.Get("WARC-Type") { + case "warcinfo": + warcFileName = rec.Header.Get("WARC-Filename") + case "response": + // Keep Content open; workers will parse and then close it later. + warcRecords = append(warcRecords, &overloadedWARCRecord{ + Record: rec, + compByteOffset: rec.Offset, + compByteLength: rec.Size, + warcFileName: warcFileName, + }) + default: + } + } + + // Nothing to do? + if len(warcRecords) == 0 { + return []*Record{}, nil + } + + // ---- Stage 2: concurrent processing of collected response records ---- + records := make([]*Record, len(warcRecords)) + + workers := runtime.GOMAXPROCS(0) // reasonable default; tune if needed + g, _ := errgroup.WithContext(context.Background()) + + // Work distribution: each worker pulls indexes from a channel + type job struct{ idx int } + jobs := make(chan job) + + // Start workers + for w := 0; w < workers; w++ { + g.Go(func() error { + for j := range jobs { + wr := warcRecords[j.idx] + rec := &Record{} + + // Build the record according to requested fields only. + for _, field := range headerFields { + switch field { + case "N": + rec.MassagedURL = surt.Massage(wr.Record.Header.Get("WARC-Target-URI")) + + case "b": + parsedTime, err := time.Parse(time.RFC3339, wr.Record.Header.Get("WARC-Date")) + if err != nil { + parsedTime = time.Time{} + } + rec.Timestamp = parsedTime + + case "a": + rec.OriginalURL = wr.Record.Header.Get("WARC-Target-URI") + + case "m": + rec.MIMEType = strings.TrimSuffix( + wr.Record.Header.Get("Content-Type"), + "; msgtype=response", + ) + + case "s": + // Only parse HTTP status if requested + rec.StatusCode = -1 + if needStatus { + httpMessage, _ := parseHTTPHeadersFromWARCRecord(wr.Record) + parts := strings.Split(httpMessage, " ") + if len(parts) >= 2 { + if sc, err := strconv.Atoi(parts[1]); err == nil { + rec.StatusCode = sc + } + } + // Reset content for any subsequent reads (e.g., digest) + if rs, ok := wr.Record.Content.(io.ReadSeeker); ok { + _, _ = rs.Seek(0, io.SeekStart) + } + } + + case "k": + // Prefer the digest from header if present; otherwise compute it. + trimmed := strings.TrimPrefix(wr.Record.Header.Get("WARC-Block-Digest"), "sha1:") + if trimmed != wr.Record.Header.Get("WARC-Block-Digest") { + rec.NewStyleChecksum = trimmed + } else if needDigest { + hasher := sha1.New() + // Ensure we start from the beginning (status parsing may have read some bytes) + if rs, ok := wr.Record.Content.(io.ReadSeeker); ok { + _, _ = rs.Seek(0, io.SeekStart) + } + if _, err := io.Copy(hasher, wr.Record.Content); err != nil { + // Close before returning error + _ = wr.Record.Content.Close() + return err + } + rec.NewStyleChecksum = base32.StdEncoding.EncodeToString(hasher.Sum(nil)) + // Reset again is not necessary since we'll close Content below + } + + case "r": + // TODO: clarify; keep placeholder + rec.Redirect = "-" + + case "M": + // TODO: ignore for now + rec.MetaTags = "-" + + case "S": + rec.CompressedRecordSize = wr.compByteLength + + case "V": + rec.CompressedArcOffset = wr.compByteOffset + + case "g": + rec.Filename = wr.warcFileName + } + } + + // Release resources for this record + if wr.Record != nil && wr.Record.Content != nil { + _ = wr.Record.Content.Close() + } + + records[j.idx] = rec + } + return nil + }) + } + + // Feed jobs + go func() { + for i := range warcRecords { + jobs <- job{idx: i} + } + close(jobs) + }() + + // Wait for all workers + if err := g.Wait(); err != nil { + return nil, err + } + + return records, nil +} + +// FormatCDX formats a Record into a CDX string based on the header format. +func (r *Record) FormatCDX(header string) (string, error) { + var result strings.Builder + headerFields := strings.FieldsSeq(header) + + for field := range headerFields { + switch field { + case "N": + result.WriteString(r.MassagedURL) + case "b": + result.WriteString(r.Timestamp.Format("20060102150405")) + case "a": + result.WriteString(r.OriginalURL) + case "m": + result.WriteString(r.MIMEType) + case "s": + result.WriteString(strconv.Itoa(r.StatusCode)) + case "k": + result.WriteString(r.NewStyleChecksum) + case "r": + result.WriteString(r.Redirect) + case "M": + result.WriteString(r.MetaTags) + case "S": + result.WriteString(strconv.FormatInt(r.CompressedRecordSize, 10)) + case "V": + result.WriteString(strconv.FormatInt(r.CompressedArcOffset, 10)) + case "g": + result.WriteString(r.Filename) + case "CDX": + continue + } + result.WriteString(" ") + } + + return strings.TrimSpace(result.String()), nil +} + +func parseHTTPHeadersFromWARCRecord(warcRecord *warc.Record) (message string, headers map[string]string) { + headers = make(map[string]string) + + var i int + scanner := bufio.NewScanner(warcRecord.Content) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + break + } + + // handle HTTP message + if i == 0 { + message = strings.Clone(line) + i++ + continue + } + + parts := strings.SplitN(line, ": ", 2) + if len(parts) == 2 { + headers[strings.ToLower(parts[0])] = strings.ToLower(parts[1]) + } + + i++ + } + + return +} diff --git a/go.mod b/go.mod index adff516..d1bbd92 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,31 @@ module github.com/internetarchive/gocdx -go 1.23.1 +go 1.24.2 + +replace github.com/internetarchive/gowarc => ../gowarc + +require ( + github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 + github.com/grafana/pyroscope-go v1.2.4 + github.com/internetarchive/gowarc v0.8.85 + golang.org/x/sync v0.13.0 +) + +require ( + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/cloudflare/circl v1.6.1 // indirect + github.com/dolthub/maphash v0.1.0 // indirect + github.com/gammazero/deque v1.0.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/maypok86/otter v1.2.4 // indirect + github.com/miekg/dns v1.1.65 // indirect + github.com/refraction-networking/utls v1.6.7 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/mod v0.24.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/tools v0.32.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9deb553 --- /dev/null +++ b/go.sum @@ -0,0 +1,62 @@ +github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 h1:ClzzXMDDuUbWfNNZqGeYq4PnYOlwlOVIvSyNaIy0ykg= +github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3/go.mod h1:we0YA5CsBbH5+/NUzC/AlMmxaDtWlXeNsqrwXjTzmzA= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ= +github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4= +github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= +github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grafana/pyroscope-go v1.2.4 h1:B22GMXz+O0nWLatxLuaP7o7L9dvP0clLvIpmeEQQM0Q= +github.com/grafana/pyroscope-go v1.2.4/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/maypok86/otter v1.2.4 h1:HhW1Pq6VdJkmWwcZZq19BlEQkHtI8xgsQzBVXJU0nfc= +github.com/maypok86/otter v1.2.4/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdHOWG4R4= +github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc= +github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= +github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/things-go/go-socks5 v0.0.6 h1:YjylIYZiND41szH4NzsVbx8aVDsS/Y8ps3QYPwQvqnI= +github.com/things-go/go-socks5 v0.0.6/go.mod h1:RF6tRutwNWzISbPfiDEChH/o1aDfRv+cXDYn2a2qkK4= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= +golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/model.go b/model.go index a3933e1..719ff45 100644 --- a/model.go +++ b/model.go @@ -2,60 +2,58 @@ package gocdx import "time" -// Record represents a single record in a CDX file +// Record represents a single record in a CDX file. +// Default fields set by the Internet Archive are (CDX N b a m s k r M S V g) as per IIPC specifications. type Record struct { - // Default fields (CDX N b a m s k r M S V g) - MassagedURL string `json:"massaged_url"` // N - Timestamp time.Time `json:"timestamp"` // b - OriginalURL string `json:"original_url"` // a - MIMEType string `json:"mime_type"` // m - StatusCode int `json:"status_code"` // s - NewStyleChecksum string `json:"new_style_checksum"` // k - Redirect string `json:"redirect"` // r - MetaTags string `json:"meta_tags"` // M - CompressedRecordSize int64 `json:"compressed_record_size"` // S - CompressedArcOffset int64 `json:"compressed_arc_offset"` // V - Filename string `json:"filename"` // g - - // Optional fields - CanonizedURL string `json:"canonized_url,omitempty"` - NewsGroup string `json:"news_group,omitempty"` - RulespaceCategory string `json:"rulespace_category,omitempty"` - CompressedDatOffset int64 `json:"compressed_dat_offset,omitempty"` - CanonizedFrame string `json:"canonized_frame,omitempty"` - LanguageDescription string `json:"language_description,omitempty"` - CanonizedHost string `json:"canonized_host,omitempty"` - CanonizedImage string `json:"canonized_image,omitempty"` - CanonizedJumpPoint string `json:"canonized_jump_point,omitempty"` - FBISChangedThing string `json:"fbis_changed_thing,omitempty"` - CanonizedLink string `json:"canonized_link,omitempty"` - CanonizedPath string `json:"canonized_path,omitempty"` - LanguageString string `json:"language_string,omitempty"` - CanonizedRedirect string `json:"canonized_redirect,omitempty"` - Uniqueness string `json:"uniqueness,omitempty"` - CanonizedURLOtherHref string `json:"canonized_url_other_href,omitempty"` - CanonizedURLOtherSrc string `json:"canonized_url_other_src,omitempty"` - CanonizedURLScript string `json:"canonized_url_script,omitempty"` - OldStyleChecksum string `json:"old_style_checksum,omitempty"` - UncompressedDatOffset int64 `json:"uncompressed_dat_offset,omitempty"` - IP string `json:"ip,omitempty"` - Frame string `json:"frame,omitempty"` - OriginalHost string `json:"original_host,omitempty"` - Image string `json:"image,omitempty"` - OriginalJumpPoint string `json:"original_jump_point,omitempty"` - Link string `json:"link,omitempty"` - ArcDocumentLength int64 `json:"arc_document_length,omitempty"` - Port int `json:"port,omitempty"` - OriginalPath string `json:"original_path,omitempty"` - Title string `json:"title,omitempty"` - UncompressedArcOffset int64 `json:"uncompressed_arc_offset,omitempty"` - URLOtherHref string `json:"url_other_href,omitempty"` - URLOtherSrc string `json:"url_other_src,omitempty"` - URLScript string `json:"url_script,omitempty"` + CanonizedURL string `json:"canonized_url,omitempty" cdx:"A"` // A + NewsGroup string `json:"news_group,omitempty" cdx:"B"` // B + RulespaceCategory string `json:"rulespace_category,omitempty" cdx:"C"` // C + CompressedDatOffset int64 `json:"compressed_dat_offset,omitempty" cdx:"D"` // D + CanonizedFrame string `json:"canonized_frame,omitempty" cdx:"F"` // F + MultiColumnLanguageDescription string `json:"language_description,omitempty" cdx:"G"` // G + CanonizedHost string `json:"canonized_host,omitempty" cdx:"H"` // H + CanonizedImage string `json:"canonized_image,omitempty" cdx:"I"` // I + CanonizedJumpPoint string `json:"canonized_jump_point,omitempty" cdx:"J"` // J + FBISChangedThing string `json:"fbis_changed_thing,omitempty" cdx:"K"` // K + CanonizedLink string `json:"canonized_link,omitempty" cdx:"L"` // L + MetaTags string `json:"meta_tags" cdx:"M"` // M + MassagedURL string `json:"massaged_url" cdx:"N"` // N + CanonizedPath string `json:"canonized_path,omitempty" cdx:"P"` // P + LanguageString string `json:"language_string,omitempty" cdx:"Q"` // Q + CanonizedRedirect string `json:"canonized_redirect,omitempty" cdx:"R"` // R + CompressedRecordSize int64 `json:"compressed_record_size" cdx:"S"` // S + Uniqueness string `json:"uniqueness,omitempty" cdx:"U"` // U + CompressedArcOffset int64 `json:"compressed_arc_offset" cdx:"V"` // V + CanonizedURLOtherHref string `json:"canonized_url_other_href,omitempty" cdx:"X"` // X + CanonizedURLOtherSrc string `json:"canonized_url_other_src,omitempty" cdx:"Y"` // Y + CanonizedURLScript string `json:"canonized_url_script,omitempty" cdx:"Z"` // Z + OriginalURL string `json:"original_url" cdx:"a"` // a + Timestamp time.Time `json:"timestamp" cdx:"b"` // b + OldStyleChecksum string `json:"old_style_checksum,omitempty" cdx:"c"` // c + UncompressedDatOffset int64 `json:"uncompressed_dat_offset,omitempty" cdx:"d"` // d + IP string `json:"ip,omitempty" cdx:"e"` // e + Frame string `json:"frame,omitempty" cdx:"f"` // f + Filename string `json:"filename" cdx:"g"` // g + OriginalHost string `json:"original_host,omitempty" cdx:"h"` // h + Image string `json:"image,omitempty" cdx:"i"` // i + OriginalJumpPoint string `json:"original_jump_point,omitempty" cdx:"j"` // j + NewStyleChecksum string `json:"new_style_checksum" cdx:"k"` // k + Link string `json:"link,omitempty" cdx:"l"` // l + MIMEType string `json:"mime_type" cdx:"m"` // m + ArcDocumentLength int64 `json:"arc_document_length,omitempty" cdx:"n"` // n + Port int `json:"port,omitempty" cdx:"o"` // o + OriginalPath string `json:"original_path,omitempty" cdx:"p"` // p + Redirect string `json:"redirect" cdx:"r"` // r + StatusCode int `json:"status_code" cdx:"s"` // s + Title string `json:"title,omitempty" cdx:"t"` // t + UncompressedArcOffset int64 `json:"uncompressed_arc_offset,omitempty" cdx:"v"` // v + URLOtherHref string `json:"url_other_href,omitempty" cdx:"x"` // x + URLOtherSrc string `json:"url_other_src,omitempty" cdx:"y"` // y + URLScript string `json:"url_script,omitempty" cdx:"z"` // z } // FieldIndex represents the indices of fields in the CDX file type FieldIndex map[byte]int -// DefaultFields represents the default CDX fields in order +// DefaultFields represents the defaultorder var DefaultFields = []byte{'N', 'b', 'a', 'm', 's', 'k', 'r', 'M', 'S', 'V', 'g'} diff --git a/parse.go b/parse.go index 146671e..32452d1 100644 --- a/parse.go +++ b/parse.go @@ -10,6 +10,8 @@ import ( "time" ) +// Parse reads a CDX file from the provided reader and returns a slice of Record. +// It expects the first line to be the header, which can be provided as an argument. func Parse(r io.Reader, header string) ([]Record, error) { const maxScanTokenSize = 1024 * 1024 // 1MB diff --git a/pkg/surt/cannonicalize.go b/pkg/surt/cannonicalize.go new file mode 100644 index 0000000..47b208c --- /dev/null +++ b/pkg/surt/cannonicalize.go @@ -0,0 +1,65 @@ +package surt + +import ( + "net" + "net/url" + "strings" + + "github.com/ImVexed/fasturl" +) + +// IACanonicalize returns the canonicalized URL string per the IA-style rules defined by surt tests. +func IACanonicalize(raw string) (string, error) { + parsed, err := fasturl.ParseURL(raw) + if err != nil { + return "", err + } + + // Non-HTTP(S) schemes are left untouched (e.g., "dns:..."). + scheme := strings.ToLower(parsed.Protocol) + if scheme != "http" && scheme != "https" { + // Return the input as-is + return raw, nil + } + + newURL := url.URL{} + + newURL.Scheme = scheme + + // Normalize host: lowercase & drop leading "www.". + host := strings.ToLower(parsed.Host) + port := parsed.Port + + // Strip leading "www." if present + host = strings.TrimPrefix(host, "www.") + + // Remove default ports (http->80, https->443); keep non-defaults. + defaultPort := map[string]string{ + "http": "80", + "https": "443", + }[scheme] + + if port == defaultPort { + port = "" + } + + if port != "" { + newURL.Host = net.JoinHostPort(host, port) + } else { + newURL.Host = host + } + + // Path: keep "/" at root, otherwise drop a single trailing slash. + path := parsed.Path + if path == "" { + path = "/" + } + if path != "/" && strings.HasSuffix(path, "/") { + path = strings.TrimSuffix(path, "/") + } + newURL.Path = path + + // Queries/fragments aren’t covered by the provided tests, so leave as-is. + + return newURL.String(), nil +} diff --git a/pkg/surt/cannonicalize_test.go b/pkg/surt/cannonicalize_test.go new file mode 100644 index 0000000..82edf29 --- /dev/null +++ b/pkg/surt/cannonicalize_test.go @@ -0,0 +1,28 @@ +package surt + +import "testing" + +func TestIAURLCanonicalizer(t *testing.T) { + tests := []struct { + in string + want string + }{ + {"http://ARCHIVE.ORG/", "http://archive.org/"}, + {"http://www.archive.org:80/", "http://archive.org/"}, + {"https://www.archive.org:80/", "https://archive.org:80/"}, + {"http://www.archive.org:443/", "http://archive.org:443/"}, + {"https://www.archive.org:443/", "https://archive.org/"}, + {"http://www.archive.org/big/", "http://archive.org/big"}, + {"dns:www.archive.org", "dns:www.archive.org"}, + } + + for _, tc := range tests { + got, err := IACanonicalize(tc.in) + if err != nil { + t.Fatalf("Canonicalize(%q) returned unexpected error: %v", tc.in, err) + } + if got != tc.want { + t.Errorf("Canonicalize(%q) = %q; want %q", tc.in, got, tc.want) + } + } +} diff --git a/pkg/surt/massage.go b/pkg/surt/massage.go new file mode 100644 index 0000000..63d0e80 --- /dev/null +++ b/pkg/surt/massage.go @@ -0,0 +1,275 @@ +package surt + +import ( + "sort" + "strings" + + "github.com/ImVexed/fasturl" +) + +func Massage(url string, opts ...MassageOpts) (massagedURL string) { + pu, err := fasturl.ParseURL(url) + if err != nil { + return "-" + } + + var ( + massageHost = true + trailingComma = false + withScheme = false + ) + + for _, opt := range opts { + switch opt { + case NoHostMassage: + massageHost = false + case WithTrailingComma: + trailingComma = true + case WithScheme: + withScheme = true + } + } + + if isOpaquePassThrough(pu.Protocol) { + return url + } + + if isNonHierarchical(pu.Protocol) { + return url + } + + host := strings.ToLower(pu.Host) + if massageHost { + host, _ = strings.CutPrefix(host, "www.") + } + + labels := hostToSURTLabels(host) + if len(labels) == 0 { + // No host → return "-" (defensive; not used in tests) + return "-" + } + + hostPart := strings.Join(reverse(labels), ",") + if trailingComma { + hostPart += "," + } + + // Path normalization: keep "/" for root; drop trailing "/" otherwise. + path := pu.Path + if path == "" { + path = "/" + } + if path != "/" && strings.HasSuffix(path, "/") { + path = strings.TrimSuffix(path, "/") + } + + // Query normalization (sort, decode, lowercase values, drop PHPSESSID, keep leading empty "&" if present) + rawQuery := pu.Query + qLeadingNameless := strings.HasPrefix(rawQuery, "?&") || strings.Contains(rawQuery, "?&") && strings.Index(rawQuery, "?&") == strings.Index(rawQuery, "?") + qString := normalizeQuery(rawQuery) + + // Build SURT + var b strings.Builder + if withScheme { + // with scheme uses parentheses around host + b.WriteString(pu.Protocol) + b.WriteString("://(") + b.WriteString(hostPart) + b.WriteString(")") + } else { + // no scheme: no leading "(", but keep trailing ")" + b.WriteString(hostPart) + b.WriteString(")") + } + b.WriteString(path) + + // Preserve a leading empty parameter if it was there (Yahoo bug case). + if qLeadingNameless { + if qString == "" { + b.WriteString("?&") + } else { + b.WriteString("?&") + b.WriteString(qString) + } + } else if qString != "" { + b.WriteString("?") + b.WriteString(qString) + } + + return b.String() +} + +func isOpaquePassThrough(s string) bool { + // Return exactly for these textual forms in tests. + // They include: filedesc:..., warcinfo:..., dns:..., mailto:... + l := strings.ToLower(s) + return strings.HasPrefix(l, "filedesc:") || + strings.HasPrefix(l, "warcinfo:") || + strings.HasPrefix(l, "dns:") || + strings.HasPrefix(l, "mailto:") +} + +func isNonHierarchical(scheme string) bool { + switch scheme { + case "mailto", "dns", "warcinfo", "filedesc": + return true + default: + return false + } +} + +func hostToSURTLabels(hostport string) []string { + // Drop port if present; SURT uses host labels only for these tests. + h := hostport + if i := strings.LastIndexByte(h, ':'); i >= 0 { + // crude split; acceptable for tests + h = h[:i] + } + // IPv6 literals or empty → leave as-is (tests don't cover IPv6). + if strings.HasPrefix(h, "[") { + return []string{strings.Trim(h, "[]")} + } + // Split by dot + parts := strings.Split(h, ".") + // Filter empty (defensive) + out := parts[:0] + for _, p := range parts { + if p != "" { + out = append(out, p) + } + } + return out +} + +func reverse[T any](a []T) []T { + n := len(a) + out := make([]T, n) + for i := 0; i < n; i++ { + out[i] = a[n-1-i] + } + return out +} + +// -------- Query normalization ---------- + +type kv struct { + k string + v string +} + +func normalizeQuery(raw string) string { + if raw == "" { + return "" + } + // Split on '&' preserving empty items (for sorting we’ll ignore the first nameless; + // the caller handles preserving a leading & if present). + items := strings.Split(raw, "&") + + var pairs []kv + for _, it := range items { + if it == "" { + // nameless empty; caller handles presence via qLeadingNameless + continue + } + key, val, hasEq := strings.Cut(it, "=") + if dropQueryParam(key) { + continue + } + + // Percent-decode key and value, but don't turn '+' into space (use our own decode). + key = decodePercentNoPlus(key) + val = decodePercentNoPlus(val) + + // Lowercase values (matches Yahoo UA and others); keys remain as-is (tests already lowercase keys). + if hasEq { + val = strings.ToLower(val) + } + + // Re-encode spaces as %20; leave other chars as-is (tests show parentheses, ';', ':' decoded). + key = strings.ReplaceAll(key, " ", "%20") + val = strings.ReplaceAll(val, " ", "%20") + + if hasEq { + pairs = append(pairs, kv{k: key, v: val}) + } else { + // Bare key like "a" → keep v empty + pairs = append(pairs, kv{k: key, v: ""}) + } + } + + if len(pairs) == 0 { + return "" + } + + // Sort by key, then value + sort.SliceStable(pairs, func(i, j int) bool { + if pairs[i].k == pairs[j].k { + return pairs[i].v < pairs[j].v + } + return pairs[i].k < pairs[j].k + }) + + // Recompose + var b strings.Builder + for idx, p := range pairs { + if idx > 0 { + b.WriteByte('&') + } + b.WriteString(p.k) + if p.v != "" || strings.Contains(p.k, "=") { + // if original was "a=" we can't detect here; tests don't need that nuance. + if p.v != "" { + b.WriteByte('=') + b.WriteString(p.v) + } + } + } + return b.String() +} + +func dropQueryParam(key string) bool { + // Drop PHPSESSID and similar session keys. + // Tests show "PHPSESSID", "JSESSIONID", "SID", "sessionid" (case-insensitive). + switch strings.ToLower(key) { + case "phpsessid", "jsessionid", "sid", "sessionid": + return true + default: + return false + } +} + +// decodePercentNoPlus decodes %XX but leaves '+' unchanged. +func decodePercentNoPlus(s string) string { + // fast path: if no '%', return input + if !strings.Contains(s, "%") { + return s + } + // Manually decode to avoid '+'=>space conversion. + var b strings.Builder + for i := 0; i < len(s); i++ { + if s[i] == '%' && i+2 < len(s) { + hi := fromHex(s[i+1]) + lo := fromHex(s[i+2]) + if hi >= 0 && lo >= 0 { + b.WriteByte(byte(hi<<4 | lo)) + i += 2 + continue + } + } + b.WriteByte(s[i]) + } + return b.String() +} + +func fromHex(c byte) int8 { + switch { + case '0' <= c && c <= '9': + return int8(c - '0') + case 'a' <= c && c <= 'f': + return int8(c - 'a' + 10) + case 'A' <= c && c <= 'F': + return int8(c - 'A' + 10) + default: + return -1 + } +} diff --git a/pkg/surt/massage_test.go b/pkg/surt/massage_test.go new file mode 100644 index 0000000..ea7c377 --- /dev/null +++ b/pkg/surt/massage_test.go @@ -0,0 +1,71 @@ +package surt + +import "testing" + +func TestSURT(t *testing.T) { + tests := []struct { + in string + opts []MassageOpts + want string + }{ + {in: "", want: "-"}, + {in: "filedesc:foo.arc.gz", want: "filedesc:foo.arc.gz"}, + {in: "filedesc:/foo.arc.gz", want: "filedesc:/foo.arc.gz"}, + {in: "filedesc://foo.arc.gz", want: "filedesc://foo.arc.gz"}, + {in: "warcinfo:foo.warc.gz", want: "warcinfo:foo.warc.gz"}, + {in: "dns:alexa.com", want: "dns:alexa.com"}, + {in: "dns:archive.org", want: "dns:archive.org"}, + + {in: "http://www.archive.org/", want: "org,archive)/"}, + {in: "http://archive.org/", want: "org,archive)/"}, + {in: "http://archive.org/goo/", want: "org,archive)/goo"}, + {in: "http://archive.org/goo/?", want: "org,archive)/goo"}, + {in: "http://archive.org/goo/?b&a", want: "org,archive)/goo?a&b"}, + {in: "http://archive.org/goo/?a=2&b&a=1", want: "org,archive)/goo?a=1&a=2&b"}, + + // trailing comma mode + {in: "http://archive.org/goo/?a=2&b&a=1", opts: []MassageOpts{WithTrailingComma}, want: "org,archive,)/goo?a=1&a=2&b"}, + {in: "dns:archive.org", opts: []MassageOpts{WithTrailingComma}, want: "dns:archive.org"}, + {in: "warcinfo:foo.warc.gz", opts: []MassageOpts{WithTrailingComma}, want: "warcinfo:foo.warc.gz"}, + + // PHP session id: + {in: "http://archive.org/index.php?PHPSESSID=0123456789abcdefghijklemopqrstuv&action=profile;u=4221", want: "org,archive)/index.php?action=profile;u=4221"}, + + // WHOIS url: + {in: "whois://whois.isoc.org.il/shaveh.co.il", want: "il,org,isoc,whois)/shaveh.co.il"}, + + // Simple customization + {in: "http://www.example.com/", opts: []MassageOpts{WithScheme}, want: "http://(com,example)/"}, + {in: "http://www.example.com/", opts: []MassageOpts{}, want: "com,example)/"}, + {in: "http://www.example.com/", opts: []MassageOpts{WithScheme, WithTrailingComma}, want: "http://(com,example,)/"}, + {in: "https://www.example.com/", opts: []MassageOpts{WithScheme, WithTrailingComma}, want: "https://(com,example,)/"}, + {in: "ftp://www.example.com/", opts: []MassageOpts{WithTrailingComma}, want: "com,example,)/"}, + {in: "ftp://www.example.com/", opts: []MassageOpts{}, want: "com,example)/"}, + {in: "ftp://www.example.com/", opts: []MassageOpts{WithScheme, WithTrailingComma}, want: "ftp://(com,example,)/"}, + {in: "http://www.example.com/", opts: []MassageOpts{WithScheme, NoHostMassage}, want: "http://(com,example,www)/"}, + {in: "http://www.example.com/", opts: []MassageOpts{NoHostMassage}, want: "com,example,www)/"}, + {in: "http://www.example.com/", opts: []MassageOpts{WithScheme, WithTrailingComma, NoHostMassage}, want: "http://(com,example,www,)/"}, + {in: "https://www.example.com/", opts: []MassageOpts{WithScheme, WithTrailingComma, NoHostMassage}, want: "https://(com,example,www,)/"}, + {in: "ftp://www.example.com/", opts: []MassageOpts{WithScheme, WithTrailingComma, NoHostMassage}, want: "ftp://(com,example,www,)/"}, + + {in: "mailto:foo@example.com", opts: []MassageOpts{WithScheme}, want: "mailto:foo@example.com"}, + {in: "mailto:foo@example.com", opts: []MassageOpts{WithTrailingComma}, want: "mailto:foo@example.com"}, + {in: "mailto:foo@example.com", opts: []MassageOpts{WithScheme, WithTrailingComma}, want: "mailto:foo@example.com"}, + {in: "dns:archive.org", opts: []MassageOpts{WithScheme}, want: "dns:archive.org"}, + {in: "dns:archive.org", opts: []MassageOpts{WithTrailingComma}, want: "dns:archive.org"}, + {in: "dns:archive.org", opts: []MassageOpts{WithScheme, WithTrailingComma}, want: "dns:archive.org"}, + {in: "whois://whois.isoc.org.il/shaveh.co.il", opts: []MassageOpts{WithScheme}, want: "whois://(il,org,isoc,whois)/shaveh.co.il"}, + {in: "whois://whois.isoc.org.il/shaveh.co.il", opts: []MassageOpts{WithTrailingComma}, want: "il,org,isoc,whois,)/shaveh.co.il"}, + {in: "whois://whois.isoc.org.il/shaveh.co.il", opts: []MassageOpts{WithTrailingComma, WithScheme}, want: "whois://(il,org,isoc,whois,)/shaveh.co.il"}, + {in: "warcinfo:foo.warc.gz", opts: []MassageOpts{WithTrailingComma}, want: "warcinfo:foo.warc.gz"}, + {in: "warcinfo:foo.warc.gz", opts: []MassageOpts{WithScheme}, want: "warcinfo:foo.warc.gz"}, + {in: "warcinfo:foo.warc.gz", opts: []MassageOpts{WithScheme, WithTrailingComma}, want: "warcinfo:foo.warc.gz"}, + } + + for _, tc := range tests { + got := Massage(tc.in, tc.opts...) + if got != tc.want { + t.Errorf("surt(%q, %+v) = %q, want %q", tc.in, tc.opts, got, tc.want) + } + } +} diff --git a/pkg/surt/opts.go b/pkg/surt/opts.go new file mode 100644 index 0000000..34953f7 --- /dev/null +++ b/pkg/surt/opts.go @@ -0,0 +1,14 @@ +package surt + +type MassageOpts int + +const ( + // WithScheme indicates that the massaged URL should include the scheme. e.g.: "http://example.com" -> "http://(com,example)/" + WithScheme MassageOpts = iota + // WithTrailingComma indicates that the massaged URL should end with a comma. e.g.: "http://example.com" -> "http://(com,example,)/" + WithTrailingComma + //WithIACanonicalization indicates that the massaged URL should be canonicalized according to IA rules. + WithIACanonicalization + //NoHostMassage indicates that the host shouldn't be massaged (e.g., stripping "www."). + NoHostMassage +) diff --git a/pkg/surt/surt.go b/pkg/surt/surt.go new file mode 100644 index 0000000..01a9e3b --- /dev/null +++ b/pkg/surt/surt.go @@ -0,0 +1,4 @@ +// Package surt is a best-effort reimplementation of the Sort-friendly URI Reordering Transform (SURT) python package. +// +// This essentially skips the more complex aspects of the original implementation in favor of a simpler, more idiomatic Go approach. +package surt