From 8ac69cafda97c51bf5907e3a42d11c0f399d6737 Mon Sep 17 00:00:00 2001 From: MyCode <230429220+MyCode83@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:23:52 +0100 Subject: [PATCH 1/6] test: reproduce panic on incomplete urlscan sort --- pkg/source/urlscan/urlscan_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/source/urlscan/urlscan_test.go b/pkg/source/urlscan/urlscan_test.go index 693dd0e..1d7d1ae 100644 --- a/pkg/source/urlscan/urlscan_test.go +++ b/pkg/source/urlscan/urlscan_test.go @@ -90,7 +90,7 @@ func TestRunValidResponse(t *testing.T) { "page": { "url": "https://blog.example.com/test" }, - "sort": [123, "abc"] + "sort": [123] } ], "has_more": false From 955e977437bae405e8e103f9e6830506d1208edc Mon Sep 17 00:00:00 2001 From: MyCode <230429220+MyCode83@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:07:48 +0100 Subject: [PATCH 2/6] fix: safely handle urlscan pagination sort values --- pkg/source/urlscan/urlscan.go | 61 +++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/pkg/source/urlscan/urlscan.go b/pkg/source/urlscan/urlscan.go index fdc780f..2bf5f90 100644 --- a/pkg/source/urlscan/urlscan.go +++ b/pkg/source/urlscan/urlscan.go @@ -5,7 +5,6 @@ import ( "fmt" "net/http" neturl "net/url" - "strconv" "time" jsoniter "github.com/json-iterator/go" @@ -63,6 +62,28 @@ func (s *Source) buildSearchURL(rootURL, searchAfter string) (string, error) { return parsedURL.String(), nil } +func buildSearchAfter(result Result) (string, error) { + if len(result.Sort) < 2 { + return "", fmt.Errorf("invalid urlscan sort: expected at least 2 values, got %d", len(result.Sort)) + } + + firstValue, ok := result.Sort[0].(float64) + if !ok { + return "", fmt.Errorf("invalid urlscan sort: first value must be a number") + } + + secondValue, ok := result.Sort[1].(string) + if !ok { + return "", fmt.Errorf("invalid urlscan sort: second value must be a string") + } + + if secondValue == "" { + return "", fmt.Errorf("invalid urlscan sort: second value must not be empty") + } + + return fmt.Sprintf("%d,%s", int(firstValue), secondValue), nil +} + func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session) <-chan source.Result { results := make(chan source.Result) s.errors = 0 @@ -91,6 +112,7 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session) if err != nil { results <- source.Result{ Source: s.Name(), + Type: source.Error, Error: err, } s.errors++ @@ -116,7 +138,11 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session) _ = resp.Body.Close() if resp.StatusCode == http.StatusTooManyRequests { - results <- source.Result{Source: s.Name(), Error: fmt.Errorf("urlscan rate limited")} + results <- source.Result{ + Source: s.Name(), + Error: fmt.Errorf("urlscan rate limited"), + Type: source.Error, + } s.errors++ return } @@ -127,16 +153,33 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session) s.results++ } } - if len(data.Results) > 0 { - lastResult := data.Results[len(data.Results)-1] - if len(lastResult.Sort) > 0 { - sort1 := strconv.Itoa(int(lastResult.Sort[0].(float64))) - sort2, _ := lastResult.Sort[1].(string) + hasMore := data.HasMore + if !hasMore { + break + } + + if len(data.Results) == 0 { + results <- source.Result{ + Source: s.Name(), + Type: source.Error, + Error: fmt.Errorf("urlscan returned has_more without results"), + } + s.errors++ + return + } - searchAfter = fmt.Sprintf("%s,%s", sort1, sort2) + lastResult := data.Results[len(data.Results)-1] + + searchAfter, err = buildSearchAfter(lastResult) + if err != nil { + results <- source.Result{ + Source: s.Name(), + Type: source.Error, + Error: err, } + s.errors++ + return } - hasMore = data.HasMore } }() From 4ea6404bc2f5a3403c5f5c980240f1cedd3cbf3e Mon Sep 17 00:00:00 2001 From: MyCode <230429220+MyCode83@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:18:46 +0100 Subject: [PATCH 3/6] test: cover urlscan search-after validation --- pkg/source/urlscan/urlscan_test.go | 66 ++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/pkg/source/urlscan/urlscan_test.go b/pkg/source/urlscan/urlscan_test.go index 1d7d1ae..701ce7c 100644 --- a/pkg/source/urlscan/urlscan_test.go +++ b/pkg/source/urlscan/urlscan_test.go @@ -264,3 +264,69 @@ func TestRunPaginatesWithoutAccumulatingSearchAfter(t *testing.T) { } } } + +func TestBuildSearchAfter(t *testing.T) { + tests := []struct { + name string + sort []interface{} + expected string + shouldErr bool + }{ + { + name: "valid values", + sort: []interface{}{float64(123), "abc"}, + expected: "123,abc", + }, + { + name: "zero numeric value", + sort: []interface{}{float64(0), "abc"}, + expected: "0,abc", + }, + { + name: "large numeric value", + sort: []interface{}{float64(2147483647), "cursor"}, + expected: "2147483647,cursor", + }, + { + name: "missing second value", + sort: []interface{}{float64(123)}, + shouldErr: true, + }, + { + name: "invalid first value type", + sort: []interface{}{"123", "abc"}, + shouldErr: true, + }, + { + name: "invalid second value type", + sort: []interface{}{float64(123), 456}, + shouldErr: true, + }, + { + name: "empty second value", + sort: []interface{}{float64(123), ""}, + shouldErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + value, err := buildSearchAfter(Result{Sort: test.sort}) + + if test.shouldErr { + if err == nil { + t.Fatalf("expected an error, got nil") + } + return + } + + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + + if value != test.expected { + t.Fatalf("expected %q, got %q", test.expected, value) + } + }) + } +} \ No newline at end of file From c2829200eff2f8b2583b2f3cdb71ad90c0ba8615 Mon Sep 17 00:00:00 2001 From: MyCode <230429220+MyCode83@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:29:22 +0100 Subject: [PATCH 4/6] fix: add missing source.Error type to alienvault, commoncrawl and urlscan results --- pkg/source/alienvault/alienvault.go | 4 ++-- pkg/source/commoncrawl/commoncrawl.go | 8 ++++---- pkg/source/urlscan/urlscan.go | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/source/alienvault/alienvault.go b/pkg/source/alienvault/alienvault.go index c166425..eaf03b4 100644 --- a/pkg/source/alienvault/alienvault.go +++ b/pkg/source/alienvault/alienvault.go @@ -46,7 +46,7 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session) apiURL := fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/url_list?page=%d", rootUrl, page) resp, err := sess.SimpleGet(ctx, apiURL) if err != nil && resp == nil { - results <- source.Result{Source: s.Name(), Error: err} + results <- source.Result{Source: s.Name(), Error: err, Type: source.Error} sess.DiscardHTTPResponse(resp) return } @@ -55,7 +55,7 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session) // Get the response body and decode err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { - results <- source.Result{Source: s.Name(), Error: err} + results <- source.Result{Source: s.Name(), Error: err, Type: source.Error} s.errors++ _ = resp.Body.Close() return diff --git a/pkg/source/commoncrawl/commoncrawl.go b/pkg/source/commoncrawl/commoncrawl.go index 6069375..3028403 100644 --- a/pkg/source/commoncrawl/commoncrawl.go +++ b/pkg/source/commoncrawl/commoncrawl.go @@ -45,7 +45,7 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session) resp, err := sess.SimpleGet(ctx, indexURL) if err != nil { - results <- source.Result{Source: s.Name(), Error: err} + results <- source.Result{Source: s.Name(), Error: err, Type: source.Error} s.errors++ sess.DiscardHTTPResponse(resp) return @@ -54,7 +54,7 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session) var indexes []indexResponse err = jsoniter.NewDecoder(resp.Body).Decode(&indexes) if err != nil { - results <- source.Result{Source: s.Name(), Error: err} + results <- source.Result{Source: s.Name(), Error: err, Type: source.Error} s.errors++ _ = resp.Body.Close() return @@ -122,7 +122,7 @@ func (s *Source) getURLs(ctx context.Context, searchURL, rootURL string, sess *s var headers = map[string]string{"Host": "index.commoncrawl.org"} u, err := url.Parse(searchURL) if err != nil { - results <- source.Result{Source: s.Name(), Error: err} + results <- source.Result{Source: s.Name(), Error: err, Type: source.Error} s.errors++ return false } @@ -134,7 +134,7 @@ func (s *Source) getURLs(ctx context.Context, searchURL, rootURL string, sess *s currentSearchURL := u.String() resp, err := sess.Get(ctx, currentSearchURL, "", headers) if err != nil { - results <- source.Result{Source: s.Name(), Error: err} + results <- source.Result{Source: s.Name(), Error: err, Type: source.Error} s.errors++ sess.DiscardHTTPResponse(resp) return false diff --git a/pkg/source/urlscan/urlscan.go b/pkg/source/urlscan/urlscan.go index 2bf5f90..c230ff1 100644 --- a/pkg/source/urlscan/urlscan.go +++ b/pkg/source/urlscan/urlscan.go @@ -121,7 +121,7 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session) resp, err := sess.Get(ctx, apiURL, "", headers) if err != nil { - results <- source.Result{Source: s.Name(), Error: err} + results <- source.Result{Source: s.Name(), Error: err, Type: source.Error} s.errors++ sess.DiscardHTTPResponse(resp) return @@ -130,7 +130,7 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session) var data response err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { - results <- source.Result{Source: s.Name(), Error: err} + results <- source.Result{Source: s.Name(), Error: err, Type: source.Error} s.errors++ _ = resp.Body.Close() return From 094764b84e9009a2ebd6953932bc6ae54449ed95 Mon Sep 17 00:00:00 2001 From: MyCode <230429220+MyCode83@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:34:57 +0100 Subject: [PATCH 5/6] style: apply gofmt --- pkg/source/urlscan/urlscan_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/source/urlscan/urlscan_test.go b/pkg/source/urlscan/urlscan_test.go index 701ce7c..10598ea 100644 --- a/pkg/source/urlscan/urlscan_test.go +++ b/pkg/source/urlscan/urlscan_test.go @@ -329,4 +329,4 @@ func TestBuildSearchAfter(t *testing.T) { } }) } -} \ No newline at end of file +} From 89175b07b395ffddf18827042c4bf1d0d34365dd Mon Sep 17 00:00:00 2001 From: MyCode <230429220+MyCode83@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:49:15 +0100 Subject: [PATCH 6/6] fix: reject invalid urlscan numeric cursor values --- pkg/source/urlscan/urlscan.go | 9 ++++++++- pkg/source/urlscan/urlscan_test.go | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/source/urlscan/urlscan.go b/pkg/source/urlscan/urlscan.go index c230ff1..7c4224e 100644 --- a/pkg/source/urlscan/urlscan.go +++ b/pkg/source/urlscan/urlscan.go @@ -3,8 +3,10 @@ package urlscan import ( "context" "fmt" + "math" "net/http" neturl "net/url" + "strconv" "time" jsoniter "github.com/json-iterator/go" @@ -72,6 +74,10 @@ func buildSearchAfter(result Result) (string, error) { return "", fmt.Errorf("invalid urlscan sort: first value must be a number") } + if math.IsNaN(firstValue) || math.IsInf(firstValue, 0) || firstValue != math.Trunc(firstValue) { + return "", fmt.Errorf("invalid urlscan sort: first value must be a finite integer") + } + secondValue, ok := result.Sort[1].(string) if !ok { return "", fmt.Errorf("invalid urlscan sort: second value must be a string") @@ -81,7 +87,8 @@ func buildSearchAfter(result Result) (string, error) { return "", fmt.Errorf("invalid urlscan sort: second value must not be empty") } - return fmt.Sprintf("%d,%s", int(firstValue), secondValue), nil + formattedFirstValue := strconv.FormatFloat(firstValue, 'f', -1, 64) + return fmt.Sprintf("%s,%s", formattedFirstValue, secondValue), nil } func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session) <-chan source.Result { diff --git a/pkg/source/urlscan/urlscan_test.go b/pkg/source/urlscan/urlscan_test.go index 10598ea..ef37ea6 100644 --- a/pkg/source/urlscan/urlscan_test.go +++ b/pkg/source/urlscan/urlscan_test.go @@ -307,6 +307,16 @@ func TestBuildSearchAfter(t *testing.T) { sort: []interface{}{float64(123), ""}, shouldErr: true, }, + { + name: "fractional first value", + sort: []interface{}{123.5, "abc"}, + shouldErr: true, + }, + { + name: "non-finite first value", + sort: []interface{}{math.Inf(1), "abc"}, + shouldErr: true, + }, } for _, test := range tests {