fix: handle urlscan pagination and source errors safely

pkg/source/alienvault/alienvault.go

@@ -46,7 +46,7 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session)
 			apiURL := fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/url_list?page=%d", rootUrl, page)
 			resp, err := sess.SimpleGet(ctx, apiURL)
 			if err != nil && resp == nil {
-				results <- source.Result{Source: s.Name(), Error: err}
+				results <- source.Result{Source: s.Name(), Error: err, Type: source.Error}
 				sess.DiscardHTTPResponse(resp)
 				return
 			}
@@ -55,7 +55,7 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session)
 			// Get the response body and decode
 			err = json.NewDecoder(resp.Body).Decode(&response)
 			if err != nil {
-				results <- source.Result{Source: s.Name(), Error: err}
+				results <- source.Result{Source: s.Name(), Error: err, Type: source.Error}
 				s.errors++
 				_ = resp.Body.Close()
 				return

pkg/source/commoncrawl/commoncrawl.go

@@ -45,7 +45,7 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session)
 
 		resp, err := sess.SimpleGet(ctx, indexURL)
 		if err != nil {
-			results <- source.Result{Source: s.Name(), Error: err}
+			results <- source.Result{Source: s.Name(), Error: err, Type: source.Error}
 			s.errors++
 			sess.DiscardHTTPResponse(resp)
 			return
@@ -54,7 +54,7 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session)
 		var indexes []indexResponse
 		err = jsoniter.NewDecoder(resp.Body).Decode(&indexes)
 		if err != nil {
-			results <- source.Result{Source: s.Name(), Error: err}
+			results <- source.Result{Source: s.Name(), Error: err, Type: source.Error}
 			s.errors++
 			_ = resp.Body.Close()
 			return
@@ -122,7 +122,7 @@ func (s *Source) getURLs(ctx context.Context, searchURL, rootURL string, sess *s
 			var headers = map[string]string{"Host": "index.commoncrawl.org"}
 			u, err := url.Parse(searchURL)
 			if err != nil {
-				results <- source.Result{Source: s.Name(), Error: err}
+				results <- source.Result{Source: s.Name(), Error: err, Type: source.Error}
 				s.errors++
 				return false
 			}
@@ -134,7 +134,7 @@ func (s *Source) getURLs(ctx context.Context, searchURL, rootURL string, sess *s
 			currentSearchURL := u.String()
 			resp, err := sess.Get(ctx, currentSearchURL, "", headers)
 			if err != nil {
-				results <- source.Result{Source: s.Name(), Error: err}
+				results <- source.Result{Source: s.Name(), Error: err, Type: source.Error}
 				s.errors++
 				sess.DiscardHTTPResponse(resp)
 				return false

pkg/source/urlscan/urlscan.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"net/http"
 	neturl "net/url"
-	"strconv"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
@@ -63,6 +62,28 @@ func (s *Source) buildSearchURL(rootURL, searchAfter string) (string, error) {
 	return parsedURL.String(), nil
 }
 
+func buildSearchAfter(result Result) (string, error) {
+	if len(result.Sort) < 2 {
+		return "", fmt.Errorf("invalid urlscan sort: expected at least 2 values, got %d", len(result.Sort))
+	}
+
+	firstValue, ok := result.Sort[0].(float64)
+	if !ok {
+		return "", fmt.Errorf("invalid urlscan sort: first value must be a number")
+	}
+
+	secondValue, ok := result.Sort[1].(string)
+	if !ok {
+		return "", fmt.Errorf("invalid urlscan sort: second value must be a string")
+	}
+
+	if secondValue == "" {
+		return "", fmt.Errorf("invalid urlscan sort: second value must not be empty")
+	}
+
+	return fmt.Sprintf("%d,%s", int(firstValue), secondValue), nil
+}
+
 func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session) <-chan source.Result {
 	results := make(chan source.Result)
 	s.errors = 0
@@ -91,6 +112,7 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session)
 			if err != nil {
 				results <- source.Result{
 					Source: s.Name(),
+					Type:   source.Error,
 					Error:  err,
 				}
 				s.errors++
@@ -99,7 +121,7 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session)
 
 			resp, err := sess.Get(ctx, apiURL, "", headers)
 			if err != nil {
-				results <- source.Result{Source: s.Name(), Error: err}
+				results <- source.Result{Source: s.Name(), Error: err, Type: source.Error}
 				s.errors++
 				sess.DiscardHTTPResponse(resp)
 				return
@@ -108,15 +130,19 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session)
 			var data response
 			err = jsoniter.NewDecoder(resp.Body).Decode(&data)
 			if err != nil {
-				results <- source.Result{Source: s.Name(), Error: err}
+				results <- source.Result{Source: s.Name(), Error: err, Type: source.Error}
 				s.errors++
 				_ = resp.Body.Close()
 				return
 			}
 			_ = resp.Body.Close()
 
 			if resp.StatusCode == http.StatusTooManyRequests {
-				results <- source.Result{Source: s.Name(), Error: fmt.Errorf("urlscan rate limited")}
+				results <- source.Result{
+					Source: s.Name(),
+					Error:  fmt.Errorf("urlscan rate limited"),
+					Type:   source.Error,
+				}
 				s.errors++
 				return
 			}
@@ -127,16 +153,33 @@ func (s *Source) Run(ctx context.Context, rootUrl string, sess *session.Session)
 					s.results++
 				}
 			}
-			if len(data.Results) > 0 {
-				lastResult := data.Results[len(data.Results)-1]
-				if len(lastResult.Sort) > 0 {
-					sort1 := strconv.Itoa(int(lastResult.Sort[0].(float64)))
-					sort2, _ := lastResult.Sort[1].(string)
+			hasMore := data.HasMore
+			if !hasMore {
+				break
+			}
+
+			if len(data.Results) == 0 {
+				results <- source.Result{
+					Source: s.Name(),
+					Type:   source.Error,
+					Error:  fmt.Errorf("urlscan returned has_more without results"),
+				}
+				s.errors++
+				return
+			}
 
-					searchAfter = fmt.Sprintf("%s,%s", sort1, sort2)
+			lastResult := data.Results[len(data.Results)-1]
+
+			searchAfter, err = buildSearchAfter(lastResult)
+			if err != nil {
+				results <- source.Result{
+					Source: s.Name(),
+					Type:   source.Error,
+					Error:  err,
 				}
+				s.errors++
+				return
 			}
-			hasMore = data.HasMore
 		}
 	}()
 

pkg/source/urlscan/urlscan_test.go

@@ -90,7 +90,7 @@ func TestRunValidResponse(t *testing.T) {
 					"page": {
 						"url": "https://blog.example.com/test"
 					},
-					"sort": [123, "abc"]
+					"sort": [123]
 				}
 			],
 			"has_more": false
@@ -264,3 +264,69 @@ func TestRunPaginatesWithoutAccumulatingSearchAfter(t *testing.T) {
 		}
 	}
 }
+
+func TestBuildSearchAfter(t *testing.T) {
+	tests := []struct {
+		name      string
+		sort      []interface{}
+		expected  string
+		shouldErr bool
+	}{
+		{
+			name:     "valid values",
+			sort:     []interface{}{float64(123), "abc"},
+			expected: "123,abc",
+		},
+		{
+			name:     "zero numeric value",
+			sort:     []interface{}{float64(0), "abc"},
+			expected: "0,abc",
+		},
+		{
+			name:     "large numeric value",
+			sort:     []interface{}{float64(2147483647), "cursor"},
+			expected: "2147483647,cursor",
+		},
+		{
+			name:      "missing second value",
+			sort:      []interface{}{float64(123)},
+			shouldErr: true,
+		},
+		{
+			name:      "invalid first value type",
+			sort:      []interface{}{"123", "abc"},
+			shouldErr: true,
+		},
+		{
+			name:      "invalid second value type",
+			sort:      []interface{}{float64(123), 456},
+			shouldErr: true,
+		},
+		{
+			name:      "empty second value",
+			sort:      []interface{}{float64(123), ""},
+			shouldErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			value, err := buildSearchAfter(Result{Sort: test.sort})
+
+			if test.shouldErr {
+				if err == nil {
+					t.Fatalf("expected an error, got nil")
+				}
+				return
+			}
+
+			if err != nil {
+				t.Fatalf("expected nil error, got %v", err)
+			}
+
+			if value != test.expected {
+				t.Fatalf("expected %q, got %q", test.expected, value)
+			}
+		})
+	}
+}