Read files in place for metadata, and decode only image headers

metadata/image/image.go

@@ -69,17 +69,18 @@ func (m *imageMetadata) Any() any {
 func init() {
 	// Add metadata handler for image files in general
 	metadata.AddHandler(regexp.MustCompile("^image/.*$"), "image", func(_ context.Context, r io.Reader, _ *metadata.Opts) ([]gomedia.Metadata, error) {
-		// Decode the image
-		img, format, err := image.Decode(r)
+		// Decode the image's format and size, from its header, rather than
+		// decoding the image, which can be large
+		config, format, err := image.DecodeConfig(r)
 		if err != nil {
 			return nil, err
 		}
 
 		// Return metadata
 		return []gomedia.Metadata{
 			types.Ptr(imageMetadata{"image:format", format}),
-			types.Ptr(imageMetadata{"image:width", img.Bounds().Dx()}),
-			types.Ptr(imageMetadata{"image:height", img.Bounds().Dy()}),
+			types.Ptr(imageMetadata{"image:width", config.Width}),
+			types.Ptr(imageMetadata{"image:height", config.Height}),
 		}, nil
 	}, "image")
 }

metadata/metadata.go

@@ -36,11 +36,11 @@ type entry struct {
 	handler    HandlerFunc
 }
 
-// namedReader wraps a *bytes.Reader with a Name(), so a handler can recover
-// the original input's filename (via gomedia.NamedReader) after GetMetadata
-// has buffered it into a fresh reader for each handler.
+// namedReader wraps a handler's reader of the input with a Name(), so a
+// handler can recover the original input's filename (via
+// gomedia.NamedReader) from the fresh reader GetMetadata gives each handler.
 type namedReader struct {
-	*bytes.Reader
+	*io.SectionReader
 	name string
 }
 
@@ -141,22 +141,21 @@ func GetMetadata(parent context.Context, r io.Reader, contentType string, opts .
 		return nil, nil
 	}
 
-	// Tee the reader into a buffer once, so every handler can read the
-	// data independently and concurrently. If r carries a name (see
-	// gomedia.NamedReader), preserve it on each handler's copy, since a
-	// handler may need the filename itself (e.g. to look up TMDB metadata)
+	// Give every handler its own reader of the data, so they can read it
+	// independently and concurrently, and seek in it. If r carries a name
+	// (see gomedia.NamedReader), preserve it on each handler's reader, since
+	// a handler may need the filename itself (e.g. to look up TMDB metadata)
 	// rather than the file's contents.
-	var buf bytes.Buffer
-	if _, err := io.Copy(&buf, r); err != nil {
+	section, err := sectionReader(r)
+	if err != nil {
 		return nil, err
 	}
 	source, _ := r.(gomedia.NamedReader)
 	newReader := func() io.Reader {
-		br := bytes.NewReader(buf.Bytes())
 		if source == nil {
-			return br
+			return section()
 		}
-		return namedReader{br, source.Name()}
+		return namedReader{section(), source.Name()}
 	}
 
 	var (
@@ -256,3 +255,49 @@ func containsFold(namespaces []string, s string) bool {
 	}
 	return false
 }
+
+// sectionReader returns a function which returns a new reader of the rest of
+// r, from its current position. When r can be read at an offset and seeked,
+// as a file can, each reader reads r itself, so a large file isn't read into
+// memory, and only the parts a handler needs are read. Otherwise r is read
+// into memory once, and each reader reads that.
+func sectionReader(r io.Reader) (func() *io.SectionReader, error) {
+	if ra, ok := r.(interface {
+		io.ReaderAt
+		io.Seeker
+	}); ok {
+		if start, size, err := remaining(ra); err == nil {
+			return func() *io.SectionReader {
+				return io.NewSectionReader(ra, start, size)
+			}, nil
+		}
+	}
+
+	// Read r into memory, when it can't be read at an offset, or seeked,
+	// such as a pipe
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	br := bytes.NewReader(data)
+	return func() *io.SectionReader {
+		return io.NewSectionReader(br, 0, int64(len(data)))
+	}, nil
+}
+
+// remaining returns the current position of r, and the number of bytes from
+// there to its end, leaving it at its current position
+func remaining(r io.Seeker) (int64, int64, error) {
+	start, err := r.Seek(0, io.SeekCurrent)
+	if err != nil {
+		return 0, 0, err
+	}
+	end, err := r.Seek(0, io.SeekEnd)
+	if err != nil {
+		return 0, 0, err
+	}
+	if _, err := r.Seek(start, io.SeekStart); err != nil {
+		return 0, 0, err
+	}
+	return start, end - start, nil
+}

metadata/metadata_test.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"image"
 	"io"
+	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"sync"
@@ -300,3 +302,90 @@ func Test_metadata_013(t *testing.T) {
 		t.Fatal("expected GetMetadata to wait for the handler to finish despite mid-flight cancellation")
 	}
 }
+
+// A file is read by each handler where it is, from its current position,
+// rather than being read into memory, and each handler can seek in it
+func Test_metadata_014(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "file.txt")
+	if err := os.WriteFile(path, []byte("skip:data"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	if _, err := f.Seek(5, io.SeekStart); err != nil {
+		t.Fatal(err)
+	}
+
+	var mu sync.Mutex
+	var got []string
+	handler := func(_ context.Context, r io.Reader, _ *Opts) ([]gomedia.Metadata, error) {
+		seeker, ok := r.(io.ReadSeeker)
+		if !ok {
+			return nil, errors.New("expected an io.ReadSeeker")
+		}
+		if _, err := seeker.Seek(1, io.SeekStart); err != nil {
+			return nil, err
+		}
+		data, err := io.ReadAll(seeker)
+		if err != nil {
+			return nil, err
+		}
+		name := ""
+		if named, ok := r.(gomedia.NamedReader); ok {
+			name = named.Name()
+		}
+		mu.Lock()
+		defer mu.Unlock()
+		got = append(got, string(data)+"@"+name)
+		return nil, nil
+	}
+	AddHandler(regexp.MustCompile("^x-test/014$"), "a", handler, "a")
+	AddHandler(regexp.MustCompile("^x-test/014$"), "b", handler, "b")
+
+	if _, err := GetMetadata(context.Background(), f, "x-test/014"); err != nil {
+		t.Fatal(err)
+	}
+	for _, g := range got {
+		if g != "ata@"+path {
+			t.Errorf("expected each handler to read the file from its position, got %q", g)
+		}
+	}
+	if len(got) != 2 {
+		t.Fatalf("expected two handlers to run, got %d", len(got))
+	}
+
+	// The file isn't read, so it's where it was
+	if pos, err := f.Seek(0, io.SeekCurrent); err != nil {
+		t.Fatal(err)
+	} else if pos != 5 {
+		t.Fatalf("expected the file to be at 5, got %d", pos)
+	}
+}
+
+// A reader which can't be read at an offset is read into memory, and each
+// handler can still seek in it
+func Test_metadata_015(t *testing.T) {
+	var got string
+	AddHandler(regexp.MustCompile("^x-test/015$"), "x", func(_ context.Context, r io.Reader, _ *Opts) ([]gomedia.Metadata, error) {
+		seeker, ok := r.(io.ReadSeeker)
+		if !ok {
+			return nil, errors.New("expected an io.ReadSeeker")
+		}
+		if _, err := seeker.Seek(2, io.SeekStart); err != nil {
+			return nil, err
+		}
+		data, err := io.ReadAll(seeker)
+		got = string(data)
+		return nil, err
+	}, "x")
+
+	if _, err := GetMetadata(context.Background(), io.MultiReader(strings.NewReader("da"), strings.NewReader("ta")), "x-test/015"); err != nil {
+		t.Fatal(err)
+	}
+	if got != "ta" {
+		t.Fatalf("expected the handler to read the rest of the data, got %q", got)
+	}
+}