From c5c52d9cdfe7d5c513a42d180cb2cdb2b57bf156 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Thu, 15 Oct 2026 15:02:26 +0200 Subject: [PATCH] Read files in place for metadata, and decode only image headers GetMetadata read every input into memory before running its handlers, so indexing a large video read all of it into memory, once for each file being indexed. An input which can be read at an offset and seeked, such as a file, is now read in place: each handler has its own section reader of it, so it reads only what it needs, and the input's position is unchanged. Other inputs are read into memory once, as before, and each handler can seek in them too. The image handler decodes the image's format and size from its header, rather than decoding the whole image. Co-Authored-By: Claude Opus 5.5 --- metadata/image/image.go | 9 ++-- metadata/metadata.go | 71 +++++++++++++++++++++++++------ metadata/metadata_test.go | 89 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 17 deletions(-) diff --git a/metadata/image/image.go b/metadata/image/image.go index 386bb6a..be566da 100644 --- a/metadata/image/image.go +++ b/metadata/image/image.go @@ -69,8 +69,9 @@ func (m *imageMetadata) Any() any { func init() { // Add metadata handler for image files in general metadata.AddHandler(regexp.MustCompile("^image/.*$"), "image", func(_ context.Context, r io.Reader, _ *metadata.Opts) ([]gomedia.Metadata, error) { - // Decode the image - img, format, err := image.Decode(r) + // Decode the image's format and size, from its header, rather than + // decoding the image, which can be large + config, format, err := image.DecodeConfig(r) if err != nil { return nil, err } @@ -78,8 +79,8 @@ func init() { // Return metadata return []gomedia.Metadata{ types.Ptr(imageMetadata{"image:format", format}), - types.Ptr(imageMetadata{"image:width", img.Bounds().Dx()}), - types.Ptr(imageMetadata{"image:height", img.Bounds().Dy()}), + types.Ptr(imageMetadata{"image:width", config.Width}), + types.Ptr(imageMetadata{"image:height", config.Height}), }, nil }, "image") } diff --git a/metadata/metadata.go b/metadata/metadata.go index 69912b7..35a8a66 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -36,11 +36,11 @@ type entry struct { handler HandlerFunc } -// namedReader wraps a *bytes.Reader with a Name(), so a handler can recover -// the original input's filename (via gomedia.NamedReader) after GetMetadata -// has buffered it into a fresh reader for each handler. +// namedReader wraps a handler's reader of the input with a Name(), so a +// handler can recover the original input's filename (via +// gomedia.NamedReader) from the fresh reader GetMetadata gives each handler. type namedReader struct { - *bytes.Reader + *io.SectionReader name string } @@ -141,22 +141,21 @@ func GetMetadata(parent context.Context, r io.Reader, contentType string, opts . return nil, nil } - // Tee the reader into a buffer once, so every handler can read the - // data independently and concurrently. If r carries a name (see - // gomedia.NamedReader), preserve it on each handler's copy, since a - // handler may need the filename itself (e.g. to look up TMDB metadata) + // Give every handler its own reader of the data, so they can read it + // independently and concurrently, and seek in it. If r carries a name + // (see gomedia.NamedReader), preserve it on each handler's reader, since + // a handler may need the filename itself (e.g. to look up TMDB metadata) // rather than the file's contents. - var buf bytes.Buffer - if _, err := io.Copy(&buf, r); err != nil { + section, err := sectionReader(r) + if err != nil { return nil, err } source, _ := r.(gomedia.NamedReader) newReader := func() io.Reader { - br := bytes.NewReader(buf.Bytes()) if source == nil { - return br + return section() } - return namedReader{br, source.Name()} + return namedReader{section(), source.Name()} } var ( @@ -256,3 +255,49 @@ func containsFold(namespaces []string, s string) bool { } return false } + +// sectionReader returns a function which returns a new reader of the rest of +// r, from its current position. When r can be read at an offset and seeked, +// as a file can, each reader reads r itself, so a large file isn't read into +// memory, and only the parts a handler needs are read. Otherwise r is read +// into memory once, and each reader reads that. +func sectionReader(r io.Reader) (func() *io.SectionReader, error) { + if ra, ok := r.(interface { + io.ReaderAt + io.Seeker + }); ok { + if start, size, err := remaining(ra); err == nil { + return func() *io.SectionReader { + return io.NewSectionReader(ra, start, size) + }, nil + } + } + + // Read r into memory, when it can't be read at an offset, or seeked, + // such as a pipe + data, err := io.ReadAll(r) + if err != nil { + return nil, err + } + br := bytes.NewReader(data) + return func() *io.SectionReader { + return io.NewSectionReader(br, 0, int64(len(data))) + }, nil +} + +// remaining returns the current position of r, and the number of bytes from +// there to its end, leaving it at its current position +func remaining(r io.Seeker) (int64, int64, error) { + start, err := r.Seek(0, io.SeekCurrent) + if err != nil { + return 0, 0, err + } + end, err := r.Seek(0, io.SeekEnd) + if err != nil { + return 0, 0, err + } + if _, err := r.Seek(start, io.SeekStart); err != nil { + return 0, 0, err + } + return start, end - start, nil +} diff --git a/metadata/metadata_test.go b/metadata/metadata_test.go index 4aa2399..5d48510 100644 --- a/metadata/metadata_test.go +++ b/metadata/metadata_test.go @@ -5,6 +5,8 @@ import ( "errors" "image" "io" + "os" + "path/filepath" "regexp" "strings" "sync" @@ -300,3 +302,90 @@ func Test_metadata_013(t *testing.T) { t.Fatal("expected GetMetadata to wait for the handler to finish despite mid-flight cancellation") } } + +// A file is read by each handler where it is, from its current position, +// rather than being read into memory, and each handler can seek in it +func Test_metadata_014(t *testing.T) { + path := filepath.Join(t.TempDir(), "file.txt") + if err := os.WriteFile(path, []byte("skip:data"), 0o644); err != nil { + t.Fatal(err) + } + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + if _, err := f.Seek(5, io.SeekStart); err != nil { + t.Fatal(err) + } + + var mu sync.Mutex + var got []string + handler := func(_ context.Context, r io.Reader, _ *Opts) ([]gomedia.Metadata, error) { + seeker, ok := r.(io.ReadSeeker) + if !ok { + return nil, errors.New("expected an io.ReadSeeker") + } + if _, err := seeker.Seek(1, io.SeekStart); err != nil { + return nil, err + } + data, err := io.ReadAll(seeker) + if err != nil { + return nil, err + } + name := "" + if named, ok := r.(gomedia.NamedReader); ok { + name = named.Name() + } + mu.Lock() + defer mu.Unlock() + got = append(got, string(data)+"@"+name) + return nil, nil + } + AddHandler(regexp.MustCompile("^x-test/014$"), "a", handler, "a") + AddHandler(regexp.MustCompile("^x-test/014$"), "b", handler, "b") + + if _, err := GetMetadata(context.Background(), f, "x-test/014"); err != nil { + t.Fatal(err) + } + for _, g := range got { + if g != "ata@"+path { + t.Errorf("expected each handler to read the file from its position, got %q", g) + } + } + if len(got) != 2 { + t.Fatalf("expected two handlers to run, got %d", len(got)) + } + + // The file isn't read, so it's where it was + if pos, err := f.Seek(0, io.SeekCurrent); err != nil { + t.Fatal(err) + } else if pos != 5 { + t.Fatalf("expected the file to be at 5, got %d", pos) + } +} + +// A reader which can't be read at an offset is read into memory, and each +// handler can still seek in it +func Test_metadata_015(t *testing.T) { + var got string + AddHandler(regexp.MustCompile("^x-test/015$"), "x", func(_ context.Context, r io.Reader, _ *Opts) ([]gomedia.Metadata, error) { + seeker, ok := r.(io.ReadSeeker) + if !ok { + return nil, errors.New("expected an io.ReadSeeker") + } + if _, err := seeker.Seek(2, io.SeekStart); err != nil { + return nil, err + } + data, err := io.ReadAll(seeker) + got = string(data) + return nil, err + }, "x") + + if _, err := GetMetadata(context.Background(), io.MultiReader(strings.NewReader("da"), strings.NewReader("ta")), "x-test/015"); err != nil { + t.Fatal(err) + } + if got != "ta" { + t.Fatalf("expected the handler to read the rest of the data, got %q", got) + } +}