From ed36707e6d3c0e1b8862f651a6329f2822586981 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Tue, 13 Oct 2026 14:18:04 +0200 Subject: [PATCH 1/3] Updates for content type detection --- metadata/mime.go | 59 +++++++++++++++++++++++++++++++++++++++- metadata/mime_test.go | 63 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/metadata/mime.go b/metadata/mime.go index 8d055e3..1ca1052 100644 --- a/metadata/mime.go +++ b/metadata/mime.go @@ -43,8 +43,30 @@ var extensionContentTypes = map[string]extensionRule{ ".m4a": {ContentType: "audio/mp4", Preferred: true, Authoritative: true}, ".jpg": {ContentType: "image/jpeg", Preferred: true}, ".jpeg": {ContentType: "image/jpeg"}, + // Markdown sniffs as text/plain, so its extension refines it (see + // ContentType), but isn't Authoritative, so binary content named .md + // is still detected from its bytes + ".md": {ContentType: "text/markdown", Preferred: true}, + ".markdown": {ContentType: "text/markdown"}, + // Documents which are zip archives sniff as application/zip, so their + // extensions refine it (see ContentType), and they're listed here as + // the system's types often don't include them + ".docx": {ContentType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", Preferred: true}, + ".xlsx": {ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", Preferred: true}, + ".pptx": {ContentType: "application/vnd.openxmlformats-officedocument.presentationml.presentation", Preferred: true}, + ".odt": {ContentType: "application/vnd.oasis.opendocument.text", Preferred: true}, + ".ods": {ContentType: "application/vnd.oasis.opendocument.spreadsheet", Preferred: true}, + ".odp": {ContentType: "application/vnd.oasis.opendocument.presentation", Preferred: true}, + ".epub": {ContentType: "application/epub+zip", Preferred: true}, } +// zipTypes are the prefixes and suffixes of the content types of documents +// which are zip archives +var ( + zipTypePrefixes = []string{"application/vnd.openxmlformats-officedocument.", "application/vnd.oasis.opendocument."} + zipTypeSuffixes = []string{"+zip"} +) + //////////////////////////////////////////////////////////////////////////////// // PUBLIC METHODS @@ -82,7 +104,20 @@ func ContentType(r io.Reader) (string, map[string]string, error) { return "", nil, gomedia.ErrInternalError.With(err.Error()) } if mediaType := http.DetectContentType(buf[:n]); mediaType != types.ContentTypeBinary { - return mime.ParseMediaType(mediaType) + contentType, params, err := mime.ParseMediaType(mediaType) + if err != nil { + return "", nil, err + } + // Plain text is refined by an extension with a more specific text + // type, such as text/markdown for .md, keeping the sniffed + // parameters, such as its charset, and a zip archive by an extension + // of a document which is a zip archive, such as .docx + if extType != "" { + if extContentType, _, err := mime.ParseMediaType(extType); err == nil && refines(contentType, extContentType) { + return extContentType, params, nil + } + } + return contentType, params, nil } // By extension second @@ -110,3 +145,25 @@ func ExtensionByType(contentType string) string { return exts[0] } + +// refines returns true when the content type of an extension is a more +// specific type of the sniffed content type: a text type for plain text, or a +// document which is a zip archive for a zip archive +func refines(sniffed, ext string) bool { + switch sniffed { + case "text/plain": + return strings.HasPrefix(ext, "text/") && ext != sniffed + case "application/zip": + for _, prefix := range zipTypePrefixes { + if strings.HasPrefix(ext, prefix) { + return true + } + } + for _, suffix := range zipTypeSuffixes { + if strings.HasSuffix(ext, suffix) { + return true + } + } + } + return false +} diff --git a/metadata/mime_test.go b/metadata/mime_test.go index c2f103a..aabb8a4 100644 --- a/metadata/mime_test.go +++ b/metadata/mime_test.go @@ -123,3 +123,66 @@ func Test_mime_005_jpg_extension_is_not_authoritative(t *testing.T) { t.Fatalf("expected image/png (from sniffing, not the .jpg extension), got %q", contentType) } } + +func Test_mime_006_text_refined_by_extension(t *testing.T) { + // Markdown sniffs as text/plain, and its extension refines it, keeping + // the sniffed charset + for _, name := range []string{"README.md", "notes.MD", "doc.markdown"} { + r := namedReader{Reader: bytes.NewReader([]byte("# Title\n\nSome text\n")), name: name} + contentType, params, err := ContentType(r) + if err != nil { + t.Fatal(err) + } + if contentType != "text/markdown" { + t.Fatalf("%s: expected text/markdown, got %q", name, contentType) + } + if params["charset"] != "utf-8" { + t.Fatalf("%s: expected charset utf-8, got %v", name, params) + } + } +} + +func Test_mime_007_text_extension_is_not_authoritative(t *testing.T) { + // Binary content named .md is detected from its bytes + data := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'} + r := namedReader{Reader: bytes.NewReader(data), name: "image.md"} + contentType, _, err := ContentType(r) + if err != nil { + t.Fatal(err) + } + if contentType != "image/png" { + t.Fatalf("expected image/png, got %q", contentType) + } + + // Plain text without a more specific extension is text/plain + r = namedReader{Reader: bytes.NewReader([]byte("hello")), name: "notes.txt"} + if contentType, _, err := ContentType(r); err != nil { + t.Fatal(err) + } else if contentType != "text/plain" { + t.Fatalf("expected text/plain, got %q", contentType) + } +} + +func Test_mime_008_zip_refined_by_extension(t *testing.T) { + // A document which is a zip archive sniffs as application/zip, and its + // extension refines it + zip := []byte("PK\x03\x04\x14\x00\x00\x00\x08\x00") + for name, want := range map[string]string{ + "report.docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "sheet.XLSX": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "deck.pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "letter.odt": "application/vnd.oasis.opendocument.text", + "book.epub": "application/epub+zip", + "archive.zip": "application/zip", + "image.jpg": "application/zip", + } { + r := namedReader{Reader: bytes.NewReader(zip), name: name} + contentType, _, err := ContentType(r) + if err != nil { + t.Fatal(err) + } + if contentType != want { + t.Fatalf("%s: expected %q, got %q", name, want, contentType) + } + } +} From 0174f66b2ed05445b6153735d569adcf4cf0b99c Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Tue, 13 Oct 2026 14:24:25 +0200 Subject: [PATCH 2/3] Updates from PR review - A zip archive isn't refined to an OpenDocument flat XML type, such as .fodt, as it's XML rather than a zip archive Co-Authored-By: Claude Opus 5.5 --- metadata/mime.go | 16 ++++++++++++---- metadata/mime_internal_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 metadata/mime_internal_test.go diff --git a/metadata/mime.go b/metadata/mime.go index 1ca1052..da5e9c5 100644 --- a/metadata/mime.go +++ b/metadata/mime.go @@ -61,10 +61,12 @@ var extensionContentTypes = map[string]extensionRule{ } // zipTypes are the prefixes and suffixes of the content types of documents -// which are zip archives +// which are zip archives, except the content types with the excluded +// suffixes, such as OpenDocument's flat XML documents (.fodt), which are XML var ( - zipTypePrefixes = []string{"application/vnd.openxmlformats-officedocument.", "application/vnd.oasis.opendocument."} - zipTypeSuffixes = []string{"+zip"} + zipTypePrefixes = []string{"application/vnd.openxmlformats-officedocument.", "application/vnd.oasis.opendocument."} + zipTypeSuffixes = []string{"+zip"} + zipTypeExcluding = []string{"-flat-xml"} ) //////////////////////////////////////////////////////////////////////////////// @@ -148,12 +150,18 @@ func ExtensionByType(contentType string) string { // refines returns true when the content type of an extension is a more // specific type of the sniffed content type: a text type for plain text, or a -// document which is a zip archive for a zip archive +// document which is a zip archive, and not a flat XML document, for a zip +// archive func refines(sniffed, ext string) bool { switch sniffed { case "text/plain": return strings.HasPrefix(ext, "text/") && ext != sniffed case "application/zip": + for _, suffix := range zipTypeExcluding { + if strings.HasSuffix(ext, suffix) { + return false + } + } for _, prefix := range zipTypePrefixes { if strings.HasPrefix(ext, prefix) { return true diff --git a/metadata/mime_internal_test.go b/metadata/mime_internal_test.go new file mode 100644 index 0000000..28836cb --- /dev/null +++ b/metadata/mime_internal_test.go @@ -0,0 +1,32 @@ +package metadata + +import "testing" + +func Test_mime_refines(t *testing.T) { + for _, tc := range []struct { + sniffed, ext string + want bool + }{ + // Plain text is refined by a more specific text type + {"text/plain", "text/markdown", true}, + {"text/plain", "text/plain", false}, + {"text/plain", "application/json", false}, + + // A zip archive is refined by a document which is a zip archive + {"application/zip", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", true}, + {"application/zip", "application/vnd.oasis.opendocument.text", true}, + {"application/zip", "application/epub+zip", true}, + {"application/zip", "image/jpeg", false}, + + // But not by a flat XML document, which isn't a zip archive + {"application/zip", "application/vnd.oasis.opendocument.text-flat-xml", false}, + {"application/zip", "application/vnd.oasis.opendocument.spreadsheet-flat-xml", false}, + + // Other sniffed types aren't refined + {"image/png", "image/jpeg", false}, + } { + if got := refines(tc.sniffed, tc.ext); got != tc.want { + t.Errorf("refines(%q, %q) = %v, want %v", tc.sniffed, tc.ext, got, tc.want) + } + } +} From d1b333044fb2d84bf5b8ee0f3591df83df4edae6 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Tue, 13 Oct 2026 14:58:57 +0200 Subject: [PATCH 3/3] Extensions registered with mime.AddExtensionType refine content types - The document extensions are removed from the package's table, so the types of extensions are those registered by the caller, with mime.AddExtensionType, or the system's - Plain text is refined by an application type which is text, such as application/json, or a registered type, but not by a binary type, such as an image, application/octet-stream, or a zip archive Co-Authored-By: Claude Opus 5.5 --- metadata/mime.go | 72 +++++++++++++++++----------------- metadata/mime_internal_test.go | 8 +++- metadata/mime_test.go | 44 +++++++++++++++++++++ 3 files changed, 88 insertions(+), 36 deletions(-) diff --git a/metadata/mime.go b/metadata/mime.go index da5e9c5..9b20f85 100644 --- a/metadata/mime.go +++ b/metadata/mime.go @@ -43,21 +43,6 @@ var extensionContentTypes = map[string]extensionRule{ ".m4a": {ContentType: "audio/mp4", Preferred: true, Authoritative: true}, ".jpg": {ContentType: "image/jpeg", Preferred: true}, ".jpeg": {ContentType: "image/jpeg"}, - // Markdown sniffs as text/plain, so its extension refines it (see - // ContentType), but isn't Authoritative, so binary content named .md - // is still detected from its bytes - ".md": {ContentType: "text/markdown", Preferred: true}, - ".markdown": {ContentType: "text/markdown"}, - // Documents which are zip archives sniff as application/zip, so their - // extensions refine it (see ContentType), and they're listed here as - // the system's types often don't include them - ".docx": {ContentType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", Preferred: true}, - ".xlsx": {ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", Preferred: true}, - ".pptx": {ContentType: "application/vnd.openxmlformats-officedocument.presentationml.presentation", Preferred: true}, - ".odt": {ContentType: "application/vnd.oasis.opendocument.text", Preferred: true}, - ".ods": {ContentType: "application/vnd.oasis.opendocument.spreadsheet", Preferred: true}, - ".odp": {ContentType: "application/vnd.oasis.opendocument.presentation", Preferred: true}, - ".epub": {ContentType: "application/epub+zip", Preferred: true}, } // zipTypes are the prefixes and suffixes of the content types of documents @@ -110,10 +95,10 @@ func ContentType(r io.Reader) (string, map[string]string, error) { if err != nil { return "", nil, err } - // Plain text is refined by an extension with a more specific text - // type, such as text/markdown for .md, keeping the sniffed - // parameters, such as its charset, and a zip archive by an extension - // of a document which is a zip archive, such as .docx + // Plain text, or a zip archive, is refined by the type of its + // extension, when that's more specific (see refines), such as + // text/markdown for .md, keeping the sniffed parameters, such as its + // charset. Extensions are registered with mime.AddExtensionType. if extType != "" { if extContentType, _, err := mime.ParseMediaType(extType); err == nil && refines(contentType, extContentType) { return extContentType, params, nil @@ -149,28 +134,45 @@ func ExtensionByType(contentType string) string { } // refines returns true when the content type of an extension is a more -// specific type of the sniffed content type: a text type for plain text, or a -// document which is a zip archive, and not a flat XML document, for a zip -// archive +// specific type of the sniffed content type. Plain text is refined by a +// text type, such as text/markdown, or an application type which is text, +// such as application/json, but not a binary type, such as an image or a +// zip archive, as sniffing would have detected it. A zip archive is refined +// by a document which is a zip archive, such as a Word document, but not a +// flat XML document. func refines(sniffed, ext string) bool { switch sniffed { case "text/plain": - return strings.HasPrefix(ext, "text/") && ext != sniffed + switch { + case ext == sniffed: + return false + case strings.HasPrefix(ext, "text/"): + return true + case strings.HasPrefix(ext, "application/"): + return ext != types.ContentTypeBinary && ext != "application/zip" && !isZipType(ext) + } case "application/zip": - for _, suffix := range zipTypeExcluding { - if strings.HasSuffix(ext, suffix) { - return false - } + return isZipType(ext) + } + return false +} + +// isZipType returns true when the content type is of a document which is a +// zip archive, but not a flat XML document +func isZipType(contentType string) bool { + for _, suffix := range zipTypeExcluding { + if strings.HasSuffix(contentType, suffix) { + return false } - for _, prefix := range zipTypePrefixes { - if strings.HasPrefix(ext, prefix) { - return true - } + } + for _, prefix := range zipTypePrefixes { + if strings.HasPrefix(contentType, prefix) { + return true } - for _, suffix := range zipTypeSuffixes { - if strings.HasSuffix(ext, suffix) { - return true - } + } + for _, suffix := range zipTypeSuffixes { + if strings.HasSuffix(contentType, suffix) { + return true } } return false diff --git a/metadata/mime_internal_test.go b/metadata/mime_internal_test.go index 28836cb..14015b0 100644 --- a/metadata/mime_internal_test.go +++ b/metadata/mime_internal_test.go @@ -10,7 +10,13 @@ func Test_mime_refines(t *testing.T) { // Plain text is refined by a more specific text type {"text/plain", "text/markdown", true}, {"text/plain", "text/plain", false}, - {"text/plain", "application/json", false}, + {"text/plain", "application/json", true}, + {"text/plain", "application/vnd.google-apps.document", true}, + {"text/plain", "application/octet-stream", false}, + {"text/plain", "application/zip", false}, + {"text/plain", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", false}, + {"text/plain", "image/png", false}, + {"text/plain", "audio/mpeg", false}, // A zip archive is refined by a document which is a zip archive {"application/zip", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", true}, diff --git a/metadata/mime_test.go b/metadata/mime_test.go index aabb8a4..f733c05 100644 --- a/metadata/mime_test.go +++ b/metadata/mime_test.go @@ -2,6 +2,7 @@ package metadata_test import ( "bytes" + "mime" "os" "path/filepath" "testing" @@ -124,6 +125,25 @@ func Test_mime_005_jpg_extension_is_not_authoritative(t *testing.T) { } } +// The extensions which are refined are registered by the caller, rather +// than being known to the package +func init() { + for ext, contentType := range map[string]string{ + ".md": "text/markdown", + ".markdown": "text/markdown", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ".odt": "application/vnd.oasis.opendocument.text", + ".epub": "application/epub+zip", + ".gdoc": "application/vnd.google-apps.document", + } { + if err := mime.AddExtensionType(ext, contentType); err != nil { + panic(err) + } + } +} + func Test_mime_006_text_refined_by_extension(t *testing.T) { // Markdown sniffs as text/plain, and its extension refines it, keeping // the sniffed charset @@ -186,3 +206,27 @@ func Test_mime_008_zip_refined_by_extension(t *testing.T) { } } } + +func Test_mime_009_text_refined_by_application_type(t *testing.T) { + // Plain text is refined by an application type which is text, such as a + // registered type, but not by a binary type + for name, want := range map[string]string{ + "Bader Pass.gdoc": "application/vnd.google-apps.document", + "data.json": "application/json", + "image.png": "text/plain", + "report.docx": "text/plain", + "notes.txt": "text/plain", + } { + r := namedReader{Reader: bytes.NewReader([]byte(`{"doc_id":"abc"}`)), name: name} + contentType, params, err := ContentType(r) + if err != nil { + t.Fatal(err) + } + if contentType != want { + t.Fatalf("%s: expected %q, got %q", name, want, contentType) + } + if params["charset"] != "utf-8" { + t.Fatalf("%s: expected charset utf-8, got %v", name, params) + } + } +}