diff --git a/metadata/mime.go b/metadata/mime.go index 8d055e3..9b20f85 100644 --- a/metadata/mime.go +++ b/metadata/mime.go @@ -45,6 +45,15 @@ var extensionContentTypes = map[string]extensionRule{ ".jpeg": {ContentType: "image/jpeg"}, } +// zipTypes are the prefixes and suffixes of the content types of documents +// which are zip archives, except the content types with the excluded +// suffixes, such as OpenDocument's flat XML documents (.fodt), which are XML +var ( + zipTypePrefixes = []string{"application/vnd.openxmlformats-officedocument.", "application/vnd.oasis.opendocument."} + zipTypeSuffixes = []string{"+zip"} + zipTypeExcluding = []string{"-flat-xml"} +) + //////////////////////////////////////////////////////////////////////////////// // PUBLIC METHODS @@ -82,7 +91,20 @@ func ContentType(r io.Reader) (string, map[string]string, error) { return "", nil, gomedia.ErrInternalError.With(err.Error()) } if mediaType := http.DetectContentType(buf[:n]); mediaType != types.ContentTypeBinary { - return mime.ParseMediaType(mediaType) + contentType, params, err := mime.ParseMediaType(mediaType) + if err != nil { + return "", nil, err + } + // Plain text, or a zip archive, is refined by the type of its + // extension, when that's more specific (see refines), such as + // text/markdown for .md, keeping the sniffed parameters, such as its + // charset. Extensions are registered with mime.AddExtensionType. + if extType != "" { + if extContentType, _, err := mime.ParseMediaType(extType); err == nil && refines(contentType, extContentType) { + return extContentType, params, nil + } + } + return contentType, params, nil } // By extension second @@ -110,3 +132,48 @@ func ExtensionByType(contentType string) string { return exts[0] } + +// refines returns true when the content type of an extension is a more +// specific type of the sniffed content type. Plain text is refined by a +// text type, such as text/markdown, or an application type which is text, +// such as application/json, but not a binary type, such as an image or a +// zip archive, as sniffing would have detected it. A zip archive is refined +// by a document which is a zip archive, such as a Word document, but not a +// flat XML document. +func refines(sniffed, ext string) bool { + switch sniffed { + case "text/plain": + switch { + case ext == sniffed: + return false + case strings.HasPrefix(ext, "text/"): + return true + case strings.HasPrefix(ext, "application/"): + return ext != types.ContentTypeBinary && ext != "application/zip" && !isZipType(ext) + } + case "application/zip": + return isZipType(ext) + } + return false +} + +// isZipType returns true when the content type is of a document which is a +// zip archive, but not a flat XML document +func isZipType(contentType string) bool { + for _, suffix := range zipTypeExcluding { + if strings.HasSuffix(contentType, suffix) { + return false + } + } + for _, prefix := range zipTypePrefixes { + if strings.HasPrefix(contentType, prefix) { + return true + } + } + for _, suffix := range zipTypeSuffixes { + if strings.HasSuffix(contentType, suffix) { + return true + } + } + return false +} diff --git a/metadata/mime_internal_test.go b/metadata/mime_internal_test.go new file mode 100644 index 0000000..14015b0 --- /dev/null +++ b/metadata/mime_internal_test.go @@ -0,0 +1,38 @@ +package metadata + +import "testing" + +func Test_mime_refines(t *testing.T) { + for _, tc := range []struct { + sniffed, ext string + want bool + }{ + // Plain text is refined by a more specific text type + {"text/plain", "text/markdown", true}, + {"text/plain", "text/plain", false}, + {"text/plain", "application/json", true}, + {"text/plain", "application/vnd.google-apps.document", true}, + {"text/plain", "application/octet-stream", false}, + {"text/plain", "application/zip", false}, + {"text/plain", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", false}, + {"text/plain", "image/png", false}, + {"text/plain", "audio/mpeg", false}, + + // A zip archive is refined by a document which is a zip archive + {"application/zip", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", true}, + {"application/zip", "application/vnd.oasis.opendocument.text", true}, + {"application/zip", "application/epub+zip", true}, + {"application/zip", "image/jpeg", false}, + + // But not by a flat XML document, which isn't a zip archive + {"application/zip", "application/vnd.oasis.opendocument.text-flat-xml", false}, + {"application/zip", "application/vnd.oasis.opendocument.spreadsheet-flat-xml", false}, + + // Other sniffed types aren't refined + {"image/png", "image/jpeg", false}, + } { + if got := refines(tc.sniffed, tc.ext); got != tc.want { + t.Errorf("refines(%q, %q) = %v, want %v", tc.sniffed, tc.ext, got, tc.want) + } + } +} diff --git a/metadata/mime_test.go b/metadata/mime_test.go index c2f103a..f733c05 100644 --- a/metadata/mime_test.go +++ b/metadata/mime_test.go @@ -2,6 +2,7 @@ package metadata_test import ( "bytes" + "mime" "os" "path/filepath" "testing" @@ -123,3 +124,109 @@ func Test_mime_005_jpg_extension_is_not_authoritative(t *testing.T) { t.Fatalf("expected image/png (from sniffing, not the .jpg extension), got %q", contentType) } } + +// The extensions which are refined are registered by the caller, rather +// than being known to the package +func init() { + for ext, contentType := range map[string]string{ + ".md": "text/markdown", + ".markdown": "text/markdown", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ".odt": "application/vnd.oasis.opendocument.text", + ".epub": "application/epub+zip", + ".gdoc": "application/vnd.google-apps.document", + } { + if err := mime.AddExtensionType(ext, contentType); err != nil { + panic(err) + } + } +} + +func Test_mime_006_text_refined_by_extension(t *testing.T) { + // Markdown sniffs as text/plain, and its extension refines it, keeping + // the sniffed charset + for _, name := range []string{"README.md", "notes.MD", "doc.markdown"} { + r := namedReader{Reader: bytes.NewReader([]byte("# Title\n\nSome text\n")), name: name} + contentType, params, err := ContentType(r) + if err != nil { + t.Fatal(err) + } + if contentType != "text/markdown" { + t.Fatalf("%s: expected text/markdown, got %q", name, contentType) + } + if params["charset"] != "utf-8" { + t.Fatalf("%s: expected charset utf-8, got %v", name, params) + } + } +} + +func Test_mime_007_text_extension_is_not_authoritative(t *testing.T) { + // Binary content named .md is detected from its bytes + data := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'} + r := namedReader{Reader: bytes.NewReader(data), name: "image.md"} + contentType, _, err := ContentType(r) + if err != nil { + t.Fatal(err) + } + if contentType != "image/png" { + t.Fatalf("expected image/png, got %q", contentType) + } + + // Plain text without a more specific extension is text/plain + r = namedReader{Reader: bytes.NewReader([]byte("hello")), name: "notes.txt"} + if contentType, _, err := ContentType(r); err != nil { + t.Fatal(err) + } else if contentType != "text/plain" { + t.Fatalf("expected text/plain, got %q", contentType) + } +} + +func Test_mime_008_zip_refined_by_extension(t *testing.T) { + // A document which is a zip archive sniffs as application/zip, and its + // extension refines it + zip := []byte("PK\x03\x04\x14\x00\x00\x00\x08\x00") + for name, want := range map[string]string{ + "report.docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "sheet.XLSX": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "deck.pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "letter.odt": "application/vnd.oasis.opendocument.text", + "book.epub": "application/epub+zip", + "archive.zip": "application/zip", + "image.jpg": "application/zip", + } { + r := namedReader{Reader: bytes.NewReader(zip), name: name} + contentType, _, err := ContentType(r) + if err != nil { + t.Fatal(err) + } + if contentType != want { + t.Fatalf("%s: expected %q, got %q", name, want, contentType) + } + } +} + +func Test_mime_009_text_refined_by_application_type(t *testing.T) { + // Plain text is refined by an application type which is text, such as a + // registered type, but not by a binary type + for name, want := range map[string]string{ + "Bader Pass.gdoc": "application/vnd.google-apps.document", + "data.json": "application/json", + "image.png": "text/plain", + "report.docx": "text/plain", + "notes.txt": "text/plain", + } { + r := namedReader{Reader: bytes.NewReader([]byte(`{"doc_id":"abc"}`)), name: name} + contentType, params, err := ContentType(r) + if err != nil { + t.Fatal(err) + } + if contentType != want { + t.Fatalf("%s: expected %q, got %q", name, want, contentType) + } + if params["charset"] != "utf-8" { + t.Fatalf("%s: expected charset utf-8, got %v", name, params) + } + } +}