Updates for content type detection

metadata/mime.go

@@ -43,8 +43,30 @@ var extensionContentTypes = map[string]extensionRule{
 	".m4a":  {ContentType: "audio/mp4", Preferred: true, Authoritative: true},
 	".jpg":  {ContentType: "image/jpeg", Preferred: true},
 	".jpeg": {ContentType: "image/jpeg"},
+	// Markdown sniffs as text/plain, so its extension refines it (see
+	// ContentType), but isn't Authoritative, so binary content named .md
+	// is still detected from its bytes
+	".md":       {ContentType: "text/markdown", Preferred: true},
+	".markdown": {ContentType: "text/markdown"},
+	// Documents which are zip archives sniff as application/zip, so their
+	// extensions refine it (see ContentType), and they're listed here as
+	// the system's types often don't include them
+	".docx": {ContentType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", Preferred: true},
+	".xlsx": {ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", Preferred: true},
+	".pptx": {ContentType: "application/vnd.openxmlformats-officedocument.presentationml.presentation", Preferred: true},
+	".odt":  {ContentType: "application/vnd.oasis.opendocument.text", Preferred: true},
+	".ods":  {ContentType: "application/vnd.oasis.opendocument.spreadsheet", Preferred: true},
+	".odp":  {ContentType: "application/vnd.oasis.opendocument.presentation", Preferred: true},
+	".epub": {ContentType: "application/epub+zip", Preferred: true},
 }
 
+// zipTypes are the prefixes and suffixes of the content types of documents
+// which are zip archives
+var (
+	zipTypePrefixes = []string{"application/vnd.openxmlformats-officedocument.", "application/vnd.oasis.opendocument."}
+	zipTypeSuffixes = []string{"+zip"}
+)
+
 ////////////////////////////////////////////////////////////////////////////////
 // PUBLIC METHODS
 
@@ -82,7 +104,20 @@ func ContentType(r io.Reader) (string, map[string]string, error) {
 		return "", nil, gomedia.ErrInternalError.With(err.Error())
 	}
 	if mediaType := http.DetectContentType(buf[:n]); mediaType != types.ContentTypeBinary {
-		return mime.ParseMediaType(mediaType)
+		contentType, params, err := mime.ParseMediaType(mediaType)
+		if err != nil {
+			return "", nil, err
+		}
+		// Plain text is refined by an extension with a more specific text
+		// type, such as text/markdown for .md, keeping the sniffed
+		// parameters, such as its charset, and a zip archive by an extension
+		// of a document which is a zip archive, such as .docx
+		if extType != "" {
+			if extContentType, _, err := mime.ParseMediaType(extType); err == nil && refines(contentType, extContentType) {
+				return extContentType, params, nil
+			}
+		}
+		return contentType, params, nil
 	}
 
 	// By extension second
@@ -110,3 +145,25 @@ func ExtensionByType(contentType string) string {
 
 	return exts[0]
 }
+
+// refines returns true when the content type of an extension is a more
+// specific type of the sniffed content type: a text type for plain text, or a
+// document which is a zip archive for a zip archive
+func refines(sniffed, ext string) bool {
+	switch sniffed {
+	case "text/plain":
+		return strings.HasPrefix(ext, "text/") && ext != sniffed
+	case "application/zip":
+		for _, prefix := range zipTypePrefixes {
+			if strings.HasPrefix(ext, prefix) {
+				return true
+			}
+		}
+		for _, suffix := range zipTypeSuffixes {
+			if strings.HasSuffix(ext, suffix) {
+				return true
+			}
+		}
+	}
+	return false
+}

metadata/mime_test.go

@@ -123,3 +123,66 @@ func Test_mime_005_jpg_extension_is_not_authoritative(t *testing.T) {
 		t.Fatalf("expected image/png (from sniffing, not the .jpg extension), got %q", contentType)
 	}
 }
+
+func Test_mime_006_text_refined_by_extension(t *testing.T) {
+	// Markdown sniffs as text/plain, and its extension refines it, keeping
+	// the sniffed charset
+	for _, name := range []string{"README.md", "notes.MD", "doc.markdown"} {
+		r := namedReader{Reader: bytes.NewReader([]byte("# Title\n\nSome text\n")), name: name}
+		contentType, params, err := ContentType(r)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if contentType != "text/markdown" {
+			t.Fatalf("%s: expected text/markdown, got %q", name, contentType)
+		}
+		if params["charset"] != "utf-8" {
+			t.Fatalf("%s: expected charset utf-8, got %v", name, params)
+		}
+	}
+}
+
+func Test_mime_007_text_extension_is_not_authoritative(t *testing.T) {
+	// Binary content named .md is detected from its bytes
+	data := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}
+	r := namedReader{Reader: bytes.NewReader(data), name: "image.md"}
+	contentType, _, err := ContentType(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if contentType != "image/png" {
+		t.Fatalf("expected image/png, got %q", contentType)
+	}
+
+	// Plain text without a more specific extension is text/plain
+	r = namedReader{Reader: bytes.NewReader([]byte("hello")), name: "notes.txt"}
+	if contentType, _, err := ContentType(r); err != nil {
+		t.Fatal(err)
+	} else if contentType != "text/plain" {
+		t.Fatalf("expected text/plain, got %q", contentType)
+	}
+}
+
+func Test_mime_008_zip_refined_by_extension(t *testing.T) {
+	// A document which is a zip archive sniffs as application/zip, and its
+	// extension refines it
+	zip := []byte("PK\x03\x04\x14\x00\x00\x00\x08\x00")
+	for name, want := range map[string]string{
+		"report.docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+		"sheet.XLSX":  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+		"deck.pptx":   "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+		"letter.odt":  "application/vnd.oasis.opendocument.text",
+		"book.epub":   "application/epub+zip",
+		"archive.zip": "application/zip",
+		"image.jpg":   "application/zip",
+	} {
+		r := namedReader{Reader: bytes.NewReader(zip), name: name}
+		contentType, _, err := ContentType(r)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if contentType != want {
+			t.Fatalf("%s: expected %q, got %q", name, want, contentType)
+		}
+	}
+}