Skip to content

Fix libyaml handling of truncated UTF-8 sequences #359

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ccoVeille wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix libyaml handling of truncated UTF-8 sequences #359

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions internal/libyaml/emitter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1883,6 +1883,15 @@ func (emitter *Emitter) writeDoubleQuotedScalar(value []byte, allow_breaks bool)
w, v = 3, rune(octet&0x0F)
case octet&0xF8 == 0xF0:
w, v = 4, rune(octet&0x07)
default:
return EmitterError{
Message: "invalid leading UTF-8 octet",
}
}
Comment thread
ccoVeille marked this conversation as resolved.
if len(value)-i < w {
return EmitterError{
Message: "incomplete UTF-8 octet sequence",
}
}
for k := 1; k < w; k++ {
octet = value[i+k]
Expand Down
83 changes: 83 additions & 0 deletions internal/libyaml/emitter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,86 @@ func runEmitWriterTest(t *testing.T, tc TestCase) {
"output should contain %q, got %q", expected, result)
}
}

// TestEmitterInvalidUTF8DoesNotPanic validates emitter behavior in isolation,
// without involving scanner/parser, when scalar values contain malformed UTF-8.
func TestEmitterInvalidUTF8DoesNotPanic(t *testing.T) {
for name, malformed := range map[string][]byte{
"Incomplete 2-byte UTF-8 sequence": {0xC2},
"Incomplete 3-byte UTF-8 sequence": {0xEF},
"Incomplete 4-byte UTF-8 sequence": {0xF0},
"truncated BOM sequence": {0xEF, 0xBB},
} {
Comment on lines +58 to +63
t.Run(name, func(t *testing.T) {
t.Run("plain scalar style", func(t *testing.T) {
emitter := NewEmitter()
var out []byte
emitter.SetOutputString(&out)
emitter.SetUnicode(true)

Comment thread
ccoVeille marked this conversation as resolved.
for _, event := range []struct {
Event Event
ExpectedErrorContains string
}{
{Event: NewStreamStartEvent(UTF8_ENCODING)},
{Event: NewDocumentStartEvent(nil, nil, true)},
{
Event: NewScalarEvent(nil, nil, malformed, true, false, PLAIN_SCALAR_STYLE),
ExpectedErrorContains: "incomplete UTF-8 octet sequence",
},
{Event: NewDocumentEndEvent(true)},
{Event: NewStreamEndEvent()},
} {
err := emitter.Emit(&event.Event)
if event.ExpectedErrorContains == "" {
assert.NoError(t, err)
continue
}

assert.ErrorMatches(t, event.ExpectedErrorContains, err)
if err != nil {
break // stop emitting further events after the expected error, all further events would be no-ops anyway due to the error state
}
}

// invalid UTF-8 should not be emitted, output should be empty
assert.Equal(t, "", string(out))
})
t.Run(name, func(t *testing.T) {
t.Run("double-quoted scalar style", func(t *testing.T) {
Comment on lines +99 to +100
emitter := NewEmitter()
var out []byte
emitter.SetOutputString(&out)
emitter.SetUnicode(true)

for _, event := range []struct {
Event Event
ExpectedErrorContains string
}{
{Event: NewStreamStartEvent(UTF8_ENCODING)},
{Event: NewDocumentStartEvent(nil, nil, true)},
{
Event: NewScalarEvent(nil, nil, malformed, true, false, DOUBLE_QUOTED_SCALAR_STYLE),
ExpectedErrorContains: "incomplete UTF-8 octet sequence",
},
{Event: NewDocumentEndEvent(true)},
{Event: NewStreamEndEvent()},
} {
err := emitter.Emit(&event.Event)
if event.ExpectedErrorContains == "" {
assert.NoError(t, err)
continue
}

assert.ErrorMatches(t, event.ExpectedErrorContains, err)
if err != nil {
break // stop emitting further events after the expected error, all further events would be no-ops anyway due to the error state
}
}

assert.Equal(t, "", string(out))
})
Comment thread
ccoVeille marked this conversation as resolved.
})
})
}
}
103 changes: 66 additions & 37 deletions internal/libyaml/scanner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -617,17 +617,30 @@ func isASCII(b []byte, i int) bool {
return b[i] <= 0x7F
}

// byteAt returns the byte at index i, or 0x00 if i is out of bounds.
func byteAt(b []byte, i int) byte {
if i >= 0 && i < len(b) {
return b[i]
}
return 0x00
}

// Check if the character at the start of the buffer can be printed unescaped.
func isPrintable(b []byte, i int) bool {
return ((b[i] == 0x0A) || // . == #x0A
(b[i] >= 0x20 && b[i] <= 0x7E) || // #x20 <= . <= #x7E
(b[i] == 0xC2 && b[i+1] >= 0xA0) || // #0xA0 <= . <= #xD7FF
(b[i] > 0xC2 && b[i] < 0xED) ||
(b[i] == 0xED && b[i+1] < 0xA0) ||
(b[i] == 0xEE) ||
(b[i] == 0xEF && // #xE000 <= . <= #xFFFD
!(b[i+1] == 0xBB && b[i+2] == 0xBF) && // && . != #xFEFF
!(b[i+1] == 0xBF && (b[i+2] == 0xBE || b[i+2] == 0xBF))))
c0 := b[i]
c1 := byteAt(b, i+1)
c2 := byteAt(b, i+2)
Comment on lines 629 to +632
return ((c0 == 0x0A) || // . == #x0A
(c0 >= 0x20 && c0 <= 0x7E) || // #x20 <= . <= #x7E
(c0 == 0xC2 && c1 >= 0xA0) || // #0xA0 <= . <= #xD7FF
(c0 > 0xC2 && c0 < 0xED) ||
(c0 == 0xED && c1 > 0 && c1 < 0xA0) ||
(c0 == 0xEE) ||
(c0 == 0xEF && // #xE000 <= . <= #xFFFD
c1 > 0 &&
c2 > 0 &&
!(c1 == 0xBB && c2 == 0xBF) && // && . != #xFEFF
!(c1 == 0xBF && (c2 == 0xBE || c2 == 0xBF))))
Comment thread
ccoVeille marked this conversation as resolved.
}

// Check if the character at the specified position is NUL.
Expand All @@ -637,6 +650,10 @@ func isZeroChar(b []byte, i int) bool {

// Check if the beginning of the buffer is a BOM.
func isBOM(b []byte, i int) bool {
if len(b) < 3 {
// BOM cannot be present if there are less than 3 bytes, avoid panic
return false
}
return b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF
Comment thread
ccoVeille marked this conversation as resolved.
}

Expand All @@ -658,60 +675,72 @@ func isBlank(b []byte, i int) bool {

// Check if the character at the specified position is a line break.
func isLineBreak(b []byte, i int) bool {
return (b[i] == '\r' || // CR (#xD)
b[i] == '\n' || // LF (#xA)
b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85)
b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028)
b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9) // PS (#x2029)
c0 := b[i]
c1 := byteAt(b, i+1)
c2 := byteAt(b, i+2)
return (c0 == '\r' || // CR (#xD)
c0 == '\n' || // LF (#xA)
c0 == 0xC2 && c1 == 0x85 || // NEL (#x85)
c0 == 0xE2 && c1 == 0x80 && c2 == 0xA8 || // LS (#x2028)
c0 == 0xE2 && c1 == 0x80 && c2 == 0xA9) // PS (#x2029)
}

// isCRLF checks if the position contains a CR LF sequence.
func isCRLF(b []byte, i int) bool {
return b[i] == '\r' && b[i+1] == '\n'
return b[i] == '\r' && byteAt(b, i+1) == '\n'
}

// Check if the character is a line break or NUL.
func isBreakOrZero(b []byte, i int) bool {
// return isLineBreak(b, i) || isZeroChar(b, i)
c0 := b[i]
c1 := byteAt(b, i+1)
c2 := byteAt(b, i+2)
return (
// isBreak:
b[i] == '\r' || // CR (#xD)
b[i] == '\n' || // LF (#xA)
b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85)
b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028)
b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029)
c0 == '\r' || // CR (#xD)
c0 == '\n' || // LF (#xA)
c0 == 0xC2 && c1 == 0x85 || // NEL (#x85)
c0 == 0xE2 && c1 == 0x80 && c2 == 0xA8 || // LS (#x2028)
c0 == 0xE2 && c1 == 0x80 && c2 == 0xA9 || // PS (#x2029)
Comment thread
ccoVeille marked this conversation as resolved.
// isZeroChar:
b[i] == 0)
c0 == 0)
}

// Check if the character is a line break, space, or NUL.
func isSpaceOrZero(b []byte, i int) bool {
// return isSpace(b, i) || isBreakOrZero(b, i)
c0 := b[i]
c1 := byteAt(b, i+1)
c2 := byteAt(b, i+2)
return (
// isSpace:
b[i] == ' ' ||
c0 == ' ' ||
// isBreakOrZero:
b[i] == '\r' || // CR (#xD)
b[i] == '\n' || // LF (#xA)
b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85)
b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028)
b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029)
b[i] == 0)
c0 == '\r' || // CR (#xD)
c0 == '\n' || // LF (#xA)
c0 == 0xC2 && c1 == 0x85 || // NEL (#x85)
c0 == 0xE2 && c1 == 0x80 && c2 == 0xA8 || // LS (#x2028)
c0 == 0xE2 && c1 == 0x80 && c2 == 0xA9 || // PS (#x2029)
c0 == 0)
}

// Check if the character is a line break, space, tab, or NUL.
func isBlankOrZero(b []byte, i int) bool {
// return isBlank(b, i) || isBreakOrZero(b, i)
c0 := b[i]
c1 := byteAt(b, i+1)
c2 := byteAt(b, i+2)
return (
// isBlank:
b[i] == ' ' || b[i] == '\t' ||
c0 == ' ' || c0 == '\t' ||
// isBreakOrZero:
b[i] == '\r' || // CR (#xD)
b[i] == '\n' || // LF (#xA)
b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85)
b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028)
b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029)
b[i] == 0)
c0 == '\r' || // CR (#xD)
c0 == '\n' || // LF (#xA)
c0 == 0xC2 && c1 == 0x85 || // NEL (#x85)
c0 == 0xE2 && c1 == 0x80 && c2 == 0xA8 || // LS (#x2028)
c0 == 0xE2 && c1 == 0x80 && c2 == 0xA9 || // PS (#x2029)
c0 == 0)
}

func isEndOfScalarInFlowContentChar(b []byte, i int) bool {
Expand All @@ -724,8 +753,8 @@ func isEndOfScalarInFlowContentChar(b []byte, i int) bool {
return isBlankOrZero(b, i+1)
// ": ", ":,", ":]" and ":}"
case ':':
return b[i+1] == ' ' || b[i+1] == ',' ||
b[i+1] == ']' || b[i+1] == '}'
nextChar := byteAt(b, i+1)
return nextChar == ' ' || nextChar == ',' || nextChar == ']' || nextChar == '}'
default:
return false
}
Expand Down
50 changes: 50 additions & 0 deletions internal/libyaml/scanner_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,56 @@ func TestScanner(t *testing.T) {
})
}

// TestTrailingUTF8LeadByte ensures a truncated multi-byte UTF-8 sequence at EOF
// reports a reader error and does not panic.
func TestTrailingUTF8LeadByte(t *testing.T) {
parser := NewParser()
parser.SetInputString([]byte{0xEF, 0xBB}) // Incomplete 3-byte UTF-8 sequence (missing third byte)

defer func() {
if r := recover(); r != nil {
t.Fatalf("Scan panicked: %v", r)
}
}()

var token Token
err := parser.Scan(&token)
assert.ErrorMatchesf(t, "incomplete UTF-8 octet sequence", err, "trailing UTF-8 lead byte must fail cleanly")
}

// TestPredicateMissingLookahead verifies that predicates handle truncated UTF-8
// safely when called without scanner buffer lookahead guarantees.
func TestPredicateMissingLookahead(t *testing.T) {
// Helper to check that a function doesn't panic
notPanic := func(name string, f func()) {
t.Helper()
defer func() {
if r := recover(); r != nil {
t.Errorf("%s panicked: %v", name, r)
}
}()
f()
}

// isPrintable should not panic on truncated UTF-8 sequences
notPanic("isPrintable with 0xF0", func() {
_ = isPrintable([]byte{0xF0}, 0)
})

// Test other predicates with truncated sequences
notPanic("isLineBreak with 0xC2", func() {
_ = isLineBreak([]byte{0xC2}, 0)
})

notPanic("isBOM with truncated", func() {
_ = isBOM([]byte{0xEF, 0xBB}, 0)
})

notPanic("isEndOfScalarInFlowContentChar", func() {
_ = isEndOfScalarInFlowContentChar([]byte{':'}, 0)
})
Comment on lines +58 to +74
}

// runScanTokensTest tests the scanTokens function.
//
//nolint:thelper // because this function is the real test
Expand Down
Loading
Loading