fix(scanner): report block scalar starting lines

lexer/block_scalar_position_test.go

@@ -0,0 +1,83 @@
+package lexer_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/goccy/go-yaml/lexer"
+	"github.com/goccy/go-yaml/parser"
+	"github.com/goccy/go-yaml/token"
+)
+
+func TestTokenizeBlockScalarStartLine(t *testing.T) {
+	styles := []struct {
+		header string
+		value  string
+		chomp  string
+	}{
+		{"|", "first\nsecond", "clip"},
+		{"|-", "first\nsecond", "strip"},
+		{"|+", "first\nsecond", "keep"},
+		{">", "first second", "clip"},
+		{">-", "first second", "strip"},
+		{">+", "first second", "keep"},
+	}
+	endings := []struct {
+		name     string
+		trailing string
+		next     string
+	}{
+		{"eof_without_newline", "", ""},
+		{"eof_after_newline", "\n", ""},
+		{"eof_after_blank_lines", "\n\n\n", ""},
+		{"comment", "\n\n", "# comment\n"},
+		{"mapping", "\n", "next: value\n"},
+		{"document_start", "\n\n", "---\nnext: value\n"},
+		{"document_end", "\n\n", "...\n"},
+	}
+	for _, newline := range []struct{ name, value string }{{"LF", "\n"}, {"CRLF", "\r\n"}, {"CR", "\r"}} {
+		for _, style := range styles {
+			for _, ending := range endings {
+				t.Run(newline.name+"/"+style.header+"/"+ending.name, func(t *testing.T) {
+					body := "  first\n  second" + ending.trailing
+					src := strings.ReplaceAll("\nkey: "+style.header+"\n"+body+ending.next, "\n", newline.value)
+					tokens := lexer.Tokenize(src)
+					var scalar *token.Token
+					for i, tk := range tokens {
+						if tk.Type == token.LiteralType || tk.Type == token.FoldedType {
+							if i+1 < len(tokens) {
+								scalar = tokens[i+1]
+							}
+							break
+						}
+					}
+					if scalar == nil || scalar.Type != token.StringType {
+						t.Fatalf("expected block scalar content token, got %v", scalar)
+					}
+					if scalar.Position.Line != 3 {
+						t.Errorf("content starts on line 3, got %d", scalar.Position.Line)
+					}
+					wantValue := style.value
+					switch style.chomp {
+					case "clip":
+						if ending.trailing != "" {
+							wantValue += "\n"
+						}
+					case "keep":
+						wantValue += ending.trailing
+					}
+					if scalar.Value != wantValue {
+						t.Errorf("value: got %q, want %q", scalar.Value, wantValue)
+					}
+					wantOrigin := strings.ReplaceAll(body, "\n", newline.value)
+					if scalar.Origin != wantOrigin {
+						t.Errorf("origin: got %q, want %q", scalar.Origin, wantOrigin)
+					}
+					if _, err := parser.Parse(tokens, parser.ParseComments); err != nil {
+						t.Fatalf("parse tokens: %v", err)
+					}
+				})
+			}
+		}
+	}
+}

scanner/scanner.go

@@ -74,7 +74,8 @@ func (s *Scanner) bufferedToken(ctx *Context) *token.Token {
 	column := s.column - len(ctx.buf)
 	level := s.indentLevel
 	if ctx.isMultiLine() {
-		line -= s.newLineCount(ctx.buf)
+		// Folded values replace some line breaks with spaces, so count the source.
+		line -= s.newLineCount(ctx.obuf)
 		column = strings.Index(string(ctx.obuf), string(ctx.buf)) + 1
 		// Since we are in a literal, folded or raw folded
 		// we can use the indent level from the last token.
@@ -799,7 +800,13 @@ func (s *Scanner) scanMultiLine(ctx *Context, c rune) error {
 			return ErrInvalidToken(invalidTk)
 		}
 		value := ctx.bufferedSrc()
-		ctx.addToken(token.String(string(value), string(ctx.obuf), s.pos()))
+		pos := s.pos()
+		pos.Line -= s.newLineCount(ctx.obuf)
+		if s.isNewLineChar(c) {
+			// The final line break is buffered but has not advanced s.line yet.
+			pos.Line++
+		}
+		ctx.addToken(token.String(string(value), string(ctx.obuf), pos))
 		ctx.clear()
 		s.progressColumn(ctx, 1)
 	} else if s.isNewLineChar(c) {
@@ -810,7 +817,9 @@ func (s *Scanner) scanMultiLine(ctx *Context, c rune) error {
 		if ctx.next() {
 			if s.foundDocumentSeparatorMarker(ctx.src[ctx.idx:]) {
 				value := ctx.bufferedSrc()
-				ctx.addToken(token.String(string(value), string(ctx.obuf), s.pos()))
+				pos := s.pos()
+				pos.Line -= s.newLineCount(ctx.obuf)
+				ctx.addToken(token.String(string(value), string(ctx.obuf), pos))
 				ctx.clear()
 				s.breakMultiLine(ctx)
 			}