First commit

.gitignore

@@ -0,0 +1 @@
+.idea/

comment.go

@@ -0,0 +1,50 @@
+package readers
+
+import (
+	"bytes"
+	"io"
+)
+
+type CommentReader struct {
+	r    io.Reader
+	over []byte
+	mark []string
+}
+
+var _ io.Reader = &CommentReader{}
+
+func NewCommentReader(r io.Reader, mark []string) *CommentReader {
+	return &CommentReader{r, nil, mark}
+}
+
+func (s *CommentReader) Read(p []byte) (n int, err error) {
+	if s.over != nil {
+		return 0, io.EOF
+	}
+	n, err = s.r.Read(p)
+	if err != nil {
+		return
+	}
+
+	n2 := s.matchesMark(p[:n])
+	if n2 != -1 {
+		s.over = p[n2:n]
+		n = n2
+		err = io.EOF
+	}
+	return
+}
+
+func (s *CommentReader) matchesMark(p []byte) int {
+	for _, i := range s.mark {
+		n := bytes.Index(p, []byte(i))
+		if n != -1 {
+			return n
+		}
+	}
+	return -1
+}
+
+func (s *CommentReader) Comment() io.Reader {
+	return io.MultiReader(bytes.NewReader(s.over), s.r)
+}

comment_test.go

@@ -0,0 +1,49 @@
+package readers
+
+import (
+	"github.com/stretchr/testify/assert"
+	"io"
+	"strings"
+	"testing"
+)
+
+func TestNewCommentReader(t *testing.T) {
+	t.Run("normal", func(t *testing.T) {
+		r := NewCommentReader(strings.NewReader("Hello world! # this is a comment"), []string{"#"})
+		all, err := io.ReadAll(r)
+		assert.NoError(t, err)
+		assert.Equal(t, "Hello world! ", string(all))
+		all, err = io.ReadAll(r.Comment())
+		assert.NoError(t, err)
+		assert.Equal(t, "# this is a comment", string(all))
+	})
+	t.Run("overflow start", func(t *testing.T) {
+		r := NewCommentReader(strings.NewReader(strings.Repeat("Hello world!", 2048)+" # this is a comment"), []string{"#"})
+		all, err := io.ReadAll(r)
+		assert.NoError(t, err)
+		assert.Equal(t, strings.Repeat("Hello world!", 2048)+" ", string(all))
+		all, err = io.ReadAll(r.Comment())
+		assert.NoError(t, err)
+		assert.Equal(t, "# this is a comment", string(all))
+	})
+	t.Run("overflow comment", func(t *testing.T) {
+		r := NewCommentReader(strings.NewReader(strings.Repeat("Hello world!", 2048)+" # "+strings.Repeat("this is a comment", 2048)), []string{"#"})
+		all, err := io.ReadAll(r)
+		assert.NoError(t, err)
+		assert.Equal(t, strings.Repeat("Hello world!", 2048)+" ", string(all))
+		all, err = io.ReadAll(r.Comment())
+		assert.NoError(t, err)
+		assert.Equal(t, "# "+strings.Repeat("this is a comment", 2048), string(all))
+	})
+}
+
+func FuzzCommentReader(f *testing.F) {
+	f.Fuzz(func(t *testing.T, a string) {
+		r := NewCommentReader(strings.NewReader(a), []string{"#"})
+		b1, err := io.ReadAll(r)
+		assert.NoError(t, err)
+		b2, err := io.ReadAll(r.Comment())
+		assert.NoError(t, err)
+		assert.Equal(t, string(b1)+string(b2), a)
+	})
+}

go.mod

@@ -0,0 +1,11 @@
+module readers
+
+go 1.21
+
+require github.com/stretchr/testify v1.8.4
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -0,0 +1,10 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

line.go

@@ -0,0 +1,81 @@
+package readers
+
+import (
+	"bytes"
+	"errors"
+	"io"
+)
+
+var EOL error = &eolError{}
+
+type eolError struct{}
+
+func (e *eolError) Error() string { return "EOL" }
+func (e *eolError) Unwrap() error { return io.EOF }
+
+type LineReader struct {
+	err  error
+	r    io.Reader
+	over []byte
+}
+
+var _ io.Reader = &LineReader{}
+
+func NewLineReader(r io.Reader) *LineReader {
+	return &LineReader{nil, r, nil}
+}
+
+func (l *LineReader) Err() error {
+	if errors.Is(l.err, io.EOF) {
+		return nil
+	}
+	return l.err
+}
+
+func (l *LineReader) Next() bool {
+	if l.err == nil || errors.Is(l.err, EOL) {
+		l.err = nil
+		return true
+	}
+	return false
+}
+
+func (l *LineReader) Read(p []byte) (n int, err error) {
+	if errors.Is(l.err, EOL) {
+		return 0, io.EOF
+	}
+
+	rOver := bytes.NewBuffer(l.over)
+	r := io.MultiReader(rOver, l.r)
+	n, l.err = r.Read(p)
+	if l.err != nil {
+		err = eolToEof(l.err)
+		return
+	}
+	nFull := n
+
+	nStart := n
+	n2 := bytes.IndexByte(p, '\n')
+	if n2 != -1 {
+		l.err = EOL
+		n = n2
+		nStart = n2 + 1
+	}
+
+	if n > 0 && p[n-1] == '\r' {
+		n--
+	}
+
+	l.over = p[nStart:nFull]
+	if rOver.Len() > 0 {
+		l.over = append(p[nStart:nFull], rOver.Bytes()...)
+	}
+	return
+}
+
+func eolToEof(err error) error {
+	if errors.Is(err, EOL) {
+		return io.EOF
+	}
+	return err
+}

line_test.go

@@ -0,0 +1,45 @@
+package readers
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	"go/scanner"
+	"io"
+	"strings"
+	"testing"
+)
+
+func TestNewLineReader(t *testing.T) {
+	t.Run("normal", func(t *testing.T) {
+		r := NewLineReader(strings.NewReader("Hello world!\nHello new line!\n"))
+		for r.Next() {
+			a, err := io.ReadAll(r)
+			fmt.Println(len(a), string(a))
+			assert.NoError(t, err)
+		}
+		assert.NoError(t, r.Err())
+	})
+	t.Run("really long", func(t *testing.T) {
+		r := NewLineReader(strings.NewReader(strings.Repeat("Hello world! ", 1024) + "\na\n"))
+		for r.Next() {
+			a, err := io.ReadAll(r)
+			fmt.Println(len(a), string(a))
+			assert.NoError(t, err)
+		}
+		assert.NoError(t, r.Err())
+	})
+	t.Run("against scanner", func(t *testing.T) {
+		r := NewLineReader(strings.NewReader(strings.Repeat("Hello world! ", 409600) + "\na\n"))
+		for r.Next() {
+			a, err := io.ReadAll(r)
+			fmt.Println(len(a), string(a))
+			assert.NoError(t, err)
+		}
+		assert.NoError(t, r.Err())
+
+		sc := bufio.NewScanner(strings.NewReader(strings.Repeat("Hello world! ", 409600) + "\na\n"))
+		assert.False(t, sc.Scan())
+		assert.Error(t, sc.Err(), scanner.Error{})
+	})
+}