commit 053222a2f408b838966db5db188397fd892662c4 Author: MrMelon54 Date: Sun Jan 21 23:50:07 2024 +0000 First commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9f11b75 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea/ diff --git a/comment.go b/comment.go new file mode 100644 index 0000000..756c694 --- /dev/null +++ b/comment.go @@ -0,0 +1,50 @@ +package readers + +import ( + "bytes" + "io" +) + +type CommentReader struct { + r io.Reader + over []byte + mark []string +} + +var _ io.Reader = &CommentReader{} + +func NewCommentReader(r io.Reader, mark []string) *CommentReader { + return &CommentReader{r, nil, mark} +} + +func (s *CommentReader) Read(p []byte) (n int, err error) { + if s.over != nil { + return 0, io.EOF + } + n, err = s.r.Read(p) + if err != nil { + return + } + + n2 := s.matchesMark(p[:n]) + if n2 != -1 { + s.over = p[n2:n] + n = n2 + err = io.EOF + } + return +} + +func (s *CommentReader) matchesMark(p []byte) int { + for _, i := range s.mark { + n := bytes.Index(p, []byte(i)) + if n != -1 { + return n + } + } + return -1 +} + +func (s *CommentReader) Comment() io.Reader { + return io.MultiReader(bytes.NewReader(s.over), s.r) +} diff --git a/comment_test.go b/comment_test.go new file mode 100644 index 0000000..32689a1 --- /dev/null +++ b/comment_test.go @@ -0,0 +1,49 @@ +package readers + +import ( + "github.com/stretchr/testify/assert" + "io" + "strings" + "testing" +) + +func TestNewCommentReader(t *testing.T) { + t.Run("normal", func(t *testing.T) { + r := NewCommentReader(strings.NewReader("Hello world! # this is a comment"), []string{"#"}) + all, err := io.ReadAll(r) + assert.NoError(t, err) + assert.Equal(t, "Hello world! ", string(all)) + all, err = io.ReadAll(r.Comment()) + assert.NoError(t, err) + assert.Equal(t, "# this is a comment", string(all)) + }) + t.Run("overflow start", func(t *testing.T) { + r := NewCommentReader(strings.NewReader(strings.Repeat("Hello world!", 2048)+" # this is a comment"), []string{"#"}) + all, err := io.ReadAll(r) + assert.NoError(t, err) + assert.Equal(t, strings.Repeat("Hello world!", 2048)+" ", string(all)) + all, err = io.ReadAll(r.Comment()) + assert.NoError(t, err) + assert.Equal(t, "# this is a comment", string(all)) + }) + t.Run("overflow comment", func(t *testing.T) { + r := NewCommentReader(strings.NewReader(strings.Repeat("Hello world!", 2048)+" # "+strings.Repeat("this is a comment", 2048)), []string{"#"}) + all, err := io.ReadAll(r) + assert.NoError(t, err) + assert.Equal(t, strings.Repeat("Hello world!", 2048)+" ", string(all)) + all, err = io.ReadAll(r.Comment()) + assert.NoError(t, err) + assert.Equal(t, "# "+strings.Repeat("this is a comment", 2048), string(all)) + }) +} + +func FuzzCommentReader(f *testing.F) { + f.Fuzz(func(t *testing.T, a string) { + r := NewCommentReader(strings.NewReader(a), []string{"#"}) + b1, err := io.ReadAll(r) + assert.NoError(t, err) + b2, err := io.ReadAll(r.Comment()) + assert.NoError(t, err) + assert.Equal(t, string(b1)+string(b2), a) + }) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..321ab3a --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module readers + +go 1.21 + +require github.com/stretchr/testify v1.8.4 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..fa4b6e6 --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/line.go b/line.go new file mode 100644 index 0000000..f932e3e --- /dev/null +++ b/line.go @@ -0,0 +1,81 @@ +package readers + +import ( + "bytes" + "errors" + "io" +) + +var EOL error = &eolError{} + +type eolError struct{} + +func (e *eolError) Error() string { return "EOL" } +func (e *eolError) Unwrap() error { return io.EOF } + +type LineReader struct { + err error + r io.Reader + over []byte +} + +var _ io.Reader = &LineReader{} + +func NewLineReader(r io.Reader) *LineReader { + return &LineReader{nil, r, nil} +} + +func (l *LineReader) Err() error { + if errors.Is(l.err, io.EOF) { + return nil + } + return l.err +} + +func (l *LineReader) Next() bool { + if l.err == nil || errors.Is(l.err, EOL) { + l.err = nil + return true + } + return false +} + +func (l *LineReader) Read(p []byte) (n int, err error) { + if errors.Is(l.err, EOL) { + return 0, io.EOF + } + + rOver := bytes.NewBuffer(l.over) + r := io.MultiReader(rOver, l.r) + n, l.err = r.Read(p) + if l.err != nil { + err = eolToEof(l.err) + return + } + nFull := n + + nStart := n + n2 := bytes.IndexByte(p, '\n') + if n2 != -1 { + l.err = EOL + n = n2 + nStart = n2 + 1 + } + + if n > 0 && p[n-1] == '\r' { + n-- + } + + l.over = p[nStart:nFull] + if rOver.Len() > 0 { + l.over = append(p[nStart:nFull], rOver.Bytes()...) + } + return +} + +func eolToEof(err error) error { + if errors.Is(err, EOL) { + return io.EOF + } + return err +} diff --git a/line_test.go b/line_test.go new file mode 100644 index 0000000..aa602a3 --- /dev/null +++ b/line_test.go @@ -0,0 +1,45 @@ +package readers + +import ( + "bufio" + "fmt" + "github.com/stretchr/testify/assert" + "go/scanner" + "io" + "strings" + "testing" +) + +func TestNewLineReader(t *testing.T) { + t.Run("normal", func(t *testing.T) { + r := NewLineReader(strings.NewReader("Hello world!\nHello new line!\n")) + for r.Next() { + a, err := io.ReadAll(r) + fmt.Println(len(a), string(a)) + assert.NoError(t, err) + } + assert.NoError(t, r.Err()) + }) + t.Run("really long", func(t *testing.T) { + r := NewLineReader(strings.NewReader(strings.Repeat("Hello world! ", 1024) + "\na\n")) + for r.Next() { + a, err := io.ReadAll(r) + fmt.Println(len(a), string(a)) + assert.NoError(t, err) + } + assert.NoError(t, r.Err()) + }) + t.Run("against scanner", func(t *testing.T) { + r := NewLineReader(strings.NewReader(strings.Repeat("Hello world! ", 409600) + "\na\n")) + for r.Next() { + a, err := io.ReadAll(r) + fmt.Println(len(a), string(a)) + assert.NoError(t, err) + } + assert.NoError(t, r.Err()) + + sc := bufio.NewScanner(strings.NewReader(strings.Repeat("Hello world! ", 409600) + "\na\n")) + assert.False(t, sc.Scan()) + assert.Error(t, sc.Err(), scanner.Error{}) + }) +}