Check-in [c28a763d5e]
Overview
Comment:Implemented ioutil/SizedWriter. CC at 100%.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: c28a763d5e932739911cbb29f9e01b1c2c2b7289
User & Date: spaskalev on 2014-12-24 19:07:41
Other Links: manifest | tags
Context
2014-12-24
21:40
Fixed SizedWriter behavior so that it follows io.Writer's Write(...) contract. Added more tests for 100% CC on the ioutil package. Predictor's compressor now uses SizedWriter and no longer has to do any internal buffering. check-in: e1778aba98 user: spaskalev tags: trunk
19:07
Implemented ioutil/SizedWriter. CC at 100%. check-in: c28a763d5e user: spaskalev tags: trunk
08:24
Added an explicit copyright notice in the code. check-in: ffd1ab7b0c user: spaskalev tags: trunk
Changes

Modified src/0dev.org/ioutil/ioutil.go from [98f8caa018] to [9da21d39e0].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23





















































24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
// Package ioutil contains various constructs for io operations
package ioutil

import (
	"io"
)

// An function alias type that implements io.Writer
type WriterFunc func([]byte) (int, error)

// Delegates the call to the WriterFunc while implementing io.Writer
func (w WriterFunc) Write(b []byte) (int, error) {
	return w(b)
}

// An function alias type that implements io.Reader
type ReaderFunc func([]byte) (int, error)

// Delegates the call to the WriterFunc while implementing io.Reader
func (r ReaderFunc) Read(b []byte) (int, error) {
	return r(b)
}






















































// Returns a reader that delegates calls to Read(...) while ensuring
// that the output buffer is never smaller than the required size
// and is downsized to a multiple of the required size if larger
func SizedReader(reader io.Reader, size int) io.Reader {
	var buffer []byte = make([]byte, 0, size)

	return ReaderFunc(func(output []byte) (int, error) {
		var (
			readCount int
			err       error
		)
	start:
		// Reply with the buffered data if there is any
		if len(buffer) > 0 {
			readCount = copy(output, buffer)

			// Advance the data in the buffer
			buffer = buffer[:copy(buffer, buffer[readCount:])]

			// Return count and error if we have read the whole buffer
			if len(buffer) == 0 {
				return readCount, err
			}

			// Do not propagate an error until the buffer is exhausted
			return readCount, nil
		}

		// Delegate if the buffer is empty and the destination buffer is large enough
		if len(output) >= size {
			return reader.Read(output[:(len(output)/size)*size])
		}

		// Perform a read into the buffer
		readCount, err = reader.Read(buffer[:size])

		// Size the buffer down to the read data size
		// and restart if we have successfully read some bytes
		buffer = buffer[:readCount]
		if len(buffer) > 0 {
			goto start
		}

		// Returning on err/misbehaving noop reader
		return 0, err
	})
}
|






|


|




|


|




>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


|





|
|




|


|



|



|








|



|








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
// Package ioutil contains various constructs for io operations.
package ioutil

import (
	"io"
)

// An function alias type that implements io.Writer.
type WriterFunc func([]byte) (int, error)

// Delegates the call to the WriterFunc while implementing io.Writer.
func (w WriterFunc) Write(b []byte) (int, error) {
	return w(b)
}

// An function alias type that implements io.Reader.
type ReaderFunc func([]byte) (int, error)

// Delegates the call to the WriterFunc while implementing io.Reader.
func (r ReaderFunc) Read(b []byte) (int, error) {
	return r(b)
}

// Returns a writer that delegates calls to Write(...) while ensuring
// that it is never called with less bytes than the specified amount.
//
// Calls with fewer bytes are buffered while a call with a nil slice
// causes the buffer to be flushed to the underlying writer.
func SizedWriter(writer io.Writer, size int) io.Writer {
	var buffer []byte = make([]byte, 0, size)
	var write WriterFunc

	write = func(input []byte) (int, error) {
		var (
			count int
			err   error
		)

		// Flush the buffer when called with no bytes to write
		if input == nil {
			// Call the writer with whatever we have in store..
			count, err = writer.Write(buffer)

			// Advance the buffer
			buffer = buffer[:copy(buffer, buffer[count:])]

			return 0, err
		}

		// Delegate to the writer if the size is right
		if len(buffer) == 0 && len(input) >= size {
			return writer.Write(input)
		}

		// Append data to the buffer
		count = copy(buffer[len(buffer):size], input)
		buffer = buffer[:len(buffer)+count]

		// Return if we don't have enough bytes to write
		if len(buffer) < size {
			return len(input), nil
		}

		// Flush the buffer as it is filled
		_, err = write(nil)
		if err != nil {
			return count, err
		}

		// Handle the rest of the input
		return write(input[count:])
	}

	return write
}

// Returns a reader that delegates calls to Read(...) while ensuring
// that the output buffer is never smaller than the required size
// and is downsized to a multiple of the required size if larger.
func SizedReader(reader io.Reader, size int) io.Reader {
	var buffer []byte = make([]byte, 0, size)

	return ReaderFunc(func(output []byte) (int, error) {
		var (
			count int
			err   error
		)
	start:
		// Reply with the buffered data if there is any
		if len(buffer) > 0 {
			count = copy(output, buffer)

			// Advance the data in the buffer
			buffer = buffer[:copy(buffer, buffer[count:])]

			// Return count and error if we have read the whole buffer
			if len(buffer) == 0 {
				return count, err
			}

			// Do not propagate an error until the buffer is exhausted
			return count, nil
		}

		// Delegate if the buffer is empty and the destination buffer is large enough
		if len(output) >= size {
			return reader.Read(output[:(len(output)/size)*size])
		}

		// Perform a read into the buffer
		count, err = reader.Read(buffer[:size])

		// Size the buffer down to the read data size
		// and restart if we have successfully read some bytes
		buffer = buffer[:count]
		if len(buffer) > 0 {
			goto start
		}

		// Returning on err/misbehaving noop reader
		return 0, err
	})
}

Modified src/0dev.org/ioutil/ioutil_test.go from [f071d3739a] to [ac59af05f3].

1
2
3
4
5

6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49










































































50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
package ioutil

import (
	diff "0dev.org/diff"
	"bytes"

	"io"
	"testing"
)

func TestWriter(t *testing.T) {
	var (
		input  []byte = []byte{0, 1, 2, 3, 4, 5, 6, 7}
		output []byte

		reader *bytes.Reader = bytes.NewReader(input)
		buffer bytes.Buffer
	)

	reader.WriteTo(WriterFunc(buffer.Write))
	output = buffer.Bytes()

	// Diff the result against the initial input
	delta := diff.Diff(diff.D{len(input), len(output),
		func(i, j int) bool { return input[i] == output[j] }})
	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
		t.Error("Differences detected ", delta)
	}
}

func TestReader(t *testing.T) {
	var (
		input  []byte = []byte{0, 1, 2, 3, 4, 5, 6, 7}
		output []byte

		reader *bytes.Reader = bytes.NewReader(input)
		buffer bytes.Buffer
	)

	buffer.ReadFrom(ReaderFunc(reader.Read))
	output = buffer.Bytes()

	// Diff the result against the initial input
	delta := diff.Diff(diff.D{len(input), len(output),
		func(i, j int) bool { return input[i] == output[j] }})
	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
		t.Error("Differences detected ", delta)
	}
}











































































func TestBlockReader(t *testing.T) {
	var (
		input  []byte = []byte{0, 1, 2, 3, 4, 5, 6, 7}
		output []byte = make([]byte, 16)

		reader *bytes.Reader = bytes.NewReader(input)
		min    io.Reader     = SizedReader(reader, 4)
	)

	// Expecting a read count of 2
	count, err := min.Read(output[:2])
	if count != 2 {
		t.Error("Invalid read count from MinReader", count)
	}
	if err != nil {
		t.Error("Unexpected error from MinReader", err)
	}

	// Expecting a read count of 2 as it should have 2 bytes in its buffer
	count, err = min.Read(output[:3])
	if count != 2 {
		t.Error("Invalid read count from MinReader", count)
	}
	if err != nil {
		t.Error("Unexpected error from MinReader", err)
	}

	// Expecting a read count of 4 as the buffer should be empty
	count, err = min.Read(output[:4])
	if count != 4 {
		t.Error("Invalid read count from MinReader", count)
	}
	if err != nil {
		t.Error("Unexpected error from MinReader", err)
	}

	// Expecting a read count of 0 with an EOF as the buffer should be empty
	count, err = min.Read(output[:1])
	if count != 0 {
		t.Error("Invalid read count from MinReader", count)
	}
	if err != io.EOF {
		t.Error("Unexpected error from MinReader", err)
	}
}





>




|



















|



















>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|











|


|





|


|





|


|





|


|


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
package ioutil

import (
	diff "0dev.org/diff"
	"bytes"
	"errors"
	"io"
	"testing"
)

func TestWriterFunc(t *testing.T) {
	var (
		input  []byte = []byte{0, 1, 2, 3, 4, 5, 6, 7}
		output []byte

		reader *bytes.Reader = bytes.NewReader(input)
		buffer bytes.Buffer
	)

	reader.WriteTo(WriterFunc(buffer.Write))
	output = buffer.Bytes()

	// Diff the result against the initial input
	delta := diff.Diff(diff.D{len(input), len(output),
		func(i, j int) bool { return input[i] == output[j] }})
	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
		t.Error("Differences detected ", delta)
	}
}

func TestReaderFunc(t *testing.T) {
	var (
		input  []byte = []byte{0, 1, 2, 3, 4, 5, 6, 7}
		output []byte

		reader *bytes.Reader = bytes.NewReader(input)
		buffer bytes.Buffer
	)

	buffer.ReadFrom(ReaderFunc(reader.Read))
	output = buffer.Bytes()

	// Diff the result against the initial input
	delta := diff.Diff(diff.D{len(input), len(output),
		func(i, j int) bool { return input[i] == output[j] }})
	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
		t.Error("Differences detected ", delta)
	}
}

func TestSizedWriter(t *testing.T) {
	var (
		buffer bytes.Buffer
		writer io.Writer = SizedWriter(&buffer, 4)
	)

	count, err := writer.Write([]byte("12"))
	if count != 2 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}

	count, err = writer.Write([]byte("3456"))
	if count != 2 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "1234" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}

	// Flush the buffer
	count, err = writer.Write(nil)
	if count != 0 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "123456" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}

	count, err = writer.Write([]byte("7890"))
	if count != 4 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "1234567890" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}
}

func TestSizedWriterError(t *testing.T) {
	var (
		errorWriter io.Writer = WriterFunc(func([]byte) (int, error) {
			return 1, errors.New("Invalid write")
		})
		writer io.Writer = SizedWriter(errorWriter, 2)
	)

	count, err := writer.Write([]byte("1"))
	if count != 1 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}

	count, err = writer.Write([]byte("2"))
	if count != 1 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err == nil {
		t.Error("Unexpected lack of error from SizedWriter")
	}
}

func TestSizedReader(t *testing.T) {
	var (
		input  []byte = []byte{0, 1, 2, 3, 4, 5, 6, 7}
		output []byte = make([]byte, 16)

		reader *bytes.Reader = bytes.NewReader(input)
		min    io.Reader     = SizedReader(reader, 4)
	)

	// Expecting a read count of 2
	count, err := min.Read(output[:2])
	if count != 2 {
		t.Error("Invalid read count from SizedReader", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedReader", err)
	}

	// Expecting a read count of 2 as it should have 2 bytes in its buffer
	count, err = min.Read(output[:3])
	if count != 2 {
		t.Error("Invalid read count from SizedReader", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedReader", err)
	}

	// Expecting a read count of 4 as the buffer should be empty
	count, err = min.Read(output[:4])
	if count != 4 {
		t.Error("Invalid read count from SizedReader", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedReader", err)
	}

	// Expecting a read count of 0 with an EOF as the buffer should be empty
	count, err = min.Read(output[:1])
	if count != 0 {
		t.Error("Invalid read count from SizedReader", count)
	}
	if err != io.EOF {
		t.Error("Unexpected error from SizedReader", err)
	}
}