Check-in [e1778aba98]
Overview
Comment:Fixed SizedWriter behavior so that it follows io.Writer's Write(...) contract. Added more tests for 100% CC on the ioutil package. Predictor's compressor now uses SizedWriter and no longer has to do any internal buffering.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: e1778aba9806c59c858de655886f0802045e6e7a
User & Date: spaskalev on 2014-12-24 21:40:11
Other Links: manifest | tags
Context
2014-12-24
22:32
Use 0dev.org/ioutil.SizedWriter as an output buffer for pdc in both compress and decompress modes. check-in: 1717bfae3b user: spaskalev tags: trunk
21:40
Fixed SizedWriter behavior so that it follows io.Writer's Write(...) contract. Added more tests for 100% CC on the ioutil package. Predictor's compressor now uses SizedWriter and no longer has to do any internal buffering. check-in: e1778aba98 user: spaskalev tags: trunk
19:07
Implemented ioutil/SizedWriter. CC at 100%. check-in: c28a763d5e user: spaskalev tags: trunk
Changes

Modified src/0dev.org/ioutil/ioutil.go from [9da21d39e0] to [bca057263b].

45
46
47
48
49
50
51

52







53
54
55
56
57
58
59
			buffer = buffer[:copy(buffer, buffer[count:])]

			return 0, err
		}

		// Delegate to the writer if the size is right
		if len(buffer) == 0 && len(input) >= size {

			return writer.Write(input)







		}

		// Append data to the buffer
		count = copy(buffer[len(buffer):size], input)
		buffer = buffer[:len(buffer)+count]

		// Return if we don't have enough bytes to write







>
|
>
>
>
>
>
>
>







45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
			buffer = buffer[:copy(buffer, buffer[count:])]

			return 0, err
		}

		// Delegate to the writer if the size is right
		if len(buffer) == 0 && len(input) >= size {
			reduced := (len(input) / size) * size
			count, err = writer.Write(input[:reduced])
			if count < reduced || err != nil {
				return count, err
			}

			// Stage any remaining data in the buffer
			buffer = append(buffer, input[count:]...)
			return len(input), nil
		}

		// Append data to the buffer
		count = copy(buffer[len(buffer):size], input)
		buffer = buffer[:len(buffer)+count]

		// Return if we don't have enough bytes to write

Modified src/0dev.org/ioutil/ioutil_test.go from [ac59af05f3] to [91aaac4713].

93
94
95
96
97
98
99






























100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

















117
118
119
120
121
122
123
		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "1234567890" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}
}































func TestSizedWriterError(t *testing.T) {
	var (
		errorWriter io.Writer = WriterFunc(func([]byte) (int, error) {
			return 1, errors.New("Invalid write")
		})
		writer io.Writer = SizedWriter(errorWriter, 2)
	)

	count, err := writer.Write([]byte("1"))
	if count != 1 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}

	count, err = writer.Write([]byte("2"))

















	if count != 1 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err == nil {
		t.Error("Unexpected lack of error from SizedWriter")
	}
}







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
















>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "1234567890" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}
}

func TestSizeWriterLarger(t *testing.T) {
	var (
		input  []byte = []byte("0123456789AB")
		buffer bytes.Buffer
		writer = SizedWriter(&buffer, 8)
	)

	count, err := writer.Write(input)
	if count != 12 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "01234567" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}

	count, err = writer.Write(nil)
	if count != 0 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "0123456789AB" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}
}

func TestSizedWriterError1(t *testing.T) {
	var (
		errorWriter io.Writer = WriterFunc(func([]byte) (int, error) {
			return 1, errors.New("Invalid write")
		})
		writer io.Writer = SizedWriter(errorWriter, 2)
	)

	count, err := writer.Write([]byte("1"))
	if count != 1 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}

	count, err = writer.Write([]byte("2"))
	if count != 1 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err == nil {
		t.Error("Unexpected lack of error from SizedWriter")
	}
}

func TestSizedWriterError2(t *testing.T) {
	var (
		errorWriter io.Writer = WriterFunc(func([]byte) (int, error) {
			return 1, errors.New("Invalid write")
		})
		writer io.Writer = SizedWriter(errorWriter, 1)
	)

	count, err := writer.Write([]byte("12"))
	if count != 1 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err == nil {
		t.Error("Unexpected lack of error from SizedWriter")
	}
}

Modified src/0dev.org/predictor/predictor.go from [1bc0c5d728] to [43e7f751ed].

26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
// Returns an io.Writer implementation that wraps the provided io.Writer
// and compresses data according to the predictor algorithm
//
// It can buffer data as the predictor mandates 8-byte blocks with a header.
// A call with no data will force a flush.
func Compressor(writer io.Writer) io.Writer {
	var ctx context
	ctx.input = make([]byte, 0, 8)

	// Forward declaration as it is required for recursion
	var write iou.WriterFunc

	write = func(data []byte) (int, error) {
		var (
			blockSize    int = 8
			bufferLength int = len(ctx.input)
			datalength   int = len(data)
		)

		// Force a flush if we are called with no data to write
		if datalength == 0 {
			// Nothing to flush if the buffer is empty though
			if len(ctx.input) == 0 {
				return 0, nil
			}
			// We can't have more than 7 bytes in the buffer so this is safe
			data, datalength = ctx.input, len(ctx.input)
			blockSize, bufferLength = datalength, 0
		}

		// Check if there are pending bytes in the buffer
		if datalength < blockSize || bufferLength > 0 {

			// If the current buffer + new data can fit into a block
			if (datalength + bufferLength) <= blockSize {
				ctx.input = append(ctx.input, data...)

				// Flush the block if the buffer fills it
				if len(ctx.input) == blockSize {
					return write(nil)
				}
				// ... otherwise just return
				return datalength, nil
			}

			// The current buffer + new data overflow the block size
			// Complete the block, flush it ...
			ctx.input = append(ctx.input, data[:blockSize-bufferLength]...)
			if c, err := write(nil); err != nil {
				return c, err
			}
			// ... and stage the rest of the data in the buffer
			ctx.input = append(ctx.input, data[blockSize-bufferLength:]...)
			return datalength, nil
		}

		var buf []byte = make([]byte, 1, blockSize+1)
		for block := 0; block < datalength/blockSize; block++ {
			for i := 0; i < blockSize; i++ {
				var current byte = data[(block*blockSize)+i]
				if ctx.table[ctx.hash] == current {







<

<
<
<
|

|
<
|


<

<
<
|
|
<
<
<
|
<
<
|
<
<
<
<
<
<
<
<
<
<
|
<
<
<
<
<
<
<
<
<
<
<







26
27
28
29
30
31
32

33



34
35
36

37
38
39

40


41
42



43


44










45











46
47
48
49
50
51
52
// Returns an io.Writer implementation that wraps the provided io.Writer
// and compresses data according to the predictor algorithm
//
// It can buffer data as the predictor mandates 8-byte blocks with a header.
// A call with no data will force a flush.
func Compressor(writer io.Writer) io.Writer {
	var ctx context





	return iou.SizedWriter(iou.WriterFunc(func(data []byte) (int, error) {
		var (
			blockSize  int = 8

			datalength int = len(data)
		)


		if datalength == 0 {


			return 0, nil
		}






		if datalength < blockSize {










			blockSize = datalength











		}

		var buf []byte = make([]byte, 1, blockSize+1)
		for block := 0; block < datalength/blockSize; block++ {
			for i := 0; i < blockSize; i++ {
				var current byte = data[(block*blockSize)+i]
				if ctx.table[ctx.hash] == current {
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
				return (block * blockSize) + c, err
			}

			// Reset the flags and buffer for the next iteration
			buf, buf[0] = buf[:1], 0
		}

		if remaining := datalength % blockSize; remaining > 0 {
			ctx.input = ctx.input[:remaining]
			copy(ctx.input, data[datalength-remaining:])
		} else {
			ctx.input = ctx.input[:0]
		}

		return datalength, nil
	}

	return write
}

// Returns an io.Reader implementation that wraps the provided io.Reader
// and decompresses data according to the predictor algorithm
func Decompressor(reader io.Reader) io.Reader {
	var ctx context
	ctx.input = make([]byte, 0, 8)







<
<
<
<
<
<
<

|
<
<







64
65
66
67
68
69
70







71
72


73
74
75
76
77
78
79
				return (block * blockSize) + c, err
			}

			// Reset the flags and buffer for the next iteration
			buf, buf[0] = buf[:1], 0
		}








		return datalength, nil
	}), 8)


}

// Returns an io.Reader implementation that wraps the provided io.Reader
// and decompresses data according to the predictor algorithm
func Decompressor(reader io.Reader) io.Reader {
	var ctx context
	ctx.input = make([]byte, 0, 8)

Modified src/0dev.org/predictor/predictor_test.go from [d9c16d84c9] to [0fb2d9a4c8].

78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
			t.Error(err)
		}
	}
}

func TestStepCycle(t *testing.T) {
	for i := 0; i < len(testData); i++ {
		for j := 1; j < len(testData); j++ {
			if err := cycle(testData[i], j); err != nil {
				t.Error("Error for testData[", i, "], step[", j, "] ", err)
			}
		}
	}
}








|







78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
			t.Error(err)
		}
	}
}

func TestStepCycle(t *testing.T) {
	for i := 0; i < len(testData); i++ {
		for j := 1; j < len(testData[i]); j++ {
			if err := cycle(testData[i], j); err != nil {
				t.Error("Error for testData[", i, "], step[", j, "] ", err)
			}
		}
	}
}

111
112
113
114
115
116
117

118
119
120
121
122
123
124

			trace = append(trace, data[:step]...)

			_, err = compressor.Write(data[:step])
			if err != nil {
				return err
			}

			data = data[step:]
		} else {
			step = len(data)
		}
	}

	// Flush the compressor







>







111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

			trace = append(trace, data[:step]...)

			_, err = compressor.Write(data[:step])
			if err != nil {
				return err
			}

			data = data[step:]
		} else {
			step = len(data)
		}
	}

	// Flush the compressor
136
137
138
139
140
141
142
143
144
145
146
147
148
149

	// Diff the result against the initial input
	delta := diff.Diff(diff.D{len(input), len(decompressed),
		func(i, j int) bool { return input[i] == decompressed[j] }})

	// Return a well-formated error if any differences are found
	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
		return fmt.Errorf("Unexpected decompressed output %v\ninput:  (%d) %#x\ntrace:  (%d) %#x\noutput: (%d) %#x\n",
			delta, len(input), input, len(trace), trace, len(decompressed), decompressed)
	}

	// All is good :)
	return nil
}







|
|





137
138
139
140
141
142
143
144
145
146
147
148
149
150

	// Diff the result against the initial input
	delta := diff.Diff(diff.D{len(input), len(decompressed),
		func(i, j int) bool { return input[i] == decompressed[j] }})

	// Return a well-formated error if any differences are found
	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
		return fmt.Errorf("Unexpected decompressed output for step %d, delta %v\ninput:  (%d) %#x\ntrace:  (%d) %#x\noutput: (%d) %#x\n",
			step, delta, len(input), input, len(trace), trace, len(decompressed), decompressed)
	}

	// All is good :)
	return nil
}