Check-in [e1778aba98]
Overview
SHA1:e1778aba9806c59c858de655886f0802045e6e7a
Date: 2014-12-24 21:40:11
User: spaskalev
Comment:Fixed SizedWriter behavior so that it follows io.Writer's Write(...) contract. Added more tests for 100% CC on the ioutil package. Predictor's compressor now uses SizedWriter and no longer has to do any internal buffering.
Timelines: family | ancestors | descendants | both | trunk
Downloads: Tarball | ZIP archive
Other Links: files | file ages | folders | manifest
Tags And Properties
Context
2014-12-24
22:32
[1717bfae3b] Use 0dev.org/ioutil.SizedWriter as an output buffer for pdc in both compress and decompress modes. (user: spaskalev, tags: trunk)
21:40
[e1778aba98] Fixed SizedWriter behavior so that it follows io.Writer's Write(...) contract. Added more tests for 100% CC on the ioutil package. Predictor's compressor now uses SizedWriter and no longer has to do any internal buffering. (user: spaskalev, tags: trunk)
19:07
[c28a763d5e] Implemented ioutil/SizedWriter. CC at 100%. (user: spaskalev, tags: trunk)
Changes

Modified src/0dev.org/ioutil/ioutil.go from [9da21d39e0] to [bca057263b].

45
46
47
48
49
50
51

52







53
54
55
56
57
58
59
			buffer = buffer[:copy(buffer, buffer[count:])]

			return 0, err
		}

		// Delegate to the writer if the size is right
		if len(buffer) == 0 && len(input) >= size {

			return writer.Write(input)







		}

		// Append data to the buffer
		count = copy(buffer[len(buffer):size], input)
		buffer = buffer[:len(buffer)+count]

		// Return if we don't have enough bytes to write







>
|
>
>
>
>
>
>
>







45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
			buffer = buffer[:copy(buffer, buffer[count:])]

			return 0, err
		}

		// Delegate to the writer if the size is right
		if len(buffer) == 0 && len(input) >= size {
			reduced := (len(input) / size) * size
			count, err = writer.Write(input[:reduced])
			if count < reduced || err != nil {
				return count, err
			}

			// Stage any remaining data in the buffer
			buffer = append(buffer, input[count:]...)
			return len(input), nil
		}

		// Append data to the buffer
		count = copy(buffer[len(buffer):size], input)
		buffer = buffer[:len(buffer)+count]

		// Return if we don't have enough bytes to write

Modified src/0dev.org/ioutil/ioutil_test.go from [ac59af05f3] to [91aaac4713].

93
94
95
96
97
98
99






























100
101
102
103
104
105
106
107
...
110
111
112
113
114
115
116

















117
118
119
120
121
122
123
		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "1234567890" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}
}































func TestSizedWriterError(t *testing.T) {
	var (
		errorWriter io.Writer = WriterFunc(func([]byte) (int, error) {
			return 1, errors.New("Invalid write")
		})
		writer io.Writer = SizedWriter(errorWriter, 2)
	)

................................................................................
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}

	count, err = writer.Write([]byte("2"))

















	if count != 1 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err == nil {
		t.Error("Unexpected lack of error from SizedWriter")
	}
}







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
...
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "1234567890" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}
}

func TestSizeWriterLarger(t *testing.T) {
	var (
		input  []byte = []byte("0123456789AB")
		buffer bytes.Buffer
		writer = SizedWriter(&buffer, 8)
	)

	count, err := writer.Write(input)
	if count != 12 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "01234567" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}

	count, err = writer.Write(nil)
	if count != 0 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "0123456789AB" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}
}

func TestSizedWriterError1(t *testing.T) {
	var (
		errorWriter io.Writer = WriterFunc(func([]byte) (int, error) {
			return 1, errors.New("Invalid write")
		})
		writer io.Writer = SizedWriter(errorWriter, 2)
	)

................................................................................
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}

	count, err = writer.Write([]byte("2"))
	if count != 1 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err == nil {
		t.Error("Unexpected lack of error from SizedWriter")
	}
}

func TestSizedWriterError2(t *testing.T) {
	var (
		errorWriter io.Writer = WriterFunc(func([]byte) (int, error) {
			return 1, errors.New("Invalid write")
		})
		writer io.Writer = SizedWriter(errorWriter, 1)
	)

	count, err := writer.Write([]byte("12"))
	if count != 1 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err == nil {
		t.Error("Unexpected lack of error from SizedWriter")
	}
}

Modified src/0dev.org/predictor/predictor.go from [1bc0c5d728] to [43e7f751ed].

26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
..
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

116
117
118
119
120
121
122
// Returns an io.Writer implementation that wraps the provided io.Writer
// and compresses data according to the predictor algorithm
//
// It can buffer data as the predictor mandates 8-byte blocks with a header.
// A call with no data will force a flush.
func Compressor(writer io.Writer) io.Writer {
	var ctx context
	ctx.input = make([]byte, 0, 8)

	// Forward declaration as it is required for recursion
	var write iou.WriterFunc

	write = func(data []byte) (int, error) {
		var (
			blockSize    int = 8
			bufferLength int = len(ctx.input)
			datalength   int = len(data)
		)

		// Force a flush if we are called with no data to write
		if datalength == 0 {
			// Nothing to flush if the buffer is empty though
			if len(ctx.input) == 0 {
				return 0, nil
			}
			// We can't have more than 7 bytes in the buffer so this is safe
			data, datalength = ctx.input, len(ctx.input)
			blockSize, bufferLength = datalength, 0
		}

		// Check if there are pending bytes in the buffer
		if datalength < blockSize || bufferLength > 0 {

			// If the current buffer + new data can fit into a block
			if (datalength + bufferLength) <= blockSize {
				ctx.input = append(ctx.input, data...)

				// Flush the block if the buffer fills it
				if len(ctx.input) == blockSize {
					return write(nil)
				}
				// ... otherwise just return
				return datalength, nil
			}

			// The current buffer + new data overflow the block size
			// Complete the block, flush it ...
			ctx.input = append(ctx.input, data[:blockSize-bufferLength]...)
			if c, err := write(nil); err != nil {
				return c, err
			}
			// ... and stage the rest of the data in the buffer
			ctx.input = append(ctx.input, data[blockSize-bufferLength:]...)
			return datalength, nil
		}

		var buf []byte = make([]byte, 1, blockSize+1)
		for block := 0; block < datalength/blockSize; block++ {
			for i := 0; i < blockSize; i++ {
				var current byte = data[(block*blockSize)+i]
				if ctx.table[ctx.hash] == current {
................................................................................
				return (block * blockSize) + c, err
			}

			// Reset the flags and buffer for the next iteration
			buf, buf[0] = buf[:1], 0
		}

		if remaining := datalength % blockSize; remaining > 0 {
			ctx.input = ctx.input[:remaining]
			copy(ctx.input, data[datalength-remaining:])
		} else {
			ctx.input = ctx.input[:0]
		}

		return datalength, nil
	}

	return write

}

// Returns an io.Reader implementation that wraps the provided io.Reader
// and decompresses data according to the predictor algorithm
func Decompressor(reader io.Reader) io.Reader {
	var ctx context
	ctx.input = make([]byte, 0, 8)







<

<
<
<
|

|
<
|


<

<
<
|
|
<
<
<
|
<
<
|
<
<
<
<
<
<
<
<
<
<
|
<
<
<
<
<
<
<
<
<
<
<







 







<
<
<
<
<
<
<

<
<
<
>







26
27
28
29
30
31
32

33



34
35
36

37
38
39

40


41
42



43


44










45











46
47
48
49
50
51
52
..
64
65
66
67
68
69
70







71



72
73
74
75
76
77
78
79
// Returns an io.Writer implementation that wraps the provided io.Writer
// and compresses data according to the predictor algorithm
//
// It can buffer data as the predictor mandates 8-byte blocks with a header.
// A call with no data will force a flush.
func Compressor(writer io.Writer) io.Writer {
	var ctx context





	return iou.SizedWriter(iou.WriterFunc(func(data []byte) (int, error) {
		var (
			blockSize  int = 8

			datalength int = len(data)
		)


		if datalength == 0 {


			return 0, nil
		}






		if datalength < blockSize {










			blockSize = datalength











		}

		var buf []byte = make([]byte, 1, blockSize+1)
		for block := 0; block < datalength/blockSize; block++ {
			for i := 0; i < blockSize; i++ {
				var current byte = data[(block*blockSize)+i]
				if ctx.table[ctx.hash] == current {
................................................................................
				return (block * blockSize) + c, err
			}

			// Reset the flags and buffer for the next iteration
			buf, buf[0] = buf[:1], 0
		}








		return datalength, nil



	}), 8)
}

// Returns an io.Reader implementation that wraps the provided io.Reader
// and decompresses data according to the predictor algorithm
func Decompressor(reader io.Reader) io.Reader {
	var ctx context
	ctx.input = make([]byte, 0, 8)

Modified src/0dev.org/predictor/predictor_test.go from [d9c16d84c9] to [0fb2d9a4c8].

78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
...
111
112
113
114
115
116
117

118
119
120
121
122
123
124
...
136
137
138
139
140
141
142
143
144
145
146
147
148
149
			t.Error(err)
		}
	}
}

func TestStepCycle(t *testing.T) {
	for i := 0; i < len(testData); i++ {
		for j := 1; j < len(testData); j++ {
			if err := cycle(testData[i], j); err != nil {
				t.Error("Error for testData[", i, "], step[", j, "] ", err)
			}
		}
	}
}

................................................................................

			trace = append(trace, data[:step]...)

			_, err = compressor.Write(data[:step])
			if err != nil {
				return err
			}

			data = data[step:]
		} else {
			step = len(data)
		}
	}

	// Flush the compressor
................................................................................

	// Diff the result against the initial input
	delta := diff.Diff(diff.D{len(input), len(decompressed),
		func(i, j int) bool { return input[i] == decompressed[j] }})

	// Return a well-formated error if any differences are found
	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
		return fmt.Errorf("Unexpected decompressed output %v\ninput:  (%d) %#x\ntrace:  (%d) %#x\noutput: (%d) %#x\n",
			delta, len(input), input, len(trace), trace, len(decompressed), decompressed)
	}

	// All is good :)
	return nil
}







|







 







>







 







|
|





78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
...
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
...
137
138
139
140
141
142
143
144
145
146
147
148
149
150
			t.Error(err)
		}
	}
}

func TestStepCycle(t *testing.T) {
	for i := 0; i < len(testData); i++ {
		for j := 1; j < len(testData[i]); j++ {
			if err := cycle(testData[i], j); err != nil {
				t.Error("Error for testData[", i, "], step[", j, "] ", err)
			}
		}
	}
}

................................................................................

			trace = append(trace, data[:step]...)

			_, err = compressor.Write(data[:step])
			if err != nil {
				return err
			}

			data = data[step:]
		} else {
			step = len(data)
		}
	}

	// Flush the compressor
................................................................................

	// Diff the result against the initial input
	delta := diff.Diff(diff.D{len(input), len(decompressed),
		func(i, j int) bool { return input[i] == decompressed[j] }})

	// Return a well-formated error if any differences are found
	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
		return fmt.Errorf("Unexpected decompressed output for step %d, delta %v\ninput:  (%d) %#x\ntrace:  (%d) %#x\noutput: (%d) %#x\n",
			step, delta, len(input), input, len(trace), trace, len(decompressed), decompressed)
	}

	// All is good :)
	return nil
}