The Shed
Check-in [e1778aba98]
Login
HomeTimelineFilesBranchesTagsTicketsWiki
Overview
Comment:Fixed SizedWriter behavior so that it follows io.Writer's Write(...) contract. Added more tests for 100% CC on the ioutil package. Predictor's compressor now uses SizedWriter and no longer has to do any internal buffering.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: e1778aba9806c59c858de655886f0802045e6e7a
User & Date: spaskalev on 2014-12-24 21:40:11
Other Links: manifest | tags
Context
2014-12-24
22:32
Use 0dev.org/ioutil.SizedWriter as an output buffer for pdc in both compress and decompress modes. check-in: 1717bfae3b user: spaskalev tags: trunk
21:40
Fixed SizedWriter behavior so that it follows io.Writer's Write(...) contract. Added more tests for 100% CC on the ioutil package. Predictor's compressor now uses SizedWriter and no longer has to do any internal buffering. check-in: e1778aba98 user: spaskalev tags: trunk
19:07
Implemented ioutil/SizedWriter. CC at 100%. check-in: c28a763d5e user: spaskalev tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Modified src/0dev.org/ioutil/ioutil.go from [9da21d39e0] to [bca057263b].

︙
45
46
47
48
49
50
51

52








53
54
55
56
57
58
59

45
46
47
48
49
50
51
52

53
54
55
56
57
58
59
60
61
62
63
64
65
66
67








+
-
+
+
+
+
+
+
+
+








			buffer = buffer[:copy(buffer, buffer[count:])]

			return 0, err
		}

		// Delegate to the writer if the size is right
		if len(buffer) == 0 && len(input) >= size {
			reduced := (len(input) / size) * size
			return writer.Write(input)
			count, err = writer.Write(input[:reduced])
			if count < reduced || err != nil {
				return count, err
			}

			// Stage any remaining data in the buffer
			buffer = append(buffer, input[count:]...)
			return len(input), nil
		}

		// Append data to the buffer
		count = copy(buffer[len(buffer):size], input)
		buffer = buffer[:len(buffer)+count]

		// Return if we don't have enough bytes to write
︙

Modified src/0dev.org/ioutil/ioutil_test.go from [ac59af05f3] to [91aaac4713].

︙
93
94
95
96
97
98
99






























100

101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

















117
118
119
120
121
122
123

93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
















+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "1234567890" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}
}

func TestSizeWriterLarger(t *testing.T) {
	var (
		input  []byte = []byte("0123456789AB")
		buffer bytes.Buffer
		writer = SizedWriter(&buffer, 8)
	)

	count, err := writer.Write(input)
	if count != 12 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "01234567" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}

	count, err = writer.Write(nil)
	if count != 0 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}
	if buffer.String() != "0123456789AB" {
		t.Error("Unexpected value in wrapped writer", buffer.String())
	}
}

func TestSizedWriterError(t *testing.T) {
func TestSizedWriterError1(t *testing.T) {
	var (
		errorWriter io.Writer = WriterFunc(func([]byte) (int, error) {
			return 1, errors.New("Invalid write")
		})
		writer io.Writer = SizedWriter(errorWriter, 2)
	)

	count, err := writer.Write([]byte("1"))
	if count != 1 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err != nil {
		t.Error("Unexpected error from SizedWriter", err)
	}

	count, err = writer.Write([]byte("2"))
	if count != 1 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err == nil {
		t.Error("Unexpected lack of error from SizedWriter")
	}
}

func TestSizedWriterError2(t *testing.T) {
	var (
		errorWriter io.Writer = WriterFunc(func([]byte) (int, error) {
			return 1, errors.New("Invalid write")
		})
		writer io.Writer = SizedWriter(errorWriter, 1)
	)

	count, err := writer.Write([]byte("12"))
	if count != 1 {
		t.Error("Unexpected write count from SizedWriter", count)
	}
	if err == nil {
		t.Error("Unexpected lack of error from SizedWriter")
	}
}
︙

Modified src/0dev.org/predictor/predictor.go from [1bc0c5d728] to [43e7f751ed].

︙
26
27
28
29
30
31
32
33
34
35
36
37
38

39
40

41
42

43
44
45
46
47
48
49
50


51
52
53
54

55
56
57

58
59
60
61
62
63
64
65
66
67
68

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86

26
27
28
29
30
31
32

33




34
35

36


37
38
39

40




41
42




43



44











45











46
47
48
49
50
51
52








-

-
-
-
-
+

-
+
-
-
+


-

-
-
-
-
+
+
-
-
-
-
+
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-
-








// Returns an io.Writer implementation that wraps the provided io.Writer
// and compresses data according to the predictor algorithm
//
// It can buffer data as the predictor mandates 8-byte blocks with a header.
// A call with no data will force a flush.
func Compressor(writer io.Writer) io.Writer {
	var ctx context
	ctx.input = make([]byte, 0, 8)

	// Forward declaration as it is required for recursion
	var write iou.WriterFunc

	write = func(data []byte) (int, error) {
	return iou.SizedWriter(iou.WriterFunc(func(data []byte) (int, error) {
		var (
			blockSize    int = 8
			blockSize  int = 8
			bufferLength int = len(ctx.input)
			datalength   int = len(data)
			datalength int = len(data)
		)

		// Force a flush if we are called with no data to write
		if datalength == 0 {
			// Nothing to flush if the buffer is empty though
			if len(ctx.input) == 0 {
				return 0, nil
			}
			return 0, nil
		}
			// We can't have more than 7 bytes in the buffer so this is safe
			data, datalength = ctx.input, len(ctx.input)
			blockSize, bufferLength = datalength, 0
		}


		// Check if there are pending bytes in the buffer
		if datalength < blockSize || bufferLength > 0 {
		if datalength < blockSize {

			// If the current buffer + new data can fit into a block
			if (datalength + bufferLength) <= blockSize {
				ctx.input = append(ctx.input, data...)

				// Flush the block if the buffer fills it
				if len(ctx.input) == blockSize {
					return write(nil)
				}
				// ... otherwise just return
				return datalength, nil
			blockSize = datalength
			}

			// The current buffer + new data overflow the block size
			// Complete the block, flush it ...
			ctx.input = append(ctx.input, data[:blockSize-bufferLength]...)
			if c, err := write(nil); err != nil {
				return c, err
			}
			// ... and stage the rest of the data in the buffer
			ctx.input = append(ctx.input, data[blockSize-bufferLength:]...)
			return datalength, nil
		}

		var buf []byte = make([]byte, 1, blockSize+1)
		for block := 0; block < datalength/blockSize; block++ {
			for i := 0; i < blockSize; i++ {
				var current byte = data[(block*blockSize)+i]
				if ctx.table[ctx.hash] == current {
︙
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

114
115
116
117
118
119
120
121
122

64
65
66
67
68
69
70







71

72


73
74
75
76
77
78
79








-
-
-
-
-
-
-

-
+
-
-








				return (block * blockSize) + c, err
			}

			// Reset the flags and buffer for the next iteration
			buf, buf[0] = buf[:1], 0
		}

		if remaining := datalength % blockSize; remaining > 0 {
			ctx.input = ctx.input[:remaining]
			copy(ctx.input, data[datalength-remaining:])
		} else {
			ctx.input = ctx.input[:0]
		}

		return datalength, nil
	}
	}), 8)

	return write
}

// Returns an io.Reader implementation that wraps the provided io.Reader
// and decompresses data according to the predictor algorithm
func Decompressor(reader io.Reader) io.Reader {
	var ctx context
	ctx.input = make([]byte, 0, 8)
︙

Modified src/0dev.org/predictor/predictor_test.go from [d9c16d84c9] to [0fb2d9a4c8].

︙
78
79
80
81
82
83
84
85

86
87
88
89
90
91
92

78
79
80
81
82
83
84

85
86
87
88
89
90
91
92








-
+








			t.Error(err)
		}
	}
}

func TestStepCycle(t *testing.T) {
	for i := 0; i < len(testData); i++ {
		for j := 1; j < len(testData); j++ {
		for j := 1; j < len(testData[i]); j++ {
			if err := cycle(testData[i], j); err != nil {
				t.Error("Error for testData[", i, "], step[", j, "] ", err)
			}
		}
	}
}
︙
111
112
113
114
115
116
117

118
119
120
121
122
123
124

111
112
113
114
115
116
117
118
119
120
121
122
123
124
125








+









			trace = append(trace, data[:step]...)

			_, err = compressor.Write(data[:step])
			if err != nil {
				return err
			}

			data = data[step:]
		} else {
			step = len(data)
		}
	}

	// Flush the compressor
︙
136
137
138
139
140
141
142
143
144


145
146
147
148
149

137
138
139
140
141
142
143


144
145
146
147
148
149
150








-
-
+
+







	// Diff the result against the initial input
	delta := diff.Diff(diff.D{len(input), len(decompressed),
		func(i, j int) bool { return input[i] == decompressed[j] }})

	// Return a well-formated error if any differences are found
	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
		return fmt.Errorf("Unexpected decompressed output %v\ninput:  (%d) %#x\ntrace:  (%d) %#x\noutput: (%d) %#x\n",
			delta, len(input), input, len(trace), trace, len(decompressed), decompressed)
		return fmt.Errorf("Unexpected decompressed output for step %d, delta %v\ninput:  (%d) %#x\ntrace:  (%d) %#x\noutput: (%d) %#x\n",
			step, delta, len(input), input, len(trace), trace, len(decompressed), decompressed)
	}

	// All is good :)
	return nil
}

Powered by Fossil · RSS