Check-in [b703c38e0b]
Overview
Comment:Extracted SizedReader to a sizedReader type with a Read() method. Closures seem to be slower in Go 1.4 and there is no real need for SizedReader to be a closure.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: b703c38e0b59db785a1706b2920f0ee6ff120ad3
User & Date: spaskalev on 2014-12-24 23:45:57
Other Links: manifest | tags
Context
2014-12-24
23:58
0dev.org/ioutil/SizedReader is now as fast as bufio.Reader if not faster for a buffer of 4096 bytes. Switched pdc to use it for decompression buffering and removed profiling code from the former check-in: 2cec92909f user: spaskalev tags: trunk
23:45
Extracted SizedReader to a sizedReader type with a Read() method. Closures seem to be slower in Go 1.4 and there is no real need for SizedReader to be a closure. check-in: b703c38e0b user: spaskalev tags: trunk
23:33
Made SizedReader faster by keeping explicit buffer indices. check-in: 701ac713de user: spaskalev tags: trunk
Changes

Modified src/0dev.org/commands/pdc/main.go from [1dda5424c0] to [675059111f].

1
2
3

4
5
6

7
8
9
10
11
12

13
14
15

16
17

18
19
20




21
22
23
24
25
26
27
1
2
3
4
5
6

7
8
9
10
11
12
13
14
15
16

17
18

19
20
21
22
23
24
25
26
27
28
29
30
31
32
33



+


-
+






+


-
+

-
+



+
+
+
+







package main

import (
	pprof "0dev.org/debug/pprof"
	iou "0dev.org/ioutil"
	predictor "0dev.org/predictor"
	"bufio"
	// "bufio"
	"fmt"
	"io"
	"os"
)

func main() {
	var code int
	switch {
	case len(os.Args) == 1:
		os.Exit(compress(os.Stdout, os.Stdin))
		code = compress(os.Stdout, os.Stdin)
	case len(os.Args) == 2 && os.Args[1] == "-d":
		os.Exit(decompress(os.Stdout, os.Stdin))
		code = decompress(os.Stdout, os.Stdin)
	default:
		fmt.Fprintln(os.Stdout, "Usage: pdc [-d]")
	}

	pprof.Stop()

	os.Exit(code)
}

// Compress the data from the given io.Reader and write it to the given io.Writer
// I/O is buffered for better performance
func compress(output io.Writer, input io.Reader) int {
	var (
		err        error
54
55
56
57
58
59
60

61

62
63
64
65
66
67
68
60
61
62
63
64
65
66
67

68
69
70
71
72
73
74
75







+
-
+








// Decompress the data from the given io.Reader and write it to the given io.Writer
// I/O is buffered for better performance
func decompress(output io.Writer, input io.Reader) int {
	var (
		err          error
		buffer       io.Writer = iou.SizedWriter(output, 4096)
		decompressor io.Reader = predictor.Decompressor(iou.SizedReader(input, 4096))
		decompressor io.Reader = predictor.Decompressor(bufio.NewReader(input))
		//decompressor io.Reader = predictor.Decompressor(bufio.NewReader(input))
	)

	_, err = io.Copy(buffer, decompressor)
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error while decompressing.\n", err)
		return 1
	}

Modified src/0dev.org/ioutil/ioutil.go from [e5419d451e] to [c84493a1fe].

77
78
79
80
81
82
83






84
85
86
87
88
89
90
91




92
93
94
95
96
97
98








99
100
101
102
103




104
105
106


107
108
109
110
111




112
113
114


115
116
117
118



119
120
121
122
123




124
125
126


127
128
129
130
131
132
133






134
135
136


137
138
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94



95
96
97
98







99
100
101
102
103
104
105
106
107




108
109
110
111
112


113
114
115




116
117
118
119
120


121
122
123



124
125
126
127




128
129
130
131
132


133
134
135






136
137
138
139
140
141
142


143
144

145







+
+
+
+
+
+





-
-
-
+
+
+
+
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+

-
-
-
-
+
+
+
+

-
-
+
+

-
-
-
-
+
+
+
+

-
-
+
+

-
-
-
+
+
+

-
-
-
-
+
+
+
+

-
-
+
+

-
-
-
-
-
-
+
+
+
+
+
+

-
-
+
+
-


		// Handle the rest of the input
		return write(input[count:])
	}

	return write
}

type sizedReader struct {
	reader         io.Reader
	buffer         []byte
	from, to, size int
}

// Returns a reader that delegates calls to Read(...) while ensuring
// that the output buffer is never smaller than the required size
// and is downsized to a multiple of the required size if larger.
func SizedReader(reader io.Reader, size int) io.Reader {
	var (
		buffer   []byte = make([]byte, size)
		from, to int    = 0, 0
	var sr sizedReader
	sr.reader = reader
	sr.buffer = make([]byte, size)
	sr.size, sr.from, sr.to = size, 0, 0
	)

	return ReaderFunc(func(output []byte) (int, error) {
		var (
			count int
			err   error
		)
	return &sr
}

func (sr *sizedReader) Read(output []byte) (int, error) {
	var (
		count int
		err   error
	)

	start:
		// Reply with the buffered data if there is any
		if to-from > 0 {
			count = copy(output, buffer[from:to])
start:
	// Reply with the buffered data if there is any
	if sr.to-sr.from > 0 {
		count = copy(output, sr.buffer[sr.from:sr.to])

			// Advance the data in the buffer
			from += count
		// Advance the data in the buffer
		sr.from += count

			// Check whether we have reached the end of the buffer
			if to-from == 0 {
				// Reset the buffer
				from, to = 0, 0
		// Check whether we have reached the end of the buffer
		if sr.to-sr.from == 0 {
			// Reset the buffer
			sr.from, sr.to = 0, 0

				return count, err
			}
			return count, err
		}

			// Do not propagate an error until the buffer is exhausted
			return count, nil
		}
		// Do not propagate an error until the buffer is exhausted
		return count, nil
	}

		// Delegate if the buffer is empty and the destination buffer is large enough
		if len(output) >= size {
			return reader.Read(output[:(len(output)/size)*size])
		}
	// Delegate if the buffer is empty and the destination buffer is large enough
	if len(output) >= sr.size {
		return sr.reader.Read(output[:(len(output)/sr.size)*sr.size])
	}

		// Perform a read into the buffer
		count, err = reader.Read(buffer)
	// Perform a read into the buffer
	count, err = sr.reader.Read(sr.buffer)

		// Size the buffer down to the read data size
		// and restart if we have successfully read some bytes
		from, to = 0, count
		if to-from > 0 {
			goto start
		}
	// Size the buffer down to the read data size
	// and restart if we have successfully read some bytes
	sr.from, sr.to = 0, count
	if sr.to-sr.from > 0 {
		goto start
	}

		// Returning on err/misbehaving noop reader
		return 0, err
	// Returning on err/misbehaving noop reader
	return 0, err
	})
}