Check-in [b703c38e0b]
Overview
SHA1:b703c38e0b59db785a1706b2920f0ee6ff120ad3
Date: 2014-12-24 23:45:57
User: spaskalev
Comment:Extracted SizedReader to a sizedReader type with a Read() method. Closures seem to be slower in Go 1.4 and there is no real need for SizedReader to be a closure.
Timelines: family | ancestors | descendants | both | trunk
Downloads: Tarball | ZIP archive
Other Links: files | file ages | folders | manifest
Tags And Properties
Context
2014-12-24
23:58
[2cec92909f] 0dev.org/ioutil/SizedReader is now as fast as bufio.Reader if not faster for a buffer of 4096 bytes. Switched pdc to use it for decompression buffering and removed profiling code from the former (user: spaskalev, tags: trunk)
23:45
[b703c38e0b] Extracted SizedReader to a sizedReader type with a Read() method. Closures seem to be slower in Go 1.4 and there is no real need for SizedReader to be a closure. (user: spaskalev, tags: trunk)
23:33
[701ac713de] Made SizedReader faster by keeping explicit buffer indices. (user: spaskalev, tags: trunk)
Changes

Modified src/0dev.org/commands/pdc/main.go from [1dda5424c0] to [675059111f].

     1      1   package main
     2      2   
     3      3   import (
            4  +	pprof "0dev.org/debug/pprof"
     4      5   	iou "0dev.org/ioutil"
     5      6   	predictor "0dev.org/predictor"
     6         -	"bufio"
            7  +	// "bufio"
     7      8   	"fmt"
     8      9   	"io"
     9     10   	"os"
    10     11   )
    11     12   
    12     13   func main() {
           14  +	var code int
    13     15   	switch {
    14     16   	case len(os.Args) == 1:
    15         -		os.Exit(compress(os.Stdout, os.Stdin))
           17  +		code = compress(os.Stdout, os.Stdin)
    16     18   	case len(os.Args) == 2 && os.Args[1] == "-d":
    17         -		os.Exit(decompress(os.Stdout, os.Stdin))
           19  +		code = decompress(os.Stdout, os.Stdin)
    18     20   	default:
    19     21   		fmt.Fprintln(os.Stdout, "Usage: pdc [-d]")
    20     22   	}
           23  +
           24  +	pprof.Stop()
           25  +
           26  +	os.Exit(code)
    21     27   }
    22     28   
    23     29   // Compress the data from the given io.Reader and write it to the given io.Writer
    24     30   // I/O is buffered for better performance
    25     31   func compress(output io.Writer, input io.Reader) int {
    26     32   	var (
    27     33   		err        error
................................................................................
    54     60   
    55     61   // Decompress the data from the given io.Reader and write it to the given io.Writer
    56     62   // I/O is buffered for better performance
    57     63   func decompress(output io.Writer, input io.Reader) int {
    58     64   	var (
    59     65   		err          error
    60     66   		buffer       io.Writer = iou.SizedWriter(output, 4096)
    61         -		decompressor io.Reader = predictor.Decompressor(bufio.NewReader(input))
           67  +		decompressor io.Reader = predictor.Decompressor(iou.SizedReader(input, 4096))
           68  +		//decompressor io.Reader = predictor.Decompressor(bufio.NewReader(input))
    62     69   	)
    63     70   
    64     71   	_, err = io.Copy(buffer, decompressor)
    65     72   	if err != nil {
    66     73   		fmt.Fprintln(os.Stderr, "Error while decompressing.\n", err)
    67     74   		return 1
    68     75   	}

Modified src/0dev.org/ioutil/ioutil.go from [e5419d451e] to [c84493a1fe].

    77     77   
    78     78   		// Handle the rest of the input
    79     79   		return write(input[count:])
    80     80   	}
    81     81   
    82     82   	return write
    83     83   }
           84  +
           85  +type sizedReader struct {
           86  +	reader         io.Reader
           87  +	buffer         []byte
           88  +	from, to, size int
           89  +}
    84     90   
    85     91   // Returns a reader that delegates calls to Read(...) while ensuring
    86     92   // that the output buffer is never smaller than the required size
    87     93   // and is downsized to a multiple of the required size if larger.
    88     94   func SizedReader(reader io.Reader, size int) io.Reader {
           95  +	var sr sizedReader
           96  +	sr.reader = reader
           97  +	sr.buffer = make([]byte, size)
           98  +	sr.size, sr.from, sr.to = size, 0, 0
           99  +	return &sr
          100  +}
          101  +
          102  +func (sr *sizedReader) Read(output []byte) (int, error) {
    89    103   	var (
    90         -		buffer   []byte = make([]byte, size)
    91         -		from, to int    = 0, 0
          104  +		count int
          105  +		err   error
    92    106   	)
    93    107   
    94         -	return ReaderFunc(func(output []byte) (int, error) {
    95         -		var (
    96         -			count int
    97         -			err   error
    98         -		)
          108  +start:
          109  +	// Reply with the buffered data if there is any
          110  +	if sr.to-sr.from > 0 {
          111  +		count = copy(output, sr.buffer[sr.from:sr.to])
          112  +
          113  +		// Advance the data in the buffer
          114  +		sr.from += count
    99    115   
   100         -	start:
   101         -		// Reply with the buffered data if there is any
   102         -		if to-from > 0 {
   103         -			count = copy(output, buffer[from:to])
   104         -
   105         -			// Advance the data in the buffer
   106         -			from += count
          116  +		// Check whether we have reached the end of the buffer
          117  +		if sr.to-sr.from == 0 {
          118  +			// Reset the buffer
          119  +			sr.from, sr.to = 0, 0
   107    120   
   108         -			// Check whether we have reached the end of the buffer
   109         -			if to-from == 0 {
   110         -				// Reset the buffer
   111         -				from, to = 0, 0
   112         -
   113         -				return count, err
   114         -			}
   115         -
   116         -			// Do not propagate an error until the buffer is exhausted
   117         -			return count, nil
          121  +			return count, err
   118    122   		}
   119    123   
   120         -		// Delegate if the buffer is empty and the destination buffer is large enough
   121         -		if len(output) >= size {
   122         -			return reader.Read(output[:(len(output)/size)*size])
   123         -		}
          124  +		// Do not propagate an error until the buffer is exhausted
          125  +		return count, nil
          126  +	}
          127  +
          128  +	// Delegate if the buffer is empty and the destination buffer is large enough
          129  +	if len(output) >= sr.size {
          130  +		return sr.reader.Read(output[:(len(output)/sr.size)*sr.size])
          131  +	}
   124    132   
   125         -		// Perform a read into the buffer
   126         -		count, err = reader.Read(buffer)
          133  +	// Perform a read into the buffer
          134  +	count, err = sr.reader.Read(sr.buffer)
   127    135   
   128         -		// Size the buffer down to the read data size
   129         -		// and restart if we have successfully read some bytes
   130         -		from, to = 0, count
   131         -		if to-from > 0 {
   132         -			goto start
   133         -		}
          136  +	// Size the buffer down to the read data size
          137  +	// and restart if we have successfully read some bytes
          138  +	sr.from, sr.to = 0, count
          139  +	if sr.to-sr.from > 0 {
          140  +		goto start
          141  +	}
   134    142   
   135         -		// Returning on err/misbehaving noop reader
   136         -		return 0, err
   137         -	})
          143  +	// Returning on err/misbehaving noop reader
          144  +	return 0, err
   138    145   }