Check-in [e1778aba98]
Overview
SHA1:e1778aba9806c59c858de655886f0802045e6e7a
Date: 2014-12-24 21:40:11
User: spaskalev
Comment:Fixed SizedWriter behavior so that it follows io.Writer's Write(...) contract. Added more tests for 100% CC on the ioutil package. Predictor's compressor now uses SizedWriter and no longer has to do any internal buffering.
Timelines: family | ancestors | descendants | both | trunk
Downloads: Tarball | ZIP archive
Other Links: files | file ages | folders | manifest
Tags And Properties
Context
2014-12-24
22:32
[1717bfae3b] Use 0dev.org/ioutil.SizedWriter as an output buffer for pdc in both compress and decompress modes. (user: spaskalev, tags: trunk)
21:40
[e1778aba98] Fixed SizedWriter behavior so that it follows io.Writer's Write(...) contract. Added more tests for 100% CC on the ioutil package. Predictor's compressor now uses SizedWriter and no longer has to do any internal buffering. (user: spaskalev, tags: trunk)
19:07
[c28a763d5e] Implemented ioutil/SizedWriter. CC at 100%. (user: spaskalev, tags: trunk)
Changes

Modified src/0dev.org/ioutil/ioutil.go from [9da21d39e0] to [bca057263b].

    45     45   			buffer = buffer[:copy(buffer, buffer[count:])]
    46     46   
    47     47   			return 0, err
    48     48   		}
    49     49   
    50     50   		// Delegate to the writer if the size is right
    51     51   		if len(buffer) == 0 && len(input) >= size {
    52         -			return writer.Write(input)
           52  +			reduced := (len(input) / size) * size
           53  +			count, err = writer.Write(input[:reduced])
           54  +			if count < reduced || err != nil {
           55  +				return count, err
           56  +			}
           57  +
           58  +			// Stage any remaining data in the buffer
           59  +			buffer = append(buffer, input[count:]...)
           60  +			return len(input), nil
    53     61   		}
    54     62   
    55     63   		// Append data to the buffer
    56     64   		count = copy(buffer[len(buffer):size], input)
    57     65   		buffer = buffer[:len(buffer)+count]
    58     66   
    59     67   		// Return if we don't have enough bytes to write

Modified src/0dev.org/ioutil/ioutil_test.go from [ac59af05f3] to [91aaac4713].

    93     93   		t.Error("Unexpected error from SizedWriter", err)
    94     94   	}
    95     95   	if buffer.String() != "1234567890" {
    96     96   		t.Error("Unexpected value in wrapped writer", buffer.String())
    97     97   	}
    98     98   }
    99     99   
   100         -func TestSizedWriterError(t *testing.T) {
          100  +func TestSizeWriterLarger(t *testing.T) {
          101  +	var (
          102  +		input  []byte = []byte("0123456789AB")
          103  +		buffer bytes.Buffer
          104  +		writer = SizedWriter(&buffer, 8)
          105  +	)
          106  +
          107  +	count, err := writer.Write(input)
          108  +	if count != 12 {
          109  +		t.Error("Unexpected write count from SizedWriter", count)
          110  +	}
          111  +	if err != nil {
          112  +		t.Error("Unexpected error from SizedWriter", err)
          113  +	}
          114  +	if buffer.String() != "01234567" {
          115  +		t.Error("Unexpected value in wrapped writer", buffer.String())
          116  +	}
          117  +
          118  +	count, err = writer.Write(nil)
          119  +	if count != 0 {
          120  +		t.Error("Unexpected write count from SizedWriter", count)
          121  +	}
          122  +	if err != nil {
          123  +		t.Error("Unexpected error from SizedWriter", err)
          124  +	}
          125  +	if buffer.String() != "0123456789AB" {
          126  +		t.Error("Unexpected value in wrapped writer", buffer.String())
          127  +	}
          128  +}
          129  +
          130  +func TestSizedWriterError1(t *testing.T) {
   101    131   	var (
   102    132   		errorWriter io.Writer = WriterFunc(func([]byte) (int, error) {
   103    133   			return 1, errors.New("Invalid write")
   104    134   		})
   105    135   		writer io.Writer = SizedWriter(errorWriter, 2)
   106    136   	)
   107    137   
................................................................................
   110    140   		t.Error("Unexpected write count from SizedWriter", count)
   111    141   	}
   112    142   	if err != nil {
   113    143   		t.Error("Unexpected error from SizedWriter", err)
   114    144   	}
   115    145   
   116    146   	count, err = writer.Write([]byte("2"))
          147  +	if count != 1 {
          148  +		t.Error("Unexpected write count from SizedWriter", count)
          149  +	}
          150  +	if err == nil {
          151  +		t.Error("Unexpected lack of error from SizedWriter")
          152  +	}
          153  +}
          154  +
          155  +func TestSizedWriterError2(t *testing.T) {
          156  +	var (
          157  +		errorWriter io.Writer = WriterFunc(func([]byte) (int, error) {
          158  +			return 1, errors.New("Invalid write")
          159  +		})
          160  +		writer io.Writer = SizedWriter(errorWriter, 1)
          161  +	)
          162  +
          163  +	count, err := writer.Write([]byte("12"))
   117    164   	if count != 1 {
   118    165   		t.Error("Unexpected write count from SizedWriter", count)
   119    166   	}
   120    167   	if err == nil {
   121    168   		t.Error("Unexpected lack of error from SizedWriter")
   122    169   	}
   123    170   }

Modified src/0dev.org/predictor/predictor.go from [1bc0c5d728] to [43e7f751ed].

    26     26   // Returns an io.Writer implementation that wraps the provided io.Writer
    27     27   // and compresses data according to the predictor algorithm
    28     28   //
    29     29   // It can buffer data as the predictor mandates 8-byte blocks with a header.
    30     30   // A call with no data will force a flush.
    31     31   func Compressor(writer io.Writer) io.Writer {
    32     32   	var ctx context
    33         -	ctx.input = make([]byte, 0, 8)
    34     33   
    35         -	// Forward declaration as it is required for recursion
    36         -	var write iou.WriterFunc
    37         -
    38         -	write = func(data []byte) (int, error) {
           34  +	return iou.SizedWriter(iou.WriterFunc(func(data []byte) (int, error) {
    39     35   		var (
    40         -			blockSize    int = 8
    41         -			bufferLength int = len(ctx.input)
    42         -			datalength   int = len(data)
           36  +			blockSize  int = 8
           37  +			datalength int = len(data)
    43     38   		)
    44     39   
    45         -		// Force a flush if we are called with no data to write
    46     40   		if datalength == 0 {
    47         -			// Nothing to flush if the buffer is empty though
    48         -			if len(ctx.input) == 0 {
    49         -				return 0, nil
    50         -			}
    51         -			// We can't have more than 7 bytes in the buffer so this is safe
    52         -			data, datalength = ctx.input, len(ctx.input)
    53         -			blockSize, bufferLength = datalength, 0
           41  +			return 0, nil
    54     42   		}
    55     43   
    56         -		// Check if there are pending bytes in the buffer
    57         -		if datalength < blockSize || bufferLength > 0 {
    58         -
    59         -			// If the current buffer + new data can fit into a block
    60         -			if (datalength + bufferLength) <= blockSize {
    61         -				ctx.input = append(ctx.input, data...)
    62         -
    63         -				// Flush the block if the buffer fills it
    64         -				if len(ctx.input) == blockSize {
    65         -					return write(nil)
    66         -				}
    67         -				// ... otherwise just return
    68         -				return datalength, nil
    69         -			}
    70         -
    71         -			// The current buffer + new data overflow the block size
    72         -			// Complete the block, flush it ...
    73         -			ctx.input = append(ctx.input, data[:blockSize-bufferLength]...)
    74         -			if c, err := write(nil); err != nil {
    75         -				return c, err
    76         -			}
    77         -			// ... and stage the rest of the data in the buffer
    78         -			ctx.input = append(ctx.input, data[blockSize-bufferLength:]...)
    79         -			return datalength, nil
           44  +		if datalength < blockSize {
           45  +			blockSize = datalength
    80     46   		}
    81     47   
    82     48   		var buf []byte = make([]byte, 1, blockSize+1)
    83     49   		for block := 0; block < datalength/blockSize; block++ {
    84     50   			for i := 0; i < blockSize; i++ {
    85     51   				var current byte = data[(block*blockSize)+i]
    86     52   				if ctx.table[ctx.hash] == current {
................................................................................
    98     64   				return (block * blockSize) + c, err
    99     65   			}
   100     66   
   101     67   			// Reset the flags and buffer for the next iteration
   102     68   			buf, buf[0] = buf[:1], 0
   103     69   		}
   104     70   
   105         -		if remaining := datalength % blockSize; remaining > 0 {
   106         -			ctx.input = ctx.input[:remaining]
   107         -			copy(ctx.input, data[datalength-remaining:])
   108         -		} else {
   109         -			ctx.input = ctx.input[:0]
   110         -		}
   111         -
   112     71   		return datalength, nil
   113         -	}
   114         -
   115         -	return write
           72  +	}), 8)
   116     73   }
   117     74   
   118     75   // Returns an io.Reader implementation that wraps the provided io.Reader
   119     76   // and decompresses data according to the predictor algorithm
   120     77   func Decompressor(reader io.Reader) io.Reader {
   121     78   	var ctx context
   122     79   	ctx.input = make([]byte, 0, 8)

Modified src/0dev.org/predictor/predictor_test.go from [d9c16d84c9] to [0fb2d9a4c8].

    78     78   			t.Error(err)
    79     79   		}
    80     80   	}
    81     81   }
    82     82   
    83     83   func TestStepCycle(t *testing.T) {
    84     84   	for i := 0; i < len(testData); i++ {
    85         -		for j := 1; j < len(testData); j++ {
           85  +		for j := 1; j < len(testData[i]); j++ {
    86     86   			if err := cycle(testData[i], j); err != nil {
    87     87   				t.Error("Error for testData[", i, "], step[", j, "] ", err)
    88     88   			}
    89     89   		}
    90     90   	}
    91     91   }
    92     92   
................................................................................
   111    111   
   112    112   			trace = append(trace, data[:step]...)
   113    113   
   114    114   			_, err = compressor.Write(data[:step])
   115    115   			if err != nil {
   116    116   				return err
   117    117   			}
          118  +
   118    119   			data = data[step:]
   119    120   		} else {
   120    121   			step = len(data)
   121    122   		}
   122    123   	}
   123    124   
   124    125   	// Flush the compressor
................................................................................
   136    137   
   137    138   	// Diff the result against the initial input
   138    139   	delta := diff.Diff(diff.D{len(input), len(decompressed),
   139    140   		func(i, j int) bool { return input[i] == decompressed[j] }})
   140    141   
   141    142   	// Return a well-formated error if any differences are found
   142    143   	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
   143         -		return fmt.Errorf("Unexpected decompressed output %v\ninput:  (%d) %#x\ntrace:  (%d) %#x\noutput: (%d) %#x\n",
   144         -			delta, len(input), input, len(trace), trace, len(decompressed), decompressed)
          144  +		return fmt.Errorf("Unexpected decompressed output for step %d, delta %v\ninput:  (%d) %#x\ntrace:  (%d) %#x\noutput: (%d) %#x\n",
          145  +			step, delta, len(input), input, len(trace), trace, len(decompressed), decompressed)
   145    146   	}
   146    147   
   147    148   	// All is good :)
   148    149   	return nil
   149    150   }