Check-in [c9f3a59cb6]
Overview
Comment:Implemented commands/pdc using predictor. Made predictor's Compressor(...) return an io.Writer.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:c9f3a59cb6f8359e28b9333fb4f6a661a94c44df
User & Date: spaskalev on 2014-12-19 21:54:39
Other Links: manifest | tags
Context
2014-12-20
11:52
[predictor] Removed the buffer from the context struct, allocate the input slice buffer on creation with make. check-in: 723ffeb1fd user: spaskalev tags: trunk
2014-12-19
21:54
Implemented commands/pdc using predictor. Made predictor's Compressor(...) return an io.Writer. check-in: c9f3a59cb6 user: spaskalev tags: trunk
2014-12-16
23:29
Removed err variable from compressor check-in: 0f4bc650d1 user: spaskalev tags: trunk
Changes

Added src/0dev.org/commands/pdc/main.go version [3c0c8016e1].



























































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
package main

import (
	predictor "0dev.org/predictor"
	"bufio"
	"fmt"
	"io"
	"os"
)

func main() {
	switch {
	case len(os.Args) == 1:
		os.Exit(compress(os.Stdout, os.Stdin))
	case len(os.Args) == 2 && os.Args[1] == "-d":
		os.Exit(decompress(os.Stdout, os.Stdin))
	default:
		fmt.Fprintln(os.Stdout, "Usage: pdc [-d]")
	}
}

// Compress the data from the given io.Reader and write it to the given io.Writer
// I/O is buffered for better performance
func compress(output io.Writer, input io.Reader) int {
	var (
		err        error
		buffer     *bufio.Writer = bufio.NewWriter(output)
		compressor io.Writer     = predictor.Compressor(buffer)
	)

	_, err = io.Copy(compressor, bufio.NewReader(input))
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error while compressing.\n", err)
		return 1
	}

	// Flush the compressor
	_, err = compressor.Write(nil)
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error while flushing compresssor buffer.\n", err)
		return 1
	}

	// Flush the buffer
	err = buffer.Flush()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error while flushing output buffer.\n", err)
		return 1
	}

	return 0
}

// Decompress the data from the given io.Reader and write it to the given io.Writer
// I/O is buffered for better performance
func decompress(output io.Writer, input io.Reader) int {
	var (
		err          error
		buffer       *bufio.Writer = bufio.NewWriter(output)
		decompressor io.Reader     = predictor.Decompressor(input)
	)

	_, err = io.Copy(buffer, bufio.NewReader(decompressor))
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error while decompressing.\n", err)
		return 1
	}

	// Flush
	err = buffer.Flush()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error while flushing output buffer.\n", err)
		return 1
	}

	return 0
}

Modified src/0dev.org/commands/plaindiff/main.go from [f98671241a] to [b2d6e402f0].

6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
	"fmt"
	"hash"
	"hash/fnv"
	"io"
	"os"
)

const usage = "Usage: plaindiff <file1> <file2>"

func main() {
	var args []string = os.Args
	if len(args) != 3 {
		os.Stderr.WriteString(usage)
		os.Exit(1)
	}







|







6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
	"fmt"
	"hash"
	"hash/fnv"
	"io"
	"os"
)

const usage = "Usage: plaindiff <file1> <file2>\n"

func main() {
	var args []string = os.Args
	if len(args) != 3 {
		os.Stderr.WriteString(usage)
		os.Exit(1)
	}

Modified src/0dev.org/predictor/predictor.go from [2cc1907fb9] to [c65cc5c256].

9
10
11
12
13
14
15
16







17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
..
99
100
101
102
103
104
105

106
107

108
109
110
111
112

113
114
115
116
117
118
119
120
121
122
123
124
type context struct {
	table  [1 << 16]byte
	buffer [1 << 3]byte
	input  []byte
	hash   uint16
}

// Returns a closure over the provided writer that compresses data when called.







//
// It can buffer data as the predictor mandates 8-byte blocks with a header.
// A call with no data will force a flush.
func Compressor(writer io.Writer) func([]byte) error {
	var ctx context
	ctx.input = ctx.buffer[:0]

	// Forward declaration as it is required for recursion
	var write func(data []byte) error

	write = func(data []byte) error {
		var (
			blockSize    int = 8
			bufferLength int = len(ctx.input)
		)

................................................................................

		return nil
	}

	return write
}


type reader func([]byte) (int, error)


func (r reader) Read(output []byte) (int, error) {
	return r(output)
}

// TODO - document

func Decompressor(wrapped io.Reader) io.Reader {
	var ctx context
	ctx.input = ctx.buffer[:0]

	return reader(func(output []byte) (int, error) {
		var (
			err       error
			flags     byte
			readCount int
		)

		// Sanity check for space to read into







|
>
>
>
>
>
>
>



|




|







 







>
|

>
|



|
>




|







9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
...
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
type context struct {
	table  [1 << 16]byte
	buffer [1 << 3]byte
	input  []byte
	hash   uint16
}

type compressor func([]byte) error

func (w compressor) Write(data []byte) (int, error) {
	return len(data), w(data)
}

// Returns an io.Writer implementation that wraps the provided io.Writer
// and compresses data according to the predictor algorithm
//
// It can buffer data as the predictor mandates 8-byte blocks with a header.
// A call with no data will force a flush.
func Compressor(writer io.Writer) io.Writer {
	var ctx context
	ctx.input = ctx.buffer[:0]

	// Forward declaration as it is required for recursion
	var write compressor

	write = func(data []byte) error {
		var (
			blockSize    int = 8
			bufferLength int = len(ctx.input)
		)

................................................................................

		return nil
	}

	return write
}

// A function type alias so that it can have methods attached to it
type decompressor func([]byte) (int, error)

// Required to implement io.Reader
func (r decompressor) Read(output []byte) (int, error) {
	return r(output)
}

// Returns an io.Reader implementation that wraps the provided io.Reader
// and decompresses data according to the predictor algorithm
func Decompressor(wrapped io.Reader) io.Reader {
	var ctx context
	ctx.input = ctx.buffer[:0]

	return decompressor(func(output []byte) (int, error) {
		var (
			err       error
			flags     byte
			readCount int
		)

		// Sanity check for space to read into

Modified src/0dev.org/predictor/predictor_test.go from [73a8f932ed] to [dd11cf49de].

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
...
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
func TestCompressorSample(t *testing.T) {
	var (
		buf bytes.Buffer
		err error
	)

	out := Compressor(&buf)
	err = out(input)
	if err != nil {
		t.Error(err)
	}

	err = out(nil)
	if err != nil {
		t.Error(err)
	}

	result := buf.Bytes()
	delta := diff.Diff(diff.D{len(result), len(output), func(i, j int) bool { return result[i] == output[j] }})

................................................................................
	var trace []byte = make([]byte, 0)

	for len(data) > 0 {
		if step <= len(data) {

			trace = append(trace, data[:step]...)

			err = compressor(data[:step])
			if err != nil {
				return err
			}
			data = data[step:]
		} else {
			step = len(data)
		}
	}

	// Flush the compressor
	err = compressor(nil)
	if err != nil {
		return err
	}

	// Attempt to decompress the data
	compressed := buf.Bytes()
	decompressed, err := ioutil.ReadAll(Decompressor(bytes.NewReader(compressed)))







|




|







 







|










|







27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
...
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
func TestCompressorSample(t *testing.T) {
	var (
		buf bytes.Buffer
		err error
	)

	out := Compressor(&buf)
	_, err = out.Write(input)
	if err != nil {
		t.Error(err)
	}

	_, err = out.Write(nil)
	if err != nil {
		t.Error(err)
	}

	result := buf.Bytes()
	delta := diff.Diff(diff.D{len(result), len(output), func(i, j int) bool { return result[i] == output[j] }})

................................................................................
	var trace []byte = make([]byte, 0)

	for len(data) > 0 {
		if step <= len(data) {

			trace = append(trace, data[:step]...)

			_, err = compressor.Write(data[:step])
			if err != nil {
				return err
			}
			data = data[step:]
		} else {
			step = len(data)
		}
	}

	// Flush the compressor
	_, err = compressor.Write(nil)
	if err != nil {
		return err
	}

	// Attempt to decompress the data
	compressed := buf.Bytes()
	decompressed, err := ioutil.ReadAll(Decompressor(bytes.NewReader(compressed)))