Overview
| Comment: | Implemented commands/pdc using predictor. Made predictor's Compressor(...) return an io.Writer. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
c9f3a59cb6f8359e28b9333fb4f6a661 |
| User & Date: | spaskalev on 2014-12-19 21:54:39 |
| Other Links: | manifest | tags |
Context
|
2014-12-20
| ||
| 11:52 | [predictor] Removed the buffer from the context struct, allocate the input slice buffer on creation with make. check-in: 723ffeb1fd user: spaskalev tags: trunk | |
|
2014-12-19
| ||
| 21:54 | Implemented commands/pdc using predictor. Made predictor's Compressor(...) return an io.Writer. check-in: c9f3a59cb6 user: spaskalev tags: trunk | |
|
2014-12-16
| ||
| 23:29 | Removed err variable from compressor check-in: 0f4bc650d1 user: spaskalev tags: trunk | |
Changes
Added src/0dev.org/commands/pdc/main.go version [3c0c8016e1].
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
package main import ( predictor "0dev.org/predictor" "bufio" "fmt" "io" "os" ) func main() { switch { case len(os.Args) == 1: os.Exit(compress(os.Stdout, os.Stdin)) case len(os.Args) == 2 && os.Args[1] == "-d": os.Exit(decompress(os.Stdout, os.Stdin)) default: fmt.Fprintln(os.Stdout, "Usage: pdc [-d]") } } // Compress the data from the given io.Reader and write it to the given io.Writer // I/O is buffered for better performance func compress(output io.Writer, input io.Reader) int { var ( err error buffer *bufio.Writer = bufio.NewWriter(output) compressor io.Writer = predictor.Compressor(buffer) ) _, err = io.Copy(compressor, bufio.NewReader(input)) if err != nil { fmt.Fprintln(os.Stderr, "Error while compressing.\n", err) return 1 } // Flush the compressor _, err = compressor.Write(nil) if err != nil { fmt.Fprintln(os.Stderr, "Error while flushing compresssor buffer.\n", err) return 1 } // Flush the buffer err = buffer.Flush() if err != nil { fmt.Fprintln(os.Stderr, "Error while flushing output buffer.\n", err) return 1 } return 0 } // Decompress the data from the given io.Reader and write it to the given io.Writer // I/O is buffered for better performance func decompress(output io.Writer, input io.Reader) int { var ( err error buffer *bufio.Writer = bufio.NewWriter(output) decompressor io.Reader = predictor.Decompressor(input) ) _, err = io.Copy(buffer, bufio.NewReader(decompressor)) if err != nil { fmt.Fprintln(os.Stderr, "Error while decompressing.\n", err) return 1 } // Flush err = buffer.Flush() if err != nil { fmt.Fprintln(os.Stderr, "Error while flushing output buffer.\n", err) return 1 } return 0 } |
Modified src/0dev.org/commands/plaindiff/main.go from [f98671241a] to [b2d6e402f0].
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
"fmt"
"hash"
"hash/fnv"
"io"
"os"
)
const usage = "Usage: plaindiff <file1> <file2>"
func main() {
var args []string = os.Args
if len(args) != 3 {
os.Stderr.WriteString(usage)
os.Exit(1)
}
|
| |
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
"fmt"
"hash"
"hash/fnv"
"io"
"os"
)
const usage = "Usage: plaindiff <file1> <file2>\n"
func main() {
var args []string = os.Args
if len(args) != 3 {
os.Stderr.WriteString(usage)
os.Exit(1)
}
|
Modified src/0dev.org/predictor/predictor.go from [2cc1907fb9] to [c65cc5c256].
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
..
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
type context struct {
table [1 << 16]byte
buffer [1 << 3]byte
input []byte
hash uint16
}
// Returns a closure over the provided writer that compresses data when called.
//
// It can buffer data as the predictor mandates 8-byte blocks with a header.
// A call with no data will force a flush.
func Compressor(writer io.Writer) func([]byte) error {
var ctx context
ctx.input = ctx.buffer[:0]
// Forward declaration as it is required for recursion
var write func(data []byte) error
write = func(data []byte) error {
var (
blockSize int = 8
bufferLength int = len(ctx.input)
)
................................................................................
return nil
}
return write
}
type reader func([]byte) (int, error)
func (r reader) Read(output []byte) (int, error) {
return r(output)
}
// TODO - document
func Decompressor(wrapped io.Reader) io.Reader {
var ctx context
ctx.input = ctx.buffer[:0]
return reader(func(output []byte) (int, error) {
var (
err error
flags byte
readCount int
)
// Sanity check for space to read into
|
|
>
>
>
>
>
>
>
|
|
>
|
>
|
|
>
|
|
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
...
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
type context struct {
table [1 << 16]byte
buffer [1 << 3]byte
input []byte
hash uint16
}
type compressor func([]byte) error
func (w compressor) Write(data []byte) (int, error) {
return len(data), w(data)
}
// Returns an io.Writer implementation that wraps the provided io.Writer
// and compresses data according to the predictor algorithm
//
// It can buffer data as the predictor mandates 8-byte blocks with a header.
// A call with no data will force a flush.
func Compressor(writer io.Writer) io.Writer {
var ctx context
ctx.input = ctx.buffer[:0]
// Forward declaration as it is required for recursion
var write compressor
write = func(data []byte) error {
var (
blockSize int = 8
bufferLength int = len(ctx.input)
)
................................................................................
return nil
}
return write
}
// A function type alias so that it can have methods attached to it
type decompressor func([]byte) (int, error)
// Required to implement io.Reader
func (r decompressor) Read(output []byte) (int, error) {
return r(output)
}
// Returns an io.Reader implementation that wraps the provided io.Reader
// and decompresses data according to the predictor algorithm
func Decompressor(wrapped io.Reader) io.Reader {
var ctx context
ctx.input = ctx.buffer[:0]
return decompressor(func(output []byte) (int, error) {
var (
err error
flags byte
readCount int
)
// Sanity check for space to read into
|
Modified src/0dev.org/predictor/predictor_test.go from [73a8f932ed] to [dd11cf49de].
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
...
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
func TestCompressorSample(t *testing.T) {
var (
buf bytes.Buffer
err error
)
out := Compressor(&buf)
err = out(input)
if err != nil {
t.Error(err)
}
err = out(nil)
if err != nil {
t.Error(err)
}
result := buf.Bytes()
delta := diff.Diff(diff.D{len(result), len(output), func(i, j int) bool { return result[i] == output[j] }})
................................................................................
var trace []byte = make([]byte, 0)
for len(data) > 0 {
if step <= len(data) {
trace = append(trace, data[:step]...)
err = compressor(data[:step])
if err != nil {
return err
}
data = data[step:]
} else {
step = len(data)
}
}
// Flush the compressor
err = compressor(nil)
if err != nil {
return err
}
// Attempt to decompress the data
compressed := buf.Bytes()
decompressed, err := ioutil.ReadAll(Decompressor(bytes.NewReader(compressed)))
|
|
|
|
|
|
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
...
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
func TestCompressorSample(t *testing.T) {
var (
buf bytes.Buffer
err error
)
out := Compressor(&buf)
_, err = out.Write(input)
if err != nil {
t.Error(err)
}
_, err = out.Write(nil)
if err != nil {
t.Error(err)
}
result := buf.Bytes()
delta := diff.Diff(diff.D{len(result), len(output), func(i, j int) bool { return result[i] == output[j] }})
................................................................................
var trace []byte = make([]byte, 0)
for len(data) > 0 {
if step <= len(data) {
trace = append(trace, data[:step]...)
_, err = compressor.Write(data[:step])
if err != nil {
return err
}
data = data[step:]
} else {
step = len(data)
}
}
// Flush the compressor
_, err = compressor.Write(nil)
if err != nil {
return err
}
// Attempt to decompress the data
compressed := buf.Bytes()
decompressed, err := ioutil.ReadAll(Decompressor(bytes.NewReader(compressed)))
|