Overview
Comment: | Implemented commands/pdc using predictor. Made predictor's Compressor(...) return an io.Writer. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
c9f3a59cb6f8359e28b9333fb4f6a661 |
User & Date: | spaskalev on 2014-12-19 21:54:39 |
Other Links: | manifest | tags |
Context
2014-12-20
| ||
11:52 | [predictor] Removed the buffer from the context struct, allocate the input slice buffer on creation with make. check-in: 723ffeb1fd user: spaskalev tags: trunk | |
2014-12-19
| ||
21:54 | Implemented commands/pdc using predictor. Made predictor's Compressor(...) return an io.Writer. check-in: c9f3a59cb6 user: spaskalev tags: trunk | |
2014-12-16
| ||
23:29 | Removed err variable from compressor check-in: 0f4bc650d1 user: spaskalev tags: trunk | |
Changes
Added src/0dev.org/commands/pdc/main.go version [3c0c8016e1].
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
package main import ( predictor "0dev.org/predictor" "bufio" "fmt" "io" "os" ) func main() { switch { case len(os.Args) == 1: os.Exit(compress(os.Stdout, os.Stdin)) case len(os.Args) == 2 && os.Args[1] == "-d": os.Exit(decompress(os.Stdout, os.Stdin)) default: fmt.Fprintln(os.Stdout, "Usage: pdc [-d]") } } // Compress the data from the given io.Reader and write it to the given io.Writer // I/O is buffered for better performance func compress(output io.Writer, input io.Reader) int { var ( err error buffer *bufio.Writer = bufio.NewWriter(output) compressor io.Writer = predictor.Compressor(buffer) ) _, err = io.Copy(compressor, bufio.NewReader(input)) if err != nil { fmt.Fprintln(os.Stderr, "Error while compressing.\n", err) return 1 } // Flush the compressor _, err = compressor.Write(nil) if err != nil { fmt.Fprintln(os.Stderr, "Error while flushing compresssor buffer.\n", err) return 1 } // Flush the buffer err = buffer.Flush() if err != nil { fmt.Fprintln(os.Stderr, "Error while flushing output buffer.\n", err) return 1 } return 0 } // Decompress the data from the given io.Reader and write it to the given io.Writer // I/O is buffered for better performance func decompress(output io.Writer, input io.Reader) int { var ( err error buffer *bufio.Writer = bufio.NewWriter(output) decompressor io.Reader = predictor.Decompressor(input) ) _, err = io.Copy(buffer, bufio.NewReader(decompressor)) if err != nil { fmt.Fprintln(os.Stderr, "Error while decompressing.\n", err) return 1 } // Flush err = buffer.Flush() if err != nil { fmt.Fprintln(os.Stderr, "Error while flushing output buffer.\n", err) return 1 } return 0 } |
Modified src/0dev.org/commands/plaindiff/main.go from [f98671241a] to [b2d6e402f0].
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
"fmt" "hash" "hash/fnv" "io" "os" ) const usage = "Usage: plaindiff <file1> <file2>" func main() { var args []string = os.Args if len(args) != 3 { os.Stderr.WriteString(usage) os.Exit(1) } |
| |
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
"fmt"
"hash"
"hash/fnv"
"io"
"os"
)
const usage = "Usage: plaindiff <file1> <file2>\n"
func main() {
var args []string = os.Args
if len(args) != 3 {
os.Stderr.WriteString(usage)
os.Exit(1)
}
|
Modified src/0dev.org/predictor/predictor.go from [2cc1907fb9] to [c65cc5c256].
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
..
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
type context struct { table [1 << 16]byte buffer [1 << 3]byte input []byte hash uint16 } // Returns a closure over the provided writer that compresses data when called. // // It can buffer data as the predictor mandates 8-byte blocks with a header. // A call with no data will force a flush. func Compressor(writer io.Writer) func([]byte) error { var ctx context ctx.input = ctx.buffer[:0] // Forward declaration as it is required for recursion var write func(data []byte) error write = func(data []byte) error { var ( blockSize int = 8 bufferLength int = len(ctx.input) ) ................................................................................ return nil } return write } type reader func([]byte) (int, error) func (r reader) Read(output []byte) (int, error) { return r(output) } // TODO - document func Decompressor(wrapped io.Reader) io.Reader { var ctx context ctx.input = ctx.buffer[:0] return reader(func(output []byte) (int, error) { var ( err error flags byte readCount int ) // Sanity check for space to read into |
|
>
>
>
>
>
>
>
|
|
>
|
>
|
|
>
|
|
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
...
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
type context struct { table [1 << 16]byte buffer [1 << 3]byte input []byte hash uint16 } type compressor func([]byte) error func (w compressor) Write(data []byte) (int, error) { return len(data), w(data) } // Returns an io.Writer implementation that wraps the provided io.Writer // and compresses data according to the predictor algorithm // // It can buffer data as the predictor mandates 8-byte blocks with a header. // A call with no data will force a flush. func Compressor(writer io.Writer) io.Writer { var ctx context ctx.input = ctx.buffer[:0] // Forward declaration as it is required for recursion var write compressor write = func(data []byte) error { var ( blockSize int = 8 bufferLength int = len(ctx.input) ) ................................................................................ return nil } return write } // A function type alias so that it can have methods attached to it type decompressor func([]byte) (int, error) // Required to implement io.Reader func (r decompressor) Read(output []byte) (int, error) { return r(output) } // Returns an io.Reader implementation that wraps the provided io.Reader // and decompresses data according to the predictor algorithm func Decompressor(wrapped io.Reader) io.Reader { var ctx context ctx.input = ctx.buffer[:0] return decompressor(func(output []byte) (int, error) { var ( err error flags byte readCount int ) // Sanity check for space to read into |
Modified src/0dev.org/predictor/predictor_test.go from [73a8f932ed] to [dd11cf49de].
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
...
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
func TestCompressorSample(t *testing.T) {
var (
buf bytes.Buffer
err error
)
out := Compressor(&buf)
err = out(input)
if err != nil {
t.Error(err)
}
err = out(nil)
if err != nil {
t.Error(err)
}
result := buf.Bytes()
delta := diff.Diff(diff.D{len(result), len(output), func(i, j int) bool { return result[i] == output[j] }})
................................................................................
var trace []byte = make([]byte, 0)
for len(data) > 0 {
if step <= len(data) {
trace = append(trace, data[:step]...)
err = compressor(data[:step])
if err != nil {
return err
}
data = data[step:]
} else {
step = len(data)
}
}
// Flush the compressor
err = compressor(nil)
if err != nil {
return err
}
// Attempt to decompress the data
compressed := buf.Bytes()
decompressed, err := ioutil.ReadAll(Decompressor(bytes.NewReader(compressed)))
|
|
|
|
|
|
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
...
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
func TestCompressorSample(t *testing.T) { var ( buf bytes.Buffer err error ) out := Compressor(&buf) _, err = out.Write(input) if err != nil { t.Error(err) } _, err = out.Write(nil) if err != nil { t.Error(err) } result := buf.Bytes() delta := diff.Diff(diff.D{len(result), len(output), func(i, j int) bool { return result[i] == output[j] }}) ................................................................................ var trace []byte = make([]byte, 0) for len(data) > 0 { if step <= len(data) { trace = append(trace, data[:step]...) _, err = compressor.Write(data[:step]) if err != nil { return err } data = data[step:] } else { step = len(data) } } // Flush the compressor _, err = compressor.Write(nil) if err != nil { return err } // Attempt to decompress the data compressed := buf.Bytes() decompressed, err := ioutil.ReadAll(Decompressor(bytes.NewReader(compressed))) |