Overview
Comment: | Implemented commands/pdc using predictor. Made predictor's Compressor(...) return an io.Writer. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
c9f3a59cb6f8359e28b9333fb4f6a661 |
User & Date: | spaskalev on 2014-12-19 21:54:39 |
Other Links: | manifest | tags |
Context
2014-12-20
| ||
11:52 | [predictor] Removed the buffer from the context struct, allocate the input slice buffer on creation with make. check-in: 723ffeb1fd user: spaskalev tags: trunk | |
2014-12-19
| ||
21:54 | Implemented commands/pdc using predictor. Made predictor's Compressor(...) return an io.Writer. check-in: c9f3a59cb6 user: spaskalev tags: trunk | |
2014-12-16
| ||
23:29 | Removed err variable from compressor check-in: 0f4bc650d1 user: spaskalev tags: trunk | |
Changes
Added src/0dev.org/commands/pdc/main.go version [3c0c8016e1].
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 | package main import ( predictor "0dev.org/predictor" "bufio" "fmt" "io" "os" ) func main() { switch { case len(os.Args) == 1: os.Exit(compress(os.Stdout, os.Stdin)) case len(os.Args) == 2 && os.Args[1] == "-d": os.Exit(decompress(os.Stdout, os.Stdin)) default: fmt.Fprintln(os.Stdout, "Usage: pdc [-d]") } } // Compress the data from the given io.Reader and write it to the given io.Writer // I/O is buffered for better performance func compress(output io.Writer, input io.Reader) int { var ( err error buffer *bufio.Writer = bufio.NewWriter(output) compressor io.Writer = predictor.Compressor(buffer) ) _, err = io.Copy(compressor, bufio.NewReader(input)) if err != nil { fmt.Fprintln(os.Stderr, "Error while compressing.\n", err) return 1 } // Flush the compressor _, err = compressor.Write(nil) if err != nil { fmt.Fprintln(os.Stderr, "Error while flushing compresssor buffer.\n", err) return 1 } // Flush the buffer err = buffer.Flush() if err != nil { fmt.Fprintln(os.Stderr, "Error while flushing output buffer.\n", err) return 1 } return 0 } // Decompress the data from the given io.Reader and write it to the given io.Writer // I/O is buffered for better performance func decompress(output io.Writer, input io.Reader) int { var ( err error buffer *bufio.Writer = bufio.NewWriter(output) decompressor io.Reader = predictor.Decompressor(input) ) _, err = io.Copy(buffer, bufio.NewReader(decompressor)) if err != nil { fmt.Fprintln(os.Stderr, "Error while decompressing.\n", err) return 1 } // Flush err = buffer.Flush() if err != nil { fmt.Fprintln(os.Stderr, "Error while flushing output buffer.\n", err) return 1 } return 0 } |
Modified src/0dev.org/commands/plaindiff/main.go from [f98671241a] to [b2d6e402f0].
1 2 3 4 5 6 7 8 9 10 11 12 | package main import ( diff "0dev.org/diff" "bufio" "fmt" "hash" "hash/fnv" "io" "os" ) | | | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | package main import ( diff "0dev.org/diff" "bufio" "fmt" "hash" "hash/fnv" "io" "os" ) const usage = "Usage: plaindiff <file1> <file2>\n" func main() { var args []string = os.Args if len(args) != 3 { os.Stderr.WriteString(usage) os.Exit(1) } |
︙ | ︙ |
Modified src/0dev.org/predictor/predictor.go from [2cc1907fb9] to [c65cc5c256].
︙ | ︙ | |||
9 10 11 12 13 14 15 | type context struct { table [1 << 16]byte buffer [1 << 3]byte input []byte hash uint16 } | > | > > > > > > | | | 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | type context struct { table [1 << 16]byte buffer [1 << 3]byte input []byte hash uint16 } type compressor func([]byte) error func (w compressor) Write(data []byte) (int, error) { return len(data), w(data) } // Returns an io.Writer implementation that wraps the provided io.Writer // and compresses data according to the predictor algorithm // // It can buffer data as the predictor mandates 8-byte blocks with a header. // A call with no data will force a flush. func Compressor(writer io.Writer) io.Writer { var ctx context ctx.input = ctx.buffer[:0] // Forward declaration as it is required for recursion var write compressor write = func(data []byte) error { var ( blockSize int = 8 bufferLength int = len(ctx.input) ) |
︙ | ︙ | |||
99 100 101 102 103 104 105 | return nil } return write } | > | > | | > | | 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 | return nil } return write } // A function type alias so that it can have methods attached to it type decompressor func([]byte) (int, error) // Required to implement io.Reader func (r decompressor) Read(output []byte) (int, error) { return r(output) } // Returns an io.Reader implementation that wraps the provided io.Reader // and decompresses data according to the predictor algorithm func Decompressor(wrapped io.Reader) io.Reader { var ctx context ctx.input = ctx.buffer[:0] return decompressor(func(output []byte) (int, error) { var ( err error flags byte readCount int ) // Sanity check for space to read into |
︙ | ︙ |
Modified src/0dev.org/predictor/predictor_test.go from [73a8f932ed] to [dd11cf49de].
︙ | ︙ | |||
27 28 29 30 31 32 33 | func TestCompressorSample(t *testing.T) { var ( buf bytes.Buffer err error ) out := Compressor(&buf) | | | | 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | func TestCompressorSample(t *testing.T) { var ( buf bytes.Buffer err error ) out := Compressor(&buf) _, err = out.Write(input) if err != nil { t.Error(err) } _, err = out.Write(nil) if err != nil { t.Error(err) } result := buf.Bytes() delta := diff.Diff(diff.D{len(result), len(output), func(i, j int) bool { return result[i] == output[j] }}) |
︙ | ︙ | |||
107 108 109 110 111 112 113 | var trace []byte = make([]byte, 0) for len(data) > 0 { if step <= len(data) { trace = append(trace, data[:step]...) | | | | 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | var trace []byte = make([]byte, 0) for len(data) > 0 { if step <= len(data) { trace = append(trace, data[:step]...) _, err = compressor.Write(data[:step]) if err != nil { return err } data = data[step:] } else { step = len(data) } } // Flush the compressor _, err = compressor.Write(nil) if err != nil { return err } // Attempt to decompress the data compressed := buf.Bytes() decompressed, err := ioutil.ReadAll(Decompressor(bytes.NewReader(compressed))) |
︙ | ︙ |