Check-in [879630c89c]
Overview
Comment:Added 0dev.org/ioutil.ReadByte() function and a test for it. CC at 100%
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | bpe
Files: files | file ages | folders
SHA1: 879630c89c1c391a3aaaabf31762d7110c0d45f6
User & Date: spaskalev on 2014-12-28 14:14:36
Other Links: branch diff | manifest | tags
Context
2014-12-28
17:19
Integrate ioutil.ReadByte from the bpe branch into trunk. check-in: 00c4e0e448 user: spaskalev tags: trunk
15:21
bpe encoding flow implementation check-in: 7733ef9df8 user: spaskalev tags: bpe
14:14
Added 0dev.org/ioutil.ReadByte() function and a test for it. CC at 100% check-in: 879630c89c user: spaskalev tags: bpe
13:57
Added a symbol-to-pair replacing reader in for the bpe check-in: 0083d7bfee user: spaskalev tags: bpe
Changes

Modified src/0dev.org/commands/short/main.go from [a5a47b7f78] to [2e2860e831].

24
25
26
27
28
29
30

31
32
33
34


35
36
37
38
39
40
41
42
43
44






45






46
47
48
49
50
51
52
type recommendation struct {
	p2s map[uint16]byte
	s2p map[byte]uint16
}

func apply(rec *recommendation, reader io.Reader) {
	symbolReader := iou.SizedReader(iou.ReaderFunc(func(output []byte) (int, error) {

		for i := 0; i < len(output)-1; i++ {
			// Read a byte from the underlying reader
			count, err := reader.Read(output[i : i+1])
			if err != nil || count != 1 {


				return i + count, err
			}

			// Convert the byte to a pair if there is a mapping for it
			if pair, ok := rec.s2p[output[i]]; ok {
				output[i] = byte(pair >> 8) // extract the high byte from the pair
				i++
				output[i] = byte(pair) // leave only the low byte from the pair
			}
		}






	}), 2)






}

func recommend(pairs pairSlice, symbols symbolSlice) *recommendation {
	var (
		rec         recommendation
		pairsLength = len(pairs)
	)







>
|


|
>
>
|








|
>
>
>
>
>
>

>
>
>
>
>
>







24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
type recommendation struct {
	p2s map[uint16]byte
	s2p map[byte]uint16
}

func apply(rec *recommendation, reader io.Reader) {
	symbolReader := iou.SizedReader(iou.ReaderFunc(func(output []byte) (int, error) {
		var i int = 0
		for ; i < len(output)-1; i++ {
			// Read a byte from the underlying reader
			count, err := reader.Read(output[i : i+1])

			// If we can't read anything else - return immediatelly
			if count == 0 {
				return i, err
			}

			// Convert the byte to a pair if there is a mapping for it
			if pair, ok := rec.s2p[output[i]]; ok {
				output[i] = byte(pair >> 8) // extract the high byte from the pair
				i++
				output[i] = byte(pair) // leave only the low byte from the pair
			}

			// Return on error
			if err != nil {
				return i + 1, err
			}
		}
		return i + 1, nil
	}), 2)

	pairReader := iou.ReaderFunc(func(output []byte) (int, error) {
		for i := 0; i < len(output); i++ {

		}
	})
}

func recommend(pairs pairSlice, symbols symbolSlice) *recommendation {
	var (
		rec         recommendation
		pairsLength = len(pairs)
	)

Modified src/0dev.org/ioutil/ioutil.go from [c11fdb5a53] to [83fc64874b].

16
17
18
19
20
21
22










23
24
25
26
27
28
29
// An function alias type that implements io.Reader.
type ReaderFunc func([]byte) (int, error)

// Delegates the call to the WriterFunc while implementing io.Reader.
func (r ReaderFunc) Read(b []byte) (int, error) {
	return r(b)
}











// Returns a writer that delegates calls to Write(...) while ensuring
// that it is never called with less bytes than the specified amount.
//
// Calls with fewer bytes are buffered while a call with a nil slice
// causes the buffer to be flushed to the underlying writer.
func SizedWriter(writer io.Writer, size int) io.Writer {







>
>
>
>
>
>
>
>
>
>







16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
// An function alias type that implements io.Reader.
type ReaderFunc func([]byte) (int, error)

// Delegates the call to the WriterFunc while implementing io.Reader.
func (r ReaderFunc) Read(b []byte) (int, error) {
	return r(b)
}

// Reads a single byte from the provided io.Reader
func ReadByte(reader io.Reader) (byte, error) {
	var (
		arr [1]byte
		err error
	)
	_, err = reader.Read(arr[:])
	return arr[0], err
}

// Returns a writer that delegates calls to Write(...) while ensuring
// that it is never called with less bytes than the specified amount.
//
// Calls with fewer bytes are buffered while a call with a nil slice
// causes the buffer to be flushed to the underlying writer.
func SizedWriter(writer io.Writer, size int) io.Writer {

Modified src/0dev.org/ioutil/ioutil_test.go from [91aaac4713] to [514a112f00].

43
44
45
46
47
48
49




















50
51
52
53
54
55
56
	// Diff the result against the initial input
	delta := diff.Diff(diff.D{len(input), len(output),
		func(i, j int) bool { return input[i] == output[j] }})
	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
		t.Error("Differences detected ", delta)
	}
}





















func TestSizedWriter(t *testing.T) {
	var (
		buffer bytes.Buffer
		writer io.Writer = SizedWriter(&buffer, 4)
	)








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
	// Diff the result against the initial input
	delta := diff.Diff(diff.D{len(input), len(output),
		func(i, j int) bool { return input[i] == output[j] }})
	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
		t.Error("Differences detected ", delta)
	}
}

func TestReadByte(t *testing.T) {
	var (
		input  []byte        = []byte{255}
		reader *bytes.Reader = bytes.NewReader(input)
	)

	result, err := ReadByte(reader)
	if result != input[0] {
		t.Error("Unexpected read result from ReadByte", result)
	}
	if err != nil {
		t.Error("Unexpected error from ReadByte", err)
	}

	result, err = ReadByte(reader)
	if err != io.EOF {
		t.Error("Unexpected nil error from ReadByte, read value:", result)
	}
}

func TestSizedWriter(t *testing.T) {
	var (
		buffer bytes.Buffer
		writer io.Writer = SizedWriter(&buffer, 4)
	)