Check-in [879630c89c]
Overview
Comment:Added 0dev.org/ioutil.ReadByte() function and a test for it. CC at 100%
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | bpe
Files: files | file ages | folders
SHA1: 879630c89c1c391a3aaaabf31762d7110c0d45f6
User & Date: spaskalev on 2014-12-28 14:14:36.311
Other Links: branch diff | manifest | tags
Context
2014-12-28
17:19
Integrate ioutil.ReadByte from the bpe branch into trunk. check-in: 00c4e0e448 user: spaskalev tags: trunk
15:21
bpe encoding flow implementation check-in: 7733ef9df8 user: spaskalev tags: bpe
14:14
Added 0dev.org/ioutil.ReadByte() function and a test for it. CC at 100% check-in: 879630c89c user: spaskalev tags: bpe
13:57
Added a symbol-to-pair replacing reader in for the bpe check-in: 0083d7bfee user: spaskalev tags: bpe
Changes
24
25
26
27
28
29
30

31
32
33
34


35
36
37
38
39
40
41
42
43
44






45






46
47
48
49
50
51
52
type recommendation struct {
	p2s map[uint16]byte
	s2p map[byte]uint16
}

func apply(rec *recommendation, reader io.Reader) {
	symbolReader := iou.SizedReader(iou.ReaderFunc(func(output []byte) (int, error) {

		for i := 0; i < len(output)-1; i++ {
			// Read a byte from the underlying reader
			count, err := reader.Read(output[i : i+1])
			if err != nil || count != 1 {


				return i + count, err
			}

			// Convert the byte to a pair if there is a mapping for it
			if pair, ok := rec.s2p[output[i]]; ok {
				output[i] = byte(pair >> 8) // extract the high byte from the pair
				i++
				output[i] = byte(pair) // leave only the low byte from the pair
			}
		}






	}), 2)






}

func recommend(pairs pairSlice, symbols symbolSlice) *recommendation {
	var (
		rec         recommendation
		pairsLength = len(pairs)
	)







>
|


|
>
>
|








|
>
>
>
>
>
>

>
>
>
>
>
>







24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
type recommendation struct {
	p2s map[uint16]byte
	s2p map[byte]uint16
}

func apply(rec *recommendation, reader io.Reader) {
	symbolReader := iou.SizedReader(iou.ReaderFunc(func(output []byte) (int, error) {
		var i int = 0
		for ; i < len(output)-1; i++ {
			// Read a byte from the underlying reader
			count, err := reader.Read(output[i : i+1])

			// If we can't read anything else - return immediatelly
			if count == 0 {
				return i, err
			}

			// Convert the byte to a pair if there is a mapping for it
			if pair, ok := rec.s2p[output[i]]; ok {
				output[i] = byte(pair >> 8) // extract the high byte from the pair
				i++
				output[i] = byte(pair) // leave only the low byte from the pair
			}

			// Return on error
			if err != nil {
				return i + 1, err
			}
		}
		return i + 1, nil
	}), 2)

	pairReader := iou.ReaderFunc(func(output []byte) (int, error) {
		for i := 0; i < len(output); i++ {

		}
	})
}

func recommend(pairs pairSlice, symbols symbolSlice) *recommendation {
	var (
		rec         recommendation
		pairsLength = len(pairs)
	)
16
17
18
19
20
21
22










23
24
25
26
27
28
29
// An function alias type that implements io.Reader.
type ReaderFunc func([]byte) (int, error)

// Delegates the call to the WriterFunc while implementing io.Reader.
func (r ReaderFunc) Read(b []byte) (int, error) {
	return r(b)
}











// Returns a writer that delegates calls to Write(...) while ensuring
// that it is never called with less bytes than the specified amount.
//
// Calls with fewer bytes are buffered while a call with a nil slice
// causes the buffer to be flushed to the underlying writer.
func SizedWriter(writer io.Writer, size int) io.Writer {







>
>
>
>
>
>
>
>
>
>







16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
// An function alias type that implements io.Reader.
type ReaderFunc func([]byte) (int, error)

// Delegates the call to the WriterFunc while implementing io.Reader.
func (r ReaderFunc) Read(b []byte) (int, error) {
	return r(b)
}

// Reads a single byte from the provided io.Reader
func ReadByte(reader io.Reader) (byte, error) {
	var (
		arr [1]byte
		err error
	)
	_, err = reader.Read(arr[:])
	return arr[0], err
}

// Returns a writer that delegates calls to Write(...) while ensuring
// that it is never called with less bytes than the specified amount.
//
// Calls with fewer bytes are buffered while a call with a nil slice
// causes the buffer to be flushed to the underlying writer.
func SizedWriter(writer io.Writer, size int) io.Writer {
43
44
45
46
47
48
49




















50
51
52
53
54
55
56
	// Diff the result against the initial input
	delta := diff.Diff(diff.D{len(input), len(output),
		func(i, j int) bool { return input[i] == output[j] }})
	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
		t.Error("Differences detected ", delta)
	}
}





















func TestSizedWriter(t *testing.T) {
	var (
		buffer bytes.Buffer
		writer io.Writer = SizedWriter(&buffer, 4)
	)








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
	// Diff the result against the initial input
	delta := diff.Diff(diff.D{len(input), len(output),
		func(i, j int) bool { return input[i] == output[j] }})
	if len(delta.Added) > 0 || len(delta.Removed) > 0 {
		t.Error("Differences detected ", delta)
	}
}

func TestReadByte(t *testing.T) {
	var (
		input  []byte        = []byte{255}
		reader *bytes.Reader = bytes.NewReader(input)
	)

	result, err := ReadByte(reader)
	if result != input[0] {
		t.Error("Unexpected read result from ReadByte", result)
	}
	if err != nil {
		t.Error("Unexpected error from ReadByte", err)
	}

	result, err = ReadByte(reader)
	if err != io.EOF {
		t.Error("Unexpected nil error from ReadByte, read value:", result)
	}
}

func TestSizedWriter(t *testing.T) {
	var (
		buffer bytes.Buffer
		writer io.Writer = SizedWriter(&buffer, 4)
	)