Check-in [0083d7bfee]
Overview
Comment:Added a symbol-to-pair replacing reader in for the bpe
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | bpe
Files: files | file ages | folders
SHA1: 0083d7bfee14fcf5e8d81e0e558366da3157abd4
User & Date: spaskalev on 2014-12-28 13:57:50
Other Links: branch diff | manifest | tags
Context
2014-12-28
14:14
Added 0dev.org/ioutil.ReadByte() function and a test for it. CC at 100% check-in: 879630c89c user: spaskalev tags: bpe
13:57
Added a symbol-to-pair replacing reader in for the bpe check-in: 0083d7bfee user: spaskalev tags: bpe
12:44
Adding swaps recommendation for the bpe. check-in: a64b9a1ac3 user: spaskalev tags: bpe
Changes

Modified src/0dev.org/commands/short/main.go from [528d5e2695] to [a5a47b7f78].

    17     17   
    18     18   	pairs, symbols := analyze(f)
    19     19   
    20     20   	rec := recommend(pairs, symbols)
    21     21   	fmt.Println(*rec)
    22     22   }
    23     23   
    24         -type pair struct {
    25         -	value uint16
    26         -	count uint64
    27         -}
    28         -
    29         -type symbol struct {
    30         -	value byte
    31         -	count uint64
    32         -}
    33         -
    34     24   type recommendation struct {
    35     25   	p2s map[uint16]byte
    36     26   	s2p map[byte]uint16
    37     27   }
           28  +
           29  +func apply(rec *recommendation, reader io.Reader) {
           30  +	symbolReader := iou.SizedReader(iou.ReaderFunc(func(output []byte) (int, error) {
           31  +		for i := 0; i < len(output)-1; i++ {
           32  +			// Read a byte from the underlying reader
           33  +			count, err := reader.Read(output[i : i+1])
           34  +			if err != nil || count != 1 {
           35  +				return i + count, err
           36  +			}
           37  +
           38  +			// Convert the byte to a pair if there is a mapping for it
           39  +			if pair, ok := rec.s2p[output[i]]; ok {
           40  +				output[i] = byte(pair >> 8) // extract the high byte from the pair
           41  +				i++
           42  +				output[i] = byte(pair) // leave only the low byte from the pair
           43  +			}
           44  +		}
           45  +	}), 2)
           46  +}
    38     47   
    39     48   func recommend(pairs pairSlice, symbols symbolSlice) *recommendation {
    40     49   	var (
    41     50   		rec         recommendation
    42     51   		pairsLength = len(pairs)
    43     52   	)
    44     53   
................................................................................
   129    138   	for index, value := range symbols {
   130    139   		allSymbols = append(allSymbols, symbol{value: byte(index), count: value})
   131    140   	}
   132    141   	sort.Sort(allSymbols)
   133    142   
   134    143   	return availablePairs, allSymbols
   135    144   }
          145  +
          146  +type pair struct {
          147  +	value uint16
          148  +	count uint64
          149  +}
          150  +
          151  +type symbol struct {
          152  +	value byte
          153  +	count uint64
          154  +}
   136    155   
   137    156   // Implements fmt.Stringer, used for debugging
   138    157   func (p pair) String() string {
   139    158   	return fmt.Sprintf("[ %d %d (%d) ]", (p.value >> 8), ((p.value << 8) >> 8), p.count)
   140    159   }
   141    160   
   142    161   type pairSlice []pair