Commit 5db3dfb8 authored by Luke Champine's avatar Luke Champine

add ECC interface and Reed-Solomon benchmarks

parent 764c9970
......@@ -16,6 +16,7 @@ dependencies:
go get -u github.com/inconshreveable/go-update
go get -u github.com/inconshreveable/muxado
go get -u github.com/kardianos/osext
go get -u github.com/klauspost/reedsolomon
go get -u github.com/laher/goxc
go get -u github.com/spf13/cobra
go get -u github.com/stretchr/graceful
......
package modules
import (
"io"
"time"
"github.com/NebulousLabs/Sia/types"
......@@ -10,6 +11,26 @@ var (
RenterDir = "renter"
)
// An ECC is an error correcting code.
type ECC interface {
// ChunkSize is the size of one chunk.
ChunkSize() uint64
// NumPieces is the number of pieces per chunk.
NumPieces() int
// Encode reads a chunk from r (a byte slice of length ChunkSize) and
// splits it into equal-length pieces, with some pieces containing parity
// data. The total number of pieces is equal to NumPieces.
Encode(r io.Reader) ([][]byte, error)
// Recover recovers the original data from pieces (including parity) and
// writes it to w. pieces should be identical to the slice returned by
// Encode (length and order must be preserved), but with missing elements
// set to nil.
Recover(pieces [][]byte, w io.Writer) error
}
// FileUploadParams contains the information used by the Renter to upload a
// file.
type FileUploadParams struct {
......
package renter
import (
"io"
"github.com/klauspost/reedsolomon"
"github.com/NebulousLabs/Sia/modules"
)
// rsCode is a Reed-Solomon encoder/decoder. It implements the modules.ECC
// interface.
type rsCode struct {
enc reedsolomon.Encoder
chunk []byte // allocated during initialization to save memory
chunkSize uint64
numPieces int
}
func (rs *rsCode) ChunkSize() uint64 { return rs.chunkSize }
func (rs *rsCode) NumPieces() int { return rs.numPieces }
// Encode reads a chunk from r and splits it into equal-length pieces. If a
// full chunk cannot be read, the remainder of the chunk will contain zeros.
func (rs *rsCode) Encode(r io.Reader) ([][]byte, error) {
_, err := io.ReadFull(r, rs.chunk)
if err != nil && err != io.ErrUnexpectedEOF {
return nil, err
}
pieces, err := rs.enc.Split(rs.chunk)
if err != nil {
return nil, err
}
err = rs.enc.Encode(pieces)
if err != nil {
return nil, err
}
return pieces, nil
}
// Recover recovers the original data from pieces (including parity) and
// writes it to w. pieces should be identical to the slice returned by
// Encode (length and order must be preserved), but with missing elements
// set to nil.
func (rs *rsCode) Recover(pieces [][]byte, w io.Writer) error {
err := rs.enc.Reconstruct(pieces)
if err != nil {
return err
}
// TODO: implement this manually
return rs.enc.Join(w, pieces, int(rs.chunkSize))
}
// NewRSCode creates a new Reed-Solomon encoder/decoder using the supplied
// parameters.
func NewRSCode(nData, nParity int, chunksize uint64) (modules.ECC, error) {
enc, err := reedsolomon.New(nData, nParity)
if err != nil {
return nil, err
}
return &rsCode{
enc: enc,
chunk: make([]byte, chunksize),
chunkSize: chunksize,
numPieces: nData + nParity,
}, nil
}
package renter
import (
"bytes"
"crypto/rand"
"io"
"io/ioutil"
"testing"
)
func TestRSEncode(t *testing.T) {
ecc, err := NewRSCode(10, 3, 130)
if err != nil {
t.Fatal(err)
}
data := make([]byte, 650)
rand.Read(data)
r := bytes.NewReader(data)
buf := new(bytes.Buffer)
for {
pieces, err := ecc.Encode(r)
if err == io.EOF {
break
} else if err != nil {
t.Fatal(err)
}
err = ecc.Recover(pieces, buf)
if err != nil {
t.Fatal(err)
}
}
if bytes.Compare(data, buf.Bytes()) != 0 {
t.Fatal("recovered data does not match original")
}
}
func BenchmarkRSEncode(b *testing.B) {
ecc, err := NewRSCode(80, 20, 1<<20) // 1 MB
if err != nil {
panic(err)
}
data := make([]byte, 1<<20)
rand.Read(data)
r := bytes.NewReader(data)
b.ResetTimer()
for i := 0; i < b.N; i++ {
r.Seek(0, 0)
ecc.Encode(r)
}
}
func BenchmarkRSRecover(b *testing.B) {
ecc, err := NewRSCode(50, 200, 1<<20)
if err != nil {
panic(err)
}
pieces, err := ecc.Encode(rand.Reader)
if err != nil {
panic(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
ecc.Recover(pieces, ioutil.Discard)
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment