diff --git a/go.mod b/go.mod index 6e8005c61bbfad33ef1c5da480c205fc139a55b8..7b31a7632fa39681eceb32fde6a5051e6a82b809 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.autistici.org/ai3/tools/replds2 go 1.15 require ( - github.com/DataDog/zstd v1.4.8 + github.com/DataDog/zstd v1.5.0 github.com/armon/go-radix v1.0.0 github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/golang/protobuf v1.5.2 diff --git a/go.sum b/go.sum index 46c00860072e797d0f4b8bdec6db4f9e3070236c..e35627ca27b6be1776b136ca477dbd41b97c6a4b 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= +github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -510,10 +512,6 @@ google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/store/memlog/log.go b/store/memlog/log.go index 725e846cdfea81b5f0e787a88e6ff14497bf0e75..c40bd5e793624b8ef34f206382184bf6208681ec 100644 --- a/store/memlog/log.go +++ b/store/memlog/log.go @@ -59,9 +59,10 @@ func logWrite(w io.Writer, node *pb.Node, dirty bool) (int, error) { // Read an entry from the log, reusing the given buffer for decoding. // Returns the node, dirty bit, new buffer, error. func logRead(r io.Reader, buf []byte) (*pb.Node, bool, []byte, error) { - // Read carefully the size. + // Read carefully the size. We're using io.ReadFull here + // because zstd has a tendency to return short reads. var szbuf [4]byte - n, err := r.Read(szbuf[:]) + n, err := io.ReadFull(r, szbuf[:]) if err != nil { return nil, false, buf, err } @@ -72,7 +73,7 @@ func logRead(r io.Reader, buf []byte) (*pb.Node, bool, []byte, error) { // Read the dirty bit. var db [1]byte - _, err = r.Read(db[:]) + _, err = io.ReadFull(r, db[:]) if err != nil { return nil, false, buf, err } @@ -86,7 +87,7 @@ func logRead(r io.Reader, buf []byte) (*pb.Node, bool, []byte, error) { if len(buf) < sz { buf = make([]byte, sz) } - n, err = r.Read(buf[:sz]) + n, err = io.ReadFull(r, buf[:sz]) if err != nil { return nil, false, buf, err } diff --git a/store/memlog/log_test.go b/store/memlog/log_test.go index 2a4262b92bc389a0a3596d13fab59ebc5e936aed..553d146282163bda0dd31b19b1dd0c51bebbb0a4 100644 --- a/store/memlog/log_test.go +++ b/store/memlog/log_test.go @@ -58,7 +58,7 @@ func TestLog(t *testing.T) { if err := l.Write(node); err != nil { t.Fatal(err) } - m.set(node) + m.set(node) // nolint: errcheck } // We should be now on log #8. @@ -86,19 +86,21 @@ func TestLog(t *testing.T) { } testPath := fmt.Sprintf("/path/to/%08d", 10) - value, ok := m.m[testPath] - if !ok { + if _, ok := m.m[testPath]; !ok { t.Fatalf("expected value not found post reload") } // Modify the log, close it and reopen it, expecting it to be // dirty, and re-checkpointing on open. log.Printf(">>> writing new data, closing and re-opening log...") - l.Write(&pb.Node{ + err = l.Write(&pb.Node{ Path: testPath, Version: int64(time.Now().Unix()), Data: []byte("new data"), }) + if err != nil { + t.Fatal(err) + } l.Close() m = &testMap{m: make(map[string]*pb.Node)} @@ -106,7 +108,7 @@ func TestLog(t *testing.T) { if err != nil { t.Fatal(err) } - value, ok = m.m[testPath] + value, ok := m.m[testPath] if !ok { t.Fatal("expected value not found post reload") }