Commit 98e2528f authored by ale's avatar ale

Use a buffered Writer for WARC output

parent ee1a3d8e
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
package warc package warc
import ( import (
"bufio"
"fmt" "fmt"
"io" "io"
"time" "time"
...@@ -73,9 +74,10 @@ func NewHeader() Header { ...@@ -73,9 +74,10 @@ func NewHeader() Header {
// Writer can write records to a file in WARC format. It is safe // Writer can write records to a file in WARC format. It is safe
// for concurrent access, since writes are serialized internally. // for concurrent access, since writes are serialized internally.
type Writer struct { type Writer struct {
writer io.WriteCloser writer io.WriteCloser
gzwriter *gzip.Writer bufwriter *bufio.Writer
lockCh chan bool gzwriter *gzip.Writer
lockCh chan bool
} }
type recordWriter struct { type recordWriter struct {
...@@ -101,7 +103,7 @@ func (w *Writer) NewRecord(hdr Header) (io.WriteCloser, error) { ...@@ -101,7 +103,7 @@ func (w *Writer) NewRecord(hdr Header) (io.WriteCloser, error) {
w.gzwriter.Close() // nolint w.gzwriter.Close() // nolint
} }
var err error var err error
w.gzwriter, err = gzip.NewWriterLevel(w.writer, gzip.BestCompression) w.gzwriter, err = gzip.NewWriterLevel(w.bufwriter, gzip.BestCompression)
if err != nil { if err != nil {
return nil, err return nil, err
} }
...@@ -118,13 +120,17 @@ func (w *Writer) Close() error { ...@@ -118,13 +120,17 @@ func (w *Writer) Close() error {
if err := w.gzwriter.Close(); err != nil { if err := w.gzwriter.Close(); err != nil {
return err return err
} }
if err := w.bufwriter.Flush(); err != nil {
return err
}
return w.writer.Close() return w.writer.Close()
} }
// NewWriter initializes a new Writer and returns it. // NewWriter initializes a new Writer and returns it.
func NewWriter(w io.WriteCloser) *Writer { func NewWriter(w io.WriteCloser) *Writer {
return &Writer{ return &Writer{
writer: w, writer: w,
bufwriter: bufio.NewWriter(w),
// Buffering is important here since we're using this // Buffering is important here since we're using this
// channel as a semaphore. // channel as a semaphore.
lockCh: make(chan bool, 1), lockCh: make(chan bool, 1),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment