diff --git a/services/database/backuptool/backuptool.go b/services/database/backuptool/backuptool.go index 5e41c1b7280a9c709b8d4b17c892477a89219685..482b7ed6cb360490c00b4a87febb8b4615dff5ad 100644 --- a/services/database/backuptool/backuptool.go +++ b/services/database/backuptool/backuptool.go @@ -84,24 +84,40 @@ func scanner(db *db_client.DatabaseRpcClient, bucketCh chan string, outCh chan R defer session.Close() for bucket := range bucketCh { - iter, err := session.Scan(bucket, "", "\xff", -1) - if err != nil { - log.Printf("Scan error for bucket %s: %s", bucket, err) - continue - } - - for { - var data []byte - key, err := iter.Next(&data) + // Call Scan() in batches, to avoid overloading the server. + batchSize := 1000 + startKey, endKey := "", "\xff" + for startKey < endKey { + iter, err := session.Scan(bucket, startKey, endKey, batchSize) if err != nil { + log.Printf("Scan error for bucket %s: %s", bucket, err) break } - outCh <- Record{ - Bucket: bucket, - Key: key, - Data: data, + // Iterate over the results and send them to outCh. + n := 0 + for { + var data []byte + key, err := iter.Next(&data) + if err != nil { + break + } + + outCh <- Record{ + Bucket: bucket, + Key: key, + Data: data, + } + n++ + startKey = key + } + // An empty result means we've reached the end + // of the scan. + if n == 0 { + break } + // Start the next scan from the next key. + startKey = db_client.IncrementKey(startKey) } } }