Commit cae8ed84 authored by ale's avatar ale

Add size-related metrics to Dataset metadata

We can keep track of how large the backups are in the metadb.
parent 89474259
Pipeline #8318 passed with stages
in 1 minute and 37 seconds
ALTER TABLE log DROP COLUMN dataset_total_files;
ALTER TABLE log DROP COLUMN dataset_total_bytes;
ALTER TABLE log DROP COLUMN dataset_bytes_added;
ALTER TABLE log DROP COLUMN dataset_duration;
ALTER TABLE log ADD COLUMN dataset_total_files INTEGER DEFAULT 0;
ALTER TABLE log ADD COLUMN dataset_total_bytes INTEGER DEFAULT 0;
ALTER TABLE log ADD COLUMN dataset_bytes_added INTEGER DEFAULT 0;
ALTER TABLE log ADD COLUMN dataset_duration INTEGER DEFAULT 0;
......@@ -4,6 +4,8 @@
// migrations/1_initialize_schema.up.sql
// migrations/2_add_snapshot_id.down.sql
// migrations/2_add_snapshot_id.up.sql
// migrations/3_add_dataset_metrics.down.sql
// migrations/3_add_dataset_metrics.up.sql
// DO NOT EDIT!
package migrations
......@@ -65,7 +67,7 @@ func _1_initialize_schemaDownSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "1_initialize_schema.down.sql", size: 121, mode: os.FileMode(420), modTime: time.Unix(1560983647, 0)}
info := bindataFileInfo{name: "1_initialize_schema.down.sql", size: 121, mode: os.FileMode(420), modTime: time.Unix(1576883528, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
......@@ -100,7 +102,7 @@ func _1_initialize_schemaUpSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "1_initialize_schema.up.sql", size: 568, mode: os.FileMode(420), modTime: time.Unix(1560983647, 0)}
info := bindataFileInfo{name: "1_initialize_schema.up.sql", size: 568, mode: os.FileMode(420), modTime: time.Unix(1576883528, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
......@@ -119,7 +121,7 @@ func _2_add_snapshot_idDownSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "2_add_snapshot_id.down.sql", size: 50, mode: os.FileMode(420), modTime: time.Unix(1571647185, 0)}
info := bindataFileInfo{name: "2_add_snapshot_id.down.sql", size: 50, mode: os.FileMode(420), modTime: time.Unix(1576883528, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
......@@ -138,7 +140,51 @@ func _2_add_snapshot_idUpSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "2_add_snapshot_id.up.sql", size: 61, mode: os.FileMode(420), modTime: time.Unix(1571647193, 0)}
info := bindataFileInfo{name: "2_add_snapshot_id.up.sql", size: 61, mode: os.FileMode(420), modTime: time.Unix(1576883528, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
var __3_add_dataset_metricsDownSql = []byte(`
ALTER TABLE log DROP COLUMN dataset_total_files;
ALTER TABLE log DROP COLUMN dataset_total_bytes;
ALTER TABLE log DROP COLUMN dataset_bytes_added;
ALTER TABLE log DROP COLUMN dataset_duration;
`)
func _3_add_dataset_metricsDownSqlBytes() ([]byte, error) {
return __3_add_dataset_metricsDownSql, nil
}
func _3_add_dataset_metricsDownSql() (*asset, error) {
bytes, err := _3_add_dataset_metricsDownSqlBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "3_add_dataset_metrics.down.sql", size: 194, mode: os.FileMode(420), modTime: time.Unix(1604251834, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
var __3_add_dataset_metricsUpSql = []byte(`
ALTER TABLE log ADD COLUMN dataset_total_files INTEGER DEFAULT 0;
ALTER TABLE log ADD COLUMN dataset_total_bytes INTEGER DEFAULT 0;
ALTER TABLE log ADD COLUMN dataset_bytes_added INTEGER DEFAULT 0;
ALTER TABLE log ADD COLUMN dataset_duration INTEGER DEFAULT 0;
`)
func _3_add_dataset_metricsUpSqlBytes() ([]byte, error) {
return __3_add_dataset_metricsUpSql, nil
}
func _3_add_dataset_metricsUpSql() (*asset, error) {
bytes, err := _3_add_dataset_metricsUpSqlBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "3_add_dataset_metrics.up.sql", size: 262, mode: os.FileMode(420), modTime: time.Unix(1604251803, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
......@@ -199,6 +245,8 @@ var _bindata = map[string]func() (*asset, error){
"1_initialize_schema.up.sql": _1_initialize_schemaUpSql,
"2_add_snapshot_id.down.sql": _2_add_snapshot_idDownSql,
"2_add_snapshot_id.up.sql": _2_add_snapshot_idUpSql,
"3_add_dataset_metrics.down.sql": _3_add_dataset_metricsDownSql,
"3_add_dataset_metrics.up.sql": _3_add_dataset_metricsUpSql,
}
// AssetDir returns the file names below a certain
......@@ -245,6 +293,8 @@ var _bintree = &bintree{nil, map[string]*bintree{
"1_initialize_schema.up.sql": &bintree{_1_initialize_schemaUpSql, map[string]*bintree{}},
"2_add_snapshot_id.down.sql": &bintree{_2_add_snapshot_idDownSql, map[string]*bintree{}},
"2_add_snapshot_id.up.sql": &bintree{_2_add_snapshot_idUpSql, map[string]*bintree{}},
"3_add_dataset_metrics.down.sql": &bintree{_3_add_dataset_metricsDownSql, map[string]*bintree{}},
"3_add_dataset_metrics.up.sql": &bintree{_3_add_dataset_metricsUpSql, map[string]*bintree{}},
}}
// RestoreAsset restores an asset under the given directory
......
......@@ -7,6 +7,8 @@ import (
"io"
"net/http"
"time"
humanize "github.com/dustin/go-humanize"
)
var (
......@@ -31,6 +33,8 @@ var (
<th>Host</th>
<th>Dataset</th>
<th>Source</th>
<th>Files</th>
<th>Size</th>
</tr>
</thead>
<tbody>
......@@ -46,6 +50,9 @@ var (
{{end}}
<td>{{.DatasetID}}</td>
<td>{{.DatasetSource}}</td>
<td>{{.DatasetTotalFiles}}</td>
<td>{{humanBytes .DatasetTotalBytes}}
({{humanBytes .DatasetBytesAdded}} new)</td>
</tr>
{{end}}
{{end}}
......@@ -63,6 +70,9 @@ func init() {
latestDatasetsTemplate = template.Must(
template.New("latest").Funcs(map[string]interface{}{
"fmtDate": fmtDate,
"humanBytes": func(i int64) string {
return humanize.Bytes(uint64(i))
},
}).Parse(latestDatasetsHTML))
startTime = time.Now()
}
......@@ -71,6 +81,10 @@ type datasetDebug struct {
DatasetID string
DatasetSnapshotID string
DatasetSource string
DatasetTotalFiles int64
DatasetTotalBytes int64
DatasetBytesAdded int64
DatasetDuration int
}
type backupDebug struct {
......@@ -102,6 +116,8 @@ func (s *httpServer) handleDebug(w http.ResponseWriter, r *http.Request) {
if err := rows.Scan(
&bd.BackupID, &bd.BackupTimestamp, &bd.BackupHost,
&dd.DatasetID, &dd.DatasetSnapshotID, &dd.DatasetSource,
&dd.DatasetTotalFiles, &dd.DatasetTotalBytes,
&dd.DatasetBytesAdded, &dd.DatasetDuration,
); err != nil {
return err
}
......
......@@ -20,6 +20,10 @@ type dbAtom struct {
DatasetID string
DatasetSource string
DatasetSnapshotID string
DatasetTotalFiles int64
DatasetTotalBytes int64
DatasetBytesAdded int64
DatasetDuration int
AtomName string
AtomFullPath string
AtomPath string
......@@ -41,6 +45,10 @@ func makeAtoms(backup tabacco.Backup, ds tabacco.Dataset) []dbAtom {
DatasetID: ds.ID,
DatasetSnapshotID: ds.SnapshotID,
DatasetSource: ds.Source,
DatasetTotalFiles: ds.TotalFiles,
DatasetTotalBytes: ds.TotalBytes,
DatasetBytesAdded: ds.BytesAdded,
DatasetDuration: ds.Duration,
AtomName: atom.Name,
AtomPath: atom.Path,
AtomFullPath: path,
......@@ -62,6 +70,10 @@ func (a *dbAtom) getDataset() *tabacco.Dataset {
ID: a.DatasetID,
SnapshotID: a.DatasetSnapshotID,
Source: a.DatasetSource,
TotalFiles: a.DatasetTotalFiles,
TotalBytes: a.DatasetTotalBytes,
BytesAdded: a.DatasetBytesAdded,
Duration: a.DatasetDuration,
}
}
......@@ -191,15 +203,19 @@ var statements = map[string]string{
INSERT INTO log (
backup_id, backup_timestamp, backup_host,
dataset_id, dataset_snapshot_id, dataset_source,
dataset_total_files, dataset_total_bytes, dataset_bytes_added,
dataset_duration,
atom_name, atom_path, atom_full_path
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
`,
"get_latest_atoms": `
SELECT
backup_id, backup_timestamp, backup_host,
dataset_id, dataset_snapshot_id, dataset_source,
dataset_total_files, dataset_total_bytes, dataset_bytes_added,
dataset_duration,
atom_name, atom_path, atom_full_path
FROM log
ORDER BY backup_timestamp DESC
......@@ -208,7 +224,9 @@ var statements = map[string]string{
"get_latest_datasets": `
SELECT
backup_id, backup_timestamp, backup_host,
dataset_id, dataset_snapshot_id, dataset_source
dataset_id, dataset_snapshot_id, dataset_source,
dataset_total_files, dataset_total_bytes, dataset_bytes_added,
dataset_duration
FROM log
WHERE backup_id IN (
SELECT backup_id
......@@ -218,7 +236,9 @@ var statements = map[string]string{
LIMIT ?)
GROUP BY
backup_id, backup_timestamp, backup_host,
dataset_id, dataset_snapshot_id, dataset_source
dataset_id, dataset_snapshot_id, dataset_source,
dataset_total_files, dataset_total_bytes, dataset_bytes_added,
dataset_duration
`,
}
......@@ -236,6 +256,10 @@ func (s *Service) AddDataset(ctx context.Context, backup tabacco.Backup, ds taba
dbAtom.DatasetID,
dbAtom.DatasetSnapshotID,
dbAtom.DatasetSource,
dbAtom.DatasetTotalFiles,
dbAtom.DatasetTotalBytes,
dbAtom.DatasetBytesAdded,
dbAtom.DatasetDuration,
dbAtom.AtomName,
dbAtom.AtomPath,
dbAtom.AtomFullPath,
......@@ -276,6 +300,8 @@ func (s *Service) FindAtoms(ctx context.Context, req *tabacco.FindRequest) ([]*t
`SELECT
backup_id, backup_timestamp, backup_host,
dataset_id, dataset_snapshot_id, dataset_source,
dataset_total_files, dataset_total_bytes, dataset_bytes_added,
dataset_duration,
atom_name, atom_path, atom_full_path
FROM log WHERE %s
ORDER BY backup_timestamp DESC`,
......@@ -295,6 +321,8 @@ func (s *Service) FindAtoms(ctx context.Context, req *tabacco.FindRequest) ([]*t
if err := rows.Scan(
&a.BackupID, &a.BackupTimestamp, &a.BackupHost,
&a.DatasetID, &a.DatasetSnapshotID, &a.DatasetSource,
&a.DatasetTotalFiles, &a.DatasetTotalBytes,
&a.DatasetBytesAdded, &a.DatasetDuration,
&a.AtomName, &a.AtomPath, &a.AtomFullPath,
); err != nil {
log.Printf("bad row: %v", err)
......
......@@ -32,6 +32,10 @@ func addTestEntry(t *testing.T, svc *Service, backupID, host, dsName string) {
Path: "/path/dataset1/sub2",
},
},
TotalBytes: 42,
TotalFiles: 2,
BytesAdded: 42,
Duration: 1,
},
)
if err != nil {
......
......@@ -312,6 +312,10 @@ func (r *resticRepository) RunBackup(ctx context.Context, shell *Shell, backup *
if msg.MessageType == "summary" {
log.Printf("dataset %s: total_bytes=%d, new=%d", ds.ID, msg.TotalBytesProcessed, msg.DataAdded)
ds.SnapshotID = msg.SnapshotID
ds.TotalFiles = msg.TotalFilesProcessed
ds.TotalBytes = msg.TotalBytesProcessed
ds.BytesAdded = msg.DataAdded
ds.Duration = int(msg.TotalDuration)
}
})
}
......
......@@ -87,6 +87,18 @@ type Dataset struct {
// Snapshot ID (repository-specific).
SnapshotID string `json:"snapshot_id"`
// Number of files in this dataset.
TotalFiles int64 `json:"total_files"`
// Number of bytes in this dataset.
TotalBytes int64 `json:"total_bytes"`
// Number of bytes that were added / removed in this backup.
BytesAdded int64 `json:"bytes_added"`
// Duration in seconds.
Duration int `json:"duration"`
}
// FindRequest specifies search criteria for atoms.
......
Copyright (c) 2005-2008 Dustin Sallings <dustin@spy.net>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
<http://www.opensource.org/licenses/mit-license.php>
# Humane Units [![Build Status](https://travis-ci.org/dustin/go-humanize.svg?branch=master)](https://travis-ci.org/dustin/go-humanize) [![GoDoc](https://godoc.org/github.com/dustin/go-humanize?status.svg)](https://godoc.org/github.com/dustin/go-humanize)
Just a few functions for helping humanize times and sizes.
`go get` it as `github.com/dustin/go-humanize`, import it as
`"github.com/dustin/go-humanize"`, use it as `humanize`.
See [godoc](https://pkg.go.dev/github.com/dustin/go-humanize) for
complete documentation.
## Sizes
This lets you take numbers like `82854982` and convert them to useful
strings like, `83 MB` or `79 MiB` (whichever you prefer).
Example:
```go
fmt.Printf("That file is %s.", humanize.Bytes(82854982)) // That file is 83 MB.
```
## Times
This lets you take a `time.Time` and spit it out in relative terms.
For example, `12 seconds ago` or `3 days from now`.
Example:
```go
fmt.Printf("This was touched %s.", humanize.Time(someTimeInstance)) // This was touched 7 hours ago.
```
Thanks to Kyle Lemons for the time implementation from an IRC
conversation one day. It's pretty neat.
## Ordinals
From a [mailing list discussion][odisc] where a user wanted to be able
to label ordinals.
0 -> 0th
1 -> 1st
2 -> 2nd
3 -> 3rd
4 -> 4th
[...]
Example:
```go
fmt.Printf("You're my %s best friend.", humanize.Ordinal(193)) // You are my 193rd best friend.
```
## Commas
Want to shove commas into numbers? Be my guest.
0 -> 0
100 -> 100
1000 -> 1,000
1000000000 -> 1,000,000,000
-100000 -> -100,000
Example:
```go
fmt.Printf("You owe $%s.\n", humanize.Comma(6582491)) // You owe $6,582,491.
```
## Ftoa
Nicer float64 formatter that removes trailing zeros.
```go
fmt.Printf("%f", 2.24) // 2.240000
fmt.Printf("%s", humanize.Ftoa(2.24)) // 2.24
fmt.Printf("%f", 2.0) // 2.000000
fmt.Printf("%s", humanize.Ftoa(2.0)) // 2
```
## SI notation
Format numbers with [SI notation][sinotation].
Example:
```go
humanize.SI(0.00000000223, "M") // 2.23 nM
```
## English-specific functions
The following functions are in the `humanize/english` subpackage.
### Plurals
Simple English pluralization
```go
english.PluralWord(1, "object", "") // object
english.PluralWord(42, "object", "") // objects
english.PluralWord(2, "bus", "") // buses
english.PluralWord(99, "locus", "loci") // loci
english.Plural(1, "object", "") // 1 object
english.Plural(42, "object", "") // 42 objects
english.Plural(2, "bus", "") // 2 buses
english.Plural(99, "locus", "loci") // 99 loci
```
### Word series
Format comma-separated words lists with conjuctions:
```go
english.WordSeries([]string{"foo"}, "and") // foo
english.WordSeries([]string{"foo", "bar"}, "and") // foo and bar
english.WordSeries([]string{"foo", "bar", "baz"}, "and") // foo, bar and baz
english.OxfordWordSeries([]string{"foo", "bar", "baz"}, "and") // foo, bar, and baz
```
[odisc]: https://groups.google.com/d/topic/golang-nuts/l8NhI74jl-4/discussion
[sinotation]: http://en.wikipedia.org/wiki/Metric_prefix
package humanize
import (
"math/big"
)
// order of magnitude (to a max order)
func oomm(n, b *big.Int, maxmag int) (float64, int) {
mag := 0
m := &big.Int{}
for n.Cmp(b) >= 0 {
n.DivMod(n, b, m)
mag++
if mag == maxmag && maxmag >= 0 {
break
}
}
return float64(n.Int64()) + (float64(m.Int64()) / float64(b.Int64())), mag
}
// total order of magnitude
// (same as above, but with no upper limit)
func oom(n, b *big.Int) (float64, int) {
mag := 0
m := &big.Int{}
for n.Cmp(b) >= 0 {
n.DivMod(n, b, m)
mag++
}
return float64(n.Int64()) + (float64(m.Int64()) / float64(b.Int64())), mag
}
package humanize
import (
"fmt"
"math/big"
"strings"
"unicode"
)
var (
bigIECExp = big.NewInt(1024)
// BigByte is one byte in bit.Ints
BigByte = big.NewInt(1)
// BigKiByte is 1,024 bytes in bit.Ints
BigKiByte = (&big.Int{}).Mul(BigByte, bigIECExp)
// BigMiByte is 1,024 k bytes in bit.Ints
BigMiByte = (&big.Int{}).Mul(BigKiByte, bigIECExp)
// BigGiByte is 1,024 m bytes in bit.Ints
BigGiByte = (&big.Int{}).Mul(BigMiByte, bigIECExp)
// BigTiByte is 1,024 g bytes in bit.Ints
BigTiByte = (&big.Int{}).Mul(BigGiByte, bigIECExp)
// BigPiByte is 1,024 t bytes in bit.Ints
BigPiByte = (&big.Int{}).Mul(BigTiByte, bigIECExp)
// BigEiByte is 1,024 p bytes in bit.Ints
BigEiByte = (&big.Int{}).Mul(BigPiByte, bigIECExp)
// BigZiByte is 1,024 e bytes in bit.Ints
BigZiByte = (&big.Int{}).Mul(BigEiByte, bigIECExp)
// BigYiByte is 1,024 z bytes in bit.Ints
BigYiByte = (&big.Int{}).Mul(BigZiByte, bigIECExp)
)
var (
bigSIExp = big.NewInt(1000)
// BigSIByte is one SI byte in big.Ints
BigSIByte = big.NewInt(1)
// BigKByte is 1,000 SI bytes in big.Ints
BigKByte = (&big.Int{}).Mul(BigSIByte, bigSIExp)
// BigMByte is 1,000 SI k bytes in big.Ints
BigMByte = (&big.Int{}).Mul(BigKByte, bigSIExp)
// BigGByte is 1,000 SI m bytes in big.Ints
BigGByte = (&big.Int{}).Mul(BigMByte, bigSIExp)
// BigTByte is 1,000 SI g bytes in big.Ints
BigTByte = (&big.Int{}).Mul(BigGByte, bigSIExp)
// BigPByte is 1,000 SI t bytes in big.Ints
BigPByte = (&big.Int{}).Mul(BigTByte, bigSIExp)
// BigEByte is 1,000 SI p bytes in big.Ints
BigEByte = (&big.Int{}).Mul(BigPByte, bigSIExp)
// BigZByte is 1,000 SI e bytes in big.Ints
BigZByte = (&big.Int{}).Mul(BigEByte, bigSIExp)
// BigYByte is 1,000 SI z bytes in big.Ints
BigYByte = (&big.Int{}).Mul(BigZByte, bigSIExp)
)
var bigBytesSizeTable = map[string]*big.Int{
"b": BigByte,
"kib": BigKiByte,
"kb": BigKByte,
"mib": BigMiByte,
"mb": BigMByte,
"gib": BigGiByte,
"gb": BigGByte,
"tib": BigTiByte,
"tb": BigTByte,
"pib": BigPiByte,
"pb": BigPByte,
"eib": BigEiByte,
"eb": BigEByte,
"zib": BigZiByte,
"zb": BigZByte,
"yib": BigYiByte,
"yb": BigYByte,
// Without suffix
"": BigByte,
"ki": BigKiByte,
"k": BigKByte,
"mi": BigMiByte,
"m": BigMByte,
"gi": BigGiByte,
"g": BigGByte,
"ti": BigTiByte,
"t": BigTByte,
"pi": BigPiByte,
"p": BigPByte,
"ei": BigEiByte,
"e": BigEByte,
"z": BigZByte,
"zi": BigZiByte,
"y": BigYByte,
"yi": BigYiByte,
}
var ten = big.NewInt(10)
func humanateBigBytes(s, base *big.Int, sizes []string) string {
if s.Cmp(ten) < 0 {
return fmt.Sprintf("%d B", s)
}
c := (&big.Int{}).Set(s)
val, mag := oomm(c, base, len(sizes)-1)
suffix := sizes[mag]
f := "%.0f %s"
if val < 10 {
f = "%.1f %s"
}
return fmt.Sprintf(f, val, suffix)
}
// BigBytes produces a human readable representation of an SI size.
//
// See also: ParseBigBytes.
//
// BigBytes(82854982) -> 83 MB
func BigBytes(s *big.Int) string {
sizes := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
return humanateBigBytes(s, bigSIExp, sizes)
}
// BigIBytes produces a human readable representation of an IEC size.
//
// See also: ParseBigBytes.
//
// BigIBytes(82854982) -> 79 MiB
func BigIBytes(s *big.Int) string {
sizes := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"}
return humanateBigBytes(s, bigIECExp, sizes)
}
// ParseBigBytes parses a string representation of bytes into the number