Skip to content

Commit 76c508a

Browse files
shiva-istariShivaji Kharse
shiva-istari
and
Shivaji Kharse
authored
feat (backup): improve lsbackup tool (#9693)
Fixes #9684 ## Improve `lsbackup` Tool ### User-Facing Changes #### CLI: `dgraph lsbackup` | Flag | Type | Description | |---|---|---| | `--since-date` | `string` | Only show backups on or after this date (YYYY-MM-DD or RFC3339) | | `--until-date` | `string` | Only show backups on or before this date (YYYY-MM-DD or RFC3339) | | `--last-n-days` | `int` | Only show backups from the last N calendar days (mutually exclusive with `--since-date`) | | `--summary` | `bool` | Print summary statistics (total count, series count, oldest/newest, last full/incremental) to stderr | | `--verbose` | `bool` | **Behavior changed** — now reads the full `manifest.json` to include predicate groups and drop operations. Without it, the lighter `manifest_summary.json` is used. | **Example usage:** ```bash # List backups from the last 7 days with summary stats dgraph lsbackup --location /data/backups --last-n-days 7 --summary # List backups in a date range dgraph lsbackup --location /data/backups --since-date 2026-01-01 --until-date 2026-03-31 # Include full group/predicate detail dgraph lsbackup --location /data/backups --verbose ``` #### GraphQL: `listBackups` Query New optional input fields on the `listBackups` admin query: | Field | Type | Description | |---|---|---| | `sinceDate` | `String` | Only return backups on or after this date | | `untilDate` | `String` | Only return backups on or before this date | | `lastNDays` | `Int` | Only return backups from the last N days (mutually exclusive with `sinceDate`) | | `fullManifest` | `Boolean` | When `true`, reads the full manifest including predicate groups and drop ops. Default `false`. | **Example query:** ```graphql query { listBackups(input: { location: "/data/backups" sinceDate: "2026-01-01" untilDate: "2026-03-31" fullManifest: true }) { backupId backupNum type path encrypted since groups { groupId predicates } } } ``` #### Backup Output: New `manifest_summary.json` Every backup now writes a `manifest_summary.json` alongside `manifest.json`. This is a lightweight version that omits predicate groups and drop operations, making listing operations faster for large backup histories. The full `manifest.json` remains unchanged and is still used for restore. --- ### Internal / Developer Changes **New types in `worker` package:** - `ManifestBase` — shared fields between full and summary manifests - `MasterManifestSummary` — lightweight manifest container (no groups/drop ops) - `BackupDateFilter` — date-range filter with `Since` and `Until` pointers - `BackupListStats` — summary statistics struct **New functions in `worker` package:** - `FilterManifestsByDate()` — filters manifests by a `BackupDateFilter` - `ComputeBackupListStats()` — computes aggregate stats over a manifest list **Signature changes:** - `ListBackupManifests()` — new `forceFull bool` parameter (3rd arg) - `ProcessListBackups()` — new `forceFull bool` parameter (4th arg) --- ### Files Changed | File | Change | |---|---| | `worker/backup_manifest.go` | Core types, filtering, stats, summary read/write | | `worker/backup.go` | Write `manifest_summary.json` during backup; updated signatures | | `worker/backup_handler.go` | Updated `ProcessListBackups` signature | | `backup/run.go` | New CLI flags, filter/summary logic | | `graphql/admin/list_backups.go` | GraphQL resolver wired with filter + `fullManifest` | | `graphql/admin/endpoints.go` | New GraphQL schema fields | ### New Test Files | File | Coverage | |---|---| | `backup/run_test.go` | Date parsing, filter construction | | `graphql/admin/list_backups_test.go` | GraphQL date filter, manifest conversion | | `worker/backup_manifest_test.go` | `FilterManifestsByDate`, `ComputeBackupListStats`, summary serialization | | `systest/backup/filesystem/backup_test.go` | E2E: summary manifest creation, GraphQL filter matrix, input validation | | `systest/integration2/backup_summary_compat_test.go` | Backward compat when `manifest_summary.json` is missing | --------- Co-authored-by: Shivaji Kharse <[email protected]>
1 parent ed78681 commit 76c508a

16 files changed

Lines changed: 2520 additions & 50 deletions

File tree

‎backup/run.go‎

Lines changed: 98 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"net/url"
1414
"os"
1515
"path/filepath"
16+
"strings"
1617
"time"
1718

1819
"github.com/golang/glog"
@@ -49,7 +50,11 @@ var opt struct {
4950
destination string
5051
format string
5152
verbose bool
52-
upgrade bool // used by export backup command.
53+
upgrade bool // used by export backup command
54+
sinceDate string // date-range filter lower bound (YYYY-MM-DD or RFC3339)
55+
untilDate string // date-range filter upper bound
56+
lastNDays int // shorthand: last N calendar days
57+
summary bool // print summary stats after the listing
5358
}
5459

5560
func init() {
@@ -158,7 +163,15 @@ func initBackupLs() {
158163
flag.StringVarP(&opt.location, "location", "l", "",
159164
"Sets the source location URI (required).")
160165
flag.BoolVar(&opt.verbose, "verbose", false,
161-
"Outputs additional info in backup list.")
166+
"Show groups and drop operations. Reads the full manifest.json instead of the lightweight summary.")
167+
flag.StringVar(&opt.sinceDate, "since-date", "",
168+
"Only show backups on or after this date (YYYY-MM-DD or RFC3339).")
169+
flag.StringVar(&opt.untilDate, "until-date", "",
170+
"Only show backups on or before this date (YYYY-MM-DD or RFC3339).")
171+
flag.IntVar(&opt.lastNDays, "last-n-days", 0,
172+
"Only show backups from the last N calendar days.")
173+
flag.BoolVar(&opt.summary, "summary", false,
174+
"Print summary statistics after the backup listing.")
162175
_ = LsBackup.Cmd.MarkFlagRequired("location")
163176
}
164177

@@ -247,11 +260,70 @@ func runRestoreCmd() error {
247260
return nil
248261
}
249262

263+
// parseFilterDate parses a date string in YYYY-MM-DD or RFC3339 format.
264+
func parseFilterDate(s string) (time.Time, error) {
265+
if s == "" {
266+
return time.Time{}, fmt.Errorf("date string must not be empty")
267+
}
268+
if t, err := time.Parse("2006-01-02", s); err == nil {
269+
return t.UTC(), nil
270+
}
271+
t, err := time.Parse(time.RFC3339, s)
272+
return t.UTC(), err
273+
}
274+
275+
// buildLsFilter constructs a BackupDateFilter from lsbackup flag values.
276+
// It validates flag combinations and returns an error for invalid inputs.
277+
func buildLsFilter(sinceDate, untilDate string, lastNDays int) (worker.BackupDateFilter, error) {
278+
filter := worker.BackupDateFilter{}
279+
if lastNDays < 0 {
280+
return filter, fmt.Errorf("--last-n-days must be a positive integer, got %d", lastNDays)
281+
}
282+
if lastNDays > 0 && sinceDate != "" {
283+
return filter, fmt.Errorf("--last-n-days and --since-date are mutually exclusive")
284+
}
285+
if lastNDays > 0 {
286+
since := time.Now().UTC().AddDate(0, 0, -lastNDays).Truncate(24 * time.Hour)
287+
filter.Since = &since
288+
}
289+
if sinceDate != "" {
290+
t, err := parseFilterDate(sinceDate)
291+
if err != nil {
292+
return filter, fmt.Errorf("invalid --since-date value %q: %w", sinceDate, err)
293+
}
294+
filter.Since = &t
295+
}
296+
if untilDate != "" {
297+
t, err := parseFilterDate(untilDate)
298+
if err != nil {
299+
return filter, fmt.Errorf("invalid --until-date value %q: %w", untilDate, err)
300+
}
301+
var end time.Time
302+
if strings.Contains(untilDate, "T") {
303+
// RFC3339 datetime: user gave an exact timestamp, respect it.
304+
end = t
305+
} else {
306+
// Plain date (YYYY-MM-DD): extend to end of that calendar day.
307+
end = t.Add(24*time.Hour - time.Millisecond)
308+
}
309+
filter.Until = &end
310+
}
311+
return filter, nil
312+
}
313+
250314
func runLsbackupCmd() error {
251-
manifests, err := worker.ListBackupManifests(opt.location, nil)
315+
// Validate and build the date filter before doing any I/O.
316+
filter, err := buildLsFilter(opt.sinceDate, opt.untilDate, opt.lastNDays)
317+
if err != nil {
318+
return err
319+
}
320+
321+
// --verbose reads the full manifest.json to include groups and drop-ops.
322+
manifests, err := worker.ListBackupManifests(opt.location, nil, opt.verbose)
252323
if err != nil {
253324
return fmt.Errorf("while listing manifests: %w", err)
254325
}
326+
manifests = worker.FilterManifestsByDate(manifests, filter)
255327

256328
type backupEntry struct {
257329
Path string `json:"path"`
@@ -269,7 +341,6 @@ func runLsbackupCmd() error {
269341

270342
var output backupOutput
271343
for _, manifest := range manifests {
272-
273344
be := backupEntry{
274345
Path: manifest.Path,
275346
Since: manifest.SinceTsDeprecated,
@@ -291,6 +362,29 @@ func runLsbackupCmd() error {
291362
}
292363
_, _ = os.Stdout.Write(b)
293364
fmt.Println()
365+
366+
if opt.summary {
367+
stats := worker.ComputeBackupListStats(manifests)
368+
if opt.sinceDate != "" || opt.untilDate != "" || opt.lastNDays > 0 {
369+
fmt.Fprintf(os.Stderr, "\n--- Backup Summary (filtered) ---\n")
370+
} else {
371+
fmt.Fprintf(os.Stderr, "\n--- Backup Summary ---\n")
372+
}
373+
fmt.Fprintf(os.Stderr, "Total backups listed : %d\n", stats.Total)
374+
fmt.Fprintf(os.Stderr, "Backup series : %d\n", stats.BackupSeriesCount)
375+
if stats.OldestBackup != nil {
376+
fmt.Fprintf(os.Stderr, "Oldest backup : %s\n", stats.OldestBackup.Format(time.RFC3339))
377+
}
378+
if stats.NewestBackup != nil {
379+
fmt.Fprintf(os.Stderr, "Newest backup : %s\n", stats.NewestBackup.Format(time.RFC3339))
380+
}
381+
if stats.LastFullBackup != nil {
382+
fmt.Fprintf(os.Stderr, "Last full backup : %s\n", stats.LastFullBackup.Format(time.RFC3339))
383+
}
384+
if stats.LastIncrBackup != nil {
385+
fmt.Fprintf(os.Stderr, "Last incr backup : %s\n", stats.LastIncrBackup.Format(time.RFC3339))
386+
}
387+
}
294388
return nil
295389
}
296390

‎backup/run_test.go‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
/*
2+
* SPDX-FileCopyrightText: © 2017-2025 Istari Digital, Inc.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package backup
7+
8+
import (
9+
"testing"
10+
"time"
11+
12+
"github.com/stretchr/testify/require"
13+
)
14+
15+
func TestParseFilterDate(t *testing.T) {
16+
tests := []struct {
17+
name string
18+
input string
19+
wantErr bool
20+
wantYear int
21+
wantMonth time.Month
22+
wantDay int
23+
wantHour int
24+
wantMin int
25+
}{
26+
{
27+
name: "YYYY-MM-DD",
28+
input: "2026-04-15",
29+
wantYear: 2026,
30+
wantMonth: time.April,
31+
wantDay: 15,
32+
},
33+
{
34+
name: "RFC3339 UTC",
35+
input: "2026-01-01T00:00:00Z",
36+
wantYear: 2026,
37+
wantMonth: time.January,
38+
wantDay: 1,
39+
},
40+
{
41+
name: "RFC3339 with positive offset normalised to UTC",
42+
input: "2026-04-15T06:30:00+05:30", // 06:30 IST == 01:00 UTC
43+
wantYear: 2026,
44+
wantMonth: time.April,
45+
wantDay: 15,
46+
wantHour: 1,
47+
wantMin: 0,
48+
},
49+
{
50+
name: "RFC3339 with negative offset normalised to UTC",
51+
input: "2026-04-15T00:00:00-05:00", // midnight EST == 05:00 UTC
52+
wantYear: 2026,
53+
wantMonth: time.April,
54+
wantDay: 15,
55+
wantHour: 5,
56+
},
57+
{
58+
name: "empty string returns error",
59+
input: "",
60+
wantErr: true,
61+
},
62+
{
63+
name: "invalid string",
64+
input: "not-a-date",
65+
wantErr: true,
66+
},
67+
{
68+
name: "partial date without day",
69+
input: "2026-04",
70+
wantErr: true,
71+
},
72+
{
73+
name: "RFC3339 without timezone is invalid",
74+
input: "2026-04-15T10:00:00",
75+
wantErr: true,
76+
},
77+
}
78+
for _, tc := range tests {
79+
t.Run(tc.name, func(t *testing.T) {
80+
got, err := parseFilterDate(tc.input)
81+
if tc.wantErr {
82+
require.Error(t, err)
83+
return
84+
}
85+
require.NoError(t, err)
86+
require.Equal(t, time.UTC, got.Location(), "result must be UTC")
87+
require.Equal(t, tc.wantYear, got.Year())
88+
require.Equal(t, tc.wantMonth, got.Month())
89+
require.Equal(t, tc.wantDay, got.Day())
90+
require.Equal(t, tc.wantHour, got.Hour())
91+
require.Equal(t, tc.wantMin, got.Minute())
92+
})
93+
}
94+
}
95+
96+
func TestBuildLsFilter(t *testing.T) {
97+
t.Run("negative lastNDays returns error", func(t *testing.T) {
98+
_, err := buildLsFilter("", "", -1)
99+
require.Error(t, err)
100+
require.Contains(t, err.Error(), "--last-n-days")
101+
require.Contains(t, err.Error(), "-1")
102+
})
103+
104+
t.Run("lastNDays and sinceDate together return error", func(t *testing.T) {
105+
_, err := buildLsFilter("2026-01-01", "", 7)
106+
require.Error(t, err)
107+
require.Contains(t, err.Error(), "--last-n-days")
108+
require.Contains(t, err.Error(), "--since-date")
109+
})
110+
111+
t.Run("invalid sinceDate returns error naming the flag", func(t *testing.T) {
112+
_, err := buildLsFilter("not-a-date", "", 0)
113+
require.Error(t, err)
114+
require.Contains(t, err.Error(), "--since-date")
115+
})
116+
117+
t.Run("invalid untilDate returns error naming the flag", func(t *testing.T) {
118+
_, err := buildLsFilter("", "not-a-date", 0)
119+
require.Error(t, err)
120+
require.Contains(t, err.Error(), "--until-date")
121+
})
122+
123+
t.Run("all empty returns filter with nil Since and Until", func(t *testing.T) {
124+
f, err := buildLsFilter("", "", 0)
125+
require.NoError(t, err)
126+
require.Nil(t, f.Since)
127+
require.Nil(t, f.Until)
128+
})
129+
130+
t.Run("lastNDays=0 does not set Since", func(t *testing.T) {
131+
f, err := buildLsFilter("", "", 0)
132+
require.NoError(t, err)
133+
require.Nil(t, f.Since)
134+
})
135+
136+
t.Run("lastNDays=7 sets Since to 7 days ago midnight UTC", func(t *testing.T) {
137+
expected := time.Now().UTC().AddDate(0, 0, -7).Truncate(24 * time.Hour)
138+
f, err := buildLsFilter("", "", 7)
139+
require.NoError(t, err)
140+
require.NotNil(t, f.Since)
141+
require.Nil(t, f.Until)
142+
require.WithinDuration(t, expected, *f.Since, time.Second)
143+
require.Equal(t, 0, f.Since.Hour(), "Since must be at midnight")
144+
})
145+
146+
t.Run("sinceDate YYYY-MM-DD sets Since to midnight UTC of that date", func(t *testing.T) {
147+
f, err := buildLsFilter("2026-03-01", "", 0)
148+
require.NoError(t, err)
149+
require.NotNil(t, f.Since)
150+
require.Nil(t, f.Until)
151+
require.Equal(t, 2026, f.Since.Year())
152+
require.Equal(t, time.March, f.Since.Month())
153+
require.Equal(t, 1, f.Since.Day())
154+
require.Equal(t, 0, f.Since.Hour())
155+
})
156+
157+
t.Run("sinceDate RFC3339 uses exact time without midnight forcing", func(t *testing.T) {
158+
f, err := buildLsFilter("2026-03-01T08:30:00Z", "", 0)
159+
require.NoError(t, err)
160+
require.NotNil(t, f.Since)
161+
require.Equal(t, 8, f.Since.Hour())
162+
require.Equal(t, 30, f.Since.Minute())
163+
require.Equal(t, 0, f.Since.Second())
164+
require.Equal(t, time.UTC, f.Since.Location())
165+
})
166+
167+
t.Run("untilDate YYYY-MM-DD extends to end of that calendar day", func(t *testing.T) {
168+
f, err := buildLsFilter("", "2026-03-31", 0)
169+
require.NoError(t, err)
170+
require.Nil(t, f.Since)
171+
require.NotNil(t, f.Until)
172+
// Should be 2026-03-31 23:59:59.999 UTC
173+
require.Equal(t, 2026, f.Until.Year())
174+
require.Equal(t, time.March, f.Until.Month())
175+
require.Equal(t, 31, f.Until.Day())
176+
require.Equal(t, 23, f.Until.Hour())
177+
require.Equal(t, 59, f.Until.Minute())
178+
require.Equal(t, 59, f.Until.Second())
179+
})
180+
181+
t.Run("untilDate RFC3339 uses exact time without end-of-day extension", func(t *testing.T) {
182+
f, err := buildLsFilter("", "2026-03-31T12:00:00Z", 0)
183+
require.NoError(t, err)
184+
require.NotNil(t, f.Until)
185+
// Must be exactly 12:00:00, not extended to 23:59:59
186+
require.Equal(t, 12, f.Until.Hour())
187+
require.Equal(t, 0, f.Until.Minute())
188+
require.Equal(t, 0, f.Until.Second())
189+
})
190+
191+
t.Run("sinceDate and untilDate both set", func(t *testing.T) {
192+
f, err := buildLsFilter("2026-01-01", "2026-12-31", 0)
193+
require.NoError(t, err)
194+
require.NotNil(t, f.Since)
195+
require.NotNil(t, f.Until)
196+
require.Equal(t, time.January, f.Since.Month())
197+
require.Equal(t, 1, f.Since.Day())
198+
require.Equal(t, time.December, f.Until.Month())
199+
require.Equal(t, 31, f.Until.Day())
200+
require.Equal(t, 23, f.Until.Hour())
201+
})
202+
203+
}

‎graphql/admin/endpoints.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,27 @@ const adminTypes = `
188188
"""
189189
anonymous: Boolean
190190
191+
"""
192+
Only return backups taken on or after this date (YYYY-MM-DD or RFC3339).
193+
"""
194+
sinceDate: String
195+
196+
"""
197+
Only return backups on or before this date. For YYYY-MM-DD the entire day is
198+
included (up to 23:59:59 UTC). For RFC3339 the exact timestamp is used as the cutoff.
199+
"""
200+
untilDate: String
201+
202+
"""
203+
Only return backups from the last N calendar days. Cannot be combined with sinceDate.
204+
"""
205+
lastNDays: Int
206+
207+
"""
208+
When true, reads the full manifest.json and includes predicate groups and drop
209+
operations. Defaults to false, which reads the lightweight manifest_summary.json.
210+
"""
211+
fullManifest: Boolean
191212
}
192213
193214
type BackupGroup {

0 commit comments

Comments
 (0)