cmd/upspin-audit: add 'orphans' command to fine orphaned refs
In this case, "orphaned" means references that are present in the store
server but are not referred to by any directory entries in the user trees.
Change-Id: I5ef31e65552b782b988d3947d91b44e596c81a21
Reviewed-on: https://upspin-review.googlesource.com/17380
Reviewed-by: Rob Pike <r@golang.org>
diff --git a/cmd/upspin-audit/bytesize.go b/cmd/upspin-audit/bytesize.go
new file mode 100644
index 0000000..5a892ca
--- /dev/null
+++ b/cmd/upspin-audit/bytesize.go
@@ -0,0 +1,46 @@
+// Copyright 2017 The Upspin Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import "fmt"
+
+// ByteSize provides a way to make numbers format in nice compact form.
+// Convert a number to ByteSize and print it using its String method to see
+// 2392685154 print as 2.23GB.
+type ByteSize float64
+
+const (
+ _ = iota // ignore first value by assigning to blank identifier
+ KB ByteSize = 1 << (10 * iota)
+ MB
+ GB
+ TB
+ PB
+ EB
+ ZB
+ YB
+)
+
+func (b ByteSize) String() string {
+ switch {
+ case b >= YB:
+ return fmt.Sprintf("%.2fYB", b/YB)
+ case b >= ZB:
+ return fmt.Sprintf("%.2fZB", b/ZB)
+ case b >= EB:
+ return fmt.Sprintf("%.2fEB", b/EB)
+ case b >= PB:
+ return fmt.Sprintf("%.2fPB", b/PB)
+ case b >= TB:
+ return fmt.Sprintf("%.2fTB", b/TB)
+ case b >= GB:
+ return fmt.Sprintf("%.2fGB", b/GB)
+ case b >= MB:
+ return fmt.Sprintf("%.2fMB", b/MB)
+ case b >= KB:
+ return fmt.Sprintf("%.2fKB", b/KB)
+ }
+ return fmt.Sprintf("%.2fB", b)
+}
diff --git a/cmd/upspin-audit/main.go b/cmd/upspin-audit/main.go
index 284e439..d9d81d2 100644
--- a/cmd/upspin-audit/main.go
+++ b/cmd/upspin-audit/main.go
@@ -15,8 +15,12 @@
"os"
"path/filepath"
"sort"
+ "strconv"
+ "strings"
+ "time"
"upspin.io/config"
+ "upspin.io/errors"
"upspin.io/flags"
"upspin.io/subcmd"
"upspin.io/transports"
@@ -24,6 +28,13 @@
"upspin.io/version"
)
+const (
+ timeFormat = "2006-01-02 15:04:05"
+ dirFilePrefix = "dir_"
+ storeFilePrefix = "store_"
+ orphanFilePrefix = "orphans_"
+)
+
type State struct {
*subcmd.State
}
@@ -66,6 +77,8 @@
s.scanDirectories(flag.Args()[1:])
case "scanstore":
s.scanStore(flag.Args()[1:])
+ case "orphans":
+ s.orphans(flag.Args()[1:])
default:
usage()
}
@@ -90,45 +103,6 @@
return &dataDir
}
-// ByteSize provides a way to make numbers format in nice compact form.
-// Convert a number to ByteSize and print it using its String method to see
-// 2392685154 print as 2.23GB.
-type ByteSize float64
-
-const (
- _ = iota // ignore first value by assigning to blank identifier
- KB ByteSize = 1 << (10 * iota)
- MB
- GB
- TB
- PB
- EB
- ZB
- YB
-)
-
-func (b ByteSize) String() string {
- switch {
- case b >= YB:
- return fmt.Sprintf("%.2fYB", b/YB)
- case b >= ZB:
- return fmt.Sprintf("%.2fZB", b/ZB)
- case b >= EB:
- return fmt.Sprintf("%.2fEB", b/EB)
- case b >= PB:
- return fmt.Sprintf("%.2fPB", b/PB)
- case b >= TB:
- return fmt.Sprintf("%.2fTB", b/TB)
- case b >= GB:
- return fmt.Sprintf("%.2fGB", b/GB)
- case b >= MB:
- return fmt.Sprintf("%.2fMB", b/MB)
- case b >= KB:
- return fmt.Sprintf("%.2fKB", b/KB)
- }
- return fmt.Sprintf("%.2fB", b)
-}
-
// writeItems sorts and writes a list of reference/size pairs to file.
func (s *State) writeItems(file string, items []upspin.ListRefsItem) {
sort.Slice(items, func(i, j int) bool { return items[i].Ref < items[j].Ref })
@@ -152,3 +126,108 @@
s.Exit(err)
}
}
+
+// readItems reads a list of reference/size pairs from the given file and
+// returns them as a map. The asymmetry with writeItems, which takes a slice,
+// is to fit the most common usage pattern.
+func (s *State) readItems(file string) (map[upspin.Reference]int64, error) {
+ f, err := os.Open(file)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ sc := bufio.NewScanner(f)
+ items := make(map[upspin.Reference]int64)
+ for sc.Scan() {
+ line := sc.Text()
+ i := strings.LastIndex(line, " ")
+ if i < 0 {
+ return nil, errors.Errorf("malformed line in %q: %q", file, line)
+ }
+ quotedRef, sizeString := line[:i], line[i+1:]
+
+ ref, err := strconv.Unquote(quotedRef)
+ if err != nil {
+ return nil, errors.Errorf("malformed ref in %q: %v", file, err)
+ }
+ size, err := strconv.ParseInt(sizeString, 10, 64)
+ if err != nil {
+ return nil, errors.Errorf("malformed size in %q: %v", file, err)
+ }
+ items[upspin.Reference(ref)] = size
+ }
+ if err := sc.Err(); err != nil {
+ return nil, err
+ }
+ return items, nil
+}
+
+func itemMapToSlice(m map[upspin.Reference]int64) (items []upspin.ListRefsItem) {
+ for ref, size := range m {
+ items = append(items, upspin.ListRefsItem{Ref: ref, Size: size})
+ }
+ return
+}
+
+// fileInfo holds a description of a reference list file written by scanstore
+// or scandir. It is derived from the name of the file, not its contents.
+type fileInfo struct {
+ Path string
+ Addr upspin.NetAddr
+ User upspin.UserName // empty for store
+ Time time.Time
+}
+
+// errIgnoreFile is returned from filenameToFileInfo to signal that the given
+// file name is not one generated by scandir or scanstore. It should be handled
+// by callers to filenameToFileInfo and is not to be seen by users.
+var errIgnoreFile = errors.Str("not a file we're interested in")
+
+// filenameToFileInfo takes a file name generated by scandir or scanstore and
+// returns the information held by that file name as a fileInfo.
+func filenameToFileInfo(file string) (fi fileInfo, err error) {
+ fi.Path = file
+ file = filepath.Base(file)
+ s := file // We will consume this string.
+
+ // Check and trim prefix.
+ switch {
+ case strings.HasPrefix(s, dirFilePrefix):
+ s = strings.TrimPrefix(s, dirFilePrefix)
+ case strings.HasPrefix(s, storeFilePrefix):
+ s = strings.TrimPrefix(s, storeFilePrefix)
+ default:
+ err = errIgnoreFile
+ return
+ }
+
+ // Collect and trim endpoint name.
+ i := strings.Index(s, "_")
+ if i < 0 {
+ err = errors.Errorf("malformed file name %q", file)
+ return
+ }
+ fi.Addr = upspin.NetAddr(s[:i])
+ s = s[i+1:]
+
+ // For dir files, collect and trim user name.
+ if strings.HasPrefix(file, dirFilePrefix) {
+ i := strings.LastIndex(s, "_")
+ if i < 0 {
+ err = errors.Errorf("malformed file name %q: missing user name", file)
+ return
+ }
+ fi.User = upspin.UserName(s[:i])
+ s = s[i+1:]
+ }
+
+ // Collect time stamp.
+ ts, err := strconv.ParseInt(s, 10, 64)
+ if err != nil {
+ err = errors.Errorf("malformed file name %q: bad timestamp: %v", file, err)
+ return
+ }
+ fi.Time = time.Unix(ts, 0)
+
+ return
+}
diff --git a/cmd/upspin-audit/orphans.go b/cmd/upspin-audit/orphans.go
new file mode 100644
index 0000000..0ce39b3
--- /dev/null
+++ b/cmd/upspin-audit/orphans.go
@@ -0,0 +1,146 @@
+// Copyright 2017 The Upspin Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "flag"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "upspin.io/upspin"
+)
+
+// TODO:
+// - add a flag to run in reverse (not garbage collection mode)
+// - add a -tidy flag to remove data from old scans (maybe tidy should be its own sub-command)
+
+func (s *State) orphans(args []string) {
+ const help = `
+Audit orphans analyses previously collected scandir and scanstore runs and
+finds references that are present in the store but missing from the scanned
+directory trees, and vice versa.
+`
+ fs := flag.NewFlagSet("orphans", flag.ExitOnError)
+ dataDir := dataDirFlag(fs)
+ s.ParseFlags(fs, args, help, "audit orphans")
+
+ if fs.NArg() != 0 {
+ fs.Usage()
+ os.Exit(2)
+ }
+
+ if err := os.MkdirAll(*dataDir, 0700); err != nil {
+ s.Exit(err)
+ }
+
+ // Iterate through the files in dataDir and collect a set of the latest
+ // files for each dir endpoint/tree and store endpoint.
+ files, err := filepath.Glob(filepath.Join(*dataDir, "*"))
+ if err != nil {
+ s.Exit(err)
+ }
+ type latestKey struct {
+ Addr upspin.NetAddr
+ User upspin.UserName // empty for store
+ }
+ latest := make(map[latestKey]fileInfo)
+ for _, file := range files {
+ fi, err := filenameToFileInfo(file)
+ if err == errIgnoreFile {
+ continue
+ }
+ if err != nil {
+ s.Exit(err)
+ }
+ k := latestKey{
+ Addr: fi.Addr,
+ User: fi.User,
+ }
+ if cur, ok := latest[k]; ok && cur.Time.After(fi.Time) {
+ continue
+ }
+ latest[k] = fi
+ }
+
+ // Print a summary of the files we found.
+ nDirs, nStores := 0, 0
+ fmt.Println("Found data for these store endpoints: (scanstore output)")
+ for _, fi := range latest {
+ if fi.User == "" {
+ fmt.Printf("\t%s\t%s\n", fi.Time.Format(timeFormat), fi.Addr)
+ nStores++
+ }
+ }
+ if nStores == 0 {
+ fmt.Println("\t(none)")
+ }
+ fmt.Println("Found data for these user trees and store endpoints: (scandir output)")
+ for _, fi := range latest {
+ if fi.User != "" {
+ fmt.Printf("\t%s\t%s\t%s\n", fi.Time.Format(timeFormat), fi.Addr, fi.User)
+ nDirs++
+ }
+ }
+ if nDirs == 0 {
+ fmt.Println("\t(none)")
+ }
+ fmt.Println()
+
+ if nDirs == 0 || nStores == 0 {
+ s.Exitf("nothing to do")
+ }
+
+ // Look for orphaned references and summarize them.
+ for _, store := range latest {
+ if store.User != "" {
+ continue // Ignore dirs.
+ }
+ storeItems, err := s.readItems(store.Path)
+ if err != nil {
+ s.Exit(err)
+ }
+ dirsMissing := make(map[upspin.Reference]int64)
+ for ref, size := range storeItems {
+ dirsMissing[ref] = size
+ }
+ var users []string
+ for _, dir := range latest {
+ if dir.User == "" {
+ continue // Ignore stores.
+ }
+ if store.Addr != dir.Addr {
+ continue
+ }
+ if dir.Time.Before(store.Time) {
+ s.Exitf("scanstore must be performed before all scandir operations\n"+
+ "scandir output in\n\t%s\npredates scanstore output in\n\t%s",
+ filepath.Base(dir.Path), filepath.Base(store.Path))
+ }
+ users = append(users, string(dir.User))
+ dirItems, err := s.readItems(dir.Path)
+ if err != nil {
+ s.Exit(err)
+ }
+ storeMissing := make(map[upspin.Reference]int64)
+ for ref, size := range dirItems {
+ if _, ok := storeItems[ref]; !ok {
+ storeMissing[ref] = size
+ }
+ delete(dirsMissing, ref)
+ }
+ if len(storeMissing) > 0 {
+ fmt.Printf("Store %q missing %d references present in %q.", store.Addr, len(storeMissing), dir.User)
+ // TODO(adg): write these to a file
+ }
+ }
+ if len(dirsMissing) > 0 {
+ fmt.Printf("Store %q contains %d references not present in these trees:\n\t%s\n", store.Addr, len(dirsMissing), strings.Join(users, "\n\t"))
+ file := filepath.Join(*dataDir, fmt.Sprintf("%s%s_%d", orphanFilePrefix, store.Addr, store.Time.Unix()))
+ s.writeItems(file, itemMapToSlice(dirsMissing))
+ }
+ }
+}
diff --git a/cmd/upspin-audit/scandir.go b/cmd/upspin-audit/scandir.go
index f19dbc1..2420808 100644
--- a/cmd/upspin-audit/scandir.go
+++ b/cmd/upspin-audit/scandir.go
@@ -61,7 +61,7 @@
os.Exit(2)
}
- if err := os.MkdirAll(*dataDir, 0600); err != nil {
+ if err := os.MkdirAll(*dataDir, 0700); err != nil {
s.Exit(err)
}
@@ -153,12 +153,8 @@
// Write the data to files, one for each user/endpoint combo.
for u, size := range users {
for ep, refs := range size {
- var items []upspin.ListRefsItem
- for ref, n := range refs {
- items = append(items, upspin.ListRefsItem{Ref: ref, Size: n})
- }
- file := filepath.Join(*dataDir, fmt.Sprintf("dir.%s.%s.%d", ep.NetAddr, u, now.Unix()))
- s.writeItems(file, items)
+ file := filepath.Join(*dataDir, fmt.Sprintf("%s%s_%s_%d", dirFilePrefix, ep.NetAddr, u, now.Unix()))
+ s.writeItems(file, itemMapToSlice(refs))
}
}
}
diff --git a/cmd/upspin-audit/scanstore.go b/cmd/upspin-audit/scanstore.go
index 264389b..7f2cda6 100644
--- a/cmd/upspin-audit/scanstore.go
+++ b/cmd/upspin-audit/scanstore.go
@@ -34,7 +34,7 @@
os.Exit(2)
}
- if err := os.MkdirAll(*dataDir, 0600); err != nil {
+ if err := os.MkdirAll(*dataDir, 0700); err != nil {
s.Exit(err)
}
@@ -77,6 +77,6 @@
}
}
fmt.Printf("%s: %d bytes total (%s) in %d references\n", endpoint.NetAddr, sum, ByteSize(sum), len(items))
- file := filepath.Join(*dataDir, fmt.Sprintf("store.%s.%d", endpoint.NetAddr, now.Unix()))
+ file := filepath.Join(*dataDir, fmt.Sprintf("%s%s_%d", storeFilePrefix, endpoint.NetAddr, now.Unix()))
s.writeItems(file, items)
}
