cmd/upspin-audit: add 'delete-garbage' command to delete garbage refs

Audit delete-garbage deletes garbage references as listed by the most
recent run of find-garbage. It operates on the store endpoint of the
current user.

Also rename some commands:
	scanstore scan-store
	scandir   scan-dir
	orphans   find-garbage

Change-Id: Idd6241094a8ae7e2d10bec181183bfdb103b4f6b
Reviewed-on: https://upspin-review.googlesource.com/17440
Reviewed-by: Rob Pike <r@golang.org>
diff --git a/cmd/upspin-audit/deletegarbage.go b/cmd/upspin-audit/deletegarbage.go
new file mode 100644
index 0000000..528a6c6
--- /dev/null
+++ b/cmd/upspin-audit/deletegarbage.go
@@ -0,0 +1,98 @@
+// Copyright 2017 The Upspin Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"os"
+	"strings"
+
+	"upspin.io/bind"
+	"upspin.io/errors"
+	"upspin.io/upspin"
+)
+
+func (s *State) deleteGarbage(args []string) {
+	const help = `
+Audit delete-garbage deletes garbage references as listed by the most recent
+run of find-garbage. It operates on the store endpoint of the current user.
+
+It must be run as the same Upspin user as the store server itself,
+as only that user has permission to delete references.
+
+Misuse of this command may result in permanent data loss. Use with caution.
+`
+	fs := flag.NewFlagSet("delete-garbage", flag.ExitOnError)
+	dataDir := dataDirFlag(fs)
+	s.ParseFlags(fs, args, help, "audit delete-garbage")
+
+	if fs.NArg() != 0 {
+		fs.Usage()
+		os.Exit(2)
+	}
+
+	for _, fi := range s.latestFilesWithPrefix(*dataDir, garbageFilePrefix) {
+		if fi.Addr != s.Config.StoreEndpoint().NetAddr {
+			// Only delete from the store endpoint of the current user.
+			continue
+		}
+		garbage, err := s.readItems(fi.Path)
+		if err != nil {
+			s.Exit(err)
+		}
+		store, err := bind.StoreServer(s.Config, s.Config.StoreEndpoint())
+		if err != nil {
+			s.Exit(err)
+		}
+		const numWorkers = 10
+		d := deleter{
+			State: s,
+			store: store,
+			refs:  make(chan upspin.Reference),
+			stop:  make(chan bool, numWorkers),
+		}
+		for i := 0; i < numWorkers; i++ {
+			go d.worker()
+		}
+	loop:
+		for ref := range garbage {
+			if strings.HasPrefix(string(ref), rootRefPrefix) {
+				// Don't ever collect root backups.
+				continue
+			}
+			select {
+			case d.refs <- ref:
+			case <-d.stop:
+				break loop
+			}
+		}
+		close(d.refs)
+	}
+}
+
+// deleter holds the state of delete-garbage workers.
+type deleter struct {
+	State *State
+	store upspin.StoreServer
+	refs  chan upspin.Reference
+	stop  chan bool
+}
+
+// worker receives refs from refs and deletes them from store. If the store
+// return a permission error then worker sends a value to stop.
+func (d *deleter) worker() {
+	for ref := range d.refs {
+		err := d.store.Delete(ref)
+		if err != nil {
+			d.State.Fail(err)
+			// Stop the entire process if we get a permission error;
+			// we likely are running as the wrong user.
+			if errors.Is(errors.Permission, err) {
+				d.stop <- true
+				return
+			}
+		}
+	}
+}
diff --git a/cmd/upspin-audit/findgarbage.go b/cmd/upspin-audit/findgarbage.go
new file mode 100644
index 0000000..cefab0d
--- /dev/null
+++ b/cmd/upspin-audit/findgarbage.go
@@ -0,0 +1,116 @@
+// Copyright 2017 The Upspin Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"upspin.io/upspin"
+)
+
+func (s *State) findGarbage(args []string) {
+	const help = `
+Audit find-garbage analyses the output of scan-dir and scan-store to finds
+references that are present in the store server but missing from the scanned
+directory trees.
+`
+	fs := flag.NewFlagSet("find-garbage", flag.ExitOnError)
+	dataDir := dataDirFlag(fs)
+	s.ParseFlags(fs, args, help, "audit find-garbage")
+
+	if fs.NArg() != 0 {
+		fs.Usage()
+		os.Exit(2)
+	}
+
+	if err := os.MkdirAll(*dataDir, 0700); err != nil {
+		s.Exit(err)
+	}
+
+	// Iterate through the files in dataDir and collect a set of the latest
+	// files for each dir endpoint/tree and store endpoint.
+	latest := s.latestFilesWithPrefix(*dataDir, storeFilePrefix, dirFilePrefix)
+
+	// Print a summary of the files we found.
+	nDirs, nStores := 0, 0
+	fmt.Println("Found data for these store endpoints: (scan-store output)")
+	for _, fi := range latest {
+		if fi.User == "" {
+			fmt.Printf("\t%s\t%s\n", fi.Time.Format(timeFormat), fi.Addr)
+			nStores++
+		}
+	}
+	if nStores == 0 {
+		fmt.Println("\t(none)")
+	}
+	fmt.Println("Found data for these user trees and store endpoints: (scan-dir output)")
+	for _, fi := range latest {
+		if fi.User != "" {
+			fmt.Printf("\t%s\t%s\t%s\n", fi.Time.Format(timeFormat), fi.Addr, fi.User)
+			nDirs++
+		}
+	}
+	if nDirs == 0 {
+		fmt.Println("\t(none)")
+	}
+	fmt.Println()
+
+	if nDirs == 0 || nStores == 0 {
+		s.Exitf("nothing to do; run scan-store and scan-dir first")
+	}
+
+	// Look for garbage references and summarize them.
+	for _, store := range latest {
+		if store.User != "" {
+			continue // Ignore dirs.
+		}
+		storeItems, err := s.readItems(store.Path)
+		if err != nil {
+			s.Exit(err)
+		}
+		dirsMissing := make(map[upspin.Reference]int64)
+		for ref, size := range storeItems {
+			dirsMissing[ref] = size
+		}
+		var users []string
+		for _, dir := range latest {
+			if dir.User == "" {
+				continue // Ignore stores.
+			}
+			if store.Addr != dir.Addr {
+				continue
+			}
+			if dir.Time.Before(store.Time) {
+				s.Exitf("scan-store must be performed before all scan-dir operations\n"+
+					"scan-dir output in\n\t%s\npredates scan-store output in\n\t%s",
+					filepath.Base(dir.Path), filepath.Base(store.Path))
+			}
+			users = append(users, string(dir.User))
+			dirItems, err := s.readItems(dir.Path)
+			if err != nil {
+				s.Exit(err)
+			}
+			storeMissing := make(map[upspin.Reference]int64)
+			for ref, size := range dirItems {
+				if _, ok := storeItems[ref]; !ok {
+					storeMissing[ref] = size
+				}
+				delete(dirsMissing, ref)
+			}
+			if len(storeMissing) > 0 {
+				fmt.Printf("Store %q missing %d references present in %q.\n", store.Addr, len(storeMissing), dir.User)
+			}
+		}
+		if len(dirsMissing) > 0 {
+			fmt.Printf("Store %q contains %d references not present in these trees:\n\t%s\n", store.Addr, len(dirsMissing), strings.Join(users, "\n\t"))
+			file := filepath.Join(*dataDir, fmt.Sprintf("%s%s_%d", garbageFilePrefix, store.Addr, store.Time.Unix()))
+			s.writeItems(file, itemMapToSlice(dirsMissing))
+		}
+	}
+}
diff --git a/cmd/upspin-audit/main.go b/cmd/upspin-audit/main.go
index d9d81d2..e13856f 100644
--- a/cmd/upspin-audit/main.go
+++ b/cmd/upspin-audit/main.go
@@ -7,6 +7,11 @@
 // determined.
 package main
 
+// TODO:
+// - add failsafes to avoid misuse of delete-garbage
+// - add a command that is the reverse of find-garbage (find-missing?)
+// - add a tidy command to remove data from old scans
+
 import (
 	"bufio"
 	"flag"
@@ -29,10 +34,12 @@
 )
 
 const (
-	timeFormat       = "2006-01-02 15:04:05"
-	dirFilePrefix    = "dir_"
-	storeFilePrefix  = "store_"
-	orphanFilePrefix = "orphans_"
+	timeFormat    = "2006-01-02 15:04:05"
+	rootRefPrefix = "tree.root."
+
+	dirFilePrefix     = "dir_"
+	storeFilePrefix   = "store_"
+	garbageFilePrefix = "garbage_"
 )
 
 type State struct {
@@ -40,9 +47,32 @@
 }
 
 const help = `Upspin-audit provides subcommands for auditing storage consumption.
-It has subcommands scandir and scanstore to scan the directory and storage servers
-and report the storage consumed by those servers.
-The set of tools will grow.
+
+The subcommands are:
+
+scan-dir
+scan-store
+	Scan the directory and store servers and report the storage consumed
+	by those servers.
+
+find-garbage
+	Use the results of scan-dir and scan-store operations to create a list
+	of references that are present in a store server but not referenced
+	by the scanned directory servers.
+
+delete-garbage
+	Delete the references found by find-garbage from the store server.
+
+To delete the garbage references in a given store server:
+1. Run scan-store (as the store server user) to generate a list of references
+   in the store server.
+2. Run scan-dir for each Upspin tree that stores data in the store server (as
+   the Upspin users that own those trees) to generate lists of references
+   referred to by those trees.
+3. Run find-garbage to compile a list of references that are in the scan-store
+   output but not in the combined output of the scan-dir runs.
+4. Run delete-garbage (as the store server user) to delete the references in
+   the find-garbage output.
 `
 
 func main() {
@@ -73,12 +103,14 @@
 	s.State.Init(cfg)
 
 	switch flag.Arg(0) {
-	case "scandir":
+	case "scan-dir":
 		s.scanDirectories(flag.Args()[1:])
-	case "scanstore":
+	case "scan-store":
 		s.scanStore(flag.Args()[1:])
-	case "orphans":
-		s.orphans(flag.Args()[1:])
+	case "find-garbage":
+		s.findGarbage(flag.Args()[1:])
+	case "delete-garbage":
+		s.deleteGarbage(flag.Args()[1:])
 	default:
 		usage()
 	}
@@ -90,7 +122,8 @@
 	fmt.Fprintln(os.Stderr, help)
 	fmt.Fprintln(os.Stderr, "Usage of upspin audit:")
 	fmt.Fprintln(os.Stderr, "\tupspin [globalflags] audit <command> [flags] ...")
-	fmt.Fprintln(os.Stderr, "\twhere <command> is one of scandir, scanstore")
+	fmt.Fprintln(os.Stderr, "Commands: scan-dir, scan-store, find-garbage, delete-garbage")
+	fmt.Fprintln(os.Stderr, "Global flags:")
 	flag.PrintDefaults()
 	os.Exit(2)
 }
@@ -169,8 +202,8 @@
 	return
 }
 
-// fileInfo holds a description of a reference list file written by scanstore
-// or scandir. It is derived from the name of the file, not its contents.
+// fileInfo holds a description of a reference list file written by scan-store
+// or scan-dir. It is derived from the name of the file, not its contents.
 type fileInfo struct {
 	Path string
 	Addr upspin.NetAddr
@@ -178,25 +211,63 @@
 	Time time.Time
 }
 
+// latestFilesWithPrefix returns the most recently generated files in dir that
+// have that have the given prefixes.
+func (s *State) latestFilesWithPrefix(dir string, prefixes ...string) (files []fileInfo) {
+	paths, err := filepath.Glob(filepath.Join(dir, "*"))
+	if err != nil {
+		s.Exit(err)
+	}
+	type latestKey struct {
+		Addr upspin.NetAddr
+		User upspin.UserName // empty for store
+	}
+	latest := make(map[latestKey]fileInfo)
+	for _, file := range paths {
+		fi, err := filenameToFileInfo(file, prefixes...)
+		if err == errIgnoreFile {
+			continue
+		}
+		if err != nil {
+			s.Exit(err)
+		}
+		k := latestKey{
+			Addr: fi.Addr,
+			User: fi.User,
+		}
+		if cur, ok := latest[k]; ok && cur.Time.After(fi.Time) {
+			continue
+		}
+		latest[k] = fi
+	}
+	for _, fi := range latest {
+		files = append(files, fi)
+	}
+	return files
+}
+
 // errIgnoreFile is returned from filenameToFileInfo to signal that the given
-// file name is not one generated by scandir or scanstore. It should be handled
+// file name is not one generated by scan-dir or scan-store. It should be handled
 // by callers to filenameToFileInfo and is not to be seen by users.
 var errIgnoreFile = errors.Str("not a file we're interested in")
 
-// filenameToFileInfo takes a file name generated by scandir or scanstore and
+// filenameToFileInfo takes a file name generated by scan-dir or scan-store and
 // returns the information held by that file name as a fileInfo.
-func filenameToFileInfo(file string) (fi fileInfo, err error) {
+func filenameToFileInfo(file string, prefixes ...string) (fi fileInfo, err error) {
 	fi.Path = file
 	file = filepath.Base(file)
 	s := file // We will consume this string.
 
 	// Check and trim prefix.
-	switch {
-	case strings.HasPrefix(s, dirFilePrefix):
-		s = strings.TrimPrefix(s, dirFilePrefix)
-	case strings.HasPrefix(s, storeFilePrefix):
-		s = strings.TrimPrefix(s, storeFilePrefix)
-	default:
+	ok := false
+	for _, p := range prefixes {
+		if strings.HasPrefix(s, p) {
+			s = strings.TrimPrefix(s, p)
+			ok = true
+			break
+		}
+	}
+	if !ok {
 		err = errIgnoreFile
 		return
 	}
diff --git a/cmd/upspin-audit/orphans.go b/cmd/upspin-audit/orphans.go
deleted file mode 100644
index 0ce39b3..0000000
--- a/cmd/upspin-audit/orphans.go
+++ /dev/null
@@ -1,146 +0,0 @@
-// Copyright 2017 The Upspin Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"upspin.io/upspin"
-)
-
-// TODO:
-// - add a flag to run in reverse (not garbage collection mode)
-// - add a -tidy flag to remove data from old scans (maybe tidy should be its own sub-command)
-
-func (s *State) orphans(args []string) {
-	const help = `
-Audit orphans analyses previously collected scandir and scanstore runs and
-finds references that are present in the store but missing from the scanned
-directory trees, and vice versa.
-`
-	fs := flag.NewFlagSet("orphans", flag.ExitOnError)
-	dataDir := dataDirFlag(fs)
-	s.ParseFlags(fs, args, help, "audit orphans")
-
-	if fs.NArg() != 0 {
-		fs.Usage()
-		os.Exit(2)
-	}
-
-	if err := os.MkdirAll(*dataDir, 0700); err != nil {
-		s.Exit(err)
-	}
-
-	// Iterate through the files in dataDir and collect a set of the latest
-	// files for each dir endpoint/tree and store endpoint.
-	files, err := filepath.Glob(filepath.Join(*dataDir, "*"))
-	if err != nil {
-		s.Exit(err)
-	}
-	type latestKey struct {
-		Addr upspin.NetAddr
-		User upspin.UserName // empty for store
-	}
-	latest := make(map[latestKey]fileInfo)
-	for _, file := range files {
-		fi, err := filenameToFileInfo(file)
-		if err == errIgnoreFile {
-			continue
-		}
-		if err != nil {
-			s.Exit(err)
-		}
-		k := latestKey{
-			Addr: fi.Addr,
-			User: fi.User,
-		}
-		if cur, ok := latest[k]; ok && cur.Time.After(fi.Time) {
-			continue
-		}
-		latest[k] = fi
-	}
-
-	// Print a summary of the files we found.
-	nDirs, nStores := 0, 0
-	fmt.Println("Found data for these store endpoints: (scanstore output)")
-	for _, fi := range latest {
-		if fi.User == "" {
-			fmt.Printf("\t%s\t%s\n", fi.Time.Format(timeFormat), fi.Addr)
-			nStores++
-		}
-	}
-	if nStores == 0 {
-		fmt.Println("\t(none)")
-	}
-	fmt.Println("Found data for these user trees and store endpoints: (scandir output)")
-	for _, fi := range latest {
-		if fi.User != "" {
-			fmt.Printf("\t%s\t%s\t%s\n", fi.Time.Format(timeFormat), fi.Addr, fi.User)
-			nDirs++
-		}
-	}
-	if nDirs == 0 {
-		fmt.Println("\t(none)")
-	}
-	fmt.Println()
-
-	if nDirs == 0 || nStores == 0 {
-		s.Exitf("nothing to do")
-	}
-
-	// Look for orphaned references and summarize them.
-	for _, store := range latest {
-		if store.User != "" {
-			continue // Ignore dirs.
-		}
-		storeItems, err := s.readItems(store.Path)
-		if err != nil {
-			s.Exit(err)
-		}
-		dirsMissing := make(map[upspin.Reference]int64)
-		for ref, size := range storeItems {
-			dirsMissing[ref] = size
-		}
-		var users []string
-		for _, dir := range latest {
-			if dir.User == "" {
-				continue // Ignore stores.
-			}
-			if store.Addr != dir.Addr {
-				continue
-			}
-			if dir.Time.Before(store.Time) {
-				s.Exitf("scanstore must be performed before all scandir operations\n"+
-					"scandir output in\n\t%s\npredates scanstore output in\n\t%s",
-					filepath.Base(dir.Path), filepath.Base(store.Path))
-			}
-			users = append(users, string(dir.User))
-			dirItems, err := s.readItems(dir.Path)
-			if err != nil {
-				s.Exit(err)
-			}
-			storeMissing := make(map[upspin.Reference]int64)
-			for ref, size := range dirItems {
-				if _, ok := storeItems[ref]; !ok {
-					storeMissing[ref] = size
-				}
-				delete(dirsMissing, ref)
-			}
-			if len(storeMissing) > 0 {
-				fmt.Printf("Store %q missing %d references present in %q.", store.Addr, len(storeMissing), dir.User)
-				// TODO(adg): write these to a file
-			}
-		}
-		if len(dirsMissing) > 0 {
-			fmt.Printf("Store %q contains %d references not present in these trees:\n\t%s\n", store.Addr, len(dirsMissing), strings.Join(users, "\n\t"))
-			file := filepath.Join(*dataDir, fmt.Sprintf("%s%s_%d", orphanFilePrefix, store.Addr, store.Time.Unix()))
-			s.writeItems(file, itemMapToSlice(dirsMissing))
-		}
-	}
-}
diff --git a/cmd/upspin-audit/scandir.go b/cmd/upspin-audit/scandir.go
index 2420808..26944de 100644
--- a/cmd/upspin-audit/scandir.go
+++ b/cmd/upspin-audit/scandir.go
@@ -48,13 +48,13 @@
 
 func (s *State) scanDirectories(args []string) {
 	const help = `
-Audit scandir scans the directory tree for the named user roots.
+Audit scan-dir scans the directory tree for the named user roots.
 For now it just prints the total storage consumed.`
 
-	fs := flag.NewFlagSet("scandir", flag.ExitOnError)
+	fs := flag.NewFlagSet("scan-dir", flag.ExitOnError)
 	glob := fs.Bool("glob", true, "apply glob processing to the arguments")
 	dataDir := dataDirFlag(fs)
-	s.ParseFlags(fs, args, help, "audit scandir root ...")
+	s.ParseFlags(fs, args, help, "audit scan-dir root ...")
 
 	if fs.NArg() == 0 || fs.Arg(0) == "help" {
 		fs.Usage()
diff --git a/cmd/upspin-audit/scanstore.go b/cmd/upspin-audit/scanstore.go
index 7f2cda6..6ff0f3c 100644
--- a/cmd/upspin-audit/scanstore.go
+++ b/cmd/upspin-audit/scanstore.go
@@ -20,16 +20,20 @@
 
 func (s *State) scanStore(args []string) {
 	const help = `
-Audit scanstore scans the storage server to identify all references.
+Audit scan-store scans the storage server to identify all references.
 By default it scans the storage server mentioned in the config file.
-For now it just prints the total storage they represent.`
+For now it just prints the total storage they represent.
 
-	fs := flag.NewFlagSet("scanstore", flag.ExitOnError)
+It must be run as the same Upspin user as the store server itself,
+as only that user has permission to list references.
+`
+
+	fs := flag.NewFlagSet("scan-store", flag.ExitOnError)
 	endpointFlag := fs.String("endpoint", string(s.Config.StoreEndpoint().NetAddr), "network `address` of storage server; default is from config")
 	dataDir := dataDirFlag(fs)
-	s.ParseFlags(fs, args, help, "audit scanstore [-endpoint <storeserver address>]")
+	s.ParseFlags(fs, args, help, "audit scan-store [-endpoint <storeserver address>]")
 
-	if fs.NArg() != 0 { // "audit scanstore help" is covered by this.
+	if fs.NArg() != 0 { // "audit scan-store help" is covered by this.
 		fs.Usage()
 		os.Exit(2)
 	}