exp/upbox, exp/cmd/upbox: move out of exp directory

Now that upbox is used by the tests in cmd/upspin, it should no longer
be considered experimental.

Change-Id: Ic52f4d63b801152b43dde29f62e676d2891e5261
diff --git a/cmd/upbox/main.go b/cmd/upbox/main.go
deleted file mode 100644
index ea5a186..0000000
--- a/cmd/upbox/main.go
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2016 The Upspin Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-/*
-Command upbox builds and runs Upspin servers as specified by a schema and
-provides an upspin shell acting as the first user specified by the schema.
-
-For information on defining a schema, see the documentation for package
-upspin.io/exp/upbox.
-*/
-package main
-
-import (
-	"flag"
-	"fmt"
-	"os"
-	"os/exec"
-	"strings"
-
-	"upspin.io/exp/upbox"
-)
-
-var (
-	logLevel = flag.String("log", "error", "log `level`")
-	basePort = flag.Int("port", 8000, "base `port` number for upspin servers")
-	schema   = flag.String("schema", "", "schema `file` name")
-)
-
-func main() {
-	flag.Parse()
-
-	sc, err := upbox.SchemaFromFile(*schema, *basePort)
-	if err != nil {
-		fail(fmt.Errorf("parsing schema: %v", err))
-	}
-	sc.LogLevel = *logLevel
-
-	if err := sc.Start(); err != nil {
-		fail(err)
-	}
-
-	// Start a shell as the first user.
-	args := []string{
-		"-config=" + sc.Config(sc.Users[0].Name),
-		"-log=" + *logLevel,
-		"shell",
-	}
-	fmt.Fprintf(os.Stderr, "upbox: upspin %s\n", strings.Join(args, " "))
-	shell := exec.Command("upspin", args...)
-	shell.Stdin = os.Stdin
-	shell.Stdout = os.Stdout
-	shell.Stderr = os.Stderr
-	err = shell.Run()
-	err2 := sc.Stop()
-	if err != nil {
-		fail(err)
-	}
-	if err2 != nil {
-		fail(err2)
-	}
-}
-
-func fail(err error) {
-	fmt.Fprintln(os.Stderr, "upbox:", err)
-	os.Exit(1)
-}
diff --git a/upbox/generate_cert.go b/upbox/generate_cert.go
deleted file mode 100644
index 18786c5..0000000
--- a/upbox/generate_cert.go
+++ /dev/null
@@ -1,111 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This file is adapted from $GOROOT/src/crypto/tls/generate_cert.go
-
-// Generate a self-signed X.509 certificate for a TLS server.
-
-package upbox
-
-import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/pem"
-	"fmt"
-	"math/big"
-	"os"
-	"path/filepath"
-	"time"
-)
-
-func publicKey(priv interface{}) interface{} {
-	switch k := priv.(type) {
-	case *rsa.PrivateKey:
-		return &k.PublicKey
-	case *ecdsa.PrivateKey:
-		return &k.PublicKey
-	default:
-		return nil
-	}
-}
-
-func pemBlockForKey(priv interface{}) *pem.Block {
-	switch k := priv.(type) {
-	case *rsa.PrivateKey:
-		return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)}
-	case *ecdsa.PrivateKey:
-		b, err := x509.MarshalECPrivateKey(k)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Unable to marshal ECDSA private key: %v", err)
-			os.Exit(2)
-		}
-		return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}
-	default:
-		return nil
-	}
-}
-
-func generateCert(dir string) error {
-	priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
-	if err != nil {
-		return fmt.Errorf("failed to generate key: %s", err)
-	}
-
-	notBefore := time.Now()
-	notAfter := notBefore.Add(24 * time.Hour)
-
-	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
-	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
-	if err != nil {
-		return fmt.Errorf("failed to generate serial number: %s", err)
-	}
-
-	template := x509.Certificate{
-		SerialNumber: serialNumber,
-		Subject: pkix.Name{
-			Organization: []string{"The Upspin Project"},
-		},
-		NotBefore: notBefore,
-		NotAfter:  notAfter,
-
-		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		BasicConstraintsValid: true,
-
-		DNSNames: []string{"localhost"},
-	}
-
-	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey(priv), priv)
-	if err != nil {
-		return fmt.Errorf("Failed to create certificate: %s", err)
-	}
-
-	certOut, err := os.Create(filepath.Join(dir, "cert.pem"))
-	if err != nil {
-		return fmt.Errorf("failed to open cert.pem for writing: %s", err)
-	}
-	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
-		return fmt.Errorf("failed to encode cert.pem: %s", err)
-	}
-	if err := certOut.Close(); err != nil {
-		return fmt.Errorf("failed to write cert.pem: %s", err)
-	}
-
-	keyOut, err := os.OpenFile(filepath.Join(dir, "key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	if err != nil {
-		return fmt.Errorf("failed to open key.pem for writing: %s", err)
-	}
-	if err := pem.Encode(keyOut, pemBlockForKey(priv)); err != nil {
-		return fmt.Errorf("failed to encode key.pem: %s", err)
-	}
-	if err := keyOut.Close(); err != nil {
-		return fmt.Errorf("failed to write cert.pem: %s", err)
-	}
-
-	return nil
-}
diff --git a/upbox/schema.go b/upbox/schema.go
deleted file mode 100644
index f31231c..0000000
--- a/upbox/schema.go
+++ /dev/null
@@ -1,595 +0,0 @@
-// Copyright 2016 The Upspin Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-/*
-Package upbox provides the Schema mechanism for declaring and creating a set of
-Upspin users and servers.
-
-Schema files must be in YAML format, of this general form:
-
-	users:
-	- name: joe
-	- name: jess@example.net
-	  storeserver: store.upspin.io
-	  dirserver: dir.upspin.io
-	  packing: ee
-	servers:
-	- name: storeserver
-	- name: dirserver
-	  user: joe
-	- name: myserver
-	  importpath: github.com/user/myserver
-	  flags:
-	    debug: cockroach
-	keyserver: key.uspin.io
-	domain: example.com
-
-
-The Users and Servers lists specify the users and servers to create within this
-schema.
-
-Users
-
-Name specifies the user name of this user.
-It must be non-empty.
-It can be a full email address, or just the user component.
-In the latter case, the top-level domain field must be set.
-
-StoreServer and DirServer specify the store and directory endpoints for this
-user. If empty, they default to the servers "storeserver" and "dirserver",
-respectively. If they are of the form "$servername" then the address of the
-server "servername" is used.
-
-Packing specifies the packing method for this user.
-If empty, it defaults to "ee".
-
-Servers
-
-Name specifies a short name for this server. It must be non-empty.
-The names "keyserver", "storeserver", and "dirserver" represent useful
-defaults.
-
-User specifies the user to run this server as.
-It can be a full email address, or just the user component.
-If empty, the Name of the server is combined with the
-Schema's Domain and a user is created with that name.
-In the latter cases, the top-level Domain field must be set.
-
-ImportPath specifies the import path for this server that is built before
-starting the server. If empty, the server Name is appended to the string
-"upspin.io/cmd/".
-
-Other top-level fields
-
-KeyServer specifies the KeyServer that each user in the cluster
-should use. If it is empty, then a Server named "keyserver" must
-be included in the list of Servers, and the address of that server
-is used.
-
-Domain specifies a domain that is appended to any user names that do
-not include a domain component.
-Domain must be specified if any domain suffixes are omitted from
-User Names or if a Servers is specified with an empty User field.
-
-Default schema
-
-If no schema is specified, the default schema is used:
-
-	users:
-	  - name: user
-	servers:
-	  - name: keyserver
-	  - name: storeserver
-	  - name: dirserver
-	domain: example.com
-
-This creates the users user@example.com, keyserver@example.com,
-storeserver@example.com, and dirserver@example.com, builds and runs
-the servers keyserver, storeserver, and dirserver (running as their
-respective users), and runs "upspin shell" as user@example.com.
-
-*/
-package upbox // import "upspin.io/exp/upbox"
-
-import (
-	"bufio"
-	"bytes"
-	"crypto/tls"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"upspin.io/upspin"
-
-	yaml "gopkg.in/yaml.v2"
-)
-
-// Schema defines a set of Upspin Users and Servers.
-type Schema struct {
-	Users   []*User
-	Servers []*Server
-
-	// Domain specifies the default domain of any user names that do not
-	// include a domain component.
-	Domain string
-
-	// KeyServer specifies the KeyServer used by each user in the cluster.
-	KeyServer string
-
-	// LogLevel specifies the logging level that each server should use.
-	LogLevel string
-
-	// user and server are mappings of user names into the Users and
-	// Servers slices. They are set by SchemaFromYAML.
-	user   map[string]*User
-	server map[string]*Server
-
-	// dir is the temporary directory in which the
-	// user config files and keys were written by Start.
-	dir string
-}
-
-// User defines an Upspin user to be created and used within a schema.
-type User struct {
-	// Name specifies the user name of this user.
-	Name string
-
-	// StoreServer and DirServer specify the store and directory endpoints
-	// for this user.
-	StoreServer string
-	DirServer   string
-
-	// Packing specifies the packing method for this user.
-	Packing string
-
-	secrets string // path to user's public and private keys; set by Run
-}
-
-// Server defines an Upspin server to be created and used within a schema.
-type Server struct {
-	// Name specifies a short name for this server.
-	Name string
-
-	// User specifies the user to run this server as.
-	User string
-
-	// ImportPath specifies the import path for this server
-	// that is built before starting the server.
-	// If empty, the server Name is appended to the string
-	// "upspin.io/cmd/".
-	ImportPath string
-
-	// Flags specifies command-line flags to supply to this server.
-	Flags map[string]string
-
-	addr string // the host:port of this server; set by SchemaFromYAML
-
-	cmd *exec.Cmd // the running process; set by Start
-}
-
-// DefaultSchema is the schema that is used if none is provided.
-const DefaultSchema = `
-users:
-  - name: user
-servers:
-  - name: keyserver
-  - name: storeserver
-  - name: dirserver
-domain: example.com
-`
-
-// SchemaFromFile parses a Schema from the named file.
-// If no name is provided the DefaultSchema is used.
-func SchemaFromFile(name string, basePort int) (*Schema, error) {
-	var doc string
-	if name == "" {
-		doc = DefaultSchema
-	} else {
-		data, err := ioutil.ReadFile(name)
-		if err != nil {
-			return nil, err
-		}
-		doc = string(data)
-	}
-	return SchemaFromYAML(doc, basePort)
-}
-
-// SchemaFromYAML parses a Schema from the given YAML document.
-func SchemaFromYAML(doc string, basePort int) (*Schema, error) {
-	var sc Schema
-	if err := yaml.Unmarshal([]byte(doc), &sc); err != nil {
-		return nil, err
-	}
-
-	sc.user = map[string]*User{}
-	sc.server = map[string]*Server{}
-
-	if len(sc.Users) == 0 {
-		return nil, errors.New("at least one user must be specified")
-	}
-
-	// Add domain to usernames without domains,
-	// and default user names for servers.
-	for i, u := range sc.Users {
-		if u.Name == "" {
-			return nil, fmt.Errorf("user[%d] must specify a name", i)
-		}
-
-		// Add domain to bare user name.
-		if !strings.Contains(u.Name, "@") {
-			if sc.Domain == "" {
-				return nil, fmt.Errorf("user %q implies domain suffix, but domain not set", u.Name)
-			}
-			u.Name += "@" + sc.Domain
-		}
-		if u.Packing == "" {
-			u.Packing = "ee"
-		}
-
-		// Add to map only after name has been normalized.
-		sc.user[u.Name] = u
-	}
-
-	port := basePort
-	for i, s := range sc.Servers {
-		if s.Name == "" {
-			return nil, fmt.Errorf("server[%d] must specify a name", i)
-		}
-		sc.server[s.Name] = s
-
-		if s.User == "" {
-			// If no user specified, default to server@domain.
-			if sc.Domain == "" {
-				return nil, fmt.Errorf("server %q specifies no user, but domain must be specified to create default user", s.Name)
-			}
-			s.User = s.Name + "@" + sc.Domain
-			// If the user isn't otherwise provided, create it.
-			if _, ok := sc.user[s.User]; !ok {
-				u := newUserFor(s)
-				sc.Users = append(sc.Users, u)
-				sc.user[u.Name] = u
-			}
-		} else if !strings.Contains(s.User, "@") {
-			// Add the domain name if user is specified.
-			if sc.Domain == "" {
-				return nil, fmt.Errorf("server %q specifies user %q without domain suffix, but domain not set", s.Name, s.User)
-			}
-			s.User += "@" + sc.Domain
-		}
-
-		// Pick address for this service.
-		s.addr = fmt.Sprintf("localhost:%d", port)
-		port++
-
-		// Set the global keyserver, if none provided.
-		if s.Name == "keyserver" && sc.KeyServer == "" {
-			sc.KeyServer = s.addr
-		}
-
-		// Default to an Upspin command if no import path provided.
-		if s.ImportPath == "" {
-			s.ImportPath = "upspin.io/cmd/" + s.Name
-		}
-	}
-
-	// Check for KeyServer only after we may have set it as "keyserver" above.
-	if sc.KeyServer == "" {
-		return nil, errors.New("no keyserver in configuration")
-	}
-
-	// Set or evaluate DirServer and StoreServer fields.
-	for _, u := range sc.Users {
-		if err := setServer(&sc, &u.DirServer, "dirserver"); err != nil {
-			return nil, fmt.Errorf("user %q: %v", u.Name, err)
-		}
-		if err := setServer(&sc, &u.StoreServer, "storeserver"); err != nil {
-			return nil, fmt.Errorf("user %q: %v", u.Name, err)
-		}
-	}
-
-	return &sc, nil
-}
-
-func newUserFor(s *Server) *User {
-	u := &User{Name: s.User}
-	switch s.Name {
-	case "keyserver":
-		u.Packing = "ee"
-		u.StoreServer = "unassigned,"
-		u.DirServer = "unassigned,"
-	case "storeserver":
-		u.Packing = "ee"
-		u.StoreServer = "inprocess,"
-		u.DirServer = "unassigned,"
-	default:
-		u.Packing = "ee"
-		// StoreServer and DirServer
-		// will be set up later in this func.
-	}
-	return u
-}
-
-func setServer(sc *Schema, field *string, kind string) error {
-	if *field == "" {
-		s, ok := sc.server[kind]
-		if !ok {
-			return fmt.Errorf("needs default %s, but none found", kind)
-		}
-		*field = "remote," + s.addr
-	} else if (*field)[0] == '$' {
-		name := (*field)[1:]
-		s, ok := sc.server[name]
-		if !ok {
-			return fmt.Errorf("specifies %v %q, but none found", kind, name)
-		}
-		*field = "remote," + s.addr
-	}
-	return nil
-}
-
-// Start sets up the Users and Servers specified by the Schema.
-func (sc *Schema) Start() error {
-	// Build servers and commands.
-	args := []string{"install", "upspin.io/cmd/upspin"}
-	for _, s := range sc.Servers {
-		args = append(args, s.ImportPath)
-	}
-	cmd := exec.Command("go", args...)
-	cmd.Stdout = prefix("build: ", os.Stdout)
-	cmd.Stderr = prefix("build: ", os.Stderr)
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("build error: %v", err)
-	}
-
-	// Create temporary directory.
-	tmpDir, err := ioutil.TempDir("", "upbox")
-	if err != nil {
-		return err
-	}
-	sc.dir = tmpDir
-
-	// If anything below fails, we should clean up.
-	cleanup := true
-	defer func() {
-		if cleanup {
-			sc.Stop()
-		}
-	}()
-
-	// Generate TLS certificates.
-	if err := generateCert(sc.dir); err != nil {
-		return err
-	}
-
-	// Generate keys.
-	// Write an empty  file for use by 'upspin keygen'.
-	configKeygen := sc.Config("keygen")
-	if err := ioutil.WriteFile(configKeygen, []byte("secrets: none"), 0644); err != nil {
-		return err
-	}
-	for _, u := range sc.Users {
-		dir := filepath.Join(sc.dir, u.Name)
-		if err := os.MkdirAll(dir, 0700); err != nil {
-			return err
-		}
-		var buf bytes.Buffer
-		keygen := exec.Command("upspin", "-config="+configKeygen, "keygen", "-where="+dir)
-		keygen.Stdout = prefix("keygen: ", &buf)
-		keygen.Stderr = prefix("keygen: ", &buf)
-		if err := keygen.Run(); err != nil {
-			fmt.Fprintf(os.Stderr, "%s", buf.Bytes())
-			return err
-		}
-		u.secrets = dir
-	}
-
-	keyUser := ""
-	if s, ok := sc.server["keyserver"]; ok {
-		keyUser = s.User
-		// Start keyserver.
-		if err := sc.writeConfig(keyUser); err != nil {
-			return err
-		}
-		cmd, err := sc.startServer(s)
-		if err != nil {
-			return err
-		}
-		s.cmd = cmd
-	}
-	// Wait for the keyserver to start and add the users to it.
-	if err := waitReady(sc.KeyServer); err != nil {
-		return err
-	}
-	for _, u := range sc.Users {
-		if u.Name == keyUser {
-			continue
-		}
-
-		if err := sc.writeConfig(u.Name); err != nil {
-			return fmt.Errorf("writing config for %v: %v", u.Name, err)
-		}
-
-		if keyUser == "" {
-			continue
-		}
-		pk, err := ioutil.ReadFile(filepath.Join(sc.dir, u.Name, "public.upspinkey"))
-		if err != nil {
-			return err
-		}
-		dir, err := upspin.ParseEndpoint(u.DirServer)
-		if err != nil {
-			return err
-		}
-		store, err := upspin.ParseEndpoint(u.StoreServer)
-		if err != nil {
-			return err
-		}
-		user := &upspin.User{
-			Name:      upspin.UserName(u.Name),
-			Dirs:      []upspin.Endpoint{*dir},
-			Stores:    []upspin.Endpoint{*store},
-			PublicKey: upspin.PublicKey(pk),
-		}
-		userYAML, err := yaml.Marshal(user)
-		if err != nil {
-			return err
-		}
-		cmd := exec.Command("upspin",
-			"-config="+sc.Config(keyUser),
-			"-log="+sc.logLevel(),
-			"user", "-put",
-		)
-		cmd.Stdin = bytes.NewReader(userYAML)
-		cmd.Stdout = prefix("key-bootstrap:\t", os.Stdout)
-		cmd.Stderr = prefix("key-bootstrap:\t", os.Stderr)
-		if err := cmd.Run(); err != nil {
-			return err
-		}
-	}
-
-	// Start other servers.
-	for i := range sc.Servers {
-		s := sc.Servers[i]
-		if s.Name == "keyserver" {
-			continue
-		}
-
-		cmd, err := sc.startServer(s)
-		if err != nil {
-			return err
-		}
-		s.cmd = cmd
-	}
-	// Wait for the other servers to start.
-	for _, s := range sc.Servers {
-		if s.Name == "keyserver" {
-			continue
-		}
-		if err := waitReady(s.addr); err != nil {
-			return err
-		}
-	}
-
-	cleanup = false // Exiting successfully, so don't clean up.
-	return nil
-}
-
-// Stop terminates any running server processes and
-// deletes all temporary config files and keys.
-func (sc *Schema) Stop() error {
-	if sc.dir == "" {
-		return errors.New("cannot stop; not started")
-	}
-	for _, s := range sc.Servers {
-		if s.cmd != nil && s.cmd.Process != nil {
-			s.cmd.Process.Kill()
-		}
-	}
-	return os.RemoveAll(sc.dir)
-}
-
-// Config returns the path to the config for the given user.
-func (sc *Schema) Config(user string) string {
-	return filepath.Join(sc.dir, "config."+user)
-}
-
-// writeConfig writes a config file for user named "config.name" inside dir.
-func (sc *Schema) writeConfig(user string) error {
-	u, ok := sc.user[user]
-	if !ok {
-		return fmt.Errorf("unknown user %q", user)
-	}
-
-	configContent := []string{
-		"username: " + u.Name,
-		"secrets: " + filepath.Join(sc.dir, u.Name),
-		"tlscerts: " + sc.dir,
-		"packing: " + u.Packing,
-		"storeserver: " + u.StoreServer,
-		"dirserver: " + u.DirServer,
-	}
-	switch user {
-	case "keyserver":
-		configContent = append(configContent,
-			"keyserver: inprocess,",
-		)
-	default:
-		configContent = append(configContent,
-			"keyserver: remote,"+sc.KeyServer,
-		)
-	}
-	return ioutil.WriteFile(sc.Config(u.Name), []byte(strings.Join(configContent, "\n")), 0644)
-}
-
-// startServer starts the given server, returning the running exec.Cmd.
-func (sc *Schema) startServer(s *Server) (*exec.Cmd, error) {
-	args := []string{
-		"-config=" + sc.Config(s.User),
-		"-log=" + sc.logLevel(),
-		"-tls_cert=" + filepath.Join(sc.dir, "cert.pem"),
-		"-tls_key=" + filepath.Join(sc.dir, "key.pem"),
-		"-letscache=", // disable
-		"-https=" + s.addr,
-		"-addr=" + s.addr,
-	}
-	if s.Name == "keyserver" {
-		args = append(args,
-			"-test_user="+s.User,
-			"-test_secrets="+filepath.Join(sc.dir, s.User),
-		)
-	}
-	for k, v := range s.Flags {
-		args = append(args, fmt.Sprintf("-%s=%v", k, v))
-	}
-	cmd := exec.Command(s.Name, args...)
-	cmd.Stdout = prefix(s.Name+":\t", os.Stdout)
-	cmd.Stderr = prefix(s.Name+":\t", os.Stderr)
-	if err := cmd.Start(); err != nil {
-		return nil, fmt.Errorf("starting %v: %v", s.Name, err)
-	}
-	return cmd, nil
-}
-
-func (sc *Schema) logLevel() string {
-	if l := sc.LogLevel; l != "" {
-		return l
-	}
-	return "error"
-}
-
-func prefix(p string, out io.Writer) io.Writer {
-	r, w := io.Pipe()
-	go func() {
-		s := bufio.NewScanner(r)
-		for s.Scan() {
-			fmt.Fprintf(out, "%s%s\n", p, s.Bytes())
-		}
-	}()
-	return w
-}
-
-func waitReady(addr string) error {
-	rt := &http.Transport{
-		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: true,
-		},
-	}
-	req, _ := http.NewRequest("GET", "https://"+addr, nil)
-	for i := 0; i < 10; i++ {
-		_, err := rt.RoundTrip(req)
-		if err != nil {
-			time.Sleep(time.Second)
-			continue
-		}
-		return nil
-	}
-	return fmt.Errorf("timed out waiting for %q to come up", addr)
-}