cloud/storage/drive: initial implementation

Contains the Upspin server for Google Drive. Follow-up of CL 16460.

Updates #479

Change-Id: I489e36a833e6cb79659be66673e0a4d3da9409f5
Reviewed-on: https://upspin-review.googlesource.com/16480
Reviewed-by: Andrew Gerrand <adg@golang.org>
diff --git a/cloud/storage/drive/drive.go b/cloud/storage/drive/drive.go
new file mode 100644
index 0000000..07f8737
--- /dev/null
+++ b/cloud/storage/drive/drive.go
@@ -0,0 +1,173 @@
+// Copyright 2017 The Upspin Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package drive implements a storage.Storage that stores data in Google Drive.
+package drive // import "drive.upspin.io/cloud/storage/drive"
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"time"
+
+	"drive.upspin.io/config"
+
+	"upspin.io/cache"
+	"upspin.io/cloud/storage"
+	"upspin.io/errors"
+	"upspin.io/upspin"
+
+	"golang.org/x/oauth2"
+	"google.golang.org/api/drive/v3"
+	"google.golang.org/api/googleapi"
+)
+
+func init() {
+	storage.Register("Drive", New)
+}
+
+// lruSize holds the maximum number of entries that should live in the LRU cache.
+// Since it only maps file names to file IDs, 1024 should be affordable to any server.
+const lruSize = 1024
+
+// New initializes a new Storage which stores data to Google Drive.
+func New(o *storage.Opts) (storage.Storage, error) {
+	const op = "cloud/storage/drive.New"
+	var (
+		t   oauth2.Token
+		ok  bool
+		err error
+	)
+	t.AccessToken, ok = o.Opts["accessToken"]
+	if !ok {
+		return nil, errors.E(op, errors.Invalid, errors.Str("missing accessToken"))
+	}
+	t.TokenType, ok = o.Opts["tokenType"]
+	if !ok {
+		return nil, errors.E(op, errors.Invalid, errors.Str("missing tokenType"))
+	}
+	t.RefreshToken, ok = o.Opts["refreshToken"]
+	if !ok {
+		return nil, errors.E(op, errors.Invalid, errors.Str("missing refreshToken"))
+	}
+	t.Expiry, err = time.Parse(time.RFC3339, o.Opts["expiry"])
+	if err != nil {
+		return nil, errors.E(op, errors.Invalid, errors.Errorf("invalid expiry %q: %v", o.Opts["expiry"], err))
+	}
+	ctx := context.Background()
+	client := config.OAuth2.Client(ctx, &t)
+	svc, err := drive.New(client)
+	if err != nil {
+		return nil, errors.E(op, errors.Internal, err)
+	}
+	return &driveImpl{
+		files: svc.Files,
+		ids:   cache.NewLRU(lruSize),
+	}, nil
+}
+
+var _ storage.Storage = (*driveImpl)(nil)
+
+// driveImpl is an implementation of Storage that connects to a Google Drive backend.
+type driveImpl struct {
+	// files holds the FilesService used to interact with the Drive API.
+	files *drive.FilesService
+	// ids will map file names to file IDs to avoid hitting the HTTP API
+	// twice on each download.
+	ids *cache.LRU
+}
+
+func (d *driveImpl) LinkBase() (string, error) {
+	// Drive does have a LinkBase but it expects it to be followed by the file ID,
+	// not by the name of the file. Since we can not use the 'ref' as the file ID
+	// this service is not available.
+	return "", upspin.ErrNotSupported
+}
+
+func (d *driveImpl) Download(ref string) ([]byte, error) {
+	const op = "cloud/storage/drive.Download"
+	id, err := d.fileId(ref)
+	if os.IsNotExist(err) {
+		return nil, errors.E(op, errors.NotExist, err)
+	}
+	if err != nil {
+		return nil, errors.E(op, errors.IO, err)
+	}
+	resp, err := d.files.Get(id).Download()
+	if err != nil {
+		return nil, errors.E(op, errors.IO, err)
+	}
+	slurp, err := ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	if err != nil {
+		return nil, errors.E(op, errors.IO, err)
+	}
+	return slurp, nil
+}
+
+func (d *driveImpl) Put(ref string, contents []byte) error {
+	const op = "cloud/storage/drive.Put"
+	// Test whether file exists.
+	id, err := d.fileId(ref)
+	if err != nil && !os.IsNotExist(err) {
+		return errors.E(op, errors.IO, err)
+	}
+	if id != "" {
+		// The file exists. Delete it to ensure uniqueness because Google Drive allows
+		// multiple files with the same name to coexist in the same folder. See:
+		// https://developers.google.com/drive/v3/reference/files#properties
+		if err := d.Delete(ref); err != nil {
+			return errors.E(op, err)
+		}
+	}
+	create := d.files.Create(&drive.File{
+		Name:    ref,
+		Parents: []string{"appDataFolder"},
+	})
+	contentType := googleapi.ContentType("application/octet-stream")
+	_, err = create.Media(bytes.NewReader(contents), contentType).Do()
+	if err != nil {
+		return errors.E(op, errors.IO, err)
+	}
+	return nil
+}
+
+func (d *driveImpl) Delete(ref string) error {
+	const op = "cloud/storage/drive.Download"
+	id, err := d.fileId(ref)
+	if os.IsNotExist(err) {
+		// nothing to delete
+		return nil
+	}
+	if err != nil {
+		return errors.E(op, errors.IO, err)
+	}
+	if err := d.files.Delete(id).Do(); err != nil {
+		return errors.E(op, errors.IO, err)
+	}
+	d.ids.Remove(ref)
+	return nil
+}
+
+// fileId returns the file ID of the first file found under the given name.
+func (d *driveImpl) fileId(name string) (string, error) {
+	// try cache first
+	if id, ok := d.ids.Get(name); ok {
+		return id.(string), nil
+	}
+	q := fmt.Sprintf(`name=%q`, name)
+	list := d.files.List().Spaces("appDataFolder").Q(q).Fields("files(id)")
+	r, err := list.Do()
+	if err != nil {
+		return "", err
+	}
+	if len(r.Files) == 0 {
+		return "", os.ErrNotExist
+	}
+	id := r.Files[0].Id
+	d.ids.Add(name, id)
+	return id, nil
+}
diff --git a/cloud/storage/drive/drive_e2e_test.go b/cloud/storage/drive/drive_e2e_test.go
new file mode 100644
index 0000000..b9ba46a
--- /dev/null
+++ b/cloud/storage/drive/drive_e2e_test.go
@@ -0,0 +1,109 @@
+// Copyright 2017 The Upspin Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package drive
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"upspin.io/cloud/storage"
+	"upspin.io/errors"
+	"upspin.io/log"
+)
+
+const testFilePrefix = "Upspin-Storage-Test"
+
+var (
+	client   storage.Storage
+	testData = []byte(fmt.Sprintf("Upspin storage test at %v", time.Now()))
+	fileName = fmt.Sprintf("%s%d", testFilePrefix, time.Now().Second())
+
+	accessToken  = flag.String("access-token", "", "oauth2 access token")
+	refreshToken = flag.String("refresh-token", "", "oauth2 refresh token")
+	expiry       = flag.String("expiry", time.Now().Format(time.RFC3339), "RFC3339 format time stamp")
+	runE2E       = flag.Bool("run-e2e", false, "enable to run tests against an actual Drive account")
+)
+
+func TestPutAndDownload(t *testing.T) {
+	err := client.Put(fileName, testData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := client.Download(fileName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(got, testData) {
+		t.Errorf("Expected %q got %q", testData, got)
+	}
+}
+
+func TestDeleteAndDownload(t *testing.T) {
+	err := client.Put(fileName, testData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = client.Delete(fileName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = client.Download(fileName)
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+	if !errors.Is(errors.NotExist, err) {
+		t.Fatalf("expected NotExist error, got %v", err)
+	}
+}
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+	if !*runE2E || *accessToken == "" || *refreshToken == "" {
+		log.Printf(`
+
+cloud/storage/drive: skipping test as it requires Google Drive credentials. To enable this
+test, set the -run-e2e flag along with valid -access-token and -refresh-token
+flag values.
+
+`)
+		os.Exit(0)
+	}
+	// Set up Drive client.
+	var err error
+	client, err = storage.Dial("Drive",
+		storage.WithKeyValue("accessToken", *accessToken),
+		storage.WithKeyValue("refreshToken", *refreshToken),
+		storage.WithKeyValue("tokenType", "Bearer"),
+		storage.WithKeyValue("expiry", *expiry))
+	if err != nil {
+		log.Fatalf("cloud/storage/drive: couldn't set up client: %v", err)
+	}
+
+	code := m.Run()
+
+	// Clean up.
+	client.(*driveImpl).cleanup()
+	os.Exit(code)
+}
+
+// cleanup removes all files with the prefix testFilePrefix from Google Drive.
+func (d *driveImpl) cleanup() {
+	q := fmt.Sprintf("name contains %q", testFilePrefix)
+	call := d.files.List().Spaces("appDataFolder").Q(q).Fields("files(id, name)")
+	r, err := call.Do()
+	if err != nil {
+		log.Fatalf("cleanup: %v", err)
+	}
+	for _, f := range r.Files {
+		if err := d.files.Delete(f.Id).Do(); err != nil {
+			log.Fatalf("cleanup: %v", err)
+		}
+		d.ids.Remove(f.Name)
+	}
+}
diff --git a/cmd/upspinserver-drive/main.go b/cmd/upspinserver-drive/main.go
new file mode 100644
index 0000000..578a9d6
--- /dev/null
+++ b/cmd/upspinserver-drive/main.go
@@ -0,0 +1,21 @@
+// Copyright 2017 The Upspin Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Command upspinserver-drive is a combined DirServer and StoreServer for use on
+// stand-alone machines. It provides the production implementations of the dir and
+// store servers (dir/server and store/server) with support for storage in Google Drive.
+package main // import "drive.upspin.io/cmd/upspinserver-drive"
+
+import (
+	"upspin.io/cloud/https"
+	"upspin.io/serverutil/upspinserver"
+
+	// Storage on Google Drive.
+	_ "drive.upspin.io/cloud/storage/drive"
+)
+
+func main() {
+	ready := upspinserver.Main()
+	https.ListenAndServe(ready, https.OptionsFromFlags())
+}
diff --git a/config/oauth2.go b/config/oauth2.go
index 2db4fe5..d3ba191 100644
--- a/config/oauth2.go
+++ b/config/oauth2.go
@@ -11,6 +11,8 @@
 	"google.golang.org/api/drive/v3"
 )
 
+// OAuth2 holds OAuth configuration used by the Upspin Google Drive package. It is used by both
+// the storage and the setup process.
 var OAuth2 = &oauth2.Config{
 	ClientID:     "756365541666-dbbsja2vlrl38j0r85f32cgl3sj6n8k9.apps.googleusercontent.com",
 	ClientSecret: "RfAusHn6sSN7YO2pErac0ggs",