cloud/storage/dropbox: add Dropbox storage backend support

update #468

Change-Id: Ifa6d2b01262698095ba23e01ea923beab78e710b
Reviewed-on: https://upspin-review.googlesource.com/13740
Reviewed-by: Andrew Gerrand <adg@golang.org>
diff --git a/cloud/storage/dropbox/dropbox.go b/cloud/storage/dropbox/dropbox.go
new file mode 100644
index 0000000..3abb0ec
--- /dev/null
+++ b/cloud/storage/dropbox/dropbox.go
@@ -0,0 +1,174 @@
+// Copyright 2016 The Upspin Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package dropbox implements a storage backend that saves data to a User
+// Dropbox.
+package dropbox // import "dropbox.upspin.io/cloud/storage/dropbox"
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"upspin.io/cloud/storage"
+	"upspin.io/errors"
+	"upspin.io/upspin"
+)
+
+// apiToken is the key for the dial options in the storage.Storage interface.
+const apiToken = "token"
+
+// New initializes a Storage implementation that stores data to Dropbox.
+func New(opts *storage.Opts) (storage.Storage, error) {
+	const op = "cloud/storage/dropbox.New"
+
+	tok, ok := opts.Opts[apiToken]
+	if !ok {
+		return nil, errors.E(op, errors.Invalid, errors.Errorf("%q option is required", apiToken))
+	}
+
+	return &dropboxImpl{
+		client: http.DefaultClient,
+		token:  tok,
+	}, nil
+}
+
+func init() {
+	storage.Register("Dropbox", New)
+}
+
+// dropboxImpl is an implementation of Storage that connects to a Dropbox backend.
+type dropboxImpl struct {
+	client *http.Client
+	token  string
+}
+
+// Guarantee we implement the Storage interface
+var _ storage.Storage = (*dropboxImpl)(nil)
+
+// LinkBase implements Storage.
+func (d *dropboxImpl) LinkBase() (base string, err error) {
+	return "", upspin.ErrNotSupported
+}
+
+// Download implements Storage.
+func (d *dropboxImpl) Download(ref string) ([]byte, error) {
+	const op = "cloud/storage/dropbox.Download"
+
+	arg, _ := json.Marshal(struct {
+		Path string `json:"path"`
+	}{"/" + ref})
+
+	req, err := d.newRequest("https://content.dropboxapi.com/2/files/download", nil, string(arg))
+	if err != nil {
+		return nil, errors.E(op, errors.Other, err)
+	}
+
+	data, err := d.doRequest(req)
+	if err != nil {
+		return nil, errors.E(op, errors.IO, err)
+	}
+
+	return data, nil
+}
+
+// Put implements Storage.
+func (d *dropboxImpl) Put(ref string, contents []byte) error {
+	const op = "cloud/storage/dropbox.Put"
+
+	arg, _ := json.Marshal(struct {
+		Path   string `json:"path"`
+		Mode   string `json:"mode"`
+		Rename bool   `json:"autorename"`
+		Mute   bool   `json:"mute"`
+	}{
+		"/" + ref,
+		"overwrite",
+		true,
+		false,
+	})
+
+	body := bytes.NewReader(contents)
+
+	// The endpoint has an upload limit of 150 MB which is fine for the Upspin
+	// default blocksize. If the Upspin blocksize is set larger than this limit,
+	// the "upload_session/start" endpoint should be used.
+	req, err := d.newRequest("https://content.dropboxapi.com/2/files/upload", body, string(arg))
+	if err != nil {
+		return errors.E(op, errors.Other, err)
+	}
+
+	_, err = d.doRequest(req)
+	if err != nil {
+		return errors.E(op, errors.IO, err)
+	}
+
+	return nil
+}
+
+// Delete implements Storage.
+func (d *dropboxImpl) Delete(ref string) error {
+	const op = "cloud/storage/dropbox.Delete"
+
+	arg, _ := json.Marshal(struct {
+		Path string `json:"path"`
+	}{"/" + ref})
+
+	body := bytes.NewReader(arg)
+
+	req, err := d.newRequest("https://api.dropboxapi.com/2/files/delete_v2", body, "")
+	if err != nil {
+		return errors.E(op, errors.Other, err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	_, err = d.doRequest(req)
+	if err != nil {
+		return errors.E(op, errors.IO, err)
+	}
+
+	return nil
+}
+
+// Close implements Storage.
+func (d *dropboxImpl) Close() {
+	// not yet implemented
+}
+
+func (d *dropboxImpl) newRequest(path string, body io.Reader, arg string) (*http.Request, error) {
+	req, err := http.NewRequest("POST", path, body)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Add("Authorization", "Bearer "+d.token)
+	req.Header.Add("Content-Type", "application/octet-stream")
+
+	if arg != "" {
+		req.Header.Add("Dropbox-API-Arg", arg)
+	}
+
+	return req, nil
+}
+
+func (d *dropboxImpl) doRequest(req *http.Request) ([]byte, error) {
+	resp, err := d.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return nil, errors.Errorf(resp.Status)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return body, nil
+}
diff --git a/cloud/storage/dropbox/dropbox_test.go b/cloud/storage/dropbox/dropbox_test.go
new file mode 100644
index 0000000..7469369
--- /dev/null
+++ b/cloud/storage/dropbox/dropbox_test.go
@@ -0,0 +1,133 @@
+// Copyright 2016 The Upspin Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dropbox
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"golang.org/x/oauth2"
+
+	"upspin.io/cloud/storage"
+)
+
+var (
+	client      storage.Storage
+	testDataStr = fmt.Sprintf("This is test at %v", time.Now())
+	testData    = []byte(testDataStr)
+	fileName    = fmt.Sprintf("test-file-%d", time.Now().Second())
+
+	authCode   = flag.String("code", "", "dropbox authentication code")
+	useDropbox = flag.Bool("use_dropbox", false, "enable to run dropbox tests; requires authentication code")
+)
+
+// This is more of a regression test as it uses the running cloud
+// storage in prod. However, since Dropbox is always available, we accept
+// to rely on it.
+func TestPutGetAndDownload(t *testing.T) {
+	err := client.Put(fileName, testData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := client.Download(fileName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(data) != testDataStr {
+		t.Errorf("Expected %q got %q", testDataStr, string(data))
+	}
+	// Check that Download yields the same data
+	bytes, err := client.Download(fileName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(bytes) != testDataStr {
+		t.Errorf("Expected %q got %q", testDataStr, string(bytes))
+	}
+}
+
+func TestDelete(t *testing.T) {
+	err := client.Put(fileName, testData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = client.Delete(fileName)
+	if err != nil {
+		t.Fatalf("Expected no errors, got %v", err)
+	}
+	// Test the side effect after Delete.
+	_, err = client.Download(fileName)
+	if err == nil {
+		t.Fatal("Expected an error, but got none")
+	}
+}
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+	if !*useDropbox {
+		log.Printf(`
+cloud/storage/dropbox: skipping test as it requires Dropbox access. To enable this test,
+on the first run get an authentication code by visiting:
+
+https://www.dropbox.com/oauth2/authorize?client_id=ufhy41x7g4obzqz&response_type=code
+
+Copy the code and pass it by the -code flag. This will get an oAuth2 access token, store
+it and reuse it in successive test calls.
+
+`)
+		os.Exit(0)
+	}
+
+	t, err := token()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error in getting oauth2 token: %v.\n", err)
+	}
+
+	// Create client that writes to your Dropbox.
+	client, err = storage.Dial("Dropbox",
+		storage.WithKeyValue("token", t))
+	if err != nil {
+		log.Fatalf("cloud/storage/dropbox: couldn't set up client: %v", err)
+	}
+
+	code := m.Run()
+
+	os.Exit(code)
+}
+
+func token() (string, error) {
+	tokenFile := path.Join(os.TempDir(), "upspin-test-token")
+
+	token, _ := ioutil.ReadFile(tokenFile)
+	if err == nil {
+		return string(token), nil
+	}
+
+	conf := &oauth2.Config{
+		ClientID:     "ufhy41x7g4obzqz",
+		ClientSecret: "vuhgmucmxm93dp5",
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://www.dropbox.com/oauth2/authorize",
+			TokenURL: "https://api.dropboxapi.com/oauth2/token",
+		},
+	}
+
+	tok, err := conf.Exchange(oauth2.NoContext, *authCode)
+	if err != nil {
+		return "", err
+	}
+
+	if err := ioutil.WriteFile(tokenFile, []byte(tok.AccessToken), 0600); err != nil {
+		return "", err
+	}
+
+	return tok.AccessToken, nil
+}
diff --git a/cmd/upspinserver-dropbox/main.go b/cmd/upspinserver-dropbox/main.go
new file mode 100644
index 0000000..0c3a633
--- /dev/null
+++ b/cmd/upspinserver-dropbox/main.go
@@ -0,0 +1,22 @@
+// Copyright 2017 The Upspin Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Command upspinserver-dropbox is a combined DirServer and StoreServer for use on
+// stand-alone machines. It provides the production implementations of the
+// dir and store servers (dir/server and store/server) with support for storage
+// in Dropbox.
+package main // import "dropbox.upspin.io/cmd/upspinserver-dropbox"
+
+import (
+	"upspin.io/cloud/https"
+	"upspin.io/serverutil/upspinserver"
+
+	// Storage on Dropbox.
+	_ "dropbox.upspin.io/cloud/storage/dropbox"
+)
+
+func main() {
+	ready := upspinserver.Main()
+	https.ListenAndServe(ready, https.OptionsFromFlags())
+}