cloud/storage/openstack: Storage implementation

Updates upspin/upspin#485

Change-Id: I26766770b9006637aa20712c2c1aaf69cabbb5ef
Reviewed-on: https://upspin-review.googlesource.com/16000
Reviewed-by: Andrew Gerrand <adg@golang.org>
diff --git a/cloud/storage/openstack/openstack.go b/cloud/storage/openstack/openstack.go
new file mode 100755
index 0000000..ea21f5d
--- /dev/null
+++ b/cloud/storage/openstack/openstack.go
@@ -0,0 +1,163 @@
+// Copyright 2017 The Upspin Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package openstack implements a storage backend that saves
+// data to an OpenStack container, e.g., OVH Object Storage.
+package openstack // import "openstack.upspin.io/cloud/storage/openstack"
+
+import (
+	"bytes"
+
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/openstack"
+	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/containers"
+	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/objects"
+
+	"upspin.io/cloud/storage"
+	"upspin.io/errors"
+	"upspin.io/upspin"
+)
+
+const storageName = "OpenStack"
+
+// OpenStack specific option names.
+const (
+	openstackRegion     = "openstackRegion"
+	openstackContainer  = "openstackContainer"
+	openstackAuthURL    = "openstackAuthURL"
+	openstackTenantName = "privateOpenstackTenantName"
+	openstackUsername   = "privateOpenstackUsername"
+	openstackPassword   = "privateOpenstackPassword"
+)
+
+var requiredOpts = []string{
+	openstackRegion,
+	openstackContainer,
+	openstackAuthURL,
+	openstackTenantName,
+	openstackUsername,
+	openstackPassword,
+}
+
+// See https://docs.openstack.org/swift/latest/overview_acl.html
+const containerPublicACL = ".r:*"
+
+type openstackStorage struct {
+	client    *gophercloud.ServiceClient
+	container string
+}
+
+// New creates a new instance of the OpenStack implementation of
+// storage.Storage.
+func New(opts *storage.Opts) (storage.Storage, error) {
+	const op = "cloud/storage/openstack.New"
+
+	for _, opt := range requiredOpts {
+		if _, ok := opts.Opts[opt]; !ok {
+			return nil, errors.E(op, errors.Invalid, errors.Errorf(
+				"%q option is required", opt))
+		}
+	}
+
+	authOpts := gophercloud.AuthOptions{
+		IdentityEndpoint: opts.Opts[openstackAuthURL],
+		Username:         opts.Opts[openstackUsername],
+		Password:         opts.Opts[openstackPassword],
+		TenantName:       opts.Opts[openstackTenantName],
+	}
+
+	// When the token expires the services returns 401 and we need to be
+	// able to authenticate again.
+	authOpts.AllowReauth = true
+
+	provider, err := openstack.AuthenticatedClient(authOpts)
+	if err != nil {
+		return nil, errors.E(op, errors.Permission, errors.Errorf(
+			"Could not authenticate: %s", err))
+	}
+
+	client, err := openstack.NewObjectStorageV1(provider, gophercloud.EndpointOpts{
+		Region: opts.Opts[openstackRegion],
+	})
+	if err != nil {
+		// The error kind is "Invalid" because AFAICS this can only
+		// happen for unknown region
+		return nil, errors.E(op, errors.Invalid, errors.Errorf(
+			"Could not create object storage client: %s", err))
+	}
+
+	return &openstackStorage{
+		client:    client,
+		container: opts.Opts[openstackContainer],
+	}, nil
+}
+
+func init() {
+	err := storage.Register(storageName, New)
+	if err != nil {
+		// If more modules are registering under the same storage name,
+		// an application should not start.
+		panic(err)
+	}
+}
+
+var _ storage.Storage = (*openstackStorage)(nil)
+
+// LinkBase will return the URL if the container has read access for everybody
+// and an unsupported error in case it does not. Still, it might return an
+// error because it can't get the necessary metadata.
+func (s *openstackStorage) LinkBase() (string, error) {
+	const op = "cloud/storage/openstack.LinkBase"
+
+	r := containers.Get(s.client, s.container)
+	h, err := r.Extract()
+	if err != nil {
+		return "", errors.E(op, errors.Internal, errors.Errorf(
+			"Unable to extract header: %s", err))
+	}
+	for _, acl := range h.Read {
+		if acl == containerPublicACL {
+			return s.client.ServiceURL(s.container) + "/", nil
+		}
+	}
+	return "", upspin.ErrNotSupported
+}
+
+func (s *openstackStorage) Download(ref string) ([]byte, error) {
+	const op = "cloud/storage/openstack.Download"
+
+	r := objects.Download(s.client, s.container, ref, nil)
+	contents, err := r.ExtractContent()
+	if err != nil {
+		if _, ok := err.(gophercloud.ErrDefault404); ok {
+			return nil, errors.E(op, errors.NotExist, err)
+		}
+		return nil, errors.E(op, errors.IO, errors.Errorf(
+			"Unable to download ref %q from container %q: %s", ref, s.container, err))
+	}
+	return contents, nil
+}
+
+func (s *openstackStorage) Put(ref string, contents []byte) error {
+	const op = "cloud/storage/openstack.Put"
+
+	opts := objects.CreateOpts{Content: bytes.NewReader(contents)}
+	err := objects.Create(s.client, s.container, ref, opts).Err
+	if err != nil {
+		return errors.E(op, errors.IO, errors.Errorf(
+			"Unable to upload ref %q to container %q: %s", ref, s.container, err))
+	}
+	return nil
+}
+
+func (s *openstackStorage) Delete(ref string) error {
+	const op = "cloud/storage/openstack.Delete"
+
+	err := objects.Delete(s.client, s.container, ref, nil).Err
+	if err != nil {
+		return errors.E(op, errors.IO, errors.Errorf(
+			"Unable to delete ref %q from container %q: %s", ref, s.container, err))
+	}
+	return nil
+}
diff --git a/cloud/storage/openstack/openstack_test.go b/cloud/storage/openstack/openstack_test.go
new file mode 100755
index 0000000..cd12c57
--- /dev/null
+++ b/cloud/storage/openstack/openstack_test.go
@@ -0,0 +1,150 @@
+// Copyright 2017 The Upspin Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package openstack
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/containers"
+
+	"upspin.io/cloud/storage"
+	"upspin.io/errors"
+	"upspin.io/log"
+)
+
+const (
+	defaultTestRegion    = "BHS3"
+	defaultTestContainer = "upspin-test-container"
+)
+
+var (
+	client storage.Storage
+
+	objectName     = fmt.Sprintf("test-file-%d", time.Now().Second())
+	objectContents = []byte(fmt.Sprintf("This is test at %v", time.Now()))
+
+	testRegion    = flag.String("test_region", defaultTestRegion, "region to use for the test container")
+	testContainer = flag.String("test_container", defaultTestContainer, "container to use for testing")
+
+	useOpenStack = flag.Bool("use_openstack", false, "enable to run OpenStack tests; requires OpenStack credentials")
+)
+
+func TestPutAndDownloadFromLinkBase(t *testing.T) {
+	err := client.Put(objectName, objectContents)
+	if err != nil {
+		t.Fatalf("Could not put: %v", err)
+	}
+	base, err := client.LinkBase()
+	if err != nil {
+		t.Fatalf("Could not get container base: %v", err)
+	}
+	response, err := http.Get(base + objectName)
+	if err != nil {
+		t.Fatalf("Could not get from container base: %v", err)
+	}
+	storedBytes, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		t.Fatalf("Could not read response body: %v", err)
+	}
+	if string(storedBytes) != string(objectContents) {
+		t.Fatalf("Downloaded contents do not match, wanted %s and got %s",
+			objectContents, string(storedBytes))
+	}
+}
+
+func TestDownloadMissing(t *testing.T) {
+	_, err := client.Download("Something I never uploaded")
+	uerr, ok := err.(*errors.Error)
+	if !ok {
+		t.Fatalf("Expected Upspin error, got %v", err)
+	}
+	if uerr.Kind != errors.NotExist {
+		t.Fatalf("Expected NotExist kind, got: %v", uerr.Kind)
+	}
+}
+
+func TestPutAndDownload(t *testing.T) {
+	err := client.Put(objectName, objectContents)
+	if err != nil {
+		t.Fatalf("Could not put: %v", err)
+	}
+	storedBytes, err := client.Download(objectName)
+	if err != nil {
+		t.Fatalf("Could not download: %v", err)
+	}
+	if string(storedBytes) != string(objectContents) {
+		t.Errorf("Downloaded contents do not match, expected %q got %q",
+			string(objectContents), string(storedBytes))
+	}
+}
+
+func TestPutAndDelete(t *testing.T) {
+	err := client.Put(objectName, objectContents)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = client.Delete(objectName)
+	if err != nil {
+		t.Fatalf("Expected no errors, got %v", err)
+	}
+	_, err = client.Download(objectName)
+	if err == nil {
+		t.Fatal("Expected an error, but got none")
+	}
+}
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+	if !*useOpenStack {
+		log.Printf(`
+
+cloud/storage/openstack: skipping test as it requires OpenStack access. To
+enable this test, ensure you are properly authorized to upload to an OpenStack
+container named by flag -test_container and then set this test's flag
+-use_openstack.
+
+`)
+		os.Exit(0)
+	}
+
+	// Create client that writes to test container.
+	var err error
+	client, err = storage.Dial(
+		"OpenStack",
+		storage.WithKeyValue("openstackRegion", *testRegion),
+		storage.WithKeyValue("openstackContainer", *testContainer),
+	)
+	if err != nil {
+		log.Fatalf("cloud/storage/openstack: couldn't set up client: %v", err)
+	}
+	if err := client.(*openstackStorage).createContainer(); err != nil {
+		log.Fatalf("cloud/storage/openstack: createContainer failed: %v", err)
+	}
+
+	exitCode := m.Run()
+
+	// Clean up.
+	if err := client.(*openstackStorage).deleteContainer(); err != nil {
+		log.Fatalf("cloud/storage/openstack: deleteContainer failed: %v", err)
+	}
+
+	os.Exit(exitCode)
+}
+
+func (s *openstackStorage) createContainer() error {
+	return containers.Create(s.client, s.container, containers.CreateOpts{
+		ContainerRead: containerPublicACL,
+	}).Err
+}
+
+func (s *openstackStorage) deleteContainer() error {
+	return containers.Delete(s.client, s.container).Err
+}
diff --git a/cmd/upspinserver-openstack/main.go b/cmd/upspinserver-openstack/main.go
new file mode 100755
index 0000000..60c5720
--- /dev/null
+++ b/cmd/upspinserver-openstack/main.go
@@ -0,0 +1,22 @@
+// Copyright 2017 The Upspin Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Command upspinserver-openstack is a combined DirServer and
+// StoreServer for use on stand-alone machines. It provides the
+// production implementations of the dir and store servers
+// (dir/server and store/server) with support for storage in
+// OpenStack e.g. OVH Object Storage.
+package main // import "openstack.upspin.io/cmd/upspinserver-openstack"
+
+import (
+	_ "openstack.upspin.io/cloud/storage/openstack"
+
+	"upspin.io/cloud/https"
+	"upspin.io/serverutil/upspinserver"
+)
+
+func main() {
+	ready := upspinserver.Main()
+	https.ListenAndServe(ready, https.OptionsFromFlags())
+}