Merge pull request #37886 from obnoxxx/gluster-dp-gid

examples/experimental/persistent-volume-provisioning/README.md

@@ -78,6 +78,8 @@ parameters:
   restuser: "admin"
   secretNamespace: "default"
   secretName: "heketi-secret"
+  gidMin: "40000"
+  gidMax: "50000"
 ```
 
 * `resturl` : Gluster REST service/Heketi service url which provision gluster volumes on demand. The general format should be `IPaddress:Port` and this is a mandatory parameter for GlusterFS dynamic provisioner. If Heketi service is exposed as a routable service in openshift/kubernetes setup, this can have a format similar to
@@ -92,6 +94,8 @@ When both `restuserkey` and `secretNamespace` + `secretName` is specified, the s
 
 Example of a secret can be found in [glusterfs-provisioning-secret.yaml](glusterfs-provisioning-secret.yaml).
 
+* `gidMin` + `gidMax` : The minimum and maximum value of GID range for the storage class. A unique value (GID) in this range ( gidMin-gidMax ) will be used for dynamically provisioned volumes. These are optional values. If not specified, the volume will be provisioned with a value between 2000-4294967295 which are defaults for gidMin and gidMax respectively.
+
 Reference : ([How to configure Heketi](https://github.com/heketi/heketi/wiki/Setting-up-the-topology))
 
 When the persistent volumes are dynamically provisioned, the Gluster plugin automatically create an endpoint and a headless service in the name `gluster-dynamic-<claimname>`. This dynamic endpoint and service will be deleted automatically when the persistent volume claim is deleted.

examples/experimental/persistent-volume-provisioning/glusterfs-dp.yaml

@@ -9,3 +9,5 @@ parameters:
   restuser: "admin"
   secretNamespace: "default"
   secretName: "heketi-secret"
+  gidMin: "40000"
+  gidMax: "50000"

pkg/volume/glusterfs/BUILD

@@ -15,6 +15,7 @@ go_library(
     srcs = [
         "doc.go",
         "glusterfs.go",
+        "glusterfs_minmax.go",
         "glusterfs_util.go",
     ],
     tags = ["automanaged"],
@@ -22,13 +23,17 @@ go_library(
         "//pkg/api/errors:go_default_library",
         "//pkg/api/resource:go_default_library",
         "//pkg/api/v1:go_default_library",
+        "//pkg/apis/storage/v1beta1/util:go_default_library",
         "//pkg/client/clientset_generated/release_1_5:go_default_library",
+        "//pkg/labels:go_default_library",
+        "//pkg/registry/core/service/allocator:go_default_library",
         "//pkg/types:go_default_library",
         "//pkg/util/exec:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//pkg/util/strings:go_default_library",
         "//pkg/volume:go_default_library",
         "//pkg/volume/util:go_default_library",
+        "//pkg/volume/util/volumehelper:go_default_library",
         "//vendor:github.com/golang/glog",
         "//vendor:github.com/heketi/heketi/client/api/go-client",
         "//vendor:github.com/heketi/heketi/pkg/glusterfs/api",
@@ -37,7 +42,10 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = ["glusterfs_test.go"],
+    srcs = [
+        "glusterfs_minmax_test.go",
+        "glusterfs_test.go",
+    ],
     library = "go_default_library",
     tags = ["automanaged"],
     deps = [

pkg/volume/glusterfs/glusterfs_minmax.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+//
+// This implementation is space-efficient for a sparse
+// allocation over a big range. Could be optimized
+// for high absolute allocation number with a bitmap.
+//
+
+package glusterfs
+
+import (
+	"errors"
+	"sync"
+
+	"k8s.io/kubernetes/pkg/registry/core/service/allocator"
+)
+
+var (
+	ErrNotFound     = errors.New("number not allocated")
+	ErrConflict     = errors.New("number already allocated")
+	ErrInvalidRange = errors.New("invalid range")
+	ErrOutOfRange   = errors.New("out of range")
+	ErrRangeFull    = errors.New("range full")
+	ErrInternal     = errors.New("internal error")
+)
+
+type MinMaxAllocator struct {
+	lock sync.Mutex
+	min  int
+	max  int
+	free int
+	used map[int]bool
+}
+
+var _ Rangeable = &MinMaxAllocator{}
+
+// Rangeable is an Interface that can adjust its min/max range.
+// Rangeable should be threadsafe
+type Rangeable interface {
+	allocator.Interface
+	SetRange(min, max int) error
+}
+
+func NewMinMaxAllocator(min, max int) (*MinMaxAllocator, error) {
+	if min > max {
+		return nil, ErrInvalidRange
+	}
+	return &MinMaxAllocator{
+		min:  min,
+		max:  max,
+		free: 1 + max - min,
+		used: map[int]bool{},
+	}, nil
+}
+
+func (a *MinMaxAllocator) SetRange(min, max int) error {
+	if min > max {
+		return ErrInvalidRange
+	}
+
+	a.lock.Lock()
+	defer a.lock.Unlock()
+
+	// Check if we need to change
+	if a.min == min && a.max == max {
+		return nil
+	}
+
+	a.min = min
+	a.max = max
+
+	// Recompute how many free we have in the range
+	num_used := 0
+	for i := range a.used {
+		if a.inRange(i) {
+			num_used++
+		}
+	}
+	a.free = 1 + max - min - num_used
+
+	return nil
+}
+
+func (a *MinMaxAllocator) Allocate(i int) (bool, error) {
+	a.lock.Lock()
+	defer a.lock.Unlock()
+
+	if !a.inRange(i) {
+		return false, ErrOutOfRange
+	}
+
+	if a.has(i) {
+		return false, ErrConflict
+	}
+
+	a.used[i] = true
+	a.free--
+
+	return true, nil
+}
+
+func (a *MinMaxAllocator) AllocateNext() (int, bool, error) {
+	a.lock.Lock()
+	defer a.lock.Unlock()
+
+	// Fast check if we're out of items
+	if a.free <= 0 {
+		return 0, false, ErrRangeFull
+	}
+
+	// Scan from the minimum until we find a free item
+	for i := a.min; i <= a.max; i++ {
+		if !a.has(i) {
+			a.used[i] = true
+			a.free--
+			return i, true, nil
+		}
+	}
+
+	// no free item found, but a.free != 0
+	return 0, false, ErrInternal
+}
+
+func (a *MinMaxAllocator) Release(i int) error {
+	a.lock.Lock()
+	defer a.lock.Unlock()
+
+	if !a.has(i) {
+		return nil
+	}
+
+	delete(a.used, i)
+
+	if a.inRange(i) {
+		a.free++
+	}
+
+	return nil
+}
+
+func (a *MinMaxAllocator) has(i int) bool {
+	_, ok := a.used[i]
+	return ok
+}
+
+func (a *MinMaxAllocator) Has(i int) bool {
+	a.lock.Lock()
+	defer a.lock.Unlock()
+
+	return a.has(i)
+}
+
+func (a *MinMaxAllocator) Free() int {
+	a.lock.Lock()
+	defer a.lock.Unlock()
+	return a.free
+}
+
+func (a *MinMaxAllocator) inRange(i int) bool {
+	return a.min <= i && i <= a.max
+}

pkg/volume/glusterfs/glusterfs_minmax_test.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package glusterfs
+
+import (
+	"testing"
+)
+
+func TestNewFree(t *testing.T) {
+	min := 1
+	max := 10
+
+	m, err := NewMinMaxAllocator(min, max)
+	if err != nil {
+		t.Errorf("error creating new allocator: '%v'", err)
+	}
+
+	if f := m.Free(); f != (max - min + 1) {
+		t.Errorf("expect to get %d free, but got %d", (max - min + 1), f)
+	}
+}
+
+func TestNewInvalidRange(t *testing.T) {
+	if _, err := NewMinMaxAllocator(10, 1); err != ErrInvalidRange {
+		t.Errorf("expect to get Error '%v', got '%v'", ErrInvalidRange, err)
+	}
+}
+
+func TestSetRange(t *testing.T) {
+	min := 1
+	max := 10
+
+	m, err := NewMinMaxAllocator(min, max)
+	if err != nil {
+		t.Errorf("error creating new allocator: '%v'", err)
+	}
+
+	if err = m.SetRange(10, 1); err != ErrInvalidRange {
+		t.Errorf("expected to get error '%v', got '%v'", ErrInvalidRange, err)
+	}
+
+	if err = m.SetRange(1, 2); err != nil {
+		t.Errorf("error setting range: '%v'", err)
+	}
+
+	if f := m.Free(); f != 2 {
+		t.Errorf("expect to get %d free, but got %d", 2, f)
+	}
+
+	if ok, _ := m.Allocate(1); !ok {
+		t.Errorf("error allocate offset %v", 1)
+	}
+
+	if f := m.Free(); f != 1 {
+		t.Errorf("expect to get 1 free, but got %d", f)
+	}
+
+	if err = m.SetRange(1, 1); err != nil {
+		t.Errorf("error setting range: '%v'", err)
+	}
+
+	if f := m.Free(); f != 0 {
+		t.Errorf("expect to get 0 free, but got %d", f)
+	}
+
+	if err = m.SetRange(2, 2); err != nil {
+		t.Errorf("error setting range: '%v'", err)
+	}
+
+	if f := m.Free(); f != 1 {
+		t.Errorf("expect to get 1 free, but got %d", f)
+	}
+}
+
+func TestAllocateNext(t *testing.T) {
+	min := 1
+	max := 10
+
+	m, err := NewMinMaxAllocator(min, max)
+	if err != nil {
+		t.Errorf("error creating new allocator: '%v'", err)
+	}
+
+	el, ok, _ := m.AllocateNext()
+	if !ok {
+		t.Fatalf("unexpected error")
+	}
+
+	if !m.Has(el) {
+		t.Errorf("expect element %v allocated", el)
+	}
+
+	if f := m.Free(); f != (max-min+1)-1 {
+		t.Errorf("expect to get %d free, but got %d", (max-min+1)-1, f)
+	}
+}
+
+func TestAllocateMax(t *testing.T) {
+	min := 1
+	max := 10
+
+	m, err := NewMinMaxAllocator(min, max)
+	if err != nil {
+		t.Errorf("error creating new allocator: '%v'", err)
+	}
+
+	for i := 1; i <= max; i++ {
+		if _, ok, _ := m.AllocateNext(); !ok {
+			t.Fatalf("unexpected error")
+		}
+	}
+
+	if _, ok, _ := m.AllocateNext(); ok {
+		t.Errorf("unexpected success")
+	}
+
+	if f := m.Free(); f != 0 {
+		t.Errorf("expect to get %d free, but got %d", 0, f)
+	}
+}
+
+func TestAllocate(t *testing.T) {
+	min := 1
+	max := 10
+	offset := 3
+
+	m, err := NewMinMaxAllocator(min, max)
+	if err != nil {
+		t.Errorf("error creating new allocator: '%v'", err)
+	}
+
+	if ok, err := m.Allocate(offset); !ok {
+		t.Errorf("error allocate offset %v: %v", offset, err)
+	}
+
+	if !m.Has(offset) {
+		t.Errorf("expect element %v allocated", offset)
+	}
+
+	if f := m.Free(); f != (max-min+1)-1 {
+		t.Errorf("expect to get %d free, but got %d", (max-min+1)-1, f)
+	}
+}
+
+func TestAllocateConflict(t *testing.T) {
+	min := 1
+	max := 10
+	offset := 3
+
+	m, err := NewMinMaxAllocator(min, max)
+	if err != nil {
+		t.Errorf("error creating new allocator: '%v'", err)
+	}
+
+	if ok, err := m.Allocate(offset); !ok {
+		t.Errorf("error allocate offset %v: %v", offset, err)
+	}
+
+	ok, err := m.Allocate(offset)
+	if ok {
+		t.Errorf("unexpected success")
+	}
+	if err != ErrConflict {
+		t.Errorf("expected error '%v', got '%v'", ErrConflict, err)
+	}
+}
+
+func TestAllocateOutOfRange(t *testing.T) {
+	min := 1
+	max := 10
+	offset := 11
+
+	m, err := NewMinMaxAllocator(min, max)
+	if err != nil {
+		t.Errorf("error creating new allocator: '%v'", err)
+	}
+
+	ok, err := m.Allocate(offset)
+	if ok {
+		t.Errorf("unexpected success")
+	}
+	if err != ErrOutOfRange {
+		t.Errorf("expected error '%v', got '%v'", ErrOutOfRange, err)
+	}
+}
+
+func TestRelease(t *testing.T) {
+	min := 1
+	max := 10
+	offset := 3
+
+	m, err := NewMinMaxAllocator(min, max)
+	if err != nil {
+		t.Errorf("error creating new allocator: '%v'", err)
+	}
+
+	if ok, err := m.Allocate(offset); !ok {
+		t.Errorf("error allocate offset %v: %v", offset, err)
+	}
+
+	if !m.Has(offset) {
+		t.Errorf("expect offset %v allocated", offset)
+	}
+
+	if err = m.Release(offset); err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+
+	if m.Has(offset) {
+		t.Errorf("expect offset %v not allocated", offset)
+	}
+}

pkg/volume/glusterfs/glusterfs_test.go

@@ -269,6 +269,8 @@ func TestParseClassParameters(t *testing.T) {
 				user:        "admin",
 				userKey:     "password",
 				secretValue: "password",
+				gidMin:      2000,
+				gidMax:      4294967295,
 			},
 		},
 		{
@@ -287,6 +289,8 @@ func TestParseClassParameters(t *testing.T) {
 				secretName:      "mysecret",
 				secretNamespace: "default",
 				secretValue:     "mypassword",
+				gidMin:          2000,
+				gidMax:          4294967295,
 			},
 		},
 		{
@@ -298,7 +302,9 @@ func TestParseClassParameters(t *testing.T) {
 			&secret,
 			false, // expect error
 			&provisioningConfig{
-				url: "https://localhost:8080",
+				url:    "https://localhost:8080",
+				gidMin: 2000,
+				gidMax: 4294967295,
 			},
 		},
 		{