Adding support mirror group APIs

Signed-off-by: sp98 <[email protected]>

rbd/group.go

@@ -110,7 +110,9 @@ func GroupImageAdd(groupIoctx *rados.IOContext, groupName string,
 		cephIoctx(groupIoctx),
 		cGroupName,
 		cephIoctx(imageIoctx),
-		cImageName)
+		cImageName,
+		C.uint32_t(0),
+	)
 	return getError(ret)
 }
 
@@ -135,7 +137,8 @@ func GroupImageRemove(groupIoctx *rados.IOContext, groupName string,
 		cephIoctx(groupIoctx),
 		cGroupName,
 		cephIoctx(imageIoctx),
-		cImageName)
+		cImageName,
+		C.uint32_t(0))
 	return getError(ret)
 }
 
@@ -160,7 +163,8 @@ func GroupImageRemoveByID(groupIoctx *rados.IOContext, groupName string,
 		cephIoctx(groupIoctx),
 		cGroupName,
 		cephIoctx(imageIoctx),
-		cid)
+		cid,
+		C.uint32_t(0))
 	return getError(ret)
 }
 

rbd/mirror_group.go

@@ -0,0 +1,177 @@
+//go:build !nautilus
+// +build !nautilus
+
+package rbd
+
+// #cgo LDFLAGS: -lrbd
+// #include <stdlib.h>
+// #include <rbd/librbd.h>
+import "C"
+import (
+	"unsafe"
+
+	"github.com/ceph/go-ceph/rados"
+)
+
+// MirrorGroupEnable will enable mirroring for a group using the specified mode.
+//
+// Implements:
+//
+//	int rbd_mirror_group_enable(rados_ioctx_t p, const char *name,
+//	  							rbd_mirror_image_mode_t mirror_image_mode,
+//									uint32_t flags);
+func MirrorGroupEnable(groupIoctx *rados.IOContext, groupName string, mode ImageMirrorMode) error {
+	cGroupName := C.CString(groupName)
+	defer C.free(unsafe.Pointer(cGroupName))
+	ret := C.rbd_mirror_group_enable(
+		cephIoctx(groupIoctx),
+		cGroupName,
+		C.rbd_mirror_image_mode_t(mode),
+		(C.uint32_t)(2),
+	)
+	return getError(ret)
+}
+
+// MirrorGroupDisable will disabling mirroring for a group
+//
+// Implements:
+//
+//	int rbd_mirror_group_disable(rados_ioctx_t p, const char *name,
+//	  							bool force)
+func MirrorGroupDisable(groupIoctx *rados.IOContext, groupName string, force bool) error {
+	cGroupName := C.CString(groupName)
+	defer C.free(unsafe.Pointer(cGroupName))
+	ret := C.rbd_mirror_group_disable(
+		cephIoctx(groupIoctx),
+		cGroupName,
+		C.bool(force))
+	return getError(ret)
+}
+
+// MirrorGroupPromote will promote the mirrored group to primary status
+//
+// Implements:
+//
+//	int rbd_mirror_group_promote(rados_ioctx_t p, const char *name,
+//	  							uint32_t flags, bool force)
+func MirrorGroupPromote(groupIoctx *rados.IOContext, groupName string, force bool) error {
+	cGroupName := C.CString(groupName)
+	defer C.free(unsafe.Pointer(cGroupName))
+	ret := C.rbd_mirror_group_promote(
+		cephIoctx(groupIoctx),
+		cGroupName,
+		(C.uint32_t)(0),
+		C.bool(force))
+	return getError(ret)
+}
+
+// MirrorGroupDemote will demote the mirrored group to primary status
+//
+// Implements:
+//
+//	int rbd_mirror_group_demote(rados_ioctx_t p, const char *name,
+//	  							uint32_t flags)
+func MirrorGroupDemote(groupIoctx *rados.IOContext, groupName string) error {
+	cGroupName := C.CString(groupName)
+	defer C.free(unsafe.Pointer(cGroupName))
+	ret := C.rbd_mirror_group_demote(
+		cephIoctx(groupIoctx),
+		cGroupName,
+		(C.uint32_t)(0))
+	return getError(ret)
+}
+
+// MirrorGroupResync is used to manually resolve split-brain status by triggering
+// resynchronization
+//
+// Implements:
+//
+//	int rbd_mirror_group_resync(rados_ioctx_t p, const char *name)
+func MirrorGroupResync(groupIoctx *rados.IOContext, groupName string) error {
+	cGroupName := C.CString(groupName)
+	defer C.free(unsafe.Pointer(cGroupName))
+	ret := C.rbd_mirror_group_resync(
+		cephIoctx(groupIoctx),
+		cGroupName)
+	return getError(ret)
+}
+
+type MirrorGroupState C.rbd_mirror_group_state_t
+
+// String representation of MirrorGroupState.
+func (mgs MirrorGroupState) String() string {
+	switch mgs {
+	case MirrorGroupEnabled:
+		return "enabled"
+	case MirrorGroupDisabled:
+		return "disabled"
+	case MirrorGroupEnabling:
+		return "enabling"
+	case MirrorGrpupDisabling:
+		return "disabled"
+	default:
+		return "<unknown>"
+	}
+}
+
+const (
+	// MirrorGrpupDisabling is the representation of
+	// RBD_MIRROR_GROUP_DISABLING from librbd.
+	MirrorGrpupDisabling = MirrorGroupState(C.RBD_MIRROR_GROUP_DISABLING)
+	// MirrorGroupEnabling is the representation of
+	// RBD_MIRROR_GROUP_ENABLING from librbd
+	MirrorGroupEnabling = MirrorGroupState(C.RBD_MIRROR_GROUP_ENABLING)
+	// MirrorGroupEnabled is the representation of
+	// RBD_MIRROR_IMAGE_ENABLED from librbd.
+	MirrorGroupEnabled = MirrorGroupState(C.RBD_MIRROR_GROUP_ENABLED)
+	// MirrorGroupDisabled is the representation of
+	// RBD_MIRROR_GROUP_DISABLED from librbd.
+	MirrorGroupDisabled = MirrorGroupState(C.RBD_MIRROR_GROUP_DISABLED)
+)
+
+// MirrorGroupInfo represents the mirroring status information of group.
+type MirrorGroupInfo struct {
+	GlobalID        string
+	State           MirrorGroupState
+	MirrorImageMode ImageMirrorMode
+	Primary         bool
+}
+
+// GetMirrorGroupInfo returns the mirroring status information of the mirrored group
+//
+// Implements:
+//
+//	int rbd_mirror_group_get_info(rados_ioctx_t p, const char *name,
+//								  rbd_mirror_group_info_t *mirror_group_info,
+//								  size_t info_size)
+func GetMirrorGroupInfo(groupIoctx *rados.IOContext, groupName string) (*MirrorGroupInfo, error) {
+	var cgInfo C.rbd_mirror_group_info_t
+	cGroupName := C.CString(groupName)
+	defer C.free(unsafe.Pointer(cGroupName))
+
+	ret := C.rbd_mirror_group_get_info(
+		cephIoctx(groupIoctx),
+		cGroupName,
+		&cgInfo,
+		C.sizeof_rbd_mirror_group_info_t)
+
+	if ret < 0 {
+		return nil, getError(ret)
+	}
+
+	info := convertMirrorGroupInfo(&cgInfo)
+
+	// free C memory allocated by C.rbd_mirror_group_get_info call
+	C.rbd_mirror_group_get_info_cleanup(&cgInfo)
+	return &info, nil
+
+}
+
+func convertMirrorGroupInfo(cgInfo *C.rbd_mirror_group_info_t) MirrorGroupInfo {
+	return MirrorGroupInfo{
+		GlobalID:        C.GoString(cgInfo.global_id),
+		MirrorImageMode: ImageMirrorMode(cgInfo.mirror_image_mode),
+		State:           MirrorGroupState(cgInfo.state),
+		Primary:         bool(cgInfo.primary),
+	}
+}

rbd/mirror_group_test.go

@@ -0,0 +1,101 @@
+package rbd
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGroupMirroring(t *testing.T) {
+	mconfig := mirrorConfig()
+	if mconfig == "" {
+		t.Skip("no mirror config env var set")
+	}
+
+	conn := radosConnect(t)
+	poolName := GetUUID()
+	err := conn.MakePool(poolName)
+	require.NoError(t, err)
+	defer func() {
+		assert.NoError(t, conn.DeletePool(poolName))
+		conn.Shutdown()
+	}()
+
+	ioctx, err := conn.OpenIOContext(poolName)
+	assert.NoError(t, err)
+	defer func() {
+		ioctx.Destroy()
+	}()
+
+	// enable per-image mirroring for this pool
+	err = SetMirrorMode(ioctx, MirrorModeImage)
+	require.NoError(t, err)
+
+	name := GetUUID()
+	options := NewRbdImageOptions()
+	assert.NoError(t,
+		options.SetUint64(ImageOptionOrder, uint64(testImageOrder)))
+	err = CreateImage(ioctx, name, testImageSize, options)
+	require.NoError(t, err)
+
+	groupName := "group1"
+	err = GroupCreate(ioctx, groupName)
+	assert.NoError(t, err)
+
+	err = GroupImageAdd(ioctx, groupName, ioctx, name)
+	assert.NoError(t, err)
+
+	token, err := CreateMirrorPeerBootstrapToken(ioctx)
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(token), 4)
+
+	conn2 := radosConnectConfig(t, mconfig)
+	defer conn2.Shutdown()
+
+	err = conn2.MakePool(poolName)
+	require.NoError(t, err)
+	defer func() {
+		assert.NoError(t, conn2.DeletePool(poolName))
+	}()
+
+	ioctx2, err := conn2.OpenIOContext(poolName)
+	assert.NoError(t, err)
+	defer func() {
+		ioctx2.Destroy()
+	}()
+
+	err = SetMirrorMode(ioctx2, MirrorModeImage)
+	require.NoError(t, err)
+
+	err = ImportMirrorPeerBootstrapToken(
+		ioctx2, MirrorPeerDirectionRxTx, token)
+	assert.NoError(t, err)
+
+	// enable mirroring
+	err = MirrorGroupEnable(ioctx, groupName, ImageMirrorModeSnapshot)
+	assert.NoError(t, err)
+
+	// wait for mirroring to be enabled
+	for i := 0; i < 30; i++ {
+		resp, err := GetMirrorGroupInfo(ioctx, groupName)
+		assert.NoError(t, err)
+		if resp.Primary {
+			break
+		}
+		time.Sleep(2 * time.Second)
+	}
+
+	// resync peer mirror group
+	err = MirrorGroupResync(ioctx2, groupName)
+	assert.NoError(t, err)
+
+	// promote mirror group
+	err = MirrorGroupPromote(ioctx2, groupName, true)
+	assert.NoError(t, err)
+
+	// demote mirror group
+	err = MirrorGroupDemote(ioctx2, groupName)
+	assert.NoError(t, err)
+}