Tablet backup and restore

.github/actions/test-setup/action.yml

@@ -11,6 +11,9 @@ inputs:
   raft-enabled:
     description: "Specifies if cluster should use consistent_cluster_management option"
     required: false
+  tablets:
+    description: "Specifies if cluster should use tablets replication"
+    required: false
   start-dev-env:
     description: "Should this action run 'make start-dev-env'"
     required: false
@@ -39,5 +42,5 @@ runs:
 
     - name: Start dev env
       if: inputs.start-dev-env == 'true'
-      run: make start-dev-env SCYLLA_VERSION=${{ inputs.scylla-version }} IP_FAMILY=${{ inputs.ip-family }} RAFT_ENABLED=${{ inputs.raft-enabled }}
+      run: make start-dev-env SCYLLA_VERSION=${{ inputs.scylla-version }} IP_FAMILY=${{ inputs.ip-family }} RAFT_ENABLED=${{ inputs.raft-enabled }} TABLETS=${{ inputs.tablets }}
       shell: bash

.github/workflows/integration-tests-nightly-latest-ipv4-tablets.yaml

@@ -0,0 +1,141 @@
+name: Integration tests (Scylla Nightly latest, IPV4, tablets)
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [ opened, synchronize, reopened ]
+
+concurrency:
+  group: int-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  scylla-version: 'scylla-nightly:latest'
+  ip-family: IPV4
+  raft-enabled: true
+  tablets: true
+
+jobs:
+  #   Right now both restore-tables and restore-schema tests take way longer than any other pkg tests.
+  #   For this reason they are divided into two distinct jobs, so that the whole workflow can be executed faster.
+  restore-tables:
+    name: Test restore tables
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+
+      - name: Setup testing dependencies
+        uses: ./.github/actions/test-setup
+        with:
+          scylla-version: ${{ env.scylla-version }}
+          ip-family: ${{ env.ip-family }}
+          raft-enabled: ${{ env.raft-enabled }}
+          tablets: ${{ env.tablets }}
+
+      - name: Run tests
+        run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"'
+
+  restore-schema:
+    name: Test restore schema
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+
+      - name: Setup testing dependencies
+        uses: ./.github/actions/test-setup
+        with:
+          scylla-version: ${{ env.scylla-version }}
+          ip-family: ${{ env.ip-family }}
+          raft-enabled: ${{ env.raft-enabled }}
+          tablets: ${{ env.tablets }}
+
+        # Go does not support negative lookahead in regex expressions, so it has to be done manually.
+        # This regex ensures that all restore tests that didn't match restore-tables job will be run here.
+      - name: Run tests
+        run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"'
+
+  backup:
+    name: Test backup
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+
+      - name: Setup testing dependencies
+        uses: ./.github/actions/test-setup
+        with:
+          scylla-version: ${{ env.scylla-version }}
+          ip-family: ${{ env.ip-family }}
+          raft-enabled: ${{ env.raft-enabled }}
+          tablets: ${{ env.tablets }}
+
+      - name: Run tests
+        run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup
+
+  repair:
+    name: Test repair
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+
+      - name: Setup testing dependencies
+        uses: ./.github/actions/test-setup
+        with:
+          scylla-version: ${{ env.scylla-version }}
+          ip-family: ${{ env.ip-family }}
+          raft-enabled: ${{ env.raft-enabled }}
+          tablets: ${{ env.tablets }}
+
+      - name: Run tests
+        run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair
+
+  small-pkg:
+    name: Test other, smaller packages
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+
+      - name: Set IP_FAMILY var for all tests
+        run: |
+          echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV
+
+      - name: Setup testing dependencies
+        uses: ./.github/actions/test-setup
+        with:
+          scylla-version: ${{ env.scylla-version }}
+          ip-family: ${{ env.ip-family }}
+          raft-enabled: ${{ env.raft-enabled }}
+          tablets: ${{ env.tablets }}
+
+      - name: Run cqlping tests
+        run: make pkg-integration-test PKG=./pkg/ping/cqlping
+
+      - name: Run dynamoping tests
+        run: make pkg-integration-test PKG=./pkg/ping/dynamoping
+
+      - name: Run scyllaclient tests
+        run: make pkg-integration-test PKG=./pkg/scyllaclient
+
+      - name: Run cluster tests
+        run: make pkg-integration-test PKG=./pkg/service/cluster
+
+      - name: Run healthcheck tests
+        run: make pkg-integration-test PKG=./pkg/service/healthcheck
+
+      - name: Run scheduler tests
+        run: make pkg-integration-test PKG=./pkg/service/scheduler
+
+      - name: Run store tests
+        run: make pkg-integration-test PKG=./pkg/store
+
+      - name: Run migrate tests
+        run: make pkg-integration-test PKG=./pkg/schema/migrate
+
+      - name: Run netwait tests
+        run: make pkg-integration-test PKG=./pkg/util/netwait

Makefile

@@ -19,6 +19,7 @@ GOFILES = go list -f '{{range .GoFiles}}{{ $$.Dir }}/{{ . }} {{end}}{{range .Tes
 SCYLLA_VERSION?=scylla:5.4.1
 IP_FAMILY?=IPV4
 RAFT_ENABLED?=true
+TABLETS?=false
 SKIP_GOSSIP?=false
 
 MANAGER_CONFIG := testing/scylla-manager/scylla-manager.yaml

pkg/scyllaclient/client_scylla.go

@@ -20,14 +20,13 @@ import (
 	"github.com/pkg/errors"
 	"github.com/scylladb/go-set/strset"
 	"github.com/scylladb/scylla-manager/v3/pkg/dht"
-	"github.com/scylladb/scylla-manager/v3/pkg/util/slice"
-	"go.uber.org/multierr"
-
 	"github.com/scylladb/scylla-manager/v3/pkg/util/parallel"
 	"github.com/scylladb/scylla-manager/v3/pkg/util/pointer"
 	"github.com/scylladb/scylla-manager/v3/pkg/util/prom"
+	"github.com/scylladb/scylla-manager/v3/pkg/util/slice"
 	"github.com/scylladb/scylla-manager/v3/swagger/gen/scylla/v1/client/operations"
 	"github.com/scylladb/scylla-manager/v3/swagger/gen/scylla/v1/models"
+	"go.uber.org/multierr"
 )
 
 // ErrHostInvalidResponse is to indicate that one of the root-causes is the invalid response from scylla-server.
@@ -246,15 +245,33 @@ func (c *Client) hosts(ctx context.Context) ([]string, error) {
 	return v, nil
 }
 
-// Keyspaces return a list of all the keyspaces.
-func (c *Client) Keyspaces(ctx context.Context) ([]string, error) {
-	resp, err := c.scyllaOps.StorageServiceKeyspacesGet(&operations.StorageServiceKeyspacesGetParams{Context: ctx})
+// KeyspaceReplication describes keyspace replication type.
+type KeyspaceReplication = string
+
+// KeyspaceReplication enum.
+const (
+	ReplicationAll    = "all"
+	ReplicationVnode  = "vnodes"
+	ReplicationTablet = "tablets"
+)
+
+// ReplicationKeyspaces return a list of keyspaces with given replication.
+func (c *Client) ReplicationKeyspaces(ctx context.Context, replication KeyspaceReplication) ([]string, error) {
+	resp, err := c.scyllaOps.StorageServiceKeyspacesGet(&operations.StorageServiceKeyspacesGetParams{
+		Context:     ctx,
+		Replication: &replication,
+	})
 	if err != nil {
 		return nil, err
 	}
 	return resp.Payload, nil
 }
 
+// Keyspaces return a list of all the keyspaces.
+func (c *Client) Keyspaces(ctx context.Context) ([]string, error) {
+	return c.ReplicationKeyspaces(ctx, ReplicationAll)
+}
+
 // Tables returns a slice of table names in a given keyspace.
 func (c *Client) Tables(ctx context.Context, keyspace string) ([]string, error) {
 	resp, err := c.scyllaOps.ColumnFamilyNameGet(&operations.ColumnFamilyNameGetParams{Context: ctx})
@@ -374,12 +391,25 @@ func (c *Client) metrics(ctx context.Context, host, name string) (map[string]*pr
 	return prom.ParseText(resp.Body)
 }
 
-// DescribeRing returns a description of token range of a given keyspace.
-func (c *Client) DescribeRing(ctx context.Context, keyspace string) (Ring, error) {
-	resp, err := c.scyllaOps.StorageServiceDescribeRingByKeyspaceGet(&operations.StorageServiceDescribeRingByKeyspaceGetParams{
+// DescribeTabletRing returns a description of token range of a given tablet table.
+func (c *Client) DescribeTabletRing(ctx context.Context, keyspace, table string) (Ring, error) {
+	return c.describeRing(&operations.StorageServiceDescribeRingByKeyspaceGetParams{
 		Context:  ctx,
 		Keyspace: keyspace,
+		Table:    &table,
 	})
+}
+
+// DescribeVnodeRing returns a description of token range of a given vnode keyspace.
+func (c *Client) DescribeVnodeRing(ctx context.Context, keyspace string) (Ring, error) {
+	return c.describeRing(&operations.StorageServiceDescribeRingByKeyspaceGetParams{
+		Context:  ctx,
+		Keyspace: keyspace,
+	})
+}
+
+func (c *Client) describeRing(params *operations.StorageServiceDescribeRingByKeyspaceGetParams) (Ring, error) {
+	resp, err := c.scyllaOps.StorageServiceDescribeRingByKeyspaceGet(params)
 	if err != nil {
 		return Ring{}, err
 	}

pkg/scyllaclient/client_scylla_test.go

@@ -180,7 +180,7 @@ func TestClientDescribeRing(t *testing.T) {
 	client, closeServer := scyllaclienttest.NewFakeScyllaServer(t, "testdata/scylla_api/describe_ring_scylla_manager.json")
 	defer closeServer()
 
-	ring, err := client.DescribeRing(context.Background(), "scylla_manager")
+	ring, err := client.DescribeVnodeRing(context.Background(), "scylla_manager")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -253,7 +253,7 @@ func TestClientDescribeRingReplicationStrategy(t *testing.T) {
 			client, closeServer := scyllaclienttest.NewFakeScyllaServer(t, test.File)
 			defer closeServer()
 
-			ring, err := client.DescribeRing(context.Background(), "scylla_manager")
+			ring, err := client.DescribeVnodeRing(context.Background(), "scylla_manager")
 			if err != nil {
 				t.Fatal(err)
 			}

pkg/scyllaclient/ring_describer.go

@@ -0,0 +1,94 @@
+// Copyright (C) 2024 ScyllaDB
+
+package scyllaclient
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	"github.com/scylladb/go-set/strset"
+)
+
+// RingDescriber describes token rings on table basis for bot vnode and tablet tables.
+type RingDescriber interface {
+	DescribeRing(ctx context.Context, keyspace, table string) (Ring, error)
+}
+
+type ringDescriber struct {
+	client   *Client
+	tabletKs *strset.Set
+	cache    ringCache
+}
+
+type ringCache struct {
+	Keyspace string
+	Table    string
+	Ring     Ring
+}
+
+func NewRingDescriber(ctx context.Context, client *Client) RingDescriber {
+	return &ringDescriber{
+		client:   client,
+		tabletKs: getTabletKs(ctx, client),
+	}
+}
+
+func (rd *ringDescriber) DescribeRing(ctx context.Context, keyspace, table string) (Ring, error) {
+	if ring, ok := rd.tryGetRing(keyspace, table); ok {
+		return ring, nil
+	}
+
+	var (
+		ring Ring
+		err  error
+	)
+	if rd.tabletKs.Has(keyspace) {
+		ring, err = rd.client.DescribeTabletRing(ctx, keyspace, table)
+	} else {
+		ring, err = rd.client.DescribeVnodeRing(ctx, keyspace)
+	}
+	if err != nil {
+		return Ring{}, errors.Wrap(err, "describe ring")
+	}
+
+	rd.setRing(keyspace, table, ring)
+	return ring, nil
+}
+
+func (rd *ringDescriber) tryGetRing(keyspace, table string) (Ring, bool) {
+	if rd.cache.Keyspace == keyspace {
+		if rd.cache.Table == table || !rd.tabletKs.Has(keyspace) {
+			return rd.cache.Ring, true
+		}
+	}
+	return Ring{}, false
+}
+
+func (rd *ringDescriber) setRing(keyspace, table string, ring Ring) {
+	rd.cache.Keyspace = keyspace
+	rd.cache.Table = table
+	rd.cache.Ring = ring
+}
+
+// getTabletKs returns set of tablet replicated keyspaces.
+func getTabletKs(ctx context.Context, client *Client) *strset.Set {
+	out := strset.New()
+	// Assume that errors indicate that endpoints rejected 'replication' param,
+	// which means that given Scylla version does not support tablet API.
+	// Other errors will be handled on other API calls.
+	tablets, err := client.ReplicationKeyspaces(ctx, ReplicationTablet)
+	if err != nil {
+		return out
+	}
+	vnodes, err := client.ReplicationKeyspaces(ctx, ReplicationVnode)
+	if err != nil {
+		return out
+	}
+	// Even when both API calls succeeded, we need to validate
+	// that the 'replication' param wasn't silently ignored.
+	out.Add(tablets...)
+	if out.HasAny(vnodes...) {
+		return strset.New()
+	}
+	return out
+}

pkg/service/backup/export_test.go

@@ -35,13 +35,16 @@ func (s *Service) InitTarget(ctx context.Context, clusterID uuid.UUID, target *T
 	}
 
 	// Collect ring information
+	ringDescriber := scyllaclient.NewRingDescriber(ctx, client)
 	rings := make(map[string]scyllaclient.Ring, len(target.Units))
 	for _, u := range target.Units {
-		ring, err := client.DescribeRing(ctx, u.Keyspace)
-		if err != nil {
-			return errors.Wrap(err, "initialize: describe keyspace ring")
+		for _, tab := range u.Tables {
+			ring, err := ringDescriber.DescribeRing(ctx, u.Keyspace, tab)
+			if err != nil {
+				return errors.Wrap(err, "initialize: describe keyspace ring")
+			}
+			rings[u.Keyspace+"."+tab] = ring
 		}
-		rings[u.Keyspace] = ring
 	}
 
 	// Get live nodes