Skip to content

Commit

Permalink
Storage API uses partial width rather than logSize (#393)
Browse files Browse the repository at this point in the history
  • Loading branch information
@AlCutter
AlCutter authored Dec 9, 2024
1 parent 7adba0d commit ea9eac7
Show file tree
Hide file tree
Showing 23 changed files with 197 additions and 255 deletions.
8 changes: 4 additions & 4 deletions api/layout/example_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,19 @@ func ExampleNodeCoordsToTileAddress() {
}

func ExampleTilePath() {
tilePath := layout.TilePath(0, 1234067, 315921160)
tilePath := layout.TilePath(0, 1234067, 8)
fmt.Printf("tile path: %s", tilePath)
// Output: tile path: tile/0/x001/x234/067.p/8
}

func ExampleEntriesPath() {
entriesPath := layout.EntriesPath(1234067, 315921160)
entriesPath := layout.EntriesPath(1234067, 8)
fmt.Printf("entries path: %s", entriesPath)
// Output: entries path: tile/entries/x001/x234/067.p/8
}

func ExampleParseTileLevelIndexWidth() {
level, index, width, _ := layout.ParseTileLevelIndexWidth("0", "x001/x234/067.p/8")
func ExampleParseTileLevelIndexPartial() {
level, index, width, _ := layout.ParseTileLevelIndexPartial("0", "x001/x234/067.p/8")
fmt.Printf("level: %d, index: %d, width: %d", level, index, width)
// Output: level: 0, index: 1234067, width: 8
}
34 changes: 18 additions & 16 deletions api/layout/paths.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,27 +37,28 @@ const (
// The logSize is required so that a partial qualifier can be appended to tiles that
// would contain fewer than 256 entries.
func EntriesPathForLogIndex(seq, logSize uint64) string {
return EntriesPath(seq/256, logSize)
return EntriesPath(seq/EntryBundleWidth, PartialTileSize(0, seq, logSize))
}

// NWithSuffix returns a tiles-spec "N" path, with a partial suffix if applicable.
func NWithSuffix(l, n, logSize uint64) string {
// NWithSuffix returns a tiles-spec "N" path, with a partial suffix if p > 0.
func NWithSuffix(l, n uint64, p uint8) string {
suffix := ""
if p := partialTileSize(l, n, logSize); p > 0 {
if p > 0 {
suffix = fmt.Sprintf(".p/%d", p)
}
return fmt.Sprintf("%s%s", fmtN(n), suffix)
}

// EntriesPath returns the local path for the nth entry bundle. p denotes the partial
// tile size, or 0 if the tile is complete.
func EntriesPath(n, logSize uint64) string {
return fmt.Sprintf("tile/entries/%s", NWithSuffix(0, n, logSize))
func EntriesPath(n uint64, p uint8) string {
return fmt.Sprintf("tile/entries/%s", NWithSuffix(0, n, p))
}

// TilePath builds the path to the subtree tile with the given level and index in tile space.
func TilePath(tileLevel, tileIndex, logSize uint64) string {
return fmt.Sprintf("tile/%d/%s", tileLevel, NWithSuffix(tileLevel, tileIndex, logSize))
// If p > 0 the path represents a partial tile.
func TilePath(tileLevel, tileIndex uint64, p uint8) string {
return fmt.Sprintf("tile/%d/%s", tileLevel, NWithSuffix(tileLevel, tileIndex, p))
}

// fmtN returns the "N" part of a Tiles-spec path.
Expand All @@ -83,13 +84,13 @@ func fmtN(N uint64) string {
// Examples:
// "/tile/0/x001/x234/067" means level 0 and index 1234067 of a full tile.
// "/tile/0/x001/x234/067.p/8" means level 0, index 1234067 and width 8 of a partial tile.
func ParseTileLevelIndexWidth(level, index string) (uint64, uint64, uint64, error) {
func ParseTileLevelIndexPartial(level, index string) (uint64, uint64, uint8, error) {
l, err := ParseTileLevel(level)
if err != nil {
return 0, 0, 0, err
}

i, w, err := ParseTileIndexWidth(index)
i, w, err := ParseTileIndexPartial(index)
if err != nil {
return 0, 0, 0, err
}
Expand All @@ -107,17 +108,18 @@ func ParseTileLevel(level string) (uint64, error) {
return l, err
}

// ParseTileIndexWidth takes index in string, validates and returns the index and width in uint64.
func ParseTileIndexWidth(index string) (uint64, uint64, error) {
w := uint64(256)
// ParseTileIndexPartial takes index in string, validates and returns the index and width in uint64.
func ParseTileIndexPartial(index string) (uint64, uint8, error) {
w := uint8(0)
indexPaths := strings.Split(index, "/")

if strings.Contains(index, ".p") {
var err error
w, err = strconv.ParseUint(indexPaths[len(indexPaths)-1], 10, 64)
if err != nil || w < 1 || w > 255 {
return 0, 0, fmt.Errorf("failed to parse tile index")
w64, err := strconv.ParseUint(indexPaths[len(indexPaths)-1], 10, 64)
if err != nil || w64 < 1 || w64 >= TileWidth {
return 0, 0, fmt.Errorf("failed to parse tile width")
}
w = uint8(w64)
indexPaths[len(indexPaths)-2] = strings.TrimSuffix(indexPaths[len(indexPaths)-2], ".p")
indexPaths = indexPaths[:len(indexPaths)-1]
}
Expand Down
109 changes: 25 additions & 84 deletions api/layout/paths_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package layout

import (
"fmt"
"math"
"testing"
)

Expand Down Expand Up @@ -65,39 +64,30 @@ func TestEntriesPathForLogIndex(t *testing.T) {
func TestEntriesPath(t *testing.T) {
for _, test := range []struct {
N uint64
logSize uint64
p uint8
wantPath string
wantErr bool
}{
{
N: 0,
logSize: 289,
wantPath: "tile/entries/000",
},
{
N: 0,
logSize: 8,
p: 8,
wantPath: "tile/entries/000.p/8",
}, {
N: 255,
logSize: 256 * 256,
wantPath: "tile/entries/255",
}, {
N: 255,
logSize: 255*256 - 3,
p: 253,
wantPath: "tile/entries/255.p/253",
}, {
N: 256,
logSize: 257 * 256,
wantPath: "tile/entries/256",
}, {
N: 123456789000,
logSize: math.MaxUint64,
wantPath: "tile/entries/x123/x456/x789/000",
},
} {
desc := fmt.Sprintf("N %d", test.N)
t.Run(desc, func(t *testing.T) {
gotPath := EntriesPath(test.N, test.logSize)
gotPath := EntriesPath(test.N, test.p)
if gotPath != test.wantPath {
t.Errorf("got file %q want %q", gotPath, test.wantPath)
}
Expand All @@ -109,59 +99,37 @@ func TestTilePath(t *testing.T) {
for _, test := range []struct {
level uint64
index uint64
logSize uint64
p uint8
wantPath string
}{
{
level: 0,
index: 0,
logSize: 256,
wantPath: "tile/0/000",
}, {
level: 0,
index: 0,
logSize: 0,
wantPath: "tile/0/000",
}, {
level: 0,
index: 0,
logSize: 255,
p: 255,
wantPath: "tile/0/000.p/255",
}, {
level: 1,
index: 0,
logSize: math.MaxUint64,
wantPath: "tile/1/000",
}, {
level: 1,
index: 0,
logSize: 256,
wantPath: "tile/1/000.p/1",
}, {
level: 1,
index: 0,
logSize: 1024,
wantPath: "tile/1/000.p/4",
}, {
level: 15,
index: 455667,
logSize: math.MaxUint64,
p: 0,
wantPath: "tile/15/x455/667",
}, {
level: 3,
index: 1234567,
logSize: math.MaxUint64,
wantPath: "tile/3/x001/x234/567",
}, {
level: 15,
index: 123456789,
logSize: math.MaxUint64,
wantPath: "tile/15/x123/x456/789",
p: 41,
wantPath: "tile/15/x123/x456/789.p/41",
},
} {
desc := fmt.Sprintf("level %x index %x", test.level, test.index)
t.Run(desc, func(t *testing.T) {
gotPath := TilePath(test.level, test.index, test.logSize)
gotPath := TilePath(test.level, test.index, test.p)
if gotPath != test.wantPath {
t.Errorf("Got path %q want %q", gotPath, test.wantPath)
}
Expand All @@ -173,102 +141,75 @@ func TestNWithSuffix(t *testing.T) {
for _, test := range []struct {
level uint64
index uint64
logSize uint64
p uint8
wantPath string
}{
{
level: 0,
index: 0,
logSize: 256,
wantPath: "000",
}, {
level: 0,
index: 0,
logSize: 0,
wantPath: "000",
}, {
level: 0,
index: 0,
logSize: 255,
p: 255,
wantPath: "000.p/255",
}, {
level: 1,
index: 0,
logSize: math.MaxUint64,
wantPath: "000",
}, {
level: 1,
index: 0,
logSize: 256,
wantPath: "000.p/1",
}, {
level: 1,
index: 0,
logSize: 1024,
wantPath: "000.p/4",
}, {
level: 15,
index: 455667,
logSize: math.MaxUint64,
wantPath: "x455/667",
}, {
level: 3,
index: 1234567,
logSize: math.MaxUint64,
wantPath: "x001/x234/567",
}, {
level: 15,
index: 123456789,
logSize: math.MaxUint64,
wantPath: "x123/x456/789",
p: 65,
wantPath: "x123/x456/789.p/65",
},
} {
desc := fmt.Sprintf("level %x index %x", test.level, test.index)
t.Run(desc, func(t *testing.T) {
gotPath := NWithSuffix(test.level, test.index, test.logSize)
gotPath := NWithSuffix(test.level, test.index, test.p)
if gotPath != test.wantPath {
t.Errorf("Got path %q want %q", gotPath, test.wantPath)
}
})
}
}

func TestParseTileLevelIndexWidth(t *testing.T) {
func TestParseTileLevelIndexPartial(t *testing.T) {
for _, test := range []struct {
pathLevel string
pathIndex string
wantLevel uint64
wantIndex uint64
wantWidth uint64
wantP uint8
wantErr bool
}{
{
pathLevel: "0",
pathIndex: "x001/x234/067",
wantLevel: 0,
wantIndex: 1234067,
wantWidth: 256,
wantP: 0,
},
{
pathLevel: "0",
pathIndex: "x001/x234/067.p/89",
wantLevel: 0,
wantIndex: 1234067,
wantWidth: 89,
wantP: 89,
},
{
pathLevel: "63",
pathIndex: "x999/x999/x999/x999/x999/999.p/255",
wantLevel: 63,
wantIndex: 999999999999999999,
wantWidth: 255,
wantP: 255,
},
{
pathLevel: "0",
pathIndex: "001",
wantLevel: 0,
wantIndex: 1,
wantWidth: 256,
wantP: 0,
},
{
pathLevel: "0",
Expand Down Expand Up @@ -348,15 +289,15 @@ func TestParseTileLevelIndexWidth(t *testing.T) {
} {
desc := fmt.Sprintf("pathLevel: %q, pathIndex: %q", test.pathLevel, test.pathIndex)
t.Run(desc, func(t *testing.T) {
gotLevel, gotIndex, gotWidth, err := ParseTileLevelIndexWidth(test.pathLevel, test.pathIndex)
gotLevel, gotIndex, gotWidth, err := ParseTileLevelIndexPartial(test.pathLevel, test.pathIndex)
if gotLevel != test.wantLevel {
t.Errorf("got level %d want %d", gotLevel, test.wantLevel)
}
if gotIndex != test.wantIndex {
t.Errorf("got index %d want %d", gotIndex, test.wantIndex)
}
if gotWidth != test.wantWidth {
t.Errorf("got width %d want %d", gotWidth, test.wantWidth)
if gotWidth != test.wantP {
t.Errorf("got width %d want %d", gotWidth, test.wantP)
}
gotErr := err != nil
if gotErr != test.wantErr {
Expand Down
27 changes: 19 additions & 8 deletions api/layout/tile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,35 @@

package layout

// partialTileSize returns the expected number of leaves in a tile at the given location within
const (
// TileHeight is the maximum number of levels Merkle tree levels a tile represents.
// This is fixed at 8 by tlog-tile spec.
TileHeight = 8
// TileWidth is the maximum number of hashes which can be present in the bottom row of a tile.
TileWidth = 1 << TileHeight
// EntryBundleWidth is the maximum number of entries which can be present in an EntryBundle.
// This is defined to be the same as the width of the node tiles by tlog-tile spec.
EntryBundleWidth = TileWidth
)

// PartialTileSize returns the expected number of leaves in a tile at the given location within
// a tree of the specified logSize, or 0 if the tile is expected to be fully populated.
func partialTileSize(level, index, logSize uint64) uint64 {
sizeAtLevel := logSize >> (level * 8)
fullTiles := sizeAtLevel / 256
func PartialTileSize(level, index, logSize uint64) uint8 {
sizeAtLevel := logSize >> (level * TileHeight)
fullTiles := sizeAtLevel / TileWidth
if index < fullTiles {
return 0
}
return sizeAtLevel % 256
return uint8(sizeAtLevel % TileWidth)
}

// NodeCoordsToTileAddress returns the (TileLevel, TileIndex) in tile-space, and the
// (NodeLevel, NodeIndex) address within that tile of the specified tree node co-ordinates.
func NodeCoordsToTileAddress(treeLevel, treeIndex uint64) (uint64, uint64, uint, uint64) {
tileRowWidth := uint64(1 << (8 - treeLevel%8))
tileLevel := treeLevel / 8
tileRowWidth := uint64(1 << (TileHeight - treeLevel%TileHeight))
tileLevel := treeLevel / TileHeight
tileIndex := treeIndex / tileRowWidth
nodeLevel := uint(treeLevel % 8)
nodeLevel := uint(treeLevel % TileHeight)
nodeIndex := uint64(treeIndex % tileRowWidth)

return tileLevel, tileIndex, nodeLevel, nodeIndex
Expand Down
Loading

0 comments on commit ea9eac7

Please sign in to comment.