Added tests (#1)

.github/workflows/go.yaml

@@ -0,0 +1,28 @@
+name: Go Test
+on: [push]
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.23
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.23
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v1
+
+    - name: Test
+      run: |
+        make
+        go test -v ./...
+
+    - name: StoreBinaries
+      uses: actions/upload-artifact@v4
+      with:
+        name: Binaries
+        path: vmdk*

bin/stat.go

@@ -30,15 +30,15 @@ func doInfo() {
 	kingpin.FatalIfError(err, "Can not open filesystem")
 
 	vmdk, err := parser.GetVMDKContext(reader, int(st.Size()),
-		func(filename string) (reader io.ReaderAt, err error) {
+		func(filename string) (reader io.ReaderAt, closer func(), err error) {
 			full_path := filepath.Join(
 				filepath.Dir(*info_command_file_arg), filename)
 			fd, err := os.Open(full_path)
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 
-			reader, err := ntfs_parser.NewPagedReader(
+			reader, err = ntfs_parser.NewPagedReader(
 				getReader(fd), 1024, 10000)
 			return reader, func() { fd.Close() }, nil
 		})

go.mod

@@ -1,10 +1,10 @@
 module github.com/Velocidex/go-vmdk
 
-go 1.22.2
+go 1.23
 
 require (
-	github.com/Velocidex/go-fat v0.0.0-20230923165230-3e6c4265297a
 	github.com/alecthomas/kingpin/v2 v2.4.0
+	github.com/sebdah/goldie v1.0.0
 	www.velocidex.com/golang/go-ntfs v0.2.0
 )
 

go.sum

@@ -1,5 +1,3 @@
-github.com/Velocidex/go-fat v0.0.0-20230923165230-3e6c4265297a h1:dWHPlB3C86vh+M5P14dZxF6Hh8o2/u8FTRF/bs2EM+Q=
-github.com/Velocidex/go-fat v0.0.0-20230923165230-3e6c4265297a/go.mod h1:g74FCv59tsVP48V2o1eyIK8aKbNKPLJIJ+HuiUPVc6E=
 github.com/alecthomas/assert v1.0.0 h1:3XmGh/PSuLzDbK3W2gUbRXwgW5lqPkuqvRgeQ30FI5o=
 github.com/alecthomas/assert v1.0.0/go.mod h1:va/d2JC+M7F6s+80kl/R3G7FUiW6JzUO+hPhLyJ36ZY=
 github.com/alecthomas/colour v0.1.0 h1:nOE9rJm6dsZ66RGWYSFrXw461ZIt9A6+nHgL7FRrDUk=
@@ -22,6 +20,7 @@ github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdk
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
 github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=

parser/context.go

@@ -23,7 +23,7 @@ type VMDKContext struct {
 	profile *VMDKProfile
 	reader  io.ReaderAt
 
-	extents []*SparseExtent
+	extents []Extent
 
 	total_size int64
 }
@@ -44,51 +44,101 @@ func (self *VMDKContext) Close() {
 	}
 }
 
-func (self *VMDKContext) getGrainForOffset(offset int64) (
-	reader io.ReaderAt, start, length int64, err error) {
+func (self *VMDKContext) getExtentForOffset(offset int64) (
+	extent Extent, err error) {
 
-	n, _ := slices.BinarySearchFunc(self.extents,
-		offset, func(item *SparseExtent, offset int64) int {
-			if offset < item.offset {
+	n, found := slices.BinarySearchFunc(self.extents,
+		offset, func(item Extent, offset int64) int {
+			if offset < item.VirtualOffset() {
 				return 1
-			} else if offset == item.offset {
+			} else if offset == item.VirtualOffset() {
 				return 0
 			}
 			return -1
 		})
+	if found {
+		n++
+	}
 
 	if n < 1 || n > len(self.extents) {
-		return nil, 0, 0, io.EOF
+		return nil, io.EOF
 	}
 
-	extent := self.extents[n-1]
-	if extent.offset > offset || extent.offset+extent.total_size < offset {
-		return nil, 0, 0, io.EOF
+	extent = self.extents[n-1]
+	if extent.VirtualOffset() > offset ||
+		extent.VirtualOffset()+extent.TotalSize() < offset {
+		return nil, io.EOF
 	}
 
-	start, length, err = extent.getGrainForOffset(offset - extent.offset)
-	return extent.reader, start, length, err
+	return extent, nil
+}
+
+func (self *VMDKContext) normalizeExtents() {
+	var extents []Extent
+	var offset int64
+
+	// Insert Null Extents
+	for _, e := range self.extents {
+		if e.VirtualOffset() > offset {
+			extents = append(extents, &NullExtent{
+				SparseExtent: SparseExtent{
+					offset:     offset,
+					total_size: e.VirtualOffset() - offset,
+				},
+			})
+		}
+
+		extents = append(extents, e)
+		offset += e.TotalSize()
+	}
+
+	self.extents = extents
 }
 
 func (self *VMDKContext) ReadAt(buf []byte, offset int64) (int, error) {
 	i := int64(0)
 	buf_len := int64(len(buf))
 
+	// First check the offset is valid for the entire file.
+	if offset > self.total_size || offset < 0 {
+		return 0, io.EOF
+	}
+
+	available_length := self.total_size - offset
+	if int64(len(buf)) > available_length {
+		buf = buf[:available_length]
+	}
+
+	// Now add partial reads for each extent
 	for i < buf_len {
-		reader, start, available_length, err := self.getGrainForOffset(offset)
+		extent, err := self.getExtentForOffset(offset + i)
 		if err != nil {
-			return 0, err
+			// Missing extent - zero pad it
+			for i := 0; i < len(buf); i++ {
+				buf[i] = 0
+			}
+			return len(buf), nil
 		}
 
+		index_in_extent := offset + i - extent.VirtualOffset()
+		available_length := extent.TotalSize() - index_in_extent
+
+		// Fill as much of the buffer as possible
 		to_read := buf_len - i
 		if to_read > available_length {
 			to_read = available_length
 		}
-		n, err := reader.ReadAt(buf[i:i+to_read], start)
+
+		n, err := extent.ReadAt(buf[i:i+to_read], index_in_extent)
 		if err != nil && err != io.EOF {
 			return int(i), err
 		}
 
+		// No more data available - we cant make more progress.
+		if n == 0 {
+			break
+		}
+
 		i += int64(n)
 	}
 
@@ -161,5 +211,7 @@ func GetVMDKContext(
 		}
 	}
 
+	res.normalizeExtents()
+
 	return res, nil
 }

parser/context_test.go

@@ -0,0 +1,78 @@
+package parser
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/sebdah/goldie"
+)
+
+type MockExtent struct {
+	*SparseExtent
+
+	buf []byte
+}
+
+func (self *MockExtent) ReadAt(buf []byte, offset int64) (int, error) {
+	for i := 0; i < len(buf); i++ {
+		buf[i] = self.buf[i+int(offset)]
+	}
+
+	return len(buf), nil
+}
+
+func makeData(offset, length int) string {
+	res := ""
+	for len(res) < length {
+		res += fmt.Sprintf(" % 4d", offset+len(res))
+	}
+
+	return res
+}
+
+func NewMockExtent(offset, total_size int64) Extent {
+	return &MockExtent{
+		SparseExtent: &SparseExtent{
+			offset:     offset,
+			total_size: total_size,
+		},
+		buf: []byte(makeData(int(offset), int(total_size))),
+	}
+}
+
+func TestFindExtent(t *testing.T) {
+	res := &VMDKContext{
+		total_size: 350,
+		extents: []Extent{
+			NewMockExtent(0, 100),
+			NewMockExtent(100, 100),
+			// Gap
+			NewMockExtent(300, 50),
+		},
+	}
+
+	res.normalizeExtents()
+	var golden []string
+
+	for _, offset := range []int64{0, 5, 95, 210, 290, 340} {
+		buf := make([]byte, 20)
+
+		extent, err := res.getExtentForOffset(offset)
+		if err != nil {
+			golden = append(golden,
+				fmt.Sprintf("err for %v %v\n", offset, err))
+		} else {
+			golden = append(golden,
+				fmt.Sprintf("extent for %v %v, err %v\n",
+					offset, extent.Stats(), err))
+		}
+
+		n, err := res.ReadAt(buf, offset)
+		golden = append(golden,
+			fmt.Sprintf("Reading %v (%v) : %v (%v)\n", offset, n,
+				string(buf[:n]), err))
+	}
+
+	goldie.Assert(t, "TestFindExtent", []byte(strings.Join(golden, "\n")))
+}

parser/null.go

@@ -0,0 +1,34 @@
+package parser
+
+import "io"
+
+type NullExtent struct {
+	SparseExtent
+}
+
+func (self *NullExtent) ReadAt(buf []byte, offset int64) (int, error) {
+	if offset < 0 || offset > self.total_size {
+		return 0, io.EOF
+	}
+
+	to_read := int64(len(buf))
+	available_length := self.total_size - offset
+	if to_read > available_length {
+		to_read = available_length
+	}
+
+	for i := int64(0); i < to_read; i++ {
+		buf[i] = 0
+	}
+
+	return int(to_read), nil
+}
+
+func (self *NullExtent) Stats() ExtentStat {
+	return ExtentStat{
+		Type:          "PAD",
+		VirtualOffset: self.offset,
+		Size:          self.total_size,
+		Filename:      self.filename,
+	}
+}

parser/readers.go

@@ -0,0 +1,13 @@
+package parser
+
+import "io"
+
+type Extent interface {
+	io.ReaderAt
+
+	VirtualOffset() int64
+	TotalSize() int64
+	Stats() ExtentStat
+	Close()
+	Debug()
+}

parser/sparse.go

@@ -30,13 +30,37 @@ type SparseExtent struct {
 }
 
 func (self *SparseExtent) Close() {
-	self.closer()
+	if self.closer != nil {
+		self.closer()
+	}
 }
 
 func (self *SparseExtent) Debug() {
 	fmt.Println(self.header.DebugString())
 }
 
+func (self *SparseExtent) TotalSize() int64 {
+	return self.total_size
+}
+
+func (self *SparseExtent) VirtualOffset() int64 {
+	return self.offset
+}
+
+func (self *SparseExtent) ReadAt(buf []byte, offset int64) (int, error) {
+	start, available_length, err := self.getGrainForOffset(offset)
+	if err != nil {
+		return 0, nil
+	}
+
+	to_read := int64(len(buf))
+	if to_read > available_length {
+		to_read = available_length
+	}
+
+	return self.reader.ReadAt(buf[:to_read], start)
+}
+
 func (self *SparseExtent) getGrainForOffset(offset int64) (
 	start, length int64, err error) {