diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml new file mode 100644 index 0000000..cfb9eeb --- /dev/null +++ b/.github/workflows/go.yaml @@ -0,0 +1,28 @@ +name: Go Test +on: [push] +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.19 + uses: actions/setup-go@v1 + with: + go-version: 1.19 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v1 + + - name: Test + run: | + make + go test -v ./... + + - name: StoreBinaries + uses: actions/upload-artifact@v2 + with: + name: Binaries + path: vmdk* diff --git a/bin/stat.go b/bin/stat.go index 5bb1c10..59991ed 100644 --- a/bin/stat.go +++ b/bin/stat.go @@ -30,15 +30,15 @@ func doInfo() { kingpin.FatalIfError(err, "Can not open filesystem") vmdk, err := parser.GetVMDKContext(reader, int(st.Size()), - func(filename string) (reader io.ReaderAt, err error) { + func(filename string) (reader io.ReaderAt, closer func(), err error) { full_path := filepath.Join( filepath.Dir(*info_command_file_arg), filename) fd, err := os.Open(full_path) if err != nil { - return nil, err + return nil, nil, err } - reader, err := ntfs_parser.NewPagedReader( + reader, err = ntfs_parser.NewPagedReader( getReader(fd), 1024, 10000) return reader, func() { fd.Close() }, nil }) diff --git a/go.mod b/go.mod index 4f079f3..909a5ea 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/Velocidex/go-vmdk go 1.22.2 require ( - github.com/Velocidex/go-fat v0.0.0-20230923165230-3e6c4265297a github.com/alecthomas/kingpin/v2 v2.4.0 + github.com/sebdah/goldie v1.0.0 www.velocidex.com/golang/go-ntfs v0.2.0 ) diff --git a/go.sum b/go.sum index 80ff18c..89852ff 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/Velocidex/go-fat v0.0.0-20230923165230-3e6c4265297a h1:dWHPlB3C86vh+M5P14dZxF6Hh8o2/u8FTRF/bs2EM+Q= -github.com/Velocidex/go-fat v0.0.0-20230923165230-3e6c4265297a/go.mod h1:g74FCv59tsVP48V2o1eyIK8aKbNKPLJIJ+HuiUPVc6E= github.com/alecthomas/assert v1.0.0 h1:3XmGh/PSuLzDbK3W2gUbRXwgW5lqPkuqvRgeQ30FI5o= github.com/alecthomas/assert v1.0.0/go.mod h1:va/d2JC+M7F6s+80kl/R3G7FUiW6JzUO+hPhLyJ36ZY= github.com/alecthomas/colour v0.1.0 h1:nOE9rJm6dsZ66RGWYSFrXw461ZIt9A6+nHgL7FRrDUk= @@ -22,6 +20,7 @@ github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdk github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= diff --git a/parser/context.go b/parser/context.go index 3ca49e8..8baf5bf 100644 --- a/parser/context.go +++ b/parser/context.go @@ -23,7 +23,7 @@ type VMDKContext struct { profile *VMDKProfile reader io.ReaderAt - extents []*SparseExtent + extents []Extent total_size int64 } @@ -44,51 +44,101 @@ func (self *VMDKContext) Close() { } } -func (self *VMDKContext) getGrainForOffset(offset int64) ( - reader io.ReaderAt, start, length int64, err error) { +func (self *VMDKContext) getExtentForOffset(offset int64) ( + extent Extent, err error) { - n, _ := slices.BinarySearchFunc(self.extents, - offset, func(item *SparseExtent, offset int64) int { - if offset < item.offset { + n, found := slices.BinarySearchFunc(self.extents, + offset, func(item Extent, offset int64) int { + if offset < item.VirtualOffset() { return 1 - } else if offset == item.offset { + } else if offset == item.VirtualOffset() { return 0 } return -1 }) + if found { + n++ + } if n < 1 || n > len(self.extents) { - return nil, 0, 0, io.EOF + return nil, io.EOF } - extent := self.extents[n-1] - if extent.offset > offset || extent.offset+extent.total_size < offset { - return nil, 0, 0, io.EOF + extent = self.extents[n-1] + if extent.VirtualOffset() > offset || + extent.VirtualOffset()+extent.TotalSize() < offset { + return nil, io.EOF } - start, length, err = extent.getGrainForOffset(offset - extent.offset) - return extent.reader, start, length, err + return extent, nil +} + +func (self *VMDKContext) normalizeExtents() { + var extents []Extent + var offset int64 + + // Insert Null Extents + for _, e := range self.extents { + if e.VirtualOffset() > offset { + extents = append(extents, &NullExtent{ + SparseExtent: SparseExtent{ + offset: offset, + total_size: e.VirtualOffset() - offset, + }, + }) + } + + extents = append(extents, e) + offset += e.TotalSize() + } + + self.extents = extents } func (self *VMDKContext) ReadAt(buf []byte, offset int64) (int, error) { i := int64(0) buf_len := int64(len(buf)) + // First check the offset is valid for the entire file. + if offset > self.total_size || offset < 0 { + return 0, io.EOF + } + + available_length := self.total_size - offset + if int64(len(buf)) > available_length { + buf = buf[:available_length] + } + + // Now add partial reads for each extent for i < buf_len { - reader, start, available_length, err := self.getGrainForOffset(offset) + extent, err := self.getExtentForOffset(offset + i) if err != nil { - return 0, err + // Missing extent - zero pad it + for i := 0; i < len(buf); i++ { + buf[i] = 0 + } + return len(buf), nil } + index_in_extent := offset + i - extent.VirtualOffset() + available_length := extent.TotalSize() - index_in_extent + + // Fill as much of the buffer as possible to_read := buf_len - i if to_read > available_length { to_read = available_length } - n, err := reader.ReadAt(buf[i:i+to_read], start) + + n, err := extent.ReadAt(buf[i:i+to_read], index_in_extent) if err != nil && err != io.EOF { return int(i), err } + // No more data available - we cant make more progress. + if n == 0 { + break + } + i += int64(n) } @@ -161,5 +211,7 @@ func GetVMDKContext( } } + res.normalizeExtents() + return res, nil } diff --git a/parser/context_test.go b/parser/context_test.go new file mode 100644 index 0000000..465ddbf --- /dev/null +++ b/parser/context_test.go @@ -0,0 +1,78 @@ +package parser + +import ( + "fmt" + "strings" + "testing" + + "github.com/sebdah/goldie" +) + +type MockExtent struct { + *SparseExtent + + buf []byte +} + +func (self *MockExtent) ReadAt(buf []byte, offset int64) (int, error) { + for i := 0; i < len(buf); i++ { + buf[i] = self.buf[i+int(offset)] + } + + return len(buf), nil +} + +func makeData(offset, length int) string { + res := "" + for len(res) < length { + res += fmt.Sprintf(" % 4d", offset+len(res)) + } + + return res +} + +func NewMockExtent(offset, total_size int64) Extent { + return &MockExtent{ + SparseExtent: &SparseExtent{ + offset: offset, + total_size: total_size, + }, + buf: []byte(makeData(int(offset), int(total_size))), + } +} + +func TestFindExtent(t *testing.T) { + res := &VMDKContext{ + total_size: 350, + extents: []Extent{ + NewMockExtent(0, 100), + NewMockExtent(100, 100), + // Gap + NewMockExtent(300, 50), + }, + } + + res.normalizeExtents() + var golden []string + + for _, offset := range []int64{0, 5, 95, 210, 290, 340} { + buf := make([]byte, 20) + + extent, err := res.getExtentForOffset(offset) + if err != nil { + golden = append(golden, + fmt.Sprintf("err for %v %v\n", offset, err)) + } else { + golden = append(golden, + fmt.Sprintf("extent for %v %v, err %v\n", + offset, extent.Stats(), err)) + } + + n, err := res.ReadAt(buf, offset) + golden = append(golden, + fmt.Sprintf("Reading %v (%v) : %v (%v)\n", offset, n, + string(buf[:n]), err)) + } + + goldie.Assert(t, "TestFindExtent", []byte(strings.Join(golden, "\n"))) +} diff --git a/parser/fixtures/TestFindExtent.golden b/parser/fixtures/TestFindExtent.golden new file mode 100644 index 0000000..b1c5780 Binary files /dev/null and b/parser/fixtures/TestFindExtent.golden differ diff --git a/parser/null.go b/parser/null.go new file mode 100644 index 0000000..96764bf --- /dev/null +++ b/parser/null.go @@ -0,0 +1,34 @@ +package parser + +import "io" + +type NullExtent struct { + SparseExtent +} + +func (self *NullExtent) ReadAt(buf []byte, offset int64) (int, error) { + if offset < 0 || offset > self.total_size { + return 0, io.EOF + } + + to_read := int64(len(buf)) + available_length := self.total_size - offset + if to_read > available_length { + to_read = available_length + } + + for i := int64(0); i < to_read; i++ { + buf[i] = 0 + } + + return int(to_read), nil +} + +func (self *NullExtent) Stats() ExtentStat { + return ExtentStat{ + Type: "PAD", + VirtualOffset: self.offset, + Size: self.total_size, + Filename: self.filename, + } +} diff --git a/parser/readers.go b/parser/readers.go new file mode 100644 index 0000000..f74de07 --- /dev/null +++ b/parser/readers.go @@ -0,0 +1,13 @@ +package parser + +import "io" + +type Extent interface { + io.ReaderAt + + VirtualOffset() int64 + TotalSize() int64 + Stats() ExtentStat + Close() + Debug() +} diff --git a/parser/sparse.go b/parser/sparse.go index ce6d0d0..dd0e6e0 100644 --- a/parser/sparse.go +++ b/parser/sparse.go @@ -30,13 +30,37 @@ type SparseExtent struct { } func (self *SparseExtent) Close() { - self.closer() + if self.closer != nil { + self.closer() + } } func (self *SparseExtent) Debug() { fmt.Println(self.header.DebugString()) } +func (self *SparseExtent) TotalSize() int64 { + return self.total_size +} + +func (self *SparseExtent) VirtualOffset() int64 { + return self.offset +} + +func (self *SparseExtent) ReadAt(buf []byte, offset int64) (int, error) { + start, available_length, err := self.getGrainForOffset(offset) + if err != nil { + return 0, nil + } + + to_read := int64(len(buf)) + if to_read > available_length { + to_read = available_length + } + + return self.reader.ReadAt(buf[:to_read], start) +} + func (self *SparseExtent) getGrainForOffset(offset int64) ( start, length int64, err error) { diff --git a/parser/stats.go b/parser/stats.go index 2d188c4..81ce28a 100644 --- a/parser/stats.go +++ b/parser/stats.go @@ -12,18 +12,22 @@ type VMDKStats struct { Extents []ExtentStat `json:"Extents"` } +func (self *SparseExtent) Stats() ExtentStat { + return ExtentStat{ + Type: "SPARSE", + VirtualOffset: self.offset, + Size: self.total_size, + Filename: self.filename, + } +} + func (self *VMDKContext) Stats() VMDKStats { res := VMDKStats{ TotalSize: self.total_size, } for _, e := range self.extents { - res.Extents = append(res.Extents, ExtentStat{ - Type: "SPARSE", - VirtualOffset: e.offset, - Size: e.total_size, - Filename: e.filename, - }) + res.Extents = append(res.Extents, e.Stats()) } return res