Skip to content

Commit

Permalink
chore: cleanup resolver/downloader because no update is needed
Browse files Browse the repository at this point in the history
  • Loading branch information
jakobmoellerdev committed Dec 17, 2024
1 parent bfe7443 commit f78090d
Show file tree
Hide file tree
Showing 12 changed files with 147 additions and 274 deletions.
2 changes: 2 additions & 0 deletions api/ocm/extensions/accessmethods/git/method.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ const (
)

func init() {
// If we remove the default registration, also the docs are gone.
// so we leave the default registration in.
accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](Type, accspeccpi.WithDescription(usage)))
accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](TypeV1Alpha1, accspeccpi.WithFormatSpec(formatV1), accspeccpi.WithConfigHandler(ConfigHandler())))
}
Expand Down
45 changes: 9 additions & 36 deletions api/tech/git/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,13 @@ package git
import (
"errors"
"fmt"
"hash/fnv"
"os"
"path/filepath"
"sync"
"syscall"

"github.com/go-git/go-billy/v5"
"github.com/juju/fslock"
"github.com/mandelsoft/vfs/pkg/memoryfs"
"github.com/mandelsoft/vfs/pkg/osfs"
"github.com/mandelsoft/vfs/pkg/projectionfs"
"github.com/mandelsoft/vfs/pkg/vfs"
)
Expand All @@ -36,19 +34,24 @@ type fs struct {

var _ billy.Filesystem = &fs{}

// file is a wrapper around a vfs.File that implements billy.File.
// it uses a mutex to lock the file, so it can be used concurrently from the same process, but
// not across processes (like a flock).
type file struct {
lock *fslock.Lock
vfs.File
lockMu sync.Mutex
}

var _ billy.File = &file{}

func (f *file) Lock() error {
return f.lock.Lock()
f.lockMu.Lock()
return nil
}

func (f *file) Unlock() error {
return f.lock.Unlock()
f.lockMu.Unlock()
return nil
}

var _ billy.File = &file{}
Expand All @@ -62,39 +65,9 @@ func (f *fs) Create(filename string) (billy.File, error) {
}

// vfsToBillyFileInfo converts a vfs.File to a billy.File
// It also creates a fslock.Lock for the file to ensure that the file is lockable
// If the vfs is an osfs.OsFs, the lock is created in the same directory as the file
// If the vfs is not an osfs.OsFs, a temporary directory is created to store the lock
// because its not trivial to store the lock for jujufs on a virtual filesystem because
// juju vfs only operates on syscalls directly and without interface abstraction its not easy to get the root.
func (f *fs) vfsToBillyFileInfo(vf vfs.File) (billy.File, error) {
var lock *fslock.Lock
if f.FileSystem == osfs.OsFs {
lock = fslock.New(fmt.Sprintf("%s.lock", vf.Name()))
} else {
hash := fnv.New32()
_, _ = hash.Write([]byte(f.FileSystem.Name()))
temp, err := os.MkdirTemp("", fmt.Sprintf("git-vfs-locks-%x", hash.Sum32()))
if err != nil {
return nil, fmt.Errorf("failed to create temp dir to allow mapping vfs to git (billy) filesystem; "+
"this temporary directory is mandatory because a virtual filesystem cannot be used to accurately depict os syslocks: %w", err)
}
_, components := vfs.Components(f.FileSystem, vf.Name())
lockPath := filepath.Join(
temp,
filepath.Join(components[:len(components)-1]...),
fmt.Sprintf("%s.lock", components[len(components)-1]),
)
if err := os.MkdirAll(filepath.Dir(lockPath), 0o755); err != nil {
return nil, fmt.Errorf("failed to create temp dir to allow mapping vfs to git (billy) filesystem; "+
"this temporary directory is mandatory because a virtual filesystem cannot be used to accurately depict os syslocks: %w", err)
}
lock = fslock.New(lockPath)
}

return &file{
File: vf,
lock: lock,
}, nil
}

Expand Down
2 changes: 0 additions & 2 deletions api/tech/git/logging.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,3 @@ package git
import "ocm.software/ocm/api/utils/logging"

var REALM = logging.DefineSubRealm("git repository", "git")

var Log = logging.DynamicLogger(REALM)
126 changes: 9 additions & 117 deletions api/tech/git/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"net/http"
"os"
"sync"

"github.com/go-git/go-billy/v5"
Expand Down Expand Up @@ -63,17 +62,10 @@ type Client interface {
// given AuthMethod.
Repository(ctx context.Context) (*git.Repository, error)

// Refresh will attempt to fetch & pull the latest changes from the remote repository.
// In case there are no changes, it will do a no-op after having realized that no changes are in the remote.
Refresh(ctx context.Context) error

// Update will stage all changes in the repository, commit them with the given message and push them to the remote repository.
Update(ctx context.Context, msg string, push bool) error

// Setup will override the current filesystem with the given filesystem. This will be the filesystem where the repository will be stored.
// There can be only one filesystem per client.
// If the filesystem contains a repository already, it can be consumed by a subsequent call to Repository.
Setup(vfs.FileSystem) error
Setup(context.Context, vfs.FileSystem) error
}

type ClientOptions struct {
Expand All @@ -88,24 +80,15 @@ type ClientOptions struct {
// Commit is the commit hash to checkout after cloning the repository.
// If empty, it will default to the plumbing.HEAD of the Ref.
Commit string
// Author is the author to use for commits. If empty, it will default to the git config of the user running the process.
Author
// AuthMethod is the authentication method to use for the repository.
AuthMethod AuthMethod
}

type Author struct {
Name string
Email string
}

var _ Client = &client{}

func NewClient(opts ClientOptions) (Client, error) {
var pref plumbing.ReferenceName
if opts.Ref == "" {
pref = plumbing.HEAD
} else {
pref := plumbing.HEAD
if opts.Ref != "" {
pref = plumbing.ReferenceName(opts.Ref)
if err := pref.Validate(); err != nil {
return nil, fmt.Errorf("invalid reference %q: %w", opts.Ref, err)
Expand Down Expand Up @@ -156,7 +139,10 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) {
newRepo = true
}
if errors.Is(err, transport.ErrEmptyRemoteRepository) {
return git.Open(strg, billyFS)
repo, err = git.Open(strg, billyFS)
if err != nil {
return nil, fmt.Errorf("failed to open repository based on URL %q after it was determined to be an empty clone: %w", c.opts.URL, err)
}
}

if err != nil {
Expand All @@ -168,10 +154,6 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) {
}
}

if err := c.opts.applyToRepo(repo); err != nil {
return nil, err
}

c.repo = repo

return repo, nil
Expand Down Expand Up @@ -221,100 +203,10 @@ func GetStorage(base billy.Filesystem) (storage.Storer, error) {
), nil
}

func (c *client) TopLevelDirs(ctx context.Context) ([]os.FileInfo, error) {
repo, err := c.Repository(ctx)
if err != nil {
return nil, err
}

fs, err := repo.Worktree()
if err != nil {
return nil, err
}

return fs.Filesystem.ReadDir(".")
}

func (c *client) Refresh(ctx context.Context) error {
repo, err := c.Repository(ctx)
if err != nil {
return err
}

worktree, err := repo.Worktree()
if err != nil {
return err
}

if err := worktree.PullContext(ctx, &git.PullOptions{
Auth: c.opts.AuthMethod,
RemoteName: git.DefaultRemoteName,
}); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) && !errors.Is(err, transport.ErrEmptyRemoteRepository) {
return err
}

return nil
}

func (c *client) Update(ctx context.Context, msg string, push bool) error {
repo, err := c.Repository(ctx)
if err != nil {
return err
}

worktree, err := repo.Worktree()
if err != nil {
return err
}

if err = worktree.AddGlob("*"); err != nil {
return err
}

_, err = worktree.Commit(msg, &git.CommitOptions{})

if errors.Is(err, git.ErrEmptyCommit) {
return nil
}

if err != nil {
return err
}

if !push {
return nil
}

if err := repo.PushContext(ctx, &git.PushOptions{
RemoteName: git.DefaultRemoteName,
}); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
return err
}

return nil
}

func (c *client) Setup(system vfs.FileSystem) error {
func (c *client) Setup(ctx context.Context, system vfs.FileSystem) error {
c.vfs = system
if _, err := c.Repository(context.Background()); err != nil {
if _, err := c.Repository(ctx); err != nil {
return fmt.Errorf("failed to setup repository %q: %w", c.opts.URL, err)
}
return nil
}

func (o ClientOptions) applyToRepo(repo *git.Repository) error {
cfg, err := repo.Config()
if err != nil {
return err
}

if o.Author.Name != "" {
cfg.User.Name = o.Author.Name
}

if o.Author.Email != "" {
cfg.User.Email = o.Author.Email
}

return repo.SetConfig(cfg)
}
112 changes: 112 additions & 0 deletions api/tech/git/resolver_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
package git_test

import (
"embed"
"fmt"
"io"
"os"
"time"

"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/mandelsoft/filepath/pkg/filepath"
. "github.com/mandelsoft/goutils/testutils"
"github.com/mandelsoft/vfs/pkg/cwdfs"
"github.com/mandelsoft/vfs/pkg/osfs"
"github.com/mandelsoft/vfs/pkg/projectionfs"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"ocm.software/ocm/api/datacontext/attrs/tmpcache"
"ocm.software/ocm/api/datacontext/attrs/vfsattr"
"ocm.software/ocm/api/ocm"
self "ocm.software/ocm/api/tech/git"
)

//go:embed testdata/repo
var testData embed.FS

var _ = Describe("standard tests with local file repo", func() {
var (
ctx ocm.Context
expectedBlobContent []byte
)

ctx = ocm.New()

BeforeEach(func() {
tempVFS, err := cwdfs.New(osfs.New(), GinkgoT().TempDir())
Expect(err).ToNot(HaveOccurred())
tmpcache.Set(ctx, &tmpcache.Attribute{Path: ".", Filesystem: tempVFS})
vfsattr.Set(ctx, tempVFS)
})

var repoDir string
var repoURL string
var ref string
var commit string

BeforeEach(func() {
repoDir = GinkgoT().TempDir() + filepath.PathSeparatorString + "repo"

repo := Must(git.PlainInit(repoDir, false))

repoBase := filepath.Join("testdata", "repo")
repoTestData := Must(testData.ReadDir(repoBase))

for _, entry := range repoTestData {
path := filepath.Join(repoBase, entry.Name())
repoPath := filepath.Join(repoDir, entry.Name())

file := Must(testData.Open(path))

fileInRepo := Must(os.OpenFile(
repoPath,
os.O_CREATE|os.O_RDWR|os.O_TRUNC,
0o600,
))

Must(io.Copy(fileInRepo, file))

Expect(fileInRepo.Close()).To(Succeed())
Expect(file.Close()).To(Succeed())
}

wt := Must(repo.Worktree())
Expect(wt.AddGlob("*")).To(Succeed())
commit = Must(wt.Commit("OCM Test Commit", &git.CommitOptions{
Author: &object.Signature{
Name: "OCM Test",
Email: "[email protected]",
When: time.Now(),
},
})).String()

path := filepath.Join("testdata", "repo", "file_in_repo")
repoURL = fmt.Sprintf("file://%s", repoDir)
ref = plumbing.Master.String()

expectedBlobContent = Must(testData.ReadFile(path))
})

It("Resolver client can setup repository", func(ctx SpecContext) {
client := Must(self.NewClient(self.ClientOptions{
URL: repoURL,
Ref: ref,
Commit: commit,
}))

tempVFS, err := projectionfs.New(osfs.New(), GinkgoT().TempDir())
Expect(err).ToNot(HaveOccurred())

Expect(client.Setup(ctx, tempVFS)).To(Succeed())

repo := Must(client.Repository(ctx))
Expect(repo).ToNot(BeNil())

file := Must(tempVFS.Stat("file_in_repo"))
Expect(file.Size()).To(Equal(int64(len(expectedBlobContent))))

})
})
Loading

0 comments on commit f78090d

Please sign in to comment.