feat: [close #103] Stage sources per module

Instead of staging sources all at once, allow vib to stage sources per module and clean sources after the module by itself, this allows users to manage sources easier and also completely skip the sources step if no module defines any sources.

api/download.go

@@ -13,18 +13,27 @@ import (
 
 // Generate the destination path for the source based on its type and module name
 func GetSourcePath(source Source, moduleName string) string {
+	if len(strings.TrimSpace(source.Path)) > 0 {
+		return filepath.Join(moduleName, source.Path)
+	}
 	switch source.Type {
 	case "git":
-		var dest string
-		if source.Destination != "" {
-			repoName := strings.Split(source.URL, "/")
-			dest = filepath.Join(moduleName, strings.ReplaceAll(repoName[len(repoName)-1], ".git", ""))
+		repoName := strings.Split(source.URL, "/")
+		return filepath.Join(moduleName, strings.ReplaceAll(repoName[len(repoName)-1], ".git", ""))
+	case "tar":
+		url := strings.Split(source.URL, "/")
+		tarFile := strings.Split(url[len(url)-1], "?")[0]
+		tarParts := strings.Split(tarFile, ".")
+		if strings.TrimSpace(tarParts[len(tarParts)-2]) != "tar" {
+			return filepath.Join(moduleName, strings.Join(tarParts[:len(tarParts)-1], "."))
 		} else {
-			dest = filepath.Join(moduleName, source.Destination)
+			return filepath.Join(moduleName, strings.Join(tarParts[:len(tarParts)-2], "."))
 		}
-		return dest
-	case "tar", "file":
-		return filepath.Join(moduleName, source.Destination)
+	case "file":
+		url := strings.Split(source.URL, "/")
+		file := strings.Split(url[len(url)-1], "?")[0]
+		fileParts := strings.Split(file, ".")
+		return filepath.Join(moduleName, strings.Join(fileParts[:len(fileParts)-1], "."))
 	}
 
 	return ""
@@ -168,10 +177,14 @@ func DownloadTarSource(downloadPath string, source Source, moduleName string) er
 
 // Move downloaded sources from the download path to the sources path
 func MoveSources(downloadPath string, sourcesPath string, sources []Source, moduleName string) error {
-	fmt.Println("Moving sources")
+	fmt.Println("Moving sources for " + moduleName)
 
+	err := os.MkdirAll(filepath.Join(sourcesPath, moduleName), 0777)
+	if err != nil {
+		return err
+	}
 	for _, source := range sources {
-		err := MoveSource(downloadPath, sourcesPath, source, moduleName)
+		err = MoveSource(downloadPath, sourcesPath, source, moduleName)
 		if err != nil {
 			return err
 		}
@@ -186,6 +199,11 @@ func MoveSources(downloadPath string, sourcesPath string, sources []Source, modu
 func MoveSource(downloadPath string, sourcesPath string, source Source, moduleName string) error {
 	fmt.Printf("Moving source: %s\n", moduleName)
 
+	err := os.MkdirAll(filepath.Join(sourcesPath, moduleName), 0777)
+	if err != nil {
+		return err
+	}
+
 	switch source.Type {
 	case "git", "file":
 		dest := GetSourcePath(source, moduleName)
@@ -197,7 +215,7 @@ func MoveSource(downloadPath string, sourcesPath string, source Source, moduleNa
 		os.MkdirAll(filepath.Join(sourcesPath, GetSourcePath(source, moduleName)), 0o777)
 		cmd := exec.Command(
 			"tar",
-			"-xf", filepath.Join(downloadPath, GetSourcePath(source, moduleName), moduleName+".tar"),
+			"-xf", filepath.Join(downloadPath, GetSourcePath(source, moduleName), moduleName+".tar*"),
 			"-C", filepath.Join(sourcesPath, GetSourcePath(source, moduleName)),
 		)
 		err := cmd.Run()

api/structs.go

@@ -2,15 +2,14 @@ package api
 
 // Configuration for a source
 type Source struct {
-	URL         string   `json:"url"`
-	Checksum    string   `json:"checksum"`
-	Type        string   `json:"type"`
-	Destination string   `json:"destination"`
-	Commit      string   `json:"commit"`
-	Tag         string   `json:"tag"`
-	Branch      string   `json:"branch"`
-	Packages    []string `json:"packages"`
-	Paths       []string `json:"paths"`
+	URL      string   `json:"url"`
+	Checksum string   `json:"checksum"`
+	Type     string   `json:"type"`
+	Commit   string   `json:"commit"`
+	Tag      string   `json:"tag"`
+	Branch   string   `json:"branch"`
+	Packages []string `json:"packages"`
+	Path     string   `json:"path"`
 }
 
 // Configuration for a recipe

core/build.go

@@ -3,6 +3,7 @@ package core
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/mitchellh/mapstructure"
@@ -235,10 +236,10 @@ func BuildContainerfile(recipe *api.Recipe) error {
 		}
 
 		// SOURCES
-		_, err = containerfile.WriteString("ADD sources /sources\n")
+		/*_, err = containerfile.WriteString("ADD sources /sources\n")
 		if err != nil {
 			return err
-		}
+		}*/
 
 		for _, cmd := range cmds {
 			err = ChangeWorkingDirectory(cmd.Workdir, containerfile)
@@ -250,7 +251,7 @@ func BuildContainerfile(recipe *api.Recipe) error {
 			if err != nil {
 				return err
 			}
-			fmt.Println(strings.Join(cmd.Command, "----"))
+
 			err = RestoreWorkingDirectory(cmd.Workdir, containerfile)
 			if err != nil {
 				return err
@@ -336,7 +337,8 @@ func BuildModule(recipe *api.Recipe, moduleInterface interface{}) ([]string, err
 
 	fmt.Printf("Building module [%s] of type [%s]\n", module.Name, module.Type)
 
-	var commands []string
+	commands := []string{fmt.Sprintf("\n# Begin Module %s - %s", module.Name, module.Type)}
+
 	if len(module.Modules) > 0 {
 		for _, nestedModule := range module.Modules {
 			buildModule, err := BuildModule(recipe, nestedModule)
@@ -366,6 +368,18 @@ func BuildModule(recipe *api.Recipe, moduleInterface interface{}) ([]string, err
 		commands = append(commands, command...)
 	}
 
+	_ = os.MkdirAll(fmt.Sprintf("%s/%s", recipe.SourcesPath, module.Name), 0755)
+
+	dirInfo, err := os.Stat(filepath.Join(recipe.SourcesPath, module.Name))
+	if err != nil {
+		return []string{""}, err
+	}
+	if dirInfo.Size() > 0 {
+		commands = append([]string{fmt.Sprintf("ADD sources/%s /sources/%s", module.Name, module.Name)}, commands...)
+		commands = append(commands, fmt.Sprintf("RUN rm -rf /sources/%s", module.Name))
+	}
+	commands = append(commands, fmt.Sprintf("# End Module %s - %s\n", module.Name, module.Type))
+
 	fmt.Printf("Module [%s] built successfully\n", module.Name)
 	return commands, nil
 }

plugins/apt.go

@@ -5,12 +5,15 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"path/filepath"
 
 	"C"
 
 	"github.com/vanilla-os/vib/api"
 )
+import (
+	"path/filepath"
+	"strings"
+)
 
 // Configuration for an APT module
 type AptModule struct {
@@ -81,13 +84,23 @@ func BuildModule(moduleInterface *C.char, recipeInterface *C.char) *C.char {
 		return C.CString(fmt.Sprintf("apt-get install -y %s %s && apt-get clean", args, packages))
 	}
 
-	if len(module.Source.Paths) > 0 {
+	if len(strings.TrimSpace(module.Source.Path)) > 0 {
 		cmd := ""
-
-		for i, path := range module.Source.Paths {
-			instPath := filepath.Join(recipe.ParentPath, path+".inst")
+		installFiles, err := os.ReadDir(module.Source.Path)
+		if err != nil {
+			return C.CString(fmt.Sprintf("ERROR: %s", err.Error()))
+		}
+		for i, path := range installFiles {
+			fullPath := filepath.Join(module.Source.Path, path.Name())
+			fileInfo, err := os.Stat(fullPath)
+			if err != nil {
+				return C.CString(fmt.Sprintf("ERROR: %s", err.Error()))
+			}
+			if !fileInfo.Mode().IsRegular() {
+				continue
+			}
 			packages := ""
-			file, err := os.Open(instPath)
+			file, err := os.Open(fullPath)
 			if err != nil {
 				return C.CString(fmt.Sprintf("ERROR: %s", err.Error()))
 			}
@@ -104,7 +117,7 @@ func BuildModule(moduleInterface *C.char, recipeInterface *C.char) *C.char {
 
 			cmd += fmt.Sprintf("apt-get install -y %s %s", args, packages)
 
-			if i != len(module.Source.Paths)-1 {
+			if i != len(installFiles)-1 {
 				cmd += "&& "
 			} else {
 				cmd += "&& apt-get clean"

plugins/dpkg-buildpackage.go

@@ -4,7 +4,6 @@ import (
 	"C"
 	"encoding/json"
 	"fmt"
-	"path/filepath"
 
 	"github.com/vanilla-os/vib/api"
 )
@@ -58,12 +57,10 @@ func BuildModule(moduleInterface *C.char, recipeInterface *C.char) *C.char {
 
 	cmd := fmt.Sprintf(
 		"cd /sources/%s && dpkg-buildpackage -d -us -uc -b",
-		filepath.Join(api.GetSourcePath(module.Source, module.Name)),
+		api.GetSourcePath(module.Source, module.Name),
 	)
 
-	for _, path := range module.Source.Paths {
-		cmd += fmt.Sprintf(" && apt install -y --allow-downgrades ../%s*.deb", path)
-	}
+	cmd += fmt.Sprintf(" && apt install -y --allow-downgrades ../%s*.deb", module.Source.Path)
 
 	cmd += " && apt clean"
 	return C.CString(cmd)