diff --git a/arduino/builder/builder.go b/arduino/builder/builder.go index 7408ece5c3f..70ea2ec4a07 100644 --- a/arduino/builder/builder.go +++ b/arduino/builder/builder.go @@ -15,16 +15,34 @@ package builder -import "github.com/arduino/arduino-cli/arduino/sketch" +import ( + "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/go-paths-helper" +) + +// nolint +const ( + BuildPropertiesArchiveFile = "archive_file" + BuildPropertiesArchiveFilePath = "archive_file_path" + BuildPropertiesObjectFile = "object_file" + RecipeARPattern = "recipe.ar.pattern" + BuildPropertiesIncludes = "includes" + BuildPropertiesCompilerWarningFlags = "compiler.warning_flags" + Space = " " +) // Builder is a Sketch builder. type Builder struct { sketch *sketch.Sketch + + // core related + coreBuildCachePath *paths.Path } // NewBuilder creates a sketch Builder. -func NewBuilder(sk *sketch.Sketch) *Builder { +func NewBuilder(sk *sketch.Sketch, coreBuildCachePath *paths.Path) *Builder { return &Builder{ - sketch: sk, + sketch: sk, + coreBuildCachePath: coreBuildCachePath, } } diff --git a/arduino/builder/core.go b/arduino/builder/core.go new file mode 100644 index 00000000000..c4d096f6f27 --- /dev/null +++ b/arduino/builder/core.go @@ -0,0 +1,8 @@ +package builder + +import "github.com/arduino/go-paths-helper" + +// CoreBuildCachePath fixdoc +func (b *Builder) CoreBuildCachePath() *paths.Path { + return b.coreBuildCachePath +} diff --git a/arduino/builder/progress/progress.go b/arduino/builder/progress/progress.go new file mode 100644 index 00000000000..53a64e803bf --- /dev/null +++ b/arduino/builder/progress/progress.go @@ -0,0 +1,33 @@ +package progress + +// Struct fixdoc +type Struct struct { + Progress float32 + StepAmount float32 + Parent *Struct +} + +// AddSubSteps fixdoc +func (p *Struct) AddSubSteps(steps int) { + p.Parent = &Struct{ + Progress: p.Progress, + StepAmount: p.StepAmount, + Parent: p.Parent, + } + if p.StepAmount == 0.0 { + p.StepAmount = 100.0 + } + p.StepAmount /= float32(steps) +} + +// RemoveSubSteps fixdoc +func (p *Struct) RemoveSubSteps() { + p.Progress = p.Parent.Progress + p.StepAmount = p.Parent.StepAmount + p.Parent = p.Parent.Parent +} + +// CompleteStep fixdoc +func (p *Struct) CompleteStep() { + p.Progress += p.StepAmount +} diff --git a/legacy/builder/types/progress_test.go b/arduino/builder/progress/progress_test.go similarity index 97% rename from legacy/builder/types/progress_test.go rename to arduino/builder/progress/progress_test.go index 2d367acf898..fff54ca3415 100644 --- a/legacy/builder/types/progress_test.go +++ b/arduino/builder/progress/progress_test.go @@ -13,7 +13,7 @@ // Arduino software without disclosing the source code of your own applications. // To purchase a commercial license, send an email to license@arduino.cc. -package types +package progress import ( "fmt" @@ -23,7 +23,7 @@ import ( ) func TestProgress(t *testing.T) { - p := &ProgressStruct{} + p := &Struct{} p.AddSubSteps(3) require.Equal(t, float32(0.0), p.Progress) require.InEpsilon(t, 33.33333, p.StepAmount, 0.00001) diff --git a/arduino/builder/sizer.go b/arduino/builder/sizer.go new file mode 100644 index 00000000000..911f261af67 --- /dev/null +++ b/arduino/builder/sizer.go @@ -0,0 +1,26 @@ +package builder + +import rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" + +// ExecutableSectionSize represents a section of the executable output file +type ExecutableSectionSize struct { + Name string `json:"name"` + Size int `json:"size"` + MaxSize int `json:"max_size"` +} + +// ExecutablesFileSections is an array of ExecutablesFileSection +type ExecutablesFileSections []ExecutableSectionSize + +// ToRPCExecutableSectionSizeArray transforms this array into a []*rpc.ExecutableSectionSize +func (s ExecutablesFileSections) ToRPCExecutableSectionSizeArray() []*rpc.ExecutableSectionSize { + res := []*rpc.ExecutableSectionSize{} + for _, section := range s { + res = append(res, &rpc.ExecutableSectionSize{ + Name: section.Name, + Size: int64(section.Size), + MaxSize: int64(section.MaxSize), + }) + } + return res +} diff --git a/arduino/builder/sketch_test.go b/arduino/builder/sketch_test.go index b1fda4e6e8c..d8c1df2c4c0 100644 --- a/arduino/builder/sketch_test.go +++ b/arduino/builder/sketch_test.go @@ -48,7 +48,7 @@ func TestMergeSketchSources(t *testing.T) { } mergedSources := strings.ReplaceAll(string(mergedBytes), "%s", pathToGoldenSource) - b := NewBuilder(sk) + b := NewBuilder(sk, nil) offset, source, err := b.sketchMergeSources(nil) require.Nil(t, err) require.Equal(t, 2, offset) @@ -61,7 +61,7 @@ func TestMergeSketchSourcesArduinoIncluded(t *testing.T) { require.NotNil(t, sk) // ensure not to include Arduino.h when it's already there - b := NewBuilder(sk) + b := NewBuilder(sk, nil) _, source, err := b.sketchMergeSources(nil) require.Nil(t, err) require.Equal(t, 1, strings.Count(source, "")) @@ -76,7 +76,7 @@ func TestCopyAdditionalFiles(t *testing.T) { sk1, err := sketch.New(paths.New("testdata", t.Name())) require.Nil(t, err) require.Equal(t, sk1.AdditionalFiles.Len(), 1) - b1 := NewBuilder(sk1) + b1 := NewBuilder(sk1, nil) // copy the sketch over, create a fake main file we don't care about it // but we need it for `SketchLoad` to succeed later diff --git a/arduino/builder/utils/utils.go b/arduino/builder/utils/utils.go index ae64dacb47c..f5aad02c810 100644 --- a/arduino/builder/utils/utils.go +++ b/arduino/builder/utils/utils.go @@ -1,12 +1,25 @@ package utils import ( + "bytes" + "fmt" + "io" "os" + "path/filepath" + "runtime" "strings" + "sync" "unicode" + "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/progress" + "github.com/arduino/arduino-cli/arduino/globals" + "github.com/arduino/arduino-cli/executils" + "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" + "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/text/runes" @@ -14,6 +27,8 @@ import ( "golang.org/x/text/unicode/norm" ) +var tr = i18n.Tr + // ObjFileIsUpToDate fixdoc func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) @@ -137,13 +152,13 @@ func NormalizeUTF8(buf []byte) []byte { var sourceControlFolders = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} -// FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files -func FilterOutSCCS(file *paths.Path) bool { +// filterOutSCCS is a ReadDirFilter that excludes known VSC or project files +func filterOutSCCS(file *paths.Path) bool { return !sourceControlFolders[file.Base()] } -// FilterReadableFiles is a ReadDirFilter that accepts only readable files -func FilterReadableFiles(file *paths.Path) bool { +// filterReadableFiles is a ReadDirFilter that accepts only readable files +func filterReadableFiles(file *paths.Path) bool { // See if the file is readable by opening it f, err := file.Open() if err != nil { @@ -160,9 +175,9 @@ var filterOutHiddenFiles = paths.FilterOutPrefixes(".") func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { fileFilter := paths.AndFilter( filterOutHiddenFiles, - FilterOutSCCS, + filterOutSCCS, paths.FilterOutDirectories(), - FilterReadableFiles, + filterReadableFiles, ) if len(extensions) > 0 { fileFilter = paths.AndFilter( @@ -173,9 +188,450 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (pat if recurse { dirFilter := paths.AndFilter( filterOutHiddenFiles, - FilterOutSCCS, + filterOutSCCS, ) return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter) } return dir.ReadDir(fileFilter) } + +// nolint +const ( + Ignore = 0 // Redirect to null + Show = 1 // Show on stdout/stderr as normal + ShowIfVerbose = 2 // Show if verbose is set, Ignore otherwise + Capture = 3 // Capture into buffer +) + +func printableArgument(arg string) string { + if strings.ContainsAny(arg, "\"\\ \t") { + arg = strings.ReplaceAll(arg, "\\", "\\\\") + arg = strings.ReplaceAll(arg, "\"", "\\\"") + return "\"" + arg + "\"" + } + return arg +} + +// Convert a command and argument slice back to a printable string. +// This adds basic escaping which is sufficient for debug output, but +// probably not for shell interpretation. This essentially reverses +// ParseCommandLine. +func printableCommand(parts []string) string { + return strings.Join(f.Map(parts, printableArgument), " ") +} + +// ExecCommand fixdoc +func ExecCommand( + verbose bool, + stdoutWriter, stderrWriter io.Writer, + command *executils.Process, stdout int, stderr int, +) ([]byte, []byte, []byte, error) { + verboseInfoBuf := &bytes.Buffer{} + if verbose { + verboseInfoBuf.WriteString(printableCommand(command.GetArgs())) + } + + stdoutBuffer := &bytes.Buffer{} + if stdout == Capture { + command.RedirectStdoutTo(stdoutBuffer) + } else if stdout == Show || (stdout == ShowIfVerbose && verbose) { + if stdoutWriter != nil { + command.RedirectStdoutTo(stdoutWriter) + } else { + command.RedirectStdoutTo(os.Stdout) + } + } + + stderrBuffer := &bytes.Buffer{} + if stderr == Capture { + command.RedirectStderrTo(stderrBuffer) + } else if stderr == Show || (stderr == ShowIfVerbose && verbose) { + if stderrWriter != nil { + command.RedirectStderrTo(stderrWriter) + } else { + command.RedirectStderrTo(os.Stderr) + } + } + + err := command.Start() + if err != nil { + return verboseInfoBuf.Bytes(), nil, nil, errors.WithStack(err) + } + + err = command.Wait() + return verboseInfoBuf.Bytes(), stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err) +} + +// DirContentIsOlderThan DirContentIsOlderThan returns true if the content of the given directory is +// older than target file. If extensions are given, only the files with these +// extensions are tested. +func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...string) (bool, error) { + targetStat, err := target.Stat() + if err != nil { + return false, err + } + targetModTime := targetStat.ModTime() + + files, err := FindFilesInFolder(dir, true, extensions...) + if err != nil { + return false, err + } + for _, file := range files { + file, err := file.Stat() + if err != nil { + return false, err + } + if file.ModTime().After(targetModTime) { + return false, nil + } + } + return true, nil +} + +// PrepareCommandForRecipe fixdoc +func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) { + pattern := buildProperties.Get(recipe) + if pattern == "" { + return nil, errors.Errorf(tr("%[1]s pattern is missing"), recipe) + } + + commandLine := buildProperties.ExpandPropsInString(pattern) + if removeUnsetProperties { + commandLine = properties.DeleteUnexpandedPropsFromString(commandLine) + } + + parts, err := properties.SplitQuotedString(commandLine, `"'`, false) + if err != nil { + return nil, errors.WithStack(err) + } + + // if the overall commandline is too long for the platform + // try reducing the length by making the filenames relative + // and changing working directory to build.path + var relativePath string + if len(commandLine) > 30000 { + relativePath = buildProperties.Get("build.path") + for i, arg := range parts { + if _, err := os.Stat(arg); os.IsNotExist(err) { + continue + } + rel, err := filepath.Rel(relativePath, arg) + if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) { + parts[i] = rel + } + } + } + + command, err := executils.NewProcess(nil, parts...) + if err != nil { + return nil, errors.WithStack(err) + } + if relativePath != "" { + command.SetDir(relativePath) + } + + return command, nil +} + +// CompileFiles fixdoc +func CompileFiles( + sourceDir, buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + verbose bool, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + return compileFiles( + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + sourceDir, + false, + buildPath, buildProperties, includes, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) +} + +// CompileFilesRecursive fixdoc +func CompileFilesRecursive( + sourceDir, buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + verbose bool, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + return compileFiles( + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + sourceDir, + true, + buildPath, buildProperties, includes, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) +} + +func compileFiles( + onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + sourceDir *paths.Path, + recurse bool, + buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + verbose bool, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, + progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + validExtensions := []string{} + for ext := range globals.SourceFilesValidExtensions { + validExtensions = append(validExtensions, ext) + } + + sources, err := FindFilesInFolder(sourceDir, recurse, validExtensions...) + if err != nil { + return nil, err + } + + progress.AddSubSteps(len(sources)) + defer progress.RemoveSubSteps() + + objectFiles := paths.NewPathList() + var objectFilesMux sync.Mutex + if len(sources) == 0 { + return objectFiles, nil + } + var errorsList []error + var errorsMux sync.Mutex + + queue := make(chan *paths.Path) + job := func(source *paths.Path) { + recipe := fmt.Sprintf("recipe%s.o.pattern", source.Ext()) + if !buildProperties.ContainsKey(recipe) { + recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()]) + } + objectFile, verboseInfo, verboseStdout, stderr, err := compileFileWithRecipe( + stdoutWriter, stderrWriter, + warningsLevel, + compilationDatabase, + verbose, + onlyUpdateCompilationDatabase, + sourceDir, source, buildPath, buildProperties, includes, recipe, + ) + if verbose { + verboseStdoutFn(verboseStdout) + verboseInfoFn(string(verboseInfo)) + } + verboseStderrFn(stderr) + if err != nil { + errorsMux.Lock() + errorsList = append(errorsList, err) + errorsMux.Unlock() + } else { + objectFilesMux.Lock() + objectFiles.Add(objectFile) + objectFilesMux.Unlock() + } + } + + // Spawn jobs runners + var wg sync.WaitGroup + if jobs == 0 { + jobs = runtime.NumCPU() + } + for i := 0; i < jobs; i++ { + wg.Add(1) + go func() { + for source := range queue { + job(source) + } + wg.Done() + }() + } + + // Feed jobs until error or done + for _, source := range sources { + errorsMux.Lock() + gotError := len(errorsList) > 0 + errorsMux.Unlock() + if gotError { + break + } + queue <- source + + progress.CompleteStep() + // PushProgress + if progressCB != nil { + progressCB(&rpc.TaskProgress{ + Percent: progress.Progress, + Completed: progress.Progress >= 100.0, + }) + } + } + close(queue) + wg.Wait() + if len(errorsList) > 0 { + // output the first error + return nil, errors.WithStack(errorsList[0]) + } + objectFiles.Sort() + return objectFiles, nil +} + +func compileFileWithRecipe( + stdoutWriter, stderrWriter io.Writer, + warningsLevel string, + compilationDatabase *builder.CompilationDatabase, + verbose, onlyUpdateCompilationDatabase bool, + sourcePath *paths.Path, + source *paths.Path, + buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + recipe string, +) (*paths.Path, []byte, []byte, []byte, error) { + verboseStdout, verboseInfo, errOut := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{} + + properties := buildProperties.Clone() + properties.Set(builder.BuildPropertiesCompilerWarningFlags, properties.Get(builder.BuildPropertiesCompilerWarningFlags+"."+warningsLevel)) + properties.Set(builder.BuildPropertiesIncludes, strings.Join(includes, builder.Space)) + properties.SetPath("source_file", source) + relativeSource, err := sourcePath.RelTo(source) + if err != nil { + return nil, nil, nil, nil, errors.WithStack(err) + } + depsFile := buildPath.Join(relativeSource.String() + ".d") + objectFile := buildPath.Join(relativeSource.String() + ".o") + + properties.SetPath(builder.BuildPropertiesObjectFile, objectFile) + err = objectFile.Parent().MkdirAll() + if err != nil { + return nil, nil, nil, nil, errors.WithStack(err) + } + + objIsUpToDate, err := ObjFileIsUpToDate(source, objectFile, depsFile) + if err != nil { + return nil, nil, nil, nil, errors.WithStack(err) + } + + command, err := PrepareCommandForRecipe(properties, recipe, false) + if err != nil { + return nil, nil, nil, nil, errors.WithStack(err) + } + if compilationDatabase != nil { + compilationDatabase.Add(source, command) + } + if !objIsUpToDate && !onlyUpdateCompilationDatabase { + // Since this compile could be multithreaded, we first capture the command output + info, stdout, stderr, err := ExecCommand(verbose, stdoutWriter, stderrWriter, command, Capture, Capture) + // and transfer all at once at the end... + if verbose { + verboseInfo.Write(info) + verboseStdout.Write(stdout) + } + errOut.Write(stderr) + + // ...and then return the error + if err != nil { + return nil, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), errors.WithStack(err) + } + } else if verbose { + if objIsUpToDate { + verboseInfo.WriteString(tr("Using previously compiled file: %[1]s", objectFile)) + } else { + verboseInfo.WriteString(tr("Skipping compile of: %[1]s", objectFile)) + } + } + + return objectFile, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), nil +} + +// ArchiveCompiledFiles fixdoc +func ArchiveCompiledFiles( + buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map, + onlyUpdateCompilationDatabase, verbose bool, + stdoutWriter, stderrWriter io.Writer, +) (*paths.Path, []byte, error) { + verboseInfobuf := &bytes.Buffer{} + archiveFilePath := buildPath.JoinPath(archiveFile) + + if onlyUpdateCompilationDatabase { + if verbose { + verboseInfobuf.WriteString(tr("Skipping archive creation of: %[1]s", archiveFilePath)) + } + return archiveFilePath, verboseInfobuf.Bytes(), nil + } + + if archiveFileStat, err := archiveFilePath.Stat(); err == nil { + rebuildArchive := false + for _, objectFile := range objectFilesToArchive { + objectFileStat, err := objectFile.Stat() + if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) { + // need to rebuild the archive + rebuildArchive = true + break + } + } + + // something changed, rebuild the core archive + if rebuildArchive { + if err := archiveFilePath.Remove(); err != nil { + return nil, nil, errors.WithStack(err) + } + } else { + if verbose { + verboseInfobuf.WriteString(tr("Using previously compiled file: %[1]s", archiveFilePath)) + } + return archiveFilePath, verboseInfobuf.Bytes(), nil + } + } + + for _, objectFile := range objectFilesToArchive { + properties := buildProperties.Clone() + properties.Set(builder.BuildPropertiesArchiveFile, archiveFilePath.Base()) + properties.SetPath(builder.BuildPropertiesArchiveFilePath, archiveFilePath) + properties.SetPath(builder.BuildPropertiesObjectFile, objectFile) + + command, err := PrepareCommandForRecipe(properties, builder.RecipeARPattern, false) + if err != nil { + return nil, verboseInfobuf.Bytes(), errors.WithStack(err) + } + + verboseInfo, _, _, err := ExecCommand(verbose, stdoutWriter, stderrWriter, command, ShowIfVerbose /* stdout */, Show /* stderr */) + if verbose { + verboseInfobuf.WriteString(string(verboseInfo)) + } + if err != nil { + return nil, verboseInfobuf.Bytes(), errors.WithStack(err) + } + } + + return archiveFilePath, verboseInfobuf.Bytes(), nil +} diff --git a/arduino/builder/utils/utils_test.go b/arduino/builder/utils/utils_test.go new file mode 100644 index 00000000000..84cd9d699ec --- /dev/null +++ b/arduino/builder/utils/utils_test.go @@ -0,0 +1,26 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPrintableCommand(t *testing.T) { + parts := []string{ + "/path/to/dir with spaces/cmd", + "arg1", + "arg-\"with\"-quotes", + "specialchar-`~!@#$%^&*()-_=+[{]}\\|;:'\",<.>/?-argument", + "arg with spaces", + "arg\twith\t\ttabs", + "lastarg", + } + correct := "\"/path/to/dir with spaces/cmd\"" + + " arg1 \"arg-\\\"with\\\"-quotes\"" + + " \"specialchar-`~!@#$%^&*()-_=+[{]}\\\\|;:'\\\",<.>/?-argument\"" + + " \"arg with spaces\" \"arg\twith\t\ttabs\"" + + " lastarg" + result := printableCommand(parts) + require.Equal(t, correct, result) +} diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 1daa6f31714..aa81cbcbf75 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -155,7 +155,21 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream // cache is purged after compilation to not remove entries that might be required defer maybePurgeBuildCache() - sketchBuilder := bldr.NewBuilder(sk) + var coreBuildCachePath *paths.Path + if req.GetBuildCachePath() == "" { + coreBuildCachePath = paths.TempDir().Join("arduino", "cores") + } else { + buildCachePath, err := paths.New(req.GetBuildCachePath()).Abs() + if err != nil { + return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err} + } + if err := buildCachePath.MkdirAll(); err != nil { + return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err} + } + coreBuildCachePath = buildCachePath.Join("core") + } + + sketchBuilder := bldr.NewBuilder(sk, coreBuildCachePath) // Add build properites related to sketch data buildProperties = sketchBuilder.SetupBuildProperties(buildProperties, buildPath, req.GetOptimizeForDebug()) @@ -193,7 +207,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.BuiltInToolsDirs = configuration.BuiltinToolsDirectories(configuration.Settings) builderCtx.OtherLibrariesDirs = paths.NewPathList(req.GetLibraries()...) builderCtx.OtherLibrariesDirs.Add(configuration.LibrariesDir(configuration.Settings)) - builderCtx.LibraryDirs = paths.NewPathList(req.Library...) builderCtx.CompilationDatabase = bldr.NewCompilationDatabase( builderCtx.BuildPath.Join("compile_commands.json"), @@ -207,19 +220,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.WarningsLevel = builder.DEFAULT_WARNINGS_LEVEL } - if req.GetBuildCachePath() == "" { - builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores") - } else { - buildCachePath, err := paths.New(req.GetBuildCachePath()).Abs() - if err != nil { - return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err} - } - if err := buildCachePath.MkdirAll(); err != nil { - return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err} - } - builderCtx.CoreBuildCachePath = buildCachePath.Join("core") - } - builderCtx.BuiltInLibrariesDirs = configuration.IDEBuiltinLibrariesDir(configuration.Settings) builderCtx.Stdout = outStream @@ -255,9 +255,10 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream libsManager = lm } useCachedLibrariesResolution := req.GetSkipLibrariesDiscovery() + libraryDir := paths.NewPathList(req.Library...) libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader( useCachedLibrariesResolution, libsManager, - builderCtx.BuiltInLibrariesDirs, builderCtx.LibraryDirs, builderCtx.OtherLibrariesDirs, + builderCtx.BuiltInLibrariesDirs, libraryDir, builderCtx.OtherLibrariesDirs, builderCtx.ActualPlatform, builderCtx.TargetPlatform, ) if err != nil { diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index ff517a5a2d4..2ad23985ce5 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -23,7 +23,6 @@ import ( "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/phases" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -51,33 +50,120 @@ func (s *Builder) Run(ctx *types.Context) error { return _err }), - utils.LogIfVerbose(false, tr("Detecting libraries used...")), + logIfVerbose(false, tr("Detecting libraries used...")), findIncludes(ctx), &WarnAboutArchIncompatibleLibraries{}, - utils.LogIfVerbose(false, tr("Generating function prototypes...")), + logIfVerbose(false, tr("Generating function prototypes...")), types.BareCommand(PreprocessSketch), - utils.LogIfVerbose(false, tr("Compiling sketch...")), + logIfVerbose(false, tr("Compiling sketch...")), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.sketch.prebuild", Suffix: ".pattern"}, - &phases.SketchBuilder{}, + types.BareCommand(func(ctx *types.Context) error { + sketchObjectFiles, err := phases.SketchBuilder( + ctx.SketchBuildPath, + ctx.BuildProperties, + ctx.SketchLibrariesDetector.IncludeFolders(), + ctx.OnlyUpdateCompilationDatabase, + ctx.Verbose, + ctx.CompilationDatabase, + ctx.Jobs, + ctx.WarningsLevel, + ctx.Stdout, ctx.Stderr, + func(msg string) { ctx.Info(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, + &ctx.Progress, ctx.ProgressCB, + ) + if err != nil { + return err + } + ctx.SketchObjectFiles = sketchObjectFiles + return nil + }), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.sketch.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, - utils.LogIfVerbose(false, tr("Compiling libraries...")), + logIfVerbose(false, tr("Compiling libraries...")), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.libraries.prebuild", Suffix: ".pattern"}, &UnusedCompiledLibrariesRemover{}, - &phases.LibrariesBuilder{}, + types.BareCommand(func(ctx *types.Context) error { + librariesObjectFiles, err := phases.LibrariesBuilder( + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.SketchLibrariesDetector.IncludeFolders(), + ctx.SketchLibrariesDetector.ImportedLibraries(), + ctx.Verbose, + ctx.OnlyUpdateCompilationDatabase, + ctx.CompilationDatabase, + ctx.Jobs, + ctx.WarningsLevel, + ctx.Stdout, + ctx.Stderr, + func(msg string) { ctx.Info(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, + &ctx.Progress, ctx.ProgressCB, + ) + if err != nil { + return err + } + ctx.LibrariesObjectFiles = librariesObjectFiles + + return nil + }), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.libraries.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, - utils.LogIfVerbose(false, tr("Compiling core...")), + logIfVerbose(false, tr("Compiling core...")), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.core.prebuild", Suffix: ".pattern"}, - &phases.CoreBuilder{}, + + types.BareCommand(func(ctx *types.Context) error { + objectFiles, archiveFile, err := phases.CoreBuilder( + ctx.BuildPath, ctx.CoreBuildPath, ctx.Builder.CoreBuildCachePath(), + ctx.BuildProperties, + ctx.ActualPlatform, + ctx.Verbose, ctx.OnlyUpdateCompilationDatabase, ctx.Clean, + ctx.CompilationDatabase, + ctx.Jobs, + ctx.WarningsLevel, + ctx.Stdout, ctx.Stderr, + func(msg string) { ctx.Info(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, + &ctx.Progress, ctx.ProgressCB, + ) + + ctx.CoreObjectsFiles = objectFiles + ctx.CoreArchiveFilePath = archiveFile + + return err + }), + &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.core.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, - utils.LogIfVerbose(false, tr("Linking everything together...")), + logIfVerbose(false, tr("Linking everything together...")), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.linking.prelink", Suffix: ".pattern"}, - &phases.Linker{}, + + types.BareCommand(func(ctx *types.Context) error { + verboseInfoOut, err := phases.Linker( + ctx.OnlyUpdateCompilationDatabase, + ctx.Verbose, + ctx.SketchObjectFiles, + ctx.LibrariesObjectFiles, + ctx.CoreObjectsFiles, + ctx.CoreArchiveFilePath, + ctx.BuildPath, + ctx.BuildProperties, + ctx.Stdout, + ctx.Stderr, + ctx.WarningsLevel, + ) + if ctx.Verbose { + ctx.Info(string(verboseInfoOut)) + } + return err + }), + &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.linking.postlink", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.objcopy.preobjcopy", Suffix: ".pattern"}, @@ -118,7 +204,18 @@ func (s *Builder) Run(ctx *types.Context) error { &ExportProjectCMake{SketchError: mainErr != nil}, - &phases.Sizer{SketchError: mainErr != nil}, + types.BareCommand(func(ctx *types.Context) error { + executableSectionsSize, err := phases.Sizer( + ctx.OnlyUpdateCompilationDatabase, mainErr != nil, ctx.Verbose, + ctx.BuildProperties, + ctx.Stdout, ctx.Stderr, + func(msg string) { ctx.Info(msg) }, + func(msg string) { ctx.Warn(msg) }, + ctx.WarningsLevel, + ) + ctx.ExecutableSectionsSize = executableSectionsSize + return err + }), } for _, command := range commands { PrintRingNameIfDebug(ctx, command) @@ -140,9 +237,6 @@ func (s *Builder) Run(ctx *types.Context) error { func PreprocessSketch(ctx *types.Context) error { preprocessorImpl := preprocessor.PreprocessSketchWithCtags - if ctx.UseArduinoPreprocessor { - preprocessorImpl = preprocessor.PreprocessSketchWithArduinoPreprocessor - } normalOutput, verboseOutput, err := preprocessorImpl( ctx.Sketch, ctx.BuildPath, ctx.SketchLibrariesDetector.IncludeFolders(), ctx.LineOffset, ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase) @@ -235,3 +329,17 @@ func findIncludes(ctx *types.Context) types.BareCommand { ) }) } + +func logIfVerbose(warn bool, msg string) types.BareCommand { + return types.BareCommand(func(ctx *types.Context) error { + if !ctx.Verbose { + return nil + } + if warn { + ctx.Warn(msg) + } else { + ctx.Info(msg) + } + return nil + }) +} diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go deleted file mode 100644 index 1d53860b163..00000000000 --- a/legacy/builder/builder_utils/utils.go +++ /dev/null @@ -1,305 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder_utils - -import ( - "fmt" - "os" - "path/filepath" - "runtime" - "strings" - "sync" - - bUtils "github.com/arduino/arduino-cli/arduino/builder/utils" - "github.com/arduino/arduino-cli/arduino/globals" - "github.com/arduino/arduino-cli/executils" - "github.com/arduino/arduino-cli/i18n" - "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" - "github.com/arduino/go-paths-helper" - "github.com/arduino/go-properties-orderedmap" - "github.com/pkg/errors" -) - -var tr = i18n.Tr - -// DirContentIsOlderThan returns true if the content of the given directory is -// older than target file. If extensions are given, only the files with these -// extensions are tested. -func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...string) (bool, error) { - targetStat, err := target.Stat() - if err != nil { - return false, err - } - targetModTime := targetStat.ModTime() - - files, err := bUtils.FindFilesInFolder(dir, true, extensions...) - if err != nil { - return false, err - } - for _, file := range files { - file, err := file.Stat() - if err != nil { - return false, err - } - if file.ModTime().After(targetModTime) { - return false, nil - } - } - return true, nil -} - -func CompileFiles(ctx *types.Context, sourceDir *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - return compileFiles(ctx, sourceDir, false, buildPath, buildProperties, includes) -} - -func CompileFilesRecursive(ctx *types.Context, sourceDir *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - return compileFiles(ctx, sourceDir, true, buildPath, buildProperties, includes) -} - -func compileFiles(ctx *types.Context, sourceDir *paths.Path, recurse bool, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - validExtensions := []string{} - for ext := range globals.SourceFilesValidExtensions { - validExtensions = append(validExtensions, ext) - } - - sources, err := bUtils.FindFilesInFolder(sourceDir, recurse, validExtensions...) - if err != nil { - return nil, err - } - - ctx.Progress.AddSubSteps(len(sources)) - defer ctx.Progress.RemoveSubSteps() - - objectFiles := paths.NewPathList() - var objectFilesMux sync.Mutex - if len(sources) == 0 { - return objectFiles, nil - } - var errorsList []error - var errorsMux sync.Mutex - - queue := make(chan *paths.Path) - job := func(source *paths.Path) { - recipe := fmt.Sprintf("recipe%s.o.pattern", source.Ext()) - if !buildProperties.ContainsKey(recipe) { - recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()]) - } - objectFile, err := compileFileWithRecipe(ctx, sourceDir, source, buildPath, buildProperties, includes, recipe) - if err != nil { - errorsMux.Lock() - errorsList = append(errorsList, err) - errorsMux.Unlock() - } else { - objectFilesMux.Lock() - objectFiles.Add(objectFile) - objectFilesMux.Unlock() - } - } - - // Spawn jobs runners - var wg sync.WaitGroup - jobs := ctx.Jobs - if jobs == 0 { - jobs = runtime.NumCPU() - } - for i := 0; i < jobs; i++ { - wg.Add(1) - go func() { - for source := range queue { - job(source) - } - wg.Done() - }() - } - - // Feed jobs until error or done - for _, source := range sources { - errorsMux.Lock() - gotError := len(errorsList) > 0 - errorsMux.Unlock() - if gotError { - break - } - queue <- source - - ctx.Progress.CompleteStep() - ctx.PushProgress() - } - close(queue) - wg.Wait() - if len(errorsList) > 0 { - // output the first error - return nil, errors.WithStack(errorsList[0]) - } - objectFiles.Sort() - return objectFiles, nil -} - -func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string, recipe string) (*paths.Path, error) { - properties := buildProperties.Clone() - properties.Set(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel)) - properties.Set(constants.BUILD_PROPERTIES_INCLUDES, strings.Join(includes, constants.SPACE)) - properties.SetPath("source_file", source) - relativeSource, err := sourcePath.RelTo(source) - if err != nil { - return nil, errors.WithStack(err) - } - depsFile := buildPath.Join(relativeSource.String() + ".d") - objectFile := buildPath.Join(relativeSource.String() + ".o") - - properties.SetPath(constants.BUILD_PROPERTIES_OBJECT_FILE, objectFile) - err = objectFile.Parent().MkdirAll() - if err != nil { - return nil, errors.WithStack(err) - } - - objIsUpToDate, err := bUtils.ObjFileIsUpToDate(source, objectFile, depsFile) - if err != nil { - return nil, errors.WithStack(err) - } - - command, err := PrepareCommandForRecipe(properties, recipe, false) - if err != nil { - return nil, errors.WithStack(err) - } - if ctx.CompilationDatabase != nil { - ctx.CompilationDatabase.Add(source, command) - } - if !objIsUpToDate && !ctx.OnlyUpdateCompilationDatabase { - // Since this compile could be multithreaded, we first capture the command output - stdout, stderr, err := utils.ExecCommand(ctx, command, utils.Capture, utils.Capture) - // and transfer all at once at the end... - if ctx.Verbose { - ctx.WriteStdout(stdout) - } - ctx.WriteStderr(stderr) - - // ...and then return the error - if err != nil { - return nil, errors.WithStack(err) - } - } else if ctx.Verbose { - if objIsUpToDate { - ctx.Info(tr("Using previously compiled file: %[1]s", objectFile)) - } else { - ctx.Info(tr("Skipping compile of: %[1]s", objectFile)) - } - } - - return objectFile, nil -} - -func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map) (*paths.Path, error) { - archiveFilePath := buildPath.JoinPath(archiveFile) - - if ctx.OnlyUpdateCompilationDatabase { - if ctx.Verbose { - ctx.Info(tr("Skipping archive creation of: %[1]s", archiveFilePath)) - } - return archiveFilePath, nil - } - - if archiveFileStat, err := archiveFilePath.Stat(); err == nil { - rebuildArchive := false - for _, objectFile := range objectFilesToArchive { - objectFileStat, err := objectFile.Stat() - if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) { - // need to rebuild the archive - rebuildArchive = true - break - } - } - - // something changed, rebuild the core archive - if rebuildArchive { - if err := archiveFilePath.Remove(); err != nil { - return nil, errors.WithStack(err) - } - } else { - if ctx.Verbose { - ctx.Info(tr("Using previously compiled file: %[1]s", archiveFilePath)) - } - return archiveFilePath, nil - } - } - - for _, objectFile := range objectFilesToArchive { - properties := buildProperties.Clone() - properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE, archiveFilePath.Base()) - properties.SetPath(constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH, archiveFilePath) - properties.SetPath(constants.BUILD_PROPERTIES_OBJECT_FILE, objectFile) - - command, err := PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false) - if err != nil { - return nil, errors.WithStack(err) - } - - _, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) - if err != nil { - return nil, errors.WithStack(err) - } - } - - return archiveFilePath, nil -} - -const COMMANDLINE_LIMIT = 30000 - -func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) { - pattern := buildProperties.Get(recipe) - if pattern == "" { - return nil, errors.Errorf(tr("%[1]s pattern is missing"), recipe) - } - - commandLine := buildProperties.ExpandPropsInString(pattern) - if removeUnsetProperties { - commandLine = properties.DeleteUnexpandedPropsFromString(commandLine) - } - - parts, err := properties.SplitQuotedString(commandLine, `"'`, false) - if err != nil { - return nil, errors.WithStack(err) - } - - // if the overall commandline is too long for the platform - // try reducing the length by making the filenames relative - // and changing working directory to build.path - var relativePath string - if len(commandLine) > COMMANDLINE_LIMIT { - relativePath = buildProperties.Get("build.path") - for i, arg := range parts { - if _, err := os.Stat(arg); os.IsNotExist(err) { - continue - } - rel, err := filepath.Rel(relativePath, arg) - if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) { - parts[i] = rel - } - } - } - - command, err := executils.NewProcess(nil, parts...) - if err != nil { - return nil, errors.WithStack(err) - } - if relativePath != "" { - command.SetDir(relativePath) - } - - return command, nil -} diff --git a/legacy/builder/constants/constants.go b/legacy/builder/constants/constants.go index 19a828584e3..d1a6e4887c6 100644 --- a/legacy/builder/constants/constants.go +++ b/legacy/builder/constants/constants.go @@ -17,19 +17,13 @@ package constants const BUILD_OPTIONS_FILE = "build.options.json" -const BUILD_PROPERTIES_ARCHIVE_FILE = "archive_file" -const BUILD_PROPERTIES_ARCHIVE_FILE_PATH = "archive_file_path" const BUILD_PROPERTIES_ARCH_OVERRIDE_CHECK = "architecture.override_check" const BUILD_PROPERTIES_BOOTLOADER_FILE = "bootloader.file" const BUILD_PROPERTIES_BOOTLOADER_NOBLINK = "bootloader.noblink" const BUILD_PROPERTIES_BUILD_BOARD = "build.board" const BUILD_PROPERTIES_BUILD_MCU = "build.mcu" -const BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS = "compiler.c.elf.flags" const BUILD_PROPERTIES_COMPILER_LDFLAGS = "compiler.ldflags" const BUILD_PROPERTIES_COMPILER_CPP_FLAGS = "compiler.cpp.flags" -const BUILD_PROPERTIES_COMPILER_WARNING_FLAGS = "compiler.warning_flags" -const BUILD_PROPERTIES_INCLUDES = "includes" -const BUILD_PROPERTIES_OBJECT_FILE = "object_file" const BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH = "runtime.platform.path" const EMPTY_STRING = "" const FOLDER_BOOTLOADERS = "bootloaders" @@ -50,9 +44,7 @@ const PACKAGE_TOOLS = "tools" const PLATFORM_ARCHITECTURE = "architecture" const PLATFORM_URL = "url" const PLATFORM_VERSION = "version" -const RECIPE_AR_PATTERN = "recipe.ar.pattern" const RECIPE_C_COMBINE_PATTERN = "recipe.c.combine.pattern" -const SPACE = " " const TOOL_NAME = "name" const TOOL_URL = "url" const TOOL_VERSION = "version" diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 6d6b67ae1d6..9ce4b068832 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -28,7 +28,6 @@ import ( "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" ) @@ -367,7 +366,7 @@ func extractCompileFlags(ctx *types.Context, recipe string, defines, dynamicLibs return target } - command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true) + command, _ := utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true) for _, arg := range command.GetArgs() { if strings.HasPrefix(arg, "-D") { diff --git a/legacy/builder/ctags_target_file_saver.go b/legacy/builder/ctags_target_file_saver.go deleted file mode 100644 index 261bc7c759a..00000000000 --- a/legacy/builder/ctags_target_file_saver.go +++ /dev/null @@ -1,16 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder diff --git a/legacy/builder/fail_if_buildpath_equals_sketchpath.go b/legacy/builder/fail_if_buildpath_equals_sketchpath.go deleted file mode 100644 index 2ec5c0da662..00000000000 --- a/legacy/builder/fail_if_buildpath_equals_sketchpath.go +++ /dev/null @@ -1,33 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/pkg/errors" -) - -type FailIfBuildPathEqualsSketchPath struct{} - -func (s *FailIfBuildPathEqualsSketchPath) Run(ctx *types.Context) error { - buildPath := ctx.BuildPath.Canonical() - sketchPath := ctx.Sketch.FullPath.Canonical() - if buildPath.EqualsTo(sketchPath) { - return errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) - } - - return nil -} diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index 0b629365eba..5c5e0339e41 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -19,74 +19,111 @@ import ( "crypto/md5" "encoding/hex" "fmt" + "io" "os" "strings" + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/cpp" + "github.com/arduino/arduino-cli/arduino/builder/progress" + "github.com/arduino/arduino-cli/arduino/builder/utils" + "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/buildcache" "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/types" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) -type CoreBuilder struct{} - var tr = i18n.Tr -func (s *CoreBuilder) Run(ctx *types.Context) error { - coreBuildPath := ctx.CoreBuildPath - coreBuildCachePath := ctx.CoreBuildCachePath - buildProperties := ctx.BuildProperties - +func CoreBuilder( + buildPath, coreBuildPath, coreBuildCachePath *paths.Path, + buildProperties *properties.Map, + actualPlatform *cores.PlatformRelease, + verbose, onlyUpdateCompilationDatabase, clean bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, *paths.Path, error) { if err := coreBuildPath.MkdirAll(); err != nil { - return errors.WithStack(err) + return nil, nil, errors.WithStack(err) } if coreBuildCachePath != nil { - if _, err := coreBuildCachePath.RelTo(ctx.BuildPath); err != nil { - ctx.Info(tr("Couldn't deeply cache core build: %[1]s", err)) - ctx.Info(tr("Running normal build of the core...")) + if _, err := coreBuildCachePath.RelTo(buildPath); err != nil { + verboseInfoFn(tr("Couldn't deeply cache core build: %[1]s", err)) + verboseInfoFn(tr("Running normal build of the core...")) coreBuildCachePath = nil - ctx.CoreBuildCachePath = nil } else if err := coreBuildCachePath.MkdirAll(); err != nil { - return errors.WithStack(err) + return nil, nil, errors.WithStack(err) } } - archiveFile, objectFiles, err := compileCore(ctx, coreBuildPath, coreBuildCachePath, buildProperties) + archiveFile, objectFiles, err := compileCore( + verbose, onlyUpdateCompilationDatabase, clean, + actualPlatform, + coreBuildPath, coreBuildCachePath, + buildProperties, + compilationDatabase, + jobs, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, + verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) if err != nil { - return errors.WithStack(err) + return nil, nil, errors.WithStack(err) } - ctx.CoreArchiveFilePath = archiveFile - ctx.CoreObjectsFiles = objectFiles - - return nil + return objectFiles, archiveFile, nil } -func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *paths.Path, buildProperties *properties.Map) (*paths.Path, paths.PathList, error) { +func compileCore( + verbose, onlyUpdateCompilationDatabase, clean bool, + actualPlatform *cores.PlatformRelease, + buildPath, buildCachePath *paths.Path, + buildProperties *properties.Map, + compilationDatabase *builder.CompilationDatabase, + jobs int, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (*paths.Path, paths.PathList, error) { coreFolder := buildProperties.GetPath("build.core.path") variantFolder := buildProperties.GetPath("build.variant.path") - targetCoreFolder := buildProperties.GetPath(constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH) - includes := []string{} - includes = append(includes, coreFolder.String()) + includes := []string{coreFolder.String()} if variantFolder != nil && variantFolder.IsDir() { includes = append(includes, variantFolder.String()) } includes = f.Map(includes, cpp.WrapWithHyphenI) var err error - variantObjectFiles := paths.NewPathList() if variantFolder != nil && variantFolder.IsDir() { - variantObjectFiles, err = builder_utils.CompileFilesRecursive(ctx, variantFolder, buildPath, buildProperties, includes) + variantObjectFiles, err = utils.CompileFilesRecursive( + variantFolder, buildPath, buildProperties, includes, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) if err != nil { return nil, nil, errors.WithStack(err) } @@ -98,7 +135,8 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path archivedCoreName := GetCachedCoreArchiveDirName( buildProperties.Get("build.fqbn"), buildProperties.Get("compiler.optimization_flags"), - realCoreFolder) + realCoreFolder, + ) targetArchivedCore = buildCachePath.Join(archivedCoreName, "core.a") if _, err := buildcache.New(buildCachePath).GetOrCreate(archivedCoreName); errors.Is(err, buildcache.CreateDirErr) { @@ -106,14 +144,14 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path } var canUseArchivedCore bool - if ctx.OnlyUpdateCompilationDatabase || ctx.Clean { + if onlyUpdateCompilationDatabase || clean { canUseArchivedCore = false - } else if isOlder, err := builder_utils.DirContentIsOlderThan(realCoreFolder, targetArchivedCore); err != nil || !isOlder { + } else if isOlder, err := utils.DirContentIsOlderThan(realCoreFolder, targetArchivedCore); err != nil || !isOlder { // Recreate the archive if ANY of the core files (including platform.txt) has changed canUseArchivedCore = false } else if targetCoreFolder == nil || realCoreFolder.EquivalentTo(targetCoreFolder) { canUseArchivedCore = true - } else if isOlder, err := builder_utils.DirContentIsOlderThan(targetCoreFolder, targetArchivedCore); err != nil || !isOlder { + } else if isOlder, err := utils.DirContentIsOlderThan(targetCoreFolder, targetArchivedCore); err != nil || !isOlder { // Recreate the archive if ANY of the build core files (including platform.txt) has changed canUseArchivedCore = false } else { @@ -122,35 +160,51 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path if canUseArchivedCore { // use archived core - if ctx.Verbose { - ctx.Info(tr("Using precompiled core: %[1]s", targetArchivedCore)) + if verbose { + verboseInfoFn(tr("Using precompiled core: %[1]s", targetArchivedCore)) } return targetArchivedCore, variantObjectFiles, nil } } - coreObjectFiles, err := builder_utils.CompileFilesRecursive(ctx, coreFolder, buildPath, buildProperties, includes) + coreObjectFiles, err := utils.CompileFilesRecursive( + coreFolder, buildPath, buildProperties, includes, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) if err != nil { return nil, nil, errors.WithStack(err) } - archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, buildPath, paths.New("core.a"), coreObjectFiles, buildProperties) + archiveFile, verboseInfo, err := utils.ArchiveCompiledFiles( + buildPath, paths.New("core.a"), coreObjectFiles, buildProperties, + onlyUpdateCompilationDatabase, verbose, stdoutWriter, stderrWriter, + ) + if verbose { + verboseInfoFn(string(verboseInfo)) + } if err != nil { return nil, nil, errors.WithStack(err) } // archive core.a - if targetArchivedCore != nil && !ctx.OnlyUpdateCompilationDatabase { + if targetArchivedCore != nil && !onlyUpdateCompilationDatabase { err := archiveFile.CopyTo(targetArchivedCore) - if ctx.Verbose { + if verbose { if err == nil { - ctx.Info(tr("Archiving built core (caching) in: %[1]s", targetArchivedCore)) + verboseInfoFn(tr("Archiving built core (caching) in: %[1]s", targetArchivedCore)) } else if os.IsNotExist(err) { - ctx.Info(tr("Unable to cache built core, please tell %[1]s maintainers to follow %[2]s", - ctx.ActualPlatform, + verboseInfoFn(tr("Unable to cache built core, please tell %[1]s maintainers to follow %[2]s", + actualPlatform, "https://arduino.github.io/arduino-cli/latest/platform-specification/#recipes-to-build-the-corea-archive-file")) } else { - ctx.Info(tr("Error archiving built core (caching) in %[1]s: %[2]s", targetArchivedCore, err)) + verboseInfoFn(tr("Error archiving built core (caching) in %[1]s: %[2]s", targetArchivedCore, err)) } } } @@ -161,8 +215,8 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path // GetCachedCoreArchiveDirName returns the directory name to be used to store // the global cached core.a. func GetCachedCoreArchiveDirName(fqbn string, optimizationFlags string, coreFolder *paths.Path) string { - fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1) - fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1) + fqbnToUnderscore := strings.ReplaceAll(fqbn, ":", "_") + fqbnToUnderscore = strings.ReplaceAll(fqbnToUnderscore, "=", "_") if absCoreFolder, err := coreFolder.Abs(); err == nil { coreFolder = absCoreFolder } // silently continue if absolute path can't be detected diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index 832bc9b0f8f..8608b292916 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -16,14 +16,17 @@ package phases import ( + "io" "strings" + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/cpp" + "github.com/arduino/arduino-cli/arduino/builder/progress" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/libraries" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/types" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -32,26 +35,43 @@ import ( var FLOAT_ABI_CFLAG = "float-abi" var FPU_CFLAG = "fpu" -type LibrariesBuilder struct{} - -func (s *LibrariesBuilder) Run(ctx *types.Context) error { - librariesBuildPath := ctx.LibrariesBuildPath - buildProperties := ctx.BuildProperties - includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() +func LibrariesBuilder( + librariesBuildPath *paths.Path, + buildProperties *properties.Map, + includesFolders paths.PathList, + importedLibraries libraries.List, + verbose, onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { includes := f.Map(includesFolders.AsStrings(), cpp.WrapWithHyphenI) - libs := ctx.SketchLibrariesDetector.ImportedLibraries() + libs := importedLibraries if err := librariesBuildPath.MkdirAll(); err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - objectFiles, err := compileLibraries(ctx, libs, librariesBuildPath, buildProperties, includes) + librariesObjectFiles, err := compileLibraries( + libs, librariesBuildPath, buildProperties, includes, + verbose, onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, + verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - ctx.LibrariesObjectFiles = objectFiles - return nil + return librariesObjectFiles, nil } func directoryContainsFile(folder *paths.Path) bool { @@ -62,12 +82,16 @@ func directoryContainsFile(folder *paths.Path) bool { return false } -func findExpectedPrecompiledLibFolder(ctx *types.Context, library *libraries.Library) *paths.Path { - mcu := ctx.BuildProperties.Get(constants.BUILD_PROPERTIES_BUILD_MCU) +func findExpectedPrecompiledLibFolder( + library *libraries.Library, + buildProperties *properties.Map, + verboseInfoFn func(msg string), +) *paths.Path { + mcu := buildProperties.Get(constants.BUILD_PROPERTIES_BUILD_MCU) // Add fpu specifications if they exist // To do so, resolve recipe.cpp.o.pattern, // search for -mfpu=xxx -mfloat-abi=yyy and add to a subfolder - command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, "recipe.cpp.o.pattern", true) + command, _ := utils.PrepareCommandForRecipe(buildProperties, "recipe.cpp.o.pattern", true) fpuSpecs := "" for _, el := range command.GetArgs() { if strings.Contains(el, FPU_CFLAG) { @@ -88,50 +112,85 @@ func findExpectedPrecompiledLibFolder(ctx *types.Context, library *libraries.Lib } } - ctx.Info(tr("Library %[1]s has been declared precompiled:", library.Name)) + verboseInfoFn(tr("Library %[1]s has been declared precompiled:", library.Name)) // Try directory with full fpuSpecs first, if available if len(fpuSpecs) > 0 { fpuSpecs = strings.TrimRight(fpuSpecs, "-") fullPrecompDir := library.SourceDir.Join(mcu).Join(fpuSpecs) if fullPrecompDir.Exist() && directoryContainsFile(fullPrecompDir) { - ctx.Info(tr("Using precompiled library in %[1]s", fullPrecompDir)) + verboseInfoFn(tr("Using precompiled library in %[1]s", fullPrecompDir)) return fullPrecompDir } - ctx.Info(tr(`Precompiled library in "%[1]s" not found`, fullPrecompDir)) + verboseInfoFn(tr(`Precompiled library in "%[1]s" not found`, fullPrecompDir)) } precompDir := library.SourceDir.Join(mcu) if precompDir.Exist() && directoryContainsFile(precompDir) { - ctx.Info(tr("Using precompiled library in %[1]s", precompDir)) + verboseInfoFn(tr("Using precompiled library in %[1]s", precompDir)) return precompDir } - ctx.Info(tr(`Precompiled library in "%[1]s" not found`, precompDir)) + verboseInfoFn(tr(`Precompiled library in "%[1]s" not found`, precompDir)) return nil } -func compileLibraries(ctx *types.Context, libraries libraries.List, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - ctx.Progress.AddSubSteps(len(libraries)) - defer ctx.Progress.RemoveSubSteps() +func compileLibraries( + libraries libraries.List, buildPath *paths.Path, buildProperties *properties.Map, includes []string, + verbose, onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + progress.AddSubSteps(len(libraries)) + defer progress.RemoveSubSteps() objectFiles := paths.NewPathList() for _, library := range libraries { - libraryObjectFiles, err := compileLibrary(ctx, library, buildPath, buildProperties, includes) + libraryObjectFiles, err := compileLibrary( + library, buildPath, buildProperties, includes, + verbose, onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) if err != nil { return nil, errors.WithStack(err) } objectFiles.AddAll(libraryObjectFiles) - ctx.Progress.CompleteStep() - ctx.PushProgress() + progress.CompleteStep() + // PushProgress + if progressCB != nil { + progressCB(&rpc.TaskProgress{ + Percent: progress.Progress, + Completed: progress.Progress >= 100.0, + }) + } } return objectFiles, nil } -func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - if ctx.Verbose { - ctx.Info(tr(`Compiling library "%[1]s"`, library.Name)) +func compileLibrary( + library *libraries.Library, buildPath *paths.Path, buildProperties *properties.Map, includes []string, + verbose, onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + if verbose { + verboseInfoFn(tr(`Compiling library "%[1]s"`, library.Name)) } libraryBuildPath := buildPath.Join(library.DirName) @@ -142,11 +201,15 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p objectFiles := paths.NewPathList() if library.Precompiled { - coreSupportPrecompiled := ctx.BuildProperties.ContainsKey("compiler.libraries.ldflags") - precompiledPath := findExpectedPrecompiledLibFolder(ctx, library) + coreSupportPrecompiled := buildProperties.ContainsKey("compiler.libraries.ldflags") + precompiledPath := findExpectedPrecompiledLibFolder( + library, + buildProperties, + verboseInfoFn, + ) if !coreSupportPrecompiled { - ctx.Info(tr("The platform does not support '%[1]s' for precompiled libraries.", "compiler.libraries.ldflags")) + verboseInfoFn(tr("The platform does not support '%[1]s' for precompiled libraries.", "compiler.libraries.ldflags")) } else if precompiledPath != nil { // Find all libraries in precompiledPath libs, err := precompiledPath.ReadDir() @@ -165,8 +228,8 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p } } - currLDFlags := ctx.BuildProperties.Get("compiler.libraries.ldflags") - ctx.BuildProperties.Set("compiler.libraries.ldflags", currLDFlags+" \"-L"+precompiledPath.String()+"\" "+libsCmd+" ") + currLDFlags := buildProperties.Get("compiler.libraries.ldflags") + buildProperties.Set("compiler.libraries.ldflags", currLDFlags+" \"-L"+precompiledPath.String()+"\" "+libsCmd+" ") // TODO: This codepath is just taken for .a with unusual names that would // be ignored by -L / -l methods. @@ -186,12 +249,29 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p } if library.Layout == libraries.RecursiveLayout { - libObjectFiles, err := builder_utils.CompileFilesRecursive(ctx, library.SourceDir, libraryBuildPath, buildProperties, includes) + libObjectFiles, err := utils.CompileFilesRecursive( + library.SourceDir, libraryBuildPath, buildProperties, includes, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) if err != nil { return nil, errors.WithStack(err) } if library.DotALinkage { - archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, libraryBuildPath, paths.New(library.DirName+".a"), libObjectFiles, buildProperties) + archiveFile, verboseInfo, err := utils.ArchiveCompiledFiles( + libraryBuildPath, paths.New(library.DirName+".a"), libObjectFiles, buildProperties, + onlyUpdateCompilationDatabase, verbose, + stdoutWriter, stderrWriter, + ) + if verbose { + verboseInfoFn(string(verboseInfo)) + } if err != nil { return nil, errors.WithStack(err) } @@ -203,7 +283,17 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p if library.UtilityDir != nil { includes = append(includes, cpp.WrapWithHyphenI(library.UtilityDir.String())) } - libObjectFiles, err := builder_utils.CompileFiles(ctx, library.SourceDir, libraryBuildPath, buildProperties, includes) + libObjectFiles, err := utils.CompileFiles( + library.SourceDir, libraryBuildPath, buildProperties, includes, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) if err != nil { return nil, errors.WithStack(err) } @@ -211,7 +301,17 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p if library.UtilityDir != nil { utilityBuildPath := libraryBuildPath.Join("utility") - utilityObjectFiles, err := builder_utils.CompileFiles(ctx, library.UtilityDir, utilityBuildPath, buildProperties, includes) + utilityObjectFiles, err := utils.CompileFiles( + library.UtilityDir, utilityBuildPath, buildProperties, includes, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) if err != nil { return nil, errors.WithStack(err) } diff --git a/legacy/builder/phases/linker.go b/legacy/builder/phases/linker.go index 1f324576586..654c4e29bc4 100644 --- a/legacy/builder/phases/linker.go +++ b/legacy/builder/phases/linker.go @@ -16,55 +16,68 @@ package phases import ( + "bytes" + "io" "strings" + "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/utils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" - "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) -type Linker struct{} - -func (s *Linker) Run(ctx *types.Context) error { - if ctx.OnlyUpdateCompilationDatabase { - if ctx.Verbose { - ctx.Info(tr("Skip linking of final executable.")) +func Linker( + onlyUpdateCompilationDatabase, verbose bool, + sketchObjectFiles, librariesObjectFiles, coreObjectsFiles paths.PathList, + coreArchiveFilePath, buildPath *paths.Path, + buildProperties *properties.Map, + stdoutWriter, stderrWriter io.Writer, + warningsLevel string, +) ([]byte, error) { + verboseInfo := &bytes.Buffer{} + if onlyUpdateCompilationDatabase { + if verbose { + verboseInfo.WriteString(tr("Skip linking of final executable.")) } - return nil + return verboseInfo.Bytes(), nil } - objectFilesSketch := ctx.SketchObjectFiles - objectFilesLibraries := ctx.LibrariesObjectFiles - objectFilesCore := ctx.CoreObjectsFiles + objectFilesSketch := sketchObjectFiles + objectFilesLibraries := librariesObjectFiles + objectFilesCore := coreObjectsFiles objectFiles := paths.NewPathList() objectFiles.AddAll(objectFilesSketch) objectFiles.AddAll(objectFilesLibraries) objectFiles.AddAll(objectFilesCore) - coreArchiveFilePath := ctx.CoreArchiveFilePath - buildPath := ctx.BuildPath coreDotARelPath, err := buildPath.RelTo(coreArchiveFilePath) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - buildProperties := ctx.BuildProperties - - err = link(ctx, objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties) + verboseInfoOut, err := link( + objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties, + verbose, stdoutWriter, stderrWriter, warningsLevel, + ) + verboseInfo.Write(verboseInfoOut) if err != nil { - return errors.WithStack(err) + return verboseInfo.Bytes(), errors.WithStack(err) } - return nil + return verboseInfo.Bytes(), nil } -func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths.Path, coreArchiveFilePath *paths.Path, buildProperties *properties.Map) error { +func link( + objectFiles paths.PathList, coreDotARelPath *paths.Path, coreArchiveFilePath *paths.Path, buildProperties *properties.Map, + verbose bool, + stdoutWriter, stderrWriter io.Writer, + warningsLevel string, +) ([]byte, error) { + verboseBuffer := &bytes.Buffer{} + wrapWithDoubleQuotes := func(value string) string { return "\"" + value + "\"" } objectFileList := strings.Join(f.Map(objectFiles.AsStrings(), wrapWithDoubleQuotes), " ") // If command line length is too big (> 30000 chars), try to collect the object files into archives @@ -93,13 +106,16 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths properties.SetPath("archive_file_path", archive) properties.SetPath("object_file", object) - command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false) + command, err := utils.PrepareCommandForRecipe(properties, builder.RecipeARPattern, false) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - if _, _, err := utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */); err != nil { - return errors.WithStack(err) + if verboseInfo, _, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */); err != nil { + if verbose { + verboseBuffer.WriteString(string(verboseInfo)) + } + return verboseBuffer.Bytes(), errors.WithStack(err) } } @@ -108,21 +124,20 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths } properties := buildProperties.Clone() - properties.Set(constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS)) - properties.Set(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel)) - properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE, coreDotARelPath.String()) - properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH, coreArchiveFilePath.String()) + properties.Set("compiler.c.elf.flags", properties.Get("compiler.c.elf.flags")) + properties.Set(builder.BuildPropertiesCompilerWarningFlags, properties.Get(builder.BuildPropertiesCompilerWarningFlags+"."+warningsLevel)) + properties.Set(builder.BuildPropertiesArchiveFile, coreDotARelPath.String()) + properties.Set(builder.BuildPropertiesArchiveFilePath, coreArchiveFilePath.String()) properties.Set("object_files", objectFileList) - command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false) + command, err := utils.PrepareCommandForRecipe(properties, "recipe.c.combine.pattern", false) if err != nil { - return err -} - - _, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) - return err -} + return verboseBuffer.Bytes(), err + } -func wrapWithDoubleQuotes(value string) string { - return "\"" + value + "\"" + verboseInfo, _, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) + if verbose { + verboseBuffer.WriteString(string(verboseInfo)) + } + return verboseBuffer.Bytes(), err } diff --git a/legacy/builder/phases/sizer.go b/legacy/builder/phases/sizer.go index c064fbe717b..cfc28f987c1 100644 --- a/legacy/builder/phases/sizer.go +++ b/legacy/builder/phases/sizer.go @@ -18,46 +18,50 @@ package phases import ( "encoding/json" "fmt" + "io" "regexp" "strconv" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) -type Sizer struct { - SketchError bool -} - -func (s *Sizer) Run(ctx *types.Context) error { - if ctx.OnlyUpdateCompilationDatabase { - return nil - } - if s.SketchError { - return nil +func Sizer( + onlyUpdateCompilationDatabase, sketchError, verbose bool, + buildProperties *properties.Map, + stdoutWriter, stderrWriter io.Writer, + printInfoFn, printWarnFn func(msg string), + warningsLevel string, +) (builder.ExecutablesFileSections, error) { + if onlyUpdateCompilationDatabase || sketchError { + return nil, nil } - buildProperties := ctx.BuildProperties - if buildProperties.ContainsKey("recipe.advanced_size.pattern") { - return checkSizeAdvanced(ctx, buildProperties) + return checkSizeAdvanced(buildProperties, verbose, stdoutWriter, stderrWriter, printInfoFn, printWarnFn) } - return checkSize(ctx, buildProperties) + return checkSize(buildProperties, verbose, stdoutWriter, stderrWriter, printInfoFn, printWarnFn, warningsLevel) } -func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error { - command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.advanced_size.pattern", false) +func checkSizeAdvanced(buildProperties *properties.Map, + verbose bool, + stdoutWriter, stderrWriter io.Writer, + printInfoFn, printWarnFn func(msg string), +) (builder.ExecutablesFileSections, error) { + command, err := utils.PrepareCommandForRecipe(buildProperties, "recipe.advanced_size.pattern", false) if err != nil { - return errors.New(tr("Error while determining sketch size: %s", err)) + return nil, errors.New(tr("Error while determining sketch size: %s", err)) } - out, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.Show /* stderr */) + verboseInfo, out, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.Capture /* stdout */, utils.Show /* stderr */) + if verbose { + printInfoFn(string(verboseInfo)) + } if err != nil { - return errors.New(tr("Error while determining sketch size: %s", err)) + return nil, errors.New(tr("Error while determining sketch size: %s", err)) } type AdvancedSizerResponse struct { @@ -67,7 +71,7 @@ func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error { // likely be printed in red. Errors will stop build/upload. Severity string `json:"severity"` // Sections are the sections sizes for machine readable use - Sections []types.ExecutableSectionSize `json:"sections"` + Sections []builder.ExecutableSectionSize `json:"sections"` // ErrorMessage is a one line error message like: // "text section exceeds available space in board" // it must be set when Severity is "error" @@ -76,71 +80,76 @@ func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error { var resp AdvancedSizerResponse if err := json.Unmarshal(out, &resp); err != nil { - return errors.New(tr("Error while determining sketch size: %s", err)) + return nil, errors.New(tr("Error while determining sketch size: %s", err)) } - ctx.ExecutableSectionsSize = resp.Sections + executableSectionsSize := resp.Sections switch resp.Severity { case "error": - ctx.Warn(resp.Output) - return errors.New(resp.ErrorMessage) + printWarnFn(resp.Output) + return executableSectionsSize, errors.New(resp.ErrorMessage) case "warning": - ctx.Warn(resp.Output) + printWarnFn(resp.Output) case "info": - ctx.Info(resp.Output) + printInfoFn(resp.Output) default: - return fmt.Errorf("invalid '%s' severity from sketch sizer: it must be 'error', 'warning' or 'info'", resp.Severity) + return executableSectionsSize, fmt.Errorf("invalid '%s' severity from sketch sizer: it must be 'error', 'warning' or 'info'", resp.Severity) } - return nil + return executableSectionsSize, nil } -func checkSize(ctx *types.Context, buildProperties *properties.Map) error { +func checkSize(buildProperties *properties.Map, + verbose bool, + stdoutWriter, stderrWriter io.Writer, + printInfoFn, printWarnFn func(msg string), + warningsLevel string, +) (builder.ExecutablesFileSections, error) { properties := buildProperties.Clone() - properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+ctx.WarningsLevel)) + properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+warningsLevel)) maxTextSizeString := properties.Get("upload.maximum_size") maxDataSizeString := properties.Get("upload.maximum_data_size") if maxTextSizeString == "" { - return nil + return nil, nil } maxTextSize, err := strconv.Atoi(maxTextSizeString) if err != nil { - return err + return nil, err } maxDataSize := -1 if maxDataSizeString != "" { maxDataSize, err = strconv.Atoi(maxDataSizeString) if err != nil { - return err + return nil, err } } - textSize, dataSize, _, err := execSizeRecipe(ctx, properties) + textSize, dataSize, _, err := execSizeRecipe(properties, verbose, stdoutWriter, stderrWriter, printInfoFn) if err != nil { - ctx.Warn(tr("Couldn't determine program size")) - return nil + printWarnFn(tr("Couldn't determine program size")) + return nil, nil } - ctx.Info(tr("Sketch uses %[1]s bytes (%[3]s%%) of program storage space. Maximum is %[2]s bytes.", + printInfoFn(tr("Sketch uses %[1]s bytes (%[3]s%%) of program storage space. Maximum is %[2]s bytes.", strconv.Itoa(textSize), strconv.Itoa(maxTextSize), strconv.Itoa(textSize*100/maxTextSize))) if dataSize >= 0 { if maxDataSize > 0 { - ctx.Info(tr("Global variables use %[1]s bytes (%[3]s%%) of dynamic memory, leaving %[4]s bytes for local variables. Maximum is %[2]s bytes.", + printInfoFn(tr("Global variables use %[1]s bytes (%[3]s%%) of dynamic memory, leaving %[4]s bytes for local variables. Maximum is %[2]s bytes.", strconv.Itoa(dataSize), strconv.Itoa(maxDataSize), strconv.Itoa(dataSize*100/maxDataSize), strconv.Itoa(maxDataSize-dataSize))) } else { - ctx.Info(tr("Global variables use %[1]s bytes of dynamic memory.", strconv.Itoa(dataSize))) + printInfoFn(tr("Global variables use %[1]s bytes of dynamic memory.", strconv.Itoa(dataSize))) } } - ctx.ExecutableSectionsSize = []types.ExecutableSectionSize{ + executableSectionsSize := []builder.ExecutableSectionSize{ { Name: "text", Size: textSize, @@ -148,7 +157,7 @@ func checkSize(ctx *types.Context, buildProperties *properties.Map) error { }, } if maxDataSize > 0 { - ctx.ExecutableSectionsSize = append(ctx.ExecutableSectionsSize, types.ExecutableSectionSize{ + executableSectionsSize = append(executableSectionsSize, builder.ExecutableSectionSize{ Name: "data", Size: dataSize, MaxSize: maxDataSize, @@ -156,36 +165,43 @@ func checkSize(ctx *types.Context, buildProperties *properties.Map) error { } if textSize > maxTextSize { - ctx.Warn(tr("Sketch too big; see %[1]s for tips on reducing it.", "https://support.arduino.cc/hc/en-us/articles/360013825179")) - return errors.New(tr("text section exceeds available space in board")) + printWarnFn(tr("Sketch too big; see %[1]s for tips on reducing it.", "https://support.arduino.cc/hc/en-us/articles/360013825179")) + return executableSectionsSize, errors.New(tr("text section exceeds available space in board")) } if maxDataSize > 0 && dataSize > maxDataSize { - ctx.Warn(tr("Not enough memory; see %[1]s for tips on reducing your footprint.", "https://support.arduino.cc/hc/en-us/articles/360013825179")) - return errors.New(tr("data section exceeds available space in board")) + printWarnFn(tr("Not enough memory; see %[1]s for tips on reducing your footprint.", "https://support.arduino.cc/hc/en-us/articles/360013825179")) + return executableSectionsSize, errors.New(tr("data section exceeds available space in board")) } if w := properties.Get("build.warn_data_percentage"); w != "" { warnDataPercentage, err := strconv.Atoi(w) if err != nil { - return err + return executableSectionsSize, err } if maxDataSize > 0 && dataSize > maxDataSize*warnDataPercentage/100 { - ctx.Warn(tr("Low memory available, stability problems may occur.")) + printWarnFn(tr("Low memory available, stability problems may occur.")) } } - return nil + return executableSectionsSize, nil } -func execSizeRecipe(ctx *types.Context, properties *properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) { - command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false) +func execSizeRecipe(properties *properties.Map, + verbose bool, + stdoutWriter, stderrWriter io.Writer, + printInfoFn func(msg string), +) (textSize int, dataSize int, eepromSize int, resErr error) { + command, err := utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false) if err != nil { resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err) return } - out, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.Show /* stderr */) + verboseInfo, out, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.Capture /* stdout */, utils.Show /* stderr */) + if verbose { + printInfoFn(string(verboseInfo)) + } if err != nil { resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err) return diff --git a/legacy/builder/phases/sketch_builder.go b/legacy/builder/phases/sketch_builder.go index 51057a21835..190604ddc0a 100644 --- a/legacy/builder/phases/sketch_builder.go +++ b/legacy/builder/phases/sketch_builder.go @@ -16,41 +16,76 @@ package phases import ( + "io" + + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/cpp" + "github.com/arduino/arduino-cli/arduino/builder/progress" + "github.com/arduino/arduino-cli/arduino/builder/utils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" - "github.com/arduino/arduino-cli/legacy/builder/types" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" + "github.com/arduino/go-paths-helper" + "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) -type SketchBuilder struct{} - -func (s *SketchBuilder) Run(ctx *types.Context) error { - sketchBuildPath := ctx.SketchBuildPath - buildProperties := ctx.BuildProperties - includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() +func SketchBuilder( + sketchBuildPath *paths.Path, + buildProperties *properties.Map, + includesFolders paths.PathList, + onlyUpdateCompilationDatabase, verbose bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { includes := f.Map(includesFolders.AsStrings(), cpp.WrapWithHyphenI) if err := sketchBuildPath.MkdirAll(); err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - objectFiles, err := builder_utils.CompileFiles(ctx, sketchBuildPath, sketchBuildPath, buildProperties, includes) + sketchObjectFiles, err := utils.CompileFiles( + sketchBuildPath, sketchBuildPath, buildProperties, includes, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, + verboseStdoutFn, + verboseStderrFn, + progress, progressCB, + ) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } // The "src/" subdirectory of a sketch is compiled recursively sketchSrcPath := sketchBuildPath.Join("src") if sketchSrcPath.IsDir() { - srcObjectFiles, err := builder_utils.CompileFilesRecursive(ctx, sketchSrcPath, sketchSrcPath, buildProperties, includes) + srcObjectFiles, err := utils.CompileFilesRecursive( + sketchSrcPath, sketchSrcPath, buildProperties, includes, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, + verboseStdoutFn, + verboseStderrFn, + progress, progressCB, + ) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - objectFiles.AddAll(srcObjectFiles) + sketchObjectFiles.AddAll(srcObjectFiles) } - ctx.SketchObjectFiles = objectFiles - - return nil + return sketchObjectFiles, nil } diff --git a/legacy/builder/recipe_runner.go b/legacy/builder/recipe_runner.go index 01257267287..9e8bdcd2425 100644 --- a/legacy/builder/recipe_runner.go +++ b/legacy/builder/recipe_runner.go @@ -20,9 +20,8 @@ import ( "sort" "strings" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" properties "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -44,7 +43,7 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { for _, recipe := range recipes { logrus.Debugf(fmt.Sprintf("Running recipe: %s", recipe)) - command, err := builder_utils.PrepareCommandForRecipe(properties, recipe, false) + command, err := utils.PrepareCommandForRecipe(properties, recipe, false) if err != nil { return errors.WithStack(err) } @@ -56,7 +55,10 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { return nil } - _, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) + verboseInfo, _, _, err := utils.ExecCommand(ctx.Verbose, ctx.Stdout, ctx.Stderr, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) + if ctx.Verbose { + ctx.Info(string(verboseInfo)) + } if err != nil { return errors.WithStack(err) } diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index c4f61212cc6..deafc113154 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -84,11 +84,8 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat // NoError(t, err) fmt.Println(err) } - if !ctx.CanUseCachedTools { - if ctx.BuiltInToolsDirs != nil { - pmb.LoadToolsFromBundleDirectories(ctx.BuiltInToolsDirs) - } - ctx.CanUseCachedTools = true + if ctx.BuiltInToolsDirs != nil { + pmb.LoadToolsFromBundleDirectories(ctx.BuiltInToolsDirs) } pm := pmb.Build() pme, _ /* never release... */ := pm.NewExplorer() @@ -100,7 +97,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.Sketch = sk } - ctx.Builder = bldr.NewBuilder(ctx.Sketch) + ctx.Builder = bldr.NewBuilder(ctx.Sketch, nil) if fqbn != "" { ctx.FQBN = parseFQBN(t, fqbn) targetPackage, targetPlatform, targetBoard, buildProperties, buildPlatform, err := pme.ResolveFQBN(ctx.FQBN) @@ -125,7 +122,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat if !stepToSkip[skipLibraries] { lm, libsResolver, _, err := detector.LibrariesLoader( false, nil, - ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) require.NoError(t, err) diff --git a/legacy/builder/test/helper.go b/legacy/builder/test/helper.go index 4ca6f318157..33b9aa1e2cf 100644 --- a/legacy/builder/test/helper.go +++ b/legacy/builder/test/helper.go @@ -21,7 +21,6 @@ import ( "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" @@ -40,13 +39,6 @@ func SetupBuildPath(t *testing.T, ctx *types.Context) *paths.Path { return buildPath } -func SetupBuildCachePath(t *testing.T, ctx *types.Context) *paths.Path { - buildCachePath, err := paths.MkTempDir(constants.EMPTY_STRING, "test_build_cache") - require.NoError(t, err) - ctx.CoreBuildCachePath = buildCachePath - return buildCachePath -} - func parseFQBN(t *testing.T, fqbnIn string) *cores.FQBN { fqbn, err := cores.ParseFQBN(fqbnIn) require.NoError(t, err) diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index b265ca848a2..5866fbcfa98 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -49,7 +49,7 @@ func TestLoadLibrariesAVR(t *testing.T) { lm, libsResolver, _, err := detector.LibrariesLoader( false, nil, - ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) require.NoError(t, err) @@ -153,7 +153,7 @@ func TestLoadLibrariesSAM(t *testing.T) { lm, libsResolver, _, err := detector.LibrariesLoader( false, nil, - ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) require.NoError(t, err) @@ -230,7 +230,7 @@ func TestLoadLibrariesAVRNoDuplicateLibrariesFolders(t *testing.T) { lm, _, _, err := detector.LibrariesLoader( false, nil, - ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) require.NoError(t, err) @@ -253,7 +253,7 @@ func TestLoadLibrariesMyAVRPlatform(t *testing.T) { lm, _, _, err := detector.LibrariesLoader( false, nil, - ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) require.NoError(t, err) diff --git a/legacy/builder/test/utils_test.go b/legacy/builder/test/utils_test.go deleted file mode 100644 index a6a4dea8fb4..00000000000 --- a/legacy/builder/test/utils_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package test - -import ( - "testing" - - "github.com/arduino/arduino-cli/legacy/builder/utils" - "github.com/stretchr/testify/require" -) - -func TestPrintableCommand(t *testing.T) { - parts := []string{ - "/path/to/dir with spaces/cmd", - "arg1", - "arg-\"with\"-quotes", - "specialchar-`~!@#$%^&*()-_=+[{]}\\|;:'\",<.>/?-argument", - "arg with spaces", - "arg\twith\t\ttabs", - "lastarg", - } - correct := "\"/path/to/dir with spaces/cmd\"" + - " arg1 \"arg-\\\"with\\\"-quotes\"" + - " \"specialchar-`~!@#$%^&*()-_=+[{]}\\\\|;:'\\\",<.>/?-argument\"" + - " \"arg with spaces\" \"arg\twith\t\ttabs\"" + - " lastarg" - result := utils.PrintableCommand(parts) - require.Equal(t, correct, result) -} diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index a6570bf3268..0c9b1a18458 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -24,6 +24,7 @@ import ( "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/detector" + "github.com/arduino/arduino-cli/arduino/builder/progress" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/arduino/sketch" @@ -32,34 +33,6 @@ import ( properties "github.com/arduino/go-properties-orderedmap" ) -type ProgressStruct struct { - Progress float32 - StepAmount float32 - Parent *ProgressStruct -} - -func (p *ProgressStruct) AddSubSteps(steps int) { - p.Parent = &ProgressStruct{ - Progress: p.Progress, - StepAmount: p.StepAmount, - Parent: p.Parent, - } - if p.StepAmount == 0.0 { - p.StepAmount = 100.0 - } - p.StepAmount /= float32(steps) -} - -func (p *ProgressStruct) RemoveSubSteps() { - p.Progress = p.Parent.Progress - p.StepAmount = p.Parent.StepAmount - p.Parent = p.Parent.Parent -} - -func (p *ProgressStruct) CompleteStep() { - p.Progress += p.StepAmount -} - // Context structure type Context struct { Builder *builder.Builder @@ -70,8 +43,6 @@ type Context struct { BuiltInToolsDirs paths.PathList BuiltInLibrariesDirs *paths.Path OtherLibrariesDirs paths.PathList - LibraryDirs paths.PathList // List of paths pointing to individual library root folders - WatchedLocations paths.PathList FQBN *cores.FQBN Clean bool @@ -86,17 +57,15 @@ type Context struct { TargetPlatform *cores.PlatformRelease ActualPlatform *cores.PlatformRelease - BuildProperties *properties.Map - BuildPath *paths.Path - SketchBuildPath *paths.Path - CoreBuildPath *paths.Path - CoreBuildCachePath *paths.Path - CoreArchiveFilePath *paths.Path - CoreObjectsFiles paths.PathList - LibrariesBuildPath *paths.Path - LibrariesObjectFiles paths.PathList - SketchObjectFiles paths.PathList - IgnoreSketchFolderNameErrors bool + BuildProperties *properties.Map + BuildPath *paths.Path + SketchBuildPath *paths.Path + CoreBuildPath *paths.Path + CoreArchiveFilePath *paths.Path + CoreObjectsFiles paths.PathList + LibrariesBuildPath *paths.Path + LibrariesObjectFiles paths.PathList + SketchObjectFiles paths.PathList Sketch *sketch.Sketch WarningsLevel string @@ -108,19 +77,13 @@ type Context struct { Verbose bool // Dry run, only create progress map - Progress ProgressStruct + Progress progress.Struct // Send progress events to this callback ProgressCB rpc.TaskProgressCB // Custom build properties defined by user (line by line as "key=value" pairs) CustomBuildProperties []string - // Reuse old tools since the backing storage didn't change - CanUseCachedTools bool - - // Experimental: use arduino-preprocessor to create prototypes - UseArduinoPreprocessor bool - // Parallel processes Jobs int @@ -130,7 +93,7 @@ type Context struct { stdLock sync.Mutex // Sizer results - ExecutableSectionsSize ExecutablesFileSections + ExecutableSectionsSize builder.ExecutablesFileSections // Compilation Database to build/update CompilationDatabase *builder.CompilationDatabase @@ -143,29 +106,6 @@ type Context struct { SourceOverride map[string]string } -// ExecutableSectionSize represents a section of the executable output file -type ExecutableSectionSize struct { - Name string `json:"name"` - Size int `json:"size"` - MaxSize int `json:"max_size"` -} - -// ExecutablesFileSections is an array of ExecutablesFileSection -type ExecutablesFileSections []ExecutableSectionSize - -// ToRPCExecutableSectionSizeArray transforms this array into a []*rpc.ExecutableSectionSize -func (s ExecutablesFileSections) ToRPCExecutableSectionSizeArray() []*rpc.ExecutableSectionSize { - res := []*rpc.ExecutableSectionSize{} - for _, section := range s { - res = append(res, &rpc.ExecutableSectionSize{ - Name: section.Name, - Size: int64(section.Size), - MaxSize: int64(section.MaxSize), - }) - } - return res -} - func (ctx *Context) ExtractBuildOptions() *properties.Map { opts := properties.NewMap() opts.Set("hardwareFolders", strings.Join(ctx.HardwareDirs.AsStrings(), ",")) diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go deleted file mode 100644 index ec34d1f2c3b..00000000000 --- a/legacy/builder/utils/utils.go +++ /dev/null @@ -1,109 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package utils - -import ( - "bytes" - "os" - "strings" - - "github.com/arduino/arduino-cli/executils" - f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/pkg/errors" -) - -func printableArgument(arg string) string { - if strings.ContainsAny(arg, "\"\\ \t") { - arg = strings.Replace(arg, "\\", "\\\\", -1) - arg = strings.Replace(arg, "\"", "\\\"", -1) - return "\"" + arg + "\"" - } else { - return arg - } -} - -// Convert a command and argument slice back to a printable string. -// This adds basic escaping which is sufficient for debug output, but -// probably not for shell interpretation. This essentially reverses -// ParseCommandLine. -func PrintableCommand(parts []string) string { - return strings.Join(f.Map(parts, printableArgument), " ") -} - -const ( - Ignore = 0 // Redirect to null - Show = 1 // Show on stdout/stderr as normal - ShowIfVerbose = 2 // Show if verbose is set, Ignore otherwise - Capture = 3 // Capture into buffer -) - -func ExecCommand(ctx *types.Context, command *executils.Process, stdout int, stderr int) ([]byte, []byte, error) { - if ctx.Verbose { - ctx.Info(PrintableCommand(command.GetArgs())) - } - - stdoutBuffer := &bytes.Buffer{} - if stdout == Capture { - command.RedirectStdoutTo(stdoutBuffer) - } else if stdout == Show || (stdout == ShowIfVerbose && ctx.Verbose) { - if ctx.Stdout != nil { - command.RedirectStdoutTo(ctx.Stdout) - } else { - command.RedirectStdoutTo(os.Stdout) - } - } - - stderrBuffer := &bytes.Buffer{} - if stderr == Capture { - command.RedirectStderrTo(stderrBuffer) - } else if stderr == Show || (stderr == ShowIfVerbose && ctx.Verbose) { - if ctx.Stderr != nil { - command.RedirectStderrTo(ctx.Stderr) - } else { - command.RedirectStderrTo(os.Stderr) - } - } - - err := command.Start() - if err != nil { - return nil, nil, errors.WithStack(err) - } - - err = command.Wait() - return stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err) -} - -type loggerAction struct { - onlyIfVerbose bool - warn bool - msg string -} - -func (l *loggerAction) Run(ctx *types.Context) error { - if !l.onlyIfVerbose || ctx.Verbose { - if l.warn { - ctx.Warn(l.msg) - } else { - ctx.Info(l.msg) - } - } - return nil -} - -func LogIfVerbose(warn bool, msg string) types.Command { - return &loggerAction{onlyIfVerbose: true, warn: warn, msg: msg} -} diff --git a/legacy/builder/wipeout_build_path_if_build_options_changed.go b/legacy/builder/wipeout_build_path_if_build_options_changed.go index 2a6873de97b..72b00b8671a 100644 --- a/legacy/builder/wipeout_build_path_if_build_options_changed.go +++ b/legacy/builder/wipeout_build_path_if_build_options_changed.go @@ -19,7 +19,7 @@ import ( "encoding/json" "path/filepath" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/go-paths-helper" @@ -66,9 +66,9 @@ func (s *WipeoutBuildPathIfBuildOptionsChanged) Run(ctx *types.Context) error { coreFolder := buildProperties.GetPath("build.core.path") realCoreFolder := coreFolder.Parent().Parent() jsonPath := ctx.BuildPath.Join(constants.BUILD_OPTIONS_FILE) - coreUnchanged, _ := builder_utils.DirContentIsOlderThan(realCoreFolder, jsonPath, ".txt") + coreUnchanged, _ := utils.DirContentIsOlderThan(realCoreFolder, jsonPath, ".txt") if coreUnchanged && targetCoreFolder != nil && !realCoreFolder.EqualsTo(targetCoreFolder) { - coreUnchanged, _ = builder_utils.DirContentIsOlderThan(targetCoreFolder, jsonPath, ".txt") + coreUnchanged, _ = utils.DirContentIsOlderThan(targetCoreFolder, jsonPath, ".txt") } if coreUnchanged { return nil