Improved compile speed by running multi-threaded library discovery.

commands/service_compile.go

@@ -168,7 +168,7 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 	if buildPathArg := req.GetBuildPath(); buildPathArg != "" {
 		buildPath = paths.New(req.GetBuildPath()).Canonical()
 		if in, _ := buildPath.IsInsideDir(sk.FullPath); in && buildPath.IsDir() {
-			if sk.AdditionalFiles, err = removeBuildFromSketchFiles(sk.AdditionalFiles, buildPath); err != nil {
+			if sk.AdditionalFiles, err = removeBuildPathFromSketchFiles(sk.AdditionalFiles, buildPath); err != nil {
 				return err
 			}
 		}
@@ -219,10 +219,6 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 		return err
 	}
 
-	actualPlatform := buildPlatform
-	otherLibrariesDirs := paths.NewPathList(req.GetLibraries()...)
-	otherLibrariesDirs.Add(s.settings.LibrariesDir())
-
 	var libsManager *librariesmanager.LibrariesManager
 	if pme.GetProfile() != nil {
 		libsManager = lm
@@ -252,6 +248,11 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 	if req.GetVerbose() {
 		verbosity = logger.VerbosityVerbose
 	}
+
+	librariesDirs := paths.NewPathList(req.GetLibraries()...) // Array of collection of libraries directories
+	librariesDirs.Add(s.settings.LibrariesDir())
+	libraryDirs := paths.NewPathList(req.GetLibrary()...) // Array of single-library directories
+
 	sketchBuilder, err := builder.NewBuilder(
 		ctx,
 		sk,
@@ -263,16 +264,16 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 		int(req.GetJobs()),
 		req.GetBuildProperties(),
 		s.settings.HardwareDirectories(),
-		otherLibrariesDirs,
+		librariesDirs,
 		s.settings.IDEBuiltinLibrariesDir(),
 		fqbn,
 		req.GetClean(),
 		req.GetSourceOverride(),
 		req.GetCreateCompilationDatabaseOnly(),
-		targetPlatform, actualPlatform,
+		targetPlatform, buildPlatform,
 		req.GetSkipLibrariesDiscovery(),
 		libsManager,
-		paths.NewPathList(req.GetLibrary()...),
+		libraryDirs,
 		outStream, errStream, verbosity, req.GetWarnings(),
 		progressCB,
 		pme.GetEnvVarsForSpawnedProcess(),
@@ -461,15 +462,15 @@ func maybePurgeBuildCache(compilationsBeforePurge uint, cacheTTL time.Duration)
 	buildcache.New(paths.TempDir().Join("arduino", "sketches")).Purge(cacheTTL)
 }
 
-// removeBuildFromSketchFiles removes the files contained in the build directory from
+// removeBuildPathFromSketchFiles removes the files contained in the build directory from
 // the list of the sketch files
-func removeBuildFromSketchFiles(files paths.PathList, build *paths.Path) (paths.PathList, error) {
+func removeBuildPathFromSketchFiles(files paths.PathList, build *paths.Path) (paths.PathList, error) {
 	var res paths.PathList
 	ignored := false
 	for _, file := range files {
 		if isInside, _ := file.IsInsideDir(build); !isInside {
-			res = append(res, file)
-		} else if !ignored {
+			res.Add(file)
+		} else {
 			ignored = true
 		}
 	}

commands/service_monitor.go

@@ -27,7 +27,6 @@ import (
 	"github.com/arduino/arduino-cli/internal/arduino/cores"
 	"github.com/arduino/arduino-cli/internal/arduino/cores/packagemanager"
 	pluggableMonitor "github.com/arduino/arduino-cli/internal/arduino/monitor"
-	"github.com/arduino/arduino-cli/internal/i18n"
 	"github.com/arduino/arduino-cli/pkg/fqbn"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	"github.com/arduino/go-properties-orderedmap"
@@ -265,10 +264,7 @@ func findMonitorAndSettingsForProtocolAndBoard(pme *packagemanager.Explorer, pro
 		} else if recipe, ok := boardPlatform.MonitorsDevRecipes[protocol]; ok {
 			// If we have a recipe we must resolve it
 			cmdLine := boardProperties.ExpandPropsInString(recipe)
-			cmdArgs, err := properties.SplitQuotedString(cmdLine, `"'`, false)
-			if err != nil {
-				return nil, nil, &cmderrors.InvalidArgumentError{Message: i18n.Tr("Invalid recipe in platform.txt"), Cause: err}
-			}
+			cmdArgs, _ := properties.SplitQuotedString(cmdLine, `"'`, false)
 			id := fmt.Sprintf("%s-%s", boardPlatform, protocol)
 			return pluggableMonitor.New(id, cmdArgs...), boardSettings, nil
 		}

commands/service_upload.go

@@ -720,10 +720,7 @@ func runTool(ctx context.Context, recipeID string, props *properties.Map, outStr
 		return errors.New(i18n.Tr("no upload port provided"))
 	}
 	cmdLine := props.ExpandPropsInString(recipe)
-	cmdArgs, err := properties.SplitQuotedString(cmdLine, `"'`, false)
-	if err != nil {
-		return errors.New(i18n.Tr("invalid recipe '%[1]s': %[2]s", recipe, err))
-	}
+	cmdArgs, _ := properties.SplitQuotedString(cmdLine, `"'`, false)
 
 	// Run Tool
 	logrus.WithField("phase", "upload").Tracef("Executing upload tool: %s", cmdLine)

internal/arduino/builder/builder.go

@@ -59,9 +59,6 @@ type Builder struct {
 	// Parallel processes
 	jobs int
 
-	// Custom build properties defined by user (line by line as "key=value" pairs)
-	customBuildProperties []string
-
 	// core related
 	coreBuildCachePath       *paths.Path
 	extraCoreBuildCachePaths paths.PathList
@@ -89,7 +86,7 @@ type Builder struct {
 	lineOffset int
 
 	targetPlatform *cores.PlatformRelease
-	actualPlatform *cores.PlatformRelease
+	buildPlatform  *cores.PlatformRelease
 
 	buildArtifacts *buildArtifacts
 
@@ -125,19 +122,20 @@ func NewBuilder(
 	coreBuildCachePath *paths.Path,
 	extraCoreBuildCachePaths paths.PathList,
 	jobs int,
-	requestBuildProperties []string,
-	hardwareDirs, otherLibrariesDirs paths.PathList,
+	customBuildProperties []string,
+	hardwareDirs paths.PathList,
+	librariesDirs paths.PathList,
 	builtInLibrariesDirs *paths.Path,
 	fqbn *fqbn.FQBN,
 	clean bool,
 	sourceOverrides map[string]string,
 	onlyUpdateCompilationDatabase bool,
-	targetPlatform, actualPlatform *cores.PlatformRelease,
+	targetPlatform, buildPlatform *cores.PlatformRelease,
 	useCachedLibrariesResolution bool,
 	librariesManager *librariesmanager.LibrariesManager,
-	libraryDirs paths.PathList,
+	customLibraryDirs paths.PathList,
 	stdout, stderr io.Writer, verbosity logger.Verbosity, warningsLevel string,
-	progresCB rpc.TaskProgressCB,
+	progressCB rpc.TaskProgressCB,
 	toolEnv []string,
 ) (*Builder, error) {
 	buildProperties := properties.NewMap()
@@ -146,14 +144,12 @@ func NewBuilder(
 	}
 	if sk != nil {
 		buildProperties.SetPath("sketch_path", sk.FullPath)
+		buildProperties.Set("build.project_name", sk.MainFile.Base())
+		buildProperties.SetPath("build.source.path", sk.FullPath)
 	}
 	if buildPath != nil {
 		buildProperties.SetPath("build.path", buildPath)
 	}
-	if sk != nil {
-		buildProperties.Set("build.project_name", sk.MainFile.Base())
-		buildProperties.SetPath("build.source.path", sk.FullPath)
-	}
 	if optimizeForDebug {
 		if debugFlags, ok := buildProperties.GetOk("compiler.optimization_flags.debug"); ok {
 			buildProperties.Set("compiler.optimization_flags", debugFlags)
@@ -165,12 +161,11 @@ func NewBuilder(
 	}
 
 	// Add user provided custom build properties
-	customBuildProperties, err := properties.LoadFromSlice(requestBuildProperties)
-	if err != nil {
+	if p, err := properties.LoadFromSlice(customBuildProperties); err == nil {
+		buildProperties.Merge(p)
+	} else {
 		return nil, fmt.Errorf("invalid build properties: %w", err)
 	}
-	buildProperties.Merge(customBuildProperties)
-	customBuildPropertiesArgs := append(requestBuildProperties, "build.warn_data_percentage=75")
 
 	sketchBuildPath, err := buildPath.Join("sketch").Abs()
 	if err != nil {
@@ -190,16 +185,20 @@ func NewBuilder(
 	}
 
 	log := logger.New(stdout, stderr, verbosity, warningsLevel)
-	libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader(
-		useCachedLibrariesResolution, librariesManager,
-		builtInLibrariesDirs, libraryDirs, otherLibrariesDirs,
-		actualPlatform, targetPlatform,
+	libsResolver, libsLoadingWarnings, err := detector.LibrariesLoader(
+		useCachedLibrariesResolution,
+		librariesManager,
+		builtInLibrariesDirs,
+		customLibraryDirs,
+		librariesDirs,
+		buildPlatform,
+		targetPlatform,
 	)
 	if err != nil {
 		return nil, err
 	}
 	if log.VerbosityLevel() == logger.VerbosityVerbose {
-		log.Warn(string(verboseOut))
+		log.Warn(string(libsLoadingWarnings))
 	}
 
 	diagnosticStore := diagnostics.NewStore()
@@ -212,25 +211,26 @@ func NewBuilder(
 		coreBuildPath:                 coreBuildPath,
 		librariesBuildPath:            librariesBuildPath,
 		jobs:                          jobs,
-		customBuildProperties:         customBuildPropertiesArgs,
 		coreBuildCachePath:            coreBuildCachePath,
 		extraCoreBuildCachePaths:      extraCoreBuildCachePaths,
 		logger:                        log,
 		clean:                         clean,
 		sourceOverrides:               sourceOverrides,
 		onlyUpdateCompilationDatabase: onlyUpdateCompilationDatabase,
 		compilationDatabase:           compilation.NewDatabase(buildPath.Join("compile_commands.json")),
-		Progress:                      progress.New(progresCB),
+		Progress:                      progress.New(progressCB),
 		executableSectionsSize:        []ExecutableSectionSize{},
 		buildArtifacts:                &buildArtifacts{},
 		targetPlatform:                targetPlatform,
-		actualPlatform:                actualPlatform,
+		buildPlatform:                 buildPlatform,
 		toolEnv:                       toolEnv,
 		buildOptions: newBuildOptions(
-			hardwareDirs, otherLibrariesDirs,
-			builtInLibrariesDirs, buildPath,
+			hardwareDirs,
+			librariesDirs,
+			builtInLibrariesDirs,
+			buildPath,
 			sk,
-			customBuildPropertiesArgs,
+			customBuildProperties,
 			fqbn,
 			clean,
 			buildProperties.Get("compiler.optimization_flags"),
@@ -239,7 +239,7 @@ func NewBuilder(
 		),
 		diagnosticStore: diagnosticStore,
 		libsDetector: detector.NewSketchLibrariesDetector(
-			libsManager, libsResolver,
+			libsResolver,
 			useCachedLibrariesResolution,
 			onlyUpdateCompilationDatabase,
 			log,
@@ -322,10 +322,19 @@ func (b *Builder) preprocess() error {
 		b.librariesBuildPath,
 		b.buildProperties,
 		b.targetPlatform.Platform.Architecture,
+		b.jobs,
 	)
 	if err != nil {
 		return err
 	}
+	if b.libsDetector.IncludeFoldersChanged() && b.librariesBuildPath.Exist() {
+		if b.logger.VerbosityLevel() == logger.VerbosityVerbose {
+			b.logger.Info(i18n.Tr("The list of included libraries has been changed... rebuilding all libraries."))
+		}
+		if err := b.librariesBuildPath.RemoveAll(); err != nil {
+			return err
+		}
+	}
 	b.Progress.CompleteStep()
 
 	b.warnAboutArchIncompatibleLibraries(b.libsDetector.ImportedLibraries())
@@ -492,29 +501,26 @@ func (b *Builder) prepareCommandForRecipe(buildProperties *properties.Map, recip
 		commandLine = properties.DeleteUnexpandedPropsFromString(commandLine)
 	}
 
-	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
-	if err != nil {
-		return nil, err
-	}
+	args, _ := properties.SplitQuotedString(commandLine, `"'`, false)
 
 	// if the overall commandline is too long for the platform
 	// try reducing the length by making the filenames relative
 	// and changing working directory to build.path
 	var relativePath string
 	if len(commandLine) > 30000 {
 		relativePath = buildProperties.Get("build.path")
-		for i, arg := range parts {
+		for i, arg := range args {
 			if _, err := os.Stat(arg); os.IsNotExist(err) {
 				continue
 			}
 			rel, err := filepath.Rel(relativePath, arg)
 			if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) {
-				parts[i] = rel
+				args[i] = rel
 			}
 		}
 	}
 
-	command, err := paths.NewProcess(b.toolEnv, parts...)
+	command, err := paths.NewProcess(b.toolEnv, args...)
 	if err != nil {
 		return nil, err
 	}

internal/arduino/builder/compilation.go

@@ -111,78 +111,79 @@ func (b *Builder) compileFiles(
 	return objectFiles, nil
 }
 
-// CompileFilesRecursive fixdoc
 func (b *Builder) compileFileWithRecipe(
 	sourcePath *paths.Path,
 	source *paths.Path,
 	buildPath *paths.Path,
 	includes []string,
 	recipe string,
 ) (*paths.Path, error) {
-	properties := b.buildProperties.Clone()
-	properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+b.logger.WarningsLevel()))
-	properties.Set("includes", strings.Join(includes, " "))
-	properties.SetPath("source_file", source)
 	relativeSource, err := sourcePath.RelTo(source)
 	if err != nil {
 		return nil, err
 	}
 	depsFile := buildPath.Join(relativeSource.String() + ".d")
 	objectFile := buildPath.Join(relativeSource.String() + ".o")
-
-	properties.SetPath("object_file", objectFile)
-	err = objectFile.Parent().MkdirAll()
-	if err != nil {
-		return nil, err
-	}
-
-	objIsUpToDate, err := utils.ObjFileIsUpToDate(source, objectFile, depsFile)
-	if err != nil {
+	if err := objectFile.Parent().MkdirAll(); err != nil {
 		return nil, err
 	}
 
+	properties := b.buildProperties.Clone()
+	properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+b.logger.WarningsLevel()))
+	properties.Set("includes", strings.Join(includes, " "))
+	properties.SetPath("source_file", source)
+	properties.SetPath("object_file", objectFile)
 	command, err := b.prepareCommandForRecipe(properties, recipe, false)
 	if err != nil {
 		return nil, err
 	}
 	if b.compilationDatabase != nil {
 		b.compilationDatabase.Add(source, command)
 	}
-	if !objIsUpToDate && !b.onlyUpdateCompilationDatabase {
-		commandStdout, commandStderr := &bytes.Buffer{}, &bytes.Buffer{}
-		command.RedirectStdoutTo(commandStdout)
-		command.RedirectStderrTo(commandStderr)
 
+	objIsUpToDate, err := utils.ObjFileIsUpToDate(source, objectFile, depsFile)
+	if err != nil {
+		return nil, err
+	}
+	if objIsUpToDate {
 		if b.logger.VerbosityLevel() == logger.VerbosityVerbose {
-			b.logger.Info(utils.PrintableCommand(command.GetArgs()))
-		}
-		// Since this compile could be multithreaded, we first capture the command output
-		if err := command.Start(); err != nil {
-			return nil, err
+			b.logger.Info(i18n.Tr("Using previously compiled file: %[1]s", objectFile))
 		}
-		err := command.Wait()
-		// and transfer all at once at the end...
+		return objectFile, nil
+	}
+	if b.onlyUpdateCompilationDatabase {
 		if b.logger.VerbosityLevel() == logger.VerbosityVerbose {
-			b.logger.WriteStdout(commandStdout.Bytes())
+			b.logger.Info(i18n.Tr("Skipping compile of: %[1]s", objectFile))
 		}
-		b.logger.WriteStderr(commandStderr.Bytes())
+		return objectFile, nil
+	}
 
-		// Parse the output of the compiler to gather errors and warnings...
-		if b.diagnosticStore != nil {
-			b.diagnosticStore.Parse(command.GetArgs(), commandStdout.Bytes())
-			b.diagnosticStore.Parse(command.GetArgs(), commandStderr.Bytes())
-		}
+	commandStdout, commandStderr := &bytes.Buffer{}, &bytes.Buffer{}
+	command.RedirectStdoutTo(commandStdout)
+	command.RedirectStderrTo(commandStderr)
+	if b.logger.VerbosityLevel() == logger.VerbosityVerbose {
+		b.logger.Info(utils.PrintableCommand(command.GetArgs()))
+	}
+	// Since this compile could be multithreaded, we first capture the command output
+	if err := command.Start(); err != nil {
+		return nil, err
+	}
+	err = command.Wait()
+	// and transfer all at once at the end...
+	if b.logger.VerbosityLevel() == logger.VerbosityVerbose {
+		b.logger.WriteStdout(commandStdout.Bytes())
+	}
+	b.logger.WriteStderr(commandStderr.Bytes())
 
-		// ...and then return the error
-		if err != nil {
-			return nil, err
-		}
-	} else if b.logger.VerbosityLevel() == logger.VerbosityVerbose {
-		if objIsUpToDate {
-			b.logger.Info(i18n.Tr("Using previously compiled file: %[1]s", objectFile))
-		} else {
-			b.logger.Info(i18n.Tr("Skipping compile of: %[1]s", objectFile))
-		}
+	// Parse the output of the compiler to gather errors and warnings...
+	if b.diagnosticStore != nil {
+		b.diagnosticStore.Parse(command.GetArgs(), commandStdout.Bytes())
+		b.diagnosticStore.Parse(command.GetArgs(), commandStderr.Bytes())
+	}
+
+	// ...and then return the error
+	if err != nil {
+		return nil, err
 	}
 
 	return objectFile, nil

internal/arduino/builder/core.go

@@ -164,7 +164,7 @@ func (b *Builder) compileCore() (*paths.Path, paths.PathList, error) {
 				b.logger.Info(i18n.Tr("Archiving built core (caching) in: %[1]s", targetArchivedCore))
 			} else if os.IsNotExist(err) {
 				b.logger.Info(i18n.Tr("Unable to cache built core, please tell %[1]s maintainers to follow %[2]s",
-					b.actualPlatform,
+					b.buildPlatform,
 					"https://arduino.github.io/arduino-cli/latest/platform-specification/#recipes-to-build-the-corea-archive-file"))
 			} else {
 				b.logger.Info(i18n.Tr("Error archiving built core (caching) in %[1]s: %[2]s", targetArchivedCore, err))

internal/arduino/builder/internal/detector/cache.go

@@ -0,0 +1,132 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2024 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package detector
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
+	"github.com/arduino/go-paths-helper"
+)
+
+type detectorCache struct {
+	curr    int
+	entries []*detectorCacheEntry
+}
+
+type detectorCacheEntry struct {
+	AddedIncludePath *paths.Path  `json:"added_include_path,omitempty"`
+	Compile          *sourceFile  `json:"compile,omitempty"`
+	CompileTask      *runner.Task `json:"compile_task,omitempty"`
+	MissingIncludeH  *string      `json:"missing_include_h,omitempty"`
+}
+
+func (e *detectorCacheEntry) String() string {
+	if e.AddedIncludePath != nil {
+		return "Added include path: " + e.AddedIncludePath.String()
+	}
+	if e.Compile != nil && e.CompileTask != nil {
+		return "Compiling: " + e.Compile.String() + " / " + e.CompileTask.String()
+	}
+	if e.MissingIncludeH != nil {
+		if *e.MissingIncludeH == "" {
+			return "No missing include files detected"
+		}
+		return "Missing include file: " + *e.MissingIncludeH
+	}
+	return "No operation"
+}
+
+func (e *detectorCacheEntry) Equals(entry *detectorCacheEntry) bool {
+	return e.String() == entry.String()
+}
+
+func newDetectorCache() *detectorCache {
+	return &detectorCache{}
+}
+
+func (c *detectorCache) String() string {
+	res := ""
+	for _, entry := range c.entries {
+		res += fmt.Sprintln(entry)
+	}
+	return res
+}
+
+// Load reads a saved cache from the given file.
+// If the file do not exists, it does nothing.
+func (c *detectorCache) Load(cacheFile *paths.Path) error {
+	if exist, err := cacheFile.ExistCheck(); err != nil {
+		return err
+	} else if !exist {
+		return nil
+	}
+	data, err := cacheFile.ReadFile()
+	if err != nil {
+		return err
+	}
+	var entries []*detectorCacheEntry
+	if err := json.Unmarshal(data, &entries); err != nil {
+		return err
+	}
+	c.curr = 0
+	c.entries = entries
+	return nil
+}
+
+// Expect adds an entry to the cache and checks if it matches the next expected entry.
+func (c *detectorCache) Expect(entry *detectorCacheEntry) {
+	if c.curr < len(c.entries) {
+		if c.entries[c.curr].Equals(entry) {
+			// Cache hit, move to the next entry
+			c.curr++
+			return
+		}
+		// Cache mismatch, invalidate and cut the remainder of the cache
+		c.entries = c.entries[:c.curr]
+	}
+	c.curr++
+	c.entries = append(c.entries, entry)
+}
+
+// Peek returns the next cache entry to be expected or nil if the cache is fully consumed.
+func (c *detectorCache) Peek() *detectorCacheEntry {
+	if c.curr < len(c.entries) {
+		return c.entries[c.curr]
+	}
+	return nil
+}
+
+// EntriesAhead returns the entries that are ahead of the current cache position.
+func (c *detectorCache) EntriesAhead() []*detectorCacheEntry {
+	if c.curr < len(c.entries) {
+		return c.entries[c.curr:]
+	}
+	return nil
+}
+
+// Save writes the current cache to the given file.
+func (c *detectorCache) Save(cacheFile *paths.Path) error {
+	// Cut off the cache if it is not fully consumed
+	c.entries = c.entries[:c.curr]
+
+	data, err := json.MarshalIndent(c.entries, "", "  ")
+	if err != nil {
+		return err
+	}
+	return cacheFile.WriteFile(data)
+}

internal/arduino/builder/internal/detector/source_file.go

@@ -0,0 +1,115 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2024 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package detector
+
+import (
+	"fmt"
+	"slices"
+
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/utils"
+	"github.com/arduino/go-paths-helper"
+)
+
+type sourceFile struct {
+	// SourcePath is the path to the source file
+	SourcePath *paths.Path `json:"source_path"`
+
+	// ObjectPath is the path to the object file that will be generated
+	ObjectPath *paths.Path `json:"object_path"`
+
+	// DepfilePath is the path to the dependency file that will be generated
+	DepfilePath *paths.Path `json:"depfile_path"`
+
+	// ExtraIncludePath contains an extra include path that must be
+	// used to compile this source file.
+	// This is mainly used for source files that comes from old-style libraries
+	// (Arduino IDE <1.5) requiring an extra include path to the "utility" folder.
+	ExtraIncludePath *paths.Path `json:"extra_include_path,omitempty"`
+}
+
+func (f *sourceFile) String() string {
+	return fmt.Sprintf("SourcePath:%s SourceRoot:%s BuildRoot:%s ExtraInclude:%s",
+		f.SourcePath, f.ObjectPath, f.DepfilePath, f.ExtraIncludePath)
+}
+
+// Equals checks if a sourceFile is equal to another.
+func (f *sourceFile) Equals(g *sourceFile) bool {
+	return f.SourcePath.EqualsTo(g.SourcePath) &&
+		f.ObjectPath.EqualsTo(g.ObjectPath) &&
+		f.DepfilePath.EqualsTo(g.DepfilePath) &&
+		((f.ExtraIncludePath == nil && g.ExtraIncludePath == nil) ||
+			(f.ExtraIncludePath != nil && g.ExtraIncludePath != nil && f.ExtraIncludePath.EqualsTo(g.ExtraIncludePath)))
+}
+
+// makeSourceFile create a sourceFile object for the given source file path.
+// The given sourceFilePath can be absolute, or relative within the sourceRoot root folder.
+func makeSourceFile(sourceRoot, buildRoot, sourceFilePath *paths.Path, extraIncludePaths ...*paths.Path) (*sourceFile, error) {
+	if len(extraIncludePaths) > 1 {
+		panic("only one extra include path allowed")
+	}
+	var extraIncludePath *paths.Path
+	if len(extraIncludePaths) > 0 {
+		extraIncludePath = extraIncludePaths[0]
+	}
+
+	if sourceFilePath.IsAbs() {
+		var err error
+		sourceFilePath, err = sourceRoot.RelTo(sourceFilePath)
+		if err != nil {
+			return nil, err
+		}
+	}
+	res := &sourceFile{
+		SourcePath:       sourceRoot.JoinPath(sourceFilePath),
+		ObjectPath:       buildRoot.Join(sourceFilePath.String() + ".o"),
+		DepfilePath:      buildRoot.Join(sourceFilePath.String() + ".d"),
+		ExtraIncludePath: extraIncludePath,
+	}
+	return res, nil
+}
+
+// ObjFileIsUpToDate checks if the compile object file is up to date.
+func (f *sourceFile) ObjFileIsUpToDate() (unchanged bool, err error) {
+	return utils.ObjFileIsUpToDate(f.SourcePath, f.ObjectPath, f.DepfilePath)
+}
+
+// uniqueSourceFileQueue is a queue of source files that does not allow duplicates.
+type uniqueSourceFileQueue []*sourceFile
+
+// Push adds a source file to the queue if it is not already present.
+func (queue *uniqueSourceFileQueue) Push(value *sourceFile) {
+	if !queue.Contains(value) {
+		*queue = append(*queue, value)
+	}
+}
+
+// Contains checks if the queue Contains a source file.
+func (queue uniqueSourceFileQueue) Contains(target *sourceFile) bool {
+	return slices.ContainsFunc(queue, target.Equals)
+}
+
+// Pop removes and returns the first element of the queue.
+func (queue *uniqueSourceFileQueue) Pop() *sourceFile {
+	old := *queue
+	x := old[0]
+	*queue = old[1:]
+	return x
+}
+
+// Empty returns true if the queue is Empty.
+func (queue uniqueSourceFileQueue) Empty() bool {
+	return len(queue) == 0
+}

internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go

@@ -22,6 +22,7 @@ import (
 	"path/filepath"
 	"runtime"
 
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/utils"
 	"github.com/arduino/arduino-cli/internal/arduino/sketch"
 	"github.com/arduino/arduino-cli/internal/i18n"
@@ -35,7 +36,7 @@ func PreprocessSketchWithArduinoPreprocessor(
 	ctx context.Context,
 	sk *sketch.Sketch, buildPath *paths.Path, includeFolders paths.PathList,
 	lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool,
-) (*Result, error) {
+) (*runner.Result, error) {
 	verboseOut := &bytes.Buffer{}
 	normalOut := &bytes.Buffer{}
 	if err := buildPath.Join("preproc").MkdirAll(); err != nil {
@@ -44,52 +45,48 @@ func PreprocessSketchWithArduinoPreprocessor(
 
 	sourceFile := buildPath.Join("sketch", sk.MainFile.Base()+".cpp")
 	targetFile := buildPath.Join("preproc", "sketch_merged.cpp")
-	gccResult, err := GCC(ctx, sourceFile, targetFile, includeFolders, buildProperties)
-	verboseOut.Write(gccResult.Stdout())
-	verboseOut.Write(gccResult.Stderr())
-	if err != nil {
-		return nil, err
+	gccResult := GCC(sourceFile, targetFile, includeFolders, buildProperties).Run(ctx)
+	verboseOut.Write(gccResult.Stdout)
+	verboseOut.Write(gccResult.Stderr)
+	if gccResult.Error != nil {
+		return nil, gccResult.Error
 	}
 
-	arduiniPreprocessorProperties := properties.NewMap()
-	arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}")
-	arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor")
-	arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.pattern", `"{cmd.path}" "{source_file}" -- -std=gnu++11`)
-	arduiniPreprocessorProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC")
-	arduiniPreprocessorProperties.Merge(buildProperties)
-	arduiniPreprocessorProperties.Merge(arduiniPreprocessorProperties.SubTree("tools").SubTree("arduino-preprocessor"))
-	arduiniPreprocessorProperties.SetPath("source_file", targetFile)
-	pattern := arduiniPreprocessorProperties.Get("pattern")
+	arduinoPreprocessorProperties := properties.NewMap()
+	arduinoPreprocessorProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}")
+	arduinoPreprocessorProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor")
+	arduinoPreprocessorProperties.Set("tools.arduino-preprocessor.pattern", `"{cmd.path}" "{source_file}" -- -std=gnu++11`)
+	arduinoPreprocessorProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC")
+	arduinoPreprocessorProperties.Merge(buildProperties)
+	arduinoPreprocessorProperties.Merge(arduinoPreprocessorProperties.SubTree("tools").SubTree("arduino-preprocessor"))
+	arduinoPreprocessorProperties.SetPath("source_file", targetFile)
+	pattern := arduinoPreprocessorProperties.Get("pattern")
 	if pattern == "" {
 		return nil, errors.New(i18n.Tr("arduino-preprocessor pattern is missing"))
 	}
 
-	commandLine := arduiniPreprocessorProperties.ExpandPropsInString(pattern)
-	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
-	if err != nil {
-		return nil, err
-	}
-
-	command, err := paths.NewProcess(nil, parts...)
+	commandLine := arduinoPreprocessorProperties.ExpandPropsInString(pattern)
+	args, _ := properties.SplitQuotedString(commandLine, `"'`, false)
+	command, err := paths.NewProcess(nil, args...)
 	if err != nil {
 		return nil, err
 	}
 	if runtime.GOOS == "windows" {
 		// chdir in the uppermost directory to avoid UTF-8 bug in clang (https://github.com/arduino/arduino-preprocessor/issues/2)
-		command.SetDir(filepath.VolumeName(parts[0]) + "/")
+		command.SetDir(filepath.VolumeName(args[0]) + "/")
 	}
 
 	verboseOut.WriteString(commandLine)
 	commandStdOut, commandStdErr, err := command.RunAndCaptureOutput(ctx)
 	verboseOut.Write(commandStdErr)
 	if err != nil {
-		return &Result{args: gccResult.Args(), stdout: verboseOut.Bytes(), stderr: normalOut.Bytes()}, err
+		return &runner.Result{Args: gccResult.Args, Stdout: verboseOut.Bytes(), Stderr: normalOut.Bytes()}, err
 	}
 	result := utils.NormalizeUTF8(commandStdOut)
 
 	destFile := buildPath.Join(sk.MainFile.Base() + ".cpp")
 	if err := destFile.WriteFile(result); err != nil {
-		return &Result{args: gccResult.Args(), stdout: verboseOut.Bytes(), stderr: normalOut.Bytes()}, err
+		return &runner.Result{Args: gccResult.Args, Stdout: verboseOut.Bytes(), Stderr: normalOut.Bytes()}, err
 	}
-	return &Result{args: gccResult.Args(), stdout: verboseOut.Bytes(), stderr: normalOut.Bytes()}, err
+	return &runner.Result{Args: gccResult.Args, Stdout: verboseOut.Bytes(), Stderr: normalOut.Bytes()}, err
 }

internal/arduino/builder/internal/preprocessor/ctags.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/arduino/arduino-cli/internal/arduino/builder/cpp"
 	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/preprocessor/internal/ctags"
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
 	"github.com/arduino/arduino-cli/internal/arduino/sketch"
 	"github.com/arduino/arduino-cli/internal/i18n"
 	"github.com/arduino/go-paths-helper"
@@ -43,7 +44,7 @@ func PreprocessSketchWithCtags(
 	sketch *sketch.Sketch, buildPath *paths.Path, includes paths.PathList,
 	lineOffset int, buildProperties *properties.Map,
 	onlyUpdateCompilationDatabase, verbose bool,
-) (*Result, error) {
+) (*runner.Result, error) {
 	// Create a temporary working directory
 	tmpDir, err := paths.MkTempDir("", "")
 	if err != nil {
@@ -56,30 +57,30 @@ func PreprocessSketchWithCtags(
 
 	// Run GCC preprocessor
 	sourceFile := buildPath.Join("sketch", sketch.MainFile.Base()+".cpp")
-	result, err := GCC(ctx, sourceFile, ctagsTarget, includes, buildProperties)
-	stdout.Write(result.Stdout())
-	stderr.Write(result.Stderr())
-	if err != nil {
+	result := GCC(sourceFile, ctagsTarget, includes, buildProperties).Run(ctx)
+	stdout.Write(result.Stdout)
+	stderr.Write(result.Stderr)
+	if err := result.Error; err != nil {
 		if !onlyUpdateCompilationDatabase {
-			return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+			return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 		}
 
 		// Do not bail out if we are generating the compile commands database
 		stderr.WriteString(fmt.Sprintf("%s: %s",
 			i18n.Tr("An error occurred adding prototypes"),
 			i18n.Tr("the compilation database may be incomplete or inaccurate")))
 		if err := sourceFile.CopyTo(ctagsTarget); err != nil {
-			return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+			return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 		}
 	}
 
 	if src, err := ctagsTarget.ReadFile(); err == nil {
 		filteredSource := filterSketchSource(sketch, bytes.NewReader(src), false)
 		if err := ctagsTarget.WriteFile([]byte(filteredSource)); err != nil {
-			return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+			return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 		}
 	} else {
-		return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+		return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 	}
 
 	// Run CTags on gcc-preprocessed source
@@ -89,7 +90,7 @@ func PreprocessSketchWithCtags(
 		stderr.Write(ctagsStdErr)
 	}
 	if err != nil {
-		return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+		return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 	}
 
 	// Parse CTags output
@@ -104,13 +105,13 @@ func PreprocessSketchWithCtags(
 	if sourceData, err := sourceFile.ReadFile(); err == nil {
 		source = string(sourceData)
 	} else {
-		return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+		return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 	}
 	source = strings.ReplaceAll(source, "\r\n", "\n")
 	source = strings.ReplaceAll(source, "\r", "\n")
 	sourceRows := strings.Split(source, "\n")
 	if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) {
-		return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, nil
+		return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, nil
 	}
 
 	insertionLine := firstFunctionLine + lineOffset - 1
@@ -136,7 +137,7 @@ func PreprocessSketchWithCtags(
 
 	// Write back arduino-preprocess output to the sourceFile
 	err = sourceFile.WriteFile([]byte(preprocessedSource))
-	return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err
+	return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err
 }
 
 func composePrototypeSection(line int, prototypes []*ctags.Prototype) string {

internal/arduino/builder/internal/preprocessor/gcc.go

@@ -16,13 +16,10 @@
 package preprocessor
 
 import (
-	"context"
-	"errors"
-	"fmt"
 	"strings"
 
 	"github.com/arduino/arduino-cli/internal/arduino/builder/cpp"
-	"github.com/arduino/arduino-cli/internal/i18n"
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
 	"github.com/arduino/go-paths-helper"
 	"github.com/arduino/go-properties-orderedmap"
 	"go.bug.st/f"
@@ -31,10 +28,9 @@ import (
 // GCC performs a run of the gcc preprocess (macro/includes expansion). The function outputs the result
 // to targetFilePath. Returns the stdout/stderr of gcc if any.
 func GCC(
-	ctx context.Context,
 	sourceFilePath, targetFilePath *paths.Path,
 	includes paths.PathList, buildProperties *properties.Map,
-) (Result, error) {
+) *runner.Task {
 	gccBuildProperties := properties.NewMap()
 	gccBuildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC")
 	gccBuildProperties.Merge(buildProperties)
@@ -58,29 +54,12 @@ func GCC(
 	}
 
 	pattern := gccBuildProperties.Get(gccPreprocRecipeProperty)
-	if pattern == "" {
-		return Result{}, errors.New(i18n.Tr("%s pattern is missing", gccPreprocRecipeProperty))
-	}
-
 	commandLine := gccBuildProperties.ExpandPropsInString(pattern)
 	commandLine = properties.DeleteUnexpandedPropsFromString(commandLine)
-	args, err := properties.SplitQuotedString(commandLine, `"'`, false)
-	if err != nil {
-		return Result{}, err
-	}
+	args, _ := properties.SplitQuotedString(commandLine, `"'`, false)
 
 	// Remove -MMD argument if present. Leaving it will make gcc try
 	// to create a /dev/null.d dependency file, which won't work.
 	args = f.Filter(args, f.NotEquals("-MMD"))
-
-	proc, err := paths.NewProcess(nil, args...)
-	if err != nil {
-		return Result{}, err
-	}
-	stdout, stderr, err := proc.RunAndCaptureOutput(ctx)
-
-	// Append gcc arguments to stdout
-	stdout = append([]byte(fmt.Sprintln(strings.Join(args, " "))), stdout...)
-
-	return Result{args: proc.GetArgs(), stdout: stdout, stderr: stderr}, err
+	return runner.NewTask(args...)
 }

internal/arduino/builder/internal/runner/runner.go

@@ -0,0 +1,117 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2024 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package runner
+
+import (
+	"context"
+	"runtime"
+	"sync"
+)
+
+// Runner is a helper to run commands in a queue, the commands are immediately exectuded
+// in a goroutine as they are enqueued. The runner can be stopped by calling Cancel.
+type Runner struct {
+	lock      sync.Mutex
+	queue     chan<- *enqueuedCommand
+	results   map[string]<-chan *Result
+	ctx       context.Context
+	ctxCancel func()
+	wg        sync.WaitGroup
+}
+
+type enqueuedCommand struct {
+	task   *Task
+	accept func(*Result)
+}
+
+func (cmd *enqueuedCommand) String() string {
+	return cmd.task.String()
+}
+
+// New creates a new Runner with the given number of workers.
+// If workers is 0, the number of workers will be the number of available CPUs.
+func New(inCtx context.Context, workers int) *Runner {
+	ctx, cancel := context.WithCancel(inCtx)
+	queue := make(chan *enqueuedCommand, 1000)
+	r := &Runner{
+		ctx:       ctx,
+		ctxCancel: cancel,
+		queue:     queue,
+		results:   map[string]<-chan *Result{},
+	}
+
+	// Spawn workers
+	if workers == 0 {
+		workers = runtime.NumCPU()
+	}
+	for i := 0; i < workers; i++ {
+		r.wg.Add(1)
+		go func() {
+			worker(ctx, queue)
+			r.wg.Done()
+		}()
+	}
+
+	return r
+}
+
+func worker(ctx context.Context, queue <-chan *enqueuedCommand) {
+	done := ctx.Done()
+	for {
+		select {
+		case <-done:
+			return
+		default:
+		}
+
+		select {
+		case <-done:
+			return
+		case cmd := <-queue:
+			result := cmd.task.Run(ctx)
+			cmd.accept(result)
+		}
+	}
+}
+
+func (r *Runner) Enqueue(task *Task) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	result := make(chan *Result, 1)
+	r.results[task.String()] = result
+	r.queue <- &enqueuedCommand{
+		task: task,
+		accept: func(res *Result) {
+			result <- res
+		},
+	}
+}
+
+func (r *Runner) Results(task *Task) *Result {
+	r.lock.Lock()
+	result, ok := r.results[task.String()]
+	r.lock.Unlock()
+	if !ok {
+		return nil
+	}
+	return <-result
+}
+
+func (r *Runner) Cancel() {
+	r.ctxCancel()
+	r.wg.Wait()
+}

internal/arduino/builder/internal/runner/runner_test.go

@@ -0,0 +1,53 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2024 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package runner_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRunMultipleTask(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	r := runner.New(ctx, 0)
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 1 ; echo -n 0"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 2 ; echo -n 1"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 3 ; echo -n 2"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 4 ; echo -n 3"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 5 ; echo -n 4"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 6 ; echo -n 5"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 7 ; echo -n 6"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 8 ; echo -n 7"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 9 ; echo -n 8"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 10 ; echo -n 9"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 11 ; echo -n 10"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 12 ; echo -n 11"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 13 ; echo -n 12"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 14 ; echo -n 13"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 15 ; echo -n 14"))
+	r.Enqueue(runner.NewTask("bash", "-c", "sleep 16 ; echo -n 15"))
+	require.Nil(t, r.Results(runner.NewTask("bash", "-c", "echo -n 5")))
+	fmt.Println(string(r.Results(runner.NewTask("bash", "-c", "sleep 3 ; echo -n 2")).Stdout))
+	fmt.Println("Cancelling")
+	r.Cancel()
+	fmt.Println("Runner completed")
+}

internal/arduino/builder/internal/runner/task.go

@@ -0,0 +1,60 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2024 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package runner
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/arduino/go-paths-helper"
+)
+
+// Task is a command to be executed
+type Task struct {
+	Args []string `json:"args"`
+}
+
+// NewTask creates a new Task
+func NewTask(args ...string) *Task {
+	return &Task{Args: args}
+}
+
+func (t *Task) String() string {
+	return strings.Join(t.Args, " ")
+}
+
+// Result contains the output of a command execution
+type Result struct {
+	Args   []string
+	Stdout []byte
+	Stderr []byte
+	Error  error
+}
+
+// Run executes the command and returns the result
+func (t *Task) Run(ctx context.Context) *Result {
+	proc, err := paths.NewProcess(nil, t.Args...)
+	if err != nil {
+		return &Result{Args: t.Args, Error: err}
+	}
+	stdout, stderr, err := proc.RunAndCaptureOutput(ctx)
+
+	// Append arguments to stdout
+	stdout = append([]byte(fmt.Sprintln(t)), stdout...)
+
+	return &Result{Args: proc.GetArgs(), Stdout: stdout, Stderr: stderr, Error: err}
+}

internal/arduino/builder/preprocess_sketch.go

@@ -32,10 +32,10 @@ func (b *Builder) preprocessSketch(includes paths.PathList) error {
 	)
 	if result != nil {
 		if b.logger.VerbosityLevel() == logger.VerbosityVerbose {
-			b.logger.WriteStdout(result.Stdout())
+			b.logger.WriteStdout(result.Stdout)
 		}
-		b.logger.WriteStderr(result.Stderr())
-		b.diagnosticStore.Parse(result.Args(), result.Stderr())
+		b.logger.WriteStderr(result.Stderr)
+		b.diagnosticStore.Parse(result.Args, result.Stderr)
 	}
 
 	return err

internal/arduino/builder/sizer.go

@@ -199,15 +199,17 @@ func (b *Builder) checkSize() (ExecutablesFileSections, error) {
 		return executableSectionsSize, errors.New(i18n.Tr("data section exceeds available space in board"))
 	}
 
+	warnDataPercentage := 75
 	if w := properties.Get("build.warn_data_percentage"); w != "" {
-		warnDataPercentage, err := strconv.Atoi(w)
-		if err != nil {
-			return executableSectionsSize, err
-		}
-		if maxDataSize > 0 && dataSize > maxDataSize*warnDataPercentage/100 {
-			b.logger.Warn(i18n.Tr("Low memory available, stability problems may occur."))
+		if p, err := strconv.Atoi(w); err == nil {
+			warnDataPercentage = p
+		} else {
+			b.logger.Warn(i18n.Tr("Invalid value for build.warn_data_percentage: %s", w))
 		}
 	}
+	if maxDataSize > 0 && dataSize > maxDataSize*warnDataPercentage/100 {
+		b.logger.Warn(i18n.Tr("Low memory available, stability problems may occur."))
+	}
 
 	return executableSectionsSize, nil
 }

internal/arduino/cores/packagemanager/loader.go

@@ -721,11 +721,8 @@ func (pme *Explorer) loadDiscoveries(release *cores.PlatformRelease) []error {
 		}
 
 		cmd := configuration.ExpandPropsInString(pattern)
-		if cmdArgs, err := properties.SplitQuotedString(cmd, `"'`, true); err != nil {
-			merr = append(merr, err)
-		} else {
-			pme.discoveryManager.Add(discoveryID, cmdArgs...)
-		}
+		cmdArgs, _ := properties.SplitQuotedString(cmd, `"'`, true)
+		pme.discoveryManager.Add(discoveryID, cmdArgs...)
 	}
 
 	return merr

internal/integrationtest/compile_4/compile_test.go

@@ -1045,15 +1045,14 @@ func TestBuildOptionsFile(t *testing.T) {
 		"sketchLocation"
 	]`)
 	requirejson.Query(t, buildOptionsBytes, ".fqbn", `"arduino:avr:uno"`)
-	requirejson.Query(t, buildOptionsBytes, ".customBuildProperties", `"build.warn_data_percentage=75"`)
 
 	// Recompiling a second time should provide the same result
 	_, _, err = cli.Run("compile", "-b", "arduino:avr:uno", "--build-path", buildPath.String(), sketchPath.String())
 	require.NoError(t, err)
 
-	buildOptionsBytes, err = buildPath.Join("build.options.json").ReadFile()
+	buildOptionsBytes2, err := buildPath.Join("build.options.json").ReadFile()
 	require.NoError(t, err)
-	requirejson.Query(t, buildOptionsBytes, ".customBuildProperties", `"build.warn_data_percentage=75"`)
+	require.Equal(t, buildOptionsBytes, buildOptionsBytes2)
 
 	// Recompiling with a new build option must produce a new `build.options.json`
 	_, _, err = cli.Run("compile", "-b", "arduino:avr:uno", "--build-path", buildPath.String(),
@@ -1062,7 +1061,7 @@ func TestBuildOptionsFile(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	buildOptionsBytes, err = buildPath.Join("build.options.json").ReadFile()
+	buildOptionsBytes3, err := buildPath.Join("build.options.json").ReadFile()
 	require.NoError(t, err)
-	requirejson.Query(t, buildOptionsBytes, ".customBuildProperties", `"custom=prop,build.warn_data_percentage=75"`)
+	require.NotEqual(t, buildOptionsBytes, buildOptionsBytes3)
 }

internal/integrationtest/compile_4/lib_discovery_caching_test.go

@@ -0,0 +1,110 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package compile_test
+
+import (
+	"testing"
+
+	"github.com/arduino/arduino-cli/internal/integrationtest"
+	"github.com/arduino/go-paths-helper"
+	"github.com/stretchr/testify/require"
+	"go.bug.st/testifyjson/requirejson"
+)
+
+func TestLibDiscoveryCache(t *testing.T) {
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	t.Cleanup(env.CleanUp)
+
+	// Install Arduino AVR Boards
+	_, _, err := cli.Run("core", "install", "arduino:avr@1.8.6")
+	require.NoError(t, err)
+
+	// Copy the testdata sketchbook
+	testdata, err := paths.New("testdata", "libraries_discovery_caching").Abs()
+	require.NoError(t, err)
+	sketchbook := cli.SketchbookDir()
+	require.NoError(t, sketchbook.RemoveAll())
+	require.NoError(t, testdata.CopyDirTo(cli.SketchbookDir()))
+
+	buildpath, err := paths.MkTempDir("", "tmpbuildpath")
+	require.NoError(t, err)
+	t.Cleanup(func() { buildpath.RemoveAll() })
+
+	{
+		sketchA := sketchbook.Join("SketchA")
+		{
+			outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String())
+			require.NoError(t, err)
+			j := requirejson.Parse(t, outjson)
+			j.MustContain(`{"builder_result":{
+		"used_libraries": [
+			{ "name": "LibA" },
+        	{ "name": "LibB" }
+    	],
+	}}`)
+		}
+
+		// Update SketchA
+		require.NoError(t, sketchA.Join("SketchA.ino").WriteFile([]byte(`
+#include <LibC.h>
+#include <LibA.h>
+void setup() {}
+void loop() {libAFunction();}
+`)))
+
+		{
+			// This compile should FAIL!
+			outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String())
+			require.Error(t, err)
+			j := requirejson.Parse(t, outjson)
+			j.MustContain(`{
+"builder_result":{
+	"used_libraries": [
+		{ "name": "LibC" },
+		{ "name": "LibA" }
+	],
+    "diagnostics": [
+		{
+			"severity": "ERROR",
+			"message": "'libAFunction' was not declared in this scope\n void loop() {libAFunction();}\n              ^~~~~~~~~~~~"
+		}
+	]
+}}`)
+			j.Query(".compiler_out").MustContain(`"The list of included libraries has been changed... rebuilding all libraries."`)
+		}
+
+		{
+			// This compile should FAIL!
+			outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String())
+			require.Error(t, err)
+			j := requirejson.Parse(t, outjson)
+			j.MustContain(`{
+"builder_result":{
+	"used_libraries": [
+		{ "name": "LibC" },
+		{ "name": "LibA" }
+	],
+    "diagnostics": [
+		{
+			"severity": "ERROR",
+			"message": "'libAFunction' was not declared in this scope\n void loop() {libAFunction();}\n              ^~~~~~~~~~~~"
+		}
+	]
+}}`)
+			j.Query(".compiler_out").MustNotContain(`"The list of included libraries has changed... rebuilding all libraries."`)
+		}
+	}
+}

internal/integrationtest/compile_4/testdata/libraries_discovery_caching/SketchA/SketchA.ino

@@ -0,0 +1,3 @@
+#include <LibA.h>
+void setup() {}
+void loop() {libAFunction();}

internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/LibA.h

@@ -0,0 +1,6 @@
+
+#include <LibB.h>
+
+#ifndef CHECK
+void libAFunction();
+#endif

internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/file1.cpp

@@ -0,0 +1,5 @@
+#include <LibB.h>
+
+#ifndef CHECK
+void libAFunction() {}
+#endif

internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibC/LibB.h

@@ -0,0 +1,2 @@
+
+#define CHECK