8000 [breaking] daemon: Fix concurrency and streamline access to PackageManager by cmaglie · Pull Request #1828 · arduino/arduino-cli · GitHub
[go: up one dir, main page]
Skip to content

[breaking] daemon: Fix concurrency and streamline access to PackageManager #1828

New issue 8000

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
Aug 26, 2022
Merged

[breaking] daemon: Fix concurrency and streamline access to PackageManager #1828

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
7ce9fd4
legacy: Removed ToolsLoader step
cmaglie Aug 5, 2022
8bec66e
Fixed some function comments
cmaglie Aug 22, 2022
1d3b99e
Thread-safe protect access to instances map
cmaglie Aug 3, 2022
ea01f19
Removed state-altering methods from PackageManager
cmaglie Aug 5, 2022
1b8a35c
Created packagemanager.Explorer to query PackageManager data
cmaglie Aug 5, 2022
6d6210c
PlatformInstall/Uninstall must release PackageManager.Explorer before…
cmaglie Aug 6, 2022
3b39d7c
Moved commands.InstanceContainer -> rpc.InstanceCommand
cmaglie Aug 11, 2022
4db186e
Created a coreInstancesContainer
cmaglie Aug 11, 2022
dd0414f
Made CoreInstance.PackageManager field private
cmaglie Aug 11, 2022
007a0ff
Moved the reminder of PackageManager functions to Explorer or Builder
cmaglie Aug 18, 2022
316d1d3
Now GetPackageManager accepts an rpc.InstanceCommand
cmaglie Aug 21, 2022
dbd6b01
Now GetLibraryManager accepts an rpc.InstanceCommand
cmaglie Aug 21, 2022
6438e58
Refactored automatic builtin-tool installation
cmaglie Aug 21, 2022
e8ef962
Added gRPC LibraryUpgrade call and fixed 'lib upgrade' command
cmaglie Aug 21, 2022
46eeee2
Explorer and Builder should not extend PackageManager
cmaglie Aug 22, 2022
04e70c5
Updated documentation
cmaglie Aug 23, 2022
df28c7c
Apply suggestions from code review
cmaglie Aug 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Moved the reminder of PackageManager functions to Explorer or Builder
  • Loading branch information
@cmaglie
cmaglie committed Aug 23, 2022
commit 007a0ff665ef76e07d40cb5a0bd106de8d35db9b
26 changes: 13 additions & 13 deletions arduino/cores/packagemanager/download.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ func (platform *PlatformReference) String() string {

// FindPlatform returns the Platform matching the PlatformReference or nil if not found.
// The PlatformVersion field of the reference is ignored.
func (pm *PackageManager) FindPlatform(ref *PlatformReference) *cores.Platform {
targetPackage, ok := pm.packages[ref.Package]
func (pme *Explorer) FindPlatform(ref *PlatformReference) *cores.Platform {
targetPackage, ok := pme.packages[ref.Package]
if !ok {
return nil
}
Expand All @@ -56,8 +56,8 @@ func (pm *PackageManager) FindPlatform(ref *PlatformReference) *cores.Platform {
}

// FindPlatformRelease returns the PlatformRelease matching the PlatformReference or nil if not found
func (pm *PackageManager) FindPlatformRelease(ref *PlatformReference) *cores.PlatformRelease {
platform := pm.FindPlatform(ref)
func (pme *Explorer) FindPlatformRelease(ref *PlatformReference) *cores.PlatformRelease {
platform := pme.FindPlatform(ref)
if platform == nil {
return nil
}
Expand All @@ -70,8 +70,8 @@ func (pm *PackageManager) FindPlatformRelease(ref *PlatformReference) *cores.Pla

// FindPlatformReleaseDependencies takes a PlatformReference and returns a set of items to download and
// a set of outputs for non existing platforms.
func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReference) (*cores.PlatformRelease, []*cores.ToolRelease, error) {
targetPackage, exists := pm.packages[item.Package]
func (pme *Explorer) FindPlatformReleaseDependencies(item *PlatformReference) (*cores.PlatformRelease, []*cores.ToolRelease, error) {
targetPackage, exists := pme.packages[item.Package]
if !exists {
return nil, nil, fmt.Errorf(tr("package %s not found"), item.Package)
}
Expand All @@ -94,22 +94,22 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc
}

// replaces "latest" with latest version too
toolDeps, err := pm.packages.GetPlatformReleaseToolDependencies(release)
toolDeps, err := pme.packages.GetPlatformReleaseToolDependencies(release)
if err != nil {
return nil, nil, fmt.Errorf(tr("getting tool dependencies for platform %[1]s: %[2]s"), release.String(), err)
}

// discovery dependencies differ from normal tool since we always want to use the latest
// available version for the platform package
discoveryDependencies, err := pm.packages.GetPlatformReleaseDiscoveryDependencies(release)
discoveryDependencies, err := pme.packages.GetPlatformReleaseDiscoveryDependencies(release)
if err != nil {
return nil, nil, fmt.Errorf(tr("getting discovery dependencies for platform %[1]s: %[2]s"), release.String(), err)
}
toolDeps = append(toolDeps, discoveryDependencies...)

// monitor dependencies differ from normal tool since we always want to use the latest
// available version for the platform package
monitorDependencies, err := pm.packages.GetPlatformReleaseMonitorDependencies(release)
monitorDependencies, err := pme.packages.GetPlatformReleaseMonitorDependencies(release)
if err != nil {
return nil, nil, fmt.Errorf(tr("getting monitor dependencies for platform %[1]s: %[2]s"), release.String(), err)
}
Expand All @@ -120,14 +120,14 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc

// DownloadToolRelease downloads a ToolRelease. If the tool is already downloaded a nil Downloader
// is returned. Uses the given downloader configuration for download, or the default config if nil.
func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config, progressCB rpc.DownloadProgressCB) error {
func (pme *Explorer) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config, progressCB rpc.DownloadProgressCB) error {
resource := tool.GetCompatibleFlavour()
if resource == nil {
return &arduino.FailedDownloadError{
Message: tr("Error downloading tool %s", tool),
Cause: errors.New(tr("no versions available for the current OS"))}
}
if err := resource.Download(pm.DownloadDir, config, tool.String(), progressCB); err != nil {
if err := resource.Download(pme.DownloadDir, config, tool.String(), progressCB); err != nil {
return &arduino.FailedDownloadError{
Message: tr("Error downloading tool %s", tool),
Cause: err}
Expand All @@ -137,9 +137,9 @@ func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *d

// DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a
// nil Downloader is returned.
func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config, progressCB rpc.DownloadProgressCB) error {
func (pme *Explorer) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config, progressCB rpc.DownloadProgressCB) error {
if platform.Resource == nil {
return &arduino.PlatformNotFoundError{Platform: platform.String()}
}
return platform.Resource.Download(pm.DownloadDir, config, platform.String(), progressCB)
return platform.Resource.Download(pme.DownloadDir, config, platform.String(), progressCB)
}
46 changes: 23 additions & 23 deletions arduino/cores/packagemanager/install_uninstall.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,32 +185,32 @@ func (pme *Explorer) DownloadAndInstallPlatformAndTools(
}

// InstallPlatform installs a specific release of a platform.
func (pm *PackageManager) InstallPlatform(platformRelease *cores.PlatformRelease) error {
destDir := pm.PackagesDir.Join(
func (pme *Explorer) InstallPlatform(platformRelease *cores.PlatformRelease) error {
destDir := pme.PackagesDir.Join(
platformRelease.Platform.Package.Name,
"hardware",
platformRelease.Platform.Architecture,
platformRelease.Version.String())
return pm.InstallPlatformInDirectory(platformRelease, destDir)
return pme.InstallPlatformInDirectory(platformRelease, destDir)
}

// InstallPlatformInDirectory installs a specific release of a platform in a specific directory.
func (pm *PackageManager) InstallPlatformInDirectory(platformRelease *cores.PlatformRelease, destDir *paths.Path) error {
if err := platformRelease.Resource.Install(pm.DownloadDir, pm.tempDir, destDir); err != nil {
func (pme *Explorer) InstallPlatformInDirectory(platformRelease *cores.PlatformRelease, destDir *paths.Path) error {
if err := platformRelease.Resource.Install(pme.DownloadDir, pme.tempDir, destDir); err != nil {
return errors.Errorf(tr("installing platform %[1]s: %[2]s"), platformRelease, err)
}
if d, err := destDir.Abs(); err == nil {
platformRelease.InstallDir = d
} else {
return err
}
if err := pm.cacheInstalledJSON(platformRelease); err != nil {
if err := pme.cacheInstalledJSON(platformRelease); err != nil {
return errors.Errorf(tr("creating installed.json in %[1]s: %[2]s"), platformRelease.InstallDir, err)
}
return nil
}

func (pm *PackageManager) cacheInstalledJSON(platformRelease *cores.PlatformRelease) error {
func (pme *Explorer) cacheInstalledJSON(platformRelease *cores.PlatformRelease) error {
index := packageindex.IndexFromPlatformRelease(platformRelease)
platformJSON, err := json.MarshalIndent(index, "", " ")
if err != nil {
Expand Down Expand Up @@ -246,15 +246,15 @@ func (pme *Explorer) RunPostInstallScript(platformRelease *cores.PlatformRelease
}

// IsManagedPlatformRelease returns true if the PlatforRelease is managed by the PackageManager
func (pm *PackageManager) IsManagedPlatformRelease(platformRelease *cores.PlatformRelease) bool {
if pm.PackagesDir == nil {
func (pme *Explorer) IsManagedPlatformRelease(platformRelease *cores.PlatformRelease) bool {
if pme.PackagesDir == nil {
return false
}
installDir := platformRelease.InstallDir.Clone()
if installDir.FollowSymLink() != nil {
return false
}
packagesDir := pm.PackagesDir.Clone()
packagesDir := pme.PackagesDir.Clone()
if packagesDir.FollowSymLink() != nil {
return false
}
Expand All @@ -266,8 +266,8 @@ func (pm *PackageManager) IsManagedPlatformRelease(platformRelease *cores.Platfo
}

// UninstallPlatform remove a PlatformRelease.
func (pm *PackageManager) UninstallPlatform(platformRelease *cores.PlatformRelease, taskCB rpc.TaskProgressCB) error {
log := pm.log.WithField("platform", platformRelease)
func (pme *Explorer) UninstallPlatform(platformRelease *cores.PlatformRelease, taskCB rpc.TaskProgressCB) error {
log := pme.log.WithField("platform", platformRelease)

log.Info("Uninstalling platform")
taskCB(&rpc.TaskProgress{Name: tr("Uninstalling %s", platformRelease)})
Expand All @@ -279,7 +279,7 @@ func (pm *PackageManager) UninstallPlatform(platformRelease *cores.PlatformRelea
}

// Safety measure
if !pm.IsManagedPlatformRelease(platformRelease) {
if !pme.IsManagedPlatformRelease(platformRelease) {
err := fmt.Errorf(tr("%s is not managed by package manager"), platformRelease)
log.WithError(err).Error("Error uninstalling")
return &arduino.FailedUninstallError{Message: err.Error()}
Expand All @@ -299,8 +299,8 @@ func (pm *PackageManager) UninstallPlatform(platformRelease *cores.PlatformRelea
}

// InstallTool installs a specific release of a tool.
func (pm *PackageManager) InstallTool(toolRelease *cores.ToolRelease, taskCB rpc.TaskProgressCB) error {
log := pm.log.WithField("Tool", toolRelease)
func (pme *Explorer) InstallTool(toolRelease *cores.ToolRelease, taskCB rpc.TaskProgressCB) error {
log := pme.log.WithField("Tool", toolRelease)

if toolRelease.IsInstalled() {
log.Warn("Tool already installed")
Expand All @@ -315,12 +315,12 @@ func (pm *PackageManager) InstallTool(toolRelease *cores.ToolRelease, taskCB rpc
if toolResource == nil {
return fmt.Errorf(tr("no compatible version of %s tools found for the current os"), toolRelease.Tool.Name)
}
destDir := pm.PackagesDir.Join(
destDir := pme.PackagesDir.Join(
toolRelease.Tool.Package.Name,
"tools",
toolRelease.Tool.Name,
toolRelease.Version.String())
err := toolResource.Install(pm.DownloadDir, pm.tempDir, destDir)
err := toolResource.Install(pme.DownloadDir, pme.tempDir, destDir)
if err != nil {
log.WithError(err).Warn("Cannot install tool")
return &arduino.FailedInstallError{Message: tr("Cannot install tool %s", toolRelease), Cause: err}
Expand All @@ -332,15 +332,15 @@ func (pm *PackageManager) InstallTool(toolRelease *cores.ToolRelease, taskCB rpc
}

// IsManagedToolRelease returns true if the ToolRelease is managed by the PackageManager
func (pm *PackageManager) IsManagedToolRelease(toolRelease *cores.ToolRelease) bool {
if pm.PackagesDir == nil {
func (pme *Explorer) IsManagedToolRelease(toolRelease *cores.ToolRelease) bool {
if pme.PackagesDir == nil {
return false
}
installDir := toolRelease.InstallDir.Clone()
if installDir.FollowSymLink() != nil {
return false
}
packagesDir := pm.PackagesDir.Clone()
packagesDir := pme.PackagesDir.Clone()
if packagesDir.FollowSymLink() != nil {
return false
}
Expand All @@ -352,16 +352,16 @@ func (pm *PackageManager) IsManagedToolRelease(toolRelease *cores.ToolRelease) b
}

// UninstallTool remove a ToolRelease.
func (pm *PackageManager) UninstallTool(toolRelease *cores.ToolRelease, taskCB rpc.TaskProgressCB) error {
log := pm.log.WithField("Tool", toolRelease)
func (pme *Explorer) UninstallTool(toolRelease *cores.ToolRelease, taskCB rpc.TaskProgressCB) error {
log := pme.log.WithField("Tool", toolRelease)
log.Info("Uninstalling tool")

if toolRelease.InstallDir == nil {
return fmt.Errorf(tr("tool not installed"))
}

// Safety measure
if !pm.IsManagedToolRelease(toolRelease) {
if !pme.IsManagedToolRelease(toolRelease) {
err := &arduino.FailedUninstallError{Message: tr("tool %s is not managed by package manager", toolRelease)}
log.WithError(err).Error("Error uninstalling")
return err
Expand Down
4 changes: 3 additions & 1 deletion arduino/cores/packagemanager/package_manager_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,8 +469,10 @@ func TestFindPlatformReleaseDependencies(t *testing.T) {
pmb := packagemanager.NewBuilder(nil, nil, nil, nil, "test")
pmb.LoadPackageIndexFromFile(paths.New("testdata", "package_tooltest_index.json"))
pm := pmb.Build()
pme, release := pm.NewExplorer()
defer release()

pl, tools, err := pm.FindPlatformReleaseDependencies(&packagemanager.PlatformReference{Package: "test", PlatformArchitecture: "avr"})
pl, tools, err := pme.FindPlatformReleaseDependencies(&packagemanager.PlatformReference{Package: "test", PlatformArchitecture: "avr"})
require.NoError(t, err)
require.NotNil(t, pl)
require.Len(t, tools, 3)
Expand Down
6 changes: 4 additions & 2 deletions arduino/cores/packagemanager/profiles.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,16 +122,18 @@ func (pmb *Builder) installMissingProfilePlatform(platformRef *sketch.ProfilePla
tmpPlatform := tmpTargetPackage.GetOrCreatePlatform(platformRef.Architecture)
tmpPlatformRelease := tmpPlatform.GetOrCreateRelease(platformRef.Version)
tmpPm := tmpPmb.Build()
tmpPme, tmpRelease := tmpPm.NewExplorer()
defer tmpRelease()

if err := tmpPm.DownloadPlatformRelease(tmpPlatformRelease, nil, downloadCB); err != nil {
if err := tmpPme.DownloadPlatformRelease(tmpPlatformRelease, nil, downloadCB); err != nil {
taskCB(&rpc.TaskProgress{Name: tr("Error downloading platform %s", tmpPlatformRelease)})
return &arduino.FailedInstallError{Message: tr("Error downloading platform %s", tmpPlatformRelease), Cause: err}
}
taskCB(&rpc.TaskProgress{Completed: true})

// Perform install
taskCB(&rpc.TaskProgress{Name: tr("Installing platform %s", tmpPlatformRelease)})
if err := tmpPm.InstallPlatformInDirectory(tmpPlatformRelease, destDir); err != nil {
if err := tmpPme.InstallPlatformInDirectory(tmpPlatformRelease, destDir); err != nil {
taskCB(&rpc.TaskProgress{Name: tr("Error installing platform %s", tmpPlatformRelease)})
return &arduino.FailedInstallError{Message: tr("Error installing platform %s", tmpPlatformRelease), Cause: err}
}
Expand Down
6 changes: 4 additions & 2 deletions cli/compile/compile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,11 +345,13 @@ func runCompileCommand(cmd *cobra.Command, args []string) {
panic(tr("Platform ID is not correct"))
}

pm := commands.GetPackageManager(inst.GetId())
platform := pm.FindPlatform(&packagemanager.PlatformReference{
// FIXME: Here we should not access PackageManager...
pme, release := commands.GetPackageManagerExplorer(compileRequest)
platform := pme.FindPlatform(&packagemanager.PlatformReference{
Package: split[0],
PlatformArchitecture: split[1],
})
release()

if profileArg.String() == "" {
if platform != nil {
Expand Down
3 changes: 2 additions & 1 deletion cli/lib/upgrade.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/arduino/arduino-cli/cli/instance"
"github.com/arduino/arduino-cli/cli/output"
"github.com/arduino/arduino-cli/commands/lib"
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
Expand All @@ -46,7 +47,7 @@ func runUpgradeCommand(cmd *cobra.Command, args []string) {
logrus.Info("Executing `arduino-cli lib upgrade`")

if len(args) == 0 {
err := lib.LibraryUpgradeAll(instance.Id, output.ProgressBar(), output.TaskProgress())
err := lib.LibraryUpgradeAll(&rpc.LibraryUpgradeAllRequest{Instance: instance}, output.ProgressBar(), output.TaskProgress())
if err != nil {
feedback.Errorf(tr("Error upgrading libraries: %v"), err)
os.Exit(errorcodes.ErrGeneric)
Expand Down
6 changes: 4 additions & 2 deletions cli/upload/upload.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,11 +121,13 @@ func runUploadCommand(command *cobra.Command, args []string) {
panic(tr("Platform ID is not correct"))
}

pm := commands.GetPackageManager(instance.GetId())
platform := pm.FindPlatform(&packagemanager.PlatformReference{
// FIXME: Here we must not access package manager...
pme, release := commands.GetPackageManagerExplorer(&rpc.UploadRequest{Instance: instance})
platform := pme.FindPlatform(&packagemanager.PlatformReference{
Package: split[0],
PlatformArchitecture: split[1],
})
release()

if platform != nil {
feedback.Errorf(tr("Try running %s", fmt.Sprintf("`%s core install %s`", globals.VersionInfo.Application, platformErr.Platform)))
Expand Down
11 changes: 6 additions & 5 deletions commands/core/download.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,11 @@ var tr = i18n.Tr

// PlatformDownload FIXMEDOC
func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadRequest, downloadCB rpc.DownloadProgressCB) (*rpc.PlatformDownloadResponse, error) {
pm := commands.GetPackageManager(req.GetInstance().GetId())
if pm == nil {
pme, release := commands.GetPackageManagerExplorer(req)
if pme == nil {
return nil, &arduino.InvalidInstanceError{}
}
defer release()

version, err := commands.ParseVersion(req)
if err != nil {
Expand All @@ -44,17 +45,17 @@ func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadRequest, dow
PlatformArchitecture: req.Architecture,
PlatformVersion: version,
}
platform, tools, err := pm.FindPlatformReleaseDependencies(ref)
platform, tools, err := pme.FindPlatformReleaseDependencies(ref)
if err != nil {
return nil, &arduino.PlatformNotFoundError{Platform: ref.String(), Cause: err}
}

if err := pm.DownloadPlatformRelease(platform, nil, downloadCB); err != nil {
if err := pme.DownloadPlatformRelease(platform, nil, downloadCB); err != nil {
return nil, err
}

for _, tool := range tools {
if err := pm.DownloadToolRelease(tool, nil, downloadCB); err != nil {
if err := pme.DownloadToolRelease(tool, nil, downloadCB); err != nil {
return nil, err
}
}
Expand Down
6 changes: 3 additions & 3 deletions commands/instances.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,16 +129,16 @@ func GetLibraryManager(id int32) *librariesmanager.LibrariesManager {
return i.lm
}

func (instance *CoreInstance) installToolIfMissing(tool *cores.ToolRelease, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) (bool, error) {
func installToolIfMissing(pme *packagemanager.Explorer, tool *cores.ToolRelease, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) (bool, error) {
if tool.IsInstalled() {
return false, nil
}
taskCB(&rpc.TaskProgress{Name: tr("Downloading missing tool %s", tool)})
if err := instance.pm.DownloadToolRelease(tool, nil, downloadCB); err != nil {
if err := pme.DownloadToolRelease(tool, nil, downloadCB); err != nil {
return false, fmt.Errorf(tr("downloading %[1]s tool: %[2]s"), tool, err)
}
taskCB(&rpc.TaskProgress{Completed: true})
if err := instance.pm.InstallTool(tool, taskCB); err != nil {
if err := pme.InstallTool(tool, taskCB); err != nil {
return false, fmt.Errorf(tr("installing %[1]s tool: %[2]s"), tool, err)
}
return true, nil
Expand Down
Loading
1241
0