Add optional 'resources' directory to plugins (#1514)

*Adding an option to create and publish plugins to Artifactory with a directory named "resources", which contains all resources that the developer wants to add to the plugin.
*When publishing a plugin make sure that you have a "resources" directory in root.
*Resources are being uploaded to Artifactory in a zip file.
*There are changes in the plugins directory hierarchy at the local '.jfrog' directory.

artifactory_test.go

@@ -5308,7 +5308,7 @@ func TestTerraformPublish(t *testing.T) {
 	assert.NoError(t, os.MkdirAll(tests.Out+"/results/", 0777))
 	runRt(t, "download", tests.TerraformRepo+"/namespace/provider/*", tests.Out+"/results/", "--explode=true")
 	// Validate
-	paths, err := fileutils.ListFilesRecursiveWalkIntoDirSymlink(tests.Out+"/results", false)
+	paths, err := fileutils.ListFilesRecursiveWalkIntoDirSymlink(filepath.Join(tests.Out, "results"), false)
 	assert.NoError(t, err)
 	assert.NoError(t, tests.ValidateListsIdentical(tests.GetTerraformModulesFilesDownload(), paths))
 }

go.mod

@@ -93,8 +93,8 @@ require (
 
 replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.2.3
 
-replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.12.1-0.20220412133016-7e9342af1eea
+replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.12.1-0.20220413150046-30f13ed79855
 
-replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.12.3
+replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.12.4-0.20220414170856-d1e29190993e
 
 replace github.com/jfrog/gocmd => github.com/jfrog/gocmd v0.6.0

plugins/commands/install.go

@@ -1,12 +1,15 @@
 package commands
 
 import (
+	ioutils "github.com/jfrog/jfrog-client-go/utils/io"
+	"github.com/mholt/archiver/v3"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
+	"github.com/jfrog/jfrog-cli-core/v2/utils/plugins"
 	commandsUtils "github.com/jfrog/jfrog-cli/plugins/commands/utils"
 	clientUtils "github.com/jfrog/jfrog-client-go/utils"
 
@@ -30,6 +33,10 @@ func InstallCmd(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	err = plugins.CheckPluginsVersionAndConvertIfNeeded()
+	if err != nil {
+		return err
+	}
 	return runInstallCmd(c.Args().Get(0))
 }
 
@@ -44,26 +51,26 @@ func runInstallCmd(requestedPlugin string) error {
 		return err
 	}
 
-	url, httpDetails, err := getServerDetails()
+	url, serverDetails, err := getServerDetails()
 	if err != nil {
 		return err
 	}
 
-	pluginRtPath, err := getRequiredPluginRtPath(pluginName, version)
+	pluginRtDirPath, err := getRequiredPluginRtDirPath(pluginName, version)
 	if err != nil {
 		return err
 	}
-	downloadUrl := clientUtils.AddTrailingSlashIfNeeded(url) + pluginRtPath
+	execDownloadUrl := clientUtils.AddTrailingSlashIfNeeded(url) + pluginRtDirPath + "/"
 
-	should, err := shouldDownloadPlugin(pluginsDir, pluginName, downloadUrl, httpDetails)
+	should, err := shouldDownloadPlugin(pluginsDir, pluginName, execDownloadUrl, commandsUtils.CreatePluginsHttpDetails(&serverDetails))
 	if err != nil {
 		return err
 	}
 	if !should {
 		return errorutils.CheckErrorf("the plugin with the requested version already exists locally")
 	}
 
-	return downloadPlugin(pluginsDir, pluginName, downloadUrl, httpDetails)
+	return downloadPlugin(pluginsDir, pluginName, execDownloadUrl, commandsUtils.CreatePluginsHttpDetails(&serverDetails))
 }
 
 // Assert repo env is not passed without server env.
@@ -99,22 +106,22 @@ func createPluginsDirIfNeeded() (string, error) {
 }
 
 // Use the server ID if provided, else use the official registry.
-func getServerDetails() (string, httputils.HttpClientDetails, error) {
+func getServerDetails() (string, config.ServerDetails, error) {
 	serverId := os.Getenv(commandsUtils.PluginsServerEnv)
 	if serverId == "" {
-		return commandsUtils.PluginsOfficialRegistryUrl, httputils.HttpClientDetails{}, nil
+		return commandsUtils.PluginsOfficialRegistryUrl, config.ServerDetails{ArtifactoryUrl: commandsUtils.PluginsOfficialRegistryUrl}, nil
 	}
 
 	rtDetails, err := config.GetSpecificConfig(serverId, false, true)
 	if err != nil {
-		return "", httputils.HttpClientDetails{}, err
+		return "", config.ServerDetails{}, err
 	}
-	return rtDetails.ArtifactoryUrl, commandsUtils.CreatePluginsHttpDetails(rtDetails), nil
+	return rtDetails.ArtifactoryUrl, *rtDetails, nil
 }
 
 // Checks if the requested plugin exists in registry and does not exists locally.
 func shouldDownloadPlugin(pluginsDir, pluginName, downloadUrl string, httpDetails httputils.HttpClientDetails) (bool, error) {
-	exists, err := fileutils.IsFileExists(filepath.Join(pluginsDir, commandsUtils.GetLocalPluginExecutableName(pluginName)), false)
+	exists, err := fileutils.IsDirExists(filepath.Join(pluginsDir, pluginName), false)
 	if err != nil {
 		return false, err
 	}
@@ -137,38 +144,30 @@ func shouldDownloadPlugin(pluginsDir, pluginName, downloadUrl string, httpDetail
 	if err != nil {
 		return false, err
 	}
-	equal, err := fileutils.IsEqualToLocalFile(filepath.Join(pluginsDir, pluginName), details.Checksum.Md5, details.Checksum.Sha1)
+	equal, err := fileutils.IsEqualToLocalFile(filepath.Join(pluginsDir, pluginName, coreutils.PluginsExecDirName, plugins.GetLocalPluginExecutableName(pluginName)), details.Checksum.Md5, details.Checksum.Sha1)
 	return !equal, err
 }
 
-// Returns the path of the plugin's executable in registry, corresponding to the local architecture.
-func getRequiredPluginRtPath(pluginName, version string) (string, error) {
+// Returns the path of the JFrog CLI plugin's directory in registry, corresponding to the local architecture.
+func getRequiredPluginRtDirPath(pluginName, version string) (pluginDirRtPath string, err error) {
 	arc, err := commandsUtils.GetLocalArchitecture()
 	if err != nil {
-		return "", err
+		return
 	}
-	return commandsUtils.GetPluginPathInArtifactory(pluginName, version, arc), nil
+	pluginDirRtPath = commandsUtils.GetPluginDirPath(pluginName, version, arc)
+	return
 }
 
 func createPluginsDir(pluginsDir string) error {
-	return os.MkdirAll(pluginsDir, 0777)
+	err := os.MkdirAll(pluginsDir, 0777)
+	if err != nil {
+		return errorutils.CheckError(err)
+	}
+	_, err = plugins.CreatePluginsConfigFile()
+	return err
 }
 
 func downloadPlugin(pluginsDir, pluginName, downloadUrl string, httpDetails httputils.HttpClientDetails) (err error) {
-	exeName := commandsUtils.GetLocalPluginExecutableName(pluginName)
-	log.Debug("Downloading plugin from: ", downloadUrl)
-	downloadDetails := &httpclient.DownloadFileDetails{
-		FileName:      pluginName,
-		DownloadPath:  downloadUrl,
-		LocalPath:     pluginsDir,
-		LocalFileName: exeName,
-		RelativePath:  exeName,
-	}
-
-	client, err := httpclient.ClientBuilder().Build()
-	if err != nil {
-		return
-	}
 	// Init progress bar.
 	progressMgr, logFile, err := progressbar.InitProgressBarIfPossible(false)
 	if err != nil {
@@ -185,19 +184,16 @@ func downloadPlugin(pluginsDir, pluginName, downloadUrl string, httpDetails http
 			}
 		}()
 	}
-	log.Info("Downloading plugin: " + pluginName)
 
-	resp, err := client.DownloadFileWithProgress(downloadDetails, "", httpDetails, false, progressMgr)
+	err = downloadPluginExec(downloadUrl, pluginName, pluginsDir, httpDetails, progressMgr)
 	if err != nil {
 		return
 	}
-	log.Debug("Artifactory response: ", resp.Status)
-	err = errorutils.CheckResponseStatus(resp, http.StatusOK)
+	err = downloadPluginsResources(downloadUrl, pluginName, pluginsDir, httpDetails, progressMgr)
 	if err != nil {
 		return
 	}
-	log.Debug("Plugin downloaded successfully.")
-	err = os.Chmod(filepath.Join(pluginsDir, exeName), 0777)
+	log.Info("Plugin downloaded successfully.")
 	return
 }
 
@@ -211,3 +207,74 @@ func getNameAndVersion(requested string) (name, version string, err error) {
 	}
 	return split[0], split[1], nil
 }
+
+func downloadPluginExec(downloadUrl, pluginName, pluginsDir string, httpDetails httputils.HttpClientDetails, progressMgr ioutils.ProgressMgr) (err error) {
+	exeName := plugins.GetLocalPluginExecutableName(pluginName)
+	downloadDetails := &httpclient.DownloadFileDetails{
+		FileName:      pluginName,
+		DownloadPath:  clientUtils.AddTrailingSlashIfNeeded(downloadUrl) + exeName,
+		LocalPath:     filepath.Join(pluginsDir, pluginName, coreutils.PluginsExecDirName),
+		LocalFileName: exeName,
+		RelativePath:  exeName,
+	}
+	log.Debug("Downloading plugin's executable from: ", downloadDetails.DownloadPath)
+	_, err = downloadFromArtifactory(downloadDetails, httpDetails, progressMgr)
+	if err != nil {
+		return
+	}
+	err = os.Chmod(filepath.Join(downloadDetails.LocalPath, downloadDetails.LocalFileName), 0777)
+	if errorutils.CheckError(err) != nil {
+		return
+	}
+	log.Debug("Plugin's executable downloaded successfully.")
+	return
+}
+
+func downloadPluginsResources(downloadUrl, pluginName, pluginsDir string, httpDetails httputils.HttpClientDetails, progressMgr ioutils.ProgressMgr) (err error) {
+	downloadDetails := &httpclient.DownloadFileDetails{
+		FileName:      pluginName,
+		DownloadPath:  clientUtils.AddTrailingSlashIfNeeded(downloadUrl) + coreutils.PluginsResourcesDirName + ".zip",
+		LocalPath:     filepath.Join(pluginsDir, pluginName),
+		LocalFileName: coreutils.PluginsResourcesDirName + ".zip",
+		RelativePath:  coreutils.PluginsResourcesDirName + ".zip",
+	}
+	log.Debug("Downloading plugin's resources from: ", downloadDetails.DownloadPath)
+	statusCode, err := downloadFromArtifactory(downloadDetails, httpDetails, progressMgr)
+	if err != nil {
+		return
+	}
+	if statusCode == http.StatusNotFound {
+		log.Debug("No resources were downloaded.")
+		return nil
+	}
+	err = archiver.Unarchive(filepath.Join(downloadDetails.LocalPath, downloadDetails.LocalFileName), filepath.Join(downloadDetails.LocalPath, coreutils.PluginsResourcesDirName)+string(os.PathSeparator))
+	if errorutils.CheckError(err) != nil {
+		return
+	}
+	err = os.Remove(filepath.Join(downloadDetails.LocalPath, downloadDetails.LocalFileName))
+	if err != nil {
+		return
+	}
+	err = coreutils.ChmodPluginsDirectoryContent()
+	if errorutils.CheckError(err) != nil {
+		return
+	}
+	log.Debug("Plugin's resources downloaded successfully.")
+	return
+}
+
+func downloadFromArtifactory(downloadDetails *httpclient.DownloadFileDetails, httpDetails httputils.HttpClientDetails, progressMgr ioutils.ProgressMgr) (statusCode int, err error) {
+	client, err := httpclient.ClientBuilder().Build()
+	if err != nil {
+		return
+	}
+	log.Info("Downloading: " + downloadDetails.FileName)
+	resp, err := client.DownloadFileWithProgress(downloadDetails, "", httpDetails, false, progressMgr)
+	if err != nil {
+		return
+	}
+	statusCode = resp.StatusCode
+	log.Debug("Artifactory response: ", statusCode)
+	err = errorutils.CheckResponseStatus(resp, http.StatusOK, http.StatusNotFound)
+	return
+}

plugins/commands/publish.go

@@ -1,11 +1,14 @@
 package commands
 
 import (
+	buildinfoutils "github.com/jfrog/build-info-go/utils"
 	"github.com/jfrog/gofrog/io"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/generic"
+	commandsutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils"
 	rtutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
 	"github.com/jfrog/jfrog-cli-core/v2/common/spec"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
+	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
 	"github.com/jfrog/jfrog-cli/plugins/commands/utils"
 	pluginsutils "github.com/jfrog/jfrog-cli/plugins/utils"
 	"github.com/jfrog/jfrog-cli/utils/cliutils"
@@ -175,28 +178,91 @@ func verifyUniqueVersion(pluginName, pluginVersion string, rtDetails *config.Ser
 }
 
 func uploadPlugin(pluginLocalPath, pluginName, pluginVersion, arc string, rtDetails *config.ServerDetails) error {
-	targetPath := utils.GetPluginPathInArtifactory(pluginName, pluginVersion, arc)
-	log.Info("Upload plugin to: " + targetPath + "...")
+	pluginDirRtPath := utils.GetPluginDirPath(pluginName, pluginVersion, arc)
+	log.Info("Upload plugin to: " + pluginDirRtPath + "...")
+	// First uploading resources directory (this is the complex part). If the upload is successful, upload the executable file.
+	// Upload plugin's resources directory if exists
+	exists, err := buildinfoutils.IsDirExists(coreutils.PluginsResourcesDirName, true)
+	if err != nil {
+		return errorutils.CheckError(err)
+	}
+	if exists {
+		empty, err := fileutils.IsDirEmpty(coreutils.PluginsResourcesDirName)
+		if err != nil {
+			return err
+		}
+		if !empty {
+			resourcesPattern := filepath.Join(coreutils.PluginsResourcesDirName, "(*)")
+			resourcesTargetPath := path.Join(pluginDirRtPath, coreutils.PluginsResourcesDirName+".zip")
+			err = uploadPluginsResources(resourcesPattern, resourcesTargetPath, rtDetails)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	// Upload plugin's executable
+	execTargetPath := path.Join(pluginDirRtPath, utils.GetPluginExecutableName(pluginName, arc))
+	err = uploadPluginsExec(pluginLocalPath, execTargetPath, rtDetails)
+	if err != nil {
+		return err
+	}
+	return nil
+}
 
-	uploadCmd := generic.NewUploadCommand()
-	uploadCmd.SetUploadConfiguration(createUploadConfiguration()).
-		SetServerDetails(rtDetails).
-		SetSpec(createUploadSpec(pluginLocalPath, targetPath))
+func uploadPluginsExec(pattern, target string, rtDetails *config.ServerDetails) error {
+	log.Debug("Upload plugin's executable to: " + target + "...")
+	result, err := createAndRunPluginsExecUploadCommand(pattern, target, rtDetails)
+	if err != nil {
+		return err
+	}
+	if result.SuccessCount() == 0 {
+		return errorutils.CheckErrorf("plugin's executable upload failed as no files were affected. Verify source path is valid")
+	}
+	if result.SuccessCount() > 1 {
+		return errorutils.CheckErrorf("while uploading plugin's executable more than one file was uploaded. Unexpected behaviour, aborting")
+	}
+	return nil
+}
 
-	err := uploadCmd.Run()
+func uploadPluginsResources(pattern, target string, rtDetails *config.ServerDetails) error {
+	log.Debug("Upload plugin's resources to: " + target + "...")
+	result, err := createAndRunPluginsResourcesUploadCommand(pattern, target, rtDetails)
 	if err != nil {
 		return err
 	}
-	result := uploadCmd.Result()
 	if result.SuccessCount() == 0 {
-		return errorutils.CheckErrorf("plugin upload failed as no files were affected. Verify source path is valid")
+		return errorutils.CheckErrorf("plugin's resources upload failed as no files were affected. Verify source path is valid")
 	}
 	if result.SuccessCount() > 1 {
-		return errorutils.CheckErrorf("more than one file was uploaded. Unexpected behaviour, aborting")
+		return errorutils.CheckErrorf("while zipping and uploading plugin's resources directory more than one file was uploaded. Unexpected behaviour, aborting")
 	}
 	return nil
 }
 
+func createAndRunPluginsExecUploadCommand(pattern, target string, rtDetails *config.ServerDetails) (*commandsutils.Result, error) {
+	uploadCmd := generic.NewUploadCommand()
+	uploadCmd.SetUploadConfiguration(createUploadConfiguration()).
+		SetServerDetails(rtDetails).
+		SetSpec(createExecUploadSpec(pattern, target))
+	err := uploadCmd.Run()
+	if err != nil {
+		return nil, err
+	}
+	return uploadCmd.Result(), nil
+}
+
+func createAndRunPluginsResourcesUploadCommand(pattern, target string, rtDetails *config.ServerDetails) (*commandsutils.Result, error) {
+	uploadCmd := generic.NewUploadCommand()
+	uploadCmd.SetUploadConfiguration(createUploadConfiguration()).
+		SetServerDetails(rtDetails).
+		SetSpec(createResourcesUploadSpec(pattern, target))
+	err := uploadCmd.Run()
+	if err != nil {
+		return nil, err
+	}
+	return uploadCmd.Result(), nil
+}
+
 // Copy the uploaded version to override latest dir.
 func copyToLatestDir(pluginName, pluginVersion string, rtDetails *config.ServerDetails) error {
 	log.Info("Copying version to latest dir...")
@@ -217,13 +283,24 @@ func createCopySpec(pluginName, pluginVersion string) *spec.SpecFiles {
 		BuildSpec()
 }
 
-func createUploadSpec(source, target string) *spec.SpecFiles {
+func createExecUploadSpec(source, target string) *spec.SpecFiles {
 	return spec.NewBuilder().
 		Pattern(source).
 		Target(target).
 		BuildSpec()
 }
 
+// Resources directory is being uploaded to Artifactory in a zip file.
+func createResourcesUploadSpec(source, target string) *spec.SpecFiles {
+	return spec.NewBuilder().
+		Pattern(source).
+		Target(target).
+		Archive("zip").
+		Recursive(true).
+		TargetPathInArchive("{1}").
+		BuildSpec()
+}
+
 func createUploadConfiguration() *rtutils.UploadConfiguration {
 	uploadConfiguration := new(rtutils.UploadConfiguration)
 	uploadConfiguration.Threads = cliutils.Threads