raystack · sravankorumilli · Sep 7, 2022 · Jul 20, 2022 · Jul 26, 2022 · Jul 26, 2022
diff --git a/.gitignore b/.gitignore
@@ -21,7 +21,8 @@ __pycache__
 optimus.yaml
 config.yaml
 coverage.txt
-
+.plugins
+yaml-plugins.zip
 /api/proto/odpf/**/*
 
 !/api/proto/odpf/optimus/

diff --git a/cmd/internal/survey/job_addhook.go b/cmd/internal/survey/job_addhook.go
@@ -44,7 +44,7 @@ func (j *JobAddHookSurvey) AskToAddHook(jobSpec models.JobSpec, pluginRepo model
 	}
 
 	var jobSpecConfigs models.JobSpecConfigs
-	if cliMod := selectedHook.CLIMod; cliMod != nil {
+	if cliMod := selectedHook.GetSurveyMod(); cliMod != nil {
 		ctx := context.Background()
 		hookAnswers, err := j.askHookQuestions(ctx, cliMod, jobSpec.Name)
 		if err != nil {

diff --git a/cmd/internal/survey/job_create.go b/cmd/internal/survey/job_create.go
@@ -111,8 +111,9 @@ func (*JobCreateSurvey) getTaskConfig(cliMod models.CommandLineMod, answers mode
 
 func (*JobCreateSurvey) getAvailableTaskNames() []string {
 	pluginRepo := models.PluginRegistry
+	plugins := pluginRepo.GetTasks()
 	var output []string
-	for _, task := range pluginRepo.GetTasks() {
+	for _, task := range plugins {
 		output = append(output, task.Info().Name)
 	}
 	return output
@@ -215,11 +216,11 @@ func (j *JobCreateSurvey) askCreateQuestions(questions []*survey.Question) (loca
 
 func (*JobCreateSurvey) getPluginCLIMod(taskName string) (models.CommandLineMod, error) {
 	pluginRepo := models.PluginRegistry
-	executionTask, err := pluginRepo.GetByName(taskName)
+	plugin, err := pluginRepo.GetByName(taskName)
 	if err != nil {
 		return nil, err
 	}
-	return executionTask.CLIMod, nil
+	return plugin.GetSurveyMod(), nil
 }
 
 func (j *JobCreateSurvey) askPluginQuestions(cliMod models.CommandLineMod, jobName string) (models.PluginAnswers, error) {

diff --git a/cmd/plugin/install.go b/cmd/plugin/install.go
@@ -6,41 +6,39 @@ import (
 
 	"github.com/odpf/optimus/cmd/internal/logger"
 	"github.com/odpf/optimus/config"
+	"github.com/odpf/optimus/plugin"
 )
 
 type installCommand struct {
-	logger       log.Logger
-	serverConfig *config.ServerConfig
+	logger         log.Logger
+	serverConfig   *config.ServerConfig
+	configFilePath string `default:"config.yaml"`
 }
 
 // NewInstallCommand initializes plugin install command
-func NewInstallCommand(serverConfig *config.ServerConfig) *cobra.Command {
-	install := &installCommand{
-		serverConfig: serverConfig,
-	}
+func NewInstallCommand() *cobra.Command {
+	install := &installCommand{}
 	cmd := &cobra.Command{
 		Use:     "install",
-		Short:   "install and extract plugins to a dir",
+		Short:   "download and extract plugins to a dir (on server)",
 		Example: "optimus plugin install",
 		RunE:    install.RunE,
 		PreRunE: install.PreRunE,
 	}
+	cmd.PersistentFlags().StringVarP(&install.configFilePath, "config", "c", install.configFilePath, "File path for server configuration")
 	return cmd
 }
 
 func (i *installCommand) PreRunE(_ *cobra.Command, _ []string) error {
-	i.logger = logger.NewClientLogger(i.serverConfig.Log)
+	c, err := config.LoadServerConfig(i.configFilePath)
+	if err != nil {
+		return err
+	}
+	i.serverConfig = c
+	i.logger = logger.NewClientLogger(c.Log)
 	return nil
 }
 
-// also used during server start
-func InstallPlugins(conf *config.ServerConfig, logger log.Logger) error {
-	dst := conf.Plugin.Dir
-	sources := conf.Plugin.Artifacts
-	pluginManger := NewPluginManager(logger)
-	return pluginManger.Install(dst, sources...)
-}
-
 func (i *installCommand) RunE(_ *cobra.Command, _ []string) error {
-	return InstallPlugins(i.serverConfig, i.logger)
+	return plugin.InstallPlugins(i.serverConfig)
 }
diff --git a/cmd/plugin/plugin.go b/cmd/plugin/plugin.go
@@ -11,39 +11,22 @@ import (
 	oPlugin "github.com/odpf/optimus/plugin"
 )
 
-type pluginCommand struct {
-	configFilePath string
-	serverConfig   *config.ServerConfig
-}
-
-// NewPluginCommand initializes command for plugin
 func NewPluginCommand() *cobra.Command {
-	plugin := pluginCommand{
-		serverConfig: &config.ServerConfig{},
-	}
 	cmd := &cobra.Command{
-		Use:    "plugin",
-		Short:  "Manage plugins",
-		Hidden: true,
+		Use:   "plugin",
+		Short: "Manage plugins",
 		Annotations: map[string]string{
 			"group:dev": "true",
 		},
-		PersistentPreRunE: plugin.PersistentPreRunE,
 	}
-	cmd.PersistentFlags().StringVarP(&plugin.configFilePath, "config", "c", plugin.configFilePath, "File path for server configuration")
-	cmd.AddCommand(NewInstallCommand(plugin.serverConfig))
+	cmd.AddCommand(
+		NewInstallCommand(),
+		NewValidateCommand(),
+		NewSyncCommand(),
+	)
 	return cmd
 }
 
-func (p *pluginCommand) PersistentPreRunE(_ *cobra.Command, _ []string) error {
-	c, err := config.LoadServerConfig(p.configFilePath)
-	if err != nil {
-		return err
-	}
-	*p.serverConfig = *c
-	return nil
-}
-
 // TriggerClientPluginsInit triggers initialization of all available plugins
 func TriggerClientPluginsInit(logLevel config.LogLevel) (cleanFn func(), err error) {
 	pluginLogLevel := hclog.Info

diff --git a/cmd/plugin/plugin_manager.go b/cmd/plugin/plugin_manager.go
diff --git a/cmd/plugin/sync.go b/cmd/plugin/sync.go
@@ -0,0 +1,103 @@
+package plugin
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+
+	"github.com/odpf/salt/log"
+	"github.com/spf13/cobra"
+
+	"github.com/odpf/optimus/cmd/internal/logger"
+	"github.com/odpf/optimus/config"
+	"github.com/odpf/optimus/plugin"
+)
+
+func NewSyncCommand() *cobra.Command {
+	sync := &syncCommand{}
+	cmd := &cobra.Command{
+		Use:     "sync",
+		Short:   "sync plugins from server",
+		Example: "optimus plugin sync -c optimus.yaml",
+		PreRunE: sync.PreRunE,
+		RunE:    sync.RunE,
+	}
+	cmd.PersistentFlags().StringVarP(&sync.configFilePath, "config", "c", sync.configFilePath, "File path for optimus configuration")
+	return cmd
+}
+
+type syncCommand struct {
+	configFilePath string
+	clientConfig   *config.ClientConfig
+	logger         log.Logger
+}
+
+func (s *syncCommand) PreRunE(_ *cobra.Command, _ []string) error {
+	c, err := config.LoadClientConfig(s.configFilePath)
+	if err != nil {
+		return errors.New("project not initialized or error in loading config : \n" + err.Error())
+	}
+	s.clientConfig = c
+	s.logger = logger.NewClientLogger(c.Log)
+	return nil
+}
+
+func getPluginDownloadURL(host string) (*url.URL, error) {
+	var downloadURL *url.URL
+	var err error
+	pluginPath := "plugins"
+	if strings.HasPrefix(host, "http://") || strings.HasPrefix(host, "https://") {
+		downloadURL, err = url.Parse(host)
+		if err != nil {
+			return nil, err
+		}
+		downloadURL.Path = pluginPath
+	} else {
+		downloadURL = &url.URL{
+			Scheme: "http",
+			Host:   host,
+			Path:   pluginPath,
+		}
+	}
+	return downloadURL, nil
+}
+
+func (s *syncCommand) downloadArchiveFromServer() error {
+	downloadURL, err := getPluginDownloadURL(s.clientConfig.Host)
+	s.logger.Info(fmt.Sprintf("download URL : %s", downloadURL.String()))
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(context.Background(), "GET", downloadURL.String(), nil)
+	if err != nil {
+		return err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	out, err := os.Create(plugin.PluginsArchiveName)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+	_, err = io.Copy(out, resp.Body)
+	return err
+}
+
+func (s *syncCommand) RunE(_ *cobra.Command, _ []string) error {
+	err := s.downloadArchiveFromServer()
+	if err != nil {
+		return err
+	}
+	return plugin.NewPluginManager().UnArchive(
+		plugin.PluginsArchiveName,
+		plugin.PluginsDir,
+	)
+}