// Package config reads, writes and edits the config file and deals with command line flags

package config

import (

"context"

"encoding/json"

"errors"

"fmt"

mathrand "math/rand"

"os"

"path/filepath"

"regexp"

"runtime"

"strings"

"time"

"github.com/mitchellh/go-homedir"

"github.com/rclone/rclone/fs"

"github.com/rclone/rclone/fs/cache"

"github.com/rclone/rclone/fs/config/configmap"

"github.com/rclone/rclone/fs/config/obscure"

"github.com/rclone/rclone/fs/fspath"

"github.com/rclone/rclone/fs/rc"

"github.com/rclone/rclone/lib/file"

"github.com/rclone/rclone/lib/random"

)

const (

configFileName = "rclone.conf"

hiddenConfigFileName = "." + configFileName

noConfigFile = "notfound"

// ConfigToken is the key used to store the token under

ConfigToken = "token"

// ConfigClientID is the config key used to store the client id

ConfigClientID = "client_id"

// ConfigClientSecret is the config key used to store the client secret

ConfigClientSecret = "client_secret"

// ConfigAuthURL is the config key used to store the auth server endpoint

ConfigAuthURL = "auth_url"

// ConfigTokenURL is the config key used to store the token server endpoint

ConfigTokenURL = "token_url"

// ConfigClientCredentials - use OAUTH2 client credentials

ConfigClientCredentials = "client_credentials"

// ConfigEncoding is the config key to change the encoding for a backend

ConfigEncoding = "encoding"

// ConfigEncodingHelp is the help for ConfigEncoding

ConfigEncodingHelp = "The encoding for the backend.\n\nSee the [encoding section in the overview](/overview/#encoding) for more info."

// ConfigAuthorize indicates that we just want "rclone authorize"

ConfigAuthorize = "config_authorize"

// ConfigAuthNoBrowser indicates that we do not want to open browser

ConfigAuthNoBrowser = "config_auth_no_browser"

// ConfigTemplate is the template content to be used in the authorization webserver

ConfigTemplate = "config_template"

// ConfigTemplateFile is the path to a template file to read into the value of `ConfigTemplate` above

ConfigTemplateFile = "config_template_file"

)

// Storage defines an interface for loading and saving config to

// persistent storage. Rclone provides a default implementation to

// load and save to a config file when this is imported

//

// import "github.com/rclone/rclone/fs/config/configfile"

// configfile.Install()

type Storage interface {

// GetSectionList returns a slice of strings with names for all the

// sections

GetSectionList() []string

// HasSection returns true if section exists in the config file

HasSection(section string) bool

// DeleteSection removes the named section and all config from the

// config file

DeleteSection(section string)

// GetKeyList returns the keys in this section

GetKeyList(section string) []string

// GetValue returns the key in section with a found flag

GetValue(section string, key string) (value string, found bool)

// SetValue sets the value under key in section

SetValue(section string, key string, value string)

// DeleteKey removes the key under section

DeleteKey(section string, key string) bool

// Load the config from permanent storage

Load() error

// Save the config to permanent storage

Save() error

// Serialize the config into a string

Serialize() (string, error)

}

// Global

var (

// Password can be used to configure the random password generator

Password = random.Password

)

var (

configPath string

cacheDir string

data Storage

dataLoaded bool

)

func init() {

// Set the function pointers up in fs

fs.ConfigFileGet = FileGetValue

fs.ConfigFileSet = SetValueAndSave

fs.ConfigFileHasSection = func(section string) bool {

return LoadedData().HasSection(section)

}

configPath = makeConfigPath()

cacheDir = makeCacheDir() // Has fallback to tempDir, so set that first

data = newDefaultStorage()

}

// Join directory with filename, and check if exists

func findFile(dir string, name string) string {

path := filepath.Join(dir, name)

if _, err := os.Stat(path); err != nil {

return ""

}

return path

}

// Find current user's home directory

func findHomeDir() (string, error) {

path, err := homedir.Dir()

if err != nil {

fs.Debugf(nil, "Home directory lookup failed and cannot be used as configuration location: %v", err)

} else if path == "" {

// On Unix homedir return success but empty string for user with empty home configured in passwd file

fs.Debugf(nil, "Home directory not defined and cannot be used as configuration location")

}

return path, err

}

// Find rclone executable directory and look for existing rclone.conf there

// (<rclone_exe_dir>/rclone.conf)

func findLocalConfig() (configDir string, configFile string) {

if exePath, err := os.Executable(); err == nil {

configDir = filepath.Dir(exePath)

configFile = findFile(configDir, configFileName)

}

return

}

// Get path to Windows AppData config subdirectory for rclone and look for existing rclone.conf there

// ($AppData/rclone/rclone.conf)

func findAppDataConfig() (configDir string, configFile string) {

if appDataDir := os.Getenv("APPDATA"); appDataDir != "" {

configDir = filepath.Join(appDataDir, "rclone")

configFile = findFile(configDir, configFileName)

} else {

fs.Debugf(nil, "Environment variable APPDATA is not defined and cannot be used as configuration location")

}

return

}

// Get path to XDG config subdirectory for rclone and look for existing rclone.conf there

// (see XDG Base Directory specification: https://specifications.freedesktop.org/basedir-spec/latest/).

// ($XDG_CONFIG_HOME\rclone\rclone.conf)

func findXDGConfig() (configDir string, configFile string) {

if xdgConfigDir := os.Getenv("XDG_CONFIG_HOME"); xdgConfigDir != "" {

configDir = filepath.Join(xdgConfigDir, "rclone")

configFile = findFile(configDir, configFileName)

}

return

}

// Get path to .config subdirectory for rclone and look for existing rclone.conf there

// (~/.config/rclone/rclone.conf)

func findDotConfigConfig(home string) (configDir string, configFile string) {

if home != "" {

configDir = filepath.Join(home, ".config", "rclone")

configFile = findFile(configDir, configFileName)

}

return

}

// Look for existing .rclone.conf (legacy hidden filename) in root of user's home directory

// (~/.rclone.conf)

func findOldHomeConfig(home string) (configDir string, configFile string) {

if home != "" {

configDir = home

configFile = findFile(home, hiddenConfigFileName)

}

return

}

// Return the path to the configuration file

func makeConfigPath() string {

// Look for existing rclone.conf in prioritized list of known locations

// Also get configuration directory to use for new config file when no existing is found.

var (

configFile string

configDir string

primaryConfigDir string

fallbackConfigDir string

)

// <rclone_exe_dir>/rclone.conf

if _, configFile = findLocalConfig(); configFile != "" {

return configFile

}

// Windows: $AppData/rclone/rclone.conf

// This is also the default location for new config when no existing is found

if runtime.GOOS == "windows" {

if primaryConfigDir, configFile = findAppDataConfig(); configFile != "" {

return configFile

}

}

// $XDG_CONFIG_HOME/rclone/rclone.conf

// Also looking for this on Windows, for backwards compatibility reasons.

if configDir, configFile = findXDGConfig(); configFile != "" {

return configFile

}

if runtime.GOOS != "windows" {

// On Unix this is also the default location for new config when no existing is found

primaryConfigDir = configDir

}

// ~/.config/rclone/rclone.conf

// This is also the fallback location for new config

// (when $AppData on Windows and $XDG_CONFIG_HOME on Unix is not defined)

homeDir, homeDirErr := findHomeDir()

if fallbackConfigDir, configFile = findDotConfigConfig(homeDir); configFile != "" {

return configFile

}

// ~/.rclone.conf

if _, configFile = findOldHomeConfig(homeDir); configFile != "" {

return configFile

}

// No existing config file found, prepare proper default for a new one.

// But first check if user supplied a --config variable or environment

// variable, since then we skip actually trying to create the default

// and report any errors related to it (we can't use pflag for this because

// it isn't initialised yet so we search the command line manually).

_, configSupplied := os.LookupEnv("RCLONE_CONFIG")

if !configSupplied {

for _, item := range os.Args {

if item == "--config" || strings.HasPrefix(item, "--config=") {

configSupplied = true

break

}

}

}

// If we found a configuration directory to be used for new config during search

// above, then create it to be ready for rclone.conf file to be written into it

// later, and also as a test of permissions to use fallback if not even able to

// create the directory.

if primaryConfigDir != "" {

configDir = primaryConfigDir

} else if fallbackConfigDir != "" {

configDir = fallbackConfigDir

} else {

configDir = ""

}

if configDir != "" {

configFile = filepath.Join(configDir, configFileName)

if configSupplied {

// User supplied custom config option, just return the default path

// as is without creating any directories, since it will not be used

// anyway and we don't want to unnecessarily create empty directory.

return configFile

}

var mkdirErr error

if mkdirErr = file.MkdirAll(configDir, os.ModePerm); mkdirErr == nil {

return configFile

}

// Problem: Try a fallback location. If we did find a home directory then

// just assume file .rclone.conf (legacy hidden filename) can be written in

// its root (~/.rclone.conf).

if homeDir != "" {

fs.Debugf(nil, "Configuration directory could not be created and will not be used: %v", mkdirErr)

return filepath.Join(homeDir, hiddenConfigFileName)

}

if !configSupplied {

fs.Errorf(nil, "Couldn't find home directory nor create configuration directory: %v", mkdirErr)

}

} else if !configSupplied {

if homeDirErr != nil {

fs.Errorf(nil, "Couldn't find configuration directory nor home directory: %v", homeDirErr)

} else {

fs.Errorf(nil, "Couldn't find configuration directory nor home directory")

}

}

// No known location that can be used: Did possibly find a configDir

// (XDG_CONFIG_HOME or APPDATA) which couldn't be created, but in any case

// did not find a home directory!

// Report it as an error, and return as last resort the path relative to current

// working directory, of .rclone.conf (legacy hidden filename).

if !configSupplied {

fs.Errorf(nil, "Defaulting to storing config in current directory.")

fs.Errorf(nil, "Use --config flag to workaround.")

}

return hiddenConfigFileName

}

// GetConfigPath returns the current config file path

func GetConfigPath() string {

return configPath

}

// SetConfigPath sets new config file path

//

// Checks for empty string, os null device, or special path, all of which indicates in-memory config.

func SetConfigPath(path string) (err error) {

var cfgPath string

if path == "" || path == os.DevNull {

cfgPath = ""

} else if filepath.Base(path) == noConfigFile {

cfgPath = ""

} else if err = file.IsReserved(path); err != nil {

return err

} else if cfgPath, err = filepath.Abs(path); err != nil {

return err

}

configPath = cfgPath

return nil

}

// SetData sets new config file storage

func SetData(newData Storage) {

// If no config file, use in-memory config (which is the default)

if configPath == "" {

return

}

data = newData

dataLoaded = false

}

// Data returns current config file storage

func Data() Storage {

return data

}

// ErrorConfigFileNotFound is returned when the config file is not found

var ErrorConfigFileNotFound = errors.New("config file not found")

// LoadedData ensures the config file storage is loaded and returns it

func LoadedData() Storage {

if !dataLoaded {

// Set RCLONE_CONFIG_DIR for backend config and subprocesses

// If empty configPath (in-memory only) the value will be "."

_ = os.Setenv("RCLONE_CONFIG_DIR", filepath.Dir(configPath))

// Load configuration from file (or initialize sensible default if no file or error)

if err := data.Load(); err == nil {

fs.Debugf(nil, "Using config file from %q", configPath)

dataLoaded = true

} else if err == ErrorConfigFileNotFound {

if configPath == "" {

fs.Debugf(nil, "Config is memory-only - using defaults")

} else {

fs.Logf(nil, "Config file %q not found - using defaults", configPath)

}

dataLoaded = true

} else {

fs.Fatalf(nil, "Failed to load config file %q: %v", configPath, err)

}

}

return data

}

// SaveConfig calling function which saves configuration file.

// if SaveConfig returns error trying again after sleep.

func SaveConfig() {

ctx := context.Background()

ci := fs.GetConfig(ctx)

var err error

for range ci.LowLevelRetries + 1 {

if err = LoadedData().Save(); err == nil {

return

}

waitingTimeMs := mathrand.Intn(1000)

time.Sleep(time.Duration(waitingTimeMs) * time.Millisecond)

}

fs.Errorf(nil, "Failed to save config after %d tries: %v", ci.LowLevelRetries, err)

}

// FileSections returns the sections in the config file

func FileSections() []string {

return LoadedData().GetSectionList()

}

// FileGetValue gets the config key under section returning the

// the value and true if found and or ("", false) otherwise

func FileGetValue(section, key string) (string, bool) {

return LoadedData().GetValue(section, key)

}

// FileSetValue sets the key in section to value.

// It doesn't save the config file.

func FileSetValue(section, key, value string) {

LoadedData().SetValue(section, key, value)

}

// FileDeleteKey deletes the config key in the config file.

// It returns true if the key was deleted,

// or returns false if the section or key didn't exist.

func FileDeleteKey(section, key string) bool {

return LoadedData().DeleteKey(section, key)

}

// GetValue gets the value for a config key from environment

// or config file under section returning the default if not set.

//

// Emulates the preference documented and normally used by rclone via

// configmap, which means environment variables before config file.

func GetValue(remote, key string) string {

envKey := fs.ConfigToEnv(remote, key)

value, found := os.LookupEnv(envKey)

if found {

return value

}

value, _ = LoadedData().GetValue(remote, key)

return value

}

// SetValueAndSave sets the key to the value and saves just that

// value in the config file. It loads the old config file in from

// disk first and overwrites the given value only.

func SetValueAndSave(remote, key, value string) error {

// Set the value in config in case we fail to reload it

FileSetValue(remote, key, value)

// Save it again

SaveConfig()

return nil

}

// Remote defines a remote with a name, type, source and description

type Remote struct {

Name string `json:"name"`

Type string `json:"type"`

Source string `json:"source"`

Description string `json:"description"`

}

var remoteEnvRe = regexp.MustCompile(`^RCLONE_CONFIG_(.+?)_TYPE=(.+)$`)

// GetRemotes returns the list of remotes defined in environment and config file.

//

// Emulates the preference documented and normally used by rclone via

// configmap, which means environment variables before config file.

func GetRemotes() []Remote {

var remotes []Remote

for _, item := range os.Environ() {

matches := remoteEnvRe.FindStringSubmatch(item)

if len(matches) == 3 {

remotes = append(remotes, Remote{

Name: strings.ToLower(matches[1]),

Type: strings.ToLower(matches[2]),

Source: "environment",

})

}

}

remoteExists := func(name string) bool {

for _, remote := range remotes {

if name == remote.Name {

return true

}

}

return false

}

sections := LoadedData().GetSectionList()

for _, section := range sections {

if !remoteExists(section) {

typeValue, found := LoadedData().GetValue(section, "type")

if found {

description, _ := LoadedData().GetValue(section, "description")

remotes = append(remotes, Remote{

Name: section,

Type: typeValue,

Source: "file",

Description: description,

})

}

}

}

return remotes

}

// GetRemoteNames returns the names of remotes defined in environment and config file.

func GetRemoteNames() []string {

remotes := GetRemotes()

var remoteNames []string

for _, remote := range remotes {

remoteNames = append(remoteNames, remote.Name)

}

return remoteNames

}

// UpdateRemoteOpt configures the remote update

type UpdateRemoteOpt struct {

// Treat all passwords as plain that need obscuring

Obscure bool `json:"obscure"`

// Treat all passwords as obscured

NoObscure bool `json:"noObscure"`

// Don't provide any output

NoOutput bool `json:"noOutput"`

// Don't interact with the user - return questions

NonInteractive bool `json:"nonInteractive"`

// If set then supply state and result parameters to continue the process

Continue bool `json:"continue"`

// If set then ask all the questions, not just the post config questions

All bool `json:"all"`

// State to restart with - used with Continue

State string `json:"state"`

// Result to return - used with Continue

Result string `json:"result"`

// If set then edit existing values

Edit bool `json:"edit"`

}

func updateRemote(ctx context.Context, name string, keyValues rc.Params, opt UpdateRemoteOpt) (out *fs.ConfigOut, err error) {

if opt.Obscure && opt.NoObscure {

return nil, errors.New("can't use --obscure and --no-obscure together")

}

err = fspath.CheckConfigName(name)

if err != nil {

return nil, err

}

interactive := !(opt.NonInteractive || opt.Continue)

if interactive && !opt.All {

ctx = suppressConfirm(ctx)

}

fsType := GetValue(name, "type")

if fsType == "" {

return nil, errors.New("couldn't find type field in config")

}

ri, err := fs.Find(fsType)

if err != nil {

return nil, fmt.Errorf("couldn't find backend for type %q", fsType)

}

// Work out which options need to be obscured

needsObscure := map[string]struct{}{}

if !opt.NoObscure {

for _, option := range ri.Options {

if option.IsPassword {

needsObscure[option.Name] = struct{}{}

}

}

}

choices := configmap.Simple{}

m := fs.ConfigMap(ri.Prefix, ri.Options, name, nil)

// Set the config

for k, v := range keyValues {

vStr := fmt.Sprint(v)

if strings.ContainsAny(k, "\n\r") || strings.ContainsAny(vStr, "\n\r") {

return nil, fmt.Errorf("update remote: invalid key or value contains \\n or \\r")

}

// Obscure parameter if necessary

if _, ok := needsObscure[k]; ok {

_, err := obscure.Reveal(vStr)

if err != nil || opt.Obscure {

// If error => not already obscured, so obscure it

// or we are forced to obscure

vStr, err = obscure.Obscure(vStr)

if err != nil {

return nil, fmt.Errorf("update remote: obscure failed: %w", err)

}

}

}

choices.Set(k, vStr)

if !strings.HasPrefix(k, fs.ConfigKeyEphemeralPrefix) {

m.Set(k, vStr)

}

}

if opt.Edit {

choices[fs.ConfigEdit] = "true"

}

if interactive {

var state = ""

if opt.All {

state = fs.ConfigAll

}

err = backendConfig(ctx, name, m, ri, choices, state)

} else {

// Start the config state machine

in := fs.ConfigIn{

State: opt.State,

Result: opt.Result,

}

if in.State == "" && opt.All {

in.State = fs.ConfigAll

}

out, err = fs.BackendConfig(ctx, name, m, ri, choices, in)

}

if err != nil {

return nil, err

}

SaveConfig()

cache.ClearConfig(name) // remove any remotes based on this config from the cache

return out, nil

}

// UpdateRemote adds the keyValues passed in to the remote of name.

// keyValues should be key, value pairs.

func UpdateRemote(ctx context.Context, name string, keyValues rc.Params, opt UpdateRemoteOpt) (out *fs.ConfigOut, err error) {

opt.Edit = true

return updateRemote(ctx, name, keyValues, opt)

}

// CreateRemote creates a new remote with name, type and a list of

// parameters which are key, value pairs. If update is set then it

// adds the new keys rather than replacing all of them.

func CreateRemote(ctx context.Context, name string, Type string, keyValues rc.Params, opts UpdateRemoteOpt) (out *fs.ConfigOut, err error) {

err = fspath.CheckConfigName(name)

if err != nil {

return nil, err

}

if !opts.Continue {

// Delete the old config if it exists

LoadedData().DeleteSection(name)

// Set the type

LoadedData().SetValue(name, "type", Type)

}

// Set the remaining values

return UpdateRemote(ctx, name, keyValues, opts)

}

// PasswordRemote adds the keyValues passed in to the remote of name.

// keyValues should be key, value pairs.

func PasswordRemote(ctx context.Context, name string, keyValues rc.Params) error {

ctx = suppressConfirm(ctx)

err := fspath.CheckConfigName(name)

if err != nil {

return err

}

for k, v := range keyValues {

keyValues[k] = obscure.MustObscure(fmt.Sprint(v))

}

_, err = UpdateRemote(ctx, name, keyValues, UpdateRemoteOpt{

NoObscure: true,

})

return err

}

// JSONListProviders prints all the providers and options in JSON format

func JSONListProviders() error {

b, err := json.MarshalIndent(fs.Registry, "", " ")

if err != nil {

return fmt.Errorf("failed to marshal examples: %w", err)

}

_, err = os.Stdout.Write(b)

if err != nil {

return fmt.Errorf("failed to write providers list: %w", err)

}

return nil

}

// fsOption returns an Option describing the possible remotes

func fsOption() *fs.Option {

o := &fs.Option{

Name: "Storage",

Help: "Type of storage to configure.",

Default: "",

Required: true,

}

for _, item := range fs.Registry {

if item.Hide {

continue

}

example := fs.OptionExample{

Value: item.Name,

Help: item.Description,

}

o.Examples = append(o.Examples, example)

}

o.Examples.Sort()

return o

}

// DumpRcRemote dumps the config for a single remote

func DumpRcRemote(name string) (dump rc.Params) {

params := rc.Params{}

for _, key := range LoadedData().GetKeyList(name) {

params[key] = GetValue(name, key)

}

return params

}

// DumpRcBlob dumps all the config as an unstructured blob suitable

// for the rc

func DumpRcBlob() (dump rc.Params) {

dump = rc.Params{}

for _, name := range LoadedData().GetSectionList() {

dump[name] = DumpRcRemote(name)

}

return dump

}

// Dump dumps all the config as a JSON file

func Dump() error {

dump := DumpRcBlob()

b, err := json.MarshalIndent(dump, "", " ")

if err != nil {

return fmt.Errorf("failed to marshal config dump: %w", err)

}

_, err = os.Stdout.Write(b)

if err != nil {

return fmt.Errorf("failed to write config dump: %w", err)

}

return nil

}

// makeCacheDir returns a directory to use for caching.

func makeCacheDir() (dir string) {

dir, err := os.UserCacheDir()

if err != nil || dir == "" {

fs.Debugf(nil, "Failed to find user cache dir, using temporary directory: %v", err)

// if no dir found then use TempDir - we will have a cachedir!

dir = os.TempDir()

}

return filepath.Join(dir, "rclone")

}

// GetCacheDir returns the default directory for cache

//

// The directory is neither guaranteed to exist nor have accessible permissions.

// Users of this should make a subdirectory and use MkdirAll() to create it

// and any parents.

func GetCacheDir() string {

return cacheDir

}

// SetCacheDir sets new default directory for cache

func SetCacheDir(path string) (err error) {

cacheDir, err = filepath.Abs(path)

return

}

// SetTempDir sets new default directory to use for temporary files.

//

// Assuming golang's os.TempDir is used to get the directory:

// "On Unix systems, it returns $TMPDIR if non-empty, else /tmp. On Windows,

// it uses GetTempPath, returning the first non-empty value from %TMP%, %TEMP%,

// %USERPROFILE%, or the Windows directory."

//

// To override the default we therefore set environment variable TMPDIR

// on Unix systems, and both TMP and TEMP on Windows (they are almost exclusively

// aliases for the same path, and programs may refer to either of them).

// This should make all libraries and forked processes use the same.

func SetTempDir(path string) (err error) {

var tempDir string

if tempDir, err = filepath.Abs(path); err != nil {

return err

}

if runtime.GOOS == "windows" {

if err = os.Setenv("TMP", tempDir); err != nil {

return err

}

if err = os.Setenv("TEMP", tempDir); err != nil {

return err

}

} else {

return os.Setenv("TMPDIR", tempDir)

}

return nil

}