// Package onedrive provides an interface to the Microsoft OneDrive

// object storage system.

package onedrive

import (

"context"

_ "embed"

"encoding/base64"

"encoding/hex"

"encoding/json"

"errors"

"fmt"

"io"

"net/http"

"net/url"

"path"

"regexp"

"strconv"

"strings"

"sync"

"time"

"github.com/rclone/rclone/backend/onedrive/api"

"github.com/rclone/rclone/backend/onedrive/quickxorhash"

"github.com/rclone/rclone/fs"

"github.com/rclone/rclone/fs/config"

"github.com/rclone/rclone/fs/config/configmap"

"github.com/rclone/rclone/fs/config/configstruct"

"github.com/rclone/rclone/fs/config/obscure"

"github.com/rclone/rclone/fs/fserrors"

"github.com/rclone/rclone/fs/fshttp"

"github.com/rclone/rclone/fs/hash"

"github.com/rclone/rclone/fs/list"

"github.com/rclone/rclone/fs/log"

"github.com/rclone/rclone/fs/operations"

"github.com/rclone/rclone/fs/walk"

"github.com/rclone/rclone/lib/atexit"

"github.com/rclone/rclone/lib/dircache"

"github.com/rclone/rclone/lib/encoder"

"github.com/rclone/rclone/lib/oauthutil"

"github.com/rclone/rclone/lib/pacer"

"github.com/rclone/rclone/lib/readers"

"github.com/rclone/rclone/lib/rest"

)

const (

rcloneClientID = "b15665d9-eda6-4092-8539-0eec376afd59"

rcloneEncryptedClientSecret = "_JUdzh3LnKNqSPcf4Wu5fgMFIQOI8glZu_akYgR8yf6egowNBg-R"

minSleep = 10 * time.Millisecond

maxSleep = 2 * time.Second

decayConstant = 2 // bigger for slower decay, exponential

configDriveID = "drive_id"

configDriveType = "drive_type"

driveTypePersonal = "personal"

driveTypeBusiness = "business"

driveTypeSharepoint = "documentLibrary"

defaultChunkSize = 10 * fs.Mebi

chunkSizeMultiple = 320 * fs.Kibi

maxSinglePartSize = 4 * fs.Mebi

regionGlobal = "global"

regionUS = "us"

regionDE = "de"

regionCN = "cn"

)

// Globals

var (

// Define the paths used for token operations

commonPathPrefix = "/common" // prefix for the paths if tenant isn't known

authPath = "/oauth2/v2.0/authorize"

tokenPath = "/oauth2/v2.0/token"

scopeAccess = fs.SpaceSepList{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "Sites.Read.All", "offline_access"}

scopeAccessWithoutSites = fs.SpaceSepList{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "offline_access"}

// When using client credential OAuth flow, scope of .default is required in order

// to use the permissions configured for the application within the tenant

scopeAccessClientCred = fs.SpaceSepList{".default"}

// Base config for how to auth

oauthConfig = &oauthutil.Config{

Scopes: scopeAccess,

ClientID: rcloneClientID,

ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),

RedirectURL: oauthutil.RedirectLocalhostURL,

}

graphAPIEndpoint = map[string]string{

"global": "https://graph.microsoft.com",

"us": "https://graph.microsoft.us",

"de": "https://graph.microsoft.de",

"cn": "https://microsoftgraph.chinacloudapi.cn",

}

authEndpoint = map[string]string{

"global": "https://login.microsoftonline.com",

"us": "https://login.microsoftonline.us",

"de": "https://login.microsoftonline.de",

"cn": "https://login.chinacloudapi.cn",

}

// QuickXorHashType is the hash.Type for OneDrive

QuickXorHashType hash.Type

//go:embed metadata.md

metadataHelp string

)

// Register with Fs

func init() {

QuickXorHashType = hash.RegisterHash("quickxor", "QuickXorHash", 40, quickxorhash.New)

fs.Register(&fs.RegInfo{

Name: "onedrive",

Description: "Microsoft OneDrive",

NewFs: NewFs,

Config: Config,

MetadataInfo: &fs.MetadataInfo{

System: systemMetadataInfo,

Help: metadataHelp,

},

Options: append(oauthutil.SharedOptions, []fs.Option{{

Name: "region",

Help: "Choose national cloud region for OneDrive.",

Default: "global",

Examples: []fs.OptionExample{

{

Value: regionGlobal,

Help: "Microsoft Cloud Global",

}, {

Value: regionUS,

Help: "Microsoft Cloud for US Government",

}, {

Value: regionDE,

Help: "Microsoft Cloud Germany (deprecated - try " + regionGlobal + " region first).",

}, {

Value: regionCN,

Help: "Azure and Office 365 operated by Vnet Group in China",

},

},

}, {

Name: "upload_cutoff",

Help: `Cutoff for switching to chunked upload.

Any files larger than this will be uploaded in chunks of chunk_size.

This is disabled by default as uploading using single part uploads

causes rclone to use twice the storage on Onedrive business as when

rclone sets the modification time after the upload Onedrive creates a

new version.

See: https://github.com/rclone/rclone/issues/1716

`,

Default: fs.SizeSuffix(-1),

Advanced: true,

}, {

Name: "chunk_size",

Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).

Above this size files will be chunked - must be multiple of 320k (327,680 bytes) and

should not exceed 250M (262,144,000 bytes) else you may encounter \"Microsoft.SharePoint.Client.InvalidClientQueryException: The request message is too big.\"

Note that the chunks will be buffered into memory.`,

Default: defaultChunkSize,

Advanced: true,

}, {

Name: "drive_id",

Help: "The ID of the drive to use.",

Default: "",

Advanced: true,

Sensitive: true,

}, {

Name: "drive_type",

Help: "The type of the drive (" + driveTypePersonal + " | " + driveTypeBusiness + " | " + driveTypeSharepoint + ").",

Default: "",

Advanced: true,

}, {

Name: "root_folder_id",

Help: `ID of the root folder.

This isn't normally needed, but in special circumstances you might

know the folder ID that you wish to access but not be able to get

there through a path traversal.

`,

Advanced: true,

Sensitive: true,

}, {

Name: "access_scopes",

Help: `Set scopes to be requested by rclone.

Choose or manually enter a custom space separated list with all scopes, that rclone should request.

`,

Default: scopeAccess,

Advanced: true,

Examples: []fs.OptionExample{

{

Value: "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access",

Help: "Read and write access to all resources",

},

{

Value: "Files.Read Files.Read.All Sites.Read.All offline_access",

Help: "Read only access to all resources",

},

{

Value: "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All offline_access",

Help: "Read and write access to all resources, without the ability to browse SharePoint sites. \nSame as if disable_site_permission was set to true",

},

},

}, {

Name: "tenant",

Help: `ID of the service principal's tenant. Also called its directory ID.

Set this if using

- Client Credential flow

`,

Sensitive: true,

}, {

Name: "disable_site_permission",

Help: `Disable the request for Sites.Read.All permission.

If set to true, you will no longer be able to search for a SharePoint site when

configuring drive ID, because rclone will not request Sites.Read.All permission.

Set it to true if your organization didn't assign Sites.Read.All permission to the

application, and your organization disallows users to consent app permission

request on their own.`,

Default: false,

Advanced: true,

Hide: fs.OptionHideBoth,

}, {

Name: "expose_onenote_files",

Help: `Set to make OneNote files show up in directory listings.

By default, rclone will hide OneNote files in directory listings because

operations like "Open" and "Update" won't work on them. But this

behaviour may also prevent you from deleting them. If you want to

delete OneNote files or otherwise want them to show up in directory

listing, set this option.`,

Default: false,

Advanced: true,

}, {

Name: "server_side_across_configs",

Default: false,

Help: `Deprecated: use --server-side-across-configs instead.

Allow server-side operations (e.g. copy) to work across different onedrive configs.

This will work if you are copying between two OneDrive *Personal* drives AND the files to

copy are already shared between them. Additionally, it should also function for a user who

has access permissions both between Onedrive for *business* and *SharePoint* under the *same

tenant*, and between *SharePoint* and another *SharePoint* under the *same tenant*. In other

cases, rclone will fall back to normal copy (which will be slightly slower).`,

Advanced: true,

}, {

Name: "list_chunk",

Help: "Size of listing chunk.",

Default: 1000,

Advanced: true,

}, {

Name: "no_versions",

Default: false,

Help: `Remove all versions on modifying operations.

Onedrive for business creates versions when rclone uploads new files

overwriting an existing one and when it sets the modification time.

These versions take up space out of the quota.

This flag checks for versions after file upload and setting

modification time and removes all but the last version.

**NB** Onedrive personal can't currently delete versions so don't use

this flag there.

`,

Advanced: true,

}, {

Name: "hard_delete",

Help: `Permanently delete files on removal.

Normally files will get sent to the recycle bin on deletion. Setting

this flag causes them to be permanently deleted. Use with care.

OneDrive personal accounts do not support the permanentDelete API,

it only applies to OneDrive for Business and SharePoint document libraries.

`,

Advanced: true,

Default: false,

}, {

Name: "link_scope",

Default: "anonymous",

Help: `Set the scope of the links created by the link command.`,

Advanced: true,

Examples: []fs.OptionExample{{

Value: "anonymous",

Help: "Anyone with the link has access, without needing to sign in.\nThis may include people outside of your organization.\nAnonymous link support may be disabled by an administrator.",

}, {

Value: "organization",

Help: "Anyone signed into your organization (tenant) can use the link to get access.\nOnly available in OneDrive for Business and SharePoint.",

}},

}, {

Name: "link_type",

Default: "view",

Help: `Set the type of the links created by the link command.`,

Advanced: true,

Examples: []fs.OptionExample{{

Value: "view",

Help: "Creates a read-only link to the item.",

}, {

Value: "edit",

Help: "Creates a read-write link to the item.",

}, {

Value: "embed",

Help: "Creates an embeddable link to the item.",

}},

}, {

Name: "link_password",

Default: "",

Help: `Set the password for links created by the link command.

At the time of writing this only works with OneDrive personal paid accounts.

`,

Advanced: true,

Sensitive: true,

}, {

Name: "hash_type",

Default: "auto",

Help: `Specify the hash in use for the backend.

This specifies the hash type in use. If set to "auto" it will use the

default hash which is QuickXorHash.

Before rclone 1.62 an SHA1 hash was used by default for Onedrive

Personal. For 1.62 and later the default is to use a QuickXorHash for

all onedrive types. If an SHA1 hash is desired then set this option

accordingly.

From July 2023 QuickXorHash will be the only available hash for

both OneDrive for Business and OneDrive Personal.

This can be set to "none" to not use any hashes.

If the hash requested does not exist on the object, it will be

returned as an empty string which is treated as a missing hash by

rclone.

`,

Examples: []fs.OptionExample{{

Value: "auto",

Help: "Rclone chooses the best hash",

}, {

Value: "quickxor",

Help: "QuickXor",

}, {

Value: "sha1",

Help: "SHA1",

}, {

Value: "sha256",

Help: "SHA256",

}, {

Value: "crc32",

Help: "CRC32",

}, {

Value: "none",

Help: "None - don't use any hashes",

}},

Advanced: true,

}, {

Name: "av_override",

Default: false,

Help: `Allows download of files the server thinks has a virus.

The onedrive/sharepoint server may check files uploaded with an Anti

Virus checker. If it detects any potential viruses or malware it will

block download of the file.

In this case you will see a message like this

server reports this file is infected with a virus - use --onedrive-av-override to download anyway: Infected (name of virus): 403 Forbidden:

If you are 100% sure you want to download this file anyway then use

the --onedrive-av-override flag, or av_override = true in the config

file.

`,

Advanced: true,

}, {

Name: "delta",

Default: false,

Help: strings.ReplaceAll(`If set rclone will use delta listing to implement recursive listings.

If this flag is set the onedrive backend will advertise |ListR|

support for recursive listings.

Setting this flag speeds up these things greatly:

rclone lsf -R onedrive:

rclone size onedrive:

rclone rc vfs/refresh recursive=true

**However** the delta listing API **only** works at the root of the

drive. If you use it not at the root then it recurses from the root

and discards all the data that is not under the directory you asked

for. So it will be correct but may not be very efficient.

This is why this flag is not set as the default.

As a rule of thumb if nearly all of your data is under rclone's root

directory (the |root/directory| in |onedrive:root/directory|) then

using this flag will be a big performance win. If your data is

mostly not under the root then using this flag will be a big

performance loss.

It is recommended if you are mounting your onedrive at the root

(or near the root when using crypt) and using rclone |rc vfs/refresh|.

`, "|", "`"),

Advanced: true,

}, {

Name: "metadata_permissions",

Help: `Control whether permissions should be read or written in metadata.

Reading permissions metadata from files can be done quickly, but it

isn't always desirable to set the permissions from the metadata.

`,

Advanced: true,

Default: rwOff,

Examples: rwExamples,

}, {

Name: config.ConfigEncoding,

Help: config.ConfigEncodingHelp,

Advanced: true,

// List of replaced characters:

// < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN

// > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN

// : (colon) -> '：' // FULLWIDTH COLON

// " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK

// \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS

// | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE

// ? (question mark) -> '？' // FULLWIDTH QUESTION MARK

// * (asterisk) -> '＊' // FULLWIDTH ASTERISK

//

// Folder names cannot begin with a tilde ('~')

// List of replaced characters:

// ~ (tilde) -> '～' // FULLWIDTH TILDE

//

// Additionally names can't begin with a space ( ) or end with a period (.) or space ( ).

// List of replaced characters:

// . (period) -> '．' // FULLWIDTH FULL STOP

// (space) -> '␠' // SYMBOL FOR SPACE

//

// Also encode invalid UTF-8 bytes as json doesn't handle them.

//

// The OneDrive API documentation lists the set of reserved characters, but

// testing showed this list is incomplete. This are the differences:

// - " (double quote) is rejected, but missing in the documentation

// - space at the end of file and folder names is rejected, but missing in the documentation

// - period at the end of file names is rejected, but missing in the documentation

//

// Adding these restrictions to the OneDrive API documentation yields exactly

// the same rules as the Windows naming conventions.

//

// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online#path-encoding

Default: (encoder.Display |

encoder.EncodeBackSlash |

encoder.EncodeLeftSpace |

encoder.EncodeLeftTilde |

encoder.EncodeRightPeriod |

encoder.EncodeRightSpace |

encoder.EncodeWin |

encoder.EncodeInvalidUtf8),

}}...),

})

}

// Get the region and graphURL from the config

func getRegionURL(m configmap.Mapper) (region, graphURL string) {

region, _ = m.Get("region")

graphURL = graphAPIEndpoint[region] + "/v1.0"

return region, graphURL

}

// Config for chooseDrive

type chooseDriveOpt struct {

opts rest.Opts

finalDriveID string

siteID string

relativePath string

}

// chooseDrive returns a query to choose which drive the user is interested in

func chooseDrive(ctx context.Context, name string, m configmap.Mapper, srv *rest.Client, opt chooseDriveOpt) (*fs.ConfigOut, error) {

_, graphURL := getRegionURL(m)

// if we use server-relative URL for finding the drive

if opt.relativePath != "" {

opt.opts = rest.Opts{

Method: "GET",

RootURL: graphURL,

Path: "/sites/root:" + opt.relativePath,

}

site := api.SiteResource{}

_, err := srv.CallJSON(ctx, &opt.opts, nil, &site)

if err != nil {

return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available site by relative path: %v", err))

}

opt.siteID = site.SiteID

}

// if we have a siteID we need to ask for the drives

if opt.siteID != "" {

opt.opts = rest.Opts{

Method: "GET",

RootURL: graphURL,

Path: "/sites/" + opt.siteID + "/drives",

}

}

drives := api.DrivesResponse{}

// We don't have the final ID yet?

// query Microsoft Graph

if opt.finalDriveID == "" {

_, err := srv.CallJSON(ctx, &opt.opts, nil, &drives)

if err != nil {

return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available drives: %v", err))

}

// Also call /me/drive as sometimes /me/drives doesn't return it #4068

if opt.opts.Path == "/me/drives" {

opt.opts.Path = "/me/drive"

meDrive := api.DriveResource{}

_, err := srv.CallJSON(ctx, &opt.opts, nil, &meDrive)

if err != nil {

return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available drives: %v", err))

}

found := false

for _, drive := range drives.Drives {

if drive.DriveID == meDrive.DriveID {

found = true

break

}

}

// add the me drive if not found already

if !found {

fs.Debugf(nil, "Adding %v to drives list from /me/drive", meDrive)

drives.Drives = append(drives.Drives, meDrive)

}

}

} else {

drives.Drives = append(drives.Drives, api.DriveResource{

DriveID: opt.finalDriveID,

DriveName: "Chosen Drive ID",

DriveType: "drive",

})

}

if len(drives.Drives) == 0 {

return fs.ConfigError("choose_type", "No drives found")

}

return fs.ConfigChoose("driveid_final", "config_driveid", "Select drive you want to use", len(drives.Drives), func(i int) (string, string) {

drive := drives.Drives[i]

return drive.DriveID, fmt.Sprintf("%s (%s)", drive.DriveName, drive.DriveType)

})

}

// Make the oauth config for the backend

func makeOauthConfig(ctx context.Context, opt *Options) (*oauthutil.Config, error) {

// Copy the default oauthConfig

oauthConfig := *oauthConfig

// Set the scopes

oauthConfig.Scopes = opt.AccessScopes

if opt.DisableSitePermission {

oauthConfig.Scopes = scopeAccessWithoutSites

}

// Construct the auth URLs

prefix := commonPathPrefix

if opt.Tenant != "" {

prefix = "/" + opt.Tenant

}

oauthConfig.TokenURL = authEndpoint[opt.Region] + prefix + tokenPath

oauthConfig.AuthURL = authEndpoint[opt.Region] + prefix + authPath

// Check to see if we are using client credentials flow

if opt.ClientCredentials {

// Override scope to .default

oauthConfig.Scopes = scopeAccessClientCred

if opt.Tenant == "" {

return nil, fmt.Errorf("tenant parameter must be set when using %s", config.ConfigClientCredentials)

}

}

return &oauthConfig, nil

}

// Config the backend

func Config(ctx context.Context, name string, m configmap.Mapper, conf fs.ConfigIn) (*fs.ConfigOut, error) {

opt := new(Options)

err := configstruct.Set(m, opt)

if err != nil {

return nil, err

}

_, graphURL := getRegionURL(m)

// Check to see if this is the start of the state machine execution

if conf.State == "" {

conf, err := makeOauthConfig(ctx, opt)

if err != nil {

return nil, err

}

return oauthutil.ConfigOut("choose_type", &oauthutil.Options{

OAuth2Config: conf,

})

}

oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)

if err != nil {

return nil, fmt.Errorf("failed to configure OneDrive: %w", err)

}

// Create a REST client, build on the OAuth client created above

srv := rest.NewClient(oAuthClient)

switch conf.State {

case "choose_type":

return fs.ConfigChooseExclusiveFixed("choose_type_done", "config_type", "Type of connection", []fs.OptionExample{{

Value: "onedrive",

Help: "OneDrive Personal or Business",

}, {

Value: "sharepoint",

Help: "Root Sharepoint site",

}, {

Value: "url",

Help: "Sharepoint site name or URL\nE.g. mysite or https://contoso.sharepoint.com/sites/mysite",

}, {

Value: "search",

Help: "Search for a Sharepoint site",

}, {

Value: "driveid",

Help: "Type in driveID (advanced)",

}, {

Value: "siteid",

Help: "Type in SiteID (advanced)",

}, {

Value: "path",

Help: "Sharepoint server-relative path (advanced)\nE.g. /teams/hr",

}})

case "choose_type_done":

// Jump to next state according to config chosen

return fs.ConfigGoto(conf.Result)

case "onedrive":

return chooseDrive(ctx, name, m, srv, chooseDriveOpt{

opts: rest.Opts{

Method: "GET",

RootURL: graphURL,

Path: "/me/drives",

},

})

case "sharepoint":

return chooseDrive(ctx, name, m, srv, chooseDriveOpt{

opts: rest.Opts{

Method: "GET",

RootURL: graphURL,

Path: "/sites/root/drives",

},

})

case "driveid":

out, err := fs.ConfigInput("driveid_end", "config_driveid_fixed", "Drive ID")

if err != nil {

return out, err

}

// Default the drive_id to the previous version in the config

out.Option.Default, _ = m.Get("drive_id")

return out, nil

case "driveid_end":

return chooseDrive(ctx, name, m, srv, chooseDriveOpt{

finalDriveID: conf.Result,

})

case "siteid":

return fs.ConfigInput("siteid_end", "config_siteid", "Site ID")

case "siteid_end":

return chooseDrive(ctx, name, m, srv, chooseDriveOpt{

siteID: conf.Result,

})

case "url":

return fs.ConfigInput("url_end", "config_site_url", `Site URL

Examples:

- "mysite"

- "https://XXX.sharepoint.com/sites/mysite"

- "https://XXX.sharepoint.com/teams/ID"

`)

case "url_end":

siteURL := conf.Result

re := regexp.MustCompile(`https://.*\.sharepoint\.com(/.*)`)

match := re.FindStringSubmatch(siteURL)

if len(match) == 2 {

return chooseDrive(ctx, name, m, srv, chooseDriveOpt{

relativePath: match[1],

})

}

return chooseDrive(ctx, name, m, srv, chooseDriveOpt{

relativePath: "/sites/" + siteURL,

})

case "path":

return fs.ConfigInput("path_end", "config_sharepoint_url", `Server-relative URL`)

case "path_end":

return chooseDrive(ctx, name, m, srv, chooseDriveOpt{

relativePath: conf.Result,

})

case "search":

return fs.ConfigInput("search_end", "config_search_term", `Search term`)

case "search_end":

searchTerm := conf.Result

opts := rest.Opts{

Method: "GET",

RootURL: graphURL,

Path: "/sites?search=" + searchTerm,

}

sites := api.SiteResponse{}

_, err := srv.CallJSON(ctx, &opts, nil, &sites)

if err != nil {

return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available sites: %v", err))

}

if len(sites.Sites) == 0 {

return fs.ConfigError("choose_type", fmt.Sprintf("search for %q returned no results", searchTerm))

}

return fs.ConfigChoose("search_sites", "config_site", `Select the Site you want to use`, len(sites.Sites), func(i int) (string, string) {

site := sites.Sites[i]

return site.SiteID, fmt.Sprintf("%s (%s)", site.SiteName, site.SiteURL)

})

case "search_sites":

return chooseDrive(ctx, name, m, srv, chooseDriveOpt{

siteID: conf.Result,

})

case "driveid_final":

finalDriveID := conf.Result

// Test the driveID and get drive type

opts := rest.Opts{

Method: "GET",

RootURL: graphURL,

Path: "/drives/" + finalDriveID + "/root",

}

var rootItem api.Item

_, err = srv.CallJSON(ctx, &opts, nil, &rootItem)

if err != nil {

return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query root for drive %q: %v", finalDriveID, err))

}

m.Set(configDriveID, finalDriveID)

m.Set(configDriveType, rootItem.ParentReference.DriveType)

return fs.ConfigConfirm("driveid_final_end", true, "config_drive_ok", fmt.Sprintf("Drive OK?\n\nFound drive %q of type %q\nURL: %s\n", rootItem.Name, rootItem.ParentReference.DriveType, rootItem.WebURL))

case "driveid_final_end":

if conf.Result == "true" {

return nil, nil

}

return fs.ConfigGoto("choose_type")

}

return nil, fmt.Errorf("unknown state %q", conf.State)

}

// Options defines the configuration for this backend

type Options struct {

Region string `config:"region"`

UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`

ChunkSize fs.SizeSuffix `config:"chunk_size"`

DriveID string `config:"drive_id"`

DriveType string `config:"drive_type"`

RootFolderID string `config:"root_folder_id"`

DisableSitePermission bool `config:"disable_site_permission"`

ClientCredentials bool `config:"client_credentials"`

AccessScopes fs.SpaceSepList `config:"access_scopes"`

Tenant string `config:"tenant"`

ExposeOneNoteFiles bool `config:"expose_onenote_files"`

ServerSideAcrossConfigs bool `config:"server_side_across_configs"`

ListChunk int64 `config:"list_chunk"`

NoVersions bool `config:"no_versions"`

HardDelete bool `config:"hard_delete"`

LinkScope string `config:"link_scope"`

LinkType string `config:"link_type"`

LinkPassword string `config:"link_password"`

HashType string `config:"hash_type"`

AVOverride bool `config:"av_override"`

Delta bool `config:"delta"`

Enc encoder.MultiEncoder `config:"encoding"`

MetadataPermissions rwChoice `config:"metadata_permissions"`

}

// Fs represents a remote OneDrive

type Fs struct {

name string // name of this remote

root string // the path we are working on

opt Options // parsed options

ci *fs.ConfigInfo // global config

features *fs.Features // optional features

srv *rest.Client // the connection to the OneDrive server

unAuth *rest.Client // no authentication connection to the OneDrive server

dirCache *dircache.DirCache // Map of directory path to directory id

pacer *fs.Pacer // pacer for API calls

tokenRenewer *oauthutil.Renew // renew the token on expiry

driveID string // ID to use for querying Microsoft Graph

driveType string // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive

hashType hash.Type // type of the hash we are using

}

// Object describes a OneDrive object

//

// Will definitely have info but maybe not meta

type Object struct {

fs *Fs // what this object is part of

remote string // The remote path

hasMetaData bool // whether info below has been set

isOneNoteFile bool // Whether the object is a OneNote file

size int64 // size of the object

modTime time.Time // modification time of the object

id string // ID of the object

hash string // Hash of the content, usually QuickXorHash but set as hash_type

mimeType string // Content-Type of object from server (may not be as uploaded)

meta *Metadata // metadata properties

}

// Directory describes a OneDrive directory

type Directory struct {

fs *Fs // what this object is part of

remote string // The remote path

size int64 // size of directory and contents or -1 if unknown

items int64 // number of objects or -1 for unknown

id string // dir ID

meta *Metadata // metadata properties

}

// ------------------------------------------------------------

// Name of the remote (as passed into NewFs)

func (f *Fs) Name() string {

return f.name

}

// Root of the remote (as passed into NewFs)

func (f *Fs) Root() string {

return f.root

}

// String converts this Fs to a string

func (f *Fs) String() string {

return fmt.Sprintf("OneDrive root '%s'", f.root)

}

// Features returns the optional features of this Fs

func (f *Fs) Features() *fs.Features {

return f.features

}

// parsePath parses a OneDrive 'url'

func parsePath(path string) (root string) {

root = strings.Trim(path, "/")

return

}

// retryErrorCodes is a slice of error codes that we will retry

var retryErrorCodes = []int{

429, // Too Many Requests.

500, // Internal Server Error

502, // Bad Gateway

503, // Service Unavailable

504, // Gateway Timeout

509, // Bandwidth Limit Exceeded

}

var (

gatewayTimeoutError sync.Once

errAsyncJobAccessDenied = errors.New("async job failed - access denied")

)

// shouldRetry returns a boolean as to whether this resp and err

// deserve to be retried. It returns the err as a convenience

func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {

if fserrors.ContextError(ctx, &err) {

return false, err

}

retry := false

if resp != nil {

switch resp.StatusCode {

case 400:

if apiErr, ok := err.(*api.Error); ok {

if apiErr.ErrorInfo.InnerError.Code == "pathIsTooLong" {

return false, fserrors.NoRetryError(err)

}

}

case 401:

if len(resp.Header["Www-Authenticate"]) == 1 && strings.Contains(resp.Header["Www-Authenticate"][0], "expired_token") {

retry = true

fs.Debugf(nil, "Should retry: %v", err)

} else if err != nil && strings.Contains(err.Error(), "Unable to initialize RPS") {

retry = true

fs.Debugf(nil, "HTTP 401: Unable to initialize RPS. Trying again.")

}

case 429, 503: // Too Many Requests, Server Too Busy

// see https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online

if values := resp.Header["Retry-After"]; len(values) == 1 && values[0] != "" {

retryAfter, parseErr := strconv.Atoi(values[0])

if parseErr != nil {

fs.Debugf(nil, "Failed to parse Retry-After: %q: %v", values[0], parseErr)

} else {

duration := time.Second * time.Duration(retryAfter)

retry = true

err = pacer.RetryAfterError(err, duration)

fs.Debugf(nil, "Too many requests. Trying again in %d seconds.", retryAfter)

}

}

case 504: // Gateway timeout

gatewayTimeoutError.Do(func() {

fs.Errorf(nil, "%v: upload chunks may be taking too long - try reducing --onedrive-chunk-size or decreasing --transfers", err)

})

case 507: // Insufficient Storage

return false, fserrors.FatalError(err)

}

}

return retry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err

}

// readMetaDataForPathRelativeToID reads the metadata for a path relative to an item that is addressed by its normalized ID.

// if `relPath` == "", it reads the metadata for the item with that ID.

//

// We address items using the pattern `drives/driveID/items/itemID:/relativePath`

// instead of simply using `drives/driveID/root:/itemPath` because it works for

// "shared with me" folders in OneDrive Personal (See #2536, #2778)

// This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480

//

// If `relPath` == ”, do not append the slash (See #3664)

func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {

opts, _ := f.newOptsCallWithIDPath(normalizedID, relPath, true, "GET", "")

err = f.pacer.Call(func() (bool, error) {

resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)

return shouldRetry(ctx, resp, err)

})

return info, resp, err

}

// readMetaDataForPath reads the metadata from the path (relative to the absolute root)

func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, resp *http.Response, err error) {

firstSlashIndex := strings.IndexRune(path, '/')

if f.driveType != driveTypePersonal || firstSlashIndex == -1 {

opts := f.newOptsCallWithPath(ctx, path, "GET", "")

opts.Path = strings.TrimSuffix(opts.Path, ":")

err = f.pacer.Call(func() (bool, error) {

resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)

return shouldRetry(ctx, resp, err)

})

return info, resp, err

}

// The following branch handles the case when we're using OneDrive Personal and the path is in a folder.

// For OneDrive Personal, we need to consider the "shared with me" folders.

// An item in such a folder can only be addressed by its ID relative to the sharer's driveID or

// by its path relative to the folder's ID relative to the sharer's driveID.

// Note: A "shared with me" folder can only be placed in the sharee's absolute root.

// So we read metadata relative to a suitable folder's normalized ID.

var dirCacheFoundRoot bool

var rootNormalizedID string

if f.dirCache != nil {

rootNormalizedID, err = f.dirCache.RootID(ctx, false)

dirCacheRootIDExists := err == nil

if f.root == "" {

// if f.root == "", it means f.root is the absolute root of the drive

// and its ID should have been found in NewFs

dirCacheFoundRoot = dirCacheRootIDExists

} else if _, err := f.dirCache.RootParentID(ctx, false); err == nil {

// if root is in a folder, it must have a parent folder, and

// if dirCache has found root in NewFs, the parent folder's ID

// should be present.

// This RootParentID() check is a fix for #3164 which describes

// a possible case where the root is not found.

dirCacheFoundRoot = dirCacheRootIDExists

}

}

relPath, insideRoot := getRelativePathInsideBase(f.root, path)

var firstDir, baseNormalizedID string

if !insideRoot || !dirCacheFoundRoot {

// We do not have the normalized ID in dirCache for our query to base on. Query it manually.

firstDir, relPath = path[:firstSlashIndex], path[firstSlashIndex+1:]

info, resp, err := f.readMetaDataForPath(ctx, firstDir)

if err != nil {

return info, resp, err

}

baseNormalizedID = info.GetID()

} else {

if f.root != "" {

// Read metadata based on root

baseNormalizedID = rootNormalizedID

} else {

// Read metadata based on firstDir

firstDir, relPath = path[:firstSlashIndex], path[firstSlashIndex+1:]

baseNormalizedID, err = f.dirCache.FindDir(ctx, firstDir, false)

if err != nil {

return nil, nil, err