change(sa): associate folder with remote(s) rather than remotes to fo…

…lders

* service account count will now be for unique paths
* handle invalid service account folders

README.md

@@ -18,12 +18,13 @@ rclone:
   stats: 30s
   live_rotate: false
   service_account_remotes:
-    tv: /opt/rclone/service_accounts/crop
-    movies: /opt/rclone/service_accounts/crop
-    music: /opt/rclone/service_accounts/crop
-    4k_movies: /opt/rclone/service_accounts/crop
-    source_4k_movies: /opt/rclone/service_accounts/crop
-    staging: /opt/rclone/service_accounts/staging
+    '/opt/rclone/service_accounts/crop':
+      - tv
+      - movies
+      - music
+      - 4k_movies
+      - source_4k_movies
+      - staging
   global_params:
     default:
       move:

cmd/manual.go

@@ -48,22 +48,27 @@ var manualCmd = &cobra.Command{
 		}
 
 		// create remote to service account map
-		remoteSaFolders := make(map[string]string)
+		remoteSaFolders := make(map[string][]string)
 
 		switch flagSaFolder != "" {
 		case true:
 			if strings.Contains(flagSrc, ":") {
 				// source is a remote
 				srcRemote := stringutils.FromLeftUntil(flagSrc, ":")
 				log.Debugf("Using service account folder for %q: %v", srcRemote, flagSaFolder)
-				remoteSaFolders[srcRemote] = flagSaFolder
+				remoteSaFolders[flagSaFolder] = []string{srcRemote}
 			}
 
 			if strings.Contains(flagDest, ":") {
 				// dest is a remote
 				dstRemote := stringutils.FromLeftUntil(flagDest, ":")
 				log.Debugf("Using service account folder for %q: %v", dstRemote, flagSaFolder)
-				remoteSaFolders[dstRemote] = flagSaFolder
+
+				if _, ok := remoteSaFolders[flagSaFolder]; ok {
+					remoteSaFolders[flagSaFolder] = append(remoteSaFolders[flagSaFolder], dstRemote)
+				} else {
+					remoteSaFolders[flagSaFolder] = []string{dstRemote}
+				}
 			}
 
 		default:

config/rclone.go

@@ -6,7 +6,7 @@ type RcloneConfig struct {
 	Stats                 string
 	LiveRotate            bool                    `mapstructure:"live_rotate"`
 	DryRun                bool                    `mapstructure:"dry_run"`
-	ServiceAccountRemotes map[string]string       `mapstructure:"service_account_remotes"`
+	ServiceAccountRemotes map[string][]string     `mapstructure:"service_account_remotes"`
 	GlobalParams          map[string]RcloneParams `mapstructure:"global_params"`
 }
 

maputils/maputils.go

@@ -27,3 +27,21 @@ func GetStringMapValue(stringMap map[string]string, key string, caseSensitive bo
 
 	return "", fmt.Errorf("key was not found in map: %q", lowerKey)
 }
+
+func GetStringKeysBySliceValue(stringMap map[string][]string, value string) ([]string, error) {
+	keys := make([]string, 0)
+
+	for k, v := range stringMap {
+		for _, r := range v {
+			if strings.EqualFold(r, value) {
+				keys = append(keys, k)
+			}
+		}
+	}
+
+	if len(keys) == 0 {
+		return keys, fmt.Errorf("value was not found in map: %q", value)
+	}
+
+	return keys, nil
+}

pathutils/find.go

@@ -31,6 +31,11 @@ func GetPathsInFolder(folder string, includeFiles bool, includeFolders bool, acc
 	var paths []Path
 	var size uint64 = 0
 
+	if _, err := os.Stat(folder); os.IsNotExist(err) {
+		log.WithError(err).Error("Failed finding paths within folder")
+		return paths, size
+	}
+
 	err := filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
 		// skip files if not wanted
 		if !includeFiles && !info.IsDir() {

rclone/sa.go

@@ -33,7 +33,7 @@ type RemoteServiceAccount struct {
 
 type ServiceAccountManager struct {
 	log                         *logrus.Entry
-	remoteServiceAccountFolders map[string]string
+	remoteServiceAccountFolders map[string][]string
 	remoteServiceAccounts       map[string]RemoteServiceAccounts
 	parallelism                 int
 }
@@ -63,7 +63,7 @@ func addServiceAccountsToTempCache(serviceAccounts []*RemoteServiceAccount) {
 
 /* Public */
 
-func NewServiceAccountManager(serviceAccountFolders map[string]string, parallelism int) *ServiceAccountManager {
+func NewServiceAccountManager(serviceAccountFolders map[string][]string, parallelism int) *ServiceAccountManager {
 	return &ServiceAccountManager{
 		log:                         logger.GetLogger("sa_manager"),
 		remoteServiceAccountFolders: serviceAccountFolders,
@@ -82,12 +82,11 @@ func (m *ServiceAccountManager) LoadServiceAccounts(remotePaths []string) error
 			continue
 		}
 
-		// parse remote name and retrieve folder
+		// parse remote name and retrieve service account folder(s)
 		remoteName := stringutils.FromLeftUntil(remotePath, ":")
-		remoteServiceAccountFolder, err := maputils.GetStringMapValue(m.remoteServiceAccountFolders, remoteName,
-			false)
+		remoteServiceAccountFolders, err := maputils.GetStringKeysBySliceValue(m.remoteServiceAccountFolders, remoteName)
 		if err != nil {
-			m.log.Tracef("Service account folder was not found for: %q, skipping...", remoteName)
+			m.log.Tracef("Service account folder(s) not found for: %q, skipping...", remoteName)
 			continue
 		}
 
@@ -96,51 +95,64 @@ func (m *ServiceAccountManager) LoadServiceAccounts(remotePaths []string) error
 			continue
 		}
 
-		// retrieve service files
-		serviceAccountFiles, _ := pathutils.GetPathsInFolder(remoteServiceAccountFolder, true,
-			false, func(path string) *string {
-				lowerPath := strings.ToLower(path)
+		// load service account files in all folders
+		totalServiceAccountFiles := make([]pathutils.Path, 0)
 
-				// ignore non json files
-				if !strings.HasSuffix(lowerPath, ".json") {
-					return nil
+		for _, remoteServiceAccountFolder := range remoteServiceAccountFolders {
+			// retrieve service files within this folder
+			serviceAccountFiles, _ := pathutils.GetPathsInFolder(remoteServiceAccountFolder, true,
+				false, func(path string) *string {
+					lowerPath := strings.ToLower(path)
+
+					// ignore non json files
+					if !strings.HasSuffix(lowerPath, ".json") {
+						return nil
+					}
+
+					return &path
+				})
+
+			// were service accounts found?
+			if len(serviceAccountFiles) == 0 {
+				m.log.Tracef("No service accounts found for %q in: %v", remoteName, remoteServiceAccountFolder)
+				continue
+			}
+
+			// sort service files
+			sort.SliceStable(serviceAccountFiles, func(i, j int) bool {
+				is := reutils.GetEveryNumber(serviceAccountFiles[i].RealPath)
+				js := reutils.GetEveryNumber(serviceAccountFiles[j].RealPath)
+
+				in, err := strconv.Atoi(is)
+				if err != nil {
+					return false
+				}
+				jn, err := strconv.Atoi(js)
+				if err != nil {
+					return false
 				}
 
-				return &path
+				return in < jn
 			})
 
+			totalServiceAccountFiles = append(totalServiceAccountFiles, serviceAccountFiles...)
+		}
+
 		// were service accounts found?
-		if len(serviceAccountFiles) == 0 {
-			m.log.Tracef("No service accounts found for %q in: %v", remoteName, remoteServiceAccountFolder)
+		if len(totalServiceAccountFiles) == 0 {
+			m.log.Tracef("No service accounts found for %q in: %v", remoteName, remoteServiceAccountFolders)
 			continue
 		}
 
-		// sort service files
-		sort.SliceStable(serviceAccountFiles, func(i, j int) bool {
-			is := reutils.GetEveryNumber(serviceAccountFiles[i].RealPath)
-			js := reutils.GetEveryNumber(serviceAccountFiles[j].RealPath)
-
-			in, err := strconv.Atoi(is)
-			if err != nil {
-				return false
-			}
-			jn, err := strconv.Atoi(js)
-			if err != nil {
-				return false
-			}
-
-			return in < jn
-		})
-
 		// add to remote service accounts var
 		v := RemoteServiceAccounts{
 			RemoteEnvVar:    ConfigToEnv(remoteName, "SERVICE_ACCOUNT_FILE"),
-			ServiceAccounts: serviceAccountFiles,
+			ServiceAccounts: totalServiceAccountFiles,
 		}
 		m.remoteServiceAccounts[remoteName] = v
 
-		m.log.Debugf("Loaded %d service accounts for remote %q (env: %v)", len(serviceAccountFiles), remoteName,
-			v.RemoteEnvVar)
+		m.log.Debugf("Loaded %d service accounts for remote %q (env: %v)", len(totalServiceAccountFiles),
+			remoteName, v.RemoteEnvVar)
 	}
 
 	return nil
@@ -252,9 +264,15 @@ func (m *ServiceAccountManager) GetServiceAccount(remotePaths ...string) ([]*Rem
 
 func (m *ServiceAccountManager) ServiceAccountsCount() int {
 	n := 0
+	t := make(map[string]int)
 
 	for _, remote := range m.remoteServiceAccounts {
-		n += len(remote.ServiceAccounts)
+		for _, sa := range remote.ServiceAccounts {
+			if _, ok := t[sa.RealPath]; !ok {
+				t[sa.RealPath] = 0
+				n++
+			}
+		}
 	}
 
 	return n